package be.nikiroo.fanfix; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.util.Map; import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.supported.BasicSupport; import be.nikiroo.utils.Cache; import be.nikiroo.utils.CacheMemory; import be.nikiroo.utils.Downloader; import be.nikiroo.utils.IOUtils; import be.nikiroo.utils.Image; import be.nikiroo.utils.ImageUtils; import be.nikiroo.utils.TraceHandler; /** * This cache will manage Internet (and local) downloads, as well as put the * downloaded files into a cache. *

* As long the cached resource is not too old, it will use it instead of * retrieving the file again. * * @author niki */ public class DataLoader { private Downloader downloader; private Cache downloadCache; private Cache cache; /** * Create a new {@link DataLoader} object. * * @param dir * the directory to use as cache * @param UA * the User-Agent to use to download the resources * @param hoursChanging * the number of hours after which a cached file that is thought * to change ~often is considered too old (or -1 for * "never too old") * @param hoursStable * the number of hours after which a LARGE cached file that is * thought to change rarely is considered too old (or -1 for * "never too old") * * @throws IOException * in case of I/O error */ public DataLoader(File dir, String UA, int hoursChanging, int hoursStable) throws IOException { downloader = new Downloader(UA); downloadCache = new Cache(dir, hoursChanging, hoursStable); cache = downloadCache; } /** * Create a new {@link DataLoader} object without disk cache (will keep a * memory cache for manual cache operations). * * @param UA * the User-Agent to use to download the resources */ public DataLoader(String UA) { downloader = new Downloader(UA); downloadCache = null; cache = new CacheMemory(); } /** * The traces handler for this {@link Cache}. * * @param tracer * the new traces handler */ public void setTraceHandler(TraceHandler tracer) { downloader.setTraceHandler(tracer); cache.setTraceHandler(tracer); if (downloadCache != null) { downloadCache.setTraceHandler(tracer); } } /** * Open a resource (will load it from the cache if possible, or save it into * the cache after downloading if not). * * @param url * the resource to open * @param support * the support to use to download the resource * @param stable * TRUE for more stable resources, FALSE when they often change * * @return the opened resource, NOT NULL * * @throws IOException * in case of I/O error */ public InputStream open(URL url, BasicSupport support, boolean stable) throws IOException { // MUST NOT return null return open(url, support, stable, url); } /** * Open a resource (will load it from the cache if possible, or save it into * the cache after downloading if not). *

* The cached resource will be assimilated to the given original {@link URL} * * @param url * the resource to open * @param support * the support to use to download the resource * @param stable * TRUE for more stable resources, FALSE when they often change * @param originalUrl * the original {@link URL} used to locate the cached resource * * @return the opened resource, NOT NULL * * @throws IOException * in case of I/O error */ public InputStream open(URL url, BasicSupport support, boolean stable, URL originalUrl) throws IOException { // MUST NOT return null try { InputStream in = null; if (downloadCache != null) { in = downloadCache.load(originalUrl, false, stable); Instance.getTraceHandler().trace( "Cache " + (in != null ? "hit" : "miss") + ": " + url); } if (in == null) { try { in = openNoCache(url, support, null, null, null); if (downloadCache != null) { downloadCache.save(in, originalUrl); // ..But we want a resetable stream in.close(); in = downloadCache.load(originalUrl, true, stable); } else { InputStream resetIn = IOUtils.forceResetableStream(in); if (resetIn != in) { in.close(); in = resetIn; } } } catch (IOException e) { throw new IOException("Cannot save the url: " + (url == null ? "null" : url.toString()), e); } } return in; } catch (IOException e) { throw new IOException("Cannot open the url: " + (url == null ? "null" : url.toString()), e); } } /** * Open the given {@link URL} without using the cache, but still update the * cookies. * * @param url * the {@link URL} to open * * @return the {@link InputStream} of the opened page * * @throws IOException * in case of I/O error */ public InputStream openNoCache(URL url) throws IOException { return downloader.open(url); } /** * Open the given {@link URL} without using the cache, but still using and * updating the cookies. * * @param url * the {@link URL} to open * @param support * the {@link BasicSupport} used for the cookies * @param postParams * the POST parameters * @param getParams * the GET parameters (priority over POST) * @param oauth * OAuth authorization (aka, "bearer XXXXXXX") * * @return the {@link InputStream} of the opened page * * @throws IOException * in case of I/O error */ public InputStream openNoCache(URL url, BasicSupport support, Map postParams, Map getParams, String oauth) throws IOException { Map cookiesValues = null; URL currentReferer = url; if (support != null) { cookiesValues = support.getCookies(); currentReferer = support.getCurrentReferer(); // priority: arguments if (oauth == null) { oauth = support.getOAuth(); } } return downloader.open(url, currentReferer, cookiesValues, postParams, getParams, oauth); } /** * Refresh the resource into cache if needed. * * @param url * the resource to open * @param support * the support to use to download the resource * @param stable * TRUE for more stable resources, FALSE when they often change * * @throws IOException * in case of I/O error */ public void refresh(URL url, BasicSupport support, boolean stable) throws IOException { if (downloadCache != null && !downloadCache.check(url, false, stable)) { open(url, support, stable).close(); } } /** * Check the resource to see if it is in the cache. * * @param url * the resource to check * @param stable * a stable file (that dones't change too often) -- parameter * used to check if the file is too old to keep or not * * @return TRUE if it is * */ public boolean check(URL url, boolean stable) { return downloadCache != null && downloadCache.check(url, false, stable); } /** * Save the given resource as an image on disk using the default image * format for content or cover -- will automatically add the extension, too. * * @param img * the resource * @param target * the target file without extension * @param cover * use the cover image format instead of the content image format * * @throws IOException * in case of I/O error */ public void saveAsImage(Image img, File target, boolean cover) throws IOException { String format; if (cover) { format = Instance.getConfig().getString(Config.IMAGE_FORMAT_COVER) .toLowerCase(); } else { format = Instance.getConfig() .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase(); } saveAsImage(img, new File(target.toString() + "." + format), format); } /** * Save the given resource as an image on disk using the given image format * for content, or with "png" format if it fails. * * @param img * the resource * @param target * the target file * @param format * the file format ("png", "jpeg", "bmp"...) * * @throws IOException * in case of I/O error */ public void saveAsImage(Image img, File target, String format) throws IOException { ImageUtils.getInstance().saveAsImage(img, target, format); } /** * Manually add this item to the cache. * * @param in * the input data * @param uniqueID * a unique ID for this resource * * * @throws IOException * in case of I/O error */ public void addToCache(InputStream in, String uniqueID) throws IOException { cache.save(in, uniqueID); } /** * Return the {@link InputStream} corresponding to the given unique ID, or * NULL if none found. * * @param uniqueID * the unique ID * * @return the content or NULL */ public InputStream getFromCache(String uniqueID) { return cache.load(uniqueID, true, true); } /** * Remove the given resource from the cache. * * @param uniqueID * a unique ID used to locate the cached resource * * @return TRUE if it was removed */ public boolean removeFromCache(String uniqueID) { return cache.remove(uniqueID); } /** * Clean the cache (delete the cached items). * * @param onlyOld * only clean the files that are considered too old * * @return the number of cleaned items */ public int cleanCache(boolean onlyOld) { return cache.clean(onlyOld); } }