src/be/nikiroo/fanfix/DataLoader.java

   1 package be.nikiroo.fanfix;
   2
   3 import java.io.File;
   4 import java.io.IOException;
   5 import java.io.InputStream;
   6 import java.net.URL;
   7 import java.util.Map;
   8
   9 import be.nikiroo.fanfix.bundles.Config;
  10 import be.nikiroo.fanfix.supported.BasicSupport;
  11 import be.nikiroo.utils.Cache;
  12 import be.nikiroo.utils.CacheMemory;
  13 import be.nikiroo.utils.Downloader;
  14 import be.nikiroo.utils.IOUtils;
  15 import be.nikiroo.utils.Image;
  16 import be.nikiroo.utils.ImageUtils;
  17 import be.nikiroo.utils.TraceHandler;
  18
  19 /**
  20  * This cache will manage Internet (and local) downloads, as well as put the
  21  * downloaded files into a cache.
  22  * <p>
  23  * As long the cached resource is not too old, it will use it instead of
  24  * retrieving the file again.
  25  *
  26  * @author niki
  27  */
  28 public class DataLoader {
  29         private Downloader downloader;
  30         private Cache downloadCache;
  31         private Cache cache;
  32
  33         /**
  34          * Create a new {@link DataLoader} object.
  35          *
  36          * @param dir
  37          *            the directory to use as cache
  38          * @param UA
  39          *            the User-Agent to use to download the resources
  40          * @param hoursChanging
  41          *            the number of hours after which a cached file that is thought
  42          *            to change ~often is considered too old (or -1 for
  43          *            "never too old")
  44          * @param hoursStable
  45          *            the number of hours after which a LARGE cached file that is
  46          *            thought to change rarely is considered too old (or -1 for
  47          *            "never too old")
  48          *
  49          * @throws IOException
  50          *             in case of I/O error
  51          */
  52         public DataLoader(File dir, String UA, int hoursChanging, int hoursStable)
  53                         throws IOException {
  54                 downloader = new Downloader(UA);
  55                 downloadCache = new Cache(dir, hoursChanging, hoursStable);
  56                 cache = downloadCache;
  57         }
  58
  59         /**
  60          * Create a new {@link DataLoader} object without disk cache (will keep a
  61          * memory cache for manual cache operations).
  62          *
  63          * @param UA
  64          *            the User-Agent to use to download the resources
  65          */
  66         public DataLoader(String UA) {
  67                 downloader = new Downloader(UA);
  68                 downloadCache = null;
  69                 cache = new CacheMemory();
  70         }
  71
  72         /**
  73          * The traces handler for this {@link Cache}.
  74          *
  75          * @param tracer
  76          *            the new traces handler
  77          */
  78         public void setTraceHandler(TraceHandler tracer) {
  79                 downloader.setTraceHandler(tracer);
  80                 cache.setTraceHandler(tracer);
  81                 if (downloadCache != null) {
  82                         downloadCache.setTraceHandler(tracer);
  83                 }
  84
  85         }
  86
  87         /**
  88          * Open a resource (will load it from the cache if possible, or save it into
  89          * the cache after downloading if not).
  90          *
  91          * @param url
  92          *            the resource to open
  93          * @param support
  94          *            the support to use to download the resource
  95          * @param stable
  96          *            TRUE for more stable resources, FALSE when they often change
  97          *
  98          * @return the opened resource, NOT NULL
  99          *
 100          * @throws IOException
 101          *             in case of I/O error
 102          */
 103         public InputStream open(URL url, BasicSupport support, boolean stable)
 104                         throws IOException {
 105                 // MUST NOT return null
 106                 return open(url, support, stable, url);
 107         }
 108
 109         /**
 110          * Open a resource (will load it from the cache if possible, or save it into
 111          * the cache after downloading if not).
 112          * <p>
 113          * The cached resource will be assimilated to the given original {@link URL}
 114          *
 115          * @param url
 116          *            the resource to open
 117          * @param support
 118          *            the support to use to download the resource
 119          * @param stable
 120          *            TRUE for more stable resources, FALSE when they often change
 121          * @param originalUrl
 122          *            the original {@link URL} used to locate the cached resource
 123          *
 124          * @return the opened resource, NOT NULL
 125          *
 126          * @throws IOException
 127          *             in case of I/O error
 128          */
 129         public InputStream open(URL url, BasicSupport support, boolean stable,
 130                         URL originalUrl) throws IOException {
 131                 // MUST NOT return null
 132                 try {
 133                         InputStream in = null;
 134
 135                         if (downloadCache != null) {
 136                                 in = downloadCache.load(originalUrl, false, stable);
 137                                 Instance.getTraceHandler().trace(
 138                                                 "Cache " + (in != null ? "hit" : "miss") + ": " + url);
 139                         }
 140
 141                         if (in == null) {
 142                                 try {
 143                                         in = openNoCache(url, support, null, null, null);
 144                                         if (downloadCache != null) {
 145                                                 downloadCache.save(in, originalUrl);
 146                                                 // ..But we want a resetable stream
 147                                                 in.close();
 148                                                 in = downloadCache.load(originalUrl, true, stable);
 149                                         } else {
 150                                                 InputStream resetIn = IOUtils.forceResetableStream(in);
 151                                                 if (resetIn != in) {
 152                                                         in.close();
 153                                                         in = resetIn;
 154                                                 }
 155                                         }
 156                                 } catch (IOException e) {
 157                                         throw new IOException("Cannot save the url: "
 158                                                         + (url == null ? "null" : url.toString()), e);
 159                                 }
 160                         }
 161
 162                         return in;
 163                 } catch (IOException e) {
 164                         throw new IOException("Cannot open the url: "
 165                                         + (url == null ? "null" : url.toString()), e);
 166                 }
 167         }
 168
 169         /**
 170          * Open the given {@link URL} without using the cache, but still update the
 171          * cookies.
 172          *
 173          * @param url
 174          *            the {@link URL} to open
 175          *
 176          * @return the {@link InputStream} of the opened page
 177          *
 178          * @throws IOException
 179          *             in case of I/O error
 180          */
 181         public InputStream openNoCache(URL url) throws IOException {
 182                 return downloader.open(url);
 183         }
 184
 185         /**
 186          * Open the given {@link URL} without using the cache, but still using and
 187          * updating the cookies.
 188          *
 189          * @param url
 190          *            the {@link URL} to open
 191          * @param support
 192          *            the {@link BasicSupport} used for the cookies
 193          * @param postParams
 194          *            the POST parameters
 195          * @param getParams
 196          *            the GET parameters (priority over POST)
 197          * @param oauth
 198          *            OAuth authorization (aka, "bearer XXXXXXX")
 199          *
 200          * @return the {@link InputStream} of the opened page
 201          *
 202          * @throws IOException
 203          *             in case of I/O error
 204          */
 205         public InputStream openNoCache(URL url, BasicSupport support,
 206                         Map<String, String> postParams, Map<String, String> getParams,
 207                         String oauth) throws IOException {
 208
 209                 Map<String, String> cookiesValues = null;
 210                 URL currentReferer = url;
 211                 if (support != null) {
 212                         cookiesValues = support.getCookies();
 213                         currentReferer = support.getCurrentReferer();
 214                         // priority: arguments
 215                         if (oauth == null) {
 216                                 oauth = support.getOAuth();
 217                         }
 218                 }
 219
 220                 return downloader.open(url, currentReferer, cookiesValues, postParams,
 221                                 getParams, oauth);
 222         }
 223
 224         /**
 225          * Refresh the resource into cache if needed.
 226          *
 227          * @param url
 228          *            the resource to open
 229          * @param support
 230          *            the support to use to download the resource
 231          * @param stable
 232          *            TRUE for more stable resources, FALSE when they often change
 233          *
 234          * @throws IOException
 235          *             in case of I/O error
 236          */
 237         public void refresh(URL url, BasicSupport support, boolean stable)
 238                         throws IOException {
 239                 if (downloadCache != null && !downloadCache.check(url, false, stable)) {
 240                         open(url, support, stable).close();
 241                 }
 242         }
 243
 244         /**
 245          * Check the resource to see if it is in the cache.
 246          *
 247          * @param url
 248          *            the resource to check
 249          * @param stable
 250          *            a stable file (that dones't change too often) -- parameter
 251          *            used to check if the file is too old to keep or not
 252          *
 253          * @return TRUE if it is
 254          *
 255          */
 256         public boolean check(URL url, boolean stable) {
 257                 return downloadCache != null && downloadCache.check(url, false, stable);
 258         }
 259
 260         /**
 261          * Save the given resource as an image on disk using the default image
 262          * format for content or cover -- will automatically add the extension, too.
 263          *
 264          * @param img
 265          *            the resource
 266          * @param target
 267          *            the target file without extension
 268          * @param cover
 269          *            use the cover image format instead of the content image format
 270          *
 271          * @throws IOException
 272          *             in case of I/O error
 273          */
 274         public void saveAsImage(Image img, File target, boolean cover)
 275                         throws IOException {
 276                 String format;
 277                 if (cover) {
 278                         format = Instance.getConfig().getString(Config.IMAGE_FORMAT_COVER)
 279                                         .toLowerCase();
 280                 } else {
 281                         format = Instance.getConfig()
 282                                         .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase();
 283                 }
 284                 saveAsImage(img, new File(target.toString() + "." + format), format);
 285         }
 286
 287         /**
 288          * Save the given resource as an image on disk using the given image format
 289          * for content, or with "png" format if it fails.
 290          *
 291          * @param img
 292          *            the resource
 293          * @param target
 294          *            the target file
 295          * @param format
 296          *            the file format ("png", "jpeg", "bmp"...)
 297          *
 298          * @throws IOException
 299          *             in case of I/O error
 300          */
 301         public void saveAsImage(Image img, File target, String format)
 302                         throws IOException {
 303                 ImageUtils.getInstance().saveAsImage(img, target, format);
 304         }
 305
 306         /**
 307          * Manually add this item to the cache.
 308          *
 309          * @param in
 310          *            the input data
 311          * @param uniqueID
 312          *            a unique ID for this resource
 313          *
 314          *
 315          * @throws IOException
 316          *             in case of I/O error
 317          */
 318         public void addToCache(InputStream in, String uniqueID) throws IOException {
 319                 cache.save(in, uniqueID);
 320         }
 321
 322         /**
 323          * Return the {@link InputStream} corresponding to the given unique ID, or
 324          * NULL if none found.
 325          *
 326          * @param uniqueID
 327          *            the unique ID
 328          *
 329          * @return the content or NULL
 330          */
 331         public InputStream getFromCache(String uniqueID) {
 332                 return cache.load(uniqueID, true, true);
 333         }
 334
 335         /**
 336          * Remove the given resource from the cache.
 337          *
 338          * @param uniqueID
 339          *            a unique ID used to locate the cached resource
 340          *
 341          * @return TRUE if it was removed
 342          */
 343         public boolean removeFromCache(String uniqueID) {
 344                 return cache.remove(uniqueID);
 345         }
 346
 347         /**
 348          * Clean the cache (delete the cached items).
 349          *
 350          * @param onlyOld
 351          *            only clean the files that are considered too old
 352          *
 353          * @return the number of cleaned items
 354          */
 355         public int cleanCache(boolean onlyOld) {
 356                 return cache.clean(onlyOld);
 357         }
 358 }