DataLoader.java

   1 package be.nikiroo.fanfix;
   2
   3 import java.io.File;
   4 import java.io.IOException;
   5 import java.io.InputStream;
   6 import java.net.URL;
   7 import java.util.Map;
   8
   9 import be.nikiroo.fanfix.bundles.Config;
  10 import be.nikiroo.fanfix.supported.BasicSupport;
  11 import be.nikiroo.utils.Cache;
  12 import be.nikiroo.utils.CacheMemory;
  13 import be.nikiroo.utils.Downloader;
  14 import be.nikiroo.utils.Image;
  15 import be.nikiroo.utils.ImageUtils;
  16 import be.nikiroo.utils.TraceHandler;
  17
  18 /**
  19  * This cache will manage Internet (and local) downloads, as well as put the
  20  * downloaded files into a cache.
  21  * <p>
  22  * As long the cached resource is not too old, it will use it instead of
  23  * retrieving the file again.
  24  *
  25  * @author niki
  26  */
  27 public class DataLoader {
  28         private Downloader downloader;
  29         private Downloader downloaderNoCache;
  30         private Cache cache;
  31
  32         /**
  33          * Create a new {@link DataLoader} object.
  34          *
  35          * @param dir
  36          *            the directory to use as cache
  37          * @param UA
  38          *            the User-Agent to use to download the resources
  39          * @param hoursChanging
  40          *            the number of hours after which a cached file that is thought
  41          *            to change ~often is considered too old (or -1 for
  42          *            "never too old")
  43          * @param hoursStable
  44          *            the number of hours after which a LARGE cached file that is
  45          *            thought to change rarely is considered too old (or -1 for
  46          *            "never too old")
  47          *
  48          * @throws IOException
  49          *             in case of I/O error
  50          */
  51         public DataLoader(File dir, String UA, int hoursChanging, int hoursStable)
  52                         throws IOException {
  53                 downloader = new Downloader(UA, new Cache(dir, hoursChanging,
  54                                 hoursStable));
  55                 downloaderNoCache = new Downloader(UA);
  56
  57                 cache = downloader.getCache();
  58         }
  59
  60         /**
  61          * Create a new {@link DataLoader} object without disk cache (will keep a
  62          * memory cache for manual cache operations).
  63          *
  64          * @param UA
  65          *            the User-Agent to use to download the resources
  66          */
  67         public DataLoader(String UA) {
  68                 downloader = new Downloader(UA);
  69                 downloaderNoCache = downloader;
  70                 cache = new CacheMemory();
  71         }
  72
  73         /**
  74          * The traces handler for this {@link Cache}.
  75          *
  76          * @param tracer
  77          *            the new traces handler
  78          */
  79         public void setTraceHandler(TraceHandler tracer) {
  80                 downloader.setTraceHandler(tracer);
  81                 downloaderNoCache.setTraceHandler(tracer);
  82                 cache.setTraceHandler(tracer);
  83                 if (downloader.getCache() != null) {
  84                         downloader.getCache().setTraceHandler(tracer);
  85                 }
  86
  87         }
  88
  89         /**
  90          * Open a resource (will load it from the cache if possible, or save it into
  91          * the cache after downloading if not).
  92          * <p>
  93          * The cached resource will be assimilated to the given original {@link URL}
  94          *
  95          * @param url
  96          *            the resource to open
  97          * @param support
  98          *            the support to use to download the resource
  99          * @param stable
 100          *            TRUE for more stable resources, FALSE when they often change
 101          *
 102          * @return the opened resource, NOT NULL
 103          *
 104          * @throws IOException
 105          *             in case of I/O error
 106          */
 107         public InputStream open(URL url, BasicSupport support, boolean stable)
 108                         throws IOException {
 109                 return open(url, url, support, stable, null, null, null);
 110         }
 111
 112         /**
 113          * Open a resource (will load it from the cache if possible, or save it into
 114          * the cache after downloading if not).
 115          * <p>
 116          * The cached resource will be assimilated to the given original {@link URL}
 117          *
 118          * @param url
 119          *            the resource to open
 120          * @param originalUrl
 121          *            the original {@link URL} before any redirection occurs, which
 122          *            is also used for the cache ID if needed (so we can retrieve
 123          *            the content with this URL if needed)
 124          * @param support
 125          *            the support to use to download the resource
 126          * @param stable
 127          *            TRUE for more stable resources, FALSE when they often change
 128          *
 129          * @return the opened resource, NOT NULL
 130          *
 131          * @throws IOException
 132          *             in case of I/O error
 133          */
 134         public InputStream open(URL url, URL originalUrl, BasicSupport support,
 135                         boolean stable) throws IOException {
 136                 return open(url, originalUrl, support, stable, null, null, null);
 137         }
 138
 139         /**
 140          * Open a resource (will load it from the cache if possible, or save it into
 141          * the cache after downloading if not).
 142          * <p>
 143          * The cached resource will be assimilated to the given original {@link URL}
 144          *
 145          * @param url
 146          *            the resource to open
 147          * @param originalUrl
 148          *            the original {@link URL} before any redirection occurs, which
 149          *            is also used for the cache ID if needed (so we can retrieve
 150          *            the content with this URL if needed)
 151          * @param support
 152          *            the support to use to download the resource
 153          * @param stable
 154          *            TRUE for more stable resources, FALSE when they often change
 155          * @param postParams
 156          *            the POST parameters
 157          * @param getParams
 158          *            the GET parameters (priority over POST)
 159          * @param oauth
 160          *            OAuth authorization (aka, "bearer XXXXXXX")
 161          *
 162          * @return the opened resource, NOT NULL
 163          *
 164          * @throws IOException
 165          *             in case of I/O error
 166          */
 167         public InputStream open(URL url, URL originalUrl, BasicSupport support,
 168                         boolean stable, Map<String, String> postParams,
 169                         Map<String, String> getParams, String oauth) throws IOException {
 170
 171                 Map<String, String> cookiesValues = null;
 172                 URL currentReferer = url;
 173
 174                 if (support != null) {
 175                         cookiesValues = support.getCookies();
 176                         currentReferer = support.getCurrentReferer();
 177                         // priority: arguments
 178                         if (oauth == null) {
 179                                 oauth = support.getOAuth();
 180                         }
 181                 }
 182
 183                 return downloader.open(url, originalUrl, currentReferer, cookiesValues,
 184                                 postParams, getParams, oauth, stable);
 185         }
 186
 187         /**
 188          * Open the given {@link URL} without using the cache, but still using and
 189          * updating the cookies.
 190          *
 191          * @param url
 192          *            the {@link URL} to open
 193          * @param support
 194          *            the {@link BasicSupport} used for the cookies
 195          * @param postParams
 196          *            the POST parameters
 197          * @param getParams
 198          *            the GET parameters (priority over POST)
 199          * @param oauth
 200          *            OAuth authorization (aka, "bearer XXXXXXX")
 201          *
 202          * @return the {@link InputStream} of the opened page
 203          *
 204          * @throws IOException
 205          *             in case of I/O error
 206          */
 207         public InputStream openNoCache(URL url, BasicSupport support,
 208                         Map<String, String> postParams, Map<String, String> getParams,
 209                         String oauth) throws IOException {
 210
 211                 Map<String, String> cookiesValues = null;
 212                 URL currentReferer = url;
 213                 if (support != null) {
 214                         cookiesValues = support.getCookies();
 215                         currentReferer = support.getCurrentReferer();
 216                         // priority: arguments
 217                         if (oauth == null) {
 218                                 oauth = support.getOAuth();
 219                         }
 220                 }
 221
 222                 return downloaderNoCache.open(url, currentReferer, cookiesValues,
 223                                 postParams, getParams, oauth);
 224         }
 225
 226         /**
 227          * Refresh the resource into cache if needed.
 228          *
 229          * @param url
 230          *            the resource to open
 231          * @param support
 232          *            the support to use to download the resource
 233          * @param stable
 234          *            TRUE for more stable resources, FALSE when they often change
 235          *
 236          * @throws IOException
 237          *             in case of I/O error
 238          */
 239         public void refresh(URL url, BasicSupport support, boolean stable)
 240                         throws IOException {
 241                 if (!check(url, stable)) {
 242                         open(url, url, support, stable, null, null, null).close();
 243                 }
 244         }
 245
 246         /**
 247          * Check the resource to see if it is in the cache.
 248          *
 249          * @param url
 250          *            the resource to check
 251          * @param stable
 252          *            a stable file (that dones't change too often) -- parameter
 253          *            used to check if the file is too old to keep or not
 254          *
 255          * @return TRUE if it is
 256          *
 257          */
 258         public boolean check(URL url, boolean stable) {
 259                 return downloader.getCache() != null
 260                                 && downloader.getCache().check(url, false, stable);
 261         }
 262
 263         /**
 264          * Save the given resource as an image on disk using the default image
 265          * format for content or cover -- will automatically add the extension, too.
 266          *
 267          * @param img
 268          *            the resource
 269          * @param target
 270          *            the target file without extension
 271          * @param cover
 272          *            use the cover image format instead of the content image format
 273          *
 274          * @throws IOException
 275          *             in case of I/O error
 276          */
 277         public void saveAsImage(Image img, File target, boolean cover)
 278                         throws IOException {
 279                 String format;
 280                 if (cover) {
 281                         format = Instance.getConfig().getString(Config.IMAGE_FORMAT_COVER)
 282                                         .toLowerCase();
 283                 } else {
 284                         format = Instance.getConfig()
 285                                         .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase();
 286                 }
 287                 saveAsImage(img, new File(target.toString() + "." + format), format);
 288         }
 289
 290         /**
 291          * Save the given resource as an image on disk using the given image format
 292          * for content, or with "png" format if it fails.
 293          *
 294          * @param img
 295          *            the resource
 296          * @param target
 297          *            the target file
 298          * @param format
 299          *            the file format ("png", "jpeg", "bmp"...)
 300          *
 301          * @throws IOException
 302          *             in case of I/O error
 303          */
 304         public void saveAsImage(Image img, File target, String format)
 305                         throws IOException {
 306                 ImageUtils.getInstance().saveAsImage(img, target, format);
 307         }
 308
 309         /**
 310          * Manually add this item to the cache.
 311          *
 312          * @param in
 313          *            the input data
 314          * @param uniqueID
 315          *            a unique ID for this resource
 316          *
 317          *
 318          * @throws IOException
 319          *             in case of I/O error
 320          */
 321         public void addToCache(InputStream in, String uniqueID) throws IOException {
 322                 cache.save(in, uniqueID);
 323         }
 324
 325         /**
 326          * Return the {@link InputStream} corresponding to the given unique ID, or
 327          * NULL if none found.
 328          *
 329          * @param uniqueID
 330          *            the unique ID
 331          *
 332          * @return the content or NULL
 333          */
 334         public InputStream getFromCache(String uniqueID) {
 335                 return cache.load(uniqueID, true, true);
 336         }
 337
 338         /**
 339          * Remove the given resource from the cache.
 340          *
 341          * @param uniqueID
 342          *            a unique ID used to locate the cached resource
 343          *
 344          * @return TRUE if it was removed
 345          */
 346         public boolean removeFromCache(String uniqueID) {
 347                 return cache.remove(uniqueID);
 348         }
 349
 350         /**
 351          * Clean the cache (delete the cached items).
 352          *
 353          * @param onlyOld
 354          *            only clean the files that are considered too old
 355          *
 356          * @return the number of cleaned items
 357          */
 358         public int cleanCache(boolean onlyOld) {
 359                 return cache.clean(onlyOld);
 360         }
 361 }