src/be/nikiroo/fanfix/DataLoader.java

   1 package be.nikiroo.fanfix;
   2
   3 import java.io.File;
   4 import java.io.IOException;
   5 import java.io.InputStream;
   6 import java.net.URL;
   7 import java.util.Map;
   8
   9 import be.nikiroo.fanfix.bundles.Config;
  10 import be.nikiroo.fanfix.supported.BasicSupport;
  11 import be.nikiroo.utils.Cache;
  12 import be.nikiroo.utils.CacheMemory;
  13 import be.nikiroo.utils.Downloader;
  14 import be.nikiroo.utils.Image;
  15 import be.nikiroo.utils.ImageUtils;
  16 import be.nikiroo.utils.TraceHandler;
  17
  18 /**
  19  * This cache will manage Internet (and local) downloads, as well as put the
  20  * downloaded files into a cache.
  21  * <p>
  22  * As long the cached resource is not too old, it will use it instead of
  23  * retrieving the file again.
  24  *
  25  * @author niki
  26  */
  27 public class DataLoader {
  28         private Downloader downloader;
  29         private Downloader downloaderNoCache;
  30         private Cache cache;
  31         private boolean offline;
  32
  33         /**
  34          * Create a new {@link DataLoader} object.
  35          *
  36          * @param dir
  37          *            the directory to use as cache
  38          * @param UA
  39          *            the User-Agent to use to download the resources
  40          * @param hoursChanging
  41          *            the number of hours after which a cached file that is thought
  42          *            to change ~often is considered too old (or -1 for
  43          *            "never too old")
  44          * @param hoursStable
  45          *            the number of hours after which a LARGE cached file that is
  46          *            thought to change rarely is considered too old (or -1 for
  47          *            "never too old")
  48          *
  49          * @throws IOException
  50          *             in case of I/O error
  51          */
  52         public DataLoader(File dir, String UA, int hoursChanging, int hoursStable)
  53                         throws IOException {
  54                 downloader = new Downloader(UA, new Cache(dir, hoursChanging,
  55                                 hoursStable));
  56                 downloaderNoCache = new Downloader(UA);
  57
  58                 cache = downloader.getCache();
  59         }
  60
  61         /**
  62          * Create a new {@link DataLoader} object without disk cache (will keep a
  63          * memory cache for manual cache operations).
  64          *
  65          * @param UA
  66          *            the User-Agent to use to download the resources
  67          */
  68         public DataLoader(String UA) {
  69                 downloader = new Downloader(UA);
  70                 downloaderNoCache = downloader;
  71                 cache = new CacheMemory();
  72         }
  73
  74         /**
  75          * This {@link Downloader} is forbidden to try and connect to the network.
  76          * <p>
  77          * If TRUE, it will only check the cache (even in no-cache mode!).
  78          * <p>
  79          * Default is FALSE.
  80          *
  81          * @return TRUE if offline
  82          */
  83         public boolean isOffline() {
  84                 return offline;
  85         }
  86
  87         /**
  88          * This {@link Downloader} is forbidden to try and connect to the network.
  89          * <p>
  90          * If TRUE, it will only check the cache (even in no-cache mode!).
  91          * <p>
  92          * Default is FALSE.
  93          *
  94          * @param offline TRUE for offline, FALSE for online
  95          */
  96         public void setOffline(boolean offline) {
  97                 this.offline = offline;
  98                 downloader.setOffline(offline);
  99                 downloaderNoCache.setOffline(offline);
 100
 101                 // If we don't, we cannot support no-cache using code in OFFLINE mode
 102                 if (offline) {
 103                         downloaderNoCache.setCache(cache);
 104                 } else {
 105                         downloaderNoCache.setCache(null);
 106                 }
 107         }
 108
 109         /**
 110          * The traces handler for this {@link Cache}.
 111          *
 112          * @param tracer
 113          *            the new traces handler
 114          */
 115         public void setTraceHandler(TraceHandler tracer) {
 116                 downloader.setTraceHandler(tracer);
 117                 downloaderNoCache.setTraceHandler(tracer);
 118                 cache.setTraceHandler(tracer);
 119                 if (downloader.getCache() != null) {
 120                         downloader.getCache().setTraceHandler(tracer);
 121                 }
 122
 123         }
 124
 125         /**
 126          * Open a resource (will load it from the cache if possible, or save it into
 127          * the cache after downloading if not).
 128          * <p>
 129          * The cached resource will be assimilated to the given original {@link URL}
 130          *
 131          * @param url
 132          *            the resource to open
 133          * @param support
 134          *            the support to use to download the resource (can be NULL)
 135          * @param stable
 136          *            TRUE for more stable resources, FALSE when they often change
 137          *
 138          * @return the opened resource, NOT NULL
 139          *
 140          * @throws IOException
 141          *             in case of I/O error
 142          */
 143         public InputStream open(URL url, BasicSupport support, boolean stable)
 144                         throws IOException {
 145                 return open(url, url, support, stable, null, null, null);
 146         }
 147
 148         /**
 149          * Open a resource (will load it from the cache if possible, or save it into
 150          * the cache after downloading if not).
 151          * <p>
 152          * The cached resource will be assimilated to the given original {@link URL}
 153          *
 154          * @param url
 155          *            the resource to open
 156          * @param originalUrl
 157          *            the original {@link URL} before any redirection occurs, which
 158          *            is also used for the cache ID if needed (so we can retrieve
 159          *            the content with this URL if needed)
 160          * @param support
 161          *            the support to use to download the resource
 162          * @param stable
 163          *            TRUE for more stable resources, FALSE when they often change
 164          *
 165          * @return the opened resource, NOT NULL
 166          *
 167          * @throws IOException
 168          *             in case of I/O error
 169          */
 170         public InputStream open(URL url, URL originalUrl, BasicSupport support,
 171                         boolean stable) throws IOException {
 172                 return open(url, originalUrl, support, stable, null, null, null);
 173         }
 174
 175         /**
 176          * Open a resource (will load it from the cache if possible, or save it into
 177          * the cache after downloading if not).
 178          * <p>
 179          * The cached resource will be assimilated to the given original {@link URL}
 180          *
 181          * @param url
 182          *            the resource to open
 183          * @param originalUrl
 184          *            the original {@link URL} before any redirection occurs, which
 185          *            is also used for the cache ID if needed (so we can retrieve
 186          *            the content with this URL if needed)
 187          * @param support
 188          *            the support to use to download the resource (can be NULL)
 189          * @param stable
 190          *            TRUE for more stable resources, FALSE when they often change
 191          * @param postParams
 192          *            the POST parameters
 193          * @param getParams
 194          *            the GET parameters (priority over POST)
 195          * @param oauth
 196          *            OAuth authorization (aka, "bearer XXXXXXX")
 197          *
 198          * @return the opened resource, NOT NULL
 199          *
 200          * @throws IOException
 201          *             in case of I/O error
 202          */
 203         public InputStream open(URL url, URL originalUrl, BasicSupport support,
 204                         boolean stable, Map<String, String> postParams,
 205                         Map<String, String> getParams, String oauth) throws IOException {
 206
 207                 Map<String, String> cookiesValues = null;
 208                 URL currentReferer = url;
 209
 210                 if (support != null) {
 211                         cookiesValues = support.getCookies();
 212                         currentReferer = support.getCurrentReferer();
 213                         // priority: arguments
 214                         if (oauth == null) {
 215                                 oauth = support.getOAuth();
 216                         }
 217                 }
 218
 219                 return downloader.open(url, originalUrl, currentReferer, cookiesValues,
 220                                 postParams, getParams, oauth, stable);
 221         }
 222
 223         /**
 224          * Open the given {@link URL} without using the cache, but still using and
 225          * updating the cookies.
 226          *
 227          * @param url
 228          *            the {@link URL} to open
 229          * @param support
 230          *            the {@link BasicSupport} used for the cookies
 231          * @param postParams
 232          *            the POST parameters
 233          * @param getParams
 234          *            the GET parameters (priority over POST)
 235          * @param oauth
 236          *            OAuth authorization (aka, "bearer XXXXXXX")
 237          *
 238          * @return the {@link InputStream} of the opened page
 239          *
 240          * @throws IOException
 241          *             in case of I/O error
 242          */
 243         public InputStream openNoCache(URL url, BasicSupport support,
 244                         Map<String, String> postParams, Map<String, String> getParams,
 245                         String oauth) throws IOException {
 246
 247                 Map<String, String> cookiesValues = null;
 248                 URL currentReferer = url;
 249                 if (support != null) {
 250                         cookiesValues = support.getCookies();
 251                         currentReferer = support.getCurrentReferer();
 252                         // priority: arguments
 253                         if (oauth == null) {
 254                                 oauth = support.getOAuth();
 255                         }
 256                 }
 257
 258                 return downloaderNoCache.open(url, currentReferer, cookiesValues,
 259                                 postParams, getParams, oauth);
 260         }
 261
 262         /**
 263          * Refresh the resource into cache if needed.
 264          *
 265          * @param url
 266          *            the resource to open
 267          * @param support
 268          *            the support to use to download the resource (can be NULL)
 269          * @param stable
 270          *            TRUE for more stable resources, FALSE when they often change
 271          *
 272          * @throws IOException
 273          *             in case of I/O error
 274          */
 275         public void refresh(URL url, BasicSupport support, boolean stable)
 276                         throws IOException {
 277                 if (!check(url, stable)) {
 278                         open(url, url, support, stable, null, null, null).close();
 279                 }
 280         }
 281
 282         /**
 283          * Check the resource to see if it is in the cache.
 284          *
 285          * @param url
 286          *            the resource to check
 287          * @param stable
 288          *            a stable file (that dones't change too often) -- parameter
 289          *            used to check if the file is too old to keep or not
 290          *
 291          * @return TRUE if it is
 292          *
 293          */
 294         public boolean check(URL url, boolean stable) {
 295                 return downloader.getCache() != null
 296                                 && downloader.getCache().check(url, false, stable);
 297         }
 298
 299         /**
 300          * Save the given resource as an image on disk using the default image
 301          * format for content or cover -- will automatically add the extension, too.
 302          *
 303          * @param img
 304          *            the resource
 305          * @param target
 306          *            the target file without extension
 307          * @param cover
 308          *            use the cover image format instead of the content image format
 309          *
 310          * @throws IOException
 311          *             in case of I/O error
 312          */
 313         public void saveAsImage(Image img, File target, boolean cover)
 314                         throws IOException {
 315                 String format;
 316                 if (cover) {
 317                         format = Instance.getInstance().getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER).toLowerCase();
 318                 } else {
 319                         format = Instance.getInstance().getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_CONTENT)
 320                                         .toLowerCase();
 321                 }
 322                 saveAsImage(img, new File(target.toString() + "." + format), format);
 323         }
 324
 325         /**
 326          * Save the given resource as an image on disk using the given image format
 327          * for content, or with "png" format if it fails.
 328          *
 329          * @param img
 330          *            the resource
 331          * @param target
 332          *            the target file
 333          * @param format
 334          *            the file format ("png", "jpeg", "bmp"...)
 335          *
 336          * @throws IOException
 337          *             in case of I/O error
 338          */
 339         public void saveAsImage(Image img, File target, String format)
 340                         throws IOException {
 341                 ImageUtils.getInstance().saveAsImage(img, target, format);
 342         }
 343
 344         /**
 345          * Manually add this item to the cache.
 346          *
 347          * @param in
 348          *            the input data
 349          * @param uniqueID
 350          *            a unique ID for this resource
 351          *
 352          *
 353          * @throws IOException
 354          *             in case of I/O error
 355          */
 356         public void addToCache(InputStream in, String uniqueID) throws IOException {
 357                 cache.save(in, uniqueID);
 358         }
 359
 360         /**
 361          * Return the {@link InputStream} corresponding to the given unique ID, or
 362          * NULL if none found.
 363          *
 364          * @param uniqueID
 365          *            the unique ID
 366          *
 367          * @return the content or NULL
 368          */
 369         public InputStream getFromCache(String uniqueID) {
 370                 return cache.load(uniqueID, true, true);
 371         }
 372
 373         /**
 374          * Remove the given resource from the cache.
 375          *
 376          * @param uniqueID
 377          *            a unique ID used to locate the cached resource
 378          *
 379          * @return TRUE if it was removed
 380          */
 381         public boolean removeFromCache(String uniqueID) {
 382                 return cache.remove(uniqueID);
 383         }
 384
 385         /**
 386          * Clean the cache (delete the cached items).
 387          *
 388          * @param onlyOld
 389          *            only clean the files that are considered too old
 390          *
 391          * @return the number of cleaned items
 392          */
 393         public int cleanCache(boolean onlyOld) {
 394                 return cache.clean(onlyOld);
 395         }
 396 }