src/be/nikiroo/fanfix/DataLoader.java

   1 package be.nikiroo.fanfix;
   2
   3 import java.awt.image.BufferedImage;
   4 import java.io.File;
   5 import java.io.IOException;
   6 import java.io.InputStream;
   7 import java.net.URL;
   8 import java.util.Map;
   9
  10 import javax.imageio.ImageIO;
  11
  12 import be.nikiroo.fanfix.bundles.Config;
  13 import be.nikiroo.fanfix.supported.BasicSupport;
  14 import be.nikiroo.utils.Cache;
  15 import be.nikiroo.utils.Downloader;
  16 import be.nikiroo.utils.ImageUtils;
  17
  18 /**
  19  * This cache will manage Internet (and local) downloads, as well as put the
  20  * downloaded files into a cache.
  21  * <p>
  22  * As long the cached resource is not too old, it will use it instead of
  23  * retrieving the file again.
  24  *
  25  * @author niki
  26  */
  27 public class DataLoader {
  28         private Cache cache;
  29         private Downloader downloader;
  30
  31         /**
  32          * Create a new {@link DataLoader} object.
  33          *
  34          * @param dir
  35          *            the directory to use as cache
  36          * @param UA
  37          *            the User-Agent to use to download the resources
  38          * @param hoursChanging
  39          *            the number of hours after which a cached file that is thought
  40          *            to change ~often is considered too old (or -1 for
  41          *            "never too old")
  42          * @param hoursStable
  43          *            the number of hours after which a LARGE cached file that is
  44          *            thought to change rarely is considered too old (or -1 for
  45          *            "never too old")
  46          *
  47          * @throws IOException
  48          *             in case of I/O error
  49          */
  50         public DataLoader(File dir, String UA, int hoursChanging, int hoursStable)
  51                         throws IOException {
  52                 cache = new Cache(dir, hoursChanging, hoursStable);
  53                 downloader = new Downloader(UA);
  54         }
  55
  56         /**
  57          * Open a resource (will load it from the cache if possible, or save it into
  58          * the cache after downloading if not).
  59          *
  60          * @param url
  61          *            the resource to open
  62          * @param support
  63          *            the support to use to download the resource
  64          * @param stable
  65          *            TRUE for more stable resources, FALSE when they often change
  66          *
  67          * @return the opened resource, NOT NULL
  68          *
  69          * @throws IOException
  70          *             in case of I/O error
  71          */
  72         public InputStream open(URL url, BasicSupport support, boolean stable)
  73                         throws IOException {
  74                 // MUST NOT return null
  75                 return open(url, support, stable, url);
  76         }
  77
  78         /**
  79          * Open a resource (will load it from the cache if possible, or save it into
  80          * the cache after downloading if not).
  81          * <p>
  82          * The cached resource will be assimilated to the given original {@link URL}
  83          *
  84          * @param url
  85          *            the resource to open
  86          * @param support
  87          *            the support to use to download the resource
  88          * @param stable
  89          *            TRUE for more stable resources, FALSE when they often change
  90          * @param originalUrl
  91          *            the original {@link URL} used to locate the cached resource
  92          *
  93          * @return the opened resource, NOT NULL
  94          *
  95          * @throws IOException
  96          *             in case of I/O error
  97          */
  98         public InputStream open(URL url, BasicSupport support, boolean stable,
  99                         URL originalUrl) throws IOException {
 100                 // MUST NOT return null
 101                 try {
 102                         InputStream in = cache.load(originalUrl, false, stable);
 103                         Instance.getTraceHandler().trace(
 104                                         "Cache " + (in != null ? "hit" : "miss") + ": " + url);
 105
 106                         if (in == null) {
 107                                 try {
 108                                         in = openNoCache(url, support, null, null, null);
 109                                         cache.save(in, originalUrl);
 110                                         // ..But we want a resetable stream
 111                                         in.close();
 112                                         in = cache.load(originalUrl, false, stable);
 113                                 } catch (IOException e) {
 114                                         throw new IOException("Cannot save the url: "
 115                                                         + (url == null ? "null" : url.toString()), e);
 116                                 }
 117                         }
 118
 119                         return in;
 120                 } catch (IOException e) {
 121                         throw new IOException("Cannot open the url: "
 122                                         + (url == null ? "null" : url.toString()), e);
 123                 }
 124         }
 125
 126         /**
 127          * Open the given {@link URL} without using the cache, but still update the
 128          * cookies.
 129          *
 130          * @param url
 131          *            the {@link URL} to open
 132          *
 133          * @return the {@link InputStream} of the opened page
 134          *
 135          * @throws IOException
 136          *             in case of I/O error
 137          */
 138         public InputStream openNoCache(URL url) throws IOException {
 139                 return downloader.open(url);
 140         }
 141
 142         /**
 143          * Open the given {@link URL} without using the cache, but still using and
 144          * updating the cookies.
 145          *
 146          * @param url
 147          *            the {@link URL} to open
 148          * @param support
 149          *            the {@link BasicSupport} used for the cookies
 150          * @param postParams
 151          *            the POST parameters
 152          * @param getParams
 153          *            the GET parameters (priority over POST)
 154          * @param oauth
 155          *            OAuth authorization (aka, "bearer XXXXXXX")
 156          *
 157          * @return the {@link InputStream} of the opened page
 158          *
 159          * @throws IOException
 160          *             in case of I/O error
 161          */
 162         public InputStream openNoCache(URL url, BasicSupport support,
 163                         Map<String, String> postParams, Map<String, String> getParams,
 164                         String oauth) throws IOException {
 165
 166                 Map<String, String> cookiesValues = null;
 167                 URL currentReferer = url;
 168                 if (support != null) {
 169                         cookiesValues = support.getCookies();
 170                         currentReferer = support.getCurrentReferer();
 171                         // priority: arguments
 172                         if (oauth == null) {
 173                                 oauth = support.getOAuth();
 174                         }
 175                 }
 176
 177                 return downloader.open(url, currentReferer, cookiesValues, postParams,
 178                                 getParams, oauth);
 179         }
 180
 181         /**
 182          * Refresh the resource into cache if needed.
 183          *
 184          * @param url
 185          *            the resource to open
 186          * @param support
 187          *            the support to use to download the resource
 188          * @param stable
 189          *            TRUE for more stable resources, FALSE when they often change
 190          *
 191          * @throws IOException
 192          *             in case of I/O error
 193          */
 194         public void refresh(URL url, BasicSupport support, boolean stable)
 195                         throws IOException {
 196                 if (!cache.check(url, false, stable)) {
 197                         open(url, support, stable).close();
 198                 }
 199         }
 200
 201         /**
 202          * Check the resource to see if it is in the cache.
 203          *
 204          * @param url
 205          *            the resource to check
 206          * @param stable
 207          *            a stable file (that dones't change too often) -- parameter
 208          *            used to check if the file is too old to keep or not
 209          *
 210          * @return TRUE if it is
 211          *
 212          */
 213         public boolean check(URL url, boolean stable) {
 214                 return cache.check(url, false, stable);
 215         }
 216
 217         /**
 218          * Save the given resource as an image on disk using the default image
 219          * format for content.
 220          *
 221          * @param url
 222          *            the resource
 223          * @param target
 224          *            the target file
 225          *
 226          * @throws IOException
 227          *             in case of I/O error
 228          */
 229         public void saveAsImage(URL url, File target) throws IOException {
 230                 InputStream in = open(url, null, true);
 231                 try {
 232                         saveAsImage(ImageUtils.fromStream(in), target);
 233                 } finally {
 234                         in.close();
 235                 }
 236         }
 237
 238         /**
 239          * Save the given resource as an image on disk using the default image
 240          * format for content.
 241          *
 242          * @param image
 243          *            the resource
 244          * @param target
 245          *            the target file
 246          *
 247          * @throws IOException
 248          *             in case of I/O error
 249          */
 250         public void saveAsImage(BufferedImage image, File target)
 251                         throws IOException {
 252                 try {
 253                         String format = Instance.getConfig()
 254                                         .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase();
 255                         boolean ok = ImageIO.write(image, format, target);
 256                         if (!ok) {
 257                                 // Some formats are not reliable
 258                                 // Second change: PNG
 259                                 if (!format.equals("png")) {
 260                                         ok = ImageIO.write(image, "png", target);
 261                                 }
 262
 263                                 if (!ok) {
 264                                         throw new IOException(
 265                                                         "Cannot find a writer for this image and format: "
 266                                                                         + format);
 267                                 }
 268                         }
 269                 } catch (IOException e) {
 270                         throw new IOException("Cannot write image to " + target, e);
 271                 }
 272         }
 273
 274         /**
 275          * Manually add this item to the cache.
 276          *
 277          * @param in
 278          *            the input data
 279          * @param uniqueID
 280          *            a unique ID for this resource
 281          *
 282          * @return the resulting {@link File}
 283          *
 284          * @throws IOException
 285          *             in case of I/O error
 286          */
 287         public File addToCache(InputStream in, String uniqueID) throws IOException {
 288                 return cache.save(in, uniqueID);
 289         }
 290
 291         /**
 292          * Return the {@link InputStream} corresponding to the given unique ID, or
 293          * NULL if none found.
 294          *
 295          * @param uniqueID
 296          *            the unique ID
 297          *
 298          * @return the content or NULL
 299          */
 300         public InputStream getFromCache(String uniqueID) {
 301                 return cache.load(uniqueID, true, true);
 302         }
 303
 304         /**
 305          * Remove the given resource from the cache.
 306          *
 307          * @param uniqueID
 308          *            a unique ID used to locate the cached resource
 309          *
 310          * @return TRUE if it was removed
 311          */
 312         public boolean removeFromCache(String uniqueID) {
 313                 return cache.remove(uniqueID);
 314         }
 315
 316         /**
 317          * Clean the cache (delete the cached items).
 318          *
 319          * @param onlyOld
 320          *            only clean the files that are considered too old
 321          *
 322          * @return the number of cleaned items
 323          */
 324         public int cleanCache(boolean onlyOld) {
 325                 return cache.clean(onlyOld);
 326         }
 327 }