src/be/nikiroo/fanfix/Cache.java

   1 package be.nikiroo.fanfix;
   2
   3 import java.io.BufferedOutputStream;
   4 import java.io.File;
   5 import java.io.FileInputStream;
   6 import java.io.FileNotFoundException;
   7 import java.io.FileOutputStream;
   8 import java.io.IOException;
   9 import java.io.InputStream;
  10 import java.io.OutputStreamWriter;
  11 import java.net.CookieHandler;
  12 import java.net.CookieManager;
  13 import java.net.CookiePolicy;
  14 import java.net.CookieStore;
  15 import java.net.HttpCookie;
  16 import java.net.HttpURLConnection;
  17 import java.net.URL;
  18 import java.net.URLConnection;
  19 import java.net.URLEncoder;
  20 import java.util.Date;
  21 import java.util.Map;
  22 import java.util.zip.GZIPInputStream;
  23
  24 import javax.imageio.ImageIO;
  25
  26 import be.nikiroo.fanfix.bundles.Config;
  27 import be.nikiroo.fanfix.supported.BasicSupport;
  28 import be.nikiroo.utils.IOUtils;
  29 import be.nikiroo.utils.ImageUtils;
  30 import be.nikiroo.utils.MarkableFileInputStream;
  31
  32 /**
  33  * This cache will manage Internet (and local) downloads, as well as put the
  34  * downloaded files into a cache.
  35  * <p>
  36  * As long the cached resource is not too old, it will use it instead of
  37  * retrieving the file again.
  38  *
  39  * @author niki
  40  */
  41 public class Cache {
  42         private File dir;
  43         private String UA;
  44         private long tooOldChanging;
  45         private long tooOldStable;
  46         private CookieManager cookies;
  47
  48         /**
  49          * Create a new {@link Cache} object.
  50          *
  51          * @param dir
  52          *            the directory to use as cache
  53          * @param UA
  54          *            the User-Agent to use to download the resources
  55          * @param hoursChanging
  56          *            the number of hours after which a cached file that is thought
  57          *            to change ~often is considered too old (or -1 for
  58          *            "never too old")
  59          * @param hoursStable
  60          *            the number of hours after which a LARGE cached file that is
  61          *            thought to change rarely is considered too old (or -1 for
  62          *            "never too old")
  63          *
  64          * @throws IOException
  65          *             in case of I/O error
  66          */
  67         public Cache(File dir, String UA, int hoursChanging, int hoursStable)
  68                         throws IOException {
  69                 this.dir = dir;
  70                 this.UA = UA;
  71                 this.tooOldChanging = 1000 * 60 * 60 * hoursChanging;
  72                 this.tooOldStable = 1000 * 60 * 60 * hoursStable;
  73
  74                 if (dir != null) {
  75                         if (!dir.exists()) {
  76                                 dir.mkdirs();
  77                         }
  78                 }
  79
  80                 if (dir == null || !dir.exists()) {
  81                         throw new IOException("Cannot create the cache directory: "
  82                                         + (dir == null ? "null" : dir.getAbsolutePath()));
  83                 }
  84
  85                 cookies = new CookieManager();
  86                 cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
  87                 CookieHandler.setDefault(cookies);
  88         }
  89
  90         /**
  91          * Clear all the cookies currently in the jar.
  92          */
  93         public void clearCookies() {
  94                 cookies.getCookieStore().removeAll();
  95         }
  96
  97         /**
  98          * Open a resource (will load it from the cache if possible, or save it into
  99          * the cache after downloading if not).
 100          *
 101          * @param url
 102          *            the resource to open
 103          * @param support
 104          *            the support to use to download the resource
 105          * @param stable
 106          *            TRUE for more stable resources, FALSE when they often change
 107          *
 108          * @return the opened resource, NOT NULL
 109          *
 110          * @throws IOException
 111          *             in case of I/O error
 112          */
 113         public InputStream open(URL url, BasicSupport support, boolean stable)
 114                         throws IOException {
 115                 // MUST NOT return null
 116                 return open(url, support, stable, url);
 117         }
 118
 119         /**
 120          * Open a resource (will load it from the cache if possible, or save it into
 121          * the cache after downloading if not).
 122          * <p>
 123          * The cached resource will be assimilated to the given original {@link URL}
 124          *
 125          * @param url
 126          *            the resource to open
 127          * @param support
 128          *            the support to use to download the resource
 129          * @param stable
 130          *            TRUE for more stable resources, FALSE when they often change
 131          * @param originalUrl
 132          *            the original {@link URL} used to locate the cached resource
 133          *
 134          * @return the opened resource, NOT NULL
 135          *
 136          * @throws IOException
 137          *             in case of I/O error
 138          */
 139         public InputStream open(URL url, BasicSupport support, boolean stable,
 140                         URL originalUrl) throws IOException {
 141                 // MUST NOT return null
 142                 try {
 143                         InputStream in = load(originalUrl, false, stable);
 144                         if (in == null) {
 145                                 try {
 146                                         save(url, support, originalUrl);
 147                                 } catch (IOException e) {
 148                                         throw new IOException("Cannot save the url: "
 149                                                         + (url == null ? "null" : url.toString()), e);
 150                                 }
 151
 152                                 // Was just saved, can load old, so, will not be null
 153                                 in = load(originalUrl, true, stable);
 154                         }
 155
 156                         return in;
 157                 } catch (IOException e) {
 158                         throw new IOException("Cannot open the url: "
 159                                         + (url == null ? "null" : url.toString()), e);
 160                 }
 161         }
 162
 163         /**
 164          * Open the given {@link URL} without using the cache, but still using and
 165          * updating the cookies.
 166          *
 167          * @param url
 168          *            the {@link URL} to open
 169          * @param support
 170          *            the {@link BasicSupport} used for the cookies
 171          *
 172          * @return the {@link InputStream} of the opened page
 173          *
 174          * @throws IOException
 175          *             in case of I/O error
 176          */
 177         public InputStream openNoCache(URL url, BasicSupport support)
 178                         throws IOException {
 179                 return openNoCache(url, support, url, null);
 180         }
 181
 182         /**
 183          * Open the given {@link URL} without using the cache, but still using and
 184          * updating the cookies.
 185          *
 186          * @param url
 187          *            the {@link URL} to open
 188          * @param support
 189          *            the {@link BasicSupport} used for the cookies
 190          * @param postParams
 191          *            the POST parameters
 192          *
 193          * @return the {@link InputStream} of the opened page
 194          *
 195          * @throws IOException
 196          *             in case of I/O error
 197          */
 198         public InputStream openNoCache(URL url, BasicSupport support,
 199                         Map<String, String> postParams) throws IOException {
 200                 return openNoCache(url, support, url, postParams);
 201         }
 202
 203         /**
 204          * Open the given {@link URL} without using the cache, but still using and
 205          * updating the cookies.
 206          *
 207          * @param url
 208          *            the {@link URL} to open
 209          * @param support
 210          *            the {@link BasicSupport} used for the cookies
 211          * @param originalUrl
 212          *            the original {@link URL} before any redirection occurs
 213          * @param postParams
 214          *            the POST parameters
 215          *
 216          * @return the {@link InputStream} of the opened page
 217          *
 218          * @throws IOException
 219          *             in case of I/O error
 220          */
 221         private InputStream openNoCache(URL url, BasicSupport support,
 222                         final URL originalUrl, Map<String, String> postParams)
 223                         throws IOException {
 224
 225                 URLConnection conn = openConnectionWithCookies(url, support);
 226                 if (postParams != null && conn instanceof HttpURLConnection) {
 227                         StringBuilder postData = new StringBuilder();
 228                         for (Map.Entry<String, String> param : postParams.entrySet()) {
 229                                 if (postData.length() != 0)
 230                                         postData.append('&');
 231                                 postData.append(URLEncoder.encode(param.getKey(), "UTF-8"));
 232                                 postData.append('=');
 233                                 postData.append(URLEncoder.encode(
 234                                                 String.valueOf(param.getValue()), "UTF-8"));
 235                         }
 236
 237                         conn.setDoOutput(true);
 238                         ((HttpURLConnection) conn).setRequestMethod("POST");
 239                         conn.setRequestProperty("Content-Type",
 240                                         "application/x-www-form-urlencoded");
 241                         conn.setRequestProperty("charset", "utf-8");
 242
 243                         OutputStreamWriter writer = new OutputStreamWriter(
 244                                         conn.getOutputStream());
 245
 246                         writer.write(postData.toString());
 247                         writer.flush();
 248                         writer.close();
 249                 }
 250
 251                 conn.connect();
 252
 253                 // Check if redirect
 254                 if (conn instanceof HttpURLConnection
 255                                 && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) {
 256                         String newUrl = conn.getHeaderField("Location");
 257                         return openNoCache(new URL(newUrl), support, originalUrl,
 258                                         postParams);
 259                 }
 260
 261                 InputStream in = conn.getInputStream();
 262                 if ("gzip".equals(conn.getContentEncoding())) {
 263                         in = new GZIPInputStream(in);
 264                 }
 265
 266                 return in;
 267         }
 268
 269         /**
 270          * Refresh the resource into cache if needed.
 271          *
 272          * @param url
 273          *            the resource to open
 274          * @param support
 275          *            the support to use to download the resource
 276          * @param stable
 277          *            TRUE for more stable resources, FALSE when they often change
 278          *
 279          * @throws IOException
 280          *             in case of I/O error
 281          */
 282         public void refresh(URL url, BasicSupport support, boolean stable)
 283                         throws IOException {
 284                 File cached = getCached(url);
 285                 if (cached.exists() && !isOld(cached, stable)) {
 286                         return;
 287                 }
 288
 289                 open(url, support, stable).close();
 290         }
 291
 292         /**
 293          * Check the resource to see if it is in the cache.
 294          *
 295          * @param url
 296          *            the resource to check
 297          *
 298          * @return TRUE if it is
 299          *
 300          */
 301         public boolean check(URL url) {
 302                 return getCached(url).exists();
 303         }
 304
 305         /**
 306          * Save the given resource as an image on disk using the default image
 307          * format for content.
 308          *
 309          * @param url
 310          *            the resource
 311          * @param target
 312          *            the target file
 313          *
 314          * @throws IOException
 315          *             in case of I/O error
 316          */
 317         public void saveAsImage(URL url, File target) throws IOException {
 318                 URL cachedUrl = new URL(url.toString());
 319                 File cached = getCached(cachedUrl);
 320
 321                 if (!cached.exists() || isOld(cached, true)) {
 322                         InputStream imageIn = open(url, null, true);
 323                         ImageIO.write(ImageUtils.fromStream(imageIn), Instance.getConfig()
 324                                         .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase(),
 325                                         cached);
 326                 }
 327
 328                 IOUtils.write(new FileInputStream(cached), target);
 329         }
 330
 331         /**
 332          * Manually add this item to the cache.
 333          *
 334          * @param in
 335          *            the input data
 336          * @param uniqueID
 337          *            a unique ID for this resource
 338          *
 339          * @return the resulting {@link File}
 340          *
 341          * @throws IOException
 342          *             in case of I/O error
 343          */
 344         public File addToCache(InputStream in, String uniqueID) throws IOException {
 345                 File file = getCached(uniqueID);
 346                 IOUtils.write(in, file);
 347                 return file;
 348         }
 349
 350         /**
 351          * Return the {@link InputStream} corresponding to the given unique ID, or
 352          * NULL if none found.
 353          *
 354          * @param uniqueID
 355          *            the unique ID
 356          *
 357          * @return the content or NULL
 358          */
 359         public InputStream getFromCache(String uniqueID) {
 360                 File file = getCached(uniqueID);
 361                 if (file.exists()) {
 362                         try {
 363                                 return new MarkableFileInputStream(new FileInputStream(file));
 364                         } catch (FileNotFoundException e) {
 365                         }
 366                 }
 367
 368                 return null;
 369         }
 370
 371         /**
 372          * Clean the cache (delete the cached items).
 373          *
 374          * @param onlyOld
 375          *            only clean the files that are considered too old
 376          *
 377          * @return the number of cleaned items
 378          */
 379         public int cleanCache(boolean onlyOld) {
 380                 int num = 0;
 381                 for (File file : dir.listFiles()) {
 382                         if (!onlyOld || isOld(file, true)) {
 383                                 if (file.delete()) {
 384                                         num++;
 385                                 } else {
 386                                         System.err.println("Cannot delete temporary file: "
 387                                                         + file.getAbsolutePath());
 388                                 }
 389                         }
 390                 }
 391                 return num;
 392         }
 393
 394         /**
 395          * Open a resource from the cache if it exists.
 396          *
 397          * @param url
 398          *            the resource to open
 399          * @param allowTooOld
 400          *            allow files even if they are considered too old
 401          * @param stable
 402          *            a stable file (that dones't change too often) -- parameter
 403          *            used to check if the file is too old to keep or not
 404          *
 405          * @return the opened resource if found, NULL i not
 406          *
 407          * @throws IOException
 408          *             in case of I/O error
 409          */
 410         private InputStream load(URL url, boolean allowTooOld, boolean stable)
 411                         throws IOException {
 412                 File cached = getCached(url);
 413                 if (cached.exists() && (allowTooOld || !isOld(cached, stable))) {
 414                         return new MarkableFileInputStream(new FileInputStream(cached));
 415                 }
 416
 417                 return null;
 418         }
 419
 420         /**
 421          * Save the given resource to the cache.
 422          *
 423          * @param url
 424          *            the resource
 425          * @param support
 426          *            the {@link BasicSupport} used to download it
 427          * @param originalUrl
 428          *            the original {@link URL} used to locate the cached resource
 429          *
 430          * @throws IOException
 431          *             in case of I/O error
 432          */
 433         private void save(URL url, BasicSupport support, URL originalUrl)
 434                         throws IOException {
 435                 InputStream in = openNoCache(url, support, originalUrl, null);
 436                 try {
 437                         File cached = getCached(originalUrl);
 438                         BufferedOutputStream out = new BufferedOutputStream(
 439                                         new FileOutputStream(cached));
 440                         try {
 441                                 byte[] buf = new byte[4096];
 442                                 int len;
 443                                 while ((len = in.read(buf)) > 0) {
 444                                         out.write(buf, 0, len);
 445                                 }
 446                         } finally {
 447                                 out.close();
 448                         }
 449                 } finally {
 450                         in.close();
 451                 }
 452         }
 453
 454         /**
 455          * Open a connection on the given {@link URL}, and manage the cookies that
 456          * come with it.
 457          *
 458          * @param url
 459          *            the {@link URL} to open
 460          * @param support
 461          *            the {@link BasicSupport} to use for cookie generation
 462          *
 463          * @return the connection
 464          *
 465          * @throws IOException
 466          *             in case of I/O error
 467          */
 468         private URLConnection openConnectionWithCookies(URL url,
 469                         BasicSupport support) throws IOException {
 470                 URLConnection conn = url.openConnection();
 471
 472                 conn.setRequestProperty("User-Agent", UA);
 473                 conn.setRequestProperty("Cookie", generateCookies(support));
 474                 conn.setRequestProperty("Accept-Encoding", "gzip");
 475                 if (support != null && support.getCurrentReferer() != null) {
 476                         conn.setRequestProperty("Referer", support.getCurrentReferer()
 477                                         .toString());
 478                         conn.setRequestProperty("Host", support.getCurrentReferer()
 479                                         .getHost());
 480                 }
 481
 482                 return conn;
 483         }
 484
 485         /**
 486          * Check if the {@link File} is too old according to
 487          * {@link Cache#tooOldChanging}.
 488          *
 489          * @param file
 490          *            the file to check
 491          * @param stable
 492          *            TRUE to denote files that are not supposed to change too often
 493          *
 494          * @return TRUE if it is
 495          */
 496         private boolean isOld(File file, boolean stable) {
 497                 long max = tooOldChanging;
 498                 if (stable) {
 499                         max = tooOldStable;
 500                 }
 501
 502                 if (max < 0) {
 503                         return false;
 504                 }
 505
 506                 long time = new Date().getTime() - file.lastModified();
 507                 if (time < 0) {
 508                         System.err.println("Timestamp in the future for file: "
 509                                         + file.getAbsolutePath());
 510                 }
 511
 512                 return time < 0 || time > max;
 513         }
 514
 515         /**
 516          * Get the cache resource from the cache if it is present for this
 517          * {@link URL}.
 518          *
 519          * @param url
 520          *            the url
 521          *
 522          * @return the cached version if present, NULL if not
 523          */
 524         private File getCached(URL url) {
 525                 String name = url.getHost();
 526                 if (name == null || name.isEmpty()) {
 527                         name = url.getFile();
 528                 } else {
 529                         name = url.toString();
 530                 }
 531
 532                 return getCached(name);
 533         }
 534
 535         /**
 536          * Get the cache resource from the cache if it is present for this unique
 537          * ID.
 538          *
 539          * @param uniqueID
 540          *            the id
 541          *
 542          * @return the cached version if present, NULL if not
 543          */
 544         private File getCached(String uniqueID) {
 545                 uniqueID = uniqueID.replace('/', '_').replace(':', '_')
 546                                 .replace("\\", "_");
 547
 548                 return new File(dir, uniqueID);
 549         }
 550
 551         /**
 552          * Generate the cookie {@link String} from the local {@link CookieStore} so
 553          * it is ready to be passed.
 554          *
 555          * @return the cookie
 556          */
 557         private String generateCookies(BasicSupport support) {
 558                 StringBuilder builder = new StringBuilder();
 559                 for (HttpCookie cookie : cookies.getCookieStore().getCookies()) {
 560                         if (builder.length() > 0) {
 561                                 builder.append(';');
 562                         }
 563
 564                         // TODO: check if format is ok
 565                         builder.append(cookie.toString());
 566                 }
 567
 568                 if (support != null) {
 569                         try {
 570                                 for (Map.Entry<String, String> set : support.getCookies()
 571                                                 .entrySet()) {
 572                                         if (builder.length() > 0) {
 573                                                 builder.append(';');
 574                                         }
 575                                         builder.append(set.getKey());
 576                                         builder.append('=');
 577                                         builder.append(set.getValue());
 578                                 }
 579                         } catch (IOException e) {
 580                                 Instance.syserr(e);
 581                         }
 582                 }
 583
 584                 return builder.toString();
 585         }
 586 }