src/be/nikiroo/fanfix/Cache.java

   1 package be.nikiroo.fanfix;
   2
   3 import java.io.BufferedOutputStream;
   4 import java.io.File;
   5 import java.io.FileInputStream;
   6 import java.io.FileNotFoundException;
   7 import java.io.FileOutputStream;
   8 import java.io.IOException;
   9 import java.io.InputStream;
  10 import java.io.OutputStreamWriter;
  11 import java.net.CookieHandler;
  12 import java.net.CookieManager;
  13 import java.net.CookiePolicy;
  14 import java.net.CookieStore;
  15 import java.net.HttpCookie;
  16 import java.net.HttpURLConnection;
  17 import java.net.URL;
  18 import java.net.URLConnection;
  19 import java.net.URLEncoder;
  20 import java.util.Date;
  21 import java.util.Map;
  22 import java.util.zip.GZIPInputStream;
  23
  24 import javax.imageio.ImageIO;
  25
  26 import be.nikiroo.fanfix.bundles.Config;
  27 import be.nikiroo.fanfix.supported.BasicSupport;
  28 import be.nikiroo.utils.IOUtils;
  29 import be.nikiroo.utils.MarkableFileInputStream;
  30
  31 /**
  32  * This cache will manage Internet (and local) downloads, as well as put the
  33  * downloaded files into a cache.
  34  * <p>
  35  * As long the cached resource is not too old, it will use it instead of
  36  * retrieving the file again.
  37  *
  38  * @author niki
  39  */
  40 public class Cache {
  41         private File dir;
  42         private String UA;
  43         private long tooOldChanging;
  44         private long tooOldStable;
  45         private CookieManager cookies;
  46
  47         /**
  48          * Create a new {@link Cache} object.
  49          *
  50          * @param dir
  51          *            the directory to use as cache
  52          * @param UA
  53          *            the User-Agent to use to download the resources
  54          * @param hoursChanging
  55          *            the number of hours after which a cached file that is thought
  56          *            to change ~often is considered too old (or -1 for
  57          *            "never too old")
  58          * @param hoursStable
  59          *            the number of hours after which a LARGE cached file that is
  60          *            thought to change rarely is considered too old (or -1 for
  61          *            "never too old")
  62          *
  63          * @throws IOException
  64          *             in case of I/O error
  65          */
  66         public Cache(File dir, String UA, int hoursChanging, int hoursStable)
  67                         throws IOException {
  68                 this.dir = dir;
  69                 this.UA = UA;
  70                 this.tooOldChanging = 1000 * 60 * 60 * hoursChanging;
  71                 this.tooOldStable = 1000 * 60 * 60 * hoursStable;
  72
  73                 if (dir != null) {
  74                         if (!dir.exists()) {
  75                                 dir.mkdirs();
  76                         }
  77                 }
  78
  79                 if (dir == null || !dir.exists()) {
  80                         throw new IOException("Cannot create the cache directory: "
  81                                         + (dir == null ? "null" : dir.getAbsolutePath()));
  82                 }
  83
  84                 cookies = new CookieManager();
  85                 cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
  86                 CookieHandler.setDefault(cookies);
  87         }
  88
  89         /**
  90          * Clear all the cookies currently in the jar.
  91          */
  92         public void clearCookies() {
  93                 cookies.getCookieStore().removeAll();
  94         }
  95
  96         /**
  97          * Open a resource (will load it from the cache if possible, or save it into
  98          * the cache after downloading if not).
  99          *
 100          * @param url
 101          *            the resource to open
 102          * @param support
 103          *            the support to use to download the resource
 104          * @param stable
 105          *            TRUE for more stable resources, FALSE when they often change
 106          *
 107          * @return the opened resource, NOT NULL
 108          *
 109          * @throws IOException
 110          *             in case of I/O error
 111          */
 112         public InputStream open(URL url, BasicSupport support, boolean stable)
 113                         throws IOException {
 114                 // MUST NOT return null
 115                 return open(url, support, stable, url);
 116         }
 117
 118         /**
 119          * Open a resource (will load it from the cache if possible, or save it into
 120          * the cache after downloading if not).
 121          * <p>
 122          * The cached resource will be assimilated to the given original {@link URL}
 123          *
 124          * @param url
 125          *            the resource to open
 126          * @param support
 127          *            the support to use to download the resource
 128          * @param stable
 129          *            TRUE for more stable resources, FALSE when they often change
 130          * @param originalUrl
 131          *            the original {@link URL} used to locate the cached resource
 132          *
 133          * @return the opened resource, NOT NULL
 134          *
 135          * @throws IOException
 136          *             in case of I/O error
 137          */
 138         public InputStream open(URL url, BasicSupport support, boolean stable,
 139                         URL originalUrl) throws IOException {
 140                 // MUST NOT return null
 141                 try {
 142                         InputStream in = load(originalUrl, false, stable);
 143                         if (in == null) {
 144                                 try {
 145                                         save(url, support, originalUrl);
 146                                 } catch (IOException e) {
 147                                         throw new IOException("Cannot save the url: "
 148                                                         + (url == null ? "null" : url.toString()), e);
 149                                 }
 150
 151                                 // Was just saved, can load old, so, will not be null
 152                                 in = load(originalUrl, true, stable);
 153                         }
 154
 155                         return in;
 156                 } catch (IOException e) {
 157                         throw new IOException("Cannot open the url: "
 158                                         + (url == null ? "null" : url.toString()), e);
 159                 }
 160         }
 161
 162         /**
 163          * Open the given {@link URL} without using the cache, but still using and
 164          * updating the cookies.
 165          *
 166          * @param url
 167          *            the {@link URL} to open
 168          * @param support
 169          *            the {@link BasicSupport} used for the cookies
 170          *
 171          * @return the {@link InputStream} of the opened page
 172          *
 173          * @throws IOException
 174          *             in case of I/O error
 175          */
 176         public InputStream openNoCache(URL url, BasicSupport support)
 177                         throws IOException {
 178                 return openNoCache(url, support, url, null);
 179         }
 180
 181         /**
 182          * Open the given {@link URL} without using the cache, but still using and
 183          * updating the cookies.
 184          *
 185          * @param url
 186          *            the {@link URL} to open
 187          * @param support
 188          *            the {@link BasicSupport} used for the cookies
 189          * @param postParams
 190          *            the POST parameters
 191          *
 192          * @return the {@link InputStream} of the opened page
 193          *
 194          * @throws IOException
 195          *             in case of I/O error
 196          */
 197         public InputStream openNoCache(URL url, BasicSupport support,
 198                         Map<String, String> postParams) throws IOException {
 199                 return openNoCache(url, support, url, postParams);
 200         }
 201
 202         /**
 203          * Open the given {@link URL} without using the cache, but still using and
 204          * updating the cookies.
 205          *
 206          * @param url
 207          *            the {@link URL} to open
 208          * @param support
 209          *            the {@link BasicSupport} used for the cookies
 210          * @param originalUrl
 211          *            the original {@link URL} before any redirection occurs
 212          * @param postParams
 213          *            the POST parameters
 214          *
 215          * @return the {@link InputStream} of the opened page
 216          *
 217          * @throws IOException
 218          *             in case of I/O error
 219          */
 220         private InputStream openNoCache(URL url, BasicSupport support,
 221                         final URL originalUrl, Map<String, String> postParams)
 222                         throws IOException {
 223
 224                 URLConnection conn = openConnectionWithCookies(url, support);
 225                 if (postParams != null && conn instanceof HttpURLConnection) {
 226                         StringBuilder postData = new StringBuilder();
 227                         for (Map.Entry<String, String> param : postParams.entrySet()) {
 228                                 if (postData.length() != 0)
 229                                         postData.append('&');
 230                                 postData.append(URLEncoder.encode(param.getKey(), "UTF-8"));
 231                                 postData.append('=');
 232                                 postData.append(URLEncoder.encode(
 233                                                 String.valueOf(param.getValue()), "UTF-8"));
 234                         }
 235
 236                         conn.setDoOutput(true);
 237                         ((HttpURLConnection) conn).setRequestMethod("POST");
 238                         conn.setRequestProperty("Content-Type",
 239                                         "application/x-www-form-urlencoded");
 240                         conn.setRequestProperty("charset", "utf-8");
 241
 242                         OutputStreamWriter writer = new OutputStreamWriter(
 243                                         conn.getOutputStream());
 244
 245                         writer.write(postData.toString());
 246                         writer.flush();
 247                         writer.close();
 248                 }
 249
 250                 conn.connect();
 251
 252                 // Check if redirect
 253                 if (conn instanceof HttpURLConnection
 254                                 && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) {
 255                         String newUrl = conn.getHeaderField("Location");
 256                         return openNoCache(new URL(newUrl), support, originalUrl,
 257                                         postParams);
 258                 }
 259
 260                 InputStream in = conn.getInputStream();
 261                 if ("gzip".equals(conn.getContentEncoding())) {
 262                         in = new GZIPInputStream(in);
 263                 }
 264
 265                 return in;
 266         }
 267
 268         /**
 269          * Refresh the resource into cache if needed.
 270          *
 271          * @param url
 272          *            the resource to open
 273          * @param support
 274          *            the support to use to download the resource
 275          * @param stable
 276          *            TRUE for more stable resources, FALSE when they often change
 277          *
 278          * @throws IOException
 279          *             in case of I/O error
 280          */
 281         public void refresh(URL url, BasicSupport support, boolean stable)
 282                         throws IOException {
 283                 File cached = getCached(url);
 284                 if (cached.exists() && !isOld(cached, stable)) {
 285                         return;
 286                 }
 287
 288                 open(url, support, stable).close();
 289         }
 290
 291         /**
 292          * Check the resource to see if it is in the cache.
 293          *
 294          * @param url
 295          *            the resource to check
 296          *
 297          * @return TRUE if it is
 298          *
 299          */
 300         public boolean check(URL url) {
 301                 return getCached(url).exists();
 302         }
 303
 304         /**
 305          * Save the given resource as an image on disk using the default image
 306          * format for content.
 307          *
 308          * @param url
 309          *            the resource
 310          * @param target
 311          *            the target file
 312          *
 313          * @throws IOException
 314          *             in case of I/O error
 315          */
 316         public void saveAsImage(URL url, File target) throws IOException {
 317                 URL cachedUrl = new URL(url.toString());
 318                 File cached = getCached(cachedUrl);
 319
 320                 if (!cached.exists() || isOld(cached, true)) {
 321                         InputStream imageIn = open(url, null, true);
 322                         ImageIO.write(IOUtils.toImage(imageIn), Instance.getConfig()
 323                                         .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase(),
 324                                         cached);
 325                 }
 326
 327                 IOUtils.write(new FileInputStream(cached), target);
 328         }
 329
 330         /**
 331          * Manually add this item to the cache.
 332          *
 333          * @param in
 334          *            the input data
 335          * @param uniqueID
 336          *            a unique ID for this resource
 337          *
 338          * @return the resulting {@link File}
 339          *
 340          * @throws IOException
 341          *             in case of I/O error
 342          */
 343         public File addToCache(InputStream in, String uniqueID) throws IOException {
 344                 File file = getCached(uniqueID);
 345                 IOUtils.write(in, file);
 346                 return file;
 347         }
 348
 349         /**
 350          * Return the {@link InputStream} corresponding to the given unique ID, or
 351          * NULL if none found.
 352          *
 353          * @param uniqueID
 354          *            the unique ID
 355          *
 356          * @return the content or NULL
 357          */
 358         public InputStream getFromCache(String uniqueID) {
 359                 File file = getCached(uniqueID);
 360                 if (file.exists()) {
 361                         try {
 362                                 return new MarkableFileInputStream(new FileInputStream(file));
 363                         } catch (FileNotFoundException e) {
 364                         }
 365                 }
 366
 367                 return null;
 368         }
 369
 370         /**
 371          * Clean the cache (delete the cached items).
 372          *
 373          * @param onlyOld
 374          *            only clean the files that are considered too old
 375          *
 376          * @return the number of cleaned items
 377          */
 378         public int cleanCache(boolean onlyOld) {
 379                 int num = 0;
 380                 for (File file : dir.listFiles()) {
 381                         if (!onlyOld || isOld(file, true)) {
 382                                 if (file.delete()) {
 383                                         num++;
 384                                 } else {
 385                                         System.err.println("Cannot delete temporary file: "
 386                                                         + file.getAbsolutePath());
 387                                 }
 388                         }
 389                 }
 390                 return num;
 391         }
 392
 393         /**
 394          * Open a resource from the cache if it exists.
 395          *
 396          * @param url
 397          *            the resource to open
 398          *
 399          * @return the opened resource if found, NULL i not
 400          *
 401          * @throws IOException
 402          *             in case of I/O error
 403          */
 404         private InputStream load(URL url, boolean allowOld, boolean stable)
 405                         throws IOException {
 406                 File cached = getCached(url);
 407                 if (cached.exists() && !isOld(cached, stable)) {
 408                         return new MarkableFileInputStream(new FileInputStream(cached));
 409                 }
 410
 411                 return null;
 412         }
 413
 414         /**
 415          * Save the given resource to the cache.
 416          *
 417          * @param url
 418          *            the resource
 419          * @param support
 420          *            the {@link BasicSupport} used to download it
 421          * @param originalUrl
 422          *            the original {@link URL} used to locate the cached resource
 423          *
 424          * @throws IOException
 425          *             in case of I/O error
 426          */
 427         private void save(URL url, BasicSupport support, URL originalUrl)
 428                         throws IOException {
 429                 InputStream in = openNoCache(url, support, originalUrl, null);
 430                 try {
 431                         File cached = getCached(originalUrl);
 432                         BufferedOutputStream out = new BufferedOutputStream(
 433                                         new FileOutputStream(cached));
 434                         try {
 435                                 byte[] buf = new byte[4096];
 436                                 int len;
 437                                 while ((len = in.read(buf)) > 0) {
 438                                         out.write(buf, 0, len);
 439                                 }
 440                         } finally {
 441                                 out.close();
 442                         }
 443                 } finally {
 444                         in.close();
 445                 }
 446         }
 447
 448         /**
 449          * Open a connection on the given {@link URL}, and manage the cookies that
 450          * come with it.
 451          *
 452          * @param url
 453          *            the {@link URL} to open
 454          * @param support
 455          *            the {@link BasicSupport} to use for cookie generation
 456          *
 457          * @return the connection
 458          *
 459          * @throws IOException
 460          *             in case of I/O error
 461          */
 462         private URLConnection openConnectionWithCookies(URL url,
 463                         BasicSupport support) throws IOException {
 464                 URLConnection conn = url.openConnection();
 465
 466                 conn.setRequestProperty("User-Agent", UA);
 467                 conn.setRequestProperty("Cookie", generateCookies(support));
 468                 conn.setRequestProperty("Accept-Encoding", "gzip");
 469                 if (support != null && support.getCurrentReferer() != null) {
 470                         conn.setRequestProperty("Referer", support.getCurrentReferer()
 471                                         .toString());
 472                         conn.setRequestProperty("Host", support.getCurrentReferer()
 473                                         .getHost());
 474                 }
 475
 476                 return conn;
 477         }
 478
 479         /**
 480          * Check if the {@link File} is too old according to
 481          * {@link Cache#tooOldChanging}.
 482          *
 483          * @param file
 484          *            the file to check
 485          * @param stable
 486          *            TRUE to denote files that are not supposed to change too often
 487          *
 488          * @return TRUE if it is
 489          */
 490         private boolean isOld(File file, boolean stable) {
 491                 long max = tooOldChanging;
 492                 if (stable) {
 493                         max = tooOldStable;
 494                 }
 495
 496                 if (max < 0) {
 497                         return false;
 498                 }
 499
 500                 long time = new Date().getTime() - file.lastModified();
 501                 if (time < 0) {
 502                         System.err.println("Timestamp in the future for file: "
 503                                         + file.getAbsolutePath());
 504                 }
 505
 506                 return time < 0 || time > max;
 507         }
 508
 509         /**
 510          * Get the cache resource from the cache if it is present for this
 511          * {@link URL}.
 512          *
 513          * @param url
 514          *            the url
 515          *
 516          * @return the cached version if present, NULL if not
 517          */
 518         private File getCached(URL url) {
 519                 String name = url.getHost();
 520                 if (name == null || name.isEmpty()) {
 521                         name = url.getFile();
 522                 } else {
 523                         name = url.toString();
 524                 }
 525
 526                 return getCached(name);
 527         }
 528
 529         /**
 530          * Get the cache resource from the cache if it is present for this unique
 531          * ID.
 532          *
 533          * @param uniqueID
 534          *            the id
 535          *
 536          * @return the cached version if present, NULL if not
 537          */
 538         private File getCached(String uniqueID) {
 539                 uniqueID = uniqueID.replace('/', '_').replace(':', '_')
 540                                 .replace("\\", "_");
 541
 542                 return new File(dir, uniqueID);
 543         }
 544
 545         /**
 546          * Generate the cookie {@link String} from the local {@link CookieStore} so
 547          * it is ready to be passed.
 548          *
 549          * @return the cookie
 550          */
 551         private String generateCookies(BasicSupport support) {
 552                 StringBuilder builder = new StringBuilder();
 553                 for (HttpCookie cookie : cookies.getCookieStore().getCookies()) {
 554                         if (builder.length() > 0) {
 555                                 builder.append(';');
 556                         }
 557
 558                         // TODO: check if format is ok
 559                         builder.append(cookie.toString());
 560                 }
 561
 562                 if (support != null) {
 563                         try {
 564                                 for (Map.Entry<String, String> set : support.getCookies()
 565                                                 .entrySet()) {
 566                                         if (builder.length() > 0) {
 567                                                 builder.append(';');
 568                                         }
 569                                         builder.append(set.getKey());
 570                                         builder.append('=');
 571                                         builder.append(set.getValue());
 572                                 }
 573                         } catch (IOException e) {
 574                                 Instance.syserr(e);
 575                         }
 576                 }
 577
 578                 return builder.toString();
 579         }
 580 }