src/be/nikiroo/fanfix/Cache.java

   1 package be.nikiroo.fanfix;
   2
   3 import java.io.BufferedOutputStream;
   4 import java.io.File;
   5 import java.io.FileInputStream;
   6 import java.io.FileNotFoundException;
   7 import java.io.FileOutputStream;
   8 import java.io.IOException;
   9 import java.io.InputStream;
  10 import java.io.OutputStreamWriter;
  11 import java.net.CookieHandler;
  12 import java.net.CookieManager;
  13 import java.net.CookiePolicy;
  14 import java.net.CookieStore;
  15 import java.net.HttpCookie;
  16 import java.net.HttpURLConnection;
  17 import java.net.URL;
  18 import java.net.URLConnection;
  19 import java.net.URLEncoder;
  20 import java.util.Date;
  21 import java.util.Map;
  22 import java.util.zip.GZIPInputStream;
  23
  24 import javax.imageio.ImageIO;
  25
  26 import be.nikiroo.fanfix.bundles.Config;
  27 import be.nikiroo.fanfix.supported.BasicSupport;
  28 import be.nikiroo.utils.IOUtils;
  29 import be.nikiroo.utils.ImageUtils;
  30 import be.nikiroo.utils.MarkableFileInputStream;
  31
  32 /**
  33  * This cache will manage Internet (and local) downloads, as well as put the
  34  * downloaded files into a cache.
  35  * <p>
  36  * As long the cached resource is not too old, it will use it instead of
  37  * retrieving the file again.
  38  *
  39  * @author niki
  40  */
  41 public class Cache {
  42         private File dir;
  43         private String UA;
  44         private long tooOldChanging;
  45         private long tooOldStable;
  46         private CookieManager cookies;
  47
  48         /**
  49          * Create a new {@link Cache} object.
  50          *
  51          * @param dir
  52          *            the directory to use as cache
  53          * @param UA
  54          *            the User-Agent to use to download the resources
  55          * @param hoursChanging
  56          *            the number of hours after which a cached file that is thought
  57          *            to change ~often is considered too old (or -1 for
  58          *            "never too old")
  59          * @param hoursStable
  60          *            the number of hours after which a LARGE cached file that is
  61          *            thought to change rarely is considered too old (or -1 for
  62          *            "never too old")
  63          *
  64          * @throws IOException
  65          *             in case of I/O error
  66          */
  67         public Cache(File dir, String UA, int hoursChanging, int hoursStable)
  68                         throws IOException {
  69                 this.dir = dir;
  70                 this.UA = UA;
  71                 this.tooOldChanging = 1000 * 60 * 60 * hoursChanging;
  72                 this.tooOldStable = 1000 * 60 * 60 * hoursStable;
  73
  74                 if (dir != null) {
  75                         if (!dir.exists()) {
  76                                 dir.mkdirs();
  77                         }
  78                 }
  79
  80                 if (dir == null || !dir.exists()) {
  81                         throw new IOException("Cannot create the cache directory: "
  82                                         + (dir == null ? "null" : dir.getAbsolutePath()));
  83                 }
  84
  85                 cookies = new CookieManager();
  86                 cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
  87                 CookieHandler.setDefault(cookies);
  88         }
  89
  90         /**
  91          * Clear all the cookies currently in the jar.
  92          */
  93         public void clearCookies() {
  94                 cookies.getCookieStore().removeAll();
  95         }
  96
  97         /**
  98          * Open a resource (will load it from the cache if possible, or save it into
  99          * the cache after downloading if not).
 100          *
 101          * @param url
 102          *            the resource to open
 103          * @param support
 104          *            the support to use to download the resource
 105          * @param stable
 106          *            TRUE for more stable resources, FALSE when they often change
 107          *
 108          * @return the opened resource, NOT NULL
 109          *
 110          * @throws IOException
 111          *             in case of I/O error
 112          */
 113         public InputStream open(URL url, BasicSupport support, boolean stable)
 114                         throws IOException {
 115                 // MUST NOT return null
 116                 return open(url, support, stable, url);
 117         }
 118
 119         /**
 120          * Open a resource (will load it from the cache if possible, or save it into
 121          * the cache after downloading if not).
 122          * <p>
 123          * The cached resource will be assimilated to the given original {@link URL}
 124          *
 125          * @param url
 126          *            the resource to open
 127          * @param support
 128          *            the support to use to download the resource
 129          * @param stable
 130          *            TRUE for more stable resources, FALSE when they often change
 131          * @param originalUrl
 132          *            the original {@link URL} used to locate the cached resource
 133          *
 134          * @return the opened resource, NOT NULL
 135          *
 136          * @throws IOException
 137          *             in case of I/O error
 138          */
 139         public InputStream open(URL url, BasicSupport support, boolean stable,
 140                         URL originalUrl) throws IOException {
 141                 // MUST NOT return null
 142                 try {
 143                         InputStream in = load(originalUrl, false, stable);
 144                         Instance.trace("Cache " + (in != null ? "hit" : "miss") + ": "
 145                                         + url);
 146
 147                         if (in == null) {
 148
 149                                 try {
 150                                         save(url, support, originalUrl);
 151                                 } catch (IOException e) {
 152                                         throw new IOException("Cannot save the url: "
 153                                                         + (url == null ? "null" : url.toString()), e);
 154                                 }
 155
 156                                 // Was just saved, can load old, so, will not be null
 157                                 in = load(originalUrl, true, stable);
 158                         }
 159
 160                         return in;
 161                 } catch (IOException e) {
 162                         throw new IOException("Cannot open the url: "
 163                                         + (url == null ? "null" : url.toString()), e);
 164                 }
 165         }
 166
 167         /**
 168          * Open the given {@link URL} without using the cache, but still using and
 169          * updating the cookies.
 170          *
 171          * @param url
 172          *            the {@link URL} to open
 173          * @param support
 174          *            the {@link BasicSupport} used for the cookies
 175          *
 176          * @return the {@link InputStream} of the opened page
 177          *
 178          * @throws IOException
 179          *             in case of I/O error
 180          */
 181         public InputStream openNoCache(URL url, BasicSupport support)
 182                         throws IOException {
 183                 return openNoCache(url, support, url, null, null, null);
 184         }
 185
 186         /**
 187          * Open the given {@link URL} without using the cache, but still using and
 188          * updating the cookies.
 189          *
 190          * @param url
 191          *            the {@link URL} to open
 192          * @param support
 193          *            the {@link BasicSupport} used for the cookies
 194          * @param postParams
 195          *            the POST parameters
 196          * @param getParams
 197          *            the GET parameters (priority over POST)
 198          * @param oauth
 199          *            OAuth authorization (aka, "bearer XXXXXXX")
 200          *
 201          * @return the {@link InputStream} of the opened page
 202          *
 203          * @throws IOException
 204          *             in case of I/O error
 205          */
 206         public InputStream openNoCache(URL url, BasicSupport support,
 207                         Map<String, String> postParams, Map<String, String> getParams,
 208                         String oauth) throws IOException {
 209                 return openNoCache(url, support, url, postParams, getParams, oauth);
 210         }
 211
 212         /**
 213          * Open the given {@link URL} without using the cache, but still using and
 214          * updating the cookies.
 215          *
 216          * @param url
 217          *            the {@link URL} to open
 218          * @param support
 219          *            the {@link BasicSupport} used for the cookies
 220          * @param originalUrl
 221          *            the original {@link URL} before any redirection occurs
 222          * @param postParams
 223          *            the POST parameters
 224          * @param getParams
 225          *            the GET parameters (priority over POST)
 226          * @param oauth
 227          *            OAuth authorisation (aka, "bearer XXXXXXX")
 228          * @return the {@link InputStream} of the opened page
 229          *
 230          * @throws IOException
 231          *             in case of I/O error
 232          */
 233         private InputStream openNoCache(URL url, BasicSupport support,
 234                         final URL originalUrl, Map<String, String> postParams,
 235                         Map<String, String> getParams, String oauth) throws IOException {
 236
 237                 Instance.trace("Open no cache: " + url);
 238
 239                 URLConnection conn = openConnectionWithCookies(url, support);
 240                 if (support != null) {
 241                         // priority: arguments
 242                         if (oauth == null) {
 243                                 oauth = support.getOAuth();
 244                         }
 245                 }
 246
 247                 // Priority: GET over POST
 248                 Map<String, String> params = getParams;
 249                 if (getParams == null) {
 250                         params = postParams;
 251                 }
 252
 253                 if ((params != null || oauth != null)
 254                                 && conn instanceof HttpURLConnection) {
 255                         StringBuilder requestData = null;
 256                         if (params != null) {
 257                                 requestData = new StringBuilder();
 258                                 for (Map.Entry<String, String> param : params.entrySet()) {
 259                                         if (requestData.length() != 0)
 260                                                 requestData.append('&');
 261                                         requestData.append(URLEncoder.encode(param.getKey(),
 262                                                         "UTF-8"));
 263                                         requestData.append('=');
 264                                         requestData.append(URLEncoder.encode(
 265                                                         String.valueOf(param.getValue()), "UTF-8"));
 266                                 }
 267
 268                                 conn.setDoOutput(true);
 269
 270                                 if (getParams == null && postParams != null) {
 271                                         ((HttpURLConnection) conn).setRequestMethod("POST");
 272                                 }
 273
 274                                 conn.setRequestProperty("Content-Type",
 275                                                 "application/x-www-form-urlencoded");
 276                                 conn.setRequestProperty("charset", "utf-8");
 277                         }
 278
 279                         if (oauth != null) {
 280                                 conn.setRequestProperty("Authorization", oauth);
 281                         }
 282
 283                         if (requestData != null) {
 284                                 OutputStreamWriter writer = new OutputStreamWriter(
 285                                                 conn.getOutputStream());
 286
 287                                 writer.write(requestData.toString());
 288                                 writer.flush();
 289                                 writer.close();
 290                         }
 291                 }
 292
 293                 conn.connect();
 294
 295                 // Check if redirect
 296                 if (conn instanceof HttpURLConnection
 297                                 && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) {
 298                         String newUrl = conn.getHeaderField("Location");
 299                         return openNoCache(new URL(newUrl), support, originalUrl,
 300                                         postParams, getParams, oauth);
 301                 }
 302
 303                 InputStream in = conn.getInputStream();
 304                 if ("gzip".equals(conn.getContentEncoding())) {
 305                         in = new GZIPInputStream(in);
 306                 }
 307
 308                 return in;
 309         }
 310
 311         /**
 312          * Refresh the resource into cache if needed.
 313          *
 314          * @param url
 315          *            the resource to open
 316          * @param support
 317          *            the support to use to download the resource
 318          * @param stable
 319          *            TRUE for more stable resources, FALSE when they often change
 320          *
 321          * @throws IOException
 322          *             in case of I/O error
 323          */
 324         public void refresh(URL url, BasicSupport support, boolean stable)
 325                         throws IOException {
 326                 File cached = getCached(url);
 327                 if (cached.exists() && !isOld(cached, stable)) {
 328                         return;
 329                 }
 330
 331                 open(url, support, stable).close();
 332         }
 333
 334         /**
 335          * Check the resource to see if it is in the cache.
 336          *
 337          * @param url
 338          *            the resource to check
 339          *
 340          * @return TRUE if it is
 341          *
 342          */
 343         public boolean check(URL url) {
 344                 return getCached(url).exists();
 345         }
 346
 347         /**
 348          * Save the given resource as an image on disk using the default image
 349          * format for content.
 350          *
 351          * @param url
 352          *            the resource
 353          * @param target
 354          *            the target file
 355          *
 356          * @throws IOException
 357          *             in case of I/O error
 358          */
 359         public void saveAsImage(URL url, File target) throws IOException {
 360                 URL cachedUrl = new URL(url.toString());
 361                 File cached = getCached(cachedUrl);
 362
 363                 if (!cached.exists() || isOld(cached, true)) {
 364                         InputStream imageIn = open(url, null, true);
 365                         ImageIO.write(ImageUtils.fromStream(imageIn), Instance.getConfig()
 366                                         .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase(),
 367                                         cached);
 368                 }
 369
 370                 IOUtils.write(new FileInputStream(cached), target);
 371         }
 372
 373         /**
 374          * Manually add this item to the cache.
 375          *
 376          * @param in
 377          *            the input data
 378          * @param uniqueID
 379          *            a unique ID for this resource
 380          *
 381          * @return the resulting {@link File}
 382          *
 383          * @throws IOException
 384          *             in case of I/O error
 385          */
 386         public File addToCache(InputStream in, String uniqueID) throws IOException {
 387                 File file = getCached(uniqueID);
 388                 File subdir = new File(file.getParentFile(), "_");
 389                 file = new File(subdir, file.getName());
 390                 subdir.mkdir();
 391                 IOUtils.write(in, file);
 392                 return file;
 393         }
 394
 395         /**
 396          * Return the {@link InputStream} corresponding to the given unique ID, or
 397          * NULL if none found.
 398          *
 399          * @param uniqueID
 400          *            the unique ID
 401          *
 402          * @return the content or NULL
 403          */
 404         public InputStream getFromCache(String uniqueID) {
 405                 File file = getCached(uniqueID);
 406                 File subdir = new File(file.getParentFile(), "_");
 407                 file = new File(subdir, file.getName());
 408                 if (file.exists()) {
 409                         try {
 410                                 return new MarkableFileInputStream(new FileInputStream(file));
 411                         } catch (FileNotFoundException e) {
 412                         }
 413                 }
 414
 415                 return null;
 416         }
 417
 418         /**
 419          * Clean the cache (delete the cached items).
 420          *
 421          * @param onlyOld
 422          *            only clean the files that are considered too old
 423          *
 424          * @return the number of cleaned items
 425          */
 426         public int cleanCache(boolean onlyOld) {
 427                 return cleanCache(onlyOld, dir);
 428         }
 429
 430         /**
 431          * Clean the cache (delete the cached items) in the given cache directory.
 432          *
 433          * @param onlyOld
 434          *            only clean the files that are considered too old
 435          * @param cacheDir
 436          *            the cache directory to clean
 437          *
 438          * @return the number of cleaned items
 439          */
 440         private int cleanCache(boolean onlyOld, File cacheDir) {
 441                 int num = 0;
 442                 for (File file : cacheDir.listFiles()) {
 443                         if (file.isDirectory()) {
 444                                 num += cleanCache(onlyOld, file);
 445                         } else {
 446                                 if (!onlyOld || isOld(file, true)) {
 447                                         if (file.delete()) {
 448                                                 num++;
 449                                         } else {
 450                                                 System.err.println("Cannot delete temporary file: "
 451                                                                 + file.getAbsolutePath());
 452                                         }
 453                                 }
 454                         }
 455                 }
 456
 457                 return num;
 458         }
 459
 460         /**
 461          * Open a resource from the cache if it exists.
 462          *
 463          * @param url
 464          *            the resource to open
 465          * @param allowTooOld
 466          *            allow files even if they are considered too old
 467          * @param stable
 468          *            a stable file (that dones't change too often) -- parameter
 469          *            used to check if the file is too old to keep or not
 470          *
 471          * @return the opened resource if found, NULL i not
 472          *
 473          * @throws IOException
 474          *             in case of I/O error
 475          */
 476         private InputStream load(URL url, boolean allowTooOld, boolean stable)
 477                         throws IOException {
 478                 File cached = getCached(url);
 479                 if (cached.exists() && (allowTooOld || !isOld(cached, stable))) {
 480                         return new MarkableFileInputStream(new FileInputStream(cached));
 481                 }
 482
 483                 return null;
 484         }
 485
 486         /**
 487          * Save the given resource to the cache.
 488          *
 489          * @param url
 490          *            the resource
 491          * @param support
 492          *            the {@link BasicSupport} used to download it
 493          * @param originalUrl
 494          *            the original {@link URL} used to locate the cached resource
 495          *
 496          * @throws IOException
 497          *             in case of I/O error
 498          */
 499         private void save(URL url, BasicSupport support, URL originalUrl)
 500                         throws IOException {
 501                 InputStream in = openNoCache(url, support, originalUrl, null, null,
 502                                 null);
 503                 try {
 504                         File cached = getCached(originalUrl);
 505                         BufferedOutputStream out = new BufferedOutputStream(
 506                                         new FileOutputStream(cached));
 507                         try {
 508                                 byte[] buf = new byte[4096];
 509                                 int len;
 510                                 while ((len = in.read(buf)) > 0) {
 511                                         out.write(buf, 0, len);
 512                                 }
 513                         } finally {
 514                                 out.close();
 515                         }
 516                 } finally {
 517                         in.close();
 518                 }
 519         }
 520
 521         /**
 522          * Open a connection on the given {@link URL}, and manage the cookies that
 523          * come with it.
 524          *
 525          * @param url
 526          *            the {@link URL} to open
 527          * @param support
 528          *            the {@link BasicSupport} to use for cookie generation
 529          *
 530          * @return the connection
 531          *
 532          * @throws IOException
 533          *             in case of I/O error
 534          */
 535         private URLConnection openConnectionWithCookies(URL url,
 536                         BasicSupport support) throws IOException {
 537                 URLConnection conn = url.openConnection();
 538
 539                 conn.setRequestProperty("User-Agent", UA);
 540                 conn.setRequestProperty("Cookie", generateCookies(support));
 541                 conn.setRequestProperty("Accept-Encoding", "gzip");
 542                 if (support != null && support.getCurrentReferer() != null) {
 543                         conn.setRequestProperty("Referer", support.getCurrentReferer()
 544                                         .toString());
 545                         conn.setRequestProperty("Host", support.getCurrentReferer()
 546                                         .getHost());
 547                 }
 548
 549                 return conn;
 550         }
 551
 552         /**
 553          * Check if the {@link File} is too old according to
 554          * {@link Cache#tooOldChanging}.
 555          *
 556          * @param file
 557          *            the file to check
 558          * @param stable
 559          *            TRUE to denote files that are not supposed to change too often
 560          *
 561          * @return TRUE if it is
 562          */
 563         private boolean isOld(File file, boolean stable) {
 564                 long max = tooOldChanging;
 565                 if (stable) {
 566                         max = tooOldStable;
 567                 }
 568
 569                 if (max < 0) {
 570                         return false;
 571                 }
 572
 573                 long time = new Date().getTime() - file.lastModified();
 574                 if (time < 0) {
 575                         System.err.println("Timestamp in the future for file: "
 576                                         + file.getAbsolutePath());
 577                 }
 578
 579                 return time < 0 || time > max;
 580         }
 581
 582         /**
 583          * Return the associated cache {@link File} from this {@link URL}.
 584          *
 585          * @param url
 586          *            the url
 587          *
 588          * @return the cached {@link File} version of this {@link URL}
 589          */
 590         private File getCached(URL url) {
 591                 File subdir = null;
 592
 593                 String name = url.getHost();
 594                 if (name == null || name.isEmpty()) {
 595                         name = url.getFile();
 596                 } else {
 597                         File cacheDir = getCached(".").getParentFile();
 598                         File subsubDir = new File(cacheDir, allowedChars(url.getHost()));
 599                         subdir = new File(subsubDir, "_" + allowedChars(url.getPath()));
 600                         name = allowedChars("_" + url.getQuery());
 601                 }
 602
 603                 File cacheFile = getCached(name);
 604                 if (subdir != null) {
 605                         cacheFile = new File(subdir, cacheFile.getName());
 606                         subdir.mkdirs();
 607                 }
 608
 609                 return cacheFile;
 610         }
 611
 612         /**
 613          * Get the basic cache resource file corresponding to this unique ID.
 614          * <p>
 615          * Note that you may need to add a sub-directory in some cases.
 616          *
 617          * @param uniqueID
 618          *            the id
 619          *
 620          * @return the cached version if present, NULL if not
 621          */
 622         private File getCached(String uniqueID) {
 623                 return new File(dir, allowedChars(uniqueID));
 624         }
 625
 626         /**
 627          * Replace not allowed chars (in a {@link File}) by "_".
 628          *
 629          * @param raw
 630          *            the raw {@link String}
 631          *
 632          * @return the sanitised {@link String}
 633          */
 634         private String allowedChars(String raw) {
 635                 return raw.replace('/', '_').replace(':', '_').replace("\\", "_");
 636         }
 637
 638         /**
 639          * Generate the cookie {@link String} from the local {@link CookieStore} so
 640          * it is ready to be passed.
 641          *
 642          * @return the cookie
 643          */
 644         private String generateCookies(BasicSupport support) {
 645                 StringBuilder builder = new StringBuilder();
 646                 for (HttpCookie cookie : cookies.getCookieStore().getCookies()) {
 647                         if (builder.length() > 0) {
 648                                 builder.append(';');
 649                         }
 650
 651                         // TODO: check if format is ok
 652                         builder.append(cookie.toString());
 653                 }
 654
 655                 if (support != null) {
 656                         for (Map.Entry<String, String> set : support.getCookies()
 657                                         .entrySet()) {
 658                                 if (builder.length() > 0) {
 659                                         builder.append(';');
 660                                 }
 661                                 builder.append(set.getKey());
 662                                 builder.append('=');
 663                                 builder.append(set.getValue());
 664                         }
 665                 }
 666
 667                 return builder.toString();
 668         }
 669 }