src/be/nikiroo/fanfix/Cache.java

   1 package be.nikiroo.fanfix;
   2
   3 import java.io.BufferedOutputStream;
   4 import java.io.File;
   5 import java.io.FileInputStream;
   6 import java.io.FileOutputStream;
   7 import java.io.IOException;
   8 import java.io.InputStream;
   9 import java.io.OutputStreamWriter;
  10 import java.net.CookieHandler;
  11 import java.net.CookieManager;
  12 import java.net.CookiePolicy;
  13 import java.net.CookieStore;
  14 import java.net.HttpCookie;
  15 import java.net.HttpURLConnection;
  16 import java.net.URISyntaxException;
  17 import java.net.URL;
  18 import java.net.URLConnection;
  19 import java.net.URLEncoder;
  20 import java.util.Date;
  21 import java.util.Map;
  22 import java.util.zip.GZIPInputStream;
  23
  24 import javax.imageio.ImageIO;
  25
  26 import be.nikiroo.fanfix.bundles.Config;
  27 import be.nikiroo.fanfix.supported.BasicSupport;
  28 import be.nikiroo.utils.IOUtils;
  29 import be.nikiroo.utils.MarkableFileInputStream;
  30
  31 /**
  32  * This cache will manage Internet (and local) downloads, as well as put the
  33  * downloaded files into a cache.
  34  * <p>
  35  * As long the cached resource is not too old, it will use it instead of
  36  * retrieving the file again.
  37  *
  38  * @author niki
  39  */
  40 public class Cache {
  41         private File dir;
  42         private String UA;
  43         private long tooOldChanging;
  44         private long tooOldStable;
  45         private CookieManager cookies;
  46
  47         /**
  48          * Create a new {@link Cache} object.
  49          *
  50          * @param dir
  51          *            the directory to use as cache
  52          * @param UA
  53          *            the User-Agent to use to download the resources
  54          * @param hoursChanging
  55          *            the number of hours after which a cached file that is thought
  56          *            to change ~often is considered too old (or -1 for
  57          *            "never too old")
  58          * @param hoursStable
  59          *            the number of hours after which a LARGE cached file that is
  60          *            thought to change rarely is considered too old (or -1 for
  61          *            "never too old")
  62          *
  63          * @throws IOException
  64          *             in case of I/O error
  65          */
  66         public Cache(File dir, String UA, int hoursChanging, int hoursStable)
  67                         throws IOException {
  68                 this.dir = dir;
  69                 this.UA = UA;
  70                 this.tooOldChanging = 1000 * 60 * 60 * hoursChanging;
  71                 this.tooOldStable = 1000 * 60 * 60 * hoursStable;
  72
  73                 if (dir != null) {
  74                         if (!dir.exists()) {
  75                                 dir.mkdirs();
  76                         }
  77                 }
  78
  79                 if (dir == null || !dir.exists()) {
  80                         throw new IOException("Cannot create the cache directory: "
  81                                         + (dir == null ? "null" : dir.getAbsolutePath()));
  82                 }
  83
  84                 cookies = new CookieManager();
  85                 cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
  86                 CookieHandler.setDefault(cookies);
  87         }
  88
  89         /**
  90          * Clear all the cookies currently in the jar.
  91          */
  92         public void clearCookies() {
  93                 cookies.getCookieStore().removeAll();
  94         }
  95
  96         /**
  97          * Open a resource (will load it from the cache if possible, or save it into
  98          * the cache after downloading if not).
  99          *
 100          * @param url
 101          *            the resource to open
 102          * @param support
 103          *            the support to use to download the resource
 104          * @param stable
 105          *            TRUE for more stable resources, FALSE when they often change
 106          *
 107          * @return the opened resource, NOT NULL
 108          *
 109          * @throws IOException
 110          *             in case of I/O error
 111          */
 112         public InputStream open(URL url, BasicSupport support, boolean stable)
 113                         throws IOException {
 114                 // MUST NOT return null
 115                 return open(url, support, stable, url);
 116         }
 117
 118         /**
 119          * Open a resource (will load it from the cache if possible, or save it into
 120          * the cache after downloading if not).
 121          * <p>
 122          * The cached resource will be assimilated to the given original {@link URL}
 123          *
 124          * @param url
 125          *            the resource to open
 126          * @param support
 127          *            the support to use to download the resource
 128          * @param stable
 129          *            TRUE for more stable resources, FALSE when they often change
 130          * @param originalUrl
 131          *            the original {@link URL} used to locate the cached resource
 132          *
 133          * @return the opened resource, NOT NULL
 134          *
 135          * @throws IOException
 136          *             in case of I/O error
 137          */
 138         public InputStream open(URL url, BasicSupport support, boolean stable,
 139                         URL originalUrl) throws IOException {
 140                 // MUST NOT return null
 141                 try {
 142                         InputStream in = load(originalUrl, false, stable);
 143                         if (in == null) {
 144                                 try {
 145                                         save(url, support, originalUrl);
 146                                 } catch (IOException e) {
 147                                         throw new IOException("Cannot save the url: "
 148                                                         + (url == null ? "null" : url.toString()), e);
 149                                 }
 150
 151                                 // Was just saved, can load old, so, will not be null
 152                                 in = load(originalUrl, true, stable);
 153                         }
 154
 155                         return in;
 156                 } catch (IOException e) {
 157                         throw new IOException("Cannot open the url: "
 158                                         + (url == null ? "null" : url.toString()), e);
 159                 }
 160         }
 161
 162         /**
 163          * Open the given {@link URL} without using the cache, but still using and
 164          * updating the cookies.
 165          *
 166          * @param url
 167          *            the {@link URL} to open
 168          * @param support
 169          *            the {@link BasicSupport} used for the cookies
 170          *
 171          * @return the {@link InputStream} of the opened page
 172          *
 173          * @throws IOException
 174          *             in case of I/O error
 175          */
 176         public InputStream openNoCache(URL url, BasicSupport support)
 177                         throws IOException {
 178                 return openNoCache(url, support, url, null);
 179         }
 180
 181         /**
 182          * Open the given {@link URL} without using the cache, but still using and
 183          * updating the cookies.
 184          *
 185          * @param url
 186          *            the {@link URL} to open
 187          * @param support
 188          *            the {@link BasicSupport} used for the cookies
 189          * @param postParams
 190          *            the POST parameters
 191          *
 192          * @return the {@link InputStream} of the opened page
 193          *
 194          * @throws IOException
 195          *             in case of I/O error
 196          */
 197         public InputStream openNoCache(URL url, BasicSupport support,
 198                         Map<String, String> postParams) throws IOException {
 199                 return openNoCache(url, support, url, postParams);
 200         }
 201
 202         /**
 203          * Open the given {@link URL} without using the cache, but still using and
 204          * updating the cookies.
 205          *
 206          * @param url
 207          *            the {@link URL} to open
 208          * @param support
 209          *            the {@link BasicSupport} used for the cookies
 210          * @param originalUrl
 211          *            the original {@link URL} before any redirection occurs
 212          * @param postParams
 213          *            the POST parameters
 214          *
 215          * @return the {@link InputStream} of the opened page
 216          *
 217          * @throws IOException
 218          *             in case of I/O error
 219          */
 220         private InputStream openNoCache(URL url, BasicSupport support,
 221                         final URL originalUrl, Map<String, String> postParams)
 222                         throws IOException {
 223
 224                 URLConnection conn = openConnectionWithCookies(url, support);
 225                 if (postParams != null && conn instanceof HttpURLConnection) {
 226                         StringBuilder postData = new StringBuilder();
 227                         for (Map.Entry<String, String> param : postParams.entrySet()) {
 228                                 if (postData.length() != 0)
 229                                         postData.append('&');
 230                                 postData.append(URLEncoder.encode(param.getKey(), "UTF-8"));
 231                                 postData.append('=');
 232                                 postData.append(URLEncoder.encode(
 233                                                 String.valueOf(param.getValue()), "UTF-8"));
 234                         }
 235
 236                         conn.setDoOutput(true);
 237                         ((HttpURLConnection) conn).setRequestMethod("POST");
 238                         conn.setRequestProperty("Content-Type",
 239                                         "application/x-www-form-urlencoded");
 240                         conn.setRequestProperty("charset", "utf-8");
 241
 242                         OutputStreamWriter writer = new OutputStreamWriter(
 243                                         conn.getOutputStream());
 244
 245                         writer.write(postData.toString());
 246                         writer.flush();
 247                         writer.close();
 248                 }
 249
 250                 conn.connect();
 251
 252                 // Check if redirect
 253                 if (conn instanceof HttpURLConnection
 254                                 && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) {
 255                         String newUrl = conn.getHeaderField("Location");
 256                         return openNoCache(new URL(newUrl), support, originalUrl,
 257                                         postParams);
 258                 }
 259
 260                 InputStream in = conn.getInputStream();
 261                 if ("gzip".equals(conn.getContentEncoding())) {
 262                         in = new GZIPInputStream(in);
 263                 }
 264
 265                 return in;
 266         }
 267
 268         /**
 269          * Refresh the resource into cache if needed.
 270          *
 271          * @param url
 272          *            the resource to open
 273          * @param support
 274          *            the support to use to download the resource
 275          * @param stable
 276          *            TRUE for more stable resources, FALSE when they often change
 277          *
 278          * @throws IOException
 279          *             in case of I/O error
 280          */
 281         public void refresh(URL url, BasicSupport support, boolean stable)
 282                         throws IOException {
 283                 File cached = getCached(url);
 284                 if (cached.exists() && !isOld(cached, stable)) {
 285                         return;
 286                 }
 287
 288                 open(url, support, stable).close();
 289         }
 290
 291         /**
 292          * Check the resource to see if it is in the cache.
 293          *
 294          * @param url
 295          *            the resource to check
 296          *
 297          * @return TRUE if it is
 298          *
 299          */
 300         public boolean check(URL url) {
 301                 return getCached(url).exists();
 302         }
 303
 304         /**
 305          * Open a resource (will load it from the cache if possible, or save it into
 306          * the cache after downloading if not) as an Image, then save it where
 307          * requested.
 308          * <p>
 309          * This version will not always work properly if the original file was not
 310          * downloaded before.
 311          *
 312          * @param url
 313          *            the resource to open
 314          *
 315          * @return the opened resource image
 316          *
 317          * @throws IOException
 318          *             in case of I/O error
 319          */
 320         public void saveAsImage(URL url, File target) throws IOException {
 321                 URL cachedUrl = new URL(url.toString());
 322                 File cached = getCached(cachedUrl);
 323
 324                 if (!cached.exists() || isOld(cached, true)) {
 325                         InputStream imageIn = open(url, null, true);
 326                         ImageIO.write(IOUtils.toImage(imageIn), Instance.getConfig()
 327                                         .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase(),
 328                                         cached);
 329                 }
 330
 331                 IOUtils.write(new FileInputStream(cached), target);
 332         }
 333
 334         /**
 335          * Manually add this item to the cache.
 336          *
 337          * @param in
 338          *            the input data
 339          * @param uniqueID
 340          *            a unique ID for this resource
 341          *
 342          * @return the resulting {@link File}
 343          *
 344          * @throws IOException
 345          *             in case of I/O error
 346          */
 347         public File addToCache(InputStream in, String uniqueID) throws IOException {
 348                 File file = getCached(new File(uniqueID).toURI().toURL());
 349                 IOUtils.write(in, file);
 350                 return file;
 351         }
 352
 353         /**
 354          * Clean the cache (delete the cached items).
 355          *
 356          * @param onlyOld
 357          *            only clean the files that are considered too old
 358          *
 359          * @return the number of cleaned items
 360          */
 361         public int cleanCache(boolean onlyOld) {
 362                 int num = 0;
 363                 for (File file : dir.listFiles()) {
 364                         if (!onlyOld || isOld(file, true)) {
 365                                 if (file.delete()) {
 366                                         num++;
 367                                 } else {
 368                                         System.err.println("Cannot delete temporary file: "
 369                                                         + file.getAbsolutePath());
 370                                 }
 371                         }
 372                 }
 373                 return num;
 374         }
 375
 376         /**
 377          * Open a resource from the cache if it exists.
 378          *
 379          * @param url
 380          *            the resource to open
 381          *
 382          * @return the opened resource if found, NULL i not
 383          *
 384          * @throws IOException
 385          *             in case of I/O error
 386          */
 387         private InputStream load(URL url, boolean allowOld, boolean stable)
 388                         throws IOException {
 389                 File cached = getCached(url);
 390                 if (cached.exists() && !isOld(cached, stable)) {
 391                         return new MarkableFileInputStream(new FileInputStream(cached));
 392                 }
 393
 394                 return null;
 395         }
 396
 397         /**
 398          * Save the given resource to the cache.
 399          *
 400          * @param url
 401          *            the resource
 402          * @param support
 403          *            the {@link BasicSupport} used to download it
 404          * @param originalUrl
 405          *            the original {@link URL} used to locate the cached resource
 406          *
 407          * @throws IOException
 408          *             in case of I/O error
 409          * @throws URISyntaxException
 410          */
 411         private void save(URL url, BasicSupport support, URL originalUrl)
 412                         throws IOException {
 413                 InputStream in = openNoCache(url, support, originalUrl, null);
 414                 try {
 415                         File cached = getCached(originalUrl);
 416                         BufferedOutputStream out = new BufferedOutputStream(
 417                                         new FileOutputStream(cached));
 418                         try {
 419                                 byte[] buf = new byte[4096];
 420                                 int len;
 421                                 while ((len = in.read(buf)) > 0) {
 422                                         out.write(buf, 0, len);
 423                                 }
 424                         } finally {
 425                                 out.close();
 426                         }
 427                 } finally {
 428                         in.close();
 429                 }
 430         }
 431
 432         /**
 433          * Open a connection on the given {@link URL}, and manage the cookies that
 434          * come with it.
 435          *
 436          * @param url
 437          *            the {@link URL} to open
 438          * @param support
 439          *            the {@link BasicSupport} to use for cookie generation
 440          *
 441          * @return the connection
 442          *
 443          * @throws IOException
 444          *             in case of I/O error
 445          */
 446         private URLConnection openConnectionWithCookies(URL url,
 447                         BasicSupport support) throws IOException {
 448                 URLConnection conn = url.openConnection();
 449
 450                 conn.setRequestProperty("User-Agent", UA);
 451                 conn.setRequestProperty("Cookie", generateCookies(support));
 452                 conn.setRequestProperty("Accept-Encoding", "gzip");
 453                 if (support != null && support.getCurrentReferer() != null) {
 454                         conn.setRequestProperty("Referer", support.getCurrentReferer()
 455                                         .toString());
 456                         conn.setRequestProperty("Host", support.getCurrentReferer()
 457                                         .getHost());
 458                 }
 459
 460                 return conn;
 461         }
 462
 463         /**
 464          * Check if the {@link File} is too old according to
 465          * {@link Cache#tooOldChanging}.
 466          *
 467          * @param file
 468          *            the file to check
 469          * @param stable
 470          *            TRUE to denote files that are not supposed to change too often
 471          *
 472          * @return TRUE if it is
 473          */
 474         private boolean isOld(File file, boolean stable) {
 475                 long max = tooOldChanging;
 476                 if (stable) {
 477                         max = tooOldStable;
 478                 }
 479
 480                 if (max < 0) {
 481                         return false;
 482                 }
 483
 484                 long time = new Date().getTime() - file.lastModified();
 485                 if (time < 0) {
 486                         System.err.println("Timestamp in the future for file: "
 487                                         + file.getAbsolutePath());
 488                 }
 489
 490                 return time < 0 || time > max;
 491         }
 492
 493         /**
 494          * Get the cache resource from the cache if it is present for this
 495          * {@link URL}.
 496          *
 497          * @param url
 498          *            the url
 499          * @return the cached version if present, NULL if not
 500          */
 501         private File getCached(URL url) {
 502                 String name = url.getHost();
 503                 if (name == null || name.length() == 0) {
 504                         name = url.getFile();
 505                 } else {
 506                         name = url.toString();
 507                 }
 508
 509                 name = name.replace('/', '_').replace(':', '_');
 510
 511                 return new File(dir, name);
 512         }
 513
 514         /**
 515          * Generate the cookie {@link String} from the local {@link CookieStore} so
 516          * it is ready to be passed.
 517          *
 518          * @return the cookie
 519          */
 520         private String generateCookies(BasicSupport support) {
 521                 StringBuilder builder = new StringBuilder();
 522                 for (HttpCookie cookie : cookies.getCookieStore().getCookies()) {
 523                         if (builder.length() > 0) {
 524                                 builder.append(';');
 525                         }
 526
 527                         // TODO: check if format is ok
 528                         builder.append(cookie.toString());
 529                 }
 530
 531                 if (support != null) {
 532                         try {
 533                                 for (Map.Entry<String, String> set : support.getCookies()
 534                                                 .entrySet()) {
 535                                         if (builder.length() > 0) {
 536                                                 builder.append(';');
 537                                         }
 538                                         builder.append(set.getKey());
 539                                         builder.append('=');
 540                                         builder.append(set.getValue());
 541                                 }
 542                         } catch (IOException e) {
 543                                 Instance.syserr(e);
 544                         }
 545                 }
 546
 547                 return builder.toString();
 548         }
 549 }