src/be/nikiroo/fanfix/Cache.java

   1 package be.nikiroo.fanfix;
   2
   3 import java.io.BufferedOutputStream;
   4 import java.io.File;
   5 import java.io.FileInputStream;
   6 import java.io.FileOutputStream;
   7 import java.io.IOException;
   8 import java.io.InputStream;
   9 import java.io.OutputStreamWriter;
  10 import java.net.CookieHandler;
  11 import java.net.CookieManager;
  12 import java.net.CookiePolicy;
  13 import java.net.CookieStore;
  14 import java.net.HttpCookie;
  15 import java.net.HttpURLConnection;
  16 import java.net.URISyntaxException;
  17 import java.net.URL;
  18 import java.net.URLConnection;
  19 import java.net.URLEncoder;
  20 import java.util.Date;
  21 import java.util.Map;
  22 import java.util.zip.GZIPInputStream;
  23
  24 import javax.imageio.ImageIO;
  25
  26 import be.nikiroo.fanfix.bundles.Config;
  27 import be.nikiroo.fanfix.supported.BasicSupport;
  28 import be.nikiroo.utils.IOUtils;
  29 import be.nikiroo.utils.MarkableFileInputStream;
  30
  31 /**
  32  * This cache will manage Internet (and local) downloads, as well as put the
  33  * downloaded files into a cache.
  34  * <p>
  35  * As long the cached resource is not too old, it will use it instead of
  36  * retrieving the file again.
  37  *
  38  * @author niki
  39  */
  40 public class Cache {
  41         private File dir;
  42         private String UA;
  43         private long tooOldChanging;
  44         private long tooOldStable;
  45         private CookieManager cookies;
  46
  47         /**
  48          * Create a new {@link Cache} object.
  49          *
  50          * @param dir
  51          *            the directory to use as cache
  52          * @param UA
  53          *            the User-Agent to use to download the resources
  54          * @param hoursChanging
  55          *            the number of hours after which a cached file that is thought
  56          *            to change ~often is considered too old (or -1 for
  57          *            "never too old")
  58          * @param hoursStable
  59          *            the number of hours after which a LARGE cached file that is
  60          *            thought to change rarely is considered too old (or -1 for
  61          *            "never too old")
  62          *
  63          * @throws IOException
  64          *             in case of I/O error
  65          */
  66         public Cache(File dir, String UA, int hoursChanging, int hoursStable)
  67                         throws IOException {
  68                 this.dir = dir;
  69                 this.UA = UA;
  70                 this.tooOldChanging = 1000 * 60 * 60 * hoursChanging;
  71                 this.tooOldStable = 1000 * 60 * 60 * hoursStable;
  72
  73                 if (dir != null) {
  74                         if (!dir.exists()) {
  75                                 dir.mkdirs();
  76                         }
  77                 }
  78
  79                 if (dir == null || !dir.exists()) {
  80                         throw new IOException("Cannot create the cache directory: "
  81                                         + (dir == null ? "null" : dir.getAbsolutePath()));
  82                 }
  83
  84                 cookies = new CookieManager();
  85                 cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
  86                 CookieHandler.setDefault(cookies);
  87         }
  88
  89         /**
  90          * Clear all the cookies currently in the jar.
  91          */
  92         public void clearCookies() {
  93                 cookies.getCookieStore().removeAll();
  94         }
  95
  96         /**
  97          * Open a resource (will load it from the cache if possible, or save it into
  98          * the cache after downloading if not).
  99          *
 100          * @param url
 101          *            the resource to open
 102          * @param support
 103          *            the support to use to download the resource
 104          * @param stable
 105          *            TRUE for more stable resources, FALSE when they often change
 106          *
 107          * @return the opened resource, NOT NULL
 108          *
 109          * @throws IOException
 110          *             in case of I/O error
 111          */
 112         public InputStream open(URL url, BasicSupport support, boolean stable)
 113                         throws IOException {
 114                 // MUST NOT return null
 115                 return open(url, support, stable, url);
 116         }
 117
 118         /**
 119          * Open a resource (will load it from the cache if possible, or save it into
 120          * the cache after downloading if not).
 121          * <p>
 122          * The cached resource will be assimilated to the given original {@link URL}
 123          *
 124          * @param url
 125          *            the resource to open
 126          * @param support
 127          *            the support to use to download the resource
 128          * @param stable
 129          *            TRUE for more stable resources, FALSE when they often change
 130          * @param originalUrl
 131          *            the original {@link URL} used to locate the cached resource
 132          *
 133          * @return the opened resource, NOT NULL
 134          *
 135          * @throws IOException
 136          *             in case of I/O error
 137          */
 138         public InputStream open(URL url, BasicSupport support, boolean stable,
 139                         URL originalUrl) throws IOException {
 140                 // MUST NOT return null
 141                 try {
 142                         InputStream in = load(originalUrl, false, stable);
 143                         if (in == null) {
 144                                 try {
 145                                         save(url, support, originalUrl);
 146                                 } catch (IOException e) {
 147                                         throw new IOException("Cannot save the url: "
 148                                                         + (url == null ? "null" : url.toString()), e);
 149                                 }
 150
 151                                 // Was just saved, can load old, so, will not be null
 152                                 in = load(originalUrl, true, stable);
 153                         }
 154
 155                         return in;
 156                 } catch (IOException e) {
 157                         throw new IOException("Cannot open the url: "
 158                                         + (url == null ? "null" : url.toString()), e);
 159                 }
 160         }
 161
 162         /**
 163          * Open the given {@link URL} without using the cache, but still using and
 164          * updating the cookies.
 165          *
 166          * @param url
 167          *            the {@link URL} to open
 168          * @param support
 169          *            the {@link BasicSupport} used for the cookies
 170          *
 171          * @return the {@link InputStream} of the opened page
 172          *
 173          * @throws IOException
 174          *             in case of I/O error
 175          */
 176         public InputStream openNoCache(URL url, BasicSupport support)
 177                         throws IOException {
 178                 return openNoCache(url, support, url, null);
 179         }
 180
 181         /**
 182          * Open the given {@link URL} without using the cache, but still using and
 183          * updating the cookies.
 184          *
 185          * @param url
 186          *            the {@link URL} to open
 187          * @param support
 188          *            the {@link BasicSupport} used for the cookies
 189          * @param postParams
 190          *            the POST parameters
 191          *
 192          * @return the {@link InputStream} of the opened page
 193          *
 194          * @throws IOException
 195          *             in case of I/O error
 196          */
 197         public InputStream openNoCache(URL url, BasicSupport support,
 198                         Map<String, String> postParams) throws IOException {
 199                 return openNoCache(url, support, url, postParams);
 200         }
 201
 202         /**
 203          * Open the given {@link URL} without using the cache, but still using and
 204          * updating the cookies.
 205          *
 206          * @param url
 207          *            the {@link URL} to open
 208          * @param support
 209          *            the {@link BasicSupport} used for the cookies
 210          * @param originalUrl
 211          *            the original {@link URL} before any redirection occurs
 212          * @param postParams
 213          *            the POST parameters
 214          *
 215          * @return the {@link InputStream} of the opened page
 216          *
 217          * @throws IOException
 218          *             in case of I/O error
 219          */
 220         private InputStream openNoCache(URL url, BasicSupport support,
 221                         final URL originalUrl, Map<String, String> postParams)
 222                         throws IOException {
 223
 224                 URLConnection conn = openConnectionWithCookies(url, support);
 225                 if (postParams != null) {
 226                         StringBuilder postData = new StringBuilder();
 227                         for (Map.Entry<String, String> param : postParams.entrySet()) {
 228                                 if (postData.length() != 0)
 229                                         postData.append('&');
 230                                 postData.append(URLEncoder.encode(param.getKey(), "UTF-8"));
 231                                 postData.append('=');
 232                                 postData.append(URLEncoder.encode(
 233                                                 String.valueOf(param.getValue()), "UTF-8"));
 234                         }
 235
 236                         conn.setDoOutput(true);
 237
 238                         OutputStreamWriter writer = new OutputStreamWriter(
 239                                         conn.getOutputStream());
 240
 241                         writer.write(postData.toString());
 242                         writer.flush();
 243                         writer.close();
 244                 }
 245
 246                 conn.connect();
 247
 248                 // Check if redirect
 249                 if (conn instanceof HttpURLConnection
 250                                 && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) {
 251                         String newUrl = conn.getHeaderField("Location");
 252                         return openNoCache(new URL(newUrl), support, originalUrl,
 253                                         postParams);
 254                 }
 255
 256                 InputStream in = conn.getInputStream();
 257                 if ("gzip".equals(conn.getContentEncoding())) {
 258                         in = new GZIPInputStream(in);
 259                 }
 260
 261                 return in;
 262         }
 263
 264         /**
 265          * Refresh the resource into cache if needed.
 266          *
 267          * @param url
 268          *            the resource to open
 269          * @param support
 270          *            the support to use to download the resource
 271          * @param stable
 272          *            TRUE for more stable resources, FALSE when they often change
 273          *
 274          * @throws IOException
 275          *             in case of I/O error
 276          */
 277         public void refresh(URL url, BasicSupport support, boolean stable)
 278                         throws IOException {
 279                 File cached = getCached(url);
 280                 if (cached.exists() && !isOld(cached, stable)) {
 281                         return;
 282                 }
 283
 284                 open(url, support, stable).close();
 285         }
 286
 287         /**
 288          * Check the resource to see if it is in the cache.
 289          *
 290          * @param url
 291          *            the resource to check
 292          *
 293          * @return TRUE if it is
 294          *
 295          */
 296         public boolean check(URL url) {
 297                 return getCached(url).exists();
 298         }
 299
 300         /**
 301          * Open a resource (will load it from the cache if possible, or save it into
 302          * the cache after downloading if not) as an Image, then save it where
 303          * requested.
 304          * <p>
 305          * This version will not always work properly if the original file was not
 306          * downloaded before.
 307          *
 308          * @param url
 309          *            the resource to open
 310          *
 311          * @return the opened resource image
 312          *
 313          * @throws IOException
 314          *             in case of I/O error
 315          */
 316         public void saveAsImage(URL url, File target) throws IOException {
 317                 URL cachedUrl = new URL(url.toString());
 318                 File cached = getCached(cachedUrl);
 319
 320                 if (!cached.exists() || isOld(cached, true)) {
 321                         InputStream imageIn = open(url, null, true);
 322                         ImageIO.write(IOUtils.toImage(imageIn), Instance.getConfig()
 323                                         .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase(),
 324                                         cached);
 325                 }
 326
 327                 IOUtils.write(new FileInputStream(cached), target);
 328         }
 329
 330         /**
 331          * Manually add this item to the cache.
 332          *
 333          * @param in
 334          *            the input data
 335          * @param uniqueID
 336          *            a unique ID for this resource
 337          *
 338          * @return the resulting {@link File}
 339          *
 340          * @throws IOException
 341          *             in case of I/O error
 342          */
 343         public File addToCache(InputStream in, String uniqueID) throws IOException {
 344                 File file = getCached(new File(uniqueID).toURI().toURL());
 345                 IOUtils.write(in, file);
 346                 return file;
 347         }
 348
 349         /**
 350          * Clean the cache (delete the cached items).
 351          *
 352          * @param onlyOld
 353          *            only clean the files that are considered too old
 354          *
 355          * @return the number of cleaned items
 356          */
 357         public int cleanCache(boolean onlyOld) {
 358                 int num = 0;
 359                 for (File file : dir.listFiles()) {
 360                         if (!onlyOld || isOld(file, true)) {
 361                                 if (file.delete()) {
 362                                         num++;
 363                                 } else {
 364                                         System.err.println("Cannot delete temporary file: "
 365                                                         + file.getAbsolutePath());
 366                                 }
 367                         }
 368                 }
 369                 return num;
 370         }
 371
 372         /**
 373          * Open a resource from the cache if it exists.
 374          *
 375          * @param url
 376          *            the resource to open
 377          *
 378          * @return the opened resource if found, NULL i not
 379          *
 380          * @throws IOException
 381          *             in case of I/O error
 382          */
 383         private InputStream load(URL url, boolean allowOld, boolean stable)
 384                         throws IOException {
 385                 File cached = getCached(url);
 386                 if (cached.exists() && !isOld(cached, stable)) {
 387                         return new MarkableFileInputStream(new FileInputStream(cached));
 388                 }
 389
 390                 return null;
 391         }
 392
 393         /**
 394          * Save the given resource to the cache.
 395          *
 396          * @param url
 397          *            the resource
 398          * @param support
 399          *            the {@link BasicSupport} used to download it
 400          * @param originalUrl
 401          *            the original {@link URL} used to locate the cached resource
 402          *
 403          * @throws IOException
 404          *             in case of I/O error
 405          * @throws URISyntaxException
 406          */
 407         private void save(URL url, BasicSupport support, URL originalUrl)
 408                         throws IOException {
 409                 InputStream in = openNoCache(url, support, originalUrl, null);
 410                 try {
 411                         File cached = getCached(originalUrl);
 412                         BufferedOutputStream out = new BufferedOutputStream(
 413                                         new FileOutputStream(cached));
 414                         try {
 415                                 byte[] buf = new byte[4096];
 416                                 int len;
 417                                 while ((len = in.read(buf)) > 0) {
 418                                         out.write(buf, 0, len);
 419                                 }
 420                         } finally {
 421                                 out.close();
 422                         }
 423                 } finally {
 424                         in.close();
 425                 }
 426         }
 427
 428         /**
 429          * Open a connection on the given {@link URL}, and manage the cookies that
 430          * come with it.
 431          *
 432          * @param url
 433          *            the {@link URL} to open
 434          * @param support
 435          *            the {@link BasicSupport} to use for cookie generation
 436          *
 437          * @return the connection
 438          *
 439          * @throws IOException
 440          *             in case of I/O error
 441          */
 442         private URLConnection openConnectionWithCookies(URL url,
 443                         BasicSupport support) throws IOException {
 444                 URLConnection conn = url.openConnection();
 445
 446                 conn.setRequestProperty("User-Agent", UA);
 447                 conn.setRequestProperty("Cookie", generateCookies(support));
 448                 conn.setRequestProperty("Accept-Encoding", "gzip");
 449                 if (support != null && support.getCurrentReferer() != null) {
 450                         conn.setRequestProperty("Referer", support.getCurrentReferer()
 451                                         .toString());
 452                         conn.setRequestProperty("Host", support.getCurrentReferer()
 453                                         .getHost());
 454                 }
 455
 456                 return conn;
 457         }
 458
 459         /**
 460          * Check if the {@link File} is too old according to
 461          * {@link Cache#tooOldChanging}.
 462          *
 463          * @param file
 464          *            the file to check
 465          * @param stable
 466          *            TRUE to denote files that are not supposed to change too often
 467          *
 468          * @return TRUE if it is
 469          */
 470         private boolean isOld(File file, boolean stable) {
 471                 long max = tooOldChanging;
 472                 if (stable) {
 473                         max = tooOldStable;
 474                 }
 475
 476                 if (max < 0) {
 477                         return false;
 478                 }
 479
 480                 long time = new Date().getTime() - file.lastModified();
 481                 if (time < 0) {
 482                         System.err.println("Timestamp in the future for file: "
 483                                         + file.getAbsolutePath());
 484                 }
 485
 486                 return time < 0 || time > max;
 487         }
 488
 489         /**
 490          * Get the cache resource from the cache if it is present for this
 491          * {@link URL}.
 492          *
 493          * @param url
 494          *            the url
 495          * @return the cached version if present, NULL if not
 496          */
 497         private File getCached(URL url) {
 498                 String name = url.getHost();
 499                 if (name == null || name.length() == 0) {
 500                         name = url.getFile();
 501                 } else {
 502                         name = url.toString();
 503                 }
 504
 505                 name = name.replace('/', '_').replace(':', '_');
 506
 507                 return new File(dir, name);
 508         }
 509
 510         /**
 511          * Generate the cookie {@link String} from the local {@link CookieStore} so
 512          * it is ready to be passed.
 513          *
 514          * @return the cookie
 515          */
 516         private String generateCookies(BasicSupport support) {
 517                 StringBuilder builder = new StringBuilder();
 518                 for (HttpCookie cookie : cookies.getCookieStore().getCookies()) {
 519                         if (builder.length() > 0) {
 520                                 builder.append(';');
 521                         }
 522
 523                         // TODO: check if format is ok
 524                         builder.append(cookie.toString());
 525                 }
 526
 527                 if (support != null) {
 528                         try {
 529                                 for (Map.Entry<String, String> set : support.getCookies()
 530                                                 .entrySet()) {
 531                                         if (builder.length() > 0) {
 532                                                 builder.append(';');
 533                                         }
 534                                         builder.append(set.getKey());
 535                                         builder.append('=');
 536                                         builder.append(set.getValue());
 537                                 }
 538                         } catch (IOException e) {
 539                                 Instance.syserr(e);
 540                         }
 541                 }
 542
 543                 return builder.toString();
 544         }
 545 }