src/be/nikiroo/fanfix/Cache.java

   1 package be.nikiroo.fanfix;
   2
   3 import java.io.BufferedOutputStream;
   4 import java.io.File;
   5 import java.io.FileInputStream;
   6 import java.io.FileOutputStream;
   7 import java.io.IOException;
   8 import java.io.InputStream;
   9 import java.net.CookieHandler;
  10 import java.net.CookieManager;
  11 import java.net.CookiePolicy;
  12 import java.net.CookieStore;
  13 import java.net.HttpCookie;
  14 import java.net.HttpURLConnection;
  15 import java.net.URISyntaxException;
  16 import java.net.URL;
  17 import java.net.URLConnection;
  18 import java.nio.file.FileAlreadyExistsException;
  19 import java.util.Date;
  20 import java.util.Map;
  21 import java.util.zip.GZIPInputStream;
  22
  23 import javax.imageio.ImageIO;
  24
  25 import be.nikiroo.fanfix.bundles.Config;
  26 import be.nikiroo.fanfix.supported.BasicSupport;
  27 import be.nikiroo.utils.IOUtils;
  28 import be.nikiroo.utils.MarkableFileInputStream;
  29 import be.nikiroo.utils.StringUtils;
  30
  31 /**
  32  * This cache will manage Internet (and local) downloads, as well as put the
  33  * downloaded files into a cache.
  34  * <p>
  35  * As long the cached resource is not too old, it will use it instead of
  36  * retrieving the file again.
  37  *
  38  * @author niki
  39  */
  40 public class Cache {
  41         private File dir;
  42         private String UA;
  43         private long tooOldChanging;
  44         private long tooOldStable;
  45         private CookieManager cookies;
  46
  47         /**
  48          * Create a new {@link Cache} object.
  49          *
  50          * @param dir
  51          *            the directory to use as cache
  52          * @param UA
  53          *            the User-Agent to use to download the resources
  54          * @param hoursChanging
  55          *            the number of hours after which a cached file that is thought
  56          *            to change ~often is considered too old (or -1 for
  57          *            "never too old")
  58          * @param hoursStable
  59          *            the number of hours after which a LARGE cached file that is
  60          *            thought to change rarely is considered too old (or -1 for
  61          *            "never too old")
  62          *
  63          * @throws IOException
  64          *             in case of I/O error
  65          */
  66         public Cache(File dir, String UA, int hoursChanging, int hoursStable)
  67                         throws IOException {
  68                 this.dir = dir;
  69                 this.UA = UA;
  70                 this.tooOldChanging = 1000 * 60 * 60 * hoursChanging;
  71                 this.tooOldStable = 1000 * 60 * 60 * hoursStable;
  72
  73                 if (dir != null) {
  74                         if (!dir.exists()) {
  75                                 dir.mkdirs();
  76                         }
  77                 }
  78
  79                 if (dir == null || !dir.exists()) {
  80                         throw new IOException("Cannot create the cache directory: "
  81                                         + (dir == null ? "null" : dir.getAbsolutePath()));
  82                 }
  83
  84                 cookies = new CookieManager();
  85                 cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
  86                 CookieHandler.setDefault(cookies);
  87         }
  88
  89         /**
  90          * Open a resource (will load it from the cache if possible, or save it into
  91          * the cache after downloading if not).
  92          *
  93          * @param url
  94          *            the resource to open
  95          * @param support
  96          *            the support to use to download the resource
  97          * @param stable
  98          *            TRUE for more stable resources, FALSE when they often change
  99          *
 100          * @return the opened resource
 101          *
 102          * @throws IOException
 103          *             in case of I/O error
 104          */
 105         public InputStream open(URL url, BasicSupport support, boolean stable)
 106                         throws IOException {
 107                 return open(url, support, stable, url);
 108         }
 109
 110         /**
 111          * Open a resource (will load it from the cache if possible, or save it into
 112          * the cache after downloading if not).
 113          * <p>
 114          * The cached resource will be assimilated to the given original {@link URL}
 115          *
 116          * @param url
 117          *            the resource to open
 118          * @param support
 119          *            the support to use to download the resource
 120          * @param stable
 121          *            TRUE for more stable resources, FALSE when they often change
 122          * @param originalUrl
 123          *            the original {@link URL} used to locate the cached resource
 124          *
 125          * @return the opened resource
 126          *
 127          * @throws IOException
 128          *             in case of I/O error
 129          */
 130         public InputStream open(URL url, BasicSupport support, boolean stable,
 131                         URL originalUrl) throws IOException {
 132                 try {
 133                         InputStream in = load(originalUrl, false, stable);
 134                         if (in == null) {
 135                                 try {
 136                                         save(url, support, originalUrl);
 137                                 } catch (IOException e) {
 138                                         throw new IOException("Cannot save the url: "
 139                                                         + (url == null ? "null" : url.toString()), e);
 140                                 }
 141
 142                                 in = load(originalUrl, true, stable);
 143                         }
 144
 145                         return in;
 146                 } catch (IOException e) {
 147                         throw new IOException("Cannot open the url: "
 148                                         + (url == null ? "null" : url.toString()), e);
 149                 }
 150         }
 151
 152         /**
 153          * Refresh the resource into cache if needed.
 154          *
 155          * @param url
 156          *            the resource to open
 157          * @param support
 158          *            the support to use to download the resource
 159          * @param stable
 160          *            TRUE for more stable resources, FALSE when they often change
 161          *
 162          * @return TRUE if it was pre-downloaded
 163          *
 164          * @throws IOException
 165          *             in case of I/O error
 166          */
 167         public void refresh(URL url, BasicSupport support, boolean stable)
 168                         throws IOException {
 169                 File cached = getCached(url);
 170                 if (cached.exists() && !isOld(cached, stable)) {
 171                         return;
 172                 }
 173
 174                 open(url, support, stable).close();
 175         }
 176
 177         /**
 178          * Check the resource to see if it is in the cache.
 179          *
 180          * @param url
 181          *            the resource to check
 182          *
 183          * @return TRUE if it is
 184          *
 185          */
 186         public boolean check(URL url) {
 187                 return getCached(url).exists();
 188         }
 189
 190         /**
 191          * Open a resource (will load it from the cache if possible, or save it into
 192          * the cache after downloading if not) as an Image, then save it where
 193          * requested.
 194          * <p>
 195          * This version will not always work properly if the original file was not
 196          * downloaded before.
 197          *
 198          * @param url
 199          *            the resource to open
 200          *
 201          * @return the opened resource image
 202          *
 203          * @throws IOException
 204          *             in case of I/O error
 205          */
 206         public void saveAsImage(URL url, File target) throws IOException {
 207                 URL cachedUrl = new URL(url.toString()
 208                                 + "."
 209                                 + Instance.getConfig().getString(Config.IMAGE_FORMAT_CONTENT)
 210                                                 .toLowerCase());
 211                 File cached = getCached(cachedUrl);
 212
 213                 if (!cached.exists() || isOld(cached, true)) {
 214                         InputStream imageIn = Instance.getCache().open(url, null, true);
 215                         ImageIO.write(StringUtils.toImage(imageIn), Instance.getConfig()
 216                                         .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase(),
 217                                         cached);
 218                 }
 219
 220                 IOUtils.write(new FileInputStream(cached), target);
 221         }
 222
 223         /**
 224          * Manually add this item to the cache.
 225          *
 226          * @param in
 227          *            the input data
 228          * @param uniqueID
 229          *            a unique ID for this resource
 230          *
 231          * @return the resulting {@link FileAlreadyExistsException}
 232          *
 233          * @throws IOException
 234          *             in case of I/O error
 235          */
 236         public File addToCache(InputStream in, String uniqueID) throws IOException {
 237                 File file = getCached(new File(uniqueID).toURI().toURL());
 238                 IOUtils.write(in, file);
 239                 return file;
 240         }
 241
 242         /**
 243          * Clean the cache (delete the cached items).
 244          *
 245          * @param onlyOld
 246          *            only clean the files that are considered too old
 247          *
 248          * @return the number of cleaned items
 249          */
 250         public int cleanCache(boolean onlyOld) {
 251                 int num = 0;
 252                 for (File file : dir.listFiles()) {
 253                         if (!onlyOld || isOld(file, true)) {
 254                                 if (file.delete()) {
 255                                         num++;
 256                                 } else {
 257                                         System.err.println("Cannot delete temporary file: "
 258                                                         + file.getAbsolutePath());
 259                                 }
 260                         }
 261                 }
 262                 return num;
 263         }
 264
 265         /**
 266          * Open a resource from the cache if it exists.
 267          *
 268          * @param url
 269          *            the resource to open
 270          * @return the opened resource
 271          * @throws IOException
 272          *             in case of I/O error
 273          */
 274         private InputStream load(URL url, boolean allowOld, boolean stable)
 275                         throws IOException {
 276                 File cached = getCached(url);
 277                 if (cached.exists() && !isOld(cached, stable)) {
 278                         return new MarkableFileInputStream(new FileInputStream(cached));
 279                 }
 280
 281                 return null;
 282         }
 283
 284         /**
 285          * Save the given resource to the cache.
 286          *
 287          * @param url
 288          *            the resource
 289          * @param support
 290          *            the {@link BasicSupport} used to download it
 291          * @param originalUrl
 292          *            the original {@link URL} used to locate the cached resource
 293          *
 294          * @throws IOException
 295          *             in case of I/O error
 296          * @throws URISyntaxException
 297          */
 298         private void save(URL url, BasicSupport support, URL originalUrl)
 299                         throws IOException {
 300                 URLConnection conn = url.openConnection();
 301
 302                 conn.setRequestProperty("User-Agent", UA);
 303                 conn.setRequestProperty("Cookie", generateCookies(support));
 304                 conn.setRequestProperty("Accept-Encoding", "gzip");
 305                 if (support != null) {
 306                         conn.setRequestProperty("Referer", support.getCurrentReferer()
 307                                         .toString());
 308                         conn.setRequestProperty("Host", support.getCurrentReferer()
 309                                         .getHost());
 310                 }
 311
 312                 conn.connect();
 313
 314                 // Check if redirect
 315                 if (conn instanceof HttpURLConnection
 316                                 && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) {
 317                         String newUrl = conn.getHeaderField("Location");
 318                         save(new URL(newUrl), support, originalUrl);
 319                         return;
 320                 }
 321
 322                 InputStream in = conn.getInputStream();
 323                 if ("gzip".equals(conn.getContentEncoding())) {
 324                         in = new GZIPInputStream(in);
 325                 }
 326
 327                 try {
 328                         File cached = getCached(originalUrl);
 329                         BufferedOutputStream out = new BufferedOutputStream(
 330                                         new FileOutputStream(cached));
 331                         try {
 332                                 byte[] buf = new byte[4096];
 333                                 int len;
 334                                 while ((len = in.read(buf)) > 0) {
 335                                         out.write(buf, 0, len);
 336                                 }
 337                         } finally {
 338                                 out.close();
 339                         }
 340                 } finally {
 341                         in.close();
 342                 }
 343         }
 344
 345         /**
 346          * Check if the {@link File} is too old according to
 347          * {@link Cache#tooOldChanging}.
 348          *
 349          * @param file
 350          *            the file to check
 351          * @param stable
 352          *            TRUE to denote files that are not supposed to change too often
 353          *
 354          * @return TRUE if it is
 355          */
 356         private boolean isOld(File file, boolean stable) {
 357                 long max = tooOldChanging;
 358                 if (stable) {
 359                         max = tooOldStable;
 360                 }
 361
 362                 if (max < 0) {
 363                         return false;
 364                 }
 365
 366                 long time = new Date().getTime() - file.lastModified();
 367                 if (time < 0) {
 368                         System.err.println("Timestamp in the future for file: "
 369                                         + file.getAbsolutePath());
 370                 }
 371
 372                 return time < 0 || time > max;
 373         }
 374
 375         /**
 376          * Get the cache resource from the cache if it is present for this
 377          * {@link URL}.
 378          *
 379          * @param url
 380          *            the url
 381          * @return the cached version if present, NULL if not
 382          */
 383         private File getCached(URL url) {
 384                 String name = url.getHost();
 385                 if (name == null || name.length() == 0) {
 386                         name = url.getFile();
 387                 } else {
 388                         name = url.toString();
 389                 }
 390
 391                 name = name.replace('/', '_').replace(':', '_');
 392
 393                 return new File(dir, name);
 394         }
 395
 396         /**
 397          * Generate the cookie {@link String} from the local {@link CookieStore} so
 398          * it is ready to be passed.
 399          *
 400          * @return the cookie
 401          */
 402         private String generateCookies(BasicSupport support) {
 403                 StringBuilder builder = new StringBuilder();
 404                 for (HttpCookie cookie : cookies.getCookieStore().getCookies()) {
 405                         if (builder.length() > 0) {
 406                                 builder.append(';');
 407                         }
 408
 409                         // TODO: check if format is ok
 410                         builder.append(cookie.toString());
 411                 }
 412
 413                 if (support != null) {
 414                         for (Map.Entry<String, String> set : support.getCookies()
 415                                         .entrySet()) {
 416                                 if (builder.length() > 0) {
 417                                         builder.append(';');
 418                                 }
 419                                 builder.append(set.getKey());
 420                                 builder.append('=');
 421                                 builder.append(set.getValue());
 422                         }
 423                 }
 424
 425                 return builder.toString();
 426         }
 427 }