src/be/nikiroo/utils/Cache.java

   1 package be.nikiroo.utils;
   2
   3 import java.io.File;
   4 import java.io.FileNotFoundException;
   5 import java.io.IOException;
   6 import java.io.InputStream;
   7 import java.net.URL;
   8 import java.util.Date;
   9
  10 import be.nikiroo.utils.streams.MarkableFileInputStream;
  11
  12 /**
  13  * A generic cache system, with special support for {@link URL}s.
  14  * <p>
  15  * This cache also manages timeout information.
  16  *
  17  * @author niki
  18  */
  19 public class Cache {
  20         private File dir;
  21         private long tooOldChanging;
  22         private long tooOldStable;
  23         private TraceHandler tracer = new TraceHandler();
  24
  25         /**
  26          * Only for inheritance.
  27          */
  28         protected Cache() {
  29         }
  30
  31         /**
  32          * Create a new {@link Cache} object.
  33          *
  34          * @param dir
  35          *            the directory to use as cache
  36          * @param hoursChanging
  37          *            the number of hours after which a cached file that is thought
  38          *            to change ~often is considered too old (or -1 for
  39          *            "never too old")
  40          * @param hoursStable
  41          *            the number of hours after which a cached file that is thought
  42          *            to change rarely is considered too old (or -1 for
  43          *            "never too old")
  44          *
  45          * @throws IOException
  46          *             in case of I/O error
  47          */
  48         public Cache(File dir, int hoursChanging, int hoursStable)
  49                         throws IOException {
  50                 this.dir = dir;
  51                 this.tooOldChanging = 1000L * 60 * 60 * hoursChanging;
  52                 this.tooOldStable = 1000L * 60 * 60 * hoursStable;
  53
  54                 if (dir != null && !dir.exists()) {
  55                         dir.mkdirs();
  56                 }
  57
  58                 if (dir == null || !dir.exists()) {
  59                         throw new IOException("Cannot create the cache directory: "
  60                                         + (dir == null ? "null" : dir.getAbsolutePath()));
  61                 }
  62         }
  63
  64         /**
  65          * The traces handler for this {@link Cache}.
  66          *
  67          * @return the traces handler
  68          */
  69         public TraceHandler getTraceHandler() {
  70                 return tracer;
  71         }
  72
  73         /**
  74          * The traces handler for this {@link Cache}.
  75          *
  76          * @param tracer
  77          *            the new traces handler
  78          */
  79         public void setTraceHandler(TraceHandler tracer) {
  80                 if (tracer == null) {
  81                         tracer = new TraceHandler(false, false, false);
  82                 }
  83
  84                 this.tracer = tracer;
  85         }
  86
  87         /**
  88          * Check the resource to see if it is in the cache.
  89          *
  90          * @param uniqueID
  91          *            the resource to check
  92          * @param allowTooOld
  93          *            allow files even if they are considered too old
  94          * @param stable
  95          *            a stable file (that dones't change too often) -- parameter
  96          *            used to check if the file is too old to keep or not
  97          *
  98          * @return TRUE if it is
  99          *
 100          */
 101         public boolean check(String uniqueID, boolean allowTooOld, boolean stable) {
 102                 return check(getCached(uniqueID), allowTooOld, stable);
 103         }
 104
 105         /**
 106          * Check the resource to see if it is in the cache.
 107          *
 108          * @param url
 109          *            the resource to check
 110          * @param allowTooOld
 111          *            allow files even if they are considered too old
 112          * @param stable
 113          *            a stable file (that dones't change too often) -- parameter
 114          *            used to check if the file is too old to keep or not
 115          *
 116          * @return TRUE if it is
 117          *
 118          */
 119         public boolean check(URL url, boolean allowTooOld, boolean stable) {
 120                 return check(getCached(url), allowTooOld, stable);
 121         }
 122
 123         /**
 124          * Check the resource to see if it is in the cache.
 125          *
 126          * @param cached
 127          *            the resource to check
 128          * @param allowTooOld
 129          *            allow files even if they are considered too old
 130          * @param stable
 131          *            a stable file (that dones't change too often) -- parameter
 132          *            used to check if the file is too old to keep or not
 133          *
 134          * @return TRUE if it is
 135          *
 136          */
 137         private boolean check(File cached, boolean allowTooOld, boolean stable) {
 138                 if (cached.exists() && cached.isFile()) {
 139                         if (!allowTooOld && isOld(cached, stable)) {
 140                                 if (!cached.delete()) {
 141                                         tracer.error("Cannot delete temporary file: "
 142                                                         + cached.getAbsolutePath());
 143                                 }
 144                         } else {
 145                                 return true;
 146                         }
 147                 }
 148
 149                 return false;
 150         }
 151
 152         /**
 153          * Clean the cache (delete the cached items).
 154          *
 155          * @param onlyOld
 156          *            only clean the files that are considered too old for a stable
 157          *            resource
 158          *
 159          * @return the number of cleaned items
 160          */
 161         public int clean(boolean onlyOld) {
 162                 long ms = System.currentTimeMillis();
 163
 164                 tracer.trace("Cleaning cache from old files...");
 165
 166                 int num = clean(onlyOld, dir, -1);
 167
 168                 tracer.trace(num + "cache items cleaned in "
 169                                 + (System.currentTimeMillis() - ms) + " ms");
 170
 171                 return num;
 172         }
 173
 174         /**
 175          * Clean the cache (delete the cached items) in the given cache directory.
 176          *
 177          * @param onlyOld
 178          *            only clean the files that are considered too old for stable
 179          *            resources
 180          * @param cacheDir
 181          *            the cache directory to clean
 182          * @param limit
 183          *            stop after limit files deleted, or -1 for unlimited
 184          *
 185          * @return the number of cleaned items
 186          */
 187         private int clean(boolean onlyOld, File cacheDir, int limit) {
 188                 int num = 0;
 189                 File[] files = cacheDir.listFiles();
 190                 if (files != null) {
 191                         for (File file : files) {
 192                                 if (limit >= 0 && num >= limit) {
 193                                         return num;
 194                                 }
 195
 196                                 if (file.isDirectory()) {
 197                                         num += clean(onlyOld, file, limit);
 198                                         file.delete(); // only if empty
 199                                 } else {
 200                                         if (!onlyOld || isOld(file, true)) {
 201                                                 if (file.delete()) {
 202                                                         num++;
 203                                                 } else {
 204                                                         tracer.error("Cannot delete temporary file: "
 205                                                                         + file.getAbsolutePath());
 206                                                 }
 207                                         }
 208                                 }
 209                         }
 210                 }
 211
 212                 return num;
 213         }
 214
 215         /**
 216          * Open a resource from the cache if it exists.
 217          *
 218          * @param uniqueID
 219          *            the unique ID
 220          * @param allowTooOld
 221          *            allow files even if they are considered too old
 222          * @param stable
 223          *            a stable file (that dones't change too often) -- parameter
 224          *            used to check if the file is too old to keep or not
 225          *
 226          * @return the opened resource if found, NULL if not
 227          */
 228         public InputStream load(String uniqueID, boolean allowTooOld, boolean stable) {
 229                 return load(getCached(uniqueID), allowTooOld, stable);
 230         }
 231
 232         /**
 233          * Open a resource from the cache if it exists.
 234          *
 235          * @param url
 236          *            the resource to open
 237          * @param allowTooOld
 238          *            allow files even if they are considered too old
 239          * @param stable
 240          *            a stable file (that doesn't change too often) -- parameter
 241          *            used to check if the file is too old to keep or not in the
 242          *            cache
 243          *
 244          * @return the opened resource if found, NULL if not
 245          */
 246         public InputStream load(URL url, boolean allowTooOld, boolean stable) {
 247                 return load(getCached(url), allowTooOld, stable);
 248         }
 249
 250         /**
 251          * Open a resource from the cache if it exists.
 252          *
 253          * @param cached
 254          *            the resource to open
 255          * @param allowTooOld
 256          *            allow files even if they are considered too old
 257          * @param stable
 258          *            a stable file (that dones't change too often) -- parameter
 259          *            used to check if the file is too old to keep or not
 260          *
 261          * @return the opened resource if found, NULL if not
 262          */
 263         private InputStream load(File cached, boolean allowTooOld, boolean stable) {
 264                 if (cached.exists() && cached.isFile()
 265                                 && (allowTooOld || !isOld(cached, stable))) {
 266                         try {
 267                                 return new MarkableFileInputStream(cached);
 268                         } catch (FileNotFoundException e) {
 269                                 return null;
 270                         }
 271                 }
 272
 273                 return null;
 274         }
 275
 276         /**
 277          * Save the given resource to the cache.
 278          *
 279          * @param in
 280          *            the input data
 281          * @param uniqueID
 282          *            a unique ID used to locate the cached resource
 283          *
 284          * @return the number of bytes written
 285          *
 286          * @throws IOException
 287          *             in case of I/O error
 288          */
 289         public long save(InputStream in, String uniqueID) throws IOException {
 290                 File cached = getCached(uniqueID);
 291                 cached.getParentFile().mkdirs();
 292                 return save(in, cached);
 293         }
 294
 295         /**
 296          * Save the given resource to the cache.
 297          *
 298          * @param in
 299          *            the input data
 300          * @param url
 301          *            the {@link URL} used to locate the cached resource
 302          *
 303          * @return the number of bytes written
 304          *
 305          * @throws IOException
 306          *             in case of I/O error
 307          */
 308         public long save(InputStream in, URL url) throws IOException {
 309                 File cached = getCached(url);
 310                 return save(in, cached);
 311         }
 312
 313         /**
 314          * Save the given resource to the cache.
 315          * <p>
 316          * Will also clean the {@link Cache} from old files.
 317          *
 318          * @param in
 319          *            the input data
 320          * @param cached
 321          *            the cached {@link File} to save to
 322          *
 323          * @return the number of bytes written
 324          *
 325          * @throws IOException
 326          *             in case of I/O error
 327          */
 328         private long save(InputStream in, File cached) throws IOException {
 329                 // We want to force at least an immediate SAVE/LOAD to work for some
 330                 // workflows, even if we don't accept cached files (times set to "0"
 331                 // -- and not "-1" or a positive value)
 332                 clean(true, dir, 10);
 333                 cached.getParentFile().mkdirs(); // in case we deleted our own parent
 334                 long bytes = IOUtils.write(in, cached);
 335                 return bytes;
 336         }
 337
 338         /**
 339          * Remove the given resource from the cache.
 340          *
 341          * @param uniqueID
 342          *            a unique ID used to locate the cached resource
 343          *
 344          * @return TRUE if it was removed
 345          */
 346         public boolean remove(String uniqueID) {
 347                 File cached = getCached(uniqueID);
 348                 return cached.delete();
 349         }
 350
 351         /**
 352          * Remove the given resource from the cache.
 353          *
 354          * @param url
 355          *            the {@link URL} used to locate the cached resource
 356          *
 357          * @return TRUE if it was removed
 358          */
 359         public boolean remove(URL url) {
 360                 File cached = getCached(url);
 361                 return cached.delete();
 362         }
 363
 364         /**
 365          * Check if the {@link File} is too old according to
 366          * {@link Cache#tooOldChanging}.
 367          *
 368          * @param file
 369          *            the file to check
 370          * @param stable
 371          *            TRUE to denote stable files, that are not supposed to change
 372          *            too often
 373          *
 374          * @return TRUE if it is
 375          */
 376         private boolean isOld(File file, boolean stable) {
 377                 long max = tooOldChanging;
 378                 if (stable) {
 379                         max = tooOldStable;
 380                 }
 381
 382                 if (max < 0) {
 383                         return false;
 384                 }
 385
 386                 long time = new Date().getTime() - file.lastModified();
 387                 if (time < 0) {
 388                         tracer.error("Timestamp in the future for file: "
 389                                         + file.getAbsolutePath());
 390                 }
 391
 392                 return time < 0 || time > max;
 393         }
 394
 395         /**
 396          * Return the associated cache {@link File} from this {@link URL}.
 397          *
 398          * @param url
 399          *            the {@link URL}
 400          *
 401          * @return the cached {@link File} version of this {@link URL}
 402          */
 403         private File getCached(URL url) {
 404                 File subdir;
 405
 406                 String name = url.getHost();
 407                 if (name == null || name.isEmpty()) {
 408                         // File
 409                         File file = new File(url.getFile());
 410                         if (file.getParent() == null) {
 411                                 subdir = new File("+");
 412                         } else {
 413                                 subdir = new File(file.getParent().replace("..", "__"));
 414                         }
 415                         subdir = new File(dir, allowedChars(subdir.getPath()));
 416                         name = allowedChars(url.getFile());
 417                 } else {
 418                         // URL
 419                         File subsubDir = new File(dir, allowedChars(url.getHost()));
 420                         subdir = new File(subsubDir, "_" + allowedChars(url.getPath()));
 421                         name = allowedChars("_" + url.getQuery());
 422                 }
 423
 424                 File cacheFile = new File(subdir, name);
 425                 subdir.mkdirs();
 426
 427                 return cacheFile;
 428         }
 429
 430         /**
 431          * Get the basic cache resource file corresponding to this unique ID.
 432          * <p>
 433          * Note that you may need to add a sub-directory in some cases.
 434          *
 435          * @param uniqueID
 436          *            the id
 437          *
 438          * @return the cached version if present, NULL if not
 439          */
 440         private File getCached(String uniqueID) {
 441                 File file = new File(dir, allowedChars(uniqueID));
 442                 File subdir = new File(file.getParentFile(), "_");
 443                 return new File(subdir, file.getName());
 444         }
 445
 446         /**
 447          * Replace not allowed chars (in a {@link File}) by "_".
 448          *
 449          * @param raw
 450          *            the raw {@link String}
 451          *
 452          * @return the sanitised {@link String}
 453          */
 454         private String allowedChars(String raw) {
 455                 return raw.replace('/', '_').replace(':', '_').replace("\\", "_");
 456         }
 457 }