X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Ffanfix%2FCache.java;h=383fa7c87a8cdd3f40b1041c9ac49e5d091e0189;hb=34eb62fc14c1f4db29e65ceb6b190811380aba7e;hp=0166a37d1689fa6152ec7a451bf568071809302b;hpb=9252c65e13ceb952626da9e1f9e6d5caef42733e;p=fanfix.git diff --git a/src/be/nikiroo/fanfix/Cache.java b/src/be/nikiroo/fanfix/Cache.java index 0166a37..383fa7c 100644 --- a/src/be/nikiroo/fanfix/Cache.java +++ b/src/be/nikiroo/fanfix/Cache.java @@ -3,18 +3,20 @@ package be.nikiroo.fanfix; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStreamWriter; import java.net.CookieHandler; import java.net.CookieManager; import java.net.CookiePolicy; import java.net.CookieStore; import java.net.HttpCookie; import java.net.HttpURLConnection; -import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; +import java.net.URLEncoder; import java.util.Date; import java.util.Map; import java.util.zip.GZIPInputStream; @@ -24,8 +26,8 @@ import javax.imageio.ImageIO; import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.supported.BasicSupport; import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.ImageUtils; import be.nikiroo.utils.MarkableFileInputStream; -import be.nikiroo.utils.StringUtils; /** * This cache will manage Internet (and local) downloads, as well as put the @@ -85,6 +87,13 @@ public class Cache { CookieHandler.setDefault(cookies); } + /** + * Clear all the cookies currently in the jar. + */ + public void clearCookies() { + cookies.getCookieStore().removeAll(); + } + /** * Open a resource (will load it from the cache if possible, or save it into * the cache after downloading if not). @@ -96,13 +105,14 @@ public class Cache { * @param stable * TRUE for more stable resources, FALSE when they often change * - * @return the opened resource + * @return the opened resource, NOT NULL * * @throws IOException * in case of I/O error */ public InputStream open(URL url, BasicSupport support, boolean stable) throws IOException { + // MUST NOT return null return open(url, support, stable, url); } @@ -121,16 +131,21 @@ public class Cache { * @param originalUrl * the original {@link URL} used to locate the cached resource * - * @return the opened resource + * @return the opened resource, NOT NULL * * @throws IOException * in case of I/O error */ public InputStream open(URL url, BasicSupport support, boolean stable, URL originalUrl) throws IOException { + // MUST NOT return null try { InputStream in = load(originalUrl, false, stable); + Instance.trace("Cache " + (in != null ? "hit" : "miss") + ": " + + url); + if (in == null) { + try { save(url, support, originalUrl); } catch (IOException e) { @@ -138,6 +153,7 @@ public class Cache { + (url == null ? "null" : url.toString()), e); } + // Was just saved, can load old, so, will not be null in = load(originalUrl, true, stable); } @@ -148,6 +164,150 @@ public class Cache { } } + /** + * Open the given {@link URL} without using the cache, but still using and + * updating the cookies. + * + * @param url + * the {@link URL} to open + * @param support + * the {@link BasicSupport} used for the cookies + * + * @return the {@link InputStream} of the opened page + * + * @throws IOException + * in case of I/O error + */ + public InputStream openNoCache(URL url, BasicSupport support) + throws IOException { + return openNoCache(url, support, url, null, null, null); + } + + /** + * Open the given {@link URL} without using the cache, but still using and + * updating the cookies. + * + * @param url + * the {@link URL} to open + * @param support + * the {@link BasicSupport} used for the cookies + * @param postParams + * the POST parameters + * @param getParams + * the GET parameters (priority over POST) + * @param oauth + * OAuth authorization (aka, "bearer XXXXXXX") + * + * @return the {@link InputStream} of the opened page + * + * @throws IOException + * in case of I/O error + */ + public InputStream openNoCache(URL url, BasicSupport support, + Map postParams, Map getParams, + String oauth) throws IOException { + return openNoCache(url, support, url, postParams, getParams, oauth); + } + + /** + * Open the given {@link URL} without using the cache, but still using and + * updating the cookies. + * + * @param url + * the {@link URL} to open + * @param support + * the {@link BasicSupport} used for the cookies + * @param originalUrl + * the original {@link URL} before any redirection occurs + * @param postParams + * the POST parameters + * @param getParams + * the GET parameters (priority over POST) + * @param oauth + * OAuth authorisation (aka, "bearer XXXXXXX") + * @return the {@link InputStream} of the opened page + * + * @throws IOException + * in case of I/O error + */ + private InputStream openNoCache(URL url, BasicSupport support, + final URL originalUrl, Map postParams, + Map getParams, String oauth) throws IOException { + + Instance.trace("Open no cache: " + url); + + URLConnection conn = openConnectionWithCookies(url, support); + if (support != null) { + // priority: arguments + if (oauth == null) { + oauth = support.getOAuth(); + } + } + + // Priority: GET over POST + Map params = getParams; + if (getParams == null) { + params = postParams; + } + + if ((params != null || oauth != null) + && conn instanceof HttpURLConnection) { + StringBuilder requestData = null; + if (params != null) { + requestData = new StringBuilder(); + for (Map.Entry param : params.entrySet()) { + if (requestData.length() != 0) + requestData.append('&'); + requestData.append(URLEncoder.encode(param.getKey(), + "UTF-8")); + requestData.append('='); + requestData.append(URLEncoder.encode( + String.valueOf(param.getValue()), "UTF-8")); + } + + conn.setDoOutput(true); + + if (getParams == null && postParams != null) { + ((HttpURLConnection) conn).setRequestMethod("POST"); + } + + conn.setRequestProperty("Content-Type", + "application/x-www-form-urlencoded"); + conn.setRequestProperty("charset", "utf-8"); + } + + if (oauth != null) { + conn.setRequestProperty("Authorization", oauth); + } + + if (requestData != null) { + OutputStreamWriter writer = new OutputStreamWriter( + conn.getOutputStream()); + + writer.write(requestData.toString()); + writer.flush(); + writer.close(); + } + } + + conn.connect(); + + // Check if redirect + if (conn instanceof HttpURLConnection + && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) { + String newUrl = conn.getHeaderField("Location"); + return openNoCache(new URL(newUrl), support, originalUrl, + postParams, getParams, oauth); + } + + InputStream in = conn.getInputStream(); + if ("gzip".equals(conn.getContentEncoding())) { + in = new GZIPInputStream(in); + } + + return in; + } + /** * Refresh the resource into cache if needed. * @@ -158,8 +318,6 @@ public class Cache { * @param stable * TRUE for more stable resources, FALSE when they often change * - * @return TRUE if it was pre-downloaded - * * @throws IOException * in case of I/O error */ @@ -187,31 +345,24 @@ public class Cache { } /** - * Open a resource (will load it from the cache if possible, or save it into - * the cache after downloading if not) as an Image, then save it where - * requested. - *

- * This version will not always work properly if the original file was not - * downloaded before. + * Save the given resource as an image on disk using the default image + * format for content. * * @param url - * the resource to open - * - * @return the opened resource image + * the resource + * @param target + * the target file * * @throws IOException * in case of I/O error */ public void saveAsImage(URL url, File target) throws IOException { - URL cachedUrl = new URL(url.toString() - + "." - + Instance.getConfig().getString(Config.IMAGE_FORMAT_CONTENT) - .toLowerCase()); + URL cachedUrl = new URL(url.toString()); File cached = getCached(cachedUrl); if (!cached.exists() || isOld(cached, true)) { - InputStream imageIn = Instance.getCache().open(url, null, true); - ImageIO.write(StringUtils.toImage(imageIn), Instance.getConfig() + InputStream imageIn = open(url, null, true); + ImageIO.write(ImageUtils.fromStream(imageIn), Instance.getConfig() .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase(), cached); } @@ -233,11 +384,37 @@ public class Cache { * in case of I/O error */ public File addToCache(InputStream in, String uniqueID) throws IOException { - File file = getCached(new File(uniqueID).toURI().toURL()); + File file = getCached(uniqueID); + File subdir = new File(file.getParentFile(), "_"); + file = new File(subdir, file.getName()); + subdir.mkdir(); IOUtils.write(in, file); return file; } + /** + * Return the {@link InputStream} corresponding to the given unique ID, or + * NULL if none found. + * + * @param uniqueID + * the unique ID + * + * @return the content or NULL + */ + public InputStream getFromCache(String uniqueID) { + File file = getCached(uniqueID); + File subdir = new File(file.getParentFile(), "_"); + file = new File(subdir, file.getName()); + if (file.exists()) { + try { + return new MarkableFileInputStream(new FileInputStream(file)); + } catch (FileNotFoundException e) { + } + } + + return null; + } + /** * Clean the cache (delete the cached items). * @@ -247,17 +424,36 @@ public class Cache { * @return the number of cleaned items */ public int cleanCache(boolean onlyOld) { + return cleanCache(onlyOld, dir); + } + + /** + * Clean the cache (delete the cached items) in the given cache directory. + * + * @param onlyOld + * only clean the files that are considered too old + * @param cacheDir + * the cache directory to clean + * + * @return the number of cleaned items + */ + private int cleanCache(boolean onlyOld, File cacheDir) { int num = 0; - for (File file : dir.listFiles()) { - if (!onlyOld || isOld(file, true)) { - if (file.delete()) { - num++; - } else { - System.err.println("Cannot delete temporary file: " - + file.getAbsolutePath()); + for (File file : cacheDir.listFiles()) { + if (file.isDirectory()) { + num += cleanCache(onlyOld, file); + } else { + if (!onlyOld || isOld(file, true)) { + if (file.delete()) { + num++; + } else { + System.err.println("Cannot delete temporary file: " + + file.getAbsolutePath()); + } } } } + return num; } @@ -266,14 +462,21 @@ public class Cache { * * @param url * the resource to open - * @return the opened resource + * @param allowTooOld + * allow files even if they are considered too old + * @param stable + * a stable file (that dones't change too often) -- parameter + * used to check if the file is too old to keep or not + * + * @return the opened resource if found, NULL i not + * * @throws IOException * in case of I/O error */ - private InputStream load(URL url, boolean allowOld, boolean stable) + private InputStream load(URL url, boolean allowTooOld, boolean stable) throws IOException { File cached = getCached(url); - if (cached.exists() && !isOld(cached, stable)) { + if (cached.exists() && (allowTooOld || !isOld(cached, stable))) { return new MarkableFileInputStream(new FileInputStream(cached)); } @@ -292,37 +495,11 @@ public class Cache { * * @throws IOException * in case of I/O error - * @throws URISyntaxException */ private void save(URL url, BasicSupport support, URL originalUrl) throws IOException { - URLConnection conn = url.openConnection(); - - conn.setRequestProperty("User-Agent", UA); - conn.setRequestProperty("Cookie", generateCookies(support)); - conn.setRequestProperty("Accept-Encoding", "gzip"); - if (support != null && support.getCurrentReferer() != null) { - conn.setRequestProperty("Referer", support.getCurrentReferer() - .toString()); - conn.setRequestProperty("Host", support.getCurrentReferer() - .getHost()); - } - - conn.connect(); - - // Check if redirect - if (conn instanceof HttpURLConnection - && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) { - String newUrl = conn.getHeaderField("Location"); - save(new URL(newUrl), support, originalUrl); - return; - } - - InputStream in = conn.getInputStream(); - if ("gzip".equals(conn.getContentEncoding())) { - in = new GZIPInputStream(in); - } - + InputStream in = openNoCache(url, support, originalUrl, null, null, + null); try { File cached = getCached(originalUrl); BufferedOutputStream out = new BufferedOutputStream( @@ -341,6 +518,37 @@ public class Cache { } } + /** + * Open a connection on the given {@link URL}, and manage the cookies that + * come with it. + * + * @param url + * the {@link URL} to open + * @param support + * the {@link BasicSupport} to use for cookie generation + * + * @return the connection + * + * @throws IOException + * in case of I/O error + */ + private URLConnection openConnectionWithCookies(URL url, + BasicSupport support) throws IOException { + URLConnection conn = url.openConnection(); + + conn.setRequestProperty("User-Agent", UA); + conn.setRequestProperty("Cookie", generateCookies(support)); + conn.setRequestProperty("Accept-Encoding", "gzip"); + if (support != null && support.getCurrentReferer() != null) { + conn.setRequestProperty("Referer", support.getCurrentReferer() + .toString()); + conn.setRequestProperty("Host", support.getCurrentReferer() + .getHost()); + } + + return conn; + } + /** * Check if the {@link File} is too old according to * {@link Cache#tooOldChanging}. @@ -372,24 +580,59 @@ public class Cache { } /** - * Get the cache resource from the cache if it is present for this - * {@link URL}. + * Return the associated cache {@link File} from this {@link URL}. * * @param url * the url - * @return the cached version if present, NULL if not + * + * @return the cached {@link File} version of this {@link URL} */ private File getCached(URL url) { + File subdir = null; + String name = url.getHost(); - if (name == null || name.length() == 0) { + if (name == null || name.isEmpty()) { name = url.getFile(); } else { - name = url.toString(); + File cacheDir = getCached(".").getParentFile(); + File subsubDir = new File(cacheDir, allowedChars(url.getHost())); + subdir = new File(subsubDir, "_" + allowedChars(url.getPath())); + name = allowedChars("_" + url.getQuery()); } - name = name.replace('/', '_').replace(':', '_'); + File cacheFile = getCached(name); + if (subdir != null) { + cacheFile = new File(subdir, cacheFile.getName()); + subdir.mkdirs(); + } - return new File(dir, name); + return cacheFile; + } + + /** + * Get the basic cache resource file corresponding to this unique ID. + *

+ * Note that you may need to add a sub-directory in some cases. + * + * @param uniqueID + * the id + * + * @return the cached version if present, NULL if not + */ + private File getCached(String uniqueID) { + return new File(dir, allowedChars(uniqueID)); + } + + /** + * Replace not allowed chars (in a {@link File}) by "_". + * + * @param raw + * the raw {@link String} + * + * @return the sanitised {@link String} + */ + private String allowedChars(String raw) { + return raw.replace('/', '_').replace(':', '_').replace("\\", "_"); } /**