X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Futils%2FDownloader.java;h=4976c483aef7ab8be33fb21ee757f205b13b292f;hb=3052163b494bf89d178638a047ca80008578a782;hp=2919ff01bf551d2d99eca89883c42a1c2e744f51;hpb=6149689f27e74830cf2638c8ceffbe0dca9b82f0;p=fanfix.git diff --git a/src/be/nikiroo/utils/Downloader.java b/src/be/nikiroo/utils/Downloader.java index 2919ff0..4976c48 100644 --- a/src/be/nikiroo/utils/Downloader.java +++ b/src/be/nikiroo/utils/Downloader.java @@ -29,6 +29,7 @@ public class Downloader { private String UA; private CookieManager cookies; private TraceHandler tracer = new TraceHandler(); + private Cache cache; /** * Create a new {@link Downloader}. @@ -40,11 +41,28 @@ public class Downloader { * only (!) */ public Downloader(String UA) { + this(UA, null); + } + + /** + * Create a new {@link Downloader}. + * + * @param UA + * the User-Agent to use to download the resources -- note that + * some websites require one, some actively blacklist real UAs + * like the one from wget, some whitelist a couple of browsers + * only (!) + * @param cache + * the {@link Cache} to use for all access (can be NULL) + */ + public Downloader(String UA, Cache cache) { this.UA = UA; cookies = new CookieManager(); cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL); CookieHandler.setDefault(cookies); + + this.cache = cache; } /** @@ -70,6 +88,25 @@ public class Downloader { this.tracer = tracer; } + /** + * The {@link Cache} to use for all access (can be NULL). + * + * @return the cache + */ + public Cache getCache() { + return cache; + } + + /** + * The {@link Cache} to use for all access (can be NULL). + * + * @param cache + * the new cache + */ + public void setCache(Cache cache) { + this.cache = cache; + } + /** * Clear all the cookies currently in the jar. *
@@ -90,7 +127,26 @@ public class Downloader {
* in case of I/O error
**/
public InputStream open(URL url) throws IOException {
- return open(url, url, url, null, null, null, null);
+ return open(url, false);
+ }
+
+ /**
+ * Open the given {@link URL} and update the cookies.
+ *
+ * @param url
+ * the {@link URL} to open
+ * @param stable
+ * stable a stable file (that doesn't change too often) --
+ * parameter used to check if the file is too old to keep or not
+ * in the cache (default is false)
+ *
+ * @return the {@link InputStream} of the opened page
+ *
+ * @throws IOException
+ * in case of I/O error
+ **/
+ public InputStream open(URL url, boolean stable) throws IOException {
+ return open(url, url, url, null, null, null, null, stable);
}
/**
@@ -117,8 +173,41 @@ public class Downloader {
public InputStream open(URL url, URL currentReferer,
Map