X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Futils%2FDownloader.java;h=f7a5f57f0a7d5cd9020d5b5bdc47389fe2ca0222;hb=8af76801d25181b38e5c2b829a2bc07cf16e3616;hp=a8a591a363958412a73eb6ef042d38ba33ae1cf9;hpb=530d4062471346d6ececf76d74a0358c91323998;p=fanfix.git diff --git a/src/be/nikiroo/utils/Downloader.java b/src/be/nikiroo/utils/Downloader.java index a8a591a..f7a5f57 100644 --- a/src/be/nikiroo/utils/Downloader.java +++ b/src/be/nikiroo/utils/Downloader.java @@ -29,6 +29,7 @@ public class Downloader { private String UA; private CookieManager cookies; private TraceHandler tracer = new TraceHandler(); + private Cache cache; /** * Create a new {@link Downloader}. @@ -40,11 +41,27 @@ public class Downloader { * only (!) */ public Downloader(String UA) { + this(UA, null); + } + + /** + * Create a new {@link Downloader}. + * + * @param UA + * the User-Agent to use to download the resources -- note that + * some websites require one, some actively blacklist real UAs + * like the one from wget, some whitelist a couple of browsers + * only (!) + * @param cache + * the {@link Cache} to use for all access (can be NULL) + */ + public Downloader(String UA, Cache cache) { this.UA = UA; - cookies = new CookieManager(); - cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL); + cookies = new CookieManager(null, CookiePolicy.ACCEPT_ALL); CookieHandler.setDefault(cookies); + + setCache(cache); } /** @@ -63,9 +80,32 @@ public class Downloader { * the new traces handler */ public void setTraceHandler(TraceHandler tracer) { + if (tracer == null) { + tracer = new TraceHandler(false, false, false); + } + this.tracer = tracer; } + /** + * The {@link Cache} to use for all access (can be NULL). + * + * @return the cache + */ + public Cache getCache() { + return cache; + } + + /** + * The {@link Cache} to use for all access (can be NULL). + * + * @param cache + * the new cache + */ + public void setCache(Cache cache) { + this.cache = cache; + } + /** * Clear all the cookies currently in the jar. *

@@ -86,7 +126,26 @@ public class Downloader { * in case of I/O error **/ public InputStream open(URL url) throws IOException { - return open(url, url, url, null, null, null, null); + return open(url, false); + } + + /** + * Open the given {@link URL} and update the cookies. + * + * @param url + * the {@link URL} to open + * @param stable + * stable a stable file (that doesn't change too often) -- + * parameter used to check if the file is too old to keep or not + * in the cache (default is false) + * + * @return the {@link InputStream} of the opened page + * + * @throws IOException + * in case of I/O error + **/ + public InputStream open(URL url, boolean stable) throws IOException { + return open(url, url, url, null, null, null, null, stable); } /** @@ -113,8 +172,41 @@ public class Downloader { public InputStream open(URL url, URL currentReferer, Map cookiesValues, Map postParams, Map getParams, String oauth) throws IOException { + return open(url, currentReferer, cookiesValues, postParams, getParams, + oauth, false); + } + + /** + * Open the given {@link URL} and update the cookies. + * + * @param url + * the {@link URL} to open + * @param currentReferer + * the current referer, for websites that needs this info + * @param cookiesValues + * the cookies + * @param postParams + * the POST parameters + * @param getParams + * the GET parameters (priority over POST) + * @param oauth + * OAuth authorization (aka, "bearer XXXXXXX") + * @param stable + * stable a stable file (that doesn't change too often) -- + * parameter used to check if the file is too old to keep or not + * in the cache (default is false) + * + * @return the {@link InputStream} of the opened page + * + * @throws IOException + * in case of I/O error + */ + public InputStream open(URL url, URL currentReferer, + Map cookiesValues, Map postParams, + Map getParams, String oauth, boolean stable) + throws IOException { return open(url, url, currentReferer, cookiesValues, postParams, - getParams, oauth); + getParams, oauth, stable); } /** @@ -123,22 +215,44 @@ public class Downloader { * @param url * the {@link URL} to open * @param originalUrl - * the original {@link URL} before any redirection occurs + * the original {@link URL} before any redirection occurs, which + * is also used for the cache ID if needed (so we can retrieve + * the content with this URL if needed) + * @param currentReferer + * the current referer, for websites that needs this info + * @param cookiesValues + * the cookies * @param postParams * the POST parameters * @param getParams * the GET parameters (priority over POST) * @param oauth * OAuth authorisation (aka, "bearer XXXXXXX") + * @param stable + * a stable file (that doesn't change too often) -- parameter + * used to check if the file is too old to keep or not in the + * cache + * * @return the {@link InputStream} of the opened page * * @throws IOException * in case of I/O error */ - private InputStream open(URL url, final URL originalUrl, - URL currentReferer, Map cookiesValues, - Map postParams, Map getParams, - String oauth) throws IOException { + public InputStream open(URL url, final URL originalUrl, URL currentReferer, + Map cookiesValues, Map postParams, + Map getParams, String oauth, boolean stable) + throws IOException { + + tracer.trace("Request: " + url); + + if (cache != null) { + InputStream in = cache.load(originalUrl, false, stable); + if (in != null) { + tracer.trace("Use the cache: " + url); + tracer.trace("Original URL : " + originalUrl); + return in; + } + } tracer.trace("Download: " + url); @@ -151,9 +265,9 @@ public class Downloader { params = postParams; } + StringBuilder requestData = null; if ((params != null || oauth != null) && conn instanceof HttpURLConnection) { - StringBuilder requestData = null; if (params != null) { requestData = new StringBuilder(); for (Map.Entry param : params.entrySet()) { @@ -166,14 +280,14 @@ public class Downloader { String.valueOf(param.getValue()), "UTF-8")); } - conn.setDoOutput(true); - if (getParams == null && postParams != null) { ((HttpURLConnection) conn).setRequestMethod("POST"); } conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + conn.setRequestProperty("Content-Length", + Integer.toString(requestData.length())); conn.setRequestProperty("charset", "utf-8"); } @@ -182,23 +296,40 @@ public class Downloader { } if (requestData != null) { + conn.setDoOutput(true); OutputStreamWriter writer = new OutputStreamWriter( conn.getOutputStream()); - - writer.write(requestData.toString()); - writer.flush(); - writer.close(); + try { + writer.write(requestData.toString()); + writer.flush(); + } finally { + writer.close(); + } } } + // Manual redirection, much better for POST data + if (conn instanceof HttpURLConnection) { + ((HttpURLConnection) conn).setInstanceFollowRedirects(false); + } + conn.connect(); // Check if redirect - if (conn instanceof HttpURLConnection - && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) { - String newUrl = conn.getHeaderField("Location"); - return open(new URL(newUrl), originalUrl, currentReferer, - cookiesValues, postParams, getParams, oauth); + // BEWARE! POST data cannot be redirected, so it is ignored here + if (conn instanceof HttpURLConnection) { + int repCode = 0; + try { + // Can fail in some circumstances + repCode = ((HttpURLConnection) conn).getResponseCode(); + } catch (IOException e) { + } + + if (repCode / 100 == 3) { + String newUrl = conn.getHeaderField("Location"); + return open(new URL(newUrl), originalUrl, currentReferer, + cookiesValues, null, getParams, oauth, stable); + } } InputStream in = conn.getInputStream(); @@ -206,6 +337,22 @@ public class Downloader { in = new GZIPInputStream(in); } + if (in != null && cache != null) { + tracer.trace("Save to cache: " + originalUrl); + try { + try { + cache.save(in, originalUrl); + } finally { + in.close(); + } + in = cache.load(originalUrl, true, false); + } catch (IOException e) { + tracer.error(new IOException( + "Cannot save URL to cache, will ignore cache: " + url, + e)); + } + } + return in; } @@ -226,9 +373,15 @@ public class Downloader { throws IOException { URLConnection conn = url.openConnection(); + String cookies = generateCookies(cookiesValues); + if (cookies != null && !cookies.isEmpty()) { + conn.setRequestProperty("Cookie", cookies); + } + conn.setRequestProperty("User-Agent", UA); - conn.setRequestProperty("Cookie", generateCookies(cookiesValues)); conn.setRequestProperty("Accept-Encoding", "gzip"); + conn.setRequestProperty("Accept", "*/*"); + if (currentReferer != null) { conn.setRequestProperty("Referer", currentReferer.toString()); conn.setRequestProperty("Host", currentReferer.getHost()); @@ -250,7 +403,6 @@ public class Downloader { builder.append(';'); } - // TODO: check if format is ok builder.append(cookie.toString()); }