X-Git-Url: http://git.nikiroo.be/?p=fanfix.git;a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Futils%2FDownloader.java;h=4191d0aea0da85511e2741c50c2a6c83806f612f;hp=f7a5f57f0a7d5cd9020d5b5bdc47389fe2ca0222;hb=HEAD;hpb=15f134727647c2c73a6c39b4404c74d1715949ce
diff --git a/src/be/nikiroo/utils/Downloader.java b/src/be/nikiroo/utils/Downloader.java
index f7a5f57..4191d0a 100644
--- a/src/be/nikiroo/utils/Downloader.java
+++ b/src/be/nikiroo/utils/Downloader.java
@@ -30,6 +30,7 @@ public class Downloader {
private CookieManager cookies;
private TraceHandler tracer = new TraceHandler();
private Cache cache;
+ private boolean offline;
/**
* Create a new {@link Downloader}.
@@ -38,7 +39,7 @@ public class Downloader {
* the User-Agent to use to download the resources -- note that
* some websites require one, some actively blacklist real UAs
* like the one from wget, some whitelist a couple of browsers
- * only (!)
+ * only (!) -- can be NULL
*/
public Downloader(String UA) {
this(UA, null);
@@ -51,7 +52,7 @@ public class Downloader {
* the User-Agent to use to download the resources -- note that
* some websites require one, some actively blacklist real UAs
* like the one from wget, some whitelist a couple of browsers
- * only (!)
+ * only (!) -- can be NULL
* @param cache
* the {@link Cache} to use for all access (can be NULL)
*/
@@ -63,6 +64,32 @@ public class Downloader {
setCache(cache);
}
+
+ /**
+ * This {@link Downloader} is forbidden to try and connect to the network.
+ *
+ * If TRUE, it will only check the cache if any.
+ *
+ * Default is FALSE.
+ *
+ * @return TRUE if offline
+ */
+ public boolean isOffline() {
+ return offline;
+ }
+
+ /**
+ * This {@link Downloader} is forbidden to try and connect to the network.
+ *
+ * If TRUE, it will only check the cache if any.
+ *
+ * Default is FALSE.
+ *
+ * @param offline TRUE for offline, FALSE for online
+ */
+ public void setOffline(boolean offline) {
+ this.offline = offline;
+ }
/**
* The traces handler for this {@link Cache}.
@@ -167,7 +194,7 @@ public class Downloader {
* @return the {@link InputStream} of the opened page
*
* @throws IOException
- * in case of I/O error
+ * in case of I/O error (including offline mode + not in cache)
*/
public InputStream open(URL url, URL currentReferer,
Map cookiesValues, Map postParams,
@@ -199,7 +226,7 @@ public class Downloader {
* @return the {@link InputStream} of the opened page
*
* @throws IOException
- * in case of I/O error
+ * in case of I/O error (including offline mode + not in cache)
*/
public InputStream open(URL url, URL currentReferer,
Map cookiesValues, Map postParams,
@@ -236,7 +263,7 @@ public class Downloader {
* @return the {@link InputStream} of the opened page
*
* @throws IOException
- * in case of I/O error
+ * in case of I/O error (including offline mode + not in cache)
*/
public InputStream open(URL url, final URL originalUrl, URL currentReferer,
Map cookiesValues, Map postParams,
@@ -254,6 +281,13 @@ public class Downloader {
}
}
+ String protocol = originalUrl == null ? null : originalUrl
+ .getProtocol();
+ if (isOffline() && !"file".equalsIgnoreCase(protocol)) {
+ tracer.error("Downloader OFFLINE, cannot proceed to URL: " + url);
+ throw new IOException("Downloader is currently OFFLINE, cannot download: " + url);
+ }
+
tracer.trace("Download: " + url);
URLConnection conn = openConnectionWithCookies(url, currentReferer,
@@ -288,7 +322,6 @@ public class Downloader {
"application/x-www-form-urlencoded");
conn.setRequestProperty("Content-Length",
Integer.toString(requestData.length()));
- conn.setRequestProperty("charset", "utf-8");
}
if (oauth != null) {
@@ -316,7 +349,8 @@ public class Downloader {
conn.connect();
// Check if redirect
- // BEWARE! POST data cannot be redirected, so it is ignored here
+ // BEWARE! POST data cannot be redirected (some webservers complain) for
+ // HTTP codes 302 and 303
if (conn instanceof HttpURLConnection) {
int repCode = 0;
try {
@@ -328,32 +362,53 @@ public class Downloader {
if (repCode / 100 == 3) {
String newUrl = conn.getHeaderField("Location");
return open(new URL(newUrl), originalUrl, currentReferer,
- cookiesValues, null, getParams, oauth, stable);
+ cookiesValues, //
+ (repCode == 302 || repCode == 303) ? null : postParams, //
+ getParams, oauth, stable);
}
}
- InputStream in = conn.getInputStream();
- if ("gzip".equals(conn.getContentEncoding())) {
- in = new GZIPInputStream(in);
- }
+ try {
+ InputStream in = conn.getInputStream();
+ if ("gzip".equals(conn.getContentEncoding())) {
+ in = new GZIPInputStream(in);
+ }
- if (in != null && cache != null) {
- tracer.trace("Save to cache: " + originalUrl);
- try {
+ if (in == null) {
+ throw new IOException("No InputStream!");
+ }
+
+ if (cache != null) {
+ String size = conn.getContentLength() < 0 ? "unknown size"
+ : StringUtils.formatNumber(conn.getContentLength())
+ + "bytes";
+ tracer.trace("Save to cache (" + size + "): " + originalUrl);
try {
- cache.save(in, originalUrl);
- } finally {
- in.close();
+ try {
+ long bytes = cache.save(in, originalUrl);
+ tracer.trace("Saved to cache: "
+ + StringUtils.formatNumber(bytes) + "bytes");
+ } finally {
+ in.close();
+ }
+ in = cache.load(originalUrl, true, true);
+ } catch (IOException e) {
+ tracer.error(new IOException(
+ "Cannot save URL to cache, will ignore cache: "
+ + url, e));
}
- in = cache.load(originalUrl, true, false);
- } catch (IOException e) {
- tracer.error(new IOException(
- "Cannot save URL to cache, will ignore cache: " + url,
- e));
}
- }
- return in;
+ if (in == null) {
+ throw new IOException(
+ "Cannot retrieve the file after storing it in the cache (??)");
+ }
+
+ return in;
+ } catch (IOException e) {
+ throw new IOException(String.format(
+ "Cannot find %s (current URL: %s)", originalUrl, url), e);
+ }
}
/**
@@ -378,9 +433,12 @@ public class Downloader {
conn.setRequestProperty("Cookie", cookies);
}
- conn.setRequestProperty("User-Agent", UA);
+ if (UA != null) {
+ conn.setRequestProperty("User-Agent", UA);
+ }
conn.setRequestProperty("Accept-Encoding", "gzip");
conn.setRequestProperty("Accept", "*/*");
+ conn.setRequestProperty("Charset", "utf-8");
if (currentReferer != null) {
conn.setRequestProperty("Referer", currentReferer.toString());