X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Futils%2FDownloader.java;h=f7a5f57f0a7d5cd9020d5b5bdc47389fe2ca0222;hb=15f134727647c2c73a6c39b4404c74d1715949ce;hp=651abc3a65932fbc9cbd7355610eb1c2292db21f;hpb=f6e8d60dbb9f124046f1b951315d74f003624f09;p=fanfix.git

diff --git a/src/be/nikiroo/utils/Downloader.java b/src/be/nikiroo/utils/Downloader.java
index 651abc3..f7a5f57 100644
--- a/src/be/nikiroo/utils/Downloader.java
+++ b/src/be/nikiroo/utils/Downloader.java
@@ -58,11 +58,10 @@ public class Downloader {
 	public Downloader(String UA, Cache cache) {
 		this.UA = UA;
 
-		cookies = new CookieManager();
-		cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
+		cookies = new CookieManager(null, CookiePolicy.ACCEPT_ALL);
 		CookieHandler.setDefault(cookies);
 
-		this.cache = cache;
+		setCache(cache);
 	}
 
 	/**
@@ -88,6 +87,25 @@ public class Downloader {
 		this.tracer = tracer;
 	}
 
+	/**
+	 * The {@link Cache} to use for all access (can be NULL).
+	 * 
+	 * @return the cache
+	 */
+	public Cache getCache() {
+		return cache;
+	}
+
+	/**
+	 * The {@link Cache} to use for all access (can be NULL).
+	 * 
+	 * @param cache
+	 *            the new cache
+	 */
+	public void setCache(Cache cache) {
+		this.cache = cache;
+	}
+
 	/**
 	 * Clear all the cookies currently in the jar.
 	 * <p>
@@ -197,7 +215,13 @@ public class Downloader {
 	 * @param url
 	 *            the {@link URL} to open
 	 * @param originalUrl
-	 *            the original {@link URL} before any redirection occurs
+	 *            the original {@link URL} before any redirection occurs, which
+	 *            is also used for the cache ID if needed (so we can retrieve
+	 *            the content with this URL if needed)
+	 * @param currentReferer
+	 *            the current referer, for websites that needs this info
+	 * @param cookiesValues
+	 *            the cookies
 	 * @param postParams
 	 *            the POST parameters
 	 * @param getParams
@@ -214,17 +238,18 @@ public class Downloader {
 	 * @throws IOException
 	 *             in case of I/O error
 	 */
-	private InputStream open(URL url, final URL originalUrl,
-			URL currentReferer, Map<String, String> cookiesValues,
-			Map<String, String> postParams, Map<String, String> getParams,
-			String oauth, boolean stable) throws IOException {
+	public InputStream open(URL url, final URL originalUrl, URL currentReferer,
+			Map<String, String> cookiesValues, Map<String, String> postParams,
+			Map<String, String> getParams, String oauth, boolean stable)
+			throws IOException {
 
 		tracer.trace("Request: " + url);
 
 		if (cache != null) {
-			InputStream in = cache.load(url, false, stable);
+			InputStream in = cache.load(originalUrl, false, stable);
 			if (in != null) {
-				tracer.trace("Take from cache: " + url);
+				tracer.trace("Use the cache: " + url);
+				tracer.trace("Original URL : " + originalUrl);
 				return in;
 			}
 		}
@@ -240,9 +265,9 @@ public class Downloader {
 			params = postParams;
 		}
 
+		StringBuilder requestData = null;
 		if ((params != null || oauth != null)
 				&& conn instanceof HttpURLConnection) {
-			StringBuilder requestData = null;
 			if (params != null) {
 				requestData = new StringBuilder();
 				for (Map.Entry<String, String> param : params.entrySet()) {
@@ -255,14 +280,14 @@ public class Downloader {
 							String.valueOf(param.getValue()), "UTF-8"));
 				}
 
-				conn.setDoOutput(true);
-
 				if (getParams == null && postParams != null) {
 					((HttpURLConnection) conn).setRequestMethod("POST");
 				}
 
 				conn.setRequestProperty("Content-Type",
 						"application/x-www-form-urlencoded");
+				conn.setRequestProperty("Content-Length",
+						Integer.toString(requestData.length()));
 				conn.setRequestProperty("charset", "utf-8");
 			}
 
@@ -271,22 +296,27 @@ public class Downloader {
 			}
 
 			if (requestData != null) {
-				OutputStreamWriter writer = null;
+				conn.setDoOutput(true);
+				OutputStreamWriter writer = new OutputStreamWriter(
+						conn.getOutputStream());
 				try {
-					writer = new OutputStreamWriter(conn.getOutputStream());
 					writer.write(requestData.toString());
 					writer.flush();
 				} finally {
-					if (writer != null) {
-						writer.close();
-					}
+					writer.close();
 				}
 			}
 		}
 
+		// Manual redirection, much better for POST data
+		if (conn instanceof HttpURLConnection) {
+			((HttpURLConnection) conn).setInstanceFollowRedirects(false);
+		}
+
 		conn.connect();
 
 		// Check if redirect
+		// BEWARE! POST data cannot be redirected, so it is ignored here
 		if (conn instanceof HttpURLConnection) {
 			int repCode = 0;
 			try {
@@ -298,7 +328,7 @@ public class Downloader {
 			if (repCode / 100 == 3) {
 				String newUrl = conn.getHeaderField("Location");
 				return open(new URL(newUrl), originalUrl, currentReferer,
-						cookiesValues, postParams, getParams, oauth, stable);
+						cookiesValues, null, getParams, oauth, stable);
 			}
 		}
 
@@ -308,9 +338,14 @@ public class Downloader {
 		}
 
 		if (in != null && cache != null) {
-			tracer.trace("Save to cache: " + url);
+			tracer.trace("Save to cache: " + originalUrl);
 			try {
-				cache.save(in, url);
+				try {
+					cache.save(in, originalUrl);
+				} finally {
+					in.close();
+				}
+				in = cache.load(originalUrl, true, false);
 			} catch (IOException e) {
 				tracer.error(new IOException(
 						"Cannot save URL to cache, will ignore cache: " + url,
@@ -338,9 +373,15 @@ public class Downloader {
 			throws IOException {
 		URLConnection conn = url.openConnection();
 
+		String cookies = generateCookies(cookiesValues);
+		if (cookies != null && !cookies.isEmpty()) {
+			conn.setRequestProperty("Cookie", cookies);
+		}
+
 		conn.setRequestProperty("User-Agent", UA);
-		conn.setRequestProperty("Cookie", generateCookies(cookiesValues));
 		conn.setRequestProperty("Accept-Encoding", "gzip");
+		conn.setRequestProperty("Accept", "*/*");
+
 		if (currentReferer != null) {
 			conn.setRequestProperty("Referer", currentReferer.toString());
 			conn.setRequestProperty("Host", currentReferer.getHost());
@@ -362,7 +403,6 @@ public class Downloader {
 				builder.append(';');
 			}
 
-			// TODO: check if format is ok
 			builder.append(cookie.toString());
 		}