Downloader: fix POST redirect bug

[fanfix.git] / src / be / nikiroo / utils / Downloader.java

diff --git a/src/be/nikiroo/utils/Downloader.java b/src/be/nikiroo/utils/Downloader.java
index 651abc3a65932fbc9cbd7355610eb1c2292db21f..f7a5f57f0a7d5cd9020d5b5bdc47389fe2ca0222 100644 (file)
--- a/src/be/nikiroo/utils/Downloader.java
+++ b/src/be/nikiroo/utils/Downloader.java
@@ -58,11 +58,10 @@ public class Downloader {
        public Downloader(String UA, Cache cache) {
                this.UA = UA;
 
-               cookies = new CookieManager();
-               cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
+               cookies = new CookieManager(null, CookiePolicy.ACCEPT_ALL);
                CookieHandler.setDefault(cookies);
 
-               this.cache = cache;
+               setCache(cache);
        }
 
        /**
@@ -88,6 +87,25 @@ public class Downloader {
                this.tracer = tracer;
        }
 
+       /**
+        * The {@link Cache} to use for all access (can be NULL).
+        * 
+        * @return the cache
+        */
+       public Cache getCache() {
+               return cache;
+       }
+
+       /**
+        * The {@link Cache} to use for all access (can be NULL).
+        * 
+        * @param cache
+        *            the new cache
+        */
+       public void setCache(Cache cache) {
+               this.cache = cache;
+       }
+
        /**
         * Clear all the cookies currently in the jar.
         * <p>
@@ -197,7 +215,13 @@ public class Downloader {
         * @param url
         *            the {@link URL} to open
         * @param originalUrl
-        *            the original {@link URL} before any redirection occurs
+        *            the original {@link URL} before any redirection occurs, which
+        *            is also used for the cache ID if needed (so we can retrieve
+        *            the content with this URL if needed)
+        * @param currentReferer
+        *            the current referer, for websites that needs this info
+        * @param cookiesValues
+        *            the cookies
         * @param postParams
         *            the POST parameters
         * @param getParams
@@ -214,17 +238,18 @@ public class Downloader {
         * @throws IOException
         *             in case of I/O error
         */
-       private InputStream open(URL url, final URL originalUrl,
-                       URL currentReferer, Map<String, String> cookiesValues,
-                       Map<String, String> postParams, Map<String, String> getParams,
-                       String oauth, boolean stable) throws IOException {
+       public InputStream open(URL url, final URL originalUrl, URL currentReferer,
+                       Map<String, String> cookiesValues, Map<String, String> postParams,
+                       Map<String, String> getParams, String oauth, boolean stable)
+                       throws IOException {
 
                tracer.trace("Request: " + url);
 
                if (cache != null) {
-                       InputStream in = cache.load(url, false, stable);
+                       InputStream in = cache.load(originalUrl, false, stable);
                        if (in != null) {
-                               tracer.trace("Take from cache: " + url);
+                               tracer.trace("Use the cache: " + url);
+                               tracer.trace("Original URL : " + originalUrl);
                                return in;
                        }
                }
@@ -240,9 +265,9 @@ public class Downloader {
                        params = postParams;
                }
 
+               StringBuilder requestData = null;
                if ((params != null || oauth != null)
                                && conn instanceof HttpURLConnection) {
-                       StringBuilder requestData = null;
                        if (params != null) {
                                requestData = new StringBuilder();
                                for (Map.Entry<String, String> param : params.entrySet()) {
@@ -255,14 +280,14 @@ public class Downloader {
                                                        String.valueOf(param.getValue()), "UTF-8"));
                                }
 
-                               conn.setDoOutput(true);
-
                                if (getParams == null && postParams != null) {
                                        ((HttpURLConnection) conn).setRequestMethod("POST");
                                }
 
                                conn.setRequestProperty("Content-Type",
                                                "application/x-www-form-urlencoded");
+                               conn.setRequestProperty("Content-Length",
+                                               Integer.toString(requestData.length()));
                                conn.setRequestProperty("charset", "utf-8");
                        }
 
@@ -271,22 +296,27 @@ public class Downloader {
                        }
 
                        if (requestData != null) {
-                               OutputStreamWriter writer = null;
+                               conn.setDoOutput(true);
+                               OutputStreamWriter writer = new OutputStreamWriter(
+                                               conn.getOutputStream());
                                try {
-                                       writer = new OutputStreamWriter(conn.getOutputStream());
                                        writer.write(requestData.toString());
                                        writer.flush();
                                } finally {
-                                       if (writer != null) {
-                                               writer.close();
-                                       }
+                                       writer.close();
                                }
                        }
                }
 
+               // Manual redirection, much better for POST data
+               if (conn instanceof HttpURLConnection) {
+                       ((HttpURLConnection) conn).setInstanceFollowRedirects(false);
+               }
+
                conn.connect();
 
                // Check if redirect
+               // BEWARE! POST data cannot be redirected, so it is ignored here
                if (conn instanceof HttpURLConnection) {
                        int repCode = 0;
                        try {
@@ -298,7 +328,7 @@ public class Downloader {
                        if (repCode / 100 == 3) {
                                String newUrl = conn.getHeaderField("Location");
                                return open(new URL(newUrl), originalUrl, currentReferer,
-                                               cookiesValues, postParams, getParams, oauth, stable);
+                                               cookiesValues, null, getParams, oauth, stable);
                        }
                }
 
@@ -308,9 +338,14 @@ public class Downloader {
                }
 
                if (in != null && cache != null) {
-                       tracer.trace("Save to cache: " + url);
+                       tracer.trace("Save to cache: " + originalUrl);
                        try {
-                               cache.save(in, url);
+                               try {
+                                       cache.save(in, originalUrl);
+                               } finally {
+                                       in.close();
+                               }
+                               in = cache.load(originalUrl, true, false);
                        } catch (IOException e) {
                                tracer.error(new IOException(
                                                "Cannot save URL to cache, will ignore cache: " + url,
@@ -338,9 +373,15 @@ public class Downloader {
                        throws IOException {
                URLConnection conn = url.openConnection();
 
+               String cookies = generateCookies(cookiesValues);
+               if (cookies != null && !cookies.isEmpty()) {
+                       conn.setRequestProperty("Cookie", cookies);
+               }
+
                conn.setRequestProperty("User-Agent", UA);
-               conn.setRequestProperty("Cookie", generateCookies(cookiesValues));
                conn.setRequestProperty("Accept-Encoding", "gzip");
+               conn.setRequestProperty("Accept", "*/*");
+
                if (currentReferer != null) {
                        conn.setRequestProperty("Referer", currentReferer.toString());
                        conn.setRequestProperty("Host", currentReferer.getHost());
@@ -362,7 +403,6 @@ public class Downloader {
                                builder.append(';');
                        }
 
-                       // TODO: check if format is ok
                        builder.append(cookie.toString());
                }