public Downloader(String UA, Cache cache) {
this.UA = UA;
- cookies = new CookieManager();
- cookies.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
+ cookies = new CookieManager(null, CookiePolicy.ACCEPT_ALL);
CookieHandler.setDefault(cookies);
- this.cache = cache;
+ setCache(cache);
}
/**
this.tracer = tracer;
}
+ /**
+ * The {@link Cache} to use for all access (can be NULL).
+ *
+ * @return the cache
+ */
+ public Cache getCache() {
+ return cache;
+ }
+
+ /**
+ * The {@link Cache} to use for all access (can be NULL).
+ *
+ * @param cache
+ * the new cache
+ */
+ public void setCache(Cache cache) {
+ this.cache = cache;
+ }
+
/**
* Clear all the cookies currently in the jar.
* <p>
* @param url
* the {@link URL} to open
* @param originalUrl
- * the original {@link URL} before any redirection occurs
+ * the original {@link URL} before any redirection occurs, which
+ * is also used for the cache ID if needed (so we can retrieve
+ * the content with this URL if needed)
+ * @param currentReferer
+ * the current referer, for websites that needs this info
+ * @param cookiesValues
+ * the cookies
* @param postParams
* the POST parameters
* @param getParams
* @throws IOException
* in case of I/O error
*/
- private InputStream open(URL url, final URL originalUrl,
- URL currentReferer, Map<String, String> cookiesValues,
- Map<String, String> postParams, Map<String, String> getParams,
- String oauth, boolean stable) throws IOException {
+ public InputStream open(URL url, final URL originalUrl, URL currentReferer,
+ Map<String, String> cookiesValues, Map<String, String> postParams,
+ Map<String, String> getParams, String oauth, boolean stable)
+ throws IOException {
tracer.trace("Request: " + url);
if (cache != null) {
- InputStream in = cache.load(url, false, stable);
+ InputStream in = cache.load(originalUrl, false, stable);
if (in != null) {
- tracer.trace("Take from cache: " + url);
+ tracer.trace("Use the cache: " + url);
+ tracer.trace("Original URL : " + originalUrl);
return in;
}
}
params = postParams;
}
+ StringBuilder requestData = null;
if ((params != null || oauth != null)
&& conn instanceof HttpURLConnection) {
- StringBuilder requestData = null;
if (params != null) {
requestData = new StringBuilder();
for (Map.Entry<String, String> param : params.entrySet()) {
String.valueOf(param.getValue()), "UTF-8"));
}
- conn.setDoOutput(true);
-
if (getParams == null && postParams != null) {
((HttpURLConnection) conn).setRequestMethod("POST");
}
conn.setRequestProperty("Content-Type",
"application/x-www-form-urlencoded");
+ conn.setRequestProperty("Content-Length",
+ Integer.toString(requestData.length()));
conn.setRequestProperty("charset", "utf-8");
}
}
if (requestData != null) {
- OutputStreamWriter writer = null;
+ conn.setDoOutput(true);
+ OutputStreamWriter writer = new OutputStreamWriter(
+ conn.getOutputStream());
try {
- writer = new OutputStreamWriter(conn.getOutputStream());
writer.write(requestData.toString());
writer.flush();
} finally {
- if (writer != null) {
- writer.close();
- }
+ writer.close();
}
}
}
+ // Manual redirection, much better for POST data
+ if (conn instanceof HttpURLConnection) {
+ ((HttpURLConnection) conn).setInstanceFollowRedirects(false);
+ }
+
conn.connect();
// Check if redirect
+ // BEWARE! POST data cannot be redirected, so it is ignored here
if (conn instanceof HttpURLConnection) {
int repCode = 0;
try {
if (repCode / 100 == 3) {
String newUrl = conn.getHeaderField("Location");
return open(new URL(newUrl), originalUrl, currentReferer,
- cookiesValues, postParams, getParams, oauth, stable);
+ cookiesValues, null, getParams, oauth, stable);
}
}
}
if (in != null && cache != null) {
- tracer.trace("Save to cache: " + url);
+ tracer.trace("Save to cache: " + originalUrl);
try {
- cache.save(in, url);
+ try {
+ cache.save(in, originalUrl);
+ } finally {
+ in.close();
+ }
+ in = cache.load(originalUrl, true, false);
} catch (IOException e) {
tracer.error(new IOException(
"Cannot save URL to cache, will ignore cache: " + url,
throws IOException {
URLConnection conn = url.openConnection();
+ String cookies = generateCookies(cookiesValues);
+ if (cookies != null && !cookies.isEmpty()) {
+ conn.setRequestProperty("Cookie", cookies);
+ }
+
conn.setRequestProperty("User-Agent", UA);
- conn.setRequestProperty("Cookie", generateCookies(cookiesValues));
conn.setRequestProperty("Accept-Encoding", "gzip");
+ conn.setRequestProperty("Accept", "*/*");
+
if (currentReferer != null) {
conn.setRequestProperty("Referer", currentReferer.toString());
conn.setRequestProperty("Host", currentReferer.getHost());
builder.append(';');
}
- // TODO: check if format is ok
builder.append(cookie.toString());
}