Merge commit '6d465e886ae46c9da99117cc4302948f700a51c4'
[fanfix.git] / Downloader.java
1 package be.nikiroo.utils;
2
3 import java.io.IOException;
4 import java.io.InputStream;
5 import java.io.OutputStreamWriter;
6 import java.net.CookieHandler;
7 import java.net.CookieManager;
8 import java.net.CookiePolicy;
9 import java.net.CookieStore;
10 import java.net.HttpCookie;
11 import java.net.HttpURLConnection;
12 import java.net.URL;
13 import java.net.URLConnection;
14 import java.net.URLEncoder;
15 import java.util.Map;
16 import java.util.zip.GZIPInputStream;
17
18 /**
19 * This class will help you download content from Internet Sites ({@link URL}
20 * based).
21 * <p>
22 * It allows you to control some options often required on web sites that do not
23 * want to simply serve HTML, but actively makes your life difficult with stupid
24 * checks.
25 *
26 * @author niki
27 */
28 public class Downloader {
29 private String UA;
30 private CookieManager cookies;
31 private TraceHandler tracer = new TraceHandler();
32 private Cache cache;
33 private boolean offline;
34
35 /**
36 * Create a new {@link Downloader}.
37 *
38 * @param UA
39 * the User-Agent to use to download the resources -- note that
40 * some websites require one, some actively blacklist real UAs
41 * like the one from wget, some whitelist a couple of browsers
42 * only (!) -- can be NULL
43 */
44 public Downloader(String UA) {
45 this(UA, null);
46 }
47
48 /**
49 * Create a new {@link Downloader}.
50 *
51 * @param UA
52 * the User-Agent to use to download the resources -- note that
53 * some websites require one, some actively blacklist real UAs
54 * like the one from wget, some whitelist a couple of browsers
55 * only (!) -- can be NULL
56 * @param cache
57 * the {@link Cache} to use for all access (can be NULL)
58 */
59 public Downloader(String UA, Cache cache) {
60 this.UA = UA;
61
62 cookies = new CookieManager(null, CookiePolicy.ACCEPT_ALL);
63 CookieHandler.setDefault(cookies);
64
65 setCache(cache);
66 }
67
68 /**
69 * This {@link Downloader} is forbidden to try and connect to the network.
70 * <p>
71 * If TRUE, it will only check the cache if any.
72 * <p>
73 * Default is FALSE.
74 *
75 * @return TRUE if offline
76 */
77 public boolean isOffline() {
78 return offline;
79 }
80
81 /**
82 * This {@link Downloader} is forbidden to try and connect to the network.
83 * <p>
84 * If TRUE, it will only check the cache if any.
85 * <p>
86 * Default is FALSE.
87 *
88 * @param offline TRUE for offline, FALSE for online
89 */
90 public void setOffline(boolean offline) {
91 this.offline = offline;
92 }
93
94 /**
95 * The traces handler for this {@link Cache}.
96 *
97 * @return the traces handler
98 */
99 public TraceHandler getTraceHandler() {
100 return tracer;
101 }
102
103 /**
104 * The traces handler for this {@link Cache}.
105 *
106 * @param tracer
107 * the new traces handler
108 */
109 public void setTraceHandler(TraceHandler tracer) {
110 if (tracer == null) {
111 tracer = new TraceHandler(false, false, false);
112 }
113
114 this.tracer = tracer;
115 }
116
117 /**
118 * The {@link Cache} to use for all access (can be NULL).
119 *
120 * @return the cache
121 */
122 public Cache getCache() {
123 return cache;
124 }
125
126 /**
127 * The {@link Cache} to use for all access (can be NULL).
128 *
129 * @param cache
130 * the new cache
131 */
132 public void setCache(Cache cache) {
133 this.cache = cache;
134 }
135
136 /**
137 * Clear all the cookies currently in the jar.
138 * <p>
139 * As long as you don't, the cookies are kept.
140 */
141 public void clearCookies() {
142 cookies.getCookieStore().removeAll();
143 }
144
145 /**
146 * Open the given {@link URL} and update the cookies.
147 *
148 * @param url
149 * the {@link URL} to open
150 * @return the {@link InputStream} of the opened page
151 *
152 * @throws IOException
153 * in case of I/O error
154 **/
155 public InputStream open(URL url) throws IOException {
156 return open(url, false);
157 }
158
159 /**
160 * Open the given {@link URL} and update the cookies.
161 *
162 * @param url
163 * the {@link URL} to open
164 * @param stable
165 * stable a stable file (that doesn't change too often) --
166 * parameter used to check if the file is too old to keep or not
167 * in the cache (default is false)
168 *
169 * @return the {@link InputStream} of the opened page
170 *
171 * @throws IOException
172 * in case of I/O error
173 **/
174 public InputStream open(URL url, boolean stable) throws IOException {
175 return open(url, url, url, null, null, null, null, stable);
176 }
177
178 /**
179 * Open the given {@link URL} and update the cookies.
180 *
181 * @param url
182 * the {@link URL} to open
183 * @param currentReferer
184 * the current referer, for websites that needs this info
185 * @param cookiesValues
186 * the cookies
187 * @param postParams
188 * the POST parameters
189 * @param getParams
190 * the GET parameters (priority over POST)
191 * @param oauth
192 * OAuth authorization (aka, "bearer XXXXXXX")
193 *
194 * @return the {@link InputStream} of the opened page
195 *
196 * @throws IOException
197 * in case of I/O error (including offline mode + not in cache)
198 */
199 public InputStream open(URL url, URL currentReferer,
200 Map<String, String> cookiesValues, Map<String, String> postParams,
201 Map<String, String> getParams, String oauth) throws IOException {
202 return open(url, currentReferer, cookiesValues, postParams, getParams,
203 oauth, false);
204 }
205
206 /**
207 * Open the given {@link URL} and update the cookies.
208 *
209 * @param url
210 * the {@link URL} to open
211 * @param currentReferer
212 * the current referer, for websites that needs this info
213 * @param cookiesValues
214 * the cookies
215 * @param postParams
216 * the POST parameters
217 * @param getParams
218 * the GET parameters (priority over POST)
219 * @param oauth
220 * OAuth authorization (aka, "bearer XXXXXXX")
221 * @param stable
222 * stable a stable file (that doesn't change too often) --
223 * parameter used to check if the file is too old to keep or not
224 * in the cache (default is false)
225 *
226 * @return the {@link InputStream} of the opened page
227 *
228 * @throws IOException
229 * in case of I/O error (including offline mode + not in cache)
230 */
231 public InputStream open(URL url, URL currentReferer,
232 Map<String, String> cookiesValues, Map<String, String> postParams,
233 Map<String, String> getParams, String oauth, boolean stable)
234 throws IOException {
235 return open(url, url, currentReferer, cookiesValues, postParams,
236 getParams, oauth, stable);
237 }
238
239 /**
240 * Open the given {@link URL} and update the cookies.
241 *
242 * @param url
243 * the {@link URL} to open
244 * @param originalUrl
245 * the original {@link URL} before any redirection occurs, which
246 * is also used for the cache ID if needed (so we can retrieve
247 * the content with this URL if needed)
248 * @param currentReferer
249 * the current referer, for websites that needs this info
250 * @param cookiesValues
251 * the cookies
252 * @param postParams
253 * the POST parameters
254 * @param getParams
255 * the GET parameters (priority over POST)
256 * @param oauth
257 * OAuth authorisation (aka, "bearer XXXXXXX")
258 * @param stable
259 * a stable file (that doesn't change too often) -- parameter
260 * used to check if the file is too old to keep or not in the
261 * cache
262 *
263 * @return the {@link InputStream} of the opened page
264 *
265 * @throws IOException
266 * in case of I/O error (including offline mode + not in cache)
267 */
268 public InputStream open(URL url, final URL originalUrl, URL currentReferer,
269 Map<String, String> cookiesValues, Map<String, String> postParams,
270 Map<String, String> getParams, String oauth, boolean stable)
271 throws IOException {
272
273 tracer.trace("Request: " + url);
274
275 if (cache != null) {
276 InputStream in = cache.load(originalUrl, false, stable);
277 if (in != null) {
278 tracer.trace("Use the cache: " + url);
279 tracer.trace("Original URL : " + originalUrl);
280 return in;
281 }
282 }
283
284 String protocol = originalUrl == null ? null : originalUrl
285 .getProtocol();
286 if (isOffline() && !"file".equalsIgnoreCase(protocol)) {
287 tracer.error("Downloader OFFLINE, cannot proceed to URL: " + url);
288 throw new IOException("Downloader is currently OFFLINE, cannot download: " + url);
289 }
290
291 tracer.trace("Download: " + url);
292
293 URLConnection conn = openConnectionWithCookies(url, currentReferer,
294 cookiesValues);
295
296 // Priority: GET over POST
297 Map<String, String> params = getParams;
298 if (getParams == null) {
299 params = postParams;
300 }
301
302 StringBuilder requestData = null;
303 if ((params != null || oauth != null)
304 && conn instanceof HttpURLConnection) {
305 if (params != null) {
306 requestData = new StringBuilder();
307 for (Map.Entry<String, String> param : params.entrySet()) {
308 if (requestData.length() != 0)
309 requestData.append('&');
310 requestData.append(URLEncoder.encode(param.getKey(),
311 "UTF-8"));
312 requestData.append('=');
313 requestData.append(URLEncoder.encode(
314 String.valueOf(param.getValue()), "UTF-8"));
315 }
316
317 if (getParams == null && postParams != null) {
318 ((HttpURLConnection) conn).setRequestMethod("POST");
319 }
320
321 conn.setRequestProperty("Content-Type",
322 "application/x-www-form-urlencoded");
323 conn.setRequestProperty("Content-Length",
324 Integer.toString(requestData.length()));
325 }
326
327 if (oauth != null) {
328 conn.setRequestProperty("Authorization", oauth);
329 }
330
331 if (requestData != null) {
332 conn.setDoOutput(true);
333 OutputStreamWriter writer = new OutputStreamWriter(
334 conn.getOutputStream());
335 try {
336 writer.write(requestData.toString());
337 writer.flush();
338 } finally {
339 writer.close();
340 }
341 }
342 }
343
344 // Manual redirection, much better for POST data
345 if (conn instanceof HttpURLConnection) {
346 ((HttpURLConnection) conn).setInstanceFollowRedirects(false);
347 }
348
349 conn.connect();
350
351 // Check if redirect
352 // BEWARE! POST data cannot be redirected (some webservers complain) for
353 // HTTP codes 302 and 303
354 if (conn instanceof HttpURLConnection) {
355 int repCode = 0;
356 try {
357 // Can fail in some circumstances
358 repCode = ((HttpURLConnection) conn).getResponseCode();
359 } catch (IOException e) {
360 }
361
362 if (repCode / 100 == 3) {
363 String newUrl = conn.getHeaderField("Location");
364 return open(new URL(newUrl), originalUrl, currentReferer,
365 cookiesValues, //
366 (repCode == 302 || repCode == 303) ? null : postParams, //
367 getParams, oauth, stable);
368 }
369 }
370
371 try {
372 InputStream in = conn.getInputStream();
373 if ("gzip".equals(conn.getContentEncoding())) {
374 in = new GZIPInputStream(in);
375 }
376
377 if (in == null) {
378 throw new IOException("No InputStream!");
379 }
380
381 if (cache != null) {
382 String size = conn.getContentLength() < 0 ? "unknown size"
383 : StringUtils.formatNumber(conn.getContentLength())
384 + "bytes";
385 tracer.trace("Save to cache (" + size + "): " + originalUrl);
386 try {
387 try {
388 long bytes = cache.save(in, originalUrl);
389 tracer.trace("Saved to cache: "
390 + StringUtils.formatNumber(bytes) + "bytes");
391 } finally {
392 in.close();
393 }
394 in = cache.load(originalUrl, true, true);
395 } catch (IOException e) {
396 tracer.error(new IOException(
397 "Cannot save URL to cache, will ignore cache: "
398 + url, e));
399 }
400 }
401
402 if (in == null) {
403 throw new IOException(
404 "Cannot retrieve the file after storing it in the cache (??)");
405 }
406
407 return in;
408 } catch (IOException e) {
409 throw new IOException(String.format(
410 "Cannot find %s (current URL: %s)", originalUrl, url), e);
411 }
412 }
413
414 /**
415 * Open a connection on the given {@link URL}, and manage the cookies that
416 * come with it.
417 *
418 * @param url
419 * the {@link URL} to open
420 *
421 * @return the connection
422 *
423 * @throws IOException
424 * in case of I/O error
425 */
426 private URLConnection openConnectionWithCookies(URL url,
427 URL currentReferer, Map<String, String> cookiesValues)
428 throws IOException {
429 URLConnection conn = url.openConnection();
430
431 String cookies = generateCookies(cookiesValues);
432 if (cookies != null && !cookies.isEmpty()) {
433 conn.setRequestProperty("Cookie", cookies);
434 }
435
436 if (UA != null) {
437 conn.setRequestProperty("User-Agent", UA);
438 }
439 conn.setRequestProperty("Accept-Encoding", "gzip");
440 conn.setRequestProperty("Accept", "*/*");
441 conn.setRequestProperty("Charset", "utf-8");
442
443 if (currentReferer != null) {
444 conn.setRequestProperty("Referer", currentReferer.toString());
445 conn.setRequestProperty("Host", currentReferer.getHost());
446 }
447
448 return conn;
449 }
450
451 /**
452 * Generate the cookie {@link String} from the local {@link CookieStore} so
453 * it is ready to be passed.
454 *
455 * @return the cookie
456 */
457 private String generateCookies(Map<String, String> cookiesValues) {
458 StringBuilder builder = new StringBuilder();
459 for (HttpCookie cookie : cookies.getCookieStore().getCookies()) {
460 if (builder.length() > 0) {
461 builder.append(';');
462 }
463
464 builder.append(cookie.toString());
465 }
466
467 if (cookiesValues != null) {
468 for (Map.Entry<String, String> set : cookiesValues.entrySet()) {
469 if (builder.length() > 0) {
470 builder.append(';');
471 }
472 builder.append(set.getKey());
473 builder.append('=');
474 builder.append(set.getValue());
475 }
476 }
477
478 return builder.toString();
479 }
480 }