fix changelog headers
[fanfix.git] / utils / Downloader.java
1 package be.nikiroo.utils;
2
3 import java.io.IOException;
4 import java.io.InputStream;
5 import java.io.OutputStreamWriter;
6 import java.net.CookieHandler;
7 import java.net.CookieManager;
8 import java.net.CookiePolicy;
9 import java.net.CookieStore;
10 import java.net.HttpCookie;
11 import java.net.HttpURLConnection;
12 import java.net.URL;
13 import java.net.URLConnection;
14 import java.net.URLEncoder;
15 import java.util.Map;
16 import java.util.zip.GZIPInputStream;
17
18 /**
19 * This class will help you download content from Internet Sites ({@link URL}
20 * based).
21 * <p>
22 * It allows you to control some options often required on web sites that do not
23 * want to simply serve HTML, but actively makes your life difficult with stupid
24 * checks.
25 *
26 * @author niki
27 */
28 public class Downloader {
29 private String UA;
30 private CookieManager cookies;
31 private TraceHandler tracer = new TraceHandler();
32 private Cache cache;
33
34 /**
35 * Create a new {@link Downloader}.
36 *
37 * @param UA
38 * the User-Agent to use to download the resources -- note that
39 * some websites require one, some actively blacklist real UAs
40 * like the one from wget, some whitelist a couple of browsers
41 * only (!)
42 */
43 public Downloader(String UA) {
44 this(UA, null);
45 }
46
47 /**
48 * Create a new {@link Downloader}.
49 *
50 * @param UA
51 * the User-Agent to use to download the resources -- note that
52 * some websites require one, some actively blacklist real UAs
53 * like the one from wget, some whitelist a couple of browsers
54 * only (!)
55 * @param cache
56 * the {@link Cache} to use for all access (can be NULL)
57 */
58 public Downloader(String UA, Cache cache) {
59 this.UA = UA;
60
61 cookies = new CookieManager(null, CookiePolicy.ACCEPT_ALL);
62 CookieHandler.setDefault(cookies);
63
64 setCache(cache);
65 }
66
67 /**
68 * The traces handler for this {@link Cache}.
69 *
70 * @return the traces handler
71 */
72 public TraceHandler getTraceHandler() {
73 return tracer;
74 }
75
76 /**
77 * The traces handler for this {@link Cache}.
78 *
79 * @param tracer
80 * the new traces handler
81 */
82 public void setTraceHandler(TraceHandler tracer) {
83 if (tracer == null) {
84 tracer = new TraceHandler(false, false, false);
85 }
86
87 this.tracer = tracer;
88 }
89
90 /**
91 * The {@link Cache} to use for all access (can be NULL).
92 *
93 * @return the cache
94 */
95 public Cache getCache() {
96 return cache;
97 }
98
99 /**
100 * The {@link Cache} to use for all access (can be NULL).
101 *
102 * @param cache
103 * the new cache
104 */
105 public void setCache(Cache cache) {
106 this.cache = cache;
107 }
108
109 /**
110 * Clear all the cookies currently in the jar.
111 * <p>
112 * As long as you don't, the cookies are kept.
113 */
114 public void clearCookies() {
115 cookies.getCookieStore().removeAll();
116 }
117
118 /**
119 * Open the given {@link URL} and update the cookies.
120 *
121 * @param url
122 * the {@link URL} to open
123 * @return the {@link InputStream} of the opened page
124 *
125 * @throws IOException
126 * in case of I/O error
127 **/
128 public InputStream open(URL url) throws IOException {
129 return open(url, false);
130 }
131
132 /**
133 * Open the given {@link URL} and update the cookies.
134 *
135 * @param url
136 * the {@link URL} to open
137 * @param stable
138 * stable a stable file (that doesn't change too often) --
139 * parameter used to check if the file is too old to keep or not
140 * in the cache (default is false)
141 *
142 * @return the {@link InputStream} of the opened page
143 *
144 * @throws IOException
145 * in case of I/O error
146 **/
147 public InputStream open(URL url, boolean stable) throws IOException {
148 return open(url, url, url, null, null, null, null, stable);
149 }
150
151 /**
152 * Open the given {@link URL} and update the cookies.
153 *
154 * @param url
155 * the {@link URL} to open
156 * @param currentReferer
157 * the current referer, for websites that needs this info
158 * @param cookiesValues
159 * the cookies
160 * @param postParams
161 * the POST parameters
162 * @param getParams
163 * the GET parameters (priority over POST)
164 * @param oauth
165 * OAuth authorization (aka, "bearer XXXXXXX")
166 *
167 * @return the {@link InputStream} of the opened page
168 *
169 * @throws IOException
170 * in case of I/O error
171 */
172 public InputStream open(URL url, URL currentReferer,
173 Map<String, String> cookiesValues, Map<String, String> postParams,
174 Map<String, String> getParams, String oauth) throws IOException {
175 return open(url, currentReferer, cookiesValues, postParams, getParams,
176 oauth, false);
177 }
178
179 /**
180 * Open the given {@link URL} and update the cookies.
181 *
182 * @param url
183 * the {@link URL} to open
184 * @param currentReferer
185 * the current referer, for websites that needs this info
186 * @param cookiesValues
187 * the cookies
188 * @param postParams
189 * the POST parameters
190 * @param getParams
191 * the GET parameters (priority over POST)
192 * @param oauth
193 * OAuth authorization (aka, "bearer XXXXXXX")
194 * @param stable
195 * stable a stable file (that doesn't change too often) --
196 * parameter used to check if the file is too old to keep or not
197 * in the cache (default is false)
198 *
199 * @return the {@link InputStream} of the opened page
200 *
201 * @throws IOException
202 * in case of I/O error
203 */
204 public InputStream open(URL url, URL currentReferer,
205 Map<String, String> cookiesValues, Map<String, String> postParams,
206 Map<String, String> getParams, String oauth, boolean stable)
207 throws IOException {
208 return open(url, url, currentReferer, cookiesValues, postParams,
209 getParams, oauth, stable);
210 }
211
212 /**
213 * Open the given {@link URL} and update the cookies.
214 *
215 * @param url
216 * the {@link URL} to open
217 * @param originalUrl
218 * the original {@link URL} before any redirection occurs, which
219 * is also used for the cache ID if needed (so we can retrieve
220 * the content with this URL if needed)
221 * @param currentReferer
222 * the current referer, for websites that needs this info
223 * @param cookiesValues
224 * the cookies
225 * @param postParams
226 * the POST parameters
227 * @param getParams
228 * the GET parameters (priority over POST)
229 * @param oauth
230 * OAuth authorisation (aka, "bearer XXXXXXX")
231 * @param stable
232 * a stable file (that doesn't change too often) -- parameter
233 * used to check if the file is too old to keep or not in the
234 * cache
235 *
236 * @return the {@link InputStream} of the opened page
237 *
238 * @throws IOException
239 * in case of I/O error
240 */
241 public InputStream open(URL url, final URL originalUrl, URL currentReferer,
242 Map<String, String> cookiesValues, Map<String, String> postParams,
243 Map<String, String> getParams, String oauth, boolean stable)
244 throws IOException {
245
246 tracer.trace("Request: " + url);
247
248 if (cache != null) {
249 InputStream in = cache.load(originalUrl, false, stable);
250 if (in != null) {
251 tracer.trace("Use the cache: " + url);
252 tracer.trace("Original URL : " + originalUrl);
253 return in;
254 }
255 }
256
257 tracer.trace("Download: " + url);
258
259 URLConnection conn = openConnectionWithCookies(url, currentReferer,
260 cookiesValues);
261
262 // Priority: GET over POST
263 Map<String, String> params = getParams;
264 if (getParams == null) {
265 params = postParams;
266 }
267
268 StringBuilder requestData = null;
269 if ((params != null || oauth != null)
270 && conn instanceof HttpURLConnection) {
271 if (params != null) {
272 requestData = new StringBuilder();
273 for (Map.Entry<String, String> param : params.entrySet()) {
274 if (requestData.length() != 0)
275 requestData.append('&');
276 requestData.append(URLEncoder.encode(param.getKey(),
277 "UTF-8"));
278 requestData.append('=');
279 requestData.append(URLEncoder.encode(
280 String.valueOf(param.getValue()), "UTF-8"));
281 }
282
283 if (getParams == null && postParams != null) {
284 ((HttpURLConnection) conn).setRequestMethod("POST");
285 }
286
287 conn.setRequestProperty("Content-Type",
288 "application/x-www-form-urlencoded");
289 conn.setRequestProperty("Content-Length",
290 Integer.toString(requestData.length()));
291 }
292
293 if (oauth != null) {
294 conn.setRequestProperty("Authorization", oauth);
295 }
296
297 if (requestData != null) {
298 conn.setDoOutput(true);
299 OutputStreamWriter writer = new OutputStreamWriter(
300 conn.getOutputStream());
301 try {
302 writer.write(requestData.toString());
303 writer.flush();
304 } finally {
305 writer.close();
306 }
307 }
308 }
309
310 // Manual redirection, much better for POST data
311 if (conn instanceof HttpURLConnection) {
312 ((HttpURLConnection) conn).setInstanceFollowRedirects(false);
313 }
314
315 conn.connect();
316
317 // Check if redirect
318 // BEWARE! POST data cannot be redirected (some webservers complain) for
319 // HTTP codes 302 and 303
320 if (conn instanceof HttpURLConnection) {
321 int repCode = 0;
322 try {
323 // Can fail in some circumstances
324 repCode = ((HttpURLConnection) conn).getResponseCode();
325 } catch (IOException e) {
326 }
327
328 if (repCode / 100 == 3) {
329 String newUrl = conn.getHeaderField("Location");
330 return open(new URL(newUrl), originalUrl, currentReferer,
331 cookiesValues, //
332 (repCode == 302 || repCode == 303) ? null : postParams, //
333 getParams, oauth, stable);
334 }
335 }
336
337 try {
338 InputStream in = conn.getInputStream();
339 if ("gzip".equals(conn.getContentEncoding())) {
340 in = new GZIPInputStream(in);
341 }
342
343 if (in == null) {
344 throw new IOException("No InputStream!");
345 }
346
347 if (cache != null) {
348 String size = conn.getContentLength() < 0 ? "unknown size"
349 : StringUtils.formatNumber(conn.getContentLength())
350 + "bytes";
351 tracer.trace("Save to cache (" + size + "): " + originalUrl);
352 try {
353 try {
354 long bytes = cache.save(in, originalUrl);
355 tracer.trace("Saved to cache: "
356 + StringUtils.formatNumber(bytes) + "bytes");
357 } finally {
358 in.close();
359 }
360 in = cache.load(originalUrl, true, true);
361 } catch (IOException e) {
362 tracer.error(new IOException(
363 "Cannot save URL to cache, will ignore cache: "
364 + url, e));
365 }
366 }
367
368 return in;
369 } catch (IOException e) {
370 throw new IOException(String.format(
371 "Cannot find %s (current URL: %s)", originalUrl, url), e);
372 }
373 }
374
375 /**
376 * Open a connection on the given {@link URL}, and manage the cookies that
377 * come with it.
378 *
379 * @param url
380 * the {@link URL} to open
381 *
382 * @return the connection
383 *
384 * @throws IOException
385 * in case of I/O error
386 */
387 private URLConnection openConnectionWithCookies(URL url,
388 URL currentReferer, Map<String, String> cookiesValues)
389 throws IOException {
390 URLConnection conn = url.openConnection();
391
392 String cookies = generateCookies(cookiesValues);
393 if (cookies != null && !cookies.isEmpty()) {
394 conn.setRequestProperty("Cookie", cookies);
395 }
396
397 conn.setRequestProperty("User-Agent", UA);
398 conn.setRequestProperty("Accept-Encoding", "gzip");
399 conn.setRequestProperty("Accept", "*/*");
400 conn.setRequestProperty("Charset", "utf-8");
401
402 if (currentReferer != null) {
403 conn.setRequestProperty("Referer", currentReferer.toString());
404 conn.setRequestProperty("Host", currentReferer.getHost());
405 }
406
407 return conn;
408 }
409
410 /**
411 * Generate the cookie {@link String} from the local {@link CookieStore} so
412 * it is ready to be passed.
413 *
414 * @return the cookie
415 */
416 private String generateCookies(Map<String, String> cookiesValues) {
417 StringBuilder builder = new StringBuilder();
418 for (HttpCookie cookie : cookies.getCookieStore().getCookies()) {
419 if (builder.length() > 0) {
420 builder.append(';');
421 }
422
423 builder.append(cookie.toString());
424 }
425
426 if (cookiesValues != null) {
427 for (Map.Entry<String, String> set : cookiesValues.entrySet()) {
428 if (builder.length() > 0) {
429 builder.append(';');
430 }
431 builder.append(set.getKey());
432 builder.append('=');
433 builder.append(set.getValue());
434 }
435 }
436
437 return builder.toString();
438 }
439 }