1 package be
.nikiroo
.fanfix
;
3 import java
.io
.BufferedOutputStream
;
5 import java
.io
.FileInputStream
;
6 import java
.io
.FileNotFoundException
;
7 import java
.io
.FileOutputStream
;
8 import java
.io
.IOException
;
9 import java
.io
.InputStream
;
10 import java
.io
.OutputStreamWriter
;
11 import java
.net
.CookieHandler
;
12 import java
.net
.CookieManager
;
13 import java
.net
.CookiePolicy
;
14 import java
.net
.CookieStore
;
15 import java
.net
.HttpCookie
;
16 import java
.net
.HttpURLConnection
;
17 import java
.net
.URISyntaxException
;
19 import java
.net
.URLConnection
;
20 import java
.net
.URLEncoder
;
21 import java
.util
.Date
;
23 import java
.util
.zip
.GZIPInputStream
;
25 import javax
.imageio
.ImageIO
;
27 import be
.nikiroo
.fanfix
.bundles
.Config
;
28 import be
.nikiroo
.fanfix
.supported
.BasicSupport
;
29 import be
.nikiroo
.utils
.IOUtils
;
30 import be
.nikiroo
.utils
.MarkableFileInputStream
;
33 * This cache will manage Internet (and local) downloads, as well as put the
34 * downloaded files into a cache.
36 * As long the cached resource is not too old, it will use it instead of
37 * retrieving the file again.
44 private long tooOldChanging
;
45 private long tooOldStable
;
46 private CookieManager cookies
;
49 * Create a new {@link Cache} object.
52 * the directory to use as cache
54 * the User-Agent to use to download the resources
55 * @param hoursChanging
56 * the number of hours after which a cached file that is thought
57 * to change ~often is considered too old (or -1 for
60 * the number of hours after which a LARGE cached file that is
61 * thought to change rarely is considered too old (or -1 for
65 * in case of I/O error
67 public Cache(File dir
, String UA
, int hoursChanging
, int hoursStable
)
71 this.tooOldChanging
= 1000 * 60 * 60 * hoursChanging
;
72 this.tooOldStable
= 1000 * 60 * 60 * hoursStable
;
80 if (dir
== null || !dir
.exists()) {
81 throw new IOException("Cannot create the cache directory: "
82 + (dir
== null ?
"null" : dir
.getAbsolutePath()));
85 cookies
= new CookieManager();
86 cookies
.setCookiePolicy(CookiePolicy
.ACCEPT_ALL
);
87 CookieHandler
.setDefault(cookies
);
91 * Clear all the cookies currently in the jar.
93 public void clearCookies() {
94 cookies
.getCookieStore().removeAll();
98 * Open a resource (will load it from the cache if possible, or save it into
99 * the cache after downloading if not).
102 * the resource to open
104 * the support to use to download the resource
106 * TRUE for more stable resources, FALSE when they often change
108 * @return the opened resource, NOT NULL
110 * @throws IOException
111 * in case of I/O error
113 public InputStream
open(URL url
, BasicSupport support
, boolean stable
)
115 // MUST NOT return null
116 return open(url
, support
, stable
, url
);
120 * Open a resource (will load it from the cache if possible, or save it into
121 * the cache after downloading if not).
123 * The cached resource will be assimilated to the given original {@link URL}
126 * the resource to open
128 * the support to use to download the resource
130 * TRUE for more stable resources, FALSE when they often change
132 * the original {@link URL} used to locate the cached resource
134 * @return the opened resource, NOT NULL
136 * @throws IOException
137 * in case of I/O error
139 public InputStream
open(URL url
, BasicSupport support
, boolean stable
,
140 URL originalUrl
) throws IOException
{
141 // MUST NOT return null
143 InputStream in
= load(originalUrl
, false, stable
);
146 save(url
, support
, originalUrl
);
147 } catch (IOException e
) {
148 throw new IOException("Cannot save the url: "
149 + (url
== null ?
"null" : url
.toString()), e
);
152 // Was just saved, can load old, so, will not be null
153 in
= load(originalUrl
, true, stable
);
157 } catch (IOException e
) {
158 throw new IOException("Cannot open the url: "
159 + (url
== null ?
"null" : url
.toString()), e
);
164 * Open the given {@link URL} without using the cache, but still using and
165 * updating the cookies.
168 * the {@link URL} to open
170 * the {@link BasicSupport} used for the cookies
172 * @return the {@link InputStream} of the opened page
174 * @throws IOException
175 * in case of I/O error
177 public InputStream
openNoCache(URL url
, BasicSupport support
)
179 return openNoCache(url
, support
, url
, null);
183 * Open the given {@link URL} without using the cache, but still using and
184 * updating the cookies.
187 * the {@link URL} to open
189 * the {@link BasicSupport} used for the cookies
191 * the POST parameters
193 * @return the {@link InputStream} of the opened page
195 * @throws IOException
196 * in case of I/O error
198 public InputStream
openNoCache(URL url
, BasicSupport support
,
199 Map
<String
, String
> postParams
) throws IOException
{
200 return openNoCache(url
, support
, url
, postParams
);
204 * Open the given {@link URL} without using the cache, but still using and
205 * updating the cookies.
208 * the {@link URL} to open
210 * the {@link BasicSupport} used for the cookies
212 * the original {@link URL} before any redirection occurs
214 * the POST parameters
216 * @return the {@link InputStream} of the opened page
218 * @throws IOException
219 * in case of I/O error
221 private InputStream
openNoCache(URL url
, BasicSupport support
,
222 final URL originalUrl
, Map
<String
, String
> postParams
)
225 URLConnection conn
= openConnectionWithCookies(url
, support
);
226 if (postParams
!= null && conn
instanceof HttpURLConnection
) {
227 StringBuilder postData
= new StringBuilder();
228 for (Map
.Entry
<String
, String
> param
: postParams
.entrySet()) {
229 if (postData
.length() != 0)
230 postData
.append('&');
231 postData
.append(URLEncoder
.encode(param
.getKey(), "UTF-8"));
232 postData
.append('=');
233 postData
.append(URLEncoder
.encode(
234 String
.valueOf(param
.getValue()), "UTF-8"));
237 conn
.setDoOutput(true);
238 ((HttpURLConnection
) conn
).setRequestMethod("POST");
239 conn
.setRequestProperty("Content-Type",
240 "application/x-www-form-urlencoded");
241 conn
.setRequestProperty("charset", "utf-8");
243 OutputStreamWriter writer
= new OutputStreamWriter(
244 conn
.getOutputStream());
246 writer
.write(postData
.toString());
254 if (conn
instanceof HttpURLConnection
255 && ((HttpURLConnection
) conn
).getResponseCode() / 100 == 3) {
256 String newUrl
= conn
.getHeaderField("Location");
257 return openNoCache(new URL(newUrl
), support
, originalUrl
,
261 InputStream in
= conn
.getInputStream();
262 if ("gzip".equals(conn
.getContentEncoding())) {
263 in
= new GZIPInputStream(in
);
270 * Refresh the resource into cache if needed.
273 * the resource to open
275 * the support to use to download the resource
277 * TRUE for more stable resources, FALSE when they often change
279 * @throws IOException
280 * in case of I/O error
282 public void refresh(URL url
, BasicSupport support
, boolean stable
)
284 File cached
= getCached(url
);
285 if (cached
.exists() && !isOld(cached
, stable
)) {
289 open(url
, support
, stable
).close();
293 * Check the resource to see if it is in the cache.
296 * the resource to check
298 * @return TRUE if it is
301 public boolean check(URL url
) {
302 return getCached(url
).exists();
306 * Open a resource (will load it from the cache if possible, or save it into
307 * the cache after downloading if not) as an Image, then save it where
310 * This version will not always work properly if the original file was not
314 * the resource to open
316 * @return the opened resource image
318 * @throws IOException
319 * in case of I/O error
321 public void saveAsImage(URL url
, File target
) throws IOException
{
322 URL cachedUrl
= new URL(url
.toString());
323 File cached
= getCached(cachedUrl
);
325 if (!cached
.exists() || isOld(cached
, true)) {
326 InputStream imageIn
= open(url
, null, true);
327 ImageIO
.write(IOUtils
.toImage(imageIn
), Instance
.getConfig()
328 .getString(Config
.IMAGE_FORMAT_CONTENT
).toLowerCase(),
332 IOUtils
.write(new FileInputStream(cached
), target
);
336 * Manually add this item to the cache.
341 * a unique ID for this resource
343 * @return the resulting {@link File}
345 * @throws IOException
346 * in case of I/O error
348 public File
addToCache(InputStream in
, String uniqueID
) throws IOException
{
349 File file
= getCached(uniqueID
);
350 IOUtils
.write(in
, file
);
355 * Return the {@link InputStream} corresponding to the given unique ID, or
356 * NULL if none found.
361 * @return the content or NULL
363 public InputStream
getFromCache(String uniqueID
) {
364 File file
= getCached(uniqueID
);
367 return new MarkableFileInputStream(new FileInputStream(file
));
368 } catch (FileNotFoundException e
) {
376 * Clean the cache (delete the cached items).
379 * only clean the files that are considered too old
381 * @return the number of cleaned items
383 public int cleanCache(boolean onlyOld
) {
385 for (File file
: dir
.listFiles()) {
386 if (!onlyOld
|| isOld(file
, true)) {
390 System
.err
.println("Cannot delete temporary file: "
391 + file
.getAbsolutePath());
399 * Open a resource from the cache if it exists.
402 * the resource to open
404 * @return the opened resource if found, NULL i not
406 * @throws IOException
407 * in case of I/O error
409 private InputStream
load(URL url
, boolean allowOld
, boolean stable
)
411 File cached
= getCached(url
);
412 if (cached
.exists() && !isOld(cached
, stable
)) {
413 return new MarkableFileInputStream(new FileInputStream(cached
));
420 * Save the given resource to the cache.
425 * the {@link BasicSupport} used to download it
427 * the original {@link URL} used to locate the cached resource
429 * @throws IOException
430 * in case of I/O error
431 * @throws URISyntaxException
433 private void save(URL url
, BasicSupport support
, URL originalUrl
)
435 InputStream in
= openNoCache(url
, support
, originalUrl
, null);
437 File cached
= getCached(originalUrl
);
438 BufferedOutputStream out
= new BufferedOutputStream(
439 new FileOutputStream(cached
));
441 byte[] buf
= new byte[4096];
443 while ((len
= in
.read(buf
)) > 0) {
444 out
.write(buf
, 0, len
);
455 * Open a connection on the given {@link URL}, and manage the cookies that
459 * the {@link URL} to open
461 * the {@link BasicSupport} to use for cookie generation
463 * @return the connection
465 * @throws IOException
466 * in case of I/O error
468 private URLConnection
openConnectionWithCookies(URL url
,
469 BasicSupport support
) throws IOException
{
470 URLConnection conn
= url
.openConnection();
472 conn
.setRequestProperty("User-Agent", UA
);
473 conn
.setRequestProperty("Cookie", generateCookies(support
));
474 conn
.setRequestProperty("Accept-Encoding", "gzip");
475 if (support
!= null && support
.getCurrentReferer() != null) {
476 conn
.setRequestProperty("Referer", support
.getCurrentReferer()
478 conn
.setRequestProperty("Host", support
.getCurrentReferer()
486 * Check if the {@link File} is too old according to
487 * {@link Cache#tooOldChanging}.
492 * TRUE to denote files that are not supposed to change too often
494 * @return TRUE if it is
496 private boolean isOld(File file
, boolean stable
) {
497 long max
= tooOldChanging
;
506 long time
= new Date().getTime() - file
.lastModified();
508 System
.err
.println("Timestamp in the future for file: "
509 + file
.getAbsolutePath());
512 return time
< 0 || time
> max
;
516 * Get the cache resource from the cache if it is present for this
521 * @return the cached version if present, NULL if not
523 private File
getCached(URL url
) {
524 String name
= url
.getHost();
525 if (name
== null || name
.isEmpty()) {
526 name
= url
.getFile();
528 name
= url
.toString();
531 return getCached(name
);
535 * Get the cache resource from the cache if it is present for this unique
540 * @return the cached version if present, NULL if not
542 private File
getCached(String uniqueID
) {
543 uniqueID
= uniqueID
.replace('/', '_').replace(':', '_')
546 return new File(dir
, uniqueID
);
550 * Generate the cookie {@link String} from the local {@link CookieStore} so
551 * it is ready to be passed.
555 private String
generateCookies(BasicSupport support
) {
556 StringBuilder builder
= new StringBuilder();
557 for (HttpCookie cookie
: cookies
.getCookieStore().getCookies()) {
558 if (builder
.length() > 0) {
562 // TODO: check if format is ok
563 builder
.append(cookie
.toString());
566 if (support
!= null) {
568 for (Map
.Entry
<String
, String
> set
: support
.getCookies()
570 if (builder
.length() > 0) {
573 builder
.append(set
.getKey());
575 builder
.append(set
.getValue());
577 } catch (IOException e
) {
582 return builder
.toString();