1 package be
.nikiroo
.fanfix
;
3 import java
.io
.BufferedOutputStream
;
5 import java
.io
.FileInputStream
;
6 import java
.io
.FileOutputStream
;
7 import java
.io
.IOException
;
8 import java
.io
.InputStream
;
9 import java
.io
.OutputStreamWriter
;
10 import java
.net
.CookieHandler
;
11 import java
.net
.CookieManager
;
12 import java
.net
.CookiePolicy
;
13 import java
.net
.CookieStore
;
14 import java
.net
.HttpCookie
;
15 import java
.net
.HttpURLConnection
;
16 import java
.net
.URISyntaxException
;
18 import java
.net
.URLConnection
;
19 import java
.net
.URLEncoder
;
20 import java
.util
.Date
;
22 import java
.util
.zip
.GZIPInputStream
;
24 import javax
.imageio
.ImageIO
;
26 import be
.nikiroo
.fanfix
.bundles
.Config
;
27 import be
.nikiroo
.fanfix
.supported
.BasicSupport
;
28 import be
.nikiroo
.utils
.IOUtils
;
29 import be
.nikiroo
.utils
.MarkableFileInputStream
;
32 * This cache will manage Internet (and local) downloads, as well as put the
33 * downloaded files into a cache.
35 * As long the cached resource is not too old, it will use it instead of
36 * retrieving the file again.
43 private long tooOldChanging
;
44 private long tooOldStable
;
45 private CookieManager cookies
;
48 * Create a new {@link Cache} object.
51 * the directory to use as cache
53 * the User-Agent to use to download the resources
54 * @param hoursChanging
55 * the number of hours after which a cached file that is thought
56 * to change ~often is considered too old (or -1 for
59 * the number of hours after which a LARGE cached file that is
60 * thought to change rarely is considered too old (or -1 for
64 * in case of I/O error
66 public Cache(File dir
, String UA
, int hoursChanging
, int hoursStable
)
70 this.tooOldChanging
= 1000 * 60 * 60 * hoursChanging
;
71 this.tooOldStable
= 1000 * 60 * 60 * hoursStable
;
79 if (dir
== null || !dir
.exists()) {
80 throw new IOException("Cannot create the cache directory: "
81 + (dir
== null ?
"null" : dir
.getAbsolutePath()));
84 cookies
= new CookieManager();
85 cookies
.setCookiePolicy(CookiePolicy
.ACCEPT_ALL
);
86 CookieHandler
.setDefault(cookies
);
90 * Clear all the cookies currently in the jar.
92 public void clearCookies() {
93 cookies
.getCookieStore().removeAll();
97 * Open a resource (will load it from the cache if possible, or save it into
98 * the cache after downloading if not).
101 * the resource to open
103 * the support to use to download the resource
105 * TRUE for more stable resources, FALSE when they often change
107 * @return the opened resource, NOT NULL
109 * @throws IOException
110 * in case of I/O error
112 public InputStream
open(URL url
, BasicSupport support
, boolean stable
)
114 // MUST NOT return null
115 return open(url
, support
, stable
, url
);
119 * Open a resource (will load it from the cache if possible, or save it into
120 * the cache after downloading if not).
122 * The cached resource will be assimilated to the given original {@link URL}
125 * the resource to open
127 * the support to use to download the resource
129 * TRUE for more stable resources, FALSE when they often change
131 * the original {@link URL} used to locate the cached resource
133 * @return the opened resource, NOT NULL
135 * @throws IOException
136 * in case of I/O error
138 public InputStream
open(URL url
, BasicSupport support
, boolean stable
,
139 URL originalUrl
) throws IOException
{
140 // MUST NOT return null
142 InputStream in
= load(originalUrl
, false, stable
);
145 save(url
, support
, originalUrl
);
146 } catch (IOException e
) {
147 throw new IOException("Cannot save the url: "
148 + (url
== null ?
"null" : url
.toString()), e
);
151 // Was just saved, can load old, so, will not be null
152 in
= load(originalUrl
, true, stable
);
156 } catch (IOException e
) {
157 throw new IOException("Cannot open the url: "
158 + (url
== null ?
"null" : url
.toString()), e
);
163 * Open the given {@link URL} without using the cache, but still using and
164 * updating the cookies.
167 * the {@link URL} to open
169 * the {@link BasicSupport} used for the cookies
171 * @return the {@link InputStream} of the opened page
173 * @throws IOException
174 * in case of I/O error
176 public InputStream
openNoCache(URL url
, BasicSupport support
)
178 return openNoCache(url
, support
, url
, null);
182 * Open the given {@link URL} without using the cache, but still using and
183 * updating the cookies.
186 * the {@link URL} to open
188 * the {@link BasicSupport} used for the cookies
190 * the POST parameters
192 * @return the {@link InputStream} of the opened page
194 * @throws IOException
195 * in case of I/O error
197 public InputStream
openNoCache(URL url
, BasicSupport support
,
198 Map
<String
, String
> postParams
) throws IOException
{
199 return openNoCache(url
, support
, url
, postParams
);
203 * Open the given {@link URL} without using the cache, but still using and
204 * updating the cookies.
207 * the {@link URL} to open
209 * the {@link BasicSupport} used for the cookies
211 * the original {@link URL} before any redirection occurs
213 * the POST parameters
215 * @return the {@link InputStream} of the opened page
217 * @throws IOException
218 * in case of I/O error
220 private InputStream
openNoCache(URL url
, BasicSupport support
,
221 final URL originalUrl
, Map
<String
, String
> postParams
)
224 URLConnection conn
= openConnectionWithCookies(url
, support
);
225 if (postParams
!= null) {
226 StringBuilder postData
= new StringBuilder();
227 for (Map
.Entry
<String
, String
> param
: postParams
.entrySet()) {
228 if (postData
.length() != 0)
229 postData
.append('&');
230 postData
.append(URLEncoder
.encode(param
.getKey(), "UTF-8"));
231 postData
.append('=');
232 postData
.append(URLEncoder
.encode(
233 String
.valueOf(param
.getValue()), "UTF-8"));
236 conn
.setDoOutput(true);
238 OutputStreamWriter writer
= new OutputStreamWriter(
239 conn
.getOutputStream());
241 writer
.write(postData
.toString());
249 if (conn
instanceof HttpURLConnection
250 && ((HttpURLConnection
) conn
).getResponseCode() / 100 == 3) {
251 String newUrl
= conn
.getHeaderField("Location");
252 return openNoCache(new URL(newUrl
), support
, originalUrl
,
256 InputStream in
= conn
.getInputStream();
257 if ("gzip".equals(conn
.getContentEncoding())) {
258 in
= new GZIPInputStream(in
);
265 * Refresh the resource into cache if needed.
268 * the resource to open
270 * the support to use to download the resource
272 * TRUE for more stable resources, FALSE when they often change
274 * @throws IOException
275 * in case of I/O error
277 public void refresh(URL url
, BasicSupport support
, boolean stable
)
279 File cached
= getCached(url
);
280 if (cached
.exists() && !isOld(cached
, stable
)) {
284 open(url
, support
, stable
).close();
288 * Check the resource to see if it is in the cache.
291 * the resource to check
293 * @return TRUE if it is
296 public boolean check(URL url
) {
297 return getCached(url
).exists();
301 * Open a resource (will load it from the cache if possible, or save it into
302 * the cache after downloading if not) as an Image, then save it where
305 * This version will not always work properly if the original file was not
309 * the resource to open
311 * @return the opened resource image
313 * @throws IOException
314 * in case of I/O error
316 public void saveAsImage(URL url
, File target
) throws IOException
{
317 URL cachedUrl
= new URL(url
.toString());
318 File cached
= getCached(cachedUrl
);
320 if (!cached
.exists() || isOld(cached
, true)) {
321 InputStream imageIn
= open(url
, null, true);
322 ImageIO
.write(IOUtils
.toImage(imageIn
), Instance
.getConfig()
323 .getString(Config
.IMAGE_FORMAT_CONTENT
).toLowerCase(),
327 IOUtils
.write(new FileInputStream(cached
), target
);
331 * Manually add this item to the cache.
336 * a unique ID for this resource
338 * @return the resulting {@link File}
340 * @throws IOException
341 * in case of I/O error
343 public File
addToCache(InputStream in
, String uniqueID
) throws IOException
{
344 File file
= getCached(new File(uniqueID
).toURI().toURL());
345 IOUtils
.write(in
, file
);
350 * Clean the cache (delete the cached items).
353 * only clean the files that are considered too old
355 * @return the number of cleaned items
357 public int cleanCache(boolean onlyOld
) {
359 for (File file
: dir
.listFiles()) {
360 if (!onlyOld
|| isOld(file
, true)) {
364 System
.err
.println("Cannot delete temporary file: "
365 + file
.getAbsolutePath());
373 * Open a resource from the cache if it exists.
376 * the resource to open
378 * @return the opened resource if found, NULL i not
380 * @throws IOException
381 * in case of I/O error
383 private InputStream
load(URL url
, boolean allowOld
, boolean stable
)
385 File cached
= getCached(url
);
386 if (cached
.exists() && !isOld(cached
, stable
)) {
387 return new MarkableFileInputStream(new FileInputStream(cached
));
394 * Save the given resource to the cache.
399 * the {@link BasicSupport} used to download it
401 * the original {@link URL} used to locate the cached resource
403 * @throws IOException
404 * in case of I/O error
405 * @throws URISyntaxException
407 private void save(URL url
, BasicSupport support
, URL originalUrl
)
409 InputStream in
= openNoCache(url
, support
, originalUrl
, null);
411 File cached
= getCached(originalUrl
);
412 BufferedOutputStream out
= new BufferedOutputStream(
413 new FileOutputStream(cached
));
415 byte[] buf
= new byte[4096];
417 while ((len
= in
.read(buf
)) > 0) {
418 out
.write(buf
, 0, len
);
429 * Open a connection on the given {@link URL}, and manage the cookies that
433 * the {@link URL} to open
435 * the {@link BasicSupport} to use for cookie generation
437 * @return the connection
439 * @throws IOException
440 * in case of I/O error
442 private URLConnection
openConnectionWithCookies(URL url
,
443 BasicSupport support
) throws IOException
{
444 URLConnection conn
= url
.openConnection();
446 conn
.setRequestProperty("User-Agent", UA
);
447 conn
.setRequestProperty("Cookie", generateCookies(support
));
448 conn
.setRequestProperty("Accept-Encoding", "gzip");
449 if (support
!= null && support
.getCurrentReferer() != null) {
450 conn
.setRequestProperty("Referer", support
.getCurrentReferer()
452 conn
.setRequestProperty("Host", support
.getCurrentReferer()
460 * Check if the {@link File} is too old according to
461 * {@link Cache#tooOldChanging}.
466 * TRUE to denote files that are not supposed to change too often
468 * @return TRUE if it is
470 private boolean isOld(File file
, boolean stable
) {
471 long max
= tooOldChanging
;
480 long time
= new Date().getTime() - file
.lastModified();
482 System
.err
.println("Timestamp in the future for file: "
483 + file
.getAbsolutePath());
486 return time
< 0 || time
> max
;
490 * Get the cache resource from the cache if it is present for this
495 * @return the cached version if present, NULL if not
497 private File
getCached(URL url
) {
498 String name
= url
.getHost();
499 if (name
== null || name
.length() == 0) {
500 name
= url
.getFile();
502 name
= url
.toString();
505 name
= name
.replace('/', '_').replace(':', '_');
507 return new File(dir
, name
);
511 * Generate the cookie {@link String} from the local {@link CookieStore} so
512 * it is ready to be passed.
516 private String
generateCookies(BasicSupport support
) {
517 StringBuilder builder
= new StringBuilder();
518 for (HttpCookie cookie
: cookies
.getCookieStore().getCookies()) {
519 if (builder
.length() > 0) {
523 // TODO: check if format is ok
524 builder
.append(cookie
.toString());
527 if (support
!= null) {
529 for (Map
.Entry
<String
, String
> set
: support
.getCookies()
531 if (builder
.length() > 0) {
534 builder
.append(set
.getKey());
536 builder
.append(set
.getValue());
538 } catch (IOException e
) {
543 return builder
.toString();