1 package be
.nikiroo
.fanfix
;
3 import java
.io
.BufferedOutputStream
;
5 import java
.io
.FileInputStream
;
6 import java
.io
.FileNotFoundException
;
7 import java
.io
.FileOutputStream
;
8 import java
.io
.IOException
;
9 import java
.io
.InputStream
;
10 import java
.io
.OutputStreamWriter
;
11 import java
.net
.CookieHandler
;
12 import java
.net
.CookieManager
;
13 import java
.net
.CookiePolicy
;
14 import java
.net
.CookieStore
;
15 import java
.net
.HttpCookie
;
16 import java
.net
.HttpURLConnection
;
18 import java
.net
.URLConnection
;
19 import java
.net
.URLEncoder
;
20 import java
.util
.Date
;
22 import java
.util
.zip
.GZIPInputStream
;
24 import javax
.imageio
.ImageIO
;
26 import be
.nikiroo
.fanfix
.bundles
.Config
;
27 import be
.nikiroo
.fanfix
.supported
.BasicSupport
;
28 import be
.nikiroo
.utils
.IOUtils
;
29 import be
.nikiroo
.utils
.ImageUtils
;
30 import be
.nikiroo
.utils
.MarkableFileInputStream
;
33 * This cache will manage Internet (and local) downloads, as well as put the
34 * downloaded files into a cache.
36 * As long the cached resource is not too old, it will use it instead of
37 * retrieving the file again.
44 private long tooOldChanging
;
45 private long tooOldStable
;
46 private CookieManager cookies
;
49 * Create a new {@link Cache} object.
52 * the directory to use as cache
54 * the User-Agent to use to download the resources
55 * @param hoursChanging
56 * the number of hours after which a cached file that is thought
57 * to change ~often is considered too old (or -1 for
60 * the number of hours after which a LARGE cached file that is
61 * thought to change rarely is considered too old (or -1 for
65 * in case of I/O error
67 public Cache(File dir
, String UA
, int hoursChanging
, int hoursStable
)
71 this.tooOldChanging
= 1000 * 60 * 60 * hoursChanging
;
72 this.tooOldStable
= 1000 * 60 * 60 * hoursStable
;
80 if (dir
== null || !dir
.exists()) {
81 throw new IOException("Cannot create the cache directory: "
82 + (dir
== null ?
"null" : dir
.getAbsolutePath()));
85 cookies
= new CookieManager();
86 cookies
.setCookiePolicy(CookiePolicy
.ACCEPT_ALL
);
87 CookieHandler
.setDefault(cookies
);
91 * Clear all the cookies currently in the jar.
93 public void clearCookies() {
94 cookies
.getCookieStore().removeAll();
98 * Open a resource (will load it from the cache if possible, or save it into
99 * the cache after downloading if not).
102 * the resource to open
104 * the support to use to download the resource
106 * TRUE for more stable resources, FALSE when they often change
108 * @return the opened resource, NOT NULL
110 * @throws IOException
111 * in case of I/O error
113 public InputStream
open(URL url
, BasicSupport support
, boolean stable
)
115 // MUST NOT return null
116 return open(url
, support
, stable
, url
);
120 * Open a resource (will load it from the cache if possible, or save it into
121 * the cache after downloading if not).
123 * The cached resource will be assimilated to the given original {@link URL}
126 * the resource to open
128 * the support to use to download the resource
130 * TRUE for more stable resources, FALSE when they often change
132 * the original {@link URL} used to locate the cached resource
134 * @return the opened resource, NOT NULL
136 * @throws IOException
137 * in case of I/O error
139 public InputStream
open(URL url
, BasicSupport support
, boolean stable
,
140 URL originalUrl
) throws IOException
{
141 // MUST NOT return null
143 InputStream in
= load(originalUrl
, false, stable
);
144 if (Instance
.isDebug()) {
145 System
.err
.println("Cache " + (in
!= null ?
"hit" : "miss")
152 save(url
, support
, originalUrl
);
153 } catch (IOException e
) {
154 throw new IOException("Cannot save the url: "
155 + (url
== null ?
"null" : url
.toString()), e
);
158 // Was just saved, can load old, so, will not be null
159 in
= load(originalUrl
, true, stable
);
163 } catch (IOException e
) {
164 throw new IOException("Cannot open the url: "
165 + (url
== null ?
"null" : url
.toString()), e
);
170 * Open the given {@link URL} without using the cache, but still using and
171 * updating the cookies.
174 * the {@link URL} to open
176 * the {@link BasicSupport} used for the cookies
178 * @return the {@link InputStream} of the opened page
180 * @throws IOException
181 * in case of I/O error
183 public InputStream
openNoCache(URL url
, BasicSupport support
)
185 return openNoCache(url
, support
, url
, null, null, null);
189 * Open the given {@link URL} without using the cache, but still using and
190 * updating the cookies.
193 * the {@link URL} to open
195 * the {@link BasicSupport} used for the cookies
197 * the POST parameters
199 * the GET parameters (priority over POST)
201 * OAuth authorization (aka, "bearer XXXXXXX")
203 * @return the {@link InputStream} of the opened page
205 * @throws IOException
206 * in case of I/O error
208 public InputStream
openNoCache(URL url
, BasicSupport support
,
209 Map
<String
, String
> postParams
, Map
<String
, String
> getParams
,
210 String oauth
) throws IOException
{
211 return openNoCache(url
, support
, url
, postParams
, getParams
, oauth
);
215 * Open the given {@link URL} without using the cache, but still using and
216 * updating the cookies.
219 * the {@link URL} to open
221 * the {@link BasicSupport} used for the cookies
223 * the original {@link URL} before any redirection occurs
225 * the POST parameters
227 * the GET parameters (priority over POST)
229 * OAuth authorization (aka, "bearer XXXXXXX")
230 * @return the {@link InputStream} of the opened page
232 * @throws IOException
233 * in case of I/O error
235 private InputStream
openNoCache(URL url
, BasicSupport support
,
236 final URL originalUrl
, Map
<String
, String
> postParams
,
237 Map
<String
, String
> getParams
, String oauth
) throws IOException
{
239 if (Instance
.isDebug()) {
240 System
.err
.println("Open no cache: " + url
);
243 URLConnection conn
= openConnectionWithCookies(url
, support
);
244 if (support
!= null) {
245 // priority: arguments
247 oauth
= support
.getOAuth();
251 // Priority: GET over POST
252 Map
<String
, String
> params
= getParams
;
253 if (getParams
== null) {
257 if ((params
!= null || oauth
!= null)
258 && conn
instanceof HttpURLConnection
) {
259 StringBuilder requestData
= null;
260 if (params
!= null) {
261 requestData
= new StringBuilder();
262 for (Map
.Entry
<String
, String
> param
: params
.entrySet()) {
263 if (requestData
.length() != 0)
264 requestData
.append('&');
265 requestData
.append(URLEncoder
.encode(param
.getKey(),
267 requestData
.append('=');
268 requestData
.append(URLEncoder
.encode(
269 String
.valueOf(param
.getValue()), "UTF-8"));
272 conn
.setDoOutput(true);
274 if (getParams
== null && postParams
!= null) {
275 ((HttpURLConnection
) conn
).setRequestMethod("POST");
278 conn
.setRequestProperty("Content-Type",
279 "application/x-www-form-urlencoded");
280 conn
.setRequestProperty("charset", "utf-8");
284 conn
.setRequestProperty("Authorization", oauth
);
287 if (requestData
!= null) {
288 OutputStreamWriter writer
= new OutputStreamWriter(
289 conn
.getOutputStream());
291 writer
.write(requestData
.toString());
300 if (conn
instanceof HttpURLConnection
301 && ((HttpURLConnection
) conn
).getResponseCode() / 100 == 3) {
302 String newUrl
= conn
.getHeaderField("Location");
303 return openNoCache(new URL(newUrl
), support
, originalUrl
,
304 postParams
, getParams
, oauth
);
307 InputStream in
= conn
.getInputStream();
308 if ("gzip".equals(conn
.getContentEncoding())) {
309 in
= new GZIPInputStream(in
);
316 * Refresh the resource into cache if needed.
319 * the resource to open
321 * the support to use to download the resource
323 * TRUE for more stable resources, FALSE when they often change
325 * @throws IOException
326 * in case of I/O error
328 public void refresh(URL url
, BasicSupport support
, boolean stable
)
330 File cached
= getCached(url
);
331 if (cached
.exists() && !isOld(cached
, stable
)) {
335 open(url
, support
, stable
).close();
339 * Check the resource to see if it is in the cache.
342 * the resource to check
344 * @return TRUE if it is
347 public boolean check(URL url
) {
348 return getCached(url
).exists();
352 * Save the given resource as an image on disk using the default image
353 * format for content.
360 * @throws IOException
361 * in case of I/O error
363 public void saveAsImage(URL url
, File target
) throws IOException
{
364 URL cachedUrl
= new URL(url
.toString());
365 File cached
= getCached(cachedUrl
);
367 if (!cached
.exists() || isOld(cached
, true)) {
368 InputStream imageIn
= open(url
, null, true);
369 ImageIO
.write(ImageUtils
.fromStream(imageIn
), Instance
.getConfig()
370 .getString(Config
.IMAGE_FORMAT_CONTENT
).toLowerCase(),
374 IOUtils
.write(new FileInputStream(cached
), target
);
378 * Manually add this item to the cache.
383 * a unique ID for this resource
385 * @return the resulting {@link File}
387 * @throws IOException
388 * in case of I/O error
390 public File
addToCache(InputStream in
, String uniqueID
) throws IOException
{
391 File file
= getCached(uniqueID
);
392 File subdir
= new File(file
.getParentFile(), "_");
393 file
= new File(subdir
, file
.getName());
395 IOUtils
.write(in
, file
);
400 * Return the {@link InputStream} corresponding to the given unique ID, or
401 * NULL if none found.
406 * @return the content or NULL
408 public InputStream
getFromCache(String uniqueID
) {
409 File file
= getCached(uniqueID
);
410 File subdir
= new File(file
.getParentFile(), "_");
411 file
= new File(subdir
, file
.getName());
414 return new MarkableFileInputStream(new FileInputStream(file
));
415 } catch (FileNotFoundException e
) {
423 * Clean the cache (delete the cached items).
426 * only clean the files that are considered too old
428 * @return the number of cleaned items
430 public int cleanCache(boolean onlyOld
) {
431 return cleanCache(onlyOld
, dir
);
435 * Clean the cache (delete the cached items) in the given cache directory.
438 * only clean the files that are considered too old
440 * the cache directory to clean
442 * @return the number of cleaned items
444 private int cleanCache(boolean onlyOld
, File cacheDir
) {
446 for (File file
: cacheDir
.listFiles()) {
447 if (file
.isDirectory()) {
448 num
+= cleanCache(onlyOld
, file
);
450 if (!onlyOld
|| isOld(file
, true)) {
454 System
.err
.println("Cannot delete temporary file: "
455 + file
.getAbsolutePath());
465 * Open a resource from the cache if it exists.
468 * the resource to open
470 * allow files even if they are considered too old
472 * a stable file (that dones't change too often) -- parameter
473 * used to check if the file is too old to keep or not
475 * @return the opened resource if found, NULL i not
477 * @throws IOException
478 * in case of I/O error
480 private InputStream
load(URL url
, boolean allowTooOld
, boolean stable
)
482 File cached
= getCached(url
);
483 if (cached
.exists() && (allowTooOld
|| !isOld(cached
, stable
))) {
484 return new MarkableFileInputStream(new FileInputStream(cached
));
491 * Save the given resource to the cache.
496 * the {@link BasicSupport} used to download it
498 * the original {@link URL} used to locate the cached resource
500 * @throws IOException
501 * in case of I/O error
503 private void save(URL url
, BasicSupport support
, URL originalUrl
)
505 InputStream in
= openNoCache(url
, support
, originalUrl
, null, null,
508 File cached
= getCached(originalUrl
);
509 BufferedOutputStream out
= new BufferedOutputStream(
510 new FileOutputStream(cached
));
512 byte[] buf
= new byte[4096];
514 while ((len
= in
.read(buf
)) > 0) {
515 out
.write(buf
, 0, len
);
526 * Open a connection on the given {@link URL}, and manage the cookies that
530 * the {@link URL} to open
532 * the {@link BasicSupport} to use for cookie generation
534 * @return the connection
536 * @throws IOException
537 * in case of I/O error
539 private URLConnection
openConnectionWithCookies(URL url
,
540 BasicSupport support
) throws IOException
{
541 URLConnection conn
= url
.openConnection();
543 conn
.setRequestProperty("User-Agent", UA
);
544 conn
.setRequestProperty("Cookie", generateCookies(support
));
545 conn
.setRequestProperty("Accept-Encoding", "gzip");
546 if (support
!= null && support
.getCurrentReferer() != null) {
547 conn
.setRequestProperty("Referer", support
.getCurrentReferer()
549 conn
.setRequestProperty("Host", support
.getCurrentReferer()
557 * Check if the {@link File} is too old according to
558 * {@link Cache#tooOldChanging}.
563 * TRUE to denote files that are not supposed to change too often
565 * @return TRUE if it is
567 private boolean isOld(File file
, boolean stable
) {
568 long max
= tooOldChanging
;
577 long time
= new Date().getTime() - file
.lastModified();
579 System
.err
.println("Timestamp in the future for file: "
580 + file
.getAbsolutePath());
583 return time
< 0 || time
> max
;
587 * Return the associated cache {@link File} from this {@link URL}.
592 * @return the cached {@link File} version of this {@link URL}
594 private File
getCached(URL url
) {
597 String name
= url
.getHost();
598 if (name
== null || name
.isEmpty()) {
599 name
= url
.getFile();
601 File cacheDir
= getCached(".").getParentFile();
602 File subsubDir
= new File(cacheDir
, allowedChars(url
.getHost()));
603 subdir
= new File(subsubDir
, "_" + allowedChars(url
.getPath()));
604 name
= allowedChars("_" + url
.getQuery());
607 File cacheFile
= getCached(name
);
608 if (subdir
!= null) {
609 cacheFile
= new File(subdir
, cacheFile
.getName());
617 * Get the basic cache resource file corresponding to this unique ID.
619 * Note that you may need to add a sub-directory in some cases.
624 * @return the cached version if present, NULL if not
626 private File
getCached(String uniqueID
) {
627 return new File(dir
, allowedChars(uniqueID
));
631 * Replace not allowed chars (in a {@link File}) by "_".
634 * the raw {@link String}
636 * @return the sanitised {@link String}
638 private String
allowedChars(String raw
) {
639 return raw
.replace('/', '_').replace(':', '_').replace("\\", "_");
643 * Generate the cookie {@link String} from the local {@link CookieStore} so
644 * it is ready to be passed.
648 private String
generateCookies(BasicSupport support
) {
649 StringBuilder builder
= new StringBuilder();
650 for (HttpCookie cookie
: cookies
.getCookieStore().getCookies()) {
651 if (builder
.length() > 0) {
655 // TODO: check if format is ok
656 builder
.append(cookie
.toString());
659 if (support
!= null) {
660 for (Map
.Entry
<String
, String
> set
: support
.getCookies()
662 if (builder
.length() > 0) {
665 builder
.append(set
.getKey());
667 builder
.append(set
.getValue());
671 return builder
.toString();