1 package be
.nikiroo
.fanfix
;
3 import java
.io
.BufferedOutputStream
;
5 import java
.io
.FileInputStream
;
6 import java
.io
.FileOutputStream
;
7 import java
.io
.IOException
;
8 import java
.io
.InputStream
;
9 import java
.net
.CookieHandler
;
10 import java
.net
.CookieManager
;
11 import java
.net
.CookiePolicy
;
12 import java
.net
.CookieStore
;
13 import java
.net
.HttpCookie
;
14 import java
.net
.HttpURLConnection
;
15 import java
.net
.URISyntaxException
;
17 import java
.net
.URLConnection
;
18 import java
.nio
.file
.FileAlreadyExistsException
;
19 import java
.util
.Date
;
21 import java
.util
.zip
.GZIPInputStream
;
23 import javax
.imageio
.ImageIO
;
25 import be
.nikiroo
.fanfix
.bundles
.Config
;
26 import be
.nikiroo
.fanfix
.supported
.BasicSupport
;
27 import be
.nikiroo
.utils
.IOUtils
;
28 import be
.nikiroo
.utils
.MarkableFileInputStream
;
29 import be
.nikiroo
.utils
.StringUtils
;
32 * This cache will manage Internet (and local) downloads, as well as put the
33 * downloaded files into a cache.
35 * As long the cached resource is not too old, it will use it instead of
36 * retrieving the file again.
43 private long tooOldChanging
;
44 private long tooOldStable
;
45 private CookieManager cookies
;
48 * Create a new {@link Cache} object.
51 * the directory to use as cache
53 * the User-Agent to use to download the resources
54 * @param hoursChanging
55 * the number of hours after which a cached file that is thought
56 * to change ~often is considered too old (or -1 for
59 * the number of hours after which a LARGE cached file that is
60 * thought to change rarely is considered too old (or -1 for
64 * in case of I/O error
66 public Cache(File dir
, String UA
, int hoursChanging
, int hoursStable
)
70 this.tooOldChanging
= 1000 * 60 * 60 * hoursChanging
;
71 this.tooOldStable
= 1000 * 60 * 60 * hoursStable
;
79 if (dir
== null || !dir
.exists()) {
80 throw new IOException("Cannot create the cache directory: "
81 + (dir
== null ?
"null" : dir
.getAbsolutePath()));
84 cookies
= new CookieManager();
85 cookies
.setCookiePolicy(CookiePolicy
.ACCEPT_ALL
);
86 CookieHandler
.setDefault(cookies
);
90 * Open a resource (will load it from the cache if possible, or save it into
91 * the cache after downloading if not).
94 * the resource to open
96 * the support to use to download the resource
98 * TRUE for more stable resources, FALSE when they often change
100 * @return the opened resource
102 * @throws IOException
103 * in case of I/O error
105 public InputStream
open(URL url
, BasicSupport support
, boolean stable
)
107 return open(url
, support
, stable
, url
);
111 * Open a resource (will load it from the cache if possible, or save it into
112 * the cache after downloading if not).
114 * The cached resource will be assimilated to the given original {@link URL}
117 * the resource to open
119 * the support to use to download the resource
121 * TRUE for more stable resources, FALSE when they often change
123 * the original {@link URL} used to locate the cached resource
125 * @return the opened resource
127 * @throws IOException
128 * in case of I/O error
130 public InputStream
open(URL url
, BasicSupport support
, boolean stable
,
131 URL originalUrl
) throws IOException
{
133 InputStream in
= load(originalUrl
, false, stable
);
136 save(url
, support
, originalUrl
);
137 } catch (IOException e
) {
138 throw new IOException("Cannot save the url: "
139 + (url
== null ?
"null" : url
.toString()), e
);
142 in
= load(originalUrl
, true, stable
);
146 } catch (IOException e
) {
147 throw new IOException("Cannot open the url: "
148 + (url
== null ?
"null" : url
.toString()), e
);
153 * Refresh the resource into cache if needed.
156 * the resource to open
158 * the support to use to download the resource
160 * TRUE for more stable resources, FALSE when they often change
162 * @return TRUE if it was pre-downloaded
164 * @throws IOException
165 * in case of I/O error
167 public void refresh(URL url
, BasicSupport support
, boolean stable
)
169 File cached
= getCached(url
);
170 if (cached
.exists() && !isOld(cached
, stable
)) {
174 open(url
, support
, stable
).close();
178 * Check the resource to see if it is in the cache.
181 * the resource to check
183 * @return TRUE if it is
186 public boolean check(URL url
) {
187 return getCached(url
).exists();
191 * Open a resource (will load it from the cache if possible, or save it into
192 * the cache after downloading if not) as an Image, then save it where
195 * This version will not always work properly if the original file was not
199 * the resource to open
201 * @return the opened resource image
203 * @throws IOException
204 * in case of I/O error
206 public void saveAsImage(URL url
, File target
) throws IOException
{
207 URL cachedUrl
= new URL(url
.toString()
209 + Instance
.getConfig().getString(Config
.IMAGE_FORMAT_CONTENT
)
211 File cached
= getCached(cachedUrl
);
213 if (!cached
.exists() || isOld(cached
, true)) {
214 InputStream imageIn
= Instance
.getCache().open(url
, null, true);
215 ImageIO
.write(StringUtils
.toImage(imageIn
), Instance
.getConfig()
216 .getString(Config
.IMAGE_FORMAT_CONTENT
).toLowerCase(),
220 IOUtils
.write(new FileInputStream(cached
), target
);
224 * Manually add this item to the cache.
229 * a unique ID for this resource
231 * @return the resulting {@link FileAlreadyExistsException}
233 * @throws IOException
234 * in case of I/O error
236 public File
addToCache(InputStream in
, String uniqueID
) throws IOException
{
237 File file
= getCached(new File(uniqueID
).toURI().toURL());
238 IOUtils
.write(in
, file
);
243 * Clean the cache (delete the cached items).
246 * only clean the files that are considered too old
248 * @return the number of cleaned items
250 public int cleanCache(boolean onlyOld
) {
252 for (File file
: dir
.listFiles()) {
253 if (!onlyOld
|| isOld(file
, true)) {
257 System
.err
.println("Cannot delete temporary file: "
258 + file
.getAbsolutePath());
266 * Open a resource from the cache if it exists.
269 * the resource to open
270 * @return the opened resource
271 * @throws IOException
272 * in case of I/O error
274 private InputStream
load(URL url
, boolean allowOld
, boolean stable
)
276 File cached
= getCached(url
);
277 if (cached
.exists() && !isOld(cached
, stable
)) {
278 return new MarkableFileInputStream(new FileInputStream(cached
));
285 * Save the given resource to the cache.
290 * the {@link BasicSupport} used to download it
292 * the original {@link URL} used to locate the cached resource
294 * @throws IOException
295 * in case of I/O error
296 * @throws URISyntaxException
298 private void save(URL url
, BasicSupport support
, URL originalUrl
)
300 URLConnection conn
= url
.openConnection();
302 conn
.setRequestProperty("User-Agent", UA
);
303 conn
.setRequestProperty("Cookie", generateCookies(support
));
304 conn
.setRequestProperty("Accept-Encoding", "gzip");
305 if (support
!= null && support
.getCurrentReferer() != null) {
306 conn
.setRequestProperty("Referer", support
.getCurrentReferer()
308 conn
.setRequestProperty("Host", support
.getCurrentReferer()
315 if (conn
instanceof HttpURLConnection
316 && ((HttpURLConnection
) conn
).getResponseCode() / 100 == 3) {
317 String newUrl
= conn
.getHeaderField("Location");
318 save(new URL(newUrl
), support
, originalUrl
);
322 InputStream in
= conn
.getInputStream();
323 if ("gzip".equals(conn
.getContentEncoding())) {
324 in
= new GZIPInputStream(in
);
328 File cached
= getCached(originalUrl
);
329 BufferedOutputStream out
= new BufferedOutputStream(
330 new FileOutputStream(cached
));
332 byte[] buf
= new byte[4096];
334 while ((len
= in
.read(buf
)) > 0) {
335 out
.write(buf
, 0, len
);
346 * Check if the {@link File} is too old according to
347 * {@link Cache#tooOldChanging}.
352 * TRUE to denote files that are not supposed to change too often
354 * @return TRUE if it is
356 private boolean isOld(File file
, boolean stable
) {
357 long max
= tooOldChanging
;
366 long time
= new Date().getTime() - file
.lastModified();
368 System
.err
.println("Timestamp in the future for file: "
369 + file
.getAbsolutePath());
372 return time
< 0 || time
> max
;
376 * Get the cache resource from the cache if it is present for this
381 * @return the cached version if present, NULL if not
383 private File
getCached(URL url
) {
384 String name
= url
.getHost();
385 if (name
== null || name
.length() == 0) {
386 name
= url
.getFile();
388 name
= url
.toString();
391 name
= name
.replace('/', '_').replace(':', '_');
393 return new File(dir
, name
);
397 * Generate the cookie {@link String} from the local {@link CookieStore} so
398 * it is ready to be passed.
402 private String
generateCookies(BasicSupport support
) {
403 StringBuilder builder
= new StringBuilder();
404 for (HttpCookie cookie
: cookies
.getCookieStore().getCookies()) {
405 if (builder
.length() > 0) {
409 // TODO: check if format is ok
410 builder
.append(cookie
.toString());
413 if (support
!= null) {
414 for (Map
.Entry
<String
, String
> set
: support
.getCookies()
416 if (builder
.length() > 0) {
419 builder
.append(set
.getKey());
421 builder
.append(set
.getValue());
425 return builder
.toString();