1 package be
.nikiroo
.fanfix
;
3 import java
.io
.BufferedOutputStream
;
5 import java
.io
.FileInputStream
;
6 import java
.io
.FileOutputStream
;
7 import java
.io
.IOException
;
8 import java
.io
.InputStream
;
9 import java
.net
.CookieHandler
;
10 import java
.net
.CookieManager
;
11 import java
.net
.CookiePolicy
;
12 import java
.net
.CookieStore
;
13 import java
.net
.HttpCookie
;
14 import java
.net
.HttpURLConnection
;
15 import java
.net
.URISyntaxException
;
17 import java
.net
.URLConnection
;
18 import java
.util
.Date
;
20 import java
.util
.zip
.GZIPInputStream
;
22 import javax
.imageio
.ImageIO
;
24 import be
.nikiroo
.fanfix
.bundles
.Config
;
25 import be
.nikiroo
.fanfix
.supported
.BasicSupport
;
26 import be
.nikiroo
.utils
.IOUtils
;
27 import be
.nikiroo
.utils
.MarkableFileInputStream
;
28 import be
.nikiroo
.utils
.StringUtils
;
31 * This cache will manage Internet (and local) downloads, as well as put the
32 * downloaded files into a cache.
34 * As long the cached resource is not too old, it will use it instead of
35 * retrieving the file again.
42 private long tooOldChanging
;
43 private long tooOldStable
;
44 private CookieManager cookies
;
47 * Create a new {@link Cache} object.
50 * the directory to use as cache
52 * the User-Agent to use to download the resources
53 * @param hoursChanging
54 * the number of hours after which a cached file that is thought
55 * to change ~often is considered too old (or -1 for
58 * the number of hours after which a LARGE cached file that is
59 * thought to change rarely is considered too old (or -1 for
63 * in case of I/O error
65 public Cache(File dir
, String UA
, int hoursChanging
, int hoursStable
)
69 this.tooOldChanging
= 1000 * 60 * 60 * hoursChanging
;
70 this.tooOldStable
= 1000 * 60 * 60 * hoursStable
;
78 if (dir
== null || !dir
.exists()) {
79 throw new IOException("Cannot create the cache directory: "
80 + (dir
== null ?
"null" : dir
.getAbsolutePath()));
83 cookies
= new CookieManager();
84 cookies
.setCookiePolicy(CookiePolicy
.ACCEPT_ALL
);
85 CookieHandler
.setDefault(cookies
);
89 * Open a resource (will load it from the cache if possible, or save it into
90 * the cache after downloading if not).
93 * the resource to open
95 * the support to use to download the resource
97 * TRUE for more stable resources, FALSE when they often change
99 * @return the opened resource
101 * @throws IOException
102 * in case of I/O error
104 public InputStream
open(URL url
, BasicSupport support
, boolean stable
)
106 return open(url
, support
, stable
, url
);
110 * Open a resource (will load it from the cache if possible, or save it into
111 * the cache after downloading if not).
113 * The cached resource will be assimilated to the given original {@link URL}
116 * the resource to open
118 * the support to use to download the resource
120 * TRUE for more stable resources, FALSE when they often change
122 * the original {@link URL} used to locate the cached resource
124 * @return the opened resource
126 * @throws IOException
127 * in case of I/O error
129 public InputStream
open(URL url
, BasicSupport support
, boolean stable
,
130 URL originalUrl
) throws IOException
{
132 InputStream in
= load(originalUrl
, false, stable
);
135 save(url
, support
, originalUrl
);
136 } catch (IOException e
) {
137 throw new IOException("Cannot save the url: "
138 + (url
== null ?
"null" : url
.toString()), e
);
141 in
= load(originalUrl
, true, stable
);
145 } catch (IOException e
) {
146 throw new IOException("Cannot open the url: "
147 + (url
== null ?
"null" : url
.toString()), e
);
152 * Refresh the resource into cache if needed.
155 * the resource to open
157 * the support to use to download the resource
159 * TRUE for more stable resources, FALSE when they often change
161 * @return TRUE if it was pre-downloaded
163 * @throws IOException
164 * in case of I/O error
166 public void refresh(URL url
, BasicSupport support
, boolean stable
)
168 File cached
= getCached(url
);
169 if (cached
.exists() && !isOld(cached
, stable
)) {
173 open(url
, support
, stable
).close();
177 * Check the resource to see if it is in the cache.
180 * the resource to check
182 * @return TRUE if it is
185 public boolean check(URL url
) {
186 return getCached(url
).exists();
190 * Open a resource (will load it from the cache if possible, or save it into
191 * the cache after downloading if not) as an Image, then save it where
194 * This version will not always work properly if the original file was not
198 * the resource to open
200 * @return the opened resource image
202 * @throws IOException
203 * in case of I/O error
205 public void saveAsImage(URL url
, File target
) throws IOException
{
206 URL cachedUrl
= new URL(url
.toString()
208 + Instance
.getConfig().getString(Config
.IMAGE_FORMAT_CONTENT
)
210 File cached
= getCached(cachedUrl
);
212 if (!cached
.exists() || isOld(cached
, true)) {
213 InputStream imageIn
= Instance
.getCache().open(url
, null, true);
214 ImageIO
.write(StringUtils
.toImage(imageIn
), Instance
.getConfig()
215 .getString(Config
.IMAGE_FORMAT_CONTENT
).toLowerCase(),
219 IOUtils
.write(new FileInputStream(cached
), target
);
223 * Manually add this item to the cache.
228 * a unique ID for this resource
230 * @return the resulting {@link File}
232 * @throws IOException
233 * in case of I/O error
235 public File
addToCache(InputStream in
, String uniqueID
) throws IOException
{
236 File file
= getCached(new File(uniqueID
).toURI().toURL());
237 IOUtils
.write(in
, file
);
242 * Clean the cache (delete the cached items).
245 * only clean the files that are considered too old
247 * @return the number of cleaned items
249 public int cleanCache(boolean onlyOld
) {
251 for (File file
: dir
.listFiles()) {
252 if (!onlyOld
|| isOld(file
, true)) {
256 System
.err
.println("Cannot delete temporary file: "
257 + file
.getAbsolutePath());
265 * Open a resource from the cache if it exists.
268 * the resource to open
269 * @return the opened resource
270 * @throws IOException
271 * in case of I/O error
273 private InputStream
load(URL url
, boolean allowOld
, boolean stable
)
275 File cached
= getCached(url
);
276 if (cached
.exists() && !isOld(cached
, stable
)) {
277 return new MarkableFileInputStream(new FileInputStream(cached
));
284 * Save the given resource to the cache.
289 * the {@link BasicSupport} used to download it
291 * the original {@link URL} used to locate the cached resource
293 * @throws IOException
294 * in case of I/O error
295 * @throws URISyntaxException
297 private void save(URL url
, BasicSupport support
, URL originalUrl
)
299 URLConnection conn
= url
.openConnection();
301 conn
.setRequestProperty("User-Agent", UA
);
302 conn
.setRequestProperty("Cookie", generateCookies(support
));
303 conn
.setRequestProperty("Accept-Encoding", "gzip");
304 if (support
!= null && support
.getCurrentReferer() != null) {
305 conn
.setRequestProperty("Referer", support
.getCurrentReferer()
307 conn
.setRequestProperty("Host", support
.getCurrentReferer()
314 if (conn
instanceof HttpURLConnection
315 && ((HttpURLConnection
) conn
).getResponseCode() / 100 == 3) {
316 String newUrl
= conn
.getHeaderField("Location");
317 save(new URL(newUrl
), support
, originalUrl
);
321 InputStream in
= conn
.getInputStream();
322 if ("gzip".equals(conn
.getContentEncoding())) {
323 in
= new GZIPInputStream(in
);
327 File cached
= getCached(originalUrl
);
328 BufferedOutputStream out
= new BufferedOutputStream(
329 new FileOutputStream(cached
));
331 byte[] buf
= new byte[4096];
333 while ((len
= in
.read(buf
)) > 0) {
334 out
.write(buf
, 0, len
);
345 * Check if the {@link File} is too old according to
346 * {@link Cache#tooOldChanging}.
351 * TRUE to denote files that are not supposed to change too often
353 * @return TRUE if it is
355 private boolean isOld(File file
, boolean stable
) {
356 long max
= tooOldChanging
;
365 long time
= new Date().getTime() - file
.lastModified();
367 System
.err
.println("Timestamp in the future for file: "
368 + file
.getAbsolutePath());
371 return time
< 0 || time
> max
;
375 * Get the cache resource from the cache if it is present for this
380 * @return the cached version if present, NULL if not
382 private File
getCached(URL url
) {
383 String name
= url
.getHost();
384 if (name
== null || name
.length() == 0) {
385 name
= url
.getFile();
387 name
= url
.toString();
390 name
= name
.replace('/', '_').replace(':', '_');
392 return new File(dir
, name
);
396 * Generate the cookie {@link String} from the local {@link CookieStore} so
397 * it is ready to be passed.
401 private String
generateCookies(BasicSupport support
) {
402 StringBuilder builder
= new StringBuilder();
403 for (HttpCookie cookie
: cookies
.getCookieStore().getCookies()) {
404 if (builder
.length() > 0) {
408 // TODO: check if format is ok
409 builder
.append(cookie
.toString());
412 if (support
!= null) {
413 for (Map
.Entry
<String
, String
> set
: support
.getCookies()
415 if (builder
.length() > 0) {
418 builder
.append(set
.getKey());
420 builder
.append(set
.getValue());
424 return builder
.toString();