1 package be
.nikiroo
.fanfix
;
3 import java
.awt
.image
.BufferedImage
;
5 import java
.io
.IOException
;
6 import java
.io
.InputStream
;
10 import javax
.imageio
.ImageIO
;
12 import be
.nikiroo
.fanfix
.bundles
.Config
;
13 import be
.nikiroo
.fanfix
.supported
.BasicSupport
;
14 import be
.nikiroo
.utils
.Cache
;
15 import be
.nikiroo
.utils
.Downloader
;
16 import be
.nikiroo
.utils
.ImageUtils
;
19 * This cache will manage Internet (and local) downloads, as well as put the
20 * downloaded files into a cache.
22 * As long the cached resource is not too old, it will use it instead of
23 * retrieving the file again.
27 public class DataLoader
{
29 private Downloader downloader
;
32 * Create a new {@link DataLoader} object.
35 * the directory to use as cache
37 * the User-Agent to use to download the resources
38 * @param hoursChanging
39 * the number of hours after which a cached file that is thought
40 * to change ~often is considered too old (or -1 for
43 * the number of hours after which a LARGE cached file that is
44 * thought to change rarely is considered too old (or -1 for
48 * in case of I/O error
50 public DataLoader(File dir
, String UA
, int hoursChanging
, int hoursStable
)
52 cache
= new Cache(dir
, hoursChanging
, hoursStable
);
53 downloader
= new Downloader(UA
);
57 * Open a resource (will load it from the cache if possible, or save it into
58 * the cache after downloading if not).
61 * the resource to open
63 * the support to use to download the resource
65 * TRUE for more stable resources, FALSE when they often change
67 * @return the opened resource, NOT NULL
70 * in case of I/O error
72 public InputStream
open(URL url
, BasicSupport support
, boolean stable
)
74 // MUST NOT return null
75 return open(url
, support
, stable
, url
);
79 * Open a resource (will load it from the cache if possible, or save it into
80 * the cache after downloading if not).
82 * The cached resource will be assimilated to the given original {@link URL}
85 * the resource to open
87 * the support to use to download the resource
89 * TRUE for more stable resources, FALSE when they often change
91 * the original {@link URL} used to locate the cached resource
93 * @return the opened resource, NOT NULL
96 * in case of I/O error
98 public InputStream
open(URL url
, BasicSupport support
, boolean stable
,
99 URL originalUrl
) throws IOException
{
100 // MUST NOT return null
102 InputStream in
= cache
.load(originalUrl
, false, stable
);
103 Instance
.getTraceHandler().trace(
104 "Cache " + (in
!= null ?
"hit" : "miss") + ": " + url
);
108 in
= openNoCache(url
, support
, null, null, null);
109 cache
.save(in
, originalUrl
);
110 // ..But we want a resetable stream
112 in
= cache
.load(originalUrl
, false, stable
);
113 } catch (IOException e
) {
114 throw new IOException("Cannot save the url: "
115 + (url
== null ?
"null" : url
.toString()), e
);
120 } catch (IOException e
) {
121 throw new IOException("Cannot open the url: "
122 + (url
== null ?
"null" : url
.toString()), e
);
127 * Open the given {@link URL} without using the cache, but still update the
131 * the {@link URL} to open
133 * @return the {@link InputStream} of the opened page
135 * @throws IOException
136 * in case of I/O error
138 public InputStream
openNoCache(URL url
) throws IOException
{
139 return downloader
.open(url
);
143 * Open the given {@link URL} without using the cache, but still using and
144 * updating the cookies.
147 * the {@link URL} to open
149 * the {@link BasicSupport} used for the cookies
151 * the POST parameters
153 * the GET parameters (priority over POST)
155 * OAuth authorization (aka, "bearer XXXXXXX")
157 * @return the {@link InputStream} of the opened page
159 * @throws IOException
160 * in case of I/O error
162 public InputStream
openNoCache(URL url
, BasicSupport support
,
163 Map
<String
, String
> postParams
, Map
<String
, String
> getParams
,
164 String oauth
) throws IOException
{
166 Map
<String
, String
> cookiesValues
= null;
167 URL currentReferer
= url
;
168 if (support
!= null) {
169 cookiesValues
= support
.getCookies();
170 currentReferer
= support
.getCurrentReferer();
171 // priority: arguments
173 oauth
= support
.getOAuth();
177 return downloader
.open(url
, currentReferer
, cookiesValues
, postParams
,
182 * Refresh the resource into cache if needed.
185 * the resource to open
187 * the support to use to download the resource
189 * TRUE for more stable resources, FALSE when they often change
191 * @throws IOException
192 * in case of I/O error
194 public void refresh(URL url
, BasicSupport support
, boolean stable
)
196 if (!cache
.check(url
, false, stable
)) {
197 open(url
, support
, stable
).close();
202 * Check the resource to see if it is in the cache.
205 * the resource to check
207 * a stable file (that dones't change too often) -- parameter
208 * used to check if the file is too old to keep or not
210 * @return TRUE if it is
213 public boolean check(URL url
, boolean stable
) {
214 return cache
.check(url
, false, stable
);
218 * Save the given resource as an image on disk using the default image
219 * format for content.
226 * @throws IOException
227 * in case of I/O error
229 public void saveAsImage(URL url
, File target
) throws IOException
{
230 InputStream in
= open(url
, null, true);
232 saveAsImage(ImageUtils
.fromStream(in
), target
);
239 * Save the given resource as an image on disk using the default image
240 * format for content.
247 * @throws IOException
248 * in case of I/O error
250 public void saveAsImage(BufferedImage image
, File target
)
253 String format
= Instance
.getConfig()
254 .getString(Config
.IMAGE_FORMAT_CONTENT
).toLowerCase();
258 ok
= ImageIO
.write(image
, format
, target
);
259 } catch (IOException e
) {
263 // Some formats are not reliable
264 // Second change: PNG
265 if (!ok
&& !format
.equals("png")) {
266 ok
= ImageIO
.write(image
, "png", target
);
270 throw new IOException(
271 "Cannot find a writer for this image and format: "
274 } catch (IOException e
) {
275 throw new IOException("Cannot write image to " + target
, e
);
280 * Manually add this item to the cache.
285 * a unique ID for this resource
287 * @return the resulting {@link File}
289 * @throws IOException
290 * in case of I/O error
292 public File
addToCache(InputStream in
, String uniqueID
) throws IOException
{
293 return cache
.save(in
, uniqueID
);
297 * Return the {@link InputStream} corresponding to the given unique ID, or
298 * NULL if none found.
303 * @return the content or NULL
305 public InputStream
getFromCache(String uniqueID
) {
306 return cache
.load(uniqueID
, true, true);
310 * Remove the given resource from the cache.
313 * a unique ID used to locate the cached resource
315 * @return TRUE if it was removed
317 public boolean removeFromCache(String uniqueID
) {
318 return cache
.remove(uniqueID
);
322 * Clean the cache (delete the cached items).
325 * only clean the files that are considered too old
327 * @return the number of cleaned items
329 public int cleanCache(boolean onlyOld
) {
330 return cache
.clean(onlyOld
);