1 package be
.nikiroo
.fanfix
;
4 import java
.io
.IOException
;
5 import java
.io
.InputStream
;
9 import javax
.imageio
.ImageIO
;
11 import be
.nikiroo
.fanfix
.bundles
.Config
;
12 import be
.nikiroo
.fanfix
.supported
.BasicSupport
;
13 import be
.nikiroo
.utils
.Cache
;
14 import be
.nikiroo
.utils
.Downloader
;
15 import be
.nikiroo
.utils
.ImageUtils
;
18 * This cache will manage Internet (and local) downloads, as well as put the
19 * downloaded files into a cache.
21 * As long the cached resource is not too old, it will use it instead of
22 * retrieving the file again.
26 public class DataLoader
{
28 private Downloader downloader
;
31 * Create a new {@link DataLoader} object.
34 * the directory to use as cache
36 * the User-Agent to use to download the resources
37 * @param hoursChanging
38 * the number of hours after which a cached file that is thought
39 * to change ~often is considered too old (or -1 for
42 * the number of hours after which a LARGE cached file that is
43 * thought to change rarely is considered too old (or -1 for
47 * in case of I/O error
49 public DataLoader(File dir
, String UA
, int hoursChanging
, int hoursStable
)
51 cache
= new Cache(dir
, hoursChanging
, hoursStable
);
52 downloader
= new Downloader(UA
);
56 * Open a resource (will load it from the cache if possible, or save it into
57 * the cache after downloading if not).
60 * the resource to open
62 * the support to use to download the resource
64 * TRUE for more stable resources, FALSE when they often change
66 * @return the opened resource, NOT NULL
69 * in case of I/O error
71 public InputStream
open(URL url
, BasicSupport support
, boolean stable
)
73 // MUST NOT return null
74 return open(url
, support
, stable
, url
);
78 * Open a resource (will load it from the cache if possible, or save it into
79 * the cache after downloading if not).
81 * The cached resource will be assimilated to the given original {@link URL}
84 * the resource to open
86 * the support to use to download the resource
88 * TRUE for more stable resources, FALSE when they often change
90 * the original {@link URL} used to locate the cached resource
92 * @return the opened resource, NOT NULL
95 * in case of I/O error
97 public InputStream
open(URL url
, BasicSupport support
, boolean stable
,
98 URL originalUrl
) throws IOException
{
99 // MUST NOT return null
101 InputStream in
= cache
.load(originalUrl
, false, stable
);
102 Instance
.trace("Cache " + (in
!= null ?
"hit" : "miss") + ": "
107 in
= openNoCache(url
, support
, null, null, null);
108 cache
.save(in
, originalUrl
);
109 // ..But we want a resetable stream
111 in
= cache
.load(originalUrl
, false, stable
);
112 } catch (IOException e
) {
113 throw new IOException("Cannot save the url: "
114 + (url
== null ?
"null" : url
.toString()), e
);
119 } catch (IOException e
) {
120 throw new IOException("Cannot open the url: "
121 + (url
== null ?
"null" : url
.toString()), e
);
126 * Open the given {@link URL} without using the cache, but still update the
130 * the {@link URL} to open
132 * @return the {@link InputStream} of the opened page
134 * @throws IOException
135 * in case of I/O error
137 public InputStream
openNoCache(URL url
) throws IOException
{
138 return downloader
.open(url
);
142 * Open the given {@link URL} without using the cache, but still using and
143 * updating the cookies.
146 * the {@link URL} to open
148 * the {@link BasicSupport} used for the cookies
150 * the POST parameters
152 * the GET parameters (priority over POST)
154 * OAuth authorization (aka, "bearer XXXXXXX")
156 * @return the {@link InputStream} of the opened page
158 * @throws IOException
159 * in case of I/O error
161 public InputStream
openNoCache(URL url
, BasicSupport support
,
162 Map
<String
, String
> postParams
, Map
<String
, String
> getParams
,
163 String oauth
) throws IOException
{
165 Map
<String
, String
> cookiesValues
= null;
166 URL currentReferer
= url
;
167 if (support
!= null) {
168 cookiesValues
= support
.getCookies();
169 currentReferer
= support
.getCurrentReferer();
170 // priority: arguments
172 oauth
= support
.getOAuth();
176 return downloader
.open(url
, currentReferer
, cookiesValues
, postParams
,
181 * Refresh the resource into cache if needed.
184 * the resource to open
186 * the support to use to download the resource
188 * TRUE for more stable resources, FALSE when they often change
190 * @throws IOException
191 * in case of I/O error
193 public void refresh(URL url
, BasicSupport support
, boolean stable
)
195 if (!cache
.check(url
, false, stable
)) {
196 open(url
, support
, stable
).close();
201 * Check the resource to see if it is in the cache.
204 * the resource to check
206 * a stable file (that dones't change too often) -- parameter
207 * used to check if the file is too old to keep or not
209 * @return TRUE if it is
212 public boolean check(URL url
, boolean stable
) {
213 return cache
.check(url
, false, stable
);
217 * Save the given resource as an image on disk using the default image
218 * format for content.
225 * @throws IOException
226 * in case of I/O error
228 public void saveAsImage(URL url
, File target
) throws IOException
{
229 InputStream in
= open(url
, null, true);
231 ImageIO
.write(ImageUtils
.fromStream(in
), Instance
.getConfig()
232 .getString(Config
.IMAGE_FORMAT_CONTENT
).toLowerCase(),
234 } catch (IOException e
) {
235 throw new IOException("Cannot write image " + url
, e
);
242 * Manually add this item to the cache.
247 * a unique ID for this resource
249 * @return the resulting {@link File}
251 * @throws IOException
252 * in case of I/O error
254 public File
addToCache(InputStream in
, String uniqueID
) throws IOException
{
255 return cache
.save(in
, uniqueID
);
259 * Return the {@link InputStream} corresponding to the given unique ID, or
260 * NULL if none found.
265 * @return the content or NULL
267 public InputStream
getFromCache(String uniqueID
) {
268 return cache
.load(uniqueID
, true, true);
272 * Remove the given resource from the cache.
275 * a unique ID used to locate the cached resource
277 * @return TRUE if it was removed
279 public boolean removeFromCache(String uniqueID
) {
280 return cache
.remove(uniqueID
);
284 * Clean the cache (delete the cached items).
287 * only clean the files that are considered too old
289 * @return the number of cleaned items
291 public int cleanCache(boolean onlyOld
) {
292 return cache
.clean(onlyOld
);