test: improve flag files
[nikiroo-utils.git] / src / be / nikiroo / fanfix / DataLoader.java
CommitLineData
f1fb834c
NR
1package be.nikiroo.fanfix;
2
3import java.io.File;
4import java.io.IOException;
5import java.io.InputStream;
6import java.net.URL;
7import java.util.Map;
8
f1fb834c
NR
9import be.nikiroo.fanfix.bundles.Config;
10import be.nikiroo.fanfix.supported.BasicSupport;
11import be.nikiroo.utils.Cache;
ae78e517 12import be.nikiroo.utils.CacheMemory;
f1fb834c 13import be.nikiroo.utils.Downloader;
16a81ef7 14import be.nikiroo.utils.Image;
f1fb834c 15import be.nikiroo.utils.ImageUtils;
ae78e517 16import be.nikiroo.utils.TraceHandler;
f1fb834c
NR
17
18/**
19 * This cache will manage Internet (and local) downloads, as well as put the
20 * downloaded files into a cache.
21 * <p>
22 * As long the cached resource is not too old, it will use it instead of
23 * retrieving the file again.
24 *
25 * @author niki
26 */
27public class DataLoader {
f1fb834c 28 private Downloader downloader;
12443642 29 private Downloader downloaderNoCache;
ae78e517 30 private Cache cache;
f1ce03f7 31 private boolean offline;
f1fb834c
NR
32
33 /**
34 * Create a new {@link DataLoader} object.
35 *
36 * @param dir
37 * the directory to use as cache
38 * @param UA
39 * the User-Agent to use to download the resources
40 * @param hoursChanging
41 * the number of hours after which a cached file that is thought
42 * to change ~often is considered too old (or -1 for
43 * "never too old")
44 * @param hoursStable
45 * the number of hours after which a LARGE cached file that is
46 * thought to change rarely is considered too old (or -1 for
47 * "never too old")
48 *
49 * @throws IOException
50 * in case of I/O error
51 */
52 public DataLoader(File dir, String UA, int hoursChanging, int hoursStable)
53 throws IOException {
12443642
NR
54 downloader = new Downloader(UA, new Cache(dir, hoursChanging,
55 hoursStable));
56 downloaderNoCache = new Downloader(UA);
57
58 cache = downloader.getCache();
ae78e517
NR
59 }
60
61 /**
62 * Create a new {@link DataLoader} object without disk cache (will keep a
63 * memory cache for manual cache operations).
64 *
65 * @param UA
66 * the User-Agent to use to download the resources
67 */
68 public DataLoader(String UA) {
69 downloader = new Downloader(UA);
12443642 70 downloaderNoCache = downloader;
ae78e517
NR
71 cache = new CacheMemory();
72 }
f1ce03f7
NR
73
74 /**
75 * This {@link Downloader} is forbidden to try and connect to the network.
76 * <p>
77 * If TRUE, it will only check the cache (even in no-cache mode!).
78 * <p>
79 * Default is FALSE.
80 *
81 * @return TRUE if offline
82 */
83 public boolean isOffline() {
84 return offline;
85 }
86
87 /**
88 * This {@link Downloader} is forbidden to try and connect to the network.
89 * <p>
90 * If TRUE, it will only check the cache (even in no-cache mode!).
91 * <p>
92 * Default is FALSE.
93 *
94 * @param offline TRUE for offline, FALSE for online
95 */
96 public void setOffline(boolean offline) {
97 this.offline = offline;
98 downloader.setOffline(offline);
99 downloaderNoCache.setOffline(offline);
100
101 // If we don't, we cannot support no-cache using code in OFFLINE mode
102 if (offline) {
103 downloaderNoCache.setCache(cache);
104 } else {
105 downloaderNoCache.setCache(null);
106 }
107 }
ae78e517
NR
108
109 /**
110 * The traces handler for this {@link Cache}.
111 *
112 * @param tracer
113 * the new traces handler
114 */
115 public void setTraceHandler(TraceHandler tracer) {
116 downloader.setTraceHandler(tracer);
12443642 117 downloaderNoCache.setTraceHandler(tracer);
ae78e517 118 cache.setTraceHandler(tracer);
12443642
NR
119 if (downloader.getCache() != null) {
120 downloader.getCache().setTraceHandler(tracer);
ae78e517
NR
121 }
122
f1fb834c
NR
123 }
124
125 /**
126 * Open a resource (will load it from the cache if possible, or save it into
127 * the cache after downloading if not).
12443642
NR
128 * <p>
129 * The cached resource will be assimilated to the given original {@link URL}
f1fb834c
NR
130 *
131 * @param url
132 * the resource to open
133 * @param support
8d59ce07 134 * the support to use to download the resource (can be NULL)
f1fb834c
NR
135 * @param stable
136 * TRUE for more stable resources, FALSE when they often change
137 *
138 * @return the opened resource, NOT NULL
139 *
140 * @throws IOException
141 * in case of I/O error
142 */
143 public InputStream open(URL url, BasicSupport support, boolean stable)
144 throws IOException {
12443642 145 return open(url, url, support, stable, null, null, null);
f1fb834c
NR
146 }
147
148 /**
149 * Open a resource (will load it from the cache if possible, or save it into
150 * the cache after downloading if not).
151 * <p>
152 * The cached resource will be assimilated to the given original {@link URL}
153 *
154 * @param url
155 * the resource to open
12443642
NR
156 * @param originalUrl
157 * the original {@link URL} before any redirection occurs, which
158 * is also used for the cache ID if needed (so we can retrieve
159 * the content with this URL if needed)
f1fb834c
NR
160 * @param support
161 * the support to use to download the resource
162 * @param stable
163 * TRUE for more stable resources, FALSE when they often change
f1fb834c
NR
164 *
165 * @return the opened resource, NOT NULL
166 *
167 * @throws IOException
168 * in case of I/O error
169 */
12443642
NR
170 public InputStream open(URL url, URL originalUrl, BasicSupport support,
171 boolean stable) throws IOException {
172 return open(url, originalUrl, support, stable, null, null, null);
f1fb834c
NR
173 }
174
175 /**
12443642
NR
176 * Open a resource (will load it from the cache if possible, or save it into
177 * the cache after downloading if not).
178 * <p>
179 * The cached resource will be assimilated to the given original {@link URL}
f1fb834c
NR
180 *
181 * @param url
12443642
NR
182 * the resource to open
183 * @param originalUrl
184 * the original {@link URL} before any redirection occurs, which
185 * is also used for the cache ID if needed (so we can retrieve
186 * the content with this URL if needed)
187 * @param support
8d59ce07 188 * the support to use to download the resource (can be NULL)
12443642
NR
189 * @param stable
190 * TRUE for more stable resources, FALSE when they often change
191 * @param postParams
192 * the POST parameters
193 * @param getParams
194 * the GET parameters (priority over POST)
195 * @param oauth
196 * OAuth authorization (aka, "bearer XXXXXXX")
f1fb834c 197 *
12443642 198 * @return the opened resource, NOT NULL
f1fb834c
NR
199 *
200 * @throws IOException
201 * in case of I/O error
202 */
12443642
NR
203 public InputStream open(URL url, URL originalUrl, BasicSupport support,
204 boolean stable, Map<String, String> postParams,
205 Map<String, String> getParams, String oauth) throws IOException {
206
207 Map<String, String> cookiesValues = null;
208 URL currentReferer = url;
209
210 if (support != null) {
211 cookiesValues = support.getCookies();
212 currentReferer = support.getCurrentReferer();
213 // priority: arguments
214 if (oauth == null) {
215 oauth = support.getOAuth();
216 }
217 }
218
219 return downloader.open(url, originalUrl, currentReferer, cookiesValues,
220 postParams, getParams, oauth, stable);
f1fb834c
NR
221 }
222
223 /**
224 * Open the given {@link URL} without using the cache, but still using and
225 * updating the cookies.
226 *
227 * @param url
228 * the {@link URL} to open
229 * @param support
230 * the {@link BasicSupport} used for the cookies
231 * @param postParams
232 * the POST parameters
233 * @param getParams
234 * the GET parameters (priority over POST)
235 * @param oauth
236 * OAuth authorization (aka, "bearer XXXXXXX")
237 *
238 * @return the {@link InputStream} of the opened page
239 *
240 * @throws IOException
241 * in case of I/O error
242 */
243 public InputStream openNoCache(URL url, BasicSupport support,
244 Map<String, String> postParams, Map<String, String> getParams,
245 String oauth) throws IOException {
246
247 Map<String, String> cookiesValues = null;
248 URL currentReferer = url;
249 if (support != null) {
250 cookiesValues = support.getCookies();
251 currentReferer = support.getCurrentReferer();
252 // priority: arguments
253 if (oauth == null) {
254 oauth = support.getOAuth();
255 }
256 }
257
12443642
NR
258 return downloaderNoCache.open(url, currentReferer, cookiesValues,
259 postParams, getParams, oauth);
f1fb834c
NR
260 }
261
262 /**
263 * Refresh the resource into cache if needed.
264 *
265 * @param url
266 * the resource to open
267 * @param support
8d59ce07 268 * the support to use to download the resource (can be NULL)
f1fb834c
NR
269 * @param stable
270 * TRUE for more stable resources, FALSE when they often change
271 *
272 * @throws IOException
273 * in case of I/O error
274 */
275 public void refresh(URL url, BasicSupport support, boolean stable)
276 throws IOException {
c289a297 277 if (!check(url, stable)) {
12443642 278 open(url, url, support, stable, null, null, null).close();
f1fb834c
NR
279 }
280 }
281
282 /**
283 * Check the resource to see if it is in the cache.
284 *
285 * @param url
286 * the resource to check
287 * @param stable
288 * a stable file (that dones't change too often) -- parameter
289 * used to check if the file is too old to keep or not
290 *
291 * @return TRUE if it is
292 *
293 */
294 public boolean check(URL url, boolean stable) {
12443642
NR
295 return downloader.getCache() != null
296 && downloader.getCache().check(url, false, stable);
f1fb834c
NR
297 }
298
299 /**
300 * Save the given resource as an image on disk using the default image
16a81ef7 301 * format for content or cover -- will automatically add the extension, too.
f1fb834c 302 *
16a81ef7 303 * @param img
f1fb834c
NR
304 * the resource
305 * @param target
16a81ef7
NR
306 * the target file without extension
307 * @param cover
308 * use the cover image format instead of the content image format
f1fb834c
NR
309 *
310 * @throws IOException
311 * in case of I/O error
312 */
16a81ef7
NR
313 public void saveAsImage(Image img, File target, boolean cover)
314 throws IOException {
315 String format;
316 if (cover) {
13fdb89a 317 format = Instance.getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER)
16a81ef7
NR
318 .toLowerCase();
319 } else {
320 format = Instance.getConfig()
13fdb89a 321 .getString(Config.FILE_FORMAT_IMAGE_FORMAT_CONTENT).toLowerCase();
f1fb834c 322 }
16a81ef7 323 saveAsImage(img, new File(target.toString() + "." + format), format);
f1fb834c
NR
324 }
325
2a25f781 326 /**
16a81ef7
NR
327 * Save the given resource as an image on disk using the given image format
328 * for content, or with "png" format if it fails.
2a25f781 329 *
16a81ef7 330 * @param img
2a25f781
NR
331 * the resource
332 * @param target
333 * the target file
16a81ef7
NR
334 * @param format
335 * the file format ("png", "jpeg", "bmp"...)
2a25f781
NR
336 *
337 * @throws IOException
338 * in case of I/O error
339 */
16a81ef7 340 public void saveAsImage(Image img, File target, String format)
2a25f781 341 throws IOException {
16a81ef7 342 ImageUtils.getInstance().saveAsImage(img, target, format);
2a25f781
NR
343 }
344
f1fb834c
NR
345 /**
346 * Manually add this item to the cache.
347 *
348 * @param in
349 * the input data
350 * @param uniqueID
351 * a unique ID for this resource
352 *
f1fb834c
NR
353 *
354 * @throws IOException
355 * in case of I/O error
356 */
ae78e517
NR
357 public void addToCache(InputStream in, String uniqueID) throws IOException {
358 cache.save(in, uniqueID);
f1fb834c
NR
359 }
360
361 /**
362 * Return the {@link InputStream} corresponding to the given unique ID, or
363 * NULL if none found.
364 *
365 * @param uniqueID
366 * the unique ID
367 *
368 * @return the content or NULL
369 */
370 public InputStream getFromCache(String uniqueID) {
371 return cache.load(uniqueID, true, true);
372 }
373
085a2f9a
NR
374 /**
375 * Remove the given resource from the cache.
376 *
377 * @param uniqueID
378 * a unique ID used to locate the cached resource
379 *
380 * @return TRUE if it was removed
381 */
382 public boolean removeFromCache(String uniqueID) {
383 return cache.remove(uniqueID);
384 }
385
f1fb834c
NR
386 /**
387 * Clean the cache (delete the cached items).
388 *
389 * @param onlyOld
390 * only clean the files that are considered too old
391 *
392 * @return the number of cleaned items
393 */
394 public int cleanCache(boolean onlyOld) {
395 return cache.clean(onlyOld);
396 }
397}