tests: fix NPE, add BasicSupportUtilities tests
[nikiroo-utils.git] / src / be / nikiroo / fanfix / DataLoader.java
1 package be.nikiroo.fanfix;
2
3 import java.io.File;
4 import java.io.IOException;
5 import java.io.InputStream;
6 import java.net.URL;
7 import java.util.Map;
8
9 import be.nikiroo.fanfix.bundles.Config;
10 import be.nikiroo.fanfix.supported.BasicSupport;
11 import be.nikiroo.utils.Cache;
12 import be.nikiroo.utils.CacheMemory;
13 import be.nikiroo.utils.Downloader;
14 import be.nikiroo.utils.Image;
15 import be.nikiroo.utils.ImageUtils;
16 import be.nikiroo.utils.TraceHandler;
17
18 /**
19 * This cache will manage Internet (and local) downloads, as well as put the
20 * downloaded files into a cache.
21 * <p>
22 * As long the cached resource is not too old, it will use it instead of
23 * retrieving the file again.
24 *
25 * @author niki
26 */
27 public class DataLoader {
28 private Downloader downloader;
29 private Downloader downloaderNoCache;
30 private Cache cache;
31
32 /**
33 * Create a new {@link DataLoader} object.
34 *
35 * @param dir
36 * the directory to use as cache
37 * @param UA
38 * the User-Agent to use to download the resources
39 * @param hoursChanging
40 * the number of hours after which a cached file that is thought
41 * to change ~often is considered too old (or -1 for
42 * "never too old")
43 * @param hoursStable
44 * the number of hours after which a LARGE cached file that is
45 * thought to change rarely is considered too old (or -1 for
46 * "never too old")
47 *
48 * @throws IOException
49 * in case of I/O error
50 */
51 public DataLoader(File dir, String UA, int hoursChanging, int hoursStable)
52 throws IOException {
53 downloader = new Downloader(UA, new Cache(dir, hoursChanging,
54 hoursStable));
55 downloaderNoCache = new Downloader(UA);
56
57 cache = downloader.getCache();
58 }
59
60 /**
61 * Create a new {@link DataLoader} object without disk cache (will keep a
62 * memory cache for manual cache operations).
63 *
64 * @param UA
65 * the User-Agent to use to download the resources
66 */
67 public DataLoader(String UA) {
68 downloader = new Downloader(UA);
69 downloaderNoCache = downloader;
70 cache = new CacheMemory();
71 }
72
73 /**
74 * The traces handler for this {@link Cache}.
75 *
76 * @param tracer
77 * the new traces handler
78 */
79 public void setTraceHandler(TraceHandler tracer) {
80 downloader.setTraceHandler(tracer);
81 downloaderNoCache.setTraceHandler(tracer);
82 cache.setTraceHandler(tracer);
83 if (downloader.getCache() != null) {
84 downloader.getCache().setTraceHandler(tracer);
85 }
86
87 }
88
89 /**
90 * Open a resource (will load it from the cache if possible, or save it into
91 * the cache after downloading if not).
92 * <p>
93 * The cached resource will be assimilated to the given original {@link URL}
94 *
95 * @param url
96 * the resource to open
97 * @param support
98 * the support to use to download the resource (can be NULL)
99 * @param stable
100 * TRUE for more stable resources, FALSE when they often change
101 *
102 * @return the opened resource, NOT NULL
103 *
104 * @throws IOException
105 * in case of I/O error
106 */
107 public InputStream open(URL url, BasicSupport support, boolean stable)
108 throws IOException {
109 return open(url, url, support, stable, null, null, null);
110 }
111
112 /**
113 * Open a resource (will load it from the cache if possible, or save it into
114 * the cache after downloading if not).
115 * <p>
116 * The cached resource will be assimilated to the given original {@link URL}
117 *
118 * @param url
119 * the resource to open
120 * @param originalUrl
121 * the original {@link URL} before any redirection occurs, which
122 * is also used for the cache ID if needed (so we can retrieve
123 * the content with this URL if needed)
124 * @param support
125 * the support to use to download the resource
126 * @param stable
127 * TRUE for more stable resources, FALSE when they often change
128 *
129 * @return the opened resource, NOT NULL
130 *
131 * @throws IOException
132 * in case of I/O error
133 */
134 public InputStream open(URL url, URL originalUrl, BasicSupport support,
135 boolean stable) throws IOException {
136 return open(url, originalUrl, support, stable, null, null, null);
137 }
138
139 /**
140 * Open a resource (will load it from the cache if possible, or save it into
141 * the cache after downloading if not).
142 * <p>
143 * The cached resource will be assimilated to the given original {@link URL}
144 *
145 * @param url
146 * the resource to open
147 * @param originalUrl
148 * the original {@link URL} before any redirection occurs, which
149 * is also used for the cache ID if needed (so we can retrieve
150 * the content with this URL if needed)
151 * @param support
152 * the support to use to download the resource (can be NULL)
153 * @param stable
154 * TRUE for more stable resources, FALSE when they often change
155 * @param postParams
156 * the POST parameters
157 * @param getParams
158 * the GET parameters (priority over POST)
159 * @param oauth
160 * OAuth authorization (aka, "bearer XXXXXXX")
161 *
162 * @return the opened resource, NOT NULL
163 *
164 * @throws IOException
165 * in case of I/O error
166 */
167 public InputStream open(URL url, URL originalUrl, BasicSupport support,
168 boolean stable, Map<String, String> postParams,
169 Map<String, String> getParams, String oauth) throws IOException {
170
171 Map<String, String> cookiesValues = null;
172 URL currentReferer = url;
173
174 if (support != null) {
175 cookiesValues = support.getCookies();
176 currentReferer = support.getCurrentReferer();
177 // priority: arguments
178 if (oauth == null) {
179 oauth = support.getOAuth();
180 }
181 }
182
183 return downloader.open(url, originalUrl, currentReferer, cookiesValues,
184 postParams, getParams, oauth, stable);
185 }
186
187 /**
188 * Open the given {@link URL} without using the cache, but still using and
189 * updating the cookies.
190 *
191 * @param url
192 * the {@link URL} to open
193 * @param support
194 * the {@link BasicSupport} used for the cookies
195 * @param postParams
196 * the POST parameters
197 * @param getParams
198 * the GET parameters (priority over POST)
199 * @param oauth
200 * OAuth authorization (aka, "bearer XXXXXXX")
201 *
202 * @return the {@link InputStream} of the opened page
203 *
204 * @throws IOException
205 * in case of I/O error
206 */
207 public InputStream openNoCache(URL url, BasicSupport support,
208 Map<String, String> postParams, Map<String, String> getParams,
209 String oauth) throws IOException {
210
211 Map<String, String> cookiesValues = null;
212 URL currentReferer = url;
213 if (support != null) {
214 cookiesValues = support.getCookies();
215 currentReferer = support.getCurrentReferer();
216 // priority: arguments
217 if (oauth == null) {
218 oauth = support.getOAuth();
219 }
220 }
221
222 return downloaderNoCache.open(url, currentReferer, cookiesValues,
223 postParams, getParams, oauth);
224 }
225
226 /**
227 * Refresh the resource into cache if needed.
228 *
229 * @param url
230 * the resource to open
231 * @param support
232 * the support to use to download the resource (can be NULL)
233 * @param stable
234 * TRUE for more stable resources, FALSE when they often change
235 *
236 * @throws IOException
237 * in case of I/O error
238 */
239 public void refresh(URL url, BasicSupport support, boolean stable)
240 throws IOException {
241 if (!check(url, stable)) {
242 open(url, url, support, stable, null, null, null).close();
243 }
244 }
245
246 /**
247 * Check the resource to see if it is in the cache.
248 *
249 * @param url
250 * the resource to check
251 * @param stable
252 * a stable file (that dones't change too often) -- parameter
253 * used to check if the file is too old to keep or not
254 *
255 * @return TRUE if it is
256 *
257 */
258 public boolean check(URL url, boolean stable) {
259 return downloader.getCache() != null
260 && downloader.getCache().check(url, false, stable);
261 }
262
263 /**
264 * Save the given resource as an image on disk using the default image
265 * format for content or cover -- will automatically add the extension, too.
266 *
267 * @param img
268 * the resource
269 * @param target
270 * the target file without extension
271 * @param cover
272 * use the cover image format instead of the content image format
273 *
274 * @throws IOException
275 * in case of I/O error
276 */
277 public void saveAsImage(Image img, File target, boolean cover)
278 throws IOException {
279 String format;
280 if (cover) {
281 format = Instance.getConfig().getString(Config.FILE_FORMAT_IMAGE_FORMAT_COVER)
282 .toLowerCase();
283 } else {
284 format = Instance.getConfig()
285 .getString(Config.FILE_FORMAT_IMAGE_FORMAT_CONTENT).toLowerCase();
286 }
287 saveAsImage(img, new File(target.toString() + "." + format), format);
288 }
289
290 /**
291 * Save the given resource as an image on disk using the given image format
292 * for content, or with "png" format if it fails.
293 *
294 * @param img
295 * the resource
296 * @param target
297 * the target file
298 * @param format
299 * the file format ("png", "jpeg", "bmp"...)
300 *
301 * @throws IOException
302 * in case of I/O error
303 */
304 public void saveAsImage(Image img, File target, String format)
305 throws IOException {
306 ImageUtils.getInstance().saveAsImage(img, target, format);
307 }
308
309 /**
310 * Manually add this item to the cache.
311 *
312 * @param in
313 * the input data
314 * @param uniqueID
315 * a unique ID for this resource
316 *
317 *
318 * @throws IOException
319 * in case of I/O error
320 */
321 public void addToCache(InputStream in, String uniqueID) throws IOException {
322 cache.save(in, uniqueID);
323 }
324
325 /**
326 * Return the {@link InputStream} corresponding to the given unique ID, or
327 * NULL if none found.
328 *
329 * @param uniqueID
330 * the unique ID
331 *
332 * @return the content or NULL
333 */
334 public InputStream getFromCache(String uniqueID) {
335 return cache.load(uniqueID, true, true);
336 }
337
338 /**
339 * Remove the given resource from the cache.
340 *
341 * @param uniqueID
342 * a unique ID used to locate the cached resource
343 *
344 * @return TRUE if it was removed
345 */
346 public boolean removeFromCache(String uniqueID) {
347 return cache.remove(uniqueID);
348 }
349
350 /**
351 * Clean the cache (delete the cached items).
352 *
353 * @param onlyOld
354 * only clean the files that are considered too old
355 *
356 * @return the number of cleaned items
357 */
358 public int cleanCache(boolean onlyOld) {
359 return cache.clean(onlyOld);
360 }
361 }