e621: supports searches
[fanfix.git] / src / be / nikiroo / fanfix / DataLoader.java
1 package be.nikiroo.fanfix;
2
3 import java.io.File;
4 import java.io.IOException;
5 import java.io.InputStream;
6 import java.net.URL;
7 import java.util.Map;
8
9 import be.nikiroo.fanfix.bundles.Config;
10 import be.nikiroo.fanfix.supported.BasicSupport;
11 import be.nikiroo.utils.Cache;
12 import be.nikiroo.utils.CacheMemory;
13 import be.nikiroo.utils.Downloader;
14 import be.nikiroo.utils.IOUtils;
15 import be.nikiroo.utils.Image;
16 import be.nikiroo.utils.ImageUtils;
17 import be.nikiroo.utils.TraceHandler;
18
19 /**
20 * This cache will manage Internet (and local) downloads, as well as put the
21 * downloaded files into a cache.
22 * <p>
23 * As long the cached resource is not too old, it will use it instead of
24 * retrieving the file again.
25 *
26 * @author niki
27 */
28 public class DataLoader {
29 private Downloader downloader;
30 private Cache downloadCache;
31 private Cache cache;
32
33 /**
34 * Create a new {@link DataLoader} object.
35 *
36 * @param dir
37 * the directory to use as cache
38 * @param UA
39 * the User-Agent to use to download the resources
40 * @param hoursChanging
41 * the number of hours after which a cached file that is thought
42 * to change ~often is considered too old (or -1 for
43 * "never too old")
44 * @param hoursStable
45 * the number of hours after which a LARGE cached file that is
46 * thought to change rarely is considered too old (or -1 for
47 * "never too old")
48 *
49 * @throws IOException
50 * in case of I/O error
51 */
52 public DataLoader(File dir, String UA, int hoursChanging, int hoursStable)
53 throws IOException {
54 downloader = new Downloader(UA);
55 downloadCache = new Cache(dir, hoursChanging, hoursStable);
56 cache = downloadCache;
57 }
58
59 /**
60 * Create a new {@link DataLoader} object without disk cache (will keep a
61 * memory cache for manual cache operations).
62 *
63 * @param UA
64 * the User-Agent to use to download the resources
65 */
66 public DataLoader(String UA) {
67 downloader = new Downloader(UA);
68 downloadCache = null;
69 cache = new CacheMemory();
70 }
71
72 /**
73 * The traces handler for this {@link Cache}.
74 *
75 * @param tracer
76 * the new traces handler
77 */
78 public void setTraceHandler(TraceHandler tracer) {
79 downloader.setTraceHandler(tracer);
80 cache.setTraceHandler(tracer);
81 if (downloadCache != null) {
82 downloadCache.setTraceHandler(tracer);
83 }
84
85 }
86
87 /**
88 * Open a resource (will load it from the cache if possible, or save it into
89 * the cache after downloading if not).
90 *
91 * @param url
92 * the resource to open
93 * @param support
94 * the support to use to download the resource
95 * @param stable
96 * TRUE for more stable resources, FALSE when they often change
97 *
98 * @return the opened resource, NOT NULL
99 *
100 * @throws IOException
101 * in case of I/O error
102 */
103 public InputStream open(URL url, BasicSupport support, boolean stable)
104 throws IOException {
105 // MUST NOT return null
106 return open(url, support, stable, url);
107 }
108
109 /**
110 * Open a resource (will load it from the cache if possible, or save it into
111 * the cache after downloading if not).
112 * <p>
113 * The cached resource will be assimilated to the given original {@link URL}
114 *
115 * @param url
116 * the resource to open
117 * @param support
118 * the support to use to download the resource
119 * @param stable
120 * TRUE for more stable resources, FALSE when they often change
121 * @param originalUrl
122 * the original {@link URL} used to locate the cached resource
123 *
124 * @return the opened resource, NOT NULL
125 *
126 * @throws IOException
127 * in case of I/O error
128 */
129 public InputStream open(URL url, BasicSupport support, boolean stable,
130 URL originalUrl) throws IOException {
131 // MUST NOT return null
132 try {
133 InputStream in = null;
134
135 if (downloadCache != null) {
136 in = downloadCache.load(originalUrl, false, stable);
137 Instance.getTraceHandler().trace(
138 "Cache " + (in != null ? "hit" : "miss") + ": " + url);
139 }
140
141 if (in == null) {
142 try {
143 in = openNoCache(url, support, null, null, null);
144 if (downloadCache != null) {
145 downloadCache.save(in, originalUrl);
146 // ..But we want a resetable stream
147 in.close();
148 in = downloadCache.load(originalUrl, true, stable);
149 } else {
150 InputStream resetIn = IOUtils.forceResetableStream(in);
151 if (resetIn != in) {
152 in.close();
153 in = resetIn;
154 }
155 }
156 } catch (IOException e) {
157 throw new IOException("Cannot save the url: "
158 + (url == null ? "null" : url.toString()), e);
159 }
160 }
161
162 return in;
163 } catch (IOException e) {
164 throw new IOException("Cannot open the url: "
165 + (url == null ? "null" : url.toString()), e);
166 }
167 }
168
169 /**
170 * Open the given {@link URL} without using the cache, but still update the
171 * cookies.
172 *
173 * @param url
174 * the {@link URL} to open
175 *
176 * @return the {@link InputStream} of the opened page
177 *
178 * @throws IOException
179 * in case of I/O error
180 */
181 public InputStream openNoCache(URL url) throws IOException {
182 return downloader.open(url);
183 }
184
185 /**
186 * Open the given {@link URL} without using the cache, but still using and
187 * updating the cookies.
188 *
189 * @param url
190 * the {@link URL} to open
191 * @param support
192 * the {@link BasicSupport} used for the cookies
193 * @param postParams
194 * the POST parameters
195 * @param getParams
196 * the GET parameters (priority over POST)
197 * @param oauth
198 * OAuth authorization (aka, "bearer XXXXXXX")
199 *
200 * @return the {@link InputStream} of the opened page
201 *
202 * @throws IOException
203 * in case of I/O error
204 */
205 public InputStream openNoCache(URL url, BasicSupport support,
206 Map<String, String> postParams, Map<String, String> getParams,
207 String oauth) throws IOException {
208
209 Map<String, String> cookiesValues = null;
210 URL currentReferer = url;
211 if (support != null) {
212 cookiesValues = support.getCookies();
213 currentReferer = support.getCurrentReferer();
214 // priority: arguments
215 if (oauth == null) {
216 oauth = support.getOAuth();
217 }
218 }
219
220 return downloader.open(url, currentReferer, cookiesValues, postParams,
221 getParams, oauth);
222 }
223
224 /**
225 * Refresh the resource into cache if needed.
226 *
227 * @param url
228 * the resource to open
229 * @param support
230 * the support to use to download the resource
231 * @param stable
232 * TRUE for more stable resources, FALSE when they often change
233 *
234 * @throws IOException
235 * in case of I/O error
236 */
237 public void refresh(URL url, BasicSupport support, boolean stable)
238 throws IOException {
239 if (downloadCache != null && !downloadCache.check(url, false, stable)) {
240 open(url, support, stable).close();
241 }
242 }
243
244 /**
245 * Check the resource to see if it is in the cache.
246 *
247 * @param url
248 * the resource to check
249 * @param stable
250 * a stable file (that dones't change too often) -- parameter
251 * used to check if the file is too old to keep or not
252 *
253 * @return TRUE if it is
254 *
255 */
256 public boolean check(URL url, boolean stable) {
257 return downloadCache != null && downloadCache.check(url, false, stable);
258 }
259
260 /**
261 * Save the given resource as an image on disk using the default image
262 * format for content or cover -- will automatically add the extension, too.
263 *
264 * @param img
265 * the resource
266 * @param target
267 * the target file without extension
268 * @param cover
269 * use the cover image format instead of the content image format
270 *
271 * @throws IOException
272 * in case of I/O error
273 */
274 public void saveAsImage(Image img, File target, boolean cover)
275 throws IOException {
276 String format;
277 if (cover) {
278 format = Instance.getConfig().getString(Config.IMAGE_FORMAT_COVER)
279 .toLowerCase();
280 } else {
281 format = Instance.getConfig()
282 .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase();
283 }
284 saveAsImage(img, new File(target.toString() + "." + format), format);
285 }
286
287 /**
288 * Save the given resource as an image on disk using the given image format
289 * for content, or with "png" format if it fails.
290 *
291 * @param img
292 * the resource
293 * @param target
294 * the target file
295 * @param format
296 * the file format ("png", "jpeg", "bmp"...)
297 *
298 * @throws IOException
299 * in case of I/O error
300 */
301 public void saveAsImage(Image img, File target, String format)
302 throws IOException {
303 ImageUtils.getInstance().saveAsImage(img, target, format);
304 }
305
306 /**
307 * Manually add this item to the cache.
308 *
309 * @param in
310 * the input data
311 * @param uniqueID
312 * a unique ID for this resource
313 *
314 *
315 * @throws IOException
316 * in case of I/O error
317 */
318 public void addToCache(InputStream in, String uniqueID) throws IOException {
319 cache.save(in, uniqueID);
320 }
321
322 /**
323 * Return the {@link InputStream} corresponding to the given unique ID, or
324 * NULL if none found.
325 *
326 * @param uniqueID
327 * the unique ID
328 *
329 * @return the content or NULL
330 */
331 public InputStream getFromCache(String uniqueID) {
332 return cache.load(uniqueID, true, true);
333 }
334
335 /**
336 * Remove the given resource from the cache.
337 *
338 * @param uniqueID
339 * a unique ID used to locate the cached resource
340 *
341 * @return TRUE if it was removed
342 */
343 public boolean removeFromCache(String uniqueID) {
344 return cache.remove(uniqueID);
345 }
346
347 /**
348 * Clean the cache (delete the cached items).
349 *
350 * @param onlyOld
351 * only clean the files that are considered too old
352 *
353 * @return the number of cleaned items
354 */
355 public int cleanCache(boolean onlyOld) {
356 return cache.clean(onlyOld);
357 }
358 }