doc
[nikiroo-utils.git] / DataLoader.java
... / ...
CommitLineData
1package be.nikiroo.fanfix;
2
3import java.io.File;
4import java.io.IOException;
5import java.io.InputStream;
6import java.net.URL;
7import java.util.Map;
8
9import javax.imageio.ImageIO;
10
11import be.nikiroo.fanfix.bundles.Config;
12import be.nikiroo.fanfix.supported.BasicSupport;
13import be.nikiroo.utils.Cache;
14import be.nikiroo.utils.Downloader;
15import be.nikiroo.utils.ImageUtils;
16
17/**
18 * This cache will manage Internet (and local) downloads, as well as put the
19 * downloaded files into a cache.
20 * <p>
21 * As long the cached resource is not too old, it will use it instead of
22 * retrieving the file again.
23 *
24 * @author niki
25 */
26public class DataLoader {
27 private Cache cache;
28 private Downloader downloader;
29
30 /**
31 * Create a new {@link DataLoader} object.
32 *
33 * @param dir
34 * the directory to use as cache
35 * @param UA
36 * the User-Agent to use to download the resources
37 * @param hoursChanging
38 * the number of hours after which a cached file that is thought
39 * to change ~often is considered too old (or -1 for
40 * "never too old")
41 * @param hoursStable
42 * the number of hours after which a LARGE cached file that is
43 * thought to change rarely is considered too old (or -1 for
44 * "never too old")
45 *
46 * @throws IOException
47 * in case of I/O error
48 */
49 public DataLoader(File dir, String UA, int hoursChanging, int hoursStable)
50 throws IOException {
51 cache = new Cache(dir, hoursChanging, hoursStable);
52 downloader = new Downloader(UA);
53 }
54
55 /**
56 * Open a resource (will load it from the cache if possible, or save it into
57 * the cache after downloading if not).
58 *
59 * @param url
60 * the resource to open
61 * @param support
62 * the support to use to download the resource
63 * @param stable
64 * TRUE for more stable resources, FALSE when they often change
65 *
66 * @return the opened resource, NOT NULL
67 *
68 * @throws IOException
69 * in case of I/O error
70 */
71 public InputStream open(URL url, BasicSupport support, boolean stable)
72 throws IOException {
73 // MUST NOT return null
74 return open(url, support, stable, url);
75 }
76
77 /**
78 * Open a resource (will load it from the cache if possible, or save it into
79 * the cache after downloading if not).
80 * <p>
81 * The cached resource will be assimilated to the given original {@link URL}
82 *
83 * @param url
84 * the resource to open
85 * @param support
86 * the support to use to download the resource
87 * @param stable
88 * TRUE for more stable resources, FALSE when they often change
89 * @param originalUrl
90 * the original {@link URL} used to locate the cached resource
91 *
92 * @return the opened resource, NOT NULL
93 *
94 * @throws IOException
95 * in case of I/O error
96 */
97 public InputStream open(URL url, BasicSupport support, boolean stable,
98 URL originalUrl) throws IOException {
99 // MUST NOT return null
100 try {
101 InputStream in = cache.load(originalUrl, false, stable);
102 Instance.trace("Cache " + (in != null ? "hit" : "miss") + ": "
103 + url);
104
105 if (in == null) {
106 try {
107 in = openNoCache(url, support, null, null, null);
108 cache.save(in, originalUrl);
109 // ..But we want a resetable stream
110 in.close();
111 in = cache.load(originalUrl, false, stable);
112 } catch (IOException e) {
113 throw new IOException("Cannot save the url: "
114 + (url == null ? "null" : url.toString()), e);
115 }
116 }
117
118 return in;
119 } catch (IOException e) {
120 throw new IOException("Cannot open the url: "
121 + (url == null ? "null" : url.toString()), e);
122 }
123 }
124
125 /**
126 * Open the given {@link URL} without using the cache, but still update the
127 * cookies.
128 *
129 * @param url
130 * the {@link URL} to open
131 *
132 * @return the {@link InputStream} of the opened page
133 *
134 * @throws IOException
135 * in case of I/O error
136 */
137 public InputStream openNoCache(URL url) throws IOException {
138 return downloader.open(url);
139 }
140
141 /**
142 * Open the given {@link URL} without using the cache, but still using and
143 * updating the cookies.
144 *
145 * @param url
146 * the {@link URL} to open
147 * @param support
148 * the {@link BasicSupport} used for the cookies
149 * @param postParams
150 * the POST parameters
151 * @param getParams
152 * the GET parameters (priority over POST)
153 * @param oauth
154 * OAuth authorization (aka, "bearer XXXXXXX")
155 *
156 * @return the {@link InputStream} of the opened page
157 *
158 * @throws IOException
159 * in case of I/O error
160 */
161 public InputStream openNoCache(URL url, BasicSupport support,
162 Map<String, String> postParams, Map<String, String> getParams,
163 String oauth) throws IOException {
164
165 Map<String, String> cookiesValues = null;
166 URL currentReferer = url;
167 if (support != null) {
168 cookiesValues = support.getCookies();
169 currentReferer = support.getCurrentReferer();
170 // priority: arguments
171 if (oauth == null) {
172 oauth = support.getOAuth();
173 }
174 }
175
176 return downloader.open(url, currentReferer, cookiesValues, postParams,
177 getParams, oauth);
178 }
179
180 /**
181 * Refresh the resource into cache if needed.
182 *
183 * @param url
184 * the resource to open
185 * @param support
186 * the support to use to download the resource
187 * @param stable
188 * TRUE for more stable resources, FALSE when they often change
189 *
190 * @throws IOException
191 * in case of I/O error
192 */
193 public void refresh(URL url, BasicSupport support, boolean stable)
194 throws IOException {
195 if (!cache.check(url, false, stable)) {
196 open(url, support, stable).close();
197 }
198 }
199
200 /**
201 * Check the resource to see if it is in the cache.
202 *
203 * @param url
204 * the resource to check
205 * @param stable
206 * a stable file (that dones't change too often) -- parameter
207 * used to check if the file is too old to keep or not
208 *
209 * @return TRUE if it is
210 *
211 */
212 public boolean check(URL url, boolean stable) {
213 return cache.check(url, false, stable);
214 }
215
216 /**
217 * Save the given resource as an image on disk using the default image
218 * format for content.
219 *
220 * @param url
221 * the resource
222 * @param target
223 * the target file
224 *
225 * @throws IOException
226 * in case of I/O error
227 */
228 public void saveAsImage(URL url, File target) throws IOException {
229 InputStream in = open(url, null, true);
230 try {
231 ImageIO.write(ImageUtils.fromStream(in), Instance.getConfig()
232 .getString(Config.IMAGE_FORMAT_CONTENT).toLowerCase(),
233 target);
234 } catch (IOException e) {
235 throw new IOException("Cannot write image " + url, e);
236 } finally {
237 in.close();
238 }
239 }
240
241 /**
242 * Manually add this item to the cache.
243 *
244 * @param in
245 * the input data
246 * @param uniqueID
247 * a unique ID for this resource
248 *
249 * @return the resulting {@link File}
250 *
251 * @throws IOException
252 * in case of I/O error
253 */
254 public File addToCache(InputStream in, String uniqueID) throws IOException {
255 return cache.save(in, uniqueID);
256 }
257
258 /**
259 * Return the {@link InputStream} corresponding to the given unique ID, or
260 * NULL if none found.
261 *
262 * @param uniqueID
263 * the unique ID
264 *
265 * @return the content or NULL
266 */
267 public InputStream getFromCache(String uniqueID) {
268 return cache.load(uniqueID, true, true);
269 }
270
271 /**
272 * Remove the given resource from the cache.
273 *
274 * @param uniqueID
275 * a unique ID used to locate the cached resource
276 *
277 * @return TRUE if it was removed
278 */
279 public boolean removeFromCache(String uniqueID) {
280 return cache.remove(uniqueID);
281 }
282
283 /**
284 * Clean the cache (delete the cached items).
285 *
286 * @param onlyOld
287 * only clean the files that are considered too old
288 *
289 * @return the number of cleaned items
290 */
291 public int cleanCache(boolean onlyOld) {
292 return cache.clean(onlyOld);
293 }
294}