Commit | Line | Data |
---|---|---|
08fe2e33 NR |
1 | package be.nikiroo.fanfix.supported; |
2 | ||
08fe2e33 NR |
3 | import java.io.IOException; |
4 | import java.io.InputStream; | |
5cf61f35 | 5 | import java.net.MalformedURLException; |
08fe2e33 | 6 | import java.net.URL; |
08fe2e33 | 7 | import java.util.ArrayList; |
793f1071 | 8 | import java.util.Date; |
08fe2e33 NR |
9 | import java.util.HashMap; |
10 | import java.util.List; | |
11 | import java.util.Map; | |
5cf61f35 | 12 | import java.util.Scanner; |
08fe2e33 | 13 | import java.util.Map.Entry; |
0ffa4754 | 14 | |
5cf61f35 | 15 | import org.json.JSONObject; |
0ffa4754 NR |
16 | import org.jsoup.helper.DataUtil; |
17 | import org.jsoup.nodes.Document; | |
18 | import org.jsoup.nodes.Element; | |
19 | import org.jsoup.nodes.Node; | |
08fe2e33 NR |
20 | |
21 | import be.nikiroo.fanfix.Instance; | |
08fe2e33 NR |
22 | import be.nikiroo.fanfix.bundles.StringId; |
23 | import be.nikiroo.fanfix.data.Chapter; | |
24 | import be.nikiroo.fanfix.data.MetaData; | |
9252c65e | 25 | import be.nikiroo.fanfix.data.Story; |
3b2b638f | 26 | import be.nikiroo.utils.Progress; |
08fe2e33 NR |
27 | import be.nikiroo.utils.StringUtils; |
28 | ||
29 | /** | |
30 | * This class is the base class used by the other support classes. It can be | |
31 | * used outside of this package, and have static method that you can use to get | |
32 | * access to the correct support class. | |
33 | * <p> | |
34 | * It will be used with 'resources' (usually web pages or files). | |
35 | * | |
36 | * @author niki | |
37 | */ | |
38 | public abstract class BasicSupport { | |
0ffa4754 NR |
39 | private Document sourceNode; |
40 | private URL source; | |
08fe2e33 | 41 | private SupportType type; |
22848428 | 42 | private URL currentReferer; // with only one 'r', as in 'HTTP'... |
8d59ce07 NR |
43 | |
44 | static protected BasicSupportHelper bsHelper = new BasicSupportHelper(); | |
45 | static protected BasicSupportImages bsImages = new BasicSupportImages(); | |
46 | static protected BasicSupportPara bsPara = new BasicSupportPara(new BasicSupportHelper(), new BasicSupportImages()); | |
08fe2e33 | 47 | |
08fe2e33 NR |
48 | /** |
49 | * Check if the given resource is supported by this {@link BasicSupport}. | |
50 | * | |
51 | * @param url | |
52 | * the resource to check for | |
53 | * | |
54 | * @return TRUE if it is | |
55 | */ | |
56 | protected abstract boolean supports(URL url); | |
57 | ||
58 | /** | |
59 | * Return TRUE if the support will return HTML encoded content values for | |
60 | * the chapters content. | |
61 | * | |
62 | * @return TRUE for HTML | |
63 | */ | |
64 | protected abstract boolean isHtml(); | |
65 | ||
0efd25e3 NR |
66 | /** |
67 | * Return the {@link MetaData} of this story. | |
68 | * | |
776ad3c6 | 69 | * @return the associated {@link MetaData}, never NULL |
0efd25e3 NR |
70 | * |
71 | * @throws IOException | |
72 | * in case of I/O error | |
73 | */ | |
0ffa4754 | 74 | protected abstract MetaData getMeta() throws IOException; |
08fe2e33 NR |
75 | |
76 | /** | |
77 | * Return the story description. | |
78 | * | |
08fe2e33 NR |
79 | * @return the description |
80 | * | |
81 | * @throws IOException | |
82 | * in case of I/O error | |
83 | */ | |
0ffa4754 | 84 | protected abstract String getDesc() throws IOException; |
08fe2e33 | 85 | |
08fe2e33 | 86 | /** |
826e4569 | 87 | * Return the list of chapters (name and resource). |
0ffa4754 NR |
88 | * <p> |
89 | * Can be NULL if this {@link BasicSupport} do no use chapters. | |
08fe2e33 | 90 | * |
ed08c171 NR |
91 | * @param pg |
92 | * the optional progress reporter | |
08fe2e33 | 93 | * |
0ffa4754 | 94 | * @return the chapters or NULL |
08fe2e33 NR |
95 | * |
96 | * @throws IOException | |
97 | * in case of I/O error | |
98 | */ | |
0ffa4754 NR |
99 | protected abstract List<Entry<String, URL>> getChapters(Progress pg) |
100 | throws IOException; | |
08fe2e33 NR |
101 | |
102 | /** | |
103 | * Return the content of the chapter (possibly HTML encoded, if | |
104 | * {@link BasicSupport#isHtml()} is TRUE). | |
105 | * | |
0ffa4754 NR |
106 | * @param chapUrl |
107 | * the chapter {@link URL} | |
08fe2e33 NR |
108 | * @param number |
109 | * the chapter number | |
ed08c171 NR |
110 | * @param pg |
111 | * the optional progress reporter | |
08fe2e33 NR |
112 | * |
113 | * @return the content | |
114 | * | |
115 | * @throws IOException | |
116 | * in case of I/O error | |
117 | */ | |
0ffa4754 NR |
118 | protected abstract String getChapterContent(URL chapUrl, int number, |
119 | Progress pg) throws IOException; | |
6e06d2cc | 120 | |
08fe2e33 NR |
121 | /** |
122 | * Return the list of cookies (values included) that must be used to | |
123 | * correctly fetch the resources. | |
124 | * <p> | |
125 | * You are expected to call the super method implementation if you override | |
126 | * it. | |
127 | * | |
128 | * @return the cookies | |
129 | */ | |
315f14ae | 130 | public Map<String, String> getCookies() { |
08fe2e33 NR |
131 | return new HashMap<String, String>(); |
132 | } | |
133 | ||
315f14ae NR |
134 | /** |
135 | * OAuth authorisation (aka, "bearer XXXXXXX"). | |
136 | * | |
137 | * @return the OAuth string | |
138 | */ | |
139 | public String getOAuth() { | |
140 | return null; | |
141 | } | |
142 | ||
a4143cd7 NR |
143 | /** |
144 | * Return the canonical form of the main {@link URL}. | |
145 | * | |
146 | * @param source | |
0ffa4754 NR |
147 | * the source {@link URL}, which can be NULL |
148 | * | |
149 | * @return the canonical form of this {@link URL} or NULL if the source was | |
150 | * NULL | |
151 | */ | |
152 | protected URL getCanonicalUrl(URL source) { | |
153 | return source; | |
154 | } | |
155 | ||
156 | /** | |
157 | * The main {@link Node} for this {@link Story}. | |
158 | * | |
159 | * @return the node | |
160 | */ | |
161 | protected Element getSourceNode() { | |
162 | return sourceNode; | |
163 | } | |
164 | ||
165 | /** | |
166 | * The main {@link URL} for this {@link Story}. | |
167 | * | |
168 | * @return the URL | |
169 | */ | |
170 | protected URL getSource() { | |
171 | return source; | |
172 | } | |
173 | ||
174 | /** | |
175 | * The current referer {@link URL} (only one 'r', as in 'HTML'...), i.e., | |
176 | * the current {@link URL} we work on. | |
177 | * | |
178 | * @return the referer | |
179 | */ | |
180 | public URL getCurrentReferer() { | |
181 | return currentReferer; | |
182 | } | |
183 | ||
184 | /** | |
185 | * The current referer {@link URL} (only one 'r', as in 'HTML'...), i.e., | |
186 | * the current {@link URL} we work on. | |
187 | * | |
188 | * @param currentReferer | |
189 | * the new referer | |
190 | */ | |
191 | protected void setCurrentReferer(URL currentReferer) { | |
192 | this.currentReferer = currentReferer; | |
193 | } | |
194 | ||
195 | /** | |
196 | * The support type. | |
197 | * | |
198 | * @return the type | |
199 | */ | |
200 | public SupportType getType() { | |
201 | return type; | |
202 | } | |
203 | ||
204 | /** | |
205 | * The support type. | |
206 | * | |
207 | * @param type | |
208 | * the new type | |
209 | */ | |
210 | protected void setType(SupportType type) { | |
211 | this.type = type; | |
212 | } | |
213 | ||
214 | /** | |
215 | * Open an input link that will be used for the support. | |
216 | * <p> | |
7445f856 NR |
217 | * Can return NULL, in which case you are supposed to work without a source |
218 | * node. | |
0ffa4754 NR |
219 | * |
220 | * @param source | |
a4143cd7 NR |
221 | * the source {@link URL} |
222 | * | |
0ffa4754 NR |
223 | * @return the {@link InputStream} |
224 | * | |
225 | * @throws IOException | |
226 | * in case of I/O error | |
227 | */ | |
228 | protected Document loadDocument(URL source) throws IOException { | |
229 | String url = getCanonicalUrl(source).toString(); | |
d66deb8d | 230 | return DataUtil.load(Instance.getInstance().getCache().open(source, this, false), "UTF-8", url.toString()); |
0ffa4754 NR |
231 | } |
232 | ||
233 | /** | |
234 | * Log into the support (can be a no-op depending upon the support). | |
a4143cd7 NR |
235 | * |
236 | * @throws IOException | |
237 | * in case of I/O error | |
238 | */ | |
0ffa4754 NR |
239 | protected void login() throws IOException { |
240 | } | |
241 | ||
0ffa4754 NR |
242 | /** |
243 | * Now that we have processed the {@link Story}, close the resources if any. | |
244 | */ | |
245 | protected void close() { | |
246 | setCurrentReferer(null); | |
a4143cd7 NR |
247 | } |
248 | ||
08fe2e33 NR |
249 | /** |
250 | * Process the given story resource into a partially filled {@link Story} | |
251 | * object containing the name and metadata. | |
252 | * | |
0efd25e3 NR |
253 | * @param getDesc |
254 | * retrieve the description of the story, or not | |
ed08c171 NR |
255 | * @param pg |
256 | * the optional progress reporter | |
08fe2e33 | 257 | * |
776ad3c6 | 258 | * @return the {@link Story}, never NULL |
08fe2e33 NR |
259 | * |
260 | * @throws IOException | |
261 | * in case of I/O error | |
262 | */ | |
0ffa4754 NR |
263 | protected Story processMeta(boolean getDesc, Progress pg) |
264 | throws IOException { | |
ed08c171 NR |
265 | if (pg == null) { |
266 | pg = new Progress(); | |
267 | } else { | |
268 | pg.setMinMax(0, 100); | |
269 | } | |
270 | ||
0ffa4754 | 271 | pg.setProgress(30); |
ed08c171 | 272 | |
0ffa4754 NR |
273 | Story story = new Story(); |
274 | MetaData meta = getMeta(); | |
bff19b54 NR |
275 | if (meta.getCreationDate() == null |
276 | || meta.getCreationDate().trim().isEmpty()) { | |
277 | meta.setCreationDate(bsHelper | |
278 | .formatDate(StringUtils.fromTime(new Date().getTime()))); | |
0ffa4754 NR |
279 | } |
280 | story.setMeta(meta); | |
920af1c7 | 281 | pg.put("meta", meta); |
ed08c171 | 282 | |
0ffa4754 | 283 | pg.setProgress(50); |
08fe2e33 | 284 | |
0ffa4754 | 285 | if (meta.getCover() == null) { |
8d59ce07 | 286 | meta.setCover(bsHelper.getDefaultCover(meta.getSubject())); |
0ffa4754 | 287 | } |
08fe2e33 | 288 | |
0ffa4754 | 289 | pg.setProgress(60); |
a4143cd7 | 290 | |
0ffa4754 | 291 | if (getDesc) { |
d66deb8d NR |
292 | String descChapterName = Instance.getInstance().getTrans().getString(StringId.DESCRIPTION); |
293 | story.getMeta().setResume(bsPara.makeChapter(this, source, 0, descChapterName, // | |
294 | getDesc(), isHtml(), null)); | |
08fe2e33 | 295 | } |
0ffa4754 | 296 | |
fdc55375 | 297 | pg.done(); |
0ffa4754 | 298 | return story; |
08fe2e33 NR |
299 | } |
300 | ||
5cf61f35 NR |
301 | /** |
302 | * Utility method to convert the given URL into a JSON object. | |
303 | * <p> | |
304 | * Note that this method expects small JSON files (everything is copied into | |
305 | * memory at least twice). | |
306 | * | |
307 | * @param url | |
308 | * the URL to parse | |
309 | * @param stable | |
310 | * TRUE for more stable resources, FALSE when they often change | |
311 | * | |
312 | * @return the JSON object | |
313 | * | |
314 | * @throws IOException | |
315 | * in case of I/O error | |
316 | */ | |
317 | protected JSONObject getJson(String url, boolean stable) | |
318 | throws IOException { | |
319 | try { | |
320 | return getJson(new URL(url), stable); | |
321 | } catch (MalformedURLException e) { | |
322 | throw new IOException("Malformed URL: " + url, e); | |
323 | } | |
324 | } | |
325 | ||
326 | /** | |
327 | * Utility method to convert the given URL into a JSON object. | |
328 | * <p> | |
329 | * Note that this method expects small JSON files (everything is copied into | |
330 | * memory at least twice). | |
331 | * | |
332 | * @param url | |
333 | * the URL to parse | |
334 | * @param stable | |
335 | * TRUE for more stable resources, FALSE when they often change | |
336 | * | |
337 | * @return the JSON object | |
338 | * | |
339 | * @throws IOException | |
340 | * in case of I/O error | |
341 | */ | |
342 | protected JSONObject getJson(URL url, boolean stable) throws IOException { | |
343 | InputStream in = Instance.getInstance().getCache().open(url, null, | |
344 | stable); | |
345 | try { | |
346 | Scanner scan = new Scanner(in); | |
347 | scan.useDelimiter("\0"); | |
348 | try { | |
349 | return new JSONObject(scan.next()); | |
350 | } finally { | |
351 | scan.close(); | |
352 | } | |
353 | } finally { | |
354 | in.close(); | |
355 | } | |
356 | } | |
357 | ||
9005532f | 358 | /** |
826e4569 NR |
359 | * Process the given story resource into a fully filled {@link Story} |
360 | * object. | |
9005532f NR |
361 | * |
362 | * @param pg | |
363 | * the optional progress reporter | |
364 | * | |
365 | * @return the {@link Story}, never NULL | |
366 | * | |
367 | * @throws IOException | |
368 | * in case of I/O error | |
369 | */ | |
6569afb4 | 370 | // TODO: ADD final when BasicSupport_Deprecated is gone |
9005532f NR |
371 | public Story process(Progress pg) throws IOException { |
372 | setCurrentReferer(source); | |
373 | login(); | |
374 | sourceNode = loadDocument(source); | |
375 | ||
376 | try { | |
377 | return doProcess(pg); | |
378 | } finally { | |
379 | close(); | |
380 | } | |
381 | } | |
382 | ||
08fe2e33 | 383 | /** |
826e4569 NR |
384 | * Actual processing step, without the calls to other methods. |
385 | * <p> | |
386 | * Will convert the story resource into a fully filled {@link Story} object. | |
08fe2e33 | 387 | * |
92fb0719 NR |
388 | * @param pg |
389 | * the optional progress reporter | |
08fe2e33 | 390 | * |
776ad3c6 | 391 | * @return the {@link Story}, never NULL |
08fe2e33 NR |
392 | * |
393 | * @throws IOException | |
394 | * in case of I/O error | |
395 | */ | |
826e4569 | 396 | protected Story doProcess(Progress pg) throws IOException { |
92fb0719 NR |
397 | if (pg == null) { |
398 | pg = new Progress(); | |
399 | } else { | |
400 | pg.setMinMax(0, 100); | |
401 | } | |
3b039231 NR |
402 | |
403 | pg.setName("Initialising"); | |
92fb0719 | 404 | |
92fb0719 | 405 | pg.setProgress(1); |
9005532f NR |
406 | Progress pgMeta = new Progress(); |
407 | pg.addProgress(pgMeta, 10); | |
408 | Story story = processMeta(true, pgMeta); | |
68328e17 | 409 | pgMeta.done(); // 10% |
920af1c7 | 410 | pg.put("meta", story.getMeta()); |
ed08c171 | 411 | |
9005532f NR |
412 | Progress pgGetChapters = new Progress(); |
413 | pg.addProgress(pgGetChapters, 10); | |
414 | story.setChapters(new ArrayList<Chapter>()); | |
415 | List<Entry<String, URL>> chapters = getChapters(pgGetChapters); | |
68328e17 | 416 | pgGetChapters.done(); // 20% |
9005532f NR |
417 | |
418 | if (chapters != null) { | |
419 | Progress pgChaps = new Progress("Extracting chapters", 0, | |
420 | chapters.size() * 300); | |
421 | pg.addProgress(pgChaps, 80); | |
422 | ||
423 | long words = 0; | |
424 | int i = 1; | |
425 | for (Entry<String, URL> chap : chapters) { | |
426 | pgChaps.setName("Extracting chapter " + i); | |
427 | URL chapUrl = chap.getValue(); | |
428 | String chapName = chap.getKey(); | |
429 | if (chapUrl != null) { | |
430 | setCurrentReferer(chapUrl); | |
431 | } | |
432 | ||
433 | pgChaps.setProgress(i * 100); | |
434 | Progress pgGetChapterContent = new Progress(); | |
435 | Progress pgMakeChapter = new Progress(); | |
436 | pgChaps.addProgress(pgGetChapterContent, 100); | |
437 | pgChaps.addProgress(pgMakeChapter, 100); | |
438 | ||
439 | String content = getChapterContent(chapUrl, i, | |
440 | pgGetChapterContent); | |
68328e17 | 441 | pgGetChapterContent.done(); |
8d59ce07 | 442 | Chapter cc = bsPara.makeChapter(this, chapUrl, i, |
9005532f | 443 | chapName, content, isHtml(), pgMakeChapter); |
68328e17 | 444 | pgMakeChapter.done(); |
ed08c171 | 445 | |
9005532f NR |
446 | words += cc.getWords(); |
447 | story.getChapters().add(cc); | |
448 | story.getMeta().setWords(words); | |
449 | ||
450 | i++; | |
08fe2e33 NR |
451 | } |
452 | ||
9005532f | 453 | pgChaps.setName("Extracting chapters"); |
fdc55375 | 454 | pgChaps.done(); |
08fe2e33 | 455 | } |
9005532f | 456 | |
68328e17 NR |
457 | pg.done(); |
458 | ||
9005532f | 459 | return story; |
08fe2e33 NR |
460 | } |
461 | ||
99d71bd7 NR |
462 | /** |
463 | * Create a chapter from the given data. | |
464 | * | |
465 | * @param source | |
466 | * the source URL for this content, which can be used to try and | |
467 | * find images if images are present in the format [image-url] | |
468 | * @param number | |
469 | * the chapter number (0 = description) | |
470 | * @param name | |
471 | * the chapter name | |
472 | * @param content | |
473 | * the content of the chapter | |
474 | * @return the {@link Chapter} | |
475 | * | |
476 | * @throws IOException | |
477 | * in case of I/O error | |
478 | */ | |
479 | public Chapter makeChapter(URL source, int number, String name, | |
480 | String content) throws IOException { | |
8d59ce07 | 481 | return bsPara.makeChapter(this, source, number, name, |
99d71bd7 NR |
482 | content, isHtml(), null); |
483 | } | |
484 | ||
08fe2e33 | 485 | /** |
0ffa4754 NR |
486 | * Return a {@link BasicSupport} implementation supporting the given |
487 | * resource if possible. | |
08fe2e33 | 488 | * |
0ffa4754 NR |
489 | * @param url |
490 | * the story resource | |
08fe2e33 | 491 | * |
0ffa4754 | 492 | * @return an implementation that supports it, or NULL |
08fe2e33 | 493 | */ |
0ffa4754 NR |
494 | public static BasicSupport getSupport(URL url) { |
495 | if (url == null) { | |
496 | return null; | |
497 | } | |
08fe2e33 | 498 | |
0ffa4754 NR |
499 | // TEXT and INFO_TEXT always support files (not URLs though) |
500 | for (SupportType type : SupportType.values()) { | |
501 | if (type != SupportType.TEXT && type != SupportType.INFO_TEXT) { | |
502 | BasicSupport support = getSupport(type, url); | |
503 | if (support != null && support.supports(url)) { | |
504 | return support; | |
505 | } | |
506 | } | |
507 | } | |
08fe2e33 | 508 | |
0ffa4754 NR |
509 | for (SupportType type : new SupportType[] { SupportType.INFO_TEXT, |
510 | SupportType.TEXT }) { | |
511 | BasicSupport support = getSupport(type, url); | |
512 | if (support != null && support.supports(url)) { | |
513 | return support; | |
514 | } | |
515 | } | |
516 | ||
517 | return null; | |
08fe2e33 NR |
518 | } |
519 | ||
520 | /** | |
0ffa4754 | 521 | * Return a {@link BasicSupport} implementation supporting the given type. |
08fe2e33 | 522 | * |
0ffa4754 | 523 | * @param type |
99d71bd7 | 524 | * the type, must not be NULL |
0ffa4754 NR |
525 | * @param url |
526 | * the {@link URL} to support (can be NULL to get an | |
727108fe NR |
527 | * "abstract support"; if not NULL, will be used as the source |
528 | * URL) | |
08fe2e33 | 529 | * |
0ffa4754 | 530 | * @return an implementation that supports it, or NULL |
08fe2e33 | 531 | */ |
0ffa4754 NR |
532 | public static BasicSupport getSupport(SupportType type, URL url) { |
533 | BasicSupport support = null; | |
08fe2e33 | 534 | |
08fe2e33 NR |
535 | switch (type) { |
536 | case EPUB: | |
0ffa4754 NR |
537 | support = new Epub(); |
538 | break; | |
08fe2e33 | 539 | case INFO_TEXT: |
0ffa4754 NR |
540 | support = new InfoText(); |
541 | break; | |
08fe2e33 | 542 | case FIMFICTION: |
315f14ae NR |
543 | try { |
544 | // Can fail if no client key or NO in options | |
0ffa4754 | 545 | support = new FimfictionApi(); |
315f14ae | 546 | } catch (IOException e) { |
0ffa4754 | 547 | support = new Fimfiction(); |
315f14ae | 548 | } |
0ffa4754 | 549 | break; |
08fe2e33 | 550 | case FANFICTION: |
0ffa4754 NR |
551 | support = new Fanfiction(); |
552 | break; | |
08fe2e33 | 553 | case TEXT: |
0ffa4754 NR |
554 | support = new Text(); |
555 | break; | |
413bcc29 NR |
556 | case MANGAHUB: |
557 | support = new MangaHub(); | |
0ffa4754 | 558 | break; |
08fe2e33 | 559 | case E621: |
0ffa4754 NR |
560 | support = new E621(); |
561 | break; | |
a4143cd7 | 562 | case YIFFSTAR: |
0ffa4754 NR |
563 | support = new YiffStar(); |
564 | break; | |
f0608ab1 | 565 | case E_HENTAI: |
0ffa4754 NR |
566 | support = new EHentai(); |
567 | break; | |
af1f506f NR |
568 | case MANGA_LEL: |
569 | support = new MangaLel(); | |
570 | break; | |
08fe2e33 | 571 | case CBZ: |
0ffa4754 NR |
572 | support = new Cbz(); |
573 | break; | |
373da363 | 574 | case HTML: |
0ffa4754 NR |
575 | support = new Html(); |
576 | break; | |
68686a37 NR |
577 | } |
578 | ||
0ffa4754 NR |
579 | if (support != null) { |
580 | support.setType(type); | |
581 | support.source = support.getCanonicalUrl(url); | |
315f14ae NR |
582 | } |
583 | ||
0ffa4754 | 584 | return support; |
315f14ae | 585 | } |
08fe2e33 | 586 | } |