Commit | Line | Data |
---|---|---|
08fe2e33 NR |
1 | package be.nikiroo.fanfix.supported; |
2 | ||
08fe2e33 NR |
3 | import java.io.IOException; |
4 | import java.io.InputStream; | |
08fe2e33 | 5 | import java.net.URL; |
08fe2e33 | 6 | import java.util.ArrayList; |
793f1071 | 7 | import java.util.Date; |
08fe2e33 NR |
8 | import java.util.HashMap; |
9 | import java.util.List; | |
10 | import java.util.Map; | |
11 | import java.util.Map.Entry; | |
0ffa4754 NR |
12 | |
13 | import org.jsoup.helper.DataUtil; | |
14 | import org.jsoup.nodes.Document; | |
15 | import org.jsoup.nodes.Element; | |
16 | import org.jsoup.nodes.Node; | |
08fe2e33 NR |
17 | |
18 | import be.nikiroo.fanfix.Instance; | |
08fe2e33 NR |
19 | import be.nikiroo.fanfix.bundles.StringId; |
20 | import be.nikiroo.fanfix.data.Chapter; | |
21 | import be.nikiroo.fanfix.data.MetaData; | |
9252c65e | 22 | import be.nikiroo.fanfix.data.Story; |
3b2b638f | 23 | import be.nikiroo.utils.Progress; |
08fe2e33 NR |
24 | import be.nikiroo.utils.StringUtils; |
25 | ||
26 | /** | |
27 | * This class is the base class used by the other support classes. It can be | |
28 | * used outside of this package, and have static method that you can use to get | |
29 | * access to the correct support class. | |
30 | * <p> | |
31 | * It will be used with 'resources' (usually web pages or files). | |
32 | * | |
33 | * @author niki | |
34 | */ | |
35 | public abstract class BasicSupport { | |
0ffa4754 NR |
36 | private Document sourceNode; |
37 | private URL source; | |
08fe2e33 | 38 | private SupportType type; |
22848428 | 39 | private URL currentReferer; // with only one 'r', as in 'HTTP'... |
8d59ce07 NR |
40 | |
41 | static protected BasicSupportHelper bsHelper = new BasicSupportHelper(); | |
42 | static protected BasicSupportImages bsImages = new BasicSupportImages(); | |
43 | static protected BasicSupportPara bsPara = new BasicSupportPara(new BasicSupportHelper(), new BasicSupportImages()); | |
08fe2e33 | 44 | |
08fe2e33 NR |
45 | /** |
46 | * Check if the given resource is supported by this {@link BasicSupport}. | |
47 | * | |
48 | * @param url | |
49 | * the resource to check for | |
50 | * | |
51 | * @return TRUE if it is | |
52 | */ | |
53 | protected abstract boolean supports(URL url); | |
54 | ||
55 | /** | |
56 | * Return TRUE if the support will return HTML encoded content values for | |
57 | * the chapters content. | |
58 | * | |
59 | * @return TRUE for HTML | |
60 | */ | |
61 | protected abstract boolean isHtml(); | |
62 | ||
0efd25e3 NR |
63 | /** |
64 | * Return the {@link MetaData} of this story. | |
65 | * | |
776ad3c6 | 66 | * @return the associated {@link MetaData}, never NULL |
0efd25e3 NR |
67 | * |
68 | * @throws IOException | |
69 | * in case of I/O error | |
70 | */ | |
0ffa4754 | 71 | protected abstract MetaData getMeta() throws IOException; |
08fe2e33 NR |
72 | |
73 | /** | |
74 | * Return the story description. | |
75 | * | |
08fe2e33 NR |
76 | * @return the description |
77 | * | |
78 | * @throws IOException | |
79 | * in case of I/O error | |
80 | */ | |
0ffa4754 | 81 | protected abstract String getDesc() throws IOException; |
08fe2e33 | 82 | |
08fe2e33 | 83 | /** |
826e4569 | 84 | * Return the list of chapters (name and resource). |
0ffa4754 NR |
85 | * <p> |
86 | * Can be NULL if this {@link BasicSupport} do no use chapters. | |
08fe2e33 | 87 | * |
ed08c171 NR |
88 | * @param pg |
89 | * the optional progress reporter | |
08fe2e33 | 90 | * |
0ffa4754 | 91 | * @return the chapters or NULL |
08fe2e33 NR |
92 | * |
93 | * @throws IOException | |
94 | * in case of I/O error | |
95 | */ | |
0ffa4754 NR |
96 | protected abstract List<Entry<String, URL>> getChapters(Progress pg) |
97 | throws IOException; | |
08fe2e33 NR |
98 | |
99 | /** | |
100 | * Return the content of the chapter (possibly HTML encoded, if | |
101 | * {@link BasicSupport#isHtml()} is TRUE). | |
102 | * | |
0ffa4754 NR |
103 | * @param chapUrl |
104 | * the chapter {@link URL} | |
08fe2e33 NR |
105 | * @param number |
106 | * the chapter number | |
ed08c171 NR |
107 | * @param pg |
108 | * the optional progress reporter | |
08fe2e33 NR |
109 | * |
110 | * @return the content | |
111 | * | |
112 | * @throws IOException | |
113 | * in case of I/O error | |
114 | */ | |
0ffa4754 NR |
115 | protected abstract String getChapterContent(URL chapUrl, int number, |
116 | Progress pg) throws IOException; | |
6e06d2cc | 117 | |
08fe2e33 NR |
118 | /** |
119 | * Return the list of cookies (values included) that must be used to | |
120 | * correctly fetch the resources. | |
121 | * <p> | |
122 | * You are expected to call the super method implementation if you override | |
123 | * it. | |
124 | * | |
125 | * @return the cookies | |
126 | */ | |
315f14ae | 127 | public Map<String, String> getCookies() { |
08fe2e33 NR |
128 | return new HashMap<String, String>(); |
129 | } | |
130 | ||
315f14ae NR |
131 | /** |
132 | * OAuth authorisation (aka, "bearer XXXXXXX"). | |
133 | * | |
134 | * @return the OAuth string | |
135 | */ | |
136 | public String getOAuth() { | |
137 | return null; | |
138 | } | |
139 | ||
a4143cd7 NR |
140 | /** |
141 | * Return the canonical form of the main {@link URL}. | |
142 | * | |
143 | * @param source | |
0ffa4754 NR |
144 | * the source {@link URL}, which can be NULL |
145 | * | |
146 | * @return the canonical form of this {@link URL} or NULL if the source was | |
147 | * NULL | |
148 | */ | |
149 | protected URL getCanonicalUrl(URL source) { | |
150 | return source; | |
151 | } | |
152 | ||
153 | /** | |
154 | * The main {@link Node} for this {@link Story}. | |
155 | * | |
156 | * @return the node | |
157 | */ | |
158 | protected Element getSourceNode() { | |
159 | return sourceNode; | |
160 | } | |
161 | ||
162 | /** | |
163 | * The main {@link URL} for this {@link Story}. | |
164 | * | |
165 | * @return the URL | |
166 | */ | |
167 | protected URL getSource() { | |
168 | return source; | |
169 | } | |
170 | ||
171 | /** | |
172 | * The current referer {@link URL} (only one 'r', as in 'HTML'...), i.e., | |
173 | * the current {@link URL} we work on. | |
174 | * | |
175 | * @return the referer | |
176 | */ | |
177 | public URL getCurrentReferer() { | |
178 | return currentReferer; | |
179 | } | |
180 | ||
181 | /** | |
182 | * The current referer {@link URL} (only one 'r', as in 'HTML'...), i.e., | |
183 | * the current {@link URL} we work on. | |
184 | * | |
185 | * @param currentReferer | |
186 | * the new referer | |
187 | */ | |
188 | protected void setCurrentReferer(URL currentReferer) { | |
189 | this.currentReferer = currentReferer; | |
190 | } | |
191 | ||
192 | /** | |
193 | * The support type. | |
194 | * | |
195 | * @return the type | |
196 | */ | |
197 | public SupportType getType() { | |
198 | return type; | |
199 | } | |
200 | ||
201 | /** | |
202 | * The support type. | |
203 | * | |
204 | * @param type | |
205 | * the new type | |
206 | */ | |
207 | protected void setType(SupportType type) { | |
208 | this.type = type; | |
209 | } | |
210 | ||
211 | /** | |
212 | * Open an input link that will be used for the support. | |
213 | * <p> | |
7445f856 NR |
214 | * Can return NULL, in which case you are supposed to work without a source |
215 | * node. | |
0ffa4754 NR |
216 | * |
217 | * @param source | |
a4143cd7 NR |
218 | * the source {@link URL} |
219 | * | |
0ffa4754 NR |
220 | * @return the {@link InputStream} |
221 | * | |
222 | * @throws IOException | |
223 | * in case of I/O error | |
224 | */ | |
225 | protected Document loadDocument(URL source) throws IOException { | |
226 | String url = getCanonicalUrl(source).toString(); | |
227 | return DataUtil.load(Instance.getCache().open(source, this, false), | |
228 | "UTF-8", url.toString()); | |
229 | } | |
230 | ||
231 | /** | |
232 | * Log into the support (can be a no-op depending upon the support). | |
a4143cd7 NR |
233 | * |
234 | * @throws IOException | |
235 | * in case of I/O error | |
236 | */ | |
0ffa4754 NR |
237 | protected void login() throws IOException { |
238 | } | |
239 | ||
0ffa4754 NR |
240 | /** |
241 | * Now that we have processed the {@link Story}, close the resources if any. | |
242 | */ | |
243 | protected void close() { | |
244 | setCurrentReferer(null); | |
a4143cd7 NR |
245 | } |
246 | ||
08fe2e33 NR |
247 | /** |
248 | * Process the given story resource into a partially filled {@link Story} | |
249 | * object containing the name and metadata. | |
250 | * | |
0efd25e3 NR |
251 | * @param getDesc |
252 | * retrieve the description of the story, or not | |
ed08c171 NR |
253 | * @param pg |
254 | * the optional progress reporter | |
08fe2e33 | 255 | * |
776ad3c6 | 256 | * @return the {@link Story}, never NULL |
08fe2e33 NR |
257 | * |
258 | * @throws IOException | |
259 | * in case of I/O error | |
260 | */ | |
0ffa4754 NR |
261 | protected Story processMeta(boolean getDesc, Progress pg) |
262 | throws IOException { | |
ed08c171 NR |
263 | if (pg == null) { |
264 | pg = new Progress(); | |
265 | } else { | |
266 | pg.setMinMax(0, 100); | |
267 | } | |
268 | ||
0ffa4754 | 269 | pg.setProgress(30); |
ed08c171 | 270 | |
0ffa4754 NR |
271 | Story story = new Story(); |
272 | MetaData meta = getMeta(); | |
273 | if (meta.getCreationDate() == null || meta.getCreationDate().isEmpty()) { | |
274 | meta.setCreationDate(StringUtils.fromTime(new Date().getTime())); | |
275 | } | |
276 | story.setMeta(meta); | |
ed08c171 | 277 | |
0ffa4754 | 278 | pg.setProgress(50); |
08fe2e33 | 279 | |
0ffa4754 | 280 | if (meta.getCover() == null) { |
8d59ce07 | 281 | meta.setCover(bsHelper.getDefaultCover(meta.getSubject())); |
0ffa4754 | 282 | } |
08fe2e33 | 283 | |
0ffa4754 | 284 | pg.setProgress(60); |
a4143cd7 | 285 | |
0ffa4754 NR |
286 | if (getDesc) { |
287 | String descChapterName = Instance.getTrans().getString( | |
288 | StringId.DESCRIPTION); | |
289 | story.getMeta().setResume( | |
8d59ce07 | 290 | bsPara.makeChapter(this, source, 0, |
0ffa4754 NR |
291 | descChapterName, // |
292 | getDesc(), isHtml(), null)); | |
08fe2e33 | 293 | } |
0ffa4754 | 294 | |
fdc55375 | 295 | pg.done(); |
0ffa4754 | 296 | return story; |
08fe2e33 NR |
297 | } |
298 | ||
9005532f | 299 | /** |
826e4569 NR |
300 | * Process the given story resource into a fully filled {@link Story} |
301 | * object. | |
9005532f NR |
302 | * |
303 | * @param pg | |
304 | * the optional progress reporter | |
305 | * | |
306 | * @return the {@link Story}, never NULL | |
307 | * | |
308 | * @throws IOException | |
309 | * in case of I/O error | |
310 | */ | |
6569afb4 | 311 | // TODO: ADD final when BasicSupport_Deprecated is gone |
9005532f NR |
312 | public Story process(Progress pg) throws IOException { |
313 | setCurrentReferer(source); | |
314 | login(); | |
315 | sourceNode = loadDocument(source); | |
316 | ||
317 | try { | |
318 | return doProcess(pg); | |
319 | } finally { | |
320 | close(); | |
321 | } | |
322 | } | |
323 | ||
08fe2e33 | 324 | /** |
826e4569 NR |
325 | * Actual processing step, without the calls to other methods. |
326 | * <p> | |
327 | * Will convert the story resource into a fully filled {@link Story} object. | |
08fe2e33 | 328 | * |
92fb0719 NR |
329 | * @param pg |
330 | * the optional progress reporter | |
08fe2e33 | 331 | * |
776ad3c6 | 332 | * @return the {@link Story}, never NULL |
08fe2e33 NR |
333 | * |
334 | * @throws IOException | |
335 | * in case of I/O error | |
336 | */ | |
826e4569 | 337 | protected Story doProcess(Progress pg) throws IOException { |
92fb0719 NR |
338 | if (pg == null) { |
339 | pg = new Progress(); | |
340 | } else { | |
341 | pg.setMinMax(0, 100); | |
342 | } | |
343 | ||
92fb0719 | 344 | pg.setProgress(1); |
9005532f NR |
345 | Progress pgMeta = new Progress(); |
346 | pg.addProgress(pgMeta, 10); | |
347 | Story story = processMeta(true, pgMeta); | |
68328e17 | 348 | pgMeta.done(); // 10% |
ed08c171 | 349 | |
9005532f | 350 | pg.setName("Retrieving " + story.getMeta().getTitle()); |
754a5bc2 | 351 | |
9005532f NR |
352 | Progress pgGetChapters = new Progress(); |
353 | pg.addProgress(pgGetChapters, 10); | |
354 | story.setChapters(new ArrayList<Chapter>()); | |
355 | List<Entry<String, URL>> chapters = getChapters(pgGetChapters); | |
68328e17 | 356 | pgGetChapters.done(); // 20% |
9005532f NR |
357 | |
358 | if (chapters != null) { | |
359 | Progress pgChaps = new Progress("Extracting chapters", 0, | |
360 | chapters.size() * 300); | |
361 | pg.addProgress(pgChaps, 80); | |
362 | ||
363 | long words = 0; | |
364 | int i = 1; | |
365 | for (Entry<String, URL> chap : chapters) { | |
366 | pgChaps.setName("Extracting chapter " + i); | |
367 | URL chapUrl = chap.getValue(); | |
368 | String chapName = chap.getKey(); | |
369 | if (chapUrl != null) { | |
370 | setCurrentReferer(chapUrl); | |
371 | } | |
372 | ||
373 | pgChaps.setProgress(i * 100); | |
374 | Progress pgGetChapterContent = new Progress(); | |
375 | Progress pgMakeChapter = new Progress(); | |
376 | pgChaps.addProgress(pgGetChapterContent, 100); | |
377 | pgChaps.addProgress(pgMakeChapter, 100); | |
378 | ||
379 | String content = getChapterContent(chapUrl, i, | |
380 | pgGetChapterContent); | |
68328e17 | 381 | pgGetChapterContent.done(); |
8d59ce07 | 382 | Chapter cc = bsPara.makeChapter(this, chapUrl, i, |
9005532f | 383 | chapName, content, isHtml(), pgMakeChapter); |
68328e17 | 384 | pgMakeChapter.done(); |
ed08c171 | 385 | |
9005532f NR |
386 | words += cc.getWords(); |
387 | story.getChapters().add(cc); | |
388 | story.getMeta().setWords(words); | |
389 | ||
390 | i++; | |
08fe2e33 NR |
391 | } |
392 | ||
9005532f | 393 | pgChaps.setName("Extracting chapters"); |
fdc55375 | 394 | pgChaps.done(); |
08fe2e33 | 395 | } |
9005532f | 396 | |
68328e17 NR |
397 | pg.done(); |
398 | ||
9005532f | 399 | return story; |
08fe2e33 NR |
400 | } |
401 | ||
99d71bd7 NR |
402 | /** |
403 | * Create a chapter from the given data. | |
404 | * | |
405 | * @param source | |
406 | * the source URL for this content, which can be used to try and | |
407 | * find images if images are present in the format [image-url] | |
408 | * @param number | |
409 | * the chapter number (0 = description) | |
410 | * @param name | |
411 | * the chapter name | |
412 | * @param content | |
413 | * the content of the chapter | |
414 | * @return the {@link Chapter} | |
415 | * | |
416 | * @throws IOException | |
417 | * in case of I/O error | |
418 | */ | |
419 | public Chapter makeChapter(URL source, int number, String name, | |
420 | String content) throws IOException { | |
8d59ce07 | 421 | return bsPara.makeChapter(this, source, number, name, |
99d71bd7 NR |
422 | content, isHtml(), null); |
423 | } | |
424 | ||
08fe2e33 | 425 | /** |
0ffa4754 NR |
426 | * Return a {@link BasicSupport} implementation supporting the given |
427 | * resource if possible. | |
08fe2e33 | 428 | * |
0ffa4754 NR |
429 | * @param url |
430 | * the story resource | |
08fe2e33 | 431 | * |
0ffa4754 | 432 | * @return an implementation that supports it, or NULL |
08fe2e33 | 433 | */ |
0ffa4754 NR |
434 | public static BasicSupport getSupport(URL url) { |
435 | if (url == null) { | |
436 | return null; | |
437 | } | |
08fe2e33 | 438 | |
0ffa4754 NR |
439 | // TEXT and INFO_TEXT always support files (not URLs though) |
440 | for (SupportType type : SupportType.values()) { | |
441 | if (type != SupportType.TEXT && type != SupportType.INFO_TEXT) { | |
442 | BasicSupport support = getSupport(type, url); | |
443 | if (support != null && support.supports(url)) { | |
444 | return support; | |
445 | } | |
446 | } | |
447 | } | |
08fe2e33 | 448 | |
0ffa4754 NR |
449 | for (SupportType type : new SupportType[] { SupportType.INFO_TEXT, |
450 | SupportType.TEXT }) { | |
451 | BasicSupport support = getSupport(type, url); | |
452 | if (support != null && support.supports(url)) { | |
453 | return support; | |
454 | } | |
455 | } | |
456 | ||
457 | return null; | |
08fe2e33 NR |
458 | } |
459 | ||
460 | /** | |
0ffa4754 | 461 | * Return a {@link BasicSupport} implementation supporting the given type. |
08fe2e33 | 462 | * |
0ffa4754 | 463 | * @param type |
99d71bd7 | 464 | * the type, must not be NULL |
0ffa4754 NR |
465 | * @param url |
466 | * the {@link URL} to support (can be NULL to get an | |
727108fe NR |
467 | * "abstract support"; if not NULL, will be used as the source |
468 | * URL) | |
08fe2e33 | 469 | * |
0ffa4754 | 470 | * @return an implementation that supports it, or NULL |
08fe2e33 | 471 | */ |
0ffa4754 NR |
472 | public static BasicSupport getSupport(SupportType type, URL url) { |
473 | BasicSupport support = null; | |
08fe2e33 | 474 | |
08fe2e33 NR |
475 | switch (type) { |
476 | case EPUB: | |
0ffa4754 NR |
477 | support = new Epub(); |
478 | break; | |
08fe2e33 | 479 | case INFO_TEXT: |
0ffa4754 NR |
480 | support = new InfoText(); |
481 | break; | |
08fe2e33 | 482 | case FIMFICTION: |
315f14ae NR |
483 | try { |
484 | // Can fail if no client key or NO in options | |
0ffa4754 | 485 | support = new FimfictionApi(); |
315f14ae | 486 | } catch (IOException e) { |
0ffa4754 | 487 | support = new Fimfiction(); |
315f14ae | 488 | } |
0ffa4754 | 489 | break; |
08fe2e33 | 490 | case FANFICTION: |
0ffa4754 NR |
491 | support = new Fanfiction(); |
492 | break; | |
08fe2e33 | 493 | case TEXT: |
0ffa4754 NR |
494 | support = new Text(); |
495 | break; | |
08fe2e33 | 496 | case MANGAFOX: |
0ffa4754 NR |
497 | support = new MangaFox(); |
498 | break; | |
08fe2e33 | 499 | case E621: |
0ffa4754 NR |
500 | support = new E621(); |
501 | break; | |
a4143cd7 | 502 | case YIFFSTAR: |
0ffa4754 NR |
503 | support = new YiffStar(); |
504 | break; | |
f0608ab1 | 505 | case E_HENTAI: |
0ffa4754 NR |
506 | support = new EHentai(); |
507 | break; | |
af1f506f NR |
508 | case MANGA_LEL: |
509 | support = new MangaLel(); | |
510 | break; | |
08fe2e33 | 511 | case CBZ: |
0ffa4754 NR |
512 | support = new Cbz(); |
513 | break; | |
373da363 | 514 | case HTML: |
0ffa4754 NR |
515 | support = new Html(); |
516 | break; | |
68686a37 NR |
517 | } |
518 | ||
0ffa4754 NR |
519 | if (support != null) { |
520 | support.setType(type); | |
521 | support.source = support.getCanonicalUrl(url); | |
315f14ae NR |
522 | } |
523 | ||
0ffa4754 | 524 | return support; |
315f14ae | 525 | } |
08fe2e33 | 526 | } |