supported/BasicSupport.java

   1 package be.nikiroo.fanfix.supported;
   2
   3 import java.io.IOException;
   4 import java.io.InputStream;
   5 import java.net.URL;
   6 import java.util.ArrayList;
   7 import java.util.Date;
   8 import java.util.HashMap;
   9 import java.util.List;
  10 import java.util.Map;
  11 import java.util.Map.Entry;
  12
  13 import org.jsoup.helper.DataUtil;
  14 import org.jsoup.nodes.Document;
  15 import org.jsoup.nodes.Element;
  16 import org.jsoup.nodes.Node;
  17
  18 import be.nikiroo.fanfix.Instance;
  19 import be.nikiroo.fanfix.bundles.StringId;
  20 import be.nikiroo.fanfix.data.Chapter;
  21 import be.nikiroo.fanfix.data.MetaData;
  22 import be.nikiroo.fanfix.data.Story;
  23 import be.nikiroo.utils.Progress;
  24 import be.nikiroo.utils.StringUtils;
  25
  26 /**
  27  * This class is the base class used by the other support classes. It can be
  28  * used outside of this package, and have static method that you can use to get
  29  * access to the correct support class.
  30  * <p>
  31  * It will be used with 'resources' (usually web pages or files).
  32  *
  33  * @author niki
  34  */
  35 public abstract class BasicSupport {
  36         private Document sourceNode;
  37         private URL source;
  38         private SupportType type;
  39         private URL currentReferer; // with only one 'r', as in 'HTTP'...
  40
  41         static protected BasicSupportHelper bsHelper = new BasicSupportHelper();
  42         static protected BasicSupportImages bsImages = new BasicSupportImages();
  43         static protected BasicSupportPara bsPara = new BasicSupportPara(new BasicSupportHelper(), new BasicSupportImages());
  44
  45         /**
  46          * Check if the given resource is supported by this {@link BasicSupport}.
  47          *
  48          * @param url
  49          *            the resource to check for
  50          *
  51          * @return TRUE if it is
  52          */
  53         protected abstract boolean supports(URL url);
  54
  55         /**
  56          * Return TRUE if the support will return HTML encoded content values for
  57          * the chapters content.
  58          *
  59          * @return TRUE for HTML
  60          */
  61         protected abstract boolean isHtml();
  62
  63         /**
  64          * Return the {@link MetaData} of this story.
  65          *
  66          * @return the associated {@link MetaData}, never NULL
  67          *
  68          * @throws IOException
  69          *             in case of I/O error
  70          */
  71         protected abstract MetaData getMeta() throws IOException;
  72
  73         /**
  74          * Return the story description.
  75          *
  76          * @return the description
  77          *
  78          * @throws IOException
  79          *             in case of I/O error
  80          */
  81         protected abstract String getDesc() throws IOException;
  82
  83         /**
  84          * Return the list of chapters (name and resource).
  85          * <p>
  86          * Can be NULL if this {@link BasicSupport} do no use chapters.
  87          *
  88          * @param pg
  89          *            the optional progress reporter
  90          *
  91          * @return the chapters or NULL
  92          *
  93          * @throws IOException
  94          *             in case of I/O error
  95          */
  96         protected abstract List<Entry<String, URL>> getChapters(Progress pg)
  97                         throws IOException;
  98
  99         /**
 100          * Return the content of the chapter (possibly HTML encoded, if
 101          * {@link BasicSupport#isHtml()} is TRUE).
 102          *
 103          * @param chapUrl
 104          *            the chapter {@link URL}
 105          * @param number
 106          *            the chapter number
 107          * @param pg
 108          *            the optional progress reporter
 109          *
 110          * @return the content
 111          *
 112          * @throws IOException
 113          *             in case of I/O error
 114          */
 115         protected abstract String getChapterContent(URL chapUrl, int number,
 116                         Progress pg) throws IOException;
 117
 118         /**
 119          * Return the list of cookies (values included) that must be used to
 120          * correctly fetch the resources.
 121          * <p>
 122          * You are expected to call the super method implementation if you override
 123          * it.
 124          *
 125          * @return the cookies
 126          */
 127         public Map<String, String> getCookies() {
 128                 return new HashMap<String, String>();
 129         }
 130
 131         /**
 132          * OAuth authorisation (aka, "bearer XXXXXXX").
 133          *
 134          * @return the OAuth string
 135          */
 136         public String getOAuth() {
 137                 return null;
 138         }
 139
 140         /**
 141          * Return the canonical form of the main {@link URL}.
 142          *
 143          * @param source
 144          *            the source {@link URL}, which can be NULL
 145          *
 146          * @return the canonical form of this {@link URL} or NULL if the source was
 147          *         NULL
 148          */
 149         protected URL getCanonicalUrl(URL source) {
 150                 return source;
 151         }
 152
 153         /**
 154          * The main {@link Node} for this {@link Story}.
 155          *
 156          * @return the node
 157          */
 158         protected Element getSourceNode() {
 159                 return sourceNode;
 160         }
 161
 162         /**
 163          * The main {@link URL} for this {@link Story}.
 164          *
 165          * @return the URL
 166          */
 167         protected URL getSource() {
 168                 return source;
 169         }
 170
 171         /**
 172          * The current referer {@link URL} (only one 'r', as in 'HTML'...), i.e.,
 173          * the current {@link URL} we work on.
 174          *
 175          * @return the referer
 176          */
 177         public URL getCurrentReferer() {
 178                 return currentReferer;
 179         }
 180
 181         /**
 182          * The current referer {@link URL} (only one 'r', as in 'HTML'...), i.e.,
 183          * the current {@link URL} we work on.
 184          *
 185          * @param currentReferer
 186          *            the new referer
 187          */
 188         protected void setCurrentReferer(URL currentReferer) {
 189                 this.currentReferer = currentReferer;
 190         }
 191
 192         /**
 193          * The support type.
 194          *
 195          * @return the type
 196          */
 197         public SupportType getType() {
 198                 return type;
 199         }
 200
 201         /**
 202          * The support type.
 203          *
 204          * @param type
 205          *            the new type
 206          */
 207         protected void setType(SupportType type) {
 208                 this.type = type;
 209         }
 210
 211         /**
 212          * Open an input link that will be used for the support.
 213          * <p>
 214          * Can return NULL, in which case you are supposed to work without a source
 215          * node.
 216          *
 217          * @param source
 218          *            the source {@link URL}
 219          *
 220          * @return the {@link InputStream}
 221          *
 222          * @throws IOException
 223          *             in case of I/O error
 224          */
 225         protected Document loadDocument(URL source) throws IOException {
 226                 String url = getCanonicalUrl(source).toString();
 227                 return DataUtil.load(Instance.getInstance().getCache().open(source, this, false), "UTF-8", url.toString());
 228         }
 229
 230         /**
 231          * Log into the support (can be a no-op depending upon the support).
 232          *
 233          * @throws IOException
 234          *             in case of I/O error
 235          */
 236         protected void login() throws IOException {
 237         }
 238
 239         /**
 240          * Now that we have processed the {@link Story}, close the resources if any.
 241          */
 242         protected void close() {
 243                 setCurrentReferer(null);
 244         }
 245
 246         /**
 247          * Process the given story resource into a partially filled {@link Story}
 248          * object containing the name and metadata.
 249          *
 250          * @param getDesc
 251          *            retrieve the description of the story, or not
 252          * @param pg
 253          *            the optional progress reporter
 254          *
 255          * @return the {@link Story}, never NULL
 256          *
 257          * @throws IOException
 258          *             in case of I/O error
 259          */
 260         protected Story processMeta(boolean getDesc, Progress pg)
 261                         throws IOException {
 262                 if (pg == null) {
 263                         pg = new Progress();
 264                 } else {
 265                         pg.setMinMax(0, 100);
 266                 }
 267
 268                 pg.setProgress(30);
 269
 270                 Story story = new Story();
 271                 MetaData meta = getMeta();
 272                 if (meta.getCreationDate() == null || meta.getCreationDate().isEmpty()) {
 273                         meta.setCreationDate(StringUtils.fromTime(new Date().getTime()));
 274                 }
 275                 story.setMeta(meta);
 276                 pg.put("meta", meta);
 277
 278                 pg.setProgress(50);
 279
 280                 if (meta.getCover() == null) {
 281                         meta.setCover(bsHelper.getDefaultCover(meta.getSubject()));
 282                 }
 283
 284                 pg.setProgress(60);
 285
 286                 if (getDesc) {
 287                         String descChapterName = Instance.getInstance().getTrans().getString(StringId.DESCRIPTION);
 288                         story.getMeta().setResume(bsPara.makeChapter(this, source, 0, descChapterName, //
 289                                         getDesc(), isHtml(), null));
 290                 }
 291
 292                 pg.done();
 293                 return story;
 294         }
 295
 296         /**
 297          * Process the given story resource into a fully filled {@link Story}
 298          * object.
 299          *
 300          * @param pg
 301          *            the optional progress reporter
 302          *
 303          * @return the {@link Story}, never NULL
 304          *
 305          * @throws IOException
 306          *             in case of I/O error
 307          */
 308         // TODO: ADD final when BasicSupport_Deprecated is gone
 309         public Story process(Progress pg) throws IOException {
 310                 setCurrentReferer(source);
 311                 login();
 312                 sourceNode = loadDocument(source);
 313
 314                 try {
 315                         return doProcess(pg);
 316                 } finally {
 317                         close();
 318                 }
 319         }
 320
 321         /**
 322          * Actual processing step, without the calls to other methods.
 323          * <p>
 324          * Will convert the story resource into a fully filled {@link Story} object.
 325          *
 326          * @param pg
 327          *            the optional progress reporter
 328          *
 329          * @return the {@link Story}, never NULL
 330          *
 331          * @throws IOException
 332          *             in case of I/O error
 333          */
 334         protected Story doProcess(Progress pg) throws IOException {
 335                 if (pg == null) {
 336                         pg = new Progress();
 337                 } else {
 338                         pg.setMinMax(0, 100);
 339                 }
 340
 341                 pg.setName("Initialising");
 342
 343                 pg.setProgress(1);
 344                 Progress pgMeta = new Progress();
 345                 pg.addProgress(pgMeta, 10);
 346                 Story story = processMeta(true, pgMeta);
 347                 pgMeta.done(); // 10%
 348                 pg.put("meta", story.getMeta());
 349
 350                 pg.setName(story.getMeta().getTitle());
 351
 352                 Progress pgGetChapters = new Progress();
 353                 pg.addProgress(pgGetChapters, 10);
 354                 story.setChapters(new ArrayList<Chapter>());
 355                 List<Entry<String, URL>> chapters = getChapters(pgGetChapters);
 356                 pgGetChapters.done(); // 20%
 357
 358                 if (chapters != null) {
 359                         Progress pgChaps = new Progress("Extracting chapters", 0,
 360                                         chapters.size() * 300);
 361                         pg.addProgress(pgChaps, 80);
 362
 363                         long words = 0;
 364                         int i = 1;
 365                         for (Entry<String, URL> chap : chapters) {
 366                                 pgChaps.setName("Extracting chapter " + i);
 367                                 URL chapUrl = chap.getValue();
 368                                 String chapName = chap.getKey();
 369                                 if (chapUrl != null) {
 370                                         setCurrentReferer(chapUrl);
 371                                 }
 372
 373                                 pgChaps.setProgress(i * 100);
 374                                 Progress pgGetChapterContent = new Progress();
 375                                 Progress pgMakeChapter = new Progress();
 376                                 pgChaps.addProgress(pgGetChapterContent, 100);
 377                                 pgChaps.addProgress(pgMakeChapter, 100);
 378
 379                                 String content = getChapterContent(chapUrl, i,
 380                                                 pgGetChapterContent);
 381                                 pgGetChapterContent.done();
 382                                 Chapter cc = bsPara.makeChapter(this, chapUrl, i,
 383                                                 chapName, content, isHtml(), pgMakeChapter);
 384                                 pgMakeChapter.done();
 385
 386                                 words += cc.getWords();
 387                                 story.getChapters().add(cc);
 388                                 story.getMeta().setWords(words);
 389
 390                                 i++;
 391                         }
 392
 393                         pgChaps.setName("Extracting chapters");
 394                         pgChaps.done();
 395                 }
 396
 397                 pg.setName(story.getMeta().getTitle());
 398                 pg.done();
 399
 400                 return story;
 401         }
 402
 403         /**
 404          * Create a chapter from the given data.
 405          *
 406          * @param source
 407          *            the source URL for this content, which can be used to try and
 408          *            find images if images are present in the format [image-url]
 409          * @param number
 410          *            the chapter number (0 = description)
 411          * @param name
 412          *            the chapter name
 413          * @param content
 414          *            the content of the chapter
 415          * @return the {@link Chapter}
 416          *
 417          * @throws IOException
 418          *             in case of I/O error
 419          */
 420         public Chapter makeChapter(URL source, int number, String name,
 421                         String content) throws IOException {
 422                 return bsPara.makeChapter(this, source, number, name,
 423                                 content, isHtml(), null);
 424         }
 425
 426         /**
 427          * Return a {@link BasicSupport} implementation supporting the given
 428          * resource if possible.
 429          *
 430          * @param url
 431          *            the story resource
 432          *
 433          * @return an implementation that supports it, or NULL
 434          */
 435         public static BasicSupport getSupport(URL url) {
 436                 if (url == null) {
 437                         return null;
 438                 }
 439
 440                 // TEXT and INFO_TEXT always support files (not URLs though)
 441                 for (SupportType type : SupportType.values()) {
 442                         if (type != SupportType.TEXT && type != SupportType.INFO_TEXT) {
 443                                 BasicSupport support = getSupport(type, url);
 444                                 if (support != null && support.supports(url)) {
 445                                         return support;
 446                                 }
 447                         }
 448                 }
 449
 450                 for (SupportType type : new SupportType[] { SupportType.INFO_TEXT,
 451                                 SupportType.TEXT }) {
 452                         BasicSupport support = getSupport(type, url);
 453                         if (support != null && support.supports(url)) {
 454                                 return support;
 455                         }
 456                 }
 457
 458                 return null;
 459         }
 460
 461         /**
 462          * Return a {@link BasicSupport} implementation supporting the given type.
 463          *
 464          * @param type
 465          *            the type, must not be NULL
 466          * @param url
 467          *            the {@link URL} to support (can be NULL to get an
 468          *            "abstract support"; if not NULL, will be used as the source
 469          *            URL)
 470          *
 471          * @return an implementation that supports it, or NULL
 472          */
 473         public static BasicSupport getSupport(SupportType type, URL url) {
 474                 BasicSupport support = null;
 475
 476                 switch (type) {
 477                 case EPUB:
 478                         support = new Epub();
 479                         break;
 480                 case INFO_TEXT:
 481                         support = new InfoText();
 482                         break;
 483                 case FIMFICTION:
 484                         try {
 485                                 // Can fail if no client key or NO in options
 486                                 support = new FimfictionApi();
 487                         } catch (IOException e) {
 488                                 support = new Fimfiction();
 489                         }
 490                         break;
 491                 case FANFICTION:
 492                         support = new Fanfiction();
 493                         break;
 494                 case TEXT:
 495                         support = new Text();
 496                         break;
 497                 case MANGAHUB:
 498                         support = new MangaHub();
 499                         break;
 500                 case E621:
 501                         support = new E621();
 502                         break;
 503                 case YIFFSTAR:
 504                         support = new YiffStar();
 505                         break;
 506                 case E_HENTAI:
 507                         support = new EHentai();
 508                         break;
 509                 case MANGA_LEL:
 510                         support = new MangaLel();
 511                         break;
 512                 case CBZ:
 513                         support = new Cbz();
 514                         break;
 515                 case HTML:
 516                         support = new Html();
 517                         break;
 518                 }
 519
 520                 if (support != null) {
 521                         support.setType(type);
 522                         support.source = support.getCanonicalUrl(url);
 523                 }
 524
 525                 return support;
 526         }
 527 }