src/be/nikiroo/fanfix/supported/Fanfiction.java

   1 package be.nikiroo.fanfix.supported;
   2
   3 import java.io.IOException;
   4 import java.io.InputStream;
   5 import java.net.MalformedURLException;
   6 import java.net.URL;
   7 import java.text.SimpleDateFormat;
   8 import java.util.AbstractMap;
   9 import java.util.ArrayList;
  10 import java.util.Date;
  11 import java.util.List;
  12 import java.util.Map.Entry;
  13 import java.util.Scanner;
  14
  15 import be.nikiroo.fanfix.Instance;
  16 import be.nikiroo.fanfix.bundles.Config;
  17 import be.nikiroo.fanfix.data.MetaData;
  18 import be.nikiroo.utils.Image;
  19 import be.nikiroo.utils.Progress;
  20 import be.nikiroo.utils.StringUtils;
  21
  22 /**
  23  * Support class for <a href="http://www.fanfiction.net/">Faniction.net</a>
  24  * stories, a website dedicated to fanfictions of many, many different
  25  * universes, from TV shows to novels to games.
  26  *
  27  * @author niki
  28  */
  29 class Fanfiction extends BasicSupport_Deprecated {
  30         @Override
  31         protected boolean isHtml() {
  32                 return true;
  33         }
  34
  35         @Override
  36         public String getSourceName() {
  37                 return "Fanfiction.net";
  38         }
  39
  40         @Override
  41         protected MetaData getMeta(URL source, InputStream in) throws IOException {
  42                 MetaData meta = new MetaData();
  43
  44                 meta.setTitle(getTitle(reset(in)));
  45                 meta.setAuthor(getAuthor(reset(in)));
  46                 meta.setDate(getDate(reset(in)));
  47                 meta.setTags(getTags(reset(in)));
  48                 meta.setSource(getSourceName());
  49                 meta.setUrl(source.toString());
  50                 meta.setPublisher(getSourceName());
  51                 meta.setUuid(source.toString());
  52                 meta.setLuid("");
  53                 meta.setLang("en"); // TODO!
  54                 meta.setSubject(getSubject(reset(in)));
  55                 meta.setType(getType().toString());
  56                 meta.setImageDocument(false);
  57                 meta.setCover(getCover(source, reset(in)));
  58
  59                 return meta;
  60         }
  61
  62         private String getSubject(InputStream in) {
  63                 String line = getLine(in, "id=pre_story_links", 0);
  64                 if (line != null) {
  65                         int pos = line.lastIndexOf('"');
  66                         if (pos >= 1) {
  67                                 line = line.substring(pos + 1);
  68                                 pos = line.indexOf('<');
  69                                 if (pos >= 0) {
  70                                         return StringUtils.unhtml(line.substring(0, pos)).trim();
  71                                 }
  72                         }
  73                 }
  74
  75                 return null;
  76         }
  77
  78         private List<String> getTags(InputStream in) {
  79                 List<String> tags = new ArrayList<String>();
  80
  81                 String key = "title=\"Send Private Message\"";
  82                 String line = getLine(in, key, 2);
  83                 if (line != null) {
  84                         key = "Rated:";
  85                         int pos = line.indexOf(key);
  86                         if (pos >= 0) {
  87                                 line = line.substring(pos + key.length());
  88                                 key = "Chapters:";
  89                                 pos = line.indexOf(key);
  90                                 if (pos >= 0) {
  91                                         line = line.substring(0, pos);
  92                                         line = StringUtils.unhtml(line).trim();
  93                                         if (line.endsWith("-")) {
  94                                                 line = line.substring(0, line.length() - 1);
  95                                         }
  96
  97                                         for (String tag : line.split("-")) {
  98                                                 tags.add(StringUtils.unhtml(tag).trim());
  99                                         }
 100                                 }
 101                         }
 102                 }
 103
 104                 return tags;
 105         }
 106
 107         private String getTitle(InputStream in) {
 108                 int i = 0;
 109                 @SuppressWarnings("resource")
 110                 Scanner scan = new Scanner(in, "UTF-8");
 111                 scan.useDelimiter("\\n");
 112                 while (scan.hasNext()) {
 113                         String line = scan.next();
 114                         if (line.contains("xcontrast_txt")) {
 115                                 if ((++i) == 2) {
 116                                         line = StringUtils.unhtml(line).trim();
 117                                         if (line.startsWith("Follow/Fav")) {
 118                                                 line = line.substring("Follow/Fav".length()).trim();
 119                                         }
 120
 121                                         return StringUtils.unhtml(line).trim();
 122                                 }
 123                         }
 124                 }
 125
 126                 return null;
 127         }
 128
 129         private String getAuthor(InputStream in) {
 130                 String author = null;
 131
 132                 int i = 0;
 133                 @SuppressWarnings("resource")
 134                 Scanner scan = new Scanner(in, "UTF-8");
 135                 scan.useDelimiter("\\n");
 136                 while (scan.hasNext()) {
 137                         String line = scan.next();
 138                         if (line.contains("xcontrast_txt")) {
 139                                 if ((++i) == 3) {
 140                                         author = StringUtils.unhtml(line).trim();
 141                                         break;
 142                                 }
 143                         }
 144                 }
 145
 146                 return BasicSupportHelper.fixAuthor(author);
 147         }
 148
 149         private String getDate(InputStream in) {
 150                 String key = "Published: <span data-xutime='";
 151                 String line = getLine(in, key, 0);
 152                 if (line != null) {
 153                         int pos = line.indexOf(key);
 154                         if (pos >= 0) {
 155                                 line = line.substring(pos + key.length());
 156                                 pos = line.indexOf('\'');
 157                                 if (pos >= 0) {
 158                                         line = line.substring(0, pos).trim();
 159                                         try {
 160                                                 SimpleDateFormat sdf = new SimpleDateFormat(
 161                                                                 "YYYY-MM-dd");
 162                                                 return sdf
 163                                                                 .format(new Date(1000 * Long.parseLong(line)));
 164                                         } catch (NumberFormatException e) {
 165                                                 Instance.getTraceHandler().error(
 166                                                                 new IOException(
 167                                                                                 "Cannot convert publication date: "
 168                                                                                                 + line, e));
 169                                         }
 170                                 }
 171                         }
 172                 }
 173
 174                 return null;
 175         }
 176
 177         @Override
 178         protected String getDesc(URL source, InputStream in) {
 179                 return getLine(in, "title=\"Send Private Message\"", 1);
 180         }
 181
 182         private Image getCover(URL url, InputStream in) {
 183                 String key = "class='cimage";
 184                 String line = getLine(in, key, 0);
 185                 if (line != null) {
 186                         int pos = line.indexOf(key);
 187                         if (pos >= 0) {
 188                                 line = line.substring(pos + key.length());
 189                                 key = "src='";
 190                                 pos = line.indexOf(key);
 191                                 if (pos >= 0) {
 192                                         line = line.substring(pos + key.length());
 193                                         pos = line.indexOf('\'');
 194                                         if (pos >= 0) {
 195                                                 line = line.substring(0, pos);
 196                                                 if (line.startsWith("//")) {
 197                                                         line = url.getProtocol() + "://"
 198                                                                         + line.substring(2);
 199                                                 } else if (line.startsWith("//")) {
 200                                                         line = url.getProtocol() + "://" + url.getHost()
 201                                                                         + "/" + line.substring(1);
 202                                                 } else {
 203                                                         line = url.getProtocol() + "://" + url.getHost()
 204                                                                         + "/" + url.getPath() + "/" + line;
 205                                                 }
 206
 207                                                 return getImage(this, null, line);
 208                                         }
 209                                 }
 210                         }
 211                 }
 212
 213                 return null;
 214         }
 215
 216         @Override
 217         protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
 218                         Progress pg) {
 219                 List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
 220
 221                 String base = source.toString();
 222                 int pos = base.lastIndexOf('/');
 223                 String suffix = base.substring(pos); // including '/' at start
 224                 base = base.substring(0, pos);
 225                 if (base.endsWith("/1")) {
 226                         base = base.substring(0, base.length() - 1); // including '/' at end
 227                 }
 228
 229                 String line = getLine(in, "id=chap_select", 0);
 230                 String key = "<option  value=";
 231                 int i = 1;
 232
 233                 if (line != null) {
 234                         for (pos = line.indexOf(key); pos >= 0; pos = line
 235                                         .indexOf(key, pos), i++) {
 236                                 pos = line.indexOf('>', pos);
 237                                 if (pos >= 0) {
 238                                         int endOfName = line.indexOf('<', pos);
 239                                         if (endOfName >= 0) {
 240                                                 String name = line.substring(pos + 1, endOfName);
 241                                                 String chapNum = i + ".";
 242                                                 if (name.startsWith(chapNum)) {
 243                                                         name = name.substring(chapNum.length(),
 244                                                                         name.length());
 245                                                 }
 246
 247                                                 try {
 248                                                         urls.add(new AbstractMap.SimpleEntry<String, URL>(
 249                                                                         name.trim(), new URL(base + i + suffix)));
 250                                                 } catch (MalformedURLException e) {
 251                                                         Instance.getTraceHandler()
 252                                                                         .error(new IOException(
 253                                                                                         "Cannot parse chapter " + i
 254                                                                                                         + " url: "
 255                                                                                                         + (base + i + suffix), e));
 256                                                 }
 257                                         }
 258                                 }
 259                         }
 260                 } else {
 261                         // only one chapter:
 262                         final String chapName = getTitle(reset(in));
 263                         final URL chapURL = source;
 264                         urls.add(new Entry<String, URL>() {
 265                                 @Override
 266                                 public URL setValue(URL value) {
 267                                         return null;
 268                                 }
 269
 270                                 @Override
 271                                 public URL getValue() {
 272                                         return chapURL;
 273                                 }
 274
 275                                 @Override
 276                                 public String getKey() {
 277                                         return chapName;
 278                                 }
 279                         });
 280                 }
 281
 282                 return urls;
 283         }
 284
 285         @Override
 286         protected String getChapterContent(URL source, InputStream in, int number,
 287                         Progress pg) {
 288                 StringBuilder builder = new StringBuilder();
 289                 String startAt = "class='storytext ";
 290                 String endAt1 = "function review_init";
 291                 String endAt2 = "id=chap_select";
 292                 boolean ok = false;
 293
 294                 @SuppressWarnings("resource")
 295                 Scanner scan = new Scanner(in, "UTF-8");
 296                 scan.useDelimiter("\\n");
 297                 while (scan.hasNext()) {
 298                         String line = scan.next();
 299                         if (!ok && line.contains(startAt)) {
 300                                 ok = true;
 301                         } else if (ok && (line.contains(endAt1) || line.contains(endAt2))) {
 302                                 ok = false;
 303                                 break;
 304                         }
 305
 306                         if (ok) {
 307                                 // First line may contain the title and chap name again
 308                                 if (builder.length() == 0) {
 309                                         int pos = line.indexOf("<hr");
 310                                         if (pos >= 0) {
 311                                                 boolean chaptered = false;
 312                                                 for (String lang : Instance.getConfig()
 313                                                                 .getString(Config.CHAPTER).split(",")) {
 314                                                         String chapterWord = Instance.getConfig()
 315                                                                         .getStringX(Config.CHAPTER, lang);
 316                                                         int posChap = line.indexOf(chapterWord + " ");
 317                                                         if (posChap < pos) {
 318                                                                 chaptered = true;
 319                                                                 break;
 320                                                         }
 321                                                 }
 322
 323                                                 if (chaptered) {
 324                                                         line = line.substring(pos);
 325                                                 }
 326                                         }
 327                                 }
 328
 329                                 builder.append(line);
 330                                 builder.append(' ');
 331                         }
 332                 }
 333
 334                 return builder.toString();
 335         }
 336
 337         @Override
 338         protected boolean supports(URL url) {
 339                 return "fanfiction.net".equals(url.getHost())
 340                                 || "www.fanfiction.net".equals(url.getHost());
 341         }
 342 }