src/be/nikiroo/fanfix/supported/E621.java

   1 package be.nikiroo.fanfix.supported;
   2
   3 import java.awt.image.BufferedImage;
   4 import java.io.IOException;
   5 import java.io.InputStream;
   6 import java.net.URL;
   7 import java.util.ArrayList;
   8 import java.util.List;
   9 import java.util.Map.Entry;
  10 import java.util.Scanner;
  11
  12 import be.nikiroo.fanfix.Instance;
  13 import be.nikiroo.fanfix.data.Chapter;
  14 import be.nikiroo.fanfix.data.MetaData;
  15 import be.nikiroo.fanfix.data.Story;
  16 import be.nikiroo.utils.Progress;
  17 import be.nikiroo.utils.StringUtils;
  18
  19 /**
  20  * Support class for <a href="http://e621.net/">e621.net</a> and <a
  21  * href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
  22  * including some of MLP.
  23  * <p>
  24  * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
  25  * comics, but it can be difficult to browse.
  26  *
  27  * @author niki
  28  */
  29 class E621 extends BasicSupport {
  30         @Override
  31         public String getSourceName() {
  32                 return "e621.net";
  33         }
  34
  35         @Override
  36         protected MetaData getMeta(URL source, InputStream in) throws IOException {
  37                 MetaData meta = new MetaData();
  38
  39                 meta.setTitle(getTitle(reset(in)));
  40                 meta.setAuthor(getAuthor(source, reset(in)));
  41                 meta.setDate("");
  42                 meta.setTags(new ArrayList<String>()); // TODDO ???
  43                 meta.setSource(getSourceName());
  44                 meta.setUrl(source.toString());
  45                 meta.setPublisher(getSourceName());
  46                 meta.setUuid(source.toString());
  47                 meta.setLuid("");
  48                 meta.setLang("EN");
  49                 meta.setSubject("Furry");
  50                 meta.setType(getType().toString());
  51                 meta.setImageDocument(true);
  52                 meta.setCover(getCover(source));
  53                 meta.setFakeCover(true);
  54
  55                 return meta;
  56         }
  57
  58         @Override
  59         public Story process(URL url, Progress pg) throws IOException {
  60                 // There is no chapters on e621, just pagination...
  61                 Story story = super.process(url, pg);
  62
  63                 Chapter only = new Chapter(1, null);
  64                 for (Chapter chap : story) {
  65                         only.getParagraphs().addAll(chap.getParagraphs());
  66                 }
  67
  68                 story.getChapters().clear();
  69                 story.getChapters().add(only);
  70
  71                 return story;
  72         }
  73
  74         @Override
  75         protected boolean supports(URL url) {
  76                 String host = url.getHost();
  77                 if (host.startsWith("www.")) {
  78                         host = host.substring("www.".length());
  79                 }
  80
  81                 return ("e621.net".equals(host) || "e926.net".equals(host))
  82                                 && url.getPath().startsWith("/pool/");
  83         }
  84
  85         @Override
  86         protected boolean isHtml() {
  87                 return true;
  88         }
  89
  90         private BufferedImage getCover(URL source) throws IOException {
  91                 InputStream in = Instance.getCache().open(source, this, true);
  92                 String images = getChapterContent(new URL(source.toString() + "?page="
  93                                 + 1), in, 1);
  94                 if (!images.isEmpty()) {
  95                         int pos = images.indexOf('\n');
  96                         if (pos >= 0) {
  97                                 images = images.substring(1, pos - 1);
  98                                 return getImage(this, null, images);
  99                         }
 100                 }
 101
 102                 return null;
 103         }
 104
 105         private String getAuthor(URL source, InputStream in) throws IOException {
 106                 String author = getLine(in, "href=\"/post/show/", 0);
 107                 if (author != null) {
 108                         String key = "href=\"";
 109                         int pos = author.indexOf(key);
 110                         if (pos >= 0) {
 111                                 author = author.substring(pos + key.length());
 112                                 pos = author.indexOf("\"");
 113                                 if (pos >= 0) {
 114                                         author = author.substring(0, pos - 1);
 115                                         String page = source.getProtocol() + "://"
 116                                                         + source.getHost() + author;
 117                                         try {
 118                                                 InputStream pageIn = Instance.getCache().open(
 119                                                                 new URL(page), this, false);
 120                                                 try {
 121                                                         key = "class=\"tag-type-artist\"";
 122                                                         author = getLine(pageIn, key, 0);
 123                                                         if (author != null) {
 124                                                                 pos = author.indexOf("<a href=\"");
 125                                                                 if (pos >= 0) {
 126                                                                         author = author.substring(pos);
 127                                                                         pos = author.indexOf("</a>");
 128                                                                         if (pos >= 0) {
 129                                                                                 author = author.substring(0, pos);
 130                                                                                 return StringUtils.unhtml(author);
 131                                                                         }
 132                                                                 }
 133                                                         }
 134                                                 } finally {
 135                                                         pageIn.close();
 136                                                 }
 137                                         } catch (Exception e) {
 138                                                 // No author found
 139                                         }
 140                                 }
 141                         }
 142                 }
 143
 144                 return null;
 145         }
 146
 147         private String getTitle(InputStream in) throws IOException {
 148                 String title = getLine(in, "<title>", 0);
 149                 if (title != null) {
 150                         int pos = title.indexOf('>');
 151                         if (pos >= 0) {
 152                                 title = title.substring(pos + 1);
 153                                 pos = title.indexOf('<');
 154                                 if (pos >= 0) {
 155                                         title = title.substring(0, pos);
 156                                 }
 157                         }
 158
 159                         if (title.startsWith("Pool:")) {
 160                                 title = title.substring("Pool:".length());
 161                         }
 162
 163                         title = StringUtils.unhtml(title).trim();
 164                 }
 165
 166                 return title;
 167         }
 168
 169         @Override
 170         protected String getDesc(URL source, InputStream in) throws IOException {
 171                 String desc = getLine(in, "margin-bottom: 2em;", 0);
 172
 173                 if (desc != null) {
 174                         StringBuilder builder = new StringBuilder();
 175
 176                         boolean inTags = false;
 177                         for (char car : desc.toCharArray()) {
 178                                 if ((inTags && car == '>') || (!inTags && car == '<')) {
 179                                         inTags = !inTags;
 180                                 }
 181
 182                                 if (inTags) {
 183                                         builder.append(car);
 184                                 }
 185                         }
 186
 187                         return builder.toString().trim();
 188                 }
 189
 190                 return null;
 191         }
 192
 193         @Override
 194         protected List<Entry<String, URL>> getChapters(URL source, InputStream in)
 195                         throws IOException {
 196                 List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
 197                 int last = 1; // no pool/show when only one page
 198
 199                 @SuppressWarnings("resource")
 200                 Scanner scan = new Scanner(in, "UTF-8");
 201                 scan.useDelimiter("\\n");
 202                 while (scan.hasNext()) {
 203                         String line = scan.next();
 204                         for (int pos = line.indexOf(source.getPath()); pos >= 0; pos = line
 205                                         .indexOf(source.getPath(), pos + source.getPath().length())) {
 206                                 int equalPos = line.indexOf("=", pos);
 207                                 int quotePos = line.indexOf("\"", pos);
 208                                 if (equalPos >= 0 && quotePos > equalPos) {
 209                                         String snum = line.substring(equalPos + 1, quotePos);
 210                                         try {
 211                                                 int num = Integer.parseInt(snum);
 212                                                 if (num > last) {
 213                                                         last = num;
 214                                                 }
 215                                         } catch (NumberFormatException e) {
 216                                         }
 217                                 }
 218                         }
 219                 }
 220
 221                 for (int i = 1; i <= last; i++) {
 222                         final String key = Integer.toString(i);
 223                         final URL value = new URL(source.toString() + "?page=" + i);
 224                         urls.add(new Entry<String, URL>() {
 225                                 public URL setValue(URL value) {
 226                                         return null;
 227                                 }
 228
 229                                 public URL getValue() {
 230                                         return value;
 231                                 }
 232
 233                                 public String getKey() {
 234                                         return key;
 235                                 }
 236                         });
 237                 }
 238
 239                 return urls;
 240         }
 241
 242         @Override
 243         protected String getChapterContent(URL source, InputStream in, int number)
 244                         throws IOException {
 245                 StringBuilder builder = new StringBuilder();
 246                 String staticSite = "https://static1.e621.net";
 247                 if (source.getHost().contains("e926")) {
 248                         staticSite = staticSite.replace("e621", "e926");
 249                 }
 250
 251                 String key = staticSite + "/data/preview/";
 252
 253                 @SuppressWarnings("resource")
 254                 Scanner scan = new Scanner(in, "UTF-8");
 255                 scan.useDelimiter("\\n");
 256                 while (scan.hasNext()) {
 257                         String line = scan.next();
 258                         if (line.contains("class=\"preview")) {
 259                                 for (int pos = line.indexOf(key); pos >= 0; pos = line.indexOf(
 260                                                 key, pos + key.length())) {
 261                                         int endPos = line.indexOf("\"", pos);
 262                                         if (endPos >= 0) {
 263                                                 String id = line.substring(pos + key.length(), endPos);
 264                                                 id = staticSite + "/data/" + id;
 265
 266                                                 int dotPos = id.lastIndexOf(".");
 267                                                 if (dotPos >= 0) {
 268                                                         id = id.substring(0, dotPos);
 269                                                         builder.append("[");
 270                                                         builder.append(id);
 271                                                         builder.append("]\n");
 272                                                 }
 273                                         }
 274                                 }
 275                         }
 276                 }
 277
 278                 return builder.toString();
 279         }
 280 }