src/be/nikiroo/fanfix/supported/E621.java

   1 package be.nikiroo.fanfix.supported;
   2
   3 import java.io.IOException;
   4 import java.io.UnsupportedEncodingException;
   5 import java.net.MalformedURLException;
   6 import java.net.URL;
   7 import java.net.URLDecoder;
   8 import java.util.AbstractMap;
   9 import java.util.ArrayList;
  10 import java.util.Date;
  11 import java.util.LinkedList;
  12 import java.util.List;
  13 import java.util.Map.Entry;
  14
  15 import org.json.JSONArray;
  16 import org.json.JSONException;
  17 import org.json.JSONObject;
  18 import org.jsoup.helper.DataUtil;
  19 import org.jsoup.nodes.Document;
  20 import org.jsoup.nodes.Element;
  21
  22 import be.nikiroo.fanfix.Instance;
  23 import be.nikiroo.fanfix.bundles.Config;
  24 import be.nikiroo.fanfix.data.MetaData;
  25 import be.nikiroo.utils.Image;
  26 import be.nikiroo.utils.Progress;
  27 import be.nikiroo.utils.StringUtils;
  28 import be.nikiroo.utils.Version;
  29
  30 /**
  31  * Support class for <a href="http://e621.net/">e621.net</a> and
  32  * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
  33  * including some of MLP.
  34  * <p>
  35  * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
  36  * comics, but it can be difficult to browse.
  37  *
  38  * @author niki
  39  */
  40 class E621 extends BasicSupport {
  41         @Override
  42         protected boolean supports(URL url) {
  43                 String host = url.getHost();
  44                 if (host.startsWith("www.")) {
  45                         host = host.substring("www.".length());
  46                 }
  47
  48                 return ("e621.net".equals(host) || "e926.net".equals(host))
  49                                 && (isPool(url) || isSearchOrSet(url));
  50         }
  51
  52         @Override
  53         protected boolean isHtml() {
  54                 return true;
  55         }
  56
  57         @Override
  58         protected MetaData getMeta() throws IOException {
  59                 MetaData meta = new MetaData();
  60
  61                 meta.setTitle(getTitle());
  62                 meta.setAuthor(getAuthor());
  63                 meta.setDate(bsHelper.formatDate(getDate()));
  64                 meta.setTags(getTags());
  65                 meta.setSource(getType().getSourceName());
  66                 meta.setUrl(getSource().toString());
  67                 meta.setPublisher(getType().getSourceName());
  68                 meta.setUuid(getSource().toString());
  69                 meta.setLuid("");
  70                 meta.setLang("en");
  71                 meta.setSubject("Furry");
  72                 meta.setType(getType().toString());
  73                 meta.setImageDocument(true);
  74                 meta.setCover(getCover());
  75                 meta.setFakeCover(true);
  76
  77                 return meta;
  78         }
  79
  80         @Override
  81         protected String getDesc() throws IOException {
  82                 if (isSearchOrSet(getSource())) {
  83                         StringBuilder builder = new StringBuilder();
  84                         builder.append("<div>");
  85                         builder.append("A collection of images from ")
  86                                         .append(getSource().getHost()) //
  87                                         .append("<br/>\n") //
  88                                         .append("&nbsp;&nbsp;&nbsp;&nbsp;Time of creation: "
  89                                                         + StringUtils.fromTime(new Date().getTime()))
  90                                         .append("<br/>\n") //
  91                                         .append("&nbsp;&nbsp;&nbsp;&nbsp;tTags: ");//
  92                         for (String tag : getTags()) {
  93                                 builder.append(
  94                                                 "\n<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;")
  95                                                 .append(tag);
  96                         }
  97                         builder.append("\n</div>");
  98
  99                         return builder.toString();
 100                 }
 101
 102                 if (isPool(getSource())) {
 103                         Element el = getSourceNode().getElementById("description");
 104                         if (el != null) {
 105                                 return el.html();
 106                         }
 107                 }
 108
 109                 return null;
 110         }
 111
 112         @Override
 113         protected List<Entry<String, URL>> getChapters(Progress pg)
 114                         throws IOException {
 115                 int i = 1;
 116                 String jsonUrl = getJsonUrl();
 117                 if (jsonUrl != null) {
 118                         for (i = 1; true; i++) {
 119                                 if (i > 1) {
 120                                         try {
 121                                                 // The API does not accept more than 2 request per sec,
 122                                                 // and asks us to limit at one per sec when possible
 123                                                 Thread.sleep(1000);
 124                                         } catch (InterruptedException e) {
 125                                         }
 126                                 }
 127
 128                                 try {
 129                                         JSONObject json = getJson(jsonUrl + "&page=" + i, false);
 130                                         if (!json.has("posts"))
 131                                                 break;
 132                                         JSONArray posts = json.getJSONArray("posts");
 133                                         if (posts.isEmpty())
 134                                                 break;
 135                                 } catch (Exception e) {
 136                                         e.printStackTrace();
 137                                 }
 138                         }
 139
 140                         // The last page was empty:
 141                         i--;
 142                 }
 143
 144                 // The pages and images are in reverse order on /posts/
 145                 List<Entry<String, URL>> chapters = new LinkedList<Entry<String, URL>>();
 146                 for (int page = i; page > 0; page--) {
 147                         chapters.add(new AbstractMap.SimpleEntry<String, URL>(
 148                                         "Page " + Integer.toString(i - page + 1),
 149                                         new URL(jsonUrl + "&page=" + page)));
 150                 }
 151
 152                 return chapters;
 153         }
 154
 155         @Override
 156         protected String getChapterContent(URL chapUrl, int number, Progress pg)
 157                         throws IOException {
 158                 StringBuilder builder = new StringBuilder();
 159
 160                 JSONObject json = getJson(chapUrl, false);
 161                 JSONArray postsArr = json.getJSONArray("posts");
 162
 163                 // The pages and images are in reverse order on /posts/
 164                 List<JSONObject> posts = new ArrayList<JSONObject>(postsArr.length());
 165                 for (int i = postsArr.length() - 1; i >= 0; i--) {
 166                         Object o = postsArr.get(i);
 167                         if (o instanceof JSONObject)
 168                                 posts.add((JSONObject) o);
 169                 }
 170
 171                 for (JSONObject post : posts) {
 172                         if (!post.has("file"))
 173                                 continue;
 174                         JSONObject file = post.getJSONObject("file");
 175                         if (!file.has("url"))
 176                                 continue;
 177
 178                         try {
 179                                 String url = file.getString("url");
 180                                 builder.append("[");
 181                                 builder.append(url);
 182                                 builder.append("]<br/>");
 183                         } catch (JSONException e) {
 184                                 // Can be NULL if filtered
 185                                 // When the value is NULL, we get an exception
 186                                 // but the "has" method still returns true
 187                                 Instance.getInstance().getTraceHandler()
 188                                                 .error("Cannot get image for chapter " + number + " of "
 189                                                                 + getSource());
 190                         }
 191                 }
 192
 193                 return builder.toString();
 194         }
 195
 196         @Override
 197         protected URL getCanonicalUrl(URL source) {
 198                 // Convert search-pools into proper pools
 199                 if (source.getPath().equals("/posts") && source.getQuery() != null
 200                                 && source.getQuery().startsWith("tags=pool%3A")) {
 201                         String poolNumber = source.getQuery()
 202                                         .substring("tags=pool%3A".length());
 203                         try {
 204                                 Integer.parseInt(poolNumber);
 205                                 String base = source.getProtocol() + "://" + source.getHost();
 206                                 if (source.getPort() != -1) {
 207                                         base = base + ":" + source.getPort();
 208                                 }
 209                                 source = new URL(base + "/pools/" + poolNumber);
 210                         } catch (NumberFormatException e) {
 211                                 // Not a simple pool, skip
 212                         } catch (MalformedURLException e) {
 213                                 // Cannot happen
 214                         }
 215                 }
 216
 217                 if (isSetOriginalUrl(source)) {
 218                         try {
 219                                 Document doc = DataUtil.load(Instance.getInstance().getCache()
 220                                                 .open(source, this, false), "UTF-8", source.toString());
 221                                 for (Element shortname : doc
 222                                                 .getElementsByClass("set-shortname")) {
 223                                         for (Element el : shortname.getElementsByTag("a")) {
 224                                                 if (!el.attr("href").isEmpty())
 225                                                         return new URL(el.absUrl("href"));
 226                                         }
 227                                 }
 228                         } catch (IOException e) {
 229                                 Instance.getInstance().getTraceHandler().error(e);
 230                         }
 231                 }
 232
 233                 if (isPool(source)) {
 234                         try {
 235                                 return new URL(
 236                                                 source.toString().replace("/pool/show/", "/pools/"));
 237                         } catch (MalformedURLException e) {
 238                         }
 239                 }
 240
 241                 return super.getCanonicalUrl(source);
 242         }
 243
 244         private String getTitle() {
 245                 String title = "";
 246
 247                 Element el = getSourceNode().getElementsByTag("title").first();
 248                 if (el != null) {
 249                         title = el.text().trim();
 250                 }
 251
 252                 for (String s : new String[] { "e621", "-", "e621", "Pool", "-" }) {
 253                         if (title.startsWith(s)) {
 254                                 title = title.substring(s.length()).trim();
 255                         }
 256                         if (title.endsWith(s)) {
 257                                 title = title.substring(0, title.length() - s.length()).trim();
 258                         }
 259                 }
 260
 261                 if (isSearchOrSet(getSource())) {
 262                         title = title.isEmpty() ? "e621" : "[e621] " + title;
 263                 }
 264
 265                 return title;
 266         }
 267
 268         private String getAuthor() {
 269                 List<String> list = new ArrayList<String>();
 270                 String jsonUrl = getJsonUrl();
 271                 if (jsonUrl != null) {
 272                         try {
 273                                 JSONObject json = getJson(jsonUrl, false);
 274                                 JSONArray posts = json.getJSONArray("posts");
 275                                 for (Object obj : posts) {
 276                                         if (!(obj instanceof JSONObject))
 277                                                 continue;
 278
 279                                         JSONObject post = (JSONObject) obj;
 280                                         if (!post.has("tags"))
 281                                                 continue;
 282
 283                                         JSONObject tags = post.getJSONObject("tags");
 284                                         if (!tags.has("artist"))
 285                                                 continue;
 286
 287                                         JSONArray artists = tags.getJSONArray("artist");
 288                                         for (Object artist : artists) {
 289                                                 if (list.contains(artist.toString()))
 290                                                         continue;
 291
 292                                                 list.add(artist.toString());
 293                                         }
 294                                 }
 295                         } catch (Exception e) {
 296                                 e.printStackTrace();
 297                         }
 298                 }
 299
 300                 StringBuilder builder = new StringBuilder();
 301                 for (String artist : list) {
 302                         if (builder.length() > 0) {
 303                                 builder.append(", ");
 304                         }
 305                         builder.append(artist);
 306                 }
 307
 308                 return builder.toString();
 309         }
 310
 311         private String getDate() {
 312                 String jsonUrl = getJsonUrl();
 313                 if (jsonUrl != null) {
 314                         try {
 315                                 JSONObject json = getJson(jsonUrl, false);
 316                                 JSONArray posts = json.getJSONArray("posts");
 317                                 for (Object obj : posts) {
 318                                         if (!(obj instanceof JSONObject))
 319                                                 continue;
 320
 321                                         JSONObject post = (JSONObject) obj;
 322                                         if (!post.has("created_at"))
 323                                                 continue;
 324
 325                                         return post.getString("created_at");
 326                                 }
 327                         } catch (Exception e) {
 328                                 e.printStackTrace();
 329                         }
 330                 }
 331
 332                 return "";
 333         }
 334
 335         // no tags for pools
 336         private List<String> getTags() {
 337                 List<String> tags = new ArrayList<String>();
 338                 if (isSearchOrSet(getSource())) {
 339                         String str = getTagsFromUrl(getSource());
 340                         for (String tag : str.split("\\+")) {
 341                                 try {
 342                                         tags.add(URLDecoder.decode(tag.trim(), "UTF-8").trim());
 343                                 } catch (UnsupportedEncodingException e) {
 344                                 }
 345                         }
 346                 }
 347
 348                 return tags;
 349         }
 350
 351         // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
 352         private String getTagsFromUrl(URL url) {
 353                 String tags = url == null ? "" : url.getQuery();
 354                 int pos = tags.indexOf("tags=");
 355
 356                 if (pos >= 0) {
 357                         tags = tags.substring(pos).substring("tags=".length());
 358                 } else {
 359                         return "";
 360                 }
 361
 362                 pos = tags.indexOf('&');
 363                 if (pos > 0) {
 364                         tags = tags.substring(0, pos);
 365                 }
 366                 pos = tags.indexOf('/');
 367                 if (pos > 0) {
 368                         tags = tags.substring(0, pos);
 369                 }
 370
 371                 return tags;
 372         }
 373
 374         private Image getCover() throws IOException {
 375                 Image image = null;
 376                 List<Entry<String, URL>> chapters = getChapters(null);
 377                 if (!chapters.isEmpty()) {
 378                         URL chap1Url = chapters.get(0).getValue();
 379                         String imgsChap1 = getChapterContent(chap1Url, 1, null);
 380                         if (!imgsChap1.isEmpty()) {
 381                                 imgsChap1 = imgsChap1.split("]")[0].substring(1).trim();
 382                                 image = bsImages.getImage(this, new URL(imgsChap1));
 383                         }
 384                 }
 385
 386                 return image;
 387         }
 388
 389         // always /posts.json/ url
 390         private String getJsonUrl() {
 391                 String url = null;
 392                 if (isSearchOrSet(getSource())) {
 393                         url = getSource().toString().replace("/posts", "/posts.json");
 394                 }
 395
 396                 if (isPool(getSource())) {
 397                         String poolNumber = getSource().getPath()
 398                                         .substring("/pools/".length());
 399                         url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber;
 400                 }
 401
 402                 if (url != null) {
 403                         // Note: one way to override the blacklist
 404                         String login = Instance.getInstance().getConfig()
 405                                         .getString(Config.LOGIN_E621_LOGIN);
 406                         String apk = Instance.getInstance().getConfig()
 407                                         .getString(Config.LOGIN_E621_APIKEY);
 408
 409                         if (login != null && !login.isEmpty() && apk != null
 410                                         && !apk.isEmpty()) {
 411                                 url = String.format("%s&login=%s&api_key=%s&_client=%s", url,
 412                                                 login, apk, "fanfix-" + Version.getCurrentVersion());
 413                         }
 414                 }
 415
 416                 return url;
 417         }
 418
 419         // note: will be removed at getCanonicalUrl()
 420         private boolean isSetOriginalUrl(URL originalUrl) {
 421                 return originalUrl.getPath().startsWith("/post_sets/");
 422         }
 423
 424         private boolean isPool(URL url) {
 425                 return url.getPath().startsWith("/pools/")
 426                                 || url.getPath().startsWith("/pool/show/");
 427         }
 428
 429         // set will be renamed into search by canonical url
 430         private boolean isSearchOrSet(URL url) {
 431                 return
 432                 // search:
 433                 (url.getPath().equals("/posts") && url.getQuery().contains("tags="))
 434                                 // or set:
 435                                 || isSetOriginalUrl(url);
 436         }
 437 }