supported/E621.java

   1 package be.nikiroo.fanfix.supported;
   2
   3 import java.io.IOException;
   4 import java.io.UnsupportedEncodingException;
   5 import java.net.MalformedURLException;
   6 import java.net.URL;
   7 import java.net.URLDecoder;
   8 import java.util.AbstractMap;
   9 import java.util.ArrayList;
  10 import java.util.Collections;
  11 import java.util.Date;
  12 import java.util.LinkedList;
  13 import java.util.List;
  14 import java.util.Map.Entry;
  15
  16 import org.json.JSONArray;
  17 import org.json.JSONException;
  18 import org.json.JSONObject;
  19 import org.jsoup.helper.DataUtil;
  20 import org.jsoup.nodes.Document;
  21 import org.jsoup.nodes.Element;
  22
  23 import be.nikiroo.fanfix.Instance;
  24 import be.nikiroo.fanfix.bundles.Config;
  25 import be.nikiroo.fanfix.data.MetaData;
  26 import be.nikiroo.utils.Image;
  27 import be.nikiroo.utils.Progress;
  28 import be.nikiroo.utils.StringUtils;
  29 import be.nikiroo.utils.Version;
  30
  31 /**
  32  * Support class for <a href="http://e621.net/">e621.net</a> and
  33  * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
  34  * including some of MLP.
  35  * <p>
  36  * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
  37  * comics, but it can be difficult to browse.
  38  *
  39  * @author niki
  40  */
  41 class E621 extends BasicSupport {
  42         @Override
  43         protected boolean supports(URL url) {
  44                 String host = url.getHost();
  45                 if (host.startsWith("www.")) {
  46                         host = host.substring("www.".length());
  47                 }
  48
  49                 return ("e621.net".equals(host) || "e926.net".equals(host))
  50                                 && (isPool(url) || isSearchOrSet(url));
  51         }
  52
  53         @Override
  54         protected boolean isHtml() {
  55                 return true;
  56         }
  57
  58         @Override
  59         protected MetaData getMeta() throws IOException {
  60                 MetaData meta = new MetaData();
  61
  62                 meta.setTitle(getTitle());
  63                 meta.setAuthor(getAuthor());
  64                 meta.setDate(getDate());
  65                 meta.setTags(getTags());
  66                 meta.setSource(getType().getSourceName());
  67                 meta.setUrl(getSource().toString());
  68                 meta.setPublisher(getType().getSourceName());
  69                 meta.setUuid(getSource().toString());
  70                 meta.setLuid("");
  71                 meta.setLang("en");
  72                 meta.setSubject("Furry");
  73                 meta.setType(getType().toString());
  74                 meta.setImageDocument(true);
  75                 meta.setCover(getCover());
  76                 meta.setFakeCover(true);
  77
  78                 return meta;
  79         }
  80
  81         @Override
  82         protected String getDesc() throws IOException {
  83                 if (isSearchOrSet(getSource())) {
  84                         StringBuilder builder = new StringBuilder();
  85                         builder.append("A collection of images from ")
  86                                         .append(getSource().getHost()).append("\n") //
  87                                         .append("\tTime of creation: "
  88                                                         + StringUtils.fromTime(new Date().getTime()))
  89                                         .append("\n") //
  90                                         .append("\tTags: ");//
  91                         for (String tag : getTags()) {
  92                                 builder.append("\t\t").append(tag);
  93                         }
  94
  95                         return builder.toString();
  96                 }
  97
  98                 if (isPool(getSource())) {
  99                         Element el = getSourceNode().getElementById("description");
 100                         if (el != null) {
 101                                 return el.text();
 102                         }
 103                 }
 104
 105                 return null;
 106         }
 107
 108         @Override
 109         protected List<Entry<String, URL>> getChapters(Progress pg)
 110                         throws IOException {
 111                 int i = 1;
 112                 String jsonUrl = getJsonUrl();
 113                 if (jsonUrl != null) {
 114                         for (i = 1; true; i++) {
 115                                 if (i > 1) {
 116                                         try {
 117                                                 // The API does not accept more than 2 request per sec,
 118                                                 // and asks us to limit at one per sec when possible
 119                                                 Thread.sleep(1000);
 120                                         } catch (InterruptedException e) {
 121                                         }
 122                                 }
 123
 124                                 try {
 125                                         JSONObject json = getJson(jsonUrl + "&page=" + i, false);
 126                                         if (!json.has("posts"))
 127                                                 break;
 128                                         JSONArray posts = json.getJSONArray("posts");
 129                                         if (posts.isEmpty())
 130                                                 break;
 131                                 } catch (Exception e) {
 132                                         e.printStackTrace();
 133                                 }
 134                         }
 135
 136                         // The last page was empty:
 137                         i--;
 138                 }
 139
 140                 // The pages and images are in reverse order on /posts/
 141                 List<Entry<String, URL>> chapters = new LinkedList<Entry<String, URL>>();
 142                 for (int page = i; page > 0; page--) {
 143                         chapters.add(new AbstractMap.SimpleEntry<String, URL>(
 144                                         "Page " + Integer.toString(i - page + 1),
 145                                         new URL(jsonUrl + "&page=" + page)));
 146                 }
 147
 148                 return chapters;
 149         }
 150
 151         @Override
 152         protected String getChapterContent(URL chapUrl, int number, Progress pg)
 153                         throws IOException {
 154                 StringBuilder builder = new StringBuilder();
 155
 156                 JSONObject json = getJson(chapUrl, false);
 157                 JSONArray postsArr = json.getJSONArray("posts");
 158
 159                 // The pages and images are in reverse order on /posts/
 160                 List<JSONObject> posts = new ArrayList<JSONObject>(postsArr.length());
 161                 for (int i = postsArr.length() - 1; i >= 0; i--) {
 162                         Object o = postsArr.get(i);
 163                         if (o instanceof JSONObject)
 164                                 posts.add((JSONObject) o);
 165                 }
 166
 167                 for (JSONObject post : posts) {
 168                         if (!post.has("file"))
 169                                 continue;
 170                         JSONObject file = post.getJSONObject("file");
 171                         if (!file.has("url"))
 172                                 continue;
 173
 174                         try {
 175                                 String url = file.getString("url");
 176                                 builder.append("[");
 177                                 builder.append(url);
 178                                 builder.append("]<br/>");
 179                         } catch (JSONException e) {
 180                                 // Can be NULL if filtered
 181                                 // When the value is NULL, we get an exception
 182                                 // but the "has" method still returns true
 183                         }
 184                 }
 185
 186                 return builder.toString();
 187         }
 188
 189         @Override
 190         protected URL getCanonicalUrl(URL source) {
 191                 // Convert search-pools into proper pools
 192                 if (source.getPath().equals("/posts") && source.getQuery() != null
 193                                 && source.getQuery().startsWith("tags=pool%3A")) {
 194                         String poolNumber = source.getQuery()
 195                                         .substring("tags=pool%3A".length());
 196                         try {
 197                                 Integer.parseInt(poolNumber);
 198                                 String base = source.getProtocol() + "://" + source.getHost();
 199                                 if (source.getPort() != -1) {
 200                                         base = base + ":" + source.getPort();
 201                                 }
 202                                 source = new URL(base + "/pools/" + poolNumber);
 203                         } catch (NumberFormatException e) {
 204                                 // Not a simple pool, skip
 205                         } catch (MalformedURLException e) {
 206                                 // Cannot happen
 207                         }
 208                 }
 209
 210                 if (isSetOriginalUrl(source)) {
 211                         try {
 212                                 Document doc = DataUtil.load(Instance.getInstance().getCache()
 213                                                 .open(source, this, false), "UTF-8", source.toString());
 214                                 for (Element shortname : doc
 215                                                 .getElementsByClass("set-shortname")) {
 216                                         for (Element el : shortname.getElementsByTag("a")) {
 217                                                 if (!el.attr("href").isEmpty())
 218                                                         return new URL(el.absUrl("href"));
 219                                         }
 220                                 }
 221                         } catch (IOException e) {
 222                                 Instance.getInstance().getTraceHandler().error(e);
 223                         }
 224                 }
 225
 226                 if (isPool(source)) {
 227                         try {
 228                                 return new URL(
 229                                                 source.toString().replace("/pool/show/", "/pools/"));
 230                         } catch (MalformedURLException e) {
 231                         }
 232                 }
 233
 234                 return super.getCanonicalUrl(source);
 235         }
 236
 237         private String getTitle() {
 238                 String title = "";
 239
 240                 Element el = getSourceNode().getElementsByTag("title").first();
 241                 if (el != null) {
 242                         title = el.text().trim();
 243                 }
 244
 245                 for (String s : new String[] { "e621", "-", "e621", "Pool", "-" }) {
 246                         if (title.startsWith(s)) {
 247                                 title = title.substring(s.length()).trim();
 248                         }
 249                         if (title.endsWith(s)) {
 250                                 title = title.substring(0, title.length() - s.length()).trim();
 251                         }
 252                 }
 253
 254                 if (isSearchOrSet(getSource())) {
 255                         title = title.isEmpty() ? "e621" : "[e621] " + title;
 256                 }
 257
 258                 return title;
 259         }
 260
 261         private String getAuthor() {
 262                 List<String> list = new ArrayList<String>();
 263                 String jsonUrl = getJsonUrl();
 264                 if (jsonUrl != null) {
 265                         try {
 266                                 JSONObject json = getJson(jsonUrl, false);
 267                                 JSONArray posts = json.getJSONArray("posts");
 268                                 for (Object obj : posts) {
 269                                         if (!(obj instanceof JSONObject))
 270                                                 continue;
 271
 272                                         JSONObject post = (JSONObject) obj;
 273                                         if (!post.has("tags"))
 274                                                 continue;
 275
 276                                         JSONObject tags = post.getJSONObject("tags");
 277                                         if (!tags.has("artist"))
 278                                                 continue;
 279
 280                                         JSONArray artists = tags.getJSONArray("artist");
 281                                         for (Object artist : artists) {
 282                                                 if (list.contains(artist.toString()))
 283                                                         continue;
 284
 285                                                 list.add(artist.toString());
 286                                         }
 287                                 }
 288                         } catch (Exception e) {
 289                                 e.printStackTrace();
 290                         }
 291                 }
 292
 293                 StringBuilder builder = new StringBuilder();
 294                 for (String artist : list) {
 295                         if (builder.length() > 0) {
 296                                 builder.append(", ");
 297                         }
 298                         builder.append(artist);
 299                 }
 300
 301                 return builder.toString();
 302         }
 303
 304         private String getDate() {
 305                 String jsonUrl = getJsonUrl();
 306                 if (jsonUrl != null) {
 307                         try {
 308                                 JSONObject json = getJson(jsonUrl, false);
 309                                 JSONArray posts = json.getJSONArray("posts");
 310                                 for (Object obj : posts) {
 311                                         if (!(obj instanceof JSONObject))
 312                                                 continue;
 313
 314                                         JSONObject post = (JSONObject) obj;
 315                                         if (!post.has("created_at"))
 316                                                 continue;
 317
 318                                         return post.getString("created_at");
 319                                 }
 320                         } catch (Exception e) {
 321                                 e.printStackTrace();
 322                         }
 323                 }
 324
 325                 return "";
 326         }
 327
 328         // no tags for pools
 329         private List<String> getTags() {
 330                 List<String> tags = new ArrayList<String>();
 331                 if (isSearchOrSet(getSource())) {
 332                         String str = getTagsFromUrl(getSource());
 333                         for (String tag : str.split("\\+")) {
 334                                 try {
 335                                         tags.add(URLDecoder.decode(tag.trim(), "UTF-8").trim());
 336                                 } catch (UnsupportedEncodingException e) {
 337                                 }
 338                         }
 339                 }
 340
 341                 return tags;
 342         }
 343
 344         // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
 345         private String getTagsFromUrl(URL url) {
 346                 String tags = url == null ? "" : url.getQuery();
 347                 int pos = tags.indexOf("tags=");
 348
 349                 if (pos >= 0) {
 350                         tags = tags.substring(pos).substring("tags=".length());
 351                 } else {
 352                         return "";
 353                 }
 354
 355                 pos = tags.indexOf('&');
 356                 if (pos > 0) {
 357                         tags = tags.substring(0, pos);
 358                 }
 359                 pos = tags.indexOf('/');
 360                 if (pos > 0) {
 361                         tags = tags.substring(0, pos);
 362                 }
 363
 364                 return tags;
 365         }
 366
 367         private Image getCover() throws IOException {
 368                 Image image = null;
 369                 List<Entry<String, URL>> chapters = getChapters(null);
 370                 if (!chapters.isEmpty()) {
 371                         URL chap1Url = chapters.get(0).getValue();
 372                         String imgsChap1 = getChapterContent(chap1Url, 1, null);
 373                         if (!imgsChap1.isEmpty()) {
 374                                 imgsChap1 = imgsChap1.split("]")[0].substring(1).trim();
 375                                 image = bsImages.getImage(this, new URL(imgsChap1));
 376                         }
 377                 }
 378
 379                 return image;
 380         }
 381
 382         // always /posts.json/ url
 383         private String getJsonUrl() {
 384                 String url = null;
 385                 if (isSearchOrSet(getSource())) {
 386                         url = getSource().toString().replace("/posts", "/posts.json");
 387                 }
 388
 389                 if (isPool(getSource())) {
 390                         String poolNumber = getSource().getPath()
 391                                         .substring("/pools/".length());
 392                         url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber;
 393                 }
 394
 395                 if (url != null) {
 396                         // Note: one way to override the blacklist
 397                         String login = Instance.getInstance().getConfig()
 398                                         .getString(Config.LOGIN_E621_LOGIN);
 399                         String apk = Instance.getInstance().getConfig()
 400                                         .getString(Config.LOGIN_E621_APIKEY);
 401
 402                         if (login != null && !login.isEmpty() && apk != null
 403                                         && !apk.isEmpty()) {
 404                                 url = String.format("%s&login=%s&api_key=%s&_client=%s", url,
 405                                                 login, apk, "fanfix-" + Version.getCurrentVersion());
 406                         }
 407                 }
 408
 409                 return url;
 410         }
 411
 412         // note: will be removed at getCanonicalUrl()
 413         private boolean isSetOriginalUrl(URL originalUrl) {
 414                 return originalUrl.getPath().startsWith("/post_sets/");
 415         }
 416
 417         private boolean isPool(URL url) {
 418                 return url.getPath().startsWith("/pools/")
 419                                 || url.getPath().startsWith("/pool/show/");
 420         }
 421
 422         // set will be renamed into search by canonical url
 423         private boolean isSearchOrSet(URL url) {
 424                 return
 425                 // search:
 426                 (url.getPath().equals("/posts") && url.getQuery().contains("tags="))
 427                                 // or set:
 428                                 || isSetOriginalUrl(url);
 429         }
 430 }