e621: use api key
authorNiki Roo <niki@nikiroo.be>
Fri, 1 May 2020 19:43:28 +0000 (21:43 +0200)
committerNiki Roo <niki@nikiroo.be>
Fri, 1 May 2020 19:43:28 +0000 (21:43 +0200)
src/be/nikiroo/fanfix/bundles/Config.java
src/be/nikiroo/fanfix/supported/BasicSupport.java
src/be/nikiroo/fanfix/supported/E621.java

index 7c8eb1c2e84762d86dac1a851f6253afef7e838c..fd27a83fba9179d01d1dd4955ffe7834906709e0 100644 (file)
@@ -164,4 +164,14 @@ public enum Config {
        @Meta(description = "The token required to use the beta APIv2 from FimFiction (see APIKEY_CLIENT_* if you want to generate a new one from your own API key)", //
        format = Format.PASSWORD, def = "Bearer WnZ5oHlzQoDocv1GcgHfcoqctHkSwL-D")
        LOGIN_FIMFICTION_APIKEY_TOKEN, //
+       
+       @Meta(description = "e621.net credentials\nYou can give your e621.net credentials here to have access to all the comics and ignore the default blacklist",//
+       group = true)
+       LOGIN_E621, //
+       @Meta(description = "Your e621.net login",//
+       format = Format.STRING)
+       LOGIN_E621_LOGIN, //
+       @Meta(description = "Your e621.net API KEY",//
+       format = Format.PASSWORD)
+       LOGIN_E621_APIKEY, //
 }
index 35a6ac353be34fe8c52607fdb9f963d895c2603b..900bcf9d5c2f59cb34b6879416b0e52565db769b 100644 (file)
@@ -2,14 +2,17 @@ package be.nikiroo.fanfix.supported;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Scanner;
 import java.util.Map.Entry;
 
+import org.json.JSONObject;
 import org.jsoup.helper.DataUtil;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -293,6 +296,63 @@ public abstract class BasicSupport {
                return story;
        }
 
+       /**
+        * Utility method to convert the given URL into a JSON object.
+        * <p>
+        * Note that this method expects small JSON files (everything is copied into
+        * memory at least twice).
+        * 
+        * @param url
+        *            the URL to parse
+        * @param stable
+        *            TRUE for more stable resources, FALSE when they often change
+        * 
+        * @return the JSON object
+        * 
+        * @throws IOException
+        *             in case of I/O error
+        */
+       protected JSONObject getJson(String url, boolean stable)
+                       throws IOException {
+               try {
+                       return getJson(new URL(url), stable);
+               } catch (MalformedURLException e) {
+                       throw new IOException("Malformed URL: " + url, e);
+               }
+       }
+
+       /**
+        * Utility method to convert the given URL into a JSON object.
+        * <p>
+        * Note that this method expects small JSON files (everything is copied into
+        * memory at least twice).
+        * 
+        * @param url
+        *            the URL to parse
+        * @param stable
+        *            TRUE for more stable resources, FALSE when they often change
+        * 
+        * @return the JSON object
+        * 
+        * @throws IOException
+        *             in case of I/O error
+        */
+       protected JSONObject getJson(URL url, boolean stable) throws IOException {
+               InputStream in = Instance.getInstance().getCache().open(url, null,
+                               stable);
+               try {
+                       Scanner scan = new Scanner(in);
+                       scan.useDelimiter("\0");
+                       try {
+                               return new JSONObject(scan.next());
+                       } finally {
+                               scan.close();
+                       }
+               } finally {
+                       in.close();
+               }
+       }
+
        /**
         * Process the given story resource into a fully filled {@link Story}
         * object.
index a8ea6e6cf3df1b2bf82f4c1aedfea800f05851fe..a019d0ced94881236adb246c517976b75cab737d 100644 (file)
@@ -1,7 +1,6 @@
 package be.nikiroo.fanfix.supported;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -14,17 +13,20 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map.Entry;
 
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
 import org.jsoup.helper.DataUtil;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
 
 import be.nikiroo.fanfix.Instance;
+import be.nikiroo.fanfix.bundles.Config;
 import be.nikiroo.fanfix.data.MetaData;
-import be.nikiroo.utils.IOUtils;
 import be.nikiroo.utils.Image;
 import be.nikiroo.utils.Progress;
 import be.nikiroo.utils.StringUtils;
+import be.nikiroo.utils.Version;
 
 /**
  * Support class for <a href="http://e621.net/">e621.net</a> and
@@ -44,7 +46,8 @@ class E621 extends BasicSupport {
                        host = host.substring("www.".length());
                }
 
-               return ("e621.net".equals(host) || "e926.net".equals(host)) && (isPool(url) || isSearchOrSet(url));
+               return ("e621.net".equals(host) || "e926.net".equals(host))
+                               && (isPool(url) || isSearchOrSet(url));
        }
 
        @Override
@@ -58,7 +61,7 @@ class E621 extends BasicSupport {
 
                meta.setTitle(getTitle());
                meta.setAuthor(getAuthor());
-               meta.setDate("");
+               meta.setDate(getDate());
                meta.setTags(getTags());
                meta.setSource(getType().getSourceName());
                meta.setUrl(getSource().toString());
@@ -79,8 +82,11 @@ class E621 extends BasicSupport {
        protected String getDesc() throws IOException {
                if (isSearchOrSet(getSource())) {
                        StringBuilder builder = new StringBuilder();
-                       builder.append("A collection of images from ").append(getSource().getHost()).append("\n") //
-                                       .append("\tTime of creation: " + StringUtils.fromTime(new Date().getTime())).append("\n") //
+                       builder.append("A collection of images from ")
+                                       .append(getSource().getHost()).append("\n") //
+                                       .append("\tTime of creation: "
+                                                       + StringUtils.fromTime(new Date().getTime()))
+                                       .append("\n") //
                                        .append("\tTags: ");//
                        for (String tag : getTags()) {
                                builder.append("\t\t").append(tag);
@@ -100,72 +106,81 @@ class E621 extends BasicSupport {
        }
 
        @Override
-       protected List<Entry<String, URL>> getChapters(Progress pg) throws IOException {
-               List<Entry<String, URL>> chapters = new LinkedList<Entry<String, URL>>();
-               
-               if (isPool(getSource())) {
-                       String baseUrl = "https://e621.net/" + getSource().getPath() + "?page=";
-                       chapters = getChapters(getSource(), pg, baseUrl, "");
-               } else if (isSearchOrSet(getSource())) {
-                       String baseUrl = "https://e621.net/posts/?page=";
-                       String search = "&tags=" + getTagsFromUrl(getSource());
-
-                       chapters = getChapters(getSource(), pg,
-                                       baseUrl, search);
-               }
-
-               // sets and some pools are sorted in reverse order on the website
-               if (getSource().getPath().startsWith("/posts")) {
-                       Collections.reverse(chapters);  
-               }
-               
-               return chapters;
-       }
-
-       private List<Entry<String, URL>> getChapters(URL source, Progress pg, String baseUrl, String parameters)
+       protected List<Entry<String, URL>> getChapters(Progress pg)
                        throws IOException {
-               List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
-
-               if (source.getHost().contains("e926")) {
-                       baseUrl = baseUrl.replace("e621", "e926");
-               }
+               int i = 1;
+               String jsonUrl = getJsonUrl();
+               if (jsonUrl != null) {
+                       for (i = 1; true; i++) {
+                               if (i > 1) {
+                                       try {
+                                               // The API does not accept more than 2 request per sec,
+                                               // and asks us to limit at one per sec when possible
+                                               Thread.sleep(1000);
+                                       } catch (InterruptedException e) {
+                                       }
+                               }
 
-               for (int i = 1; true; i++) {
-                       URL url = new URL(baseUrl + i + parameters);
-                       try {
-                               InputStream pageI = Instance.getInstance().getCache().open(url, this, false);
                                try {
-                                       if (IOUtils.readSmallStream(pageI).contains("Nobody here but us chickens!")) {
+                                       JSONObject json = getJson(jsonUrl + "&page=" + i, false);
+                                       if (!json.has("posts"))
                                                break;
-                                       }
-                                       urls.add(new AbstractMap.SimpleEntry<String, URL>("Page " + Integer.toString(i), url));
-                               } finally {
-                                       pageI.close();
+                                       JSONArray posts = json.getJSONArray("posts");
+                                       if (posts.isEmpty())
+                                               break;
+                               } catch (Exception e) {
+                                       e.printStackTrace();
                                }
-                       } catch (Exception e) {
-                               break;
                        }
+
+                       // The last page was empty:
+                       i--;
+               }
+
+               // The pages and images are in reverse order on /posts/
+               List<Entry<String, URL>> chapters = new LinkedList<Entry<String, URL>>();
+               for (int page = i; page > 0; page--) {
+                       chapters.add(new AbstractMap.SimpleEntry<String, URL>(
+                                       "Page " + Integer.toString(i - page + 1),
+                                       new URL(jsonUrl + "&page=" + page)));
                }
 
-               return urls;
+               return chapters;
        }
 
        @Override
-       protected String getChapterContent(URL chapUrl, int number, Progress pg) throws IOException {
+       protected String getChapterContent(URL chapUrl, int number, Progress pg)
+                       throws IOException {
                StringBuilder builder = new StringBuilder();
-               Document chapterNode = loadDocument(chapUrl);
-               
-               Elements articles = chapterNode.getElementsByTag("article");
-               
-               // sets and some pools are sorted in reverse order on the website
-               if (getSource().getPath().startsWith("/posts")) {
-                       Collections.reverse(articles);  
+
+               JSONObject json = getJson(chapUrl, false);
+               JSONArray postsArr = json.getJSONArray("posts");
+
+               // The pages and images are in reverse order on /posts/
+               List<JSONObject> posts = new ArrayList<JSONObject>(postsArr.length());
+               for (int i = postsArr.length() - 1; i >= 0; i--) {
+                       Object o = postsArr.get(i);
+                       if (o instanceof JSONObject)
+                               posts.add((JSONObject) o);
                }
-               
-               for (Element el : articles) {
-                       builder.append("[");
-                       builder.append(el.attr("data-file-url"));
-                       builder.append("]<br/>");
+
+               for (JSONObject post : posts) {
+                       if (!post.has("file"))
+                               continue;
+                       JSONObject file = post.getJSONObject("file");
+                       if (!file.has("url"))
+                               continue;
+
+                       try {
+                               String url = file.getString("url");
+                               builder.append("[");
+                               builder.append(url);
+                               builder.append("]<br/>");
+                       } catch (JSONException e) {
+                               // Can be NULL if filtered
+                               // When the value is NULL, we get an exception
+                               // but the "has" method still returns true
+                       }
                }
 
                return builder.toString();
@@ -191,11 +206,13 @@ class E621 extends BasicSupport {
                                // Cannot happen
                        }
                }
-               
+
                if (isSetOriginalUrl(source)) {
                        try {
-                               Document doc = DataUtil.load(Instance.getInstance().getCache().open(source, this, false), "UTF-8", source.toString());
-                               for (Element shortname : doc.getElementsByClass("set-shortname")) {
+                               Document doc = DataUtil.load(Instance.getInstance().getCache()
+                                               .open(source, this, false), "UTF-8", source.toString());
+                               for (Element shortname : doc
+                                               .getElementsByClass("set-shortname")) {
                                        for (Element el : shortname.getElementsByTag("a")) {
                                                if (!el.attr("href").isEmpty())
                                                        return new URL(el.absUrl("href"));
@@ -208,7 +225,8 @@ class E621 extends BasicSupport {
 
                if (isPool(source)) {
                        try {
-                               return new URL(source.toString().replace("/pool/show/", "/pools/"));
+                               return new URL(
+                                               source.toString().replace("/pool/show/", "/pools/"));
                        } catch (MalformedURLException e) {
                        }
                }
@@ -216,29 +234,6 @@ class E621 extends BasicSupport {
                return super.getCanonicalUrl(source);
        }
 
-       // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
-       private String getTagsFromUrl(URL url) {
-               String tags = url == null ? "" : url.getQuery();
-               int pos = tags.indexOf("tags=");
-
-               if (pos >= 0) {
-                       tags = tags.substring(pos).substring("tags=".length());
-               } else {
-                       return "";
-               }
-
-               pos = tags.indexOf('&');
-               if (pos > 0) {
-                       tags = tags.substring(0, pos);
-               }
-               pos = tags.indexOf('/');
-               if (pos > 0) {
-                       tags = tags.substring(0, pos);
-               }
-
-               return tags;
-       }
-
        private String getTitle() {
                String title = "";
 
@@ -259,64 +254,75 @@ class E621 extends BasicSupport {
                if (isSearchOrSet(getSource())) {
                        title = title.isEmpty() ? "e621" : "[e621] " + title;
                }
-               
+
                return title;
        }
 
-       private String getAuthor() throws IOException {
-               StringBuilder builder = new StringBuilder();
-
-               if (isSearchOrSet(getSource())) {
-                       for (Element el : getSourceNode().getElementsByClass("search-tag")) {
-                               if (el.attr("itemprop").equals("author")) {
-                                       if (builder.length() > 0) {
-                                               builder.append(", ");
+       private String getAuthor() {
+               List<String> list = new ArrayList<String>();
+               String jsonUrl = getJsonUrl();
+               if (jsonUrl != null) {
+                       try {
+                               JSONObject json = getJson(jsonUrl, false);
+                               JSONArray posts = json.getJSONArray("posts");
+                               for (Object obj : posts) {
+                                       if (!(obj instanceof JSONObject))
+                                               continue;
+
+                                       JSONObject post = (JSONObject) obj;
+                                       if (!post.has("tags"))
+                                               continue;
+
+                                       JSONObject tags = post.getJSONObject("tags");
+                                       if (!tags.has("artist"))
+                                               continue;
+
+                                       JSONArray artists = tags.getJSONArray("artist");
+                                       for (Object artist : artists) {
+                                               if (list.contains(artist.toString()))
+                                                       continue;
+
+                                               list.add(artist.toString());
                                        }
-                                       builder.append(el.text().trim());
                                }
+                       } catch (Exception e) {
+                               e.printStackTrace();
                        }
                }
 
-               if (isPool(getSource())) {
-                       String desc = getDesc();
-                       String descL = desc.toLowerCase();
+               StringBuilder builder = new StringBuilder();
+               for (String artist : list) {
+                       if (builder.length() > 0) {
+                               builder.append(", ");
+                       }
+                       builder.append(artist);
+               }
 
-                       if (descL.startsWith("by:") || descL.startsWith("by ")) {
-                               desc = desc.substring(3).trim();
-                               desc = desc.split("\n")[0];
+               return builder.toString();
+       }
 
-                               String tab[] = desc.split(" ");
-                               for (int i = 0; i < Math.min(tab.length, 5); i++) {
-                                       if (tab[i].startsWith("http"))
-                                               break;
-                                       builder.append(" ").append(tab[i]);
-                               }
-                       }
+       private String getDate() {
+               String jsonUrl = getJsonUrl();
+               if (jsonUrl != null) {
+                       try {
+                               JSONObject json = getJson(jsonUrl, false);
+                               JSONArray posts = json.getJSONArray("posts");
+                               for (Object obj : posts) {
+                                       if (!(obj instanceof JSONObject))
+                                               continue;
 
-                       if (builder.length() == 0) {
-                               try {
-                                       String poolNumber = getSource().getPath()
-                                                       .substring("/pools/".length());
-                                       String url = "https://e621.net/posts" + "?tags=pool%3A"
-                                                       + poolNumber;
-
-                                       Document page1 = DataUtil.load(Instance.getInstance()
-                                                       .getCache().open(getSource(), null, false), "UTF-8",
-                                                       url);
-                                       for (Element el : page1.getElementsByClass("search-tag")) {
-                                               if (el.attr("itemprop").equals("author")) {
-                                                       if (builder.length() > 0) {
-                                                               builder.append(", ");
-                                                       }
-                                                       builder.append(el.text().trim());
-                                               }
-                                       }
-                               } catch (Exception e) {
+                                       JSONObject post = (JSONObject) obj;
+                                       if (!post.has("created_at"))
+                                               continue;
+
+                                       return post.getString("created_at");
                                }
+                       } catch (Exception e) {
+                               e.printStackTrace();
                        }
                }
 
-               return builder.toString();
+               return "";
        }
 
        // no tags for pools
@@ -335,6 +341,29 @@ class E621 extends BasicSupport {
                return tags;
        }
 
+       // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
+       private String getTagsFromUrl(URL url) {
+               String tags = url == null ? "" : url.getQuery();
+               int pos = tags.indexOf("tags=");
+
+               if (pos >= 0) {
+                       tags = tags.substring(pos).substring("tags=".length());
+               } else {
+                       return "";
+               }
+
+               pos = tags.indexOf('&');
+               if (pos > 0) {
+                       tags = tags.substring(0, pos);
+               }
+               pos = tags.indexOf('/');
+               if (pos > 0) {
+                       tags = tags.substring(0, pos);
+               }
+
+               return tags;
+       }
+
        private Image getCover() throws IOException {
                Image image = null;
                List<Entry<String, URL>> chapters = getChapters(null);
@@ -350,13 +379,44 @@ class E621 extends BasicSupport {
                return image;
        }
 
+       // always /posts.json/ url
+       private String getJsonUrl() {
+               String url = null;
+               if (isSearchOrSet(getSource())) {
+                       url = getSource().toString().replace("/posts", "/posts.json");
+               }
+
+               if (isPool(getSource())) {
+                       String poolNumber = getSource().getPath()
+                                       .substring("/pools/".length());
+                       url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber;
+               }
+
+               if (url != null) {
+                       // Note: one way to override the blacklist
+                       String login = Instance.getInstance().getConfig()
+                                       .getString(Config.LOGIN_E621_LOGIN);
+                       String apk = Instance.getInstance().getConfig()
+                                       .getString(Config.LOGIN_E621_APIKEY);
+
+                       if (login != null && !login.isEmpty() && apk != null
+                                       && !apk.isEmpty()) {
+                               url = String.format("%s&login=%s&api_key=%s&_client=%s", url,
+                                               login, apk, "fanfix-" + Version.getCurrentVersion());
+                       }
+               }
+
+               return url;
+       }
+
        // note: will be removed at getCanonicalUrl()
        private boolean isSetOriginalUrl(URL originalUrl) {
                return originalUrl.getPath().startsWith("/post_sets/");
        }
 
        private boolean isPool(URL url) {
-               return url.getPath().startsWith("/pools/") || url.getPath().startsWith("/pool/show/");
+               return url.getPath().startsWith("/pools/")
+                               || url.getPath().startsWith("/pool/show/");
        }
 
        // set will be renamed into search by canonical url