From: Niki Roo Date: Fri, 1 May 2020 19:43:28 +0000 (+0200) Subject: e621: use api key X-Git-Url: http://git.nikiroo.be/?p=nikiroo-utils.git;a=commitdiff_plain;h=5cf61f350b436825d28f5f4f50d8e86fbe5485f1 e621: use api key --- diff --git a/bundles/Config.java b/bundles/Config.java index 7c8eb1c..fd27a83 100644 --- a/bundles/Config.java +++ b/bundles/Config.java @@ -164,4 +164,14 @@ public enum Config { @Meta(description = "The token required to use the beta APIv2 from FimFiction (see APIKEY_CLIENT_* if you want to generate a new one from your own API key)", // format = Format.PASSWORD, def = "Bearer WnZ5oHlzQoDocv1GcgHfcoqctHkSwL-D") LOGIN_FIMFICTION_APIKEY_TOKEN, // + + @Meta(description = "e621.net credentials\nYou can give your e621.net credentials here to have access to all the comics and ignore the default blacklist",// + group = true) + LOGIN_E621, // + @Meta(description = "Your e621.net login",// + format = Format.STRING) + LOGIN_E621_LOGIN, // + @Meta(description = "Your e621.net API KEY",// + format = Format.PASSWORD) + LOGIN_E621_APIKEY, // } diff --git a/supported/BasicSupport.java b/supported/BasicSupport.java index 35a6ac3..900bcf9 100644 --- a/supported/BasicSupport.java +++ b/supported/BasicSupport.java @@ -2,14 +2,17 @@ package be.nikiroo.fanfix.supported; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Scanner; import java.util.Map.Entry; +import org.json.JSONObject; import org.jsoup.helper.DataUtil; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -293,6 +296,63 @@ public abstract class BasicSupport { return story; } + /** + * Utility method to convert the given URL into a JSON object. + *

+ * Note that this method expects small JSON files (everything is copied into + * memory at least twice). + * + * @param url + * the URL to parse + * @param stable + * TRUE for more stable resources, FALSE when they often change + * + * @return the JSON object + * + * @throws IOException + * in case of I/O error + */ + protected JSONObject getJson(String url, boolean stable) + throws IOException { + try { + return getJson(new URL(url), stable); + } catch (MalformedURLException e) { + throw new IOException("Malformed URL: " + url, e); + } + } + + /** + * Utility method to convert the given URL into a JSON object. + *

+ * Note that this method expects small JSON files (everything is copied into + * memory at least twice). + * + * @param url + * the URL to parse + * @param stable + * TRUE for more stable resources, FALSE when they often change + * + * @return the JSON object + * + * @throws IOException + * in case of I/O error + */ + protected JSONObject getJson(URL url, boolean stable) throws IOException { + InputStream in = Instance.getInstance().getCache().open(url, null, + stable); + try { + Scanner scan = new Scanner(in); + scan.useDelimiter("\0"); + try { + return new JSONObject(scan.next()); + } finally { + scan.close(); + } + } finally { + in.close(); + } + } + /** * Process the given story resource into a fully filled {@link Story} * object. diff --git a/supported/E621.java b/supported/E621.java index a8ea6e6..a019d0c 100644 --- a/supported/E621.java +++ b/supported/E621.java @@ -1,7 +1,6 @@ package be.nikiroo.fanfix.supported; import java.io.IOException; -import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; @@ -14,17 +13,20 @@ import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; import org.jsoup.helper.DataUtil; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import be.nikiroo.fanfix.Instance; +import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.data.MetaData; -import be.nikiroo.utils.IOUtils; import be.nikiroo.utils.Image; import be.nikiroo.utils.Progress; import be.nikiroo.utils.StringUtils; +import be.nikiroo.utils.Version; /** * Support class for e621.net and @@ -44,7 +46,8 @@ class E621 extends BasicSupport { host = host.substring("www.".length()); } - return ("e621.net".equals(host) || "e926.net".equals(host)) && (isPool(url) || isSearchOrSet(url)); + return ("e621.net".equals(host) || "e926.net".equals(host)) + && (isPool(url) || isSearchOrSet(url)); } @Override @@ -58,7 +61,7 @@ class E621 extends BasicSupport { meta.setTitle(getTitle()); meta.setAuthor(getAuthor()); - meta.setDate(""); + meta.setDate(getDate()); meta.setTags(getTags()); meta.setSource(getType().getSourceName()); meta.setUrl(getSource().toString()); @@ -79,8 +82,11 @@ class E621 extends BasicSupport { protected String getDesc() throws IOException { if (isSearchOrSet(getSource())) { StringBuilder builder = new StringBuilder(); - builder.append("A collection of images from ").append(getSource().getHost()).append("\n") // - .append("\tTime of creation: " + StringUtils.fromTime(new Date().getTime())).append("\n") // + builder.append("A collection of images from ") + .append(getSource().getHost()).append("\n") // + .append("\tTime of creation: " + + StringUtils.fromTime(new Date().getTime())) + .append("\n") // .append("\tTags: ");// for (String tag : getTags()) { builder.append("\t\t").append(tag); @@ -100,72 +106,81 @@ class E621 extends BasicSupport { } @Override - protected List> getChapters(Progress pg) throws IOException { - List> chapters = new LinkedList>(); - - if (isPool(getSource())) { - String baseUrl = "https://e621.net/" + getSource().getPath() + "?page="; - chapters = getChapters(getSource(), pg, baseUrl, ""); - } else if (isSearchOrSet(getSource())) { - String baseUrl = "https://e621.net/posts/?page="; - String search = "&tags=" + getTagsFromUrl(getSource()); - - chapters = getChapters(getSource(), pg, - baseUrl, search); - } - - // sets and some pools are sorted in reverse order on the website - if (getSource().getPath().startsWith("/posts")) { - Collections.reverse(chapters); - } - - return chapters; - } - - private List> getChapters(URL source, Progress pg, String baseUrl, String parameters) + protected List> getChapters(Progress pg) throws IOException { - List> urls = new ArrayList>(); - - if (source.getHost().contains("e926")) { - baseUrl = baseUrl.replace("e621", "e926"); - } + int i = 1; + String jsonUrl = getJsonUrl(); + if (jsonUrl != null) { + for (i = 1; true; i++) { + if (i > 1) { + try { + // The API does not accept more than 2 request per sec, + // and asks us to limit at one per sec when possible + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } - for (int i = 1; true; i++) { - URL url = new URL(baseUrl + i + parameters); - try { - InputStream pageI = Instance.getInstance().getCache().open(url, this, false); try { - if (IOUtils.readSmallStream(pageI).contains("Nobody here but us chickens!")) { + JSONObject json = getJson(jsonUrl + "&page=" + i, false); + if (!json.has("posts")) break; - } - urls.add(new AbstractMap.SimpleEntry("Page " + Integer.toString(i), url)); - } finally { - pageI.close(); + JSONArray posts = json.getJSONArray("posts"); + if (posts.isEmpty()) + break; + } catch (Exception e) { + e.printStackTrace(); } - } catch (Exception e) { - break; } + + // The last page was empty: + i--; + } + + // The pages and images are in reverse order on /posts/ + List> chapters = new LinkedList>(); + for (int page = i; page > 0; page--) { + chapters.add(new AbstractMap.SimpleEntry( + "Page " + Integer.toString(i - page + 1), + new URL(jsonUrl + "&page=" + page))); } - return urls; + return chapters; } @Override - protected String getChapterContent(URL chapUrl, int number, Progress pg) throws IOException { + protected String getChapterContent(URL chapUrl, int number, Progress pg) + throws IOException { StringBuilder builder = new StringBuilder(); - Document chapterNode = loadDocument(chapUrl); - - Elements articles = chapterNode.getElementsByTag("article"); - - // sets and some pools are sorted in reverse order on the website - if (getSource().getPath().startsWith("/posts")) { - Collections.reverse(articles); + + JSONObject json = getJson(chapUrl, false); + JSONArray postsArr = json.getJSONArray("posts"); + + // The pages and images are in reverse order on /posts/ + List posts = new ArrayList(postsArr.length()); + for (int i = postsArr.length() - 1; i >= 0; i--) { + Object o = postsArr.get(i); + if (o instanceof JSONObject) + posts.add((JSONObject) o); } - - for (Element el : articles) { - builder.append("["); - builder.append(el.attr("data-file-url")); - builder.append("]
"); + + for (JSONObject post : posts) { + if (!post.has("file")) + continue; + JSONObject file = post.getJSONObject("file"); + if (!file.has("url")) + continue; + + try { + String url = file.getString("url"); + builder.append("["); + builder.append(url); + builder.append("]
"); + } catch (JSONException e) { + // Can be NULL if filtered + // When the value is NULL, we get an exception + // but the "has" method still returns true + } } return builder.toString(); @@ -191,11 +206,13 @@ class E621 extends BasicSupport { // Cannot happen } } - + if (isSetOriginalUrl(source)) { try { - Document doc = DataUtil.load(Instance.getInstance().getCache().open(source, this, false), "UTF-8", source.toString()); - for (Element shortname : doc.getElementsByClass("set-shortname")) { + Document doc = DataUtil.load(Instance.getInstance().getCache() + .open(source, this, false), "UTF-8", source.toString()); + for (Element shortname : doc + .getElementsByClass("set-shortname")) { for (Element el : shortname.getElementsByTag("a")) { if (!el.attr("href").isEmpty()) return new URL(el.absUrl("href")); @@ -208,7 +225,8 @@ class E621 extends BasicSupport { if (isPool(source)) { try { - return new URL(source.toString().replace("/pool/show/", "/pools/")); + return new URL( + source.toString().replace("/pool/show/", "/pools/")); } catch (MalformedURLException e) { } } @@ -216,29 +234,6 @@ class E621 extends BasicSupport { return super.getCanonicalUrl(source); } - // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query - private String getTagsFromUrl(URL url) { - String tags = url == null ? "" : url.getQuery(); - int pos = tags.indexOf("tags="); - - if (pos >= 0) { - tags = tags.substring(pos).substring("tags=".length()); - } else { - return ""; - } - - pos = tags.indexOf('&'); - if (pos > 0) { - tags = tags.substring(0, pos); - } - pos = tags.indexOf('/'); - if (pos > 0) { - tags = tags.substring(0, pos); - } - - return tags; - } - private String getTitle() { String title = ""; @@ -259,64 +254,75 @@ class E621 extends BasicSupport { if (isSearchOrSet(getSource())) { title = title.isEmpty() ? "e621" : "[e621] " + title; } - + return title; } - private String getAuthor() throws IOException { - StringBuilder builder = new StringBuilder(); - - if (isSearchOrSet(getSource())) { - for (Element el : getSourceNode().getElementsByClass("search-tag")) { - if (el.attr("itemprop").equals("author")) { - if (builder.length() > 0) { - builder.append(", "); + private String getAuthor() { + List list = new ArrayList(); + String jsonUrl = getJsonUrl(); + if (jsonUrl != null) { + try { + JSONObject json = getJson(jsonUrl, false); + JSONArray posts = json.getJSONArray("posts"); + for (Object obj : posts) { + if (!(obj instanceof JSONObject)) + continue; + + JSONObject post = (JSONObject) obj; + if (!post.has("tags")) + continue; + + JSONObject tags = post.getJSONObject("tags"); + if (!tags.has("artist")) + continue; + + JSONArray artists = tags.getJSONArray("artist"); + for (Object artist : artists) { + if (list.contains(artist.toString())) + continue; + + list.add(artist.toString()); } - builder.append(el.text().trim()); } + } catch (Exception e) { + e.printStackTrace(); } } - if (isPool(getSource())) { - String desc = getDesc(); - String descL = desc.toLowerCase(); + StringBuilder builder = new StringBuilder(); + for (String artist : list) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(artist); + } - if (descL.startsWith("by:") || descL.startsWith("by ")) { - desc = desc.substring(3).trim(); - desc = desc.split("\n")[0]; + return builder.toString(); + } - String tab[] = desc.split(" "); - for (int i = 0; i < Math.min(tab.length, 5); i++) { - if (tab[i].startsWith("http")) - break; - builder.append(" ").append(tab[i]); - } - } + private String getDate() { + String jsonUrl = getJsonUrl(); + if (jsonUrl != null) { + try { + JSONObject json = getJson(jsonUrl, false); + JSONArray posts = json.getJSONArray("posts"); + for (Object obj : posts) { + if (!(obj instanceof JSONObject)) + continue; - if (builder.length() == 0) { - try { - String poolNumber = getSource().getPath() - .substring("/pools/".length()); - String url = "https://e621.net/posts" + "?tags=pool%3A" - + poolNumber; - - Document page1 = DataUtil.load(Instance.getInstance() - .getCache().open(getSource(), null, false), "UTF-8", - url); - for (Element el : page1.getElementsByClass("search-tag")) { - if (el.attr("itemprop").equals("author")) { - if (builder.length() > 0) { - builder.append(", "); - } - builder.append(el.text().trim()); - } - } - } catch (Exception e) { + JSONObject post = (JSONObject) obj; + if (!post.has("created_at")) + continue; + + return post.getString("created_at"); } + } catch (Exception e) { + e.printStackTrace(); } } - return builder.toString(); + return ""; } // no tags for pools @@ -335,6 +341,29 @@ class E621 extends BasicSupport { return tags; } + // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query + private String getTagsFromUrl(URL url) { + String tags = url == null ? "" : url.getQuery(); + int pos = tags.indexOf("tags="); + + if (pos >= 0) { + tags = tags.substring(pos).substring("tags=".length()); + } else { + return ""; + } + + pos = tags.indexOf('&'); + if (pos > 0) { + tags = tags.substring(0, pos); + } + pos = tags.indexOf('/'); + if (pos > 0) { + tags = tags.substring(0, pos); + } + + return tags; + } + private Image getCover() throws IOException { Image image = null; List> chapters = getChapters(null); @@ -350,13 +379,44 @@ class E621 extends BasicSupport { return image; } + // always /posts.json/ url + private String getJsonUrl() { + String url = null; + if (isSearchOrSet(getSource())) { + url = getSource().toString().replace("/posts", "/posts.json"); + } + + if (isPool(getSource())) { + String poolNumber = getSource().getPath() + .substring("/pools/".length()); + url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber; + } + + if (url != null) { + // Note: one way to override the blacklist + String login = Instance.getInstance().getConfig() + .getString(Config.LOGIN_E621_LOGIN); + String apk = Instance.getInstance().getConfig() + .getString(Config.LOGIN_E621_APIKEY); + + if (login != null && !login.isEmpty() && apk != null + && !apk.isEmpty()) { + url = String.format("%s&login=%s&api_key=%s&_client=%s", url, + login, apk, "fanfix-" + Version.getCurrentVersion()); + } + } + + return url; + } + // note: will be removed at getCanonicalUrl() private boolean isSetOriginalUrl(URL originalUrl) { return originalUrl.getPath().startsWith("/post_sets/"); } private boolean isPool(URL url) { - return url.getPath().startsWith("/pools/") || url.getPath().startsWith("/pool/show/"); + return url.getPath().startsWith("/pools/") + || url.getPath().startsWith("/pool/show/"); } // set will be renamed into search by canonical url