X-Git-Url: https://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Ffanfix%2Fsupported%2FE621.java;h=f3a7238e534e7bf74ca0c1471da3cba6aca28b5e;hb=878ec53820a09ab01397bfe74b3e0f3884e6f9cd;hp=39bbe867d9eb2cdfc630c80df9393bc9bc47aba4;hpb=ba9b5cc4a752a651f1896b14c04cb6ade7746f97;p=fanfix.git
diff --git a/src/be/nikiroo/fanfix/supported/E621.java b/src/be/nikiroo/fanfix/supported/E621.java
index 39bbe86..f3a7238 100644
--- a/src/be/nikiroo/fanfix/supported/E621.java
+++ b/src/be/nikiroo/fanfix/supported/E621.java
@@ -1,30 +1,31 @@
package be.nikiroo.fanfix.supported;
import java.io.IOException;
-import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.AbstractMap;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
import org.jsoup.helper.DataUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
import be.nikiroo.fanfix.Instance;
+import be.nikiroo.fanfix.bundles.Config;
import be.nikiroo.fanfix.data.MetaData;
-import be.nikiroo.utils.IOUtils;
import be.nikiroo.utils.Image;
import be.nikiroo.utils.Progress;
import be.nikiroo.utils.StringUtils;
+import be.nikiroo.utils.Version;
/**
* Support class for e621.net and
@@ -44,7 +45,8 @@ class E621 extends BasicSupport {
host = host.substring("www.".length());
}
- return ("e621.net".equals(host) || "e926.net".equals(host)) && (isPool(url) || isSearchOrSet(url));
+ return ("e621.net".equals(host) || "e926.net".equals(host))
+ && (isPool(url) || isSearchOrSet(url));
}
@Override
@@ -58,16 +60,13 @@ class E621 extends BasicSupport {
meta.setTitle(getTitle());
meta.setAuthor(getAuthor());
- meta.setDate("");
+ meta.setDate(bsHelper.formatDate(getDate()));
meta.setTags(getTags());
- meta.setSource(getType().getSourceName());
meta.setUrl(getSource().toString());
- meta.setPublisher(getType().getSourceName());
meta.setUuid(getSource().toString());
meta.setLuid("");
meta.setLang("en");
meta.setSubject("Furry");
- meta.setType(getType().toString());
meta.setImageDocument(true);
meta.setCover(getCover());
meta.setFakeCover(true);
@@ -79,12 +78,20 @@ class E621 extends BasicSupport {
protected String getDesc() throws IOException {
if (isSearchOrSet(getSource())) {
StringBuilder builder = new StringBuilder();
- builder.append("A collection of images from ").append(getSource().getHost()).append("\n") //
- .append("\tTime of creation: " + StringUtils.fromTime(new Date().getTime())).append("\n") //
- .append("\tTags: ");//
+ builder.append("
");
+ builder.append("A collection of images from ")
+ .append(getSource().getHost()) //
+ .append("
\n") //
+ .append(" Time of creation: "
+ + StringUtils.fromTime(new Date().getTime()))
+ .append("
\n") //
+ .append(" tTags: ");//
for (String tag : getTags()) {
- builder.append("\t\t").append(tag);
+ builder.append(
+ "\n
")
+ .append(tag);
}
+ builder.append("\n
");
return builder.toString();
}
@@ -92,7 +99,7 @@ class E621 extends BasicSupport {
if (isPool(getSource())) {
Element el = getSourceNode().getElementById("description");
if (el != null) {
- return el.text();
+ return el.html();
}
}
@@ -100,72 +107,84 @@ class E621 extends BasicSupport {
}
@Override
- protected List> getChapters(Progress pg) throws IOException {
- List> chapters = new LinkedList>();
-
- if (isPool(getSource())) {
- String baseUrl = "https://e621.net/" + getSource().getPath() + "?page=";
- chapters = getChapters(getSource(), pg, baseUrl, "");
- } else if (isSearchOrSet(getSource())) {
- String baseUrl = "https://e621.net/posts/?page=";
- String search = "&tags=" + getTagsFromUrl(getSource());
-
- chapters = getChapters(getSource(), pg,
- baseUrl, search);
- }
-
- // sets and some pools are sorted in reverse order on the website
- if (getSource().getPath().startsWith("/posts")) {
- Collections.reverse(chapters);
- }
-
- return chapters;
- }
-
- private List> getChapters(URL source, Progress pg, String baseUrl, String parameters)
+ protected List> getChapters(Progress pg)
throws IOException {
- List> urls = new ArrayList>();
-
- if (source.getHost().contains("e926")) {
- baseUrl = baseUrl.replace("e621", "e926");
- }
+ int i = 1;
+ String jsonUrl = getJsonUrl();
+ if (jsonUrl != null) {
+ for (i = 1; true; i++) {
+ if (i > 1) {
+ try {
+ // The API does not accept more than 2 request per sec,
+ // and asks us to limit at one per sec when possible
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+ }
- for (int i = 1; true; i++) {
- URL url = new URL(baseUrl + i + parameters);
- try {
- InputStream pageI = Instance.getInstance().getCache().open(url, this, false);
try {
- if (IOUtils.readSmallStream(pageI).contains("Nobody here but us chickens!")) {
+ JSONObject json = getJson(jsonUrl + "&page=" + i, false);
+ if (!json.has("posts"))
break;
- }
- urls.add(new AbstractMap.SimpleEntry("Page " + Integer.toString(i), url));
- } finally {
- pageI.close();
+ JSONArray posts = json.getJSONArray("posts");
+ if (posts.isEmpty())
+ break;
+ } catch (Exception e) {
+ e.printStackTrace();
}
- } catch (Exception e) {
- break;
}
+
+ // The last page was empty:
+ i--;
}
- return urls;
+ // The pages and images are in reverse order on /posts/
+ List> chapters = new LinkedList>();
+ for (int page = i; page > 0; page--) {
+ chapters.add(new AbstractMap.SimpleEntry(
+ "Page " + Integer.toString(i - page + 1),
+ new URL(jsonUrl + "&page=" + page)));
+ }
+
+ return chapters;
}
@Override
- protected String getChapterContent(URL chapUrl, int number, Progress pg) throws IOException {
+ protected String getChapterContent(URL chapUrl, int number, Progress pg)
+ throws IOException {
StringBuilder builder = new StringBuilder();
- Document chapterNode = loadDocument(chapUrl);
-
- Elements articles = chapterNode.getElementsByTag("article");
-
- // sets and some pools are sorted in reverse order on the website
- if (getSource().getPath().startsWith("/posts")) {
- Collections.reverse(articles);
+
+ JSONObject json = getJson(chapUrl, false);
+ JSONArray postsArr = json.getJSONArray("posts");
+
+ // The pages and images are in reverse order on /posts/
+ List posts = new ArrayList(postsArr.length());
+ for (int i = postsArr.length() - 1; i >= 0; i--) {
+ Object o = postsArr.get(i);
+ if (o instanceof JSONObject)
+ posts.add((JSONObject) o);
}
-
- for (Element el : articles) {
- builder.append("[");
- builder.append(el.attr("data-file-url"));
- builder.append("]
");
+
+ for (JSONObject post : posts) {
+ if (!post.has("file"))
+ continue;
+ JSONObject file = post.getJSONObject("file");
+ if (!file.has("url"))
+ continue;
+
+ try {
+ String url = file.getString("url");
+ builder.append("[");
+ builder.append(url);
+ builder.append("]
");
+ } catch (JSONException e) {
+ // Can be NULL if filtered
+ // When the value is NULL, we get an exception
+ // but the "has" method still returns true
+ Instance.getInstance().getTraceHandler()
+ .error("Cannot get image for chapter " + number + " of "
+ + getSource());
+ }
}
return builder.toString();
@@ -173,10 +192,31 @@ class E621 extends BasicSupport {
@Override
protected URL getCanonicalUrl(URL source) {
+ // Convert search-pools into proper pools
+ if (source.getPath().equals("/posts") && source.getQuery() != null
+ && source.getQuery().startsWith("tags=pool%3A")) {
+ String poolNumber = source.getQuery()
+ .substring("tags=pool%3A".length());
+ try {
+ Integer.parseInt(poolNumber);
+ String base = source.getProtocol() + "://" + source.getHost();
+ if (source.getPort() != -1) {
+ base = base + ":" + source.getPort();
+ }
+ source = new URL(base + "/pools/" + poolNumber);
+ } catch (NumberFormatException e) {
+ // Not a simple pool, skip
+ } catch (MalformedURLException e) {
+ // Cannot happen
+ }
+ }
+
if (isSetOriginalUrl(source)) {
try {
- Document doc = DataUtil.load(Instance.getInstance().getCache().open(source, this, false), "UTF-8", source.toString());
- for (Element shortname : doc.getElementsByClass("set-shortname")) {
+ Document doc = DataUtil.load(Instance.getInstance().getCache()
+ .open(source, this, false), "UTF-8", source.toString());
+ for (Element shortname : doc
+ .getElementsByClass("set-shortname")) {
for (Element el : shortname.getElementsByTag("a")) {
if (!el.attr("href").isEmpty())
return new URL(el.absUrl("href"));
@@ -189,7 +229,8 @@ class E621 extends BasicSupport {
if (isPool(source)) {
try {
- return new URL(source.toString().replace("/pool/show/", "/pools/"));
+ return new URL(
+ source.toString().replace("/pool/show/", "/pools/"));
} catch (MalformedURLException e) {
}
}
@@ -197,29 +238,6 @@ class E621 extends BasicSupport {
return super.getCanonicalUrl(source);
}
- // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
- private String getTagsFromUrl(URL url) {
- String tags = url == null ? "" : url.getQuery();
- int pos = tags.indexOf("tags=");
-
- if (pos >= 0) {
- tags = tags.substring(pos).substring("tags=".length());
- } else {
- return "";
- }
-
- pos = tags.indexOf('&');
- if (pos > 0) {
- tags = tags.substring(0, pos);
- }
- pos = tags.indexOf('/');
- if (pos > 0) {
- tags = tags.substring(0, pos);
- }
-
- return tags;
- }
-
private String getTitle() {
String title = "";
@@ -228,54 +246,87 @@ class E621 extends BasicSupport {
title = el.text().trim();
}
- for (String s : new String[] { "e621", "-", "e621" }) {
+ for (String s : new String[] { "e621", "-", "e621", "Pool", "-" }) {
if (title.startsWith(s)) {
title = title.substring(s.length()).trim();
}
if (title.endsWith(s)) {
title = title.substring(0, title.length() - s.length()).trim();
}
-
}
if (isSearchOrSet(getSource())) {
title = title.isEmpty() ? "e621" : "[e621] " + title;
}
+
return title;
}
- private String getAuthor() throws IOException {
- StringBuilder builder = new StringBuilder();
-
- if (isSearchOrSet(getSource())) {
- for (Element el : getSourceNode().getElementsByClass("search-tag")) {
- if (el.attr("itemprop").equals("author")) {
- if (builder.length() > 0) {
- builder.append(", ");
+ private String getAuthor() {
+ List list = new ArrayList();
+ String jsonUrl = getJsonUrl();
+ if (jsonUrl != null) {
+ try {
+ JSONObject json = getJson(jsonUrl, false);
+ JSONArray posts = json.getJSONArray("posts");
+ for (Object obj : posts) {
+ if (!(obj instanceof JSONObject))
+ continue;
+
+ JSONObject post = (JSONObject) obj;
+ if (!post.has("tags"))
+ continue;
+
+ JSONObject tags = post.getJSONObject("tags");
+ if (!tags.has("artist"))
+ continue;
+
+ JSONArray artists = tags.getJSONArray("artist");
+ for (Object artist : artists) {
+ if (list.contains(artist.toString()))
+ continue;
+
+ list.add(artist.toString());
}
- builder.append(el.text().trim());
}
+ } catch (Exception e) {
+ e.printStackTrace();
}
}
- if (isPool(getSource())) {
- String desc = getDesc();
- String descL = desc.toLowerCase();
+ StringBuilder builder = new StringBuilder();
+ for (String artist : list) {
+ if (builder.length() > 0) {
+ builder.append(", ");
+ }
+ builder.append(artist);
+ }
+
+ return builder.toString();
+ }
- if (descL.startsWith("by:") || descL.startsWith("by ")) {
- desc = desc.substring(3).trim();
- desc = desc.split("\n")[0];
+ private String getDate() {
+ String jsonUrl = getJsonUrl();
+ if (jsonUrl != null) {
+ try {
+ JSONObject json = getJson(jsonUrl, false);
+ JSONArray posts = json.getJSONArray("posts");
+ for (Object obj : posts) {
+ if (!(obj instanceof JSONObject))
+ continue;
- String tab[] = desc.split(" ");
- for (int i = 0; i < Math.min(tab.length, 5); i++) {
- if (tab[i].startsWith("http"))
- break;
- builder.append(" ").append(tab[i]);
+ JSONObject post = (JSONObject) obj;
+ if (!post.has("created_at"))
+ continue;
+
+ return post.getString("created_at");
}
+ } catch (Exception e) {
+ e.printStackTrace();
}
}
- return builder.toString();
+ return "";
}
// no tags for pools
@@ -294,6 +345,29 @@ class E621 extends BasicSupport {
return tags;
}
+ // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
+ private String getTagsFromUrl(URL url) {
+ String tags = url == null ? "" : url.getQuery();
+ int pos = tags.indexOf("tags=");
+
+ if (pos >= 0) {
+ tags = tags.substring(pos).substring("tags=".length());
+ } else {
+ return "";
+ }
+
+ pos = tags.indexOf('&');
+ if (pos > 0) {
+ tags = tags.substring(0, pos);
+ }
+ pos = tags.indexOf('/');
+ if (pos > 0) {
+ tags = tags.substring(0, pos);
+ }
+
+ return tags;
+ }
+
private Image getCover() throws IOException {
Image image = null;
List> chapters = getChapters(null);
@@ -309,13 +383,44 @@ class E621 extends BasicSupport {
return image;
}
+ // always /posts.json/ url
+ private String getJsonUrl() {
+ String url = null;
+ if (isSearchOrSet(getSource())) {
+ url = getSource().toString().replace("/posts", "/posts.json");
+ }
+
+ if (isPool(getSource())) {
+ String poolNumber = getSource().getPath()
+ .substring("/pools/".length());
+ url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber;
+ }
+
+ if (url != null) {
+ // Note: one way to override the blacklist
+ String login = Instance.getInstance().getConfig()
+ .getString(Config.LOGIN_E621_LOGIN);
+ String apk = Instance.getInstance().getConfig()
+ .getString(Config.LOGIN_E621_APIKEY);
+
+ if (login != null && !login.isEmpty() && apk != null
+ && !apk.isEmpty()) {
+ url = String.format("%s&login=%s&api_key=%s&_client=%s", url,
+ login, apk, "fanfix-" + Version.getCurrentVersion());
+ }
+ }
+
+ return url;
+ }
+
// note: will be removed at getCanonicalUrl()
private boolean isSetOriginalUrl(URL originalUrl) {
return originalUrl.getPath().startsWith("/post_sets/");
}
private boolean isPool(URL url) {
- return url.getPath().startsWith("/pools/") || url.getPath().startsWith("/pool/show/");
+ return url.getPath().startsWith("/pools/")
+ || url.getPath().startsWith("/pool/show/");
}
// set will be renamed into search by canonical url