package be.nikiroo.fanfix.supported; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.jsoup.helper.DataUtil; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.data.MetaData; import be.nikiroo.utils.Image; import be.nikiroo.utils.Progress; import be.nikiroo.utils.StringUtils; import be.nikiroo.utils.Version; /** * Support class for e621.net and * e926.net, a Furry website supporting comics, * including some of MLP. *

* e926.net only shows the "clean" images and * comics, but it can be difficult to browse. * * @author niki */ class E621 extends BasicSupport { @Override protected boolean supports(URL url) { String host = url.getHost(); if (host.startsWith("www.")) { host = host.substring("www.".length()); } return ("e621.net".equals(host) || "e926.net".equals(host)) && (isPool(url) || isSearchOrSet(url)); } @Override protected boolean isHtml() { return true; } @Override protected MetaData getMeta() throws IOException { MetaData meta = new MetaData(); meta.setTitle(getTitle()); meta.setAuthor(getAuthor()); meta.setDate(bsHelper.formatDate(getDate())); meta.setTags(getTags()); meta.setUrl(getSource().toString()); meta.setUuid(getSource().toString()); meta.setLuid(""); meta.setLang("en"); meta.setSubject("Furry"); meta.setImageDocument(true); meta.setCover(getCover()); meta.setFakeCover(true); return meta; } @Override protected String getDesc() throws IOException { if (isSearchOrSet(getSource())) { StringBuilder builder = new StringBuilder(); builder.append("

"); builder.append("A collection of images from ") .append(getSource().getHost()) // .append("
\n") // .append("    Time of creation: " + StringUtils.fromTime(new Date().getTime())) .append("
\n") // .append("    tTags: ");// for (String tag : getTags()) { builder.append( "\n
        ") .append(tag); } builder.append("\n
"); return builder.toString(); } if (isPool(getSource())) { Element el = getSourceNode().getElementById("description"); if (el != null) { return el.html(); } } return null; } @Override protected List> getChapters(Progress pg) throws IOException { int i = 1; String jsonUrl = getJsonUrl(); if (jsonUrl != null) { for (i = 1; true; i++) { if (i > 1) { try { // The API does not accept more than 2 request per sec, // and asks us to limit at one per sec when possible Thread.sleep(1000); } catch (InterruptedException e) { } } try { JSONObject json = getJson(jsonUrl + "&page=" + i, false); if (!json.has("posts")) break; JSONArray posts = json.getJSONArray("posts"); if (posts.isEmpty()) break; } catch (Exception e) { e.printStackTrace(); } } // The last page was empty: i--; } // The pages and images are in reverse order on /posts/ List> chapters = new LinkedList>(); for (int page = i; page > 0; page--) { chapters.add(new AbstractMap.SimpleEntry( "Page " + Integer.toString(i - page + 1), new URL(jsonUrl + "&page=" + page))); } return chapters; } @Override protected String getChapterContent(URL chapUrl, int number, Progress pg) throws IOException { StringBuilder builder = new StringBuilder(); JSONObject json = getJson(chapUrl, false); JSONArray postsArr = json.getJSONArray("posts"); // The pages and images are in reverse order on /posts/ List posts = new ArrayList(postsArr.length()); for (int i = postsArr.length() - 1; i >= 0; i--) { Object o = postsArr.get(i); if (o instanceof JSONObject) posts.add((JSONObject) o); } for (JSONObject post : posts) { if (!post.has("file")) continue; JSONObject file = post.getJSONObject("file"); if (!file.has("url")) continue; try { String url = file.getString("url"); builder.append("["); builder.append(url); builder.append("]
"); } catch (JSONException e) { // Can be NULL if filtered // When the value is NULL, we get an exception // but the "has" method still returns true Instance.getInstance().getTraceHandler() .error("Cannot get image for chapter " + number + " of " + getSource()); } } return builder.toString(); } @Override protected URL getCanonicalUrl(URL source) { // Convert search-pools into proper pools if (source.getPath().equals("/posts") && source.getQuery() != null && source.getQuery().startsWith("tags=pool%3A")) { String poolNumber = source.getQuery() .substring("tags=pool%3A".length()); try { Integer.parseInt(poolNumber); String base = source.getProtocol() + "://" + source.getHost(); if (source.getPort() != -1) { base = base + ":" + source.getPort(); } source = new URL(base + "/pools/" + poolNumber); } catch (NumberFormatException e) { // Not a simple pool, skip } catch (MalformedURLException e) { // Cannot happen } } if (isSetOriginalUrl(source)) { try { Document doc = DataUtil.load(Instance.getInstance().getCache() .open(source, this, false), "UTF-8", source.toString()); for (Element shortname : doc .getElementsByClass("set-shortname")) { for (Element el : shortname.getElementsByTag("a")) { if (!el.attr("href").isEmpty()) return new URL(el.absUrl("href")); } } } catch (IOException e) { Instance.getInstance().getTraceHandler().error(e); } } if (isPool(source)) { try { return new URL( source.toString().replace("/pool/show/", "/pools/")); } catch (MalformedURLException e) { } } return super.getCanonicalUrl(source); } private String getTitle() { String title = ""; Element el = getSourceNode().getElementsByTag("title").first(); if (el != null) { title = el.text().trim(); } for (String s : new String[] { "e621", "-", "e621", "Pool", "-" }) { if (title.startsWith(s)) { title = title.substring(s.length()).trim(); } if (title.endsWith(s)) { title = title.substring(0, title.length() - s.length()).trim(); } } if (isSearchOrSet(getSource())) { title = title.isEmpty() ? "e621" : "[e621] " + title; } return title; } private String getAuthor() { List list = new ArrayList(); String jsonUrl = getJsonUrl(); if (jsonUrl != null) { try { JSONObject json = getJson(jsonUrl, false); JSONArray posts = json.getJSONArray("posts"); for (Object obj : posts) { if (!(obj instanceof JSONObject)) continue; JSONObject post = (JSONObject) obj; if (!post.has("tags")) continue; JSONObject tags = post.getJSONObject("tags"); if (!tags.has("artist")) continue; JSONArray artists = tags.getJSONArray("artist"); for (Object artist : artists) { if (list.contains(artist.toString())) continue; list.add(artist.toString()); } } } catch (Exception e) { e.printStackTrace(); } } StringBuilder builder = new StringBuilder(); for (String artist : list) { if (builder.length() > 0) { builder.append(", "); } builder.append(artist); } return builder.toString(); } private String getDate() { String jsonUrl = getJsonUrl(); if (jsonUrl != null) { try { JSONObject json = getJson(jsonUrl, false); JSONArray posts = json.getJSONArray("posts"); for (Object obj : posts) { if (!(obj instanceof JSONObject)) continue; JSONObject post = (JSONObject) obj; if (!post.has("created_at")) continue; return post.getString("created_at"); } } catch (Exception e) { e.printStackTrace(); } } return ""; } // no tags for pools private List getTags() { List tags = new ArrayList(); if (isSearchOrSet(getSource())) { String str = getTagsFromUrl(getSource()); for (String tag : str.split("\\+")) { try { tags.add(URLDecoder.decode(tag.trim(), "UTF-8").trim()); } catch (UnsupportedEncodingException e) { } } } return tags; } // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query private String getTagsFromUrl(URL url) { String tags = url == null ? "" : url.getQuery(); int pos = tags.indexOf("tags="); if (pos >= 0) { tags = tags.substring(pos).substring("tags=".length()); } else { return ""; } pos = tags.indexOf('&'); if (pos > 0) { tags = tags.substring(0, pos); } pos = tags.indexOf('/'); if (pos > 0) { tags = tags.substring(0, pos); } return tags; } private Image getCover() throws IOException { Image image = null; List> chapters = getChapters(null); if (!chapters.isEmpty()) { URL chap1Url = chapters.get(0).getValue(); String imgsChap1 = getChapterContent(chap1Url, 1, null); if (!imgsChap1.isEmpty()) { imgsChap1 = imgsChap1.split("]")[0].substring(1).trim(); image = bsImages.getImage(this, new URL(imgsChap1)); } } return image; } // always /posts.json/ url private String getJsonUrl() { String url = null; if (isSearchOrSet(getSource())) { url = getSource().toString().replace("/posts", "/posts.json"); } if (isPool(getSource())) { String poolNumber = getSource().getPath() .substring("/pools/".length()); url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber; } if (url != null) { // Note: one way to override the blacklist String login = Instance.getInstance().getConfig() .getString(Config.LOGIN_E621_LOGIN); String apk = Instance.getInstance().getConfig() .getString(Config.LOGIN_E621_APIKEY); if (login != null && !login.isEmpty() && apk != null && !apk.isEmpty()) { url = String.format("%s&login=%s&api_key=%s&_client=%s", url, login, apk, "fanfix-" + Version.getCurrentVersion()); } } return url; } // note: will be removed at getCanonicalUrl() private boolean isSetOriginalUrl(URL originalUrl) { return originalUrl.getPath().startsWith("/post_sets/"); } private boolean isPool(URL url) { return url.getPath().startsWith("/pools/") || url.getPath().startsWith("/pool/show/"); } // set will be renamed into search by canonical url private boolean isSearchOrSet(URL url) { return // search: (url.getPath().equals("/posts") && url.getQuery().contains("tags=")) // or set: || isSetOriginalUrl(url); } }