X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Ffanfix%2Fsupported%2FE621.java;h=adf8d28c8b2853a01fddc02b239346bec9f28a0e;hb=e14f67abf357c8db74aa230faf8922f93f59c7d6;hp=d6cbad40744743f27550e66b8f3357caa1be2591;hpb=9b863b20370118c95c6801b73dda951c7e507871;p=nikiroo-utils.git diff --git a/src/be/nikiroo/fanfix/supported/E621.java b/src/be/nikiroo/fanfix/supported/E621.java index d6cbad4..adf8d28 100644 --- a/src/be/nikiroo/fanfix/supported/E621.java +++ b/src/be/nikiroo/fanfix/supported/E621.java @@ -1,33 +1,35 @@ package be.nikiroo.fanfix.supported; import java.io.IOException; -import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; -import java.net.URLEncoder; import java.util.AbstractMap; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; +import java.util.Date; import java.util.LinkedList; import java.util.List; -import java.util.AbstractMap.SimpleEntry; import java.util.Map.Entry; -import java.util.Scanner; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.jsoup.helper.DataUtil; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; import be.nikiroo.fanfix.Instance; -import be.nikiroo.fanfix.data.Chapter; +import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.data.MetaData; -import be.nikiroo.fanfix.data.Story; import be.nikiroo.utils.Image; import be.nikiroo.utils.Progress; import be.nikiroo.utils.StringUtils; +import be.nikiroo.utils.Version; /** - * Support class for e621.net and e926.net, a Furry website supporting comics, + * Support class for e621.net and + * e926.net, a Furry website supporting comics, * including some of MLP. *
* e926.net only shows the "clean" images and
@@ -35,374 +37,396 @@ import be.nikiroo.utils.StringUtils;
*
* @author niki
*/
-class E621 extends BasicSupport_Deprecated {
+class E621 extends BasicSupport {
+ @Override
+ protected boolean supports(URL url) {
+ String host = url.getHost();
+ if (host.startsWith("www.")) {
+ host = host.substring("www.".length());
+ }
+
+ return ("e621.net".equals(host) || "e926.net".equals(host))
+ && (isPool(url) || isSearchOrSet(url));
+ }
+
@Override
- public String getSourceName() {
- return "e621.net";
+ protected boolean isHtml() {
+ return true;
}
@Override
- protected MetaData getMeta(URL source, InputStream in) throws IOException {
+ protected MetaData getMeta() throws IOException {
MetaData meta = new MetaData();
- meta.setTitle(getTitle(reset(in)));
- meta.setAuthor(getAuthor(source, reset(in)));
- meta.setDate("");
- meta.setTags(getTags(source, reset(in), false));
- meta.setSource(getSourceName());
- meta.setUrl(source.toString());
- meta.setPublisher(getSourceName());
- meta.setUuid(source.toString());
+ meta.setTitle(getTitle());
+ meta.setAuthor(getAuthor());
+ meta.setDate(bsHelper.formatDate(getDate()));
+ meta.setTags(getTags());
+ meta.setSource(getType().getSourceName());
+ meta.setUrl(getSource().toString());
+ meta.setPublisher(getType().getSourceName());
+ meta.setUuid(getSource().toString());
meta.setLuid("");
meta.setLang("en");
meta.setSubject("Furry");
meta.setType(getType().toString());
meta.setImageDocument(true);
- meta.setCover(getCover(source, reset(in)));
+ meta.setCover(getCover());
meta.setFakeCover(true);
return meta;
}
- private List
");
+ } catch (JSONException e) {
+ // Can be NULL if filtered
+ // When the value is NULL, we get an exception
+ // but the "has" method still returns true
+ Instance.getInstance().getTraceHandler()
+ .error("Cannot get image for chapter " + number + " of "
+ + getSource());
+ }
+ }
+
+ return builder.toString();
}
@Override
- protected boolean isHtml() {
- return true;
- }
+ protected URL getCanonicalUrl(URL source) {
+ // Convert search-pools into proper pools
+ if (source.getPath().equals("/posts") && source.getQuery() != null
+ && source.getQuery().startsWith("tags=pool%3A")) {
+ String poolNumber = source.getQuery()
+ .substring("tags=pool%3A".length());
+ try {
+ Integer.parseInt(poolNumber);
+ String base = source.getProtocol() + "://" + source.getHost();
+ if (source.getPort() != -1) {
+ base = base + ":" + source.getPort();
+ }
+ source = new URL(base + "/pools/" + poolNumber);
+ } catch (NumberFormatException e) {
+ // Not a simple pool, skip
+ } catch (MalformedURLException e) {
+ // Cannot happen
+ }
+ }
- private Image getCover(URL source, InputStream in) throws IOException {
- URL urlForCover = source;
- if (isPool(source)) {
- urlForCover = new URL(source.toString() + "?page=1");
+ if (isSetOriginalUrl(source)) {
+ try {
+ Document doc = DataUtil.load(Instance.getInstance().getCache()
+ .open(source, this, false), "UTF-8", source.toString());
+ for (Element shortname : doc
+ .getElementsByClass("set-shortname")) {
+ for (Element el : shortname.getElementsByTag("a")) {
+ if (!el.attr("href").isEmpty())
+ return new URL(el.absUrl("href"));
+ }
+ }
+ } catch (IOException e) {
+ Instance.getInstance().getTraceHandler().error(e);
+ }
}
- String images = getChapterContent(urlForCover, in, 1, null);
- if (!images.isEmpty()) {
- int pos = images.indexOf("
");
- if (pos >= 0) {
- images = images.substring(1, pos - 1);
- return getImage(this, null, images);
+ if (isPool(source)) {
+ try {
+ return new URL(
+ source.toString().replace("/pool/show/", "/pools/"));
+ } catch (MalformedURLException e) {
}
}
- return null;
+ return super.getCanonicalUrl(source);
}
- private String getAuthor(URL source, InputStream in) {
- if (isSearch(source)) {
- StringBuilder builder = new StringBuilder();
- for (String author : getTags(source, in, true)) {
- if (builder.length() > 0)
- builder.append(", ");
- builder.append(author);
- }
+ private String getTitle() {
+ String title = "";
- return builder.toString();
+ Element el = getSourceNode().getElementsByTag("title").first();
+ if (el != null) {
+ title = el.text().trim();
}
- String author = getLine(in, "href=\"/post/show/", 0);
- if (author != null) {
- String key = "href=\"";
- int pos = author.indexOf(key);
- if (pos >= 0) {
- author = author.substring(pos + key.length());
- pos = author.indexOf("\"");
- if (pos >= 0) {
- author = author.substring(0, pos - 1);
- String page = source.getProtocol() + "://"
- + source.getHost() + author;
- try {
- InputStream pageIn = Instance.getCache().open(
- new URL(page), this, false);
- try {
- key = "class=\"tag-type-artist\"";
- author = getLine(pageIn, key, 0);
- if (author != null) {
- pos = author.indexOf("= 0) {
- author = author.substring(pos);
- pos = author.indexOf("");
- if (pos >= 0) {
- author = author.substring(0, pos);
- return StringUtils.unhtml(author);
- }
- }
- }
- } finally {
- pageIn.close();
- }
- } catch (Exception e) {
- // No author found
- }
- }
+ for (String s : new String[] { "e621", "-", "e621", "Pool", "-" }) {
+ if (title.startsWith(s)) {
+ title = title.substring(s.length()).trim();
+ }
+ if (title.endsWith(s)) {
+ title = title.substring(0, title.length() - s.length()).trim();
}
}
- return null;
+ if (isSearchOrSet(getSource())) {
+ title = title.isEmpty() ? "e621" : "[e621] " + title;
+ }
+
+ return title;
}
- private String getTitle(InputStream in) {
- String title = getLine(in, "
");
- }
- }
- }
+ if (isPool(getSource())) {
+ String poolNumber = getSource().getPath()
+ .substring("/pools/".length());
+ url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber;
+ }
+
+ if (url != null) {
+ // Note: one way to override the blacklist
+ String login = Instance.getInstance().getConfig()
+ .getString(Config.LOGIN_E621_LOGIN);
+ String apk = Instance.getInstance().getConfig()
+ .getString(Config.LOGIN_E621_APIKEY);
+
+ if (login != null && !login.isEmpty() && apk != null
+ && !apk.isEmpty()) {
+ url = String.format("%s&login=%s&api_key=%s&_client=%s", url,
+ login, apk, "fanfix-" + Version.getCurrentVersion());
}
}
- return builder.toString();
+ return url;
}
- @Override
- protected URL getCanonicalUrl(URL source) {
- if (isSearch(source)) {
- // /post?tags=tag1+tag2 -> ../post/index/1/tag1%32tag2
- String key = "post?tags=";
- if (source.toString().contains(key)) {
- int pos = source.toString().indexOf(key);
- String tags = source.toString().substring(pos + key.length());
- tags = tags.replace("+", "%32");
- try {
- return new URL(source.toString().substring(0, pos)
- + "post/index/1/" + tags);
- } catch (MalformedURLException e) {
- Instance.getTraceHandler().error(e);
- }
- }
- }
- return super.getCanonicalUrl(source);
+ // note: will be removed at getCanonicalUrl()
+ private boolean isSetOriginalUrl(URL originalUrl) {
+ return originalUrl.getPath().startsWith("/post_sets/");
}
private boolean isPool(URL url) {
- return url.getPath().startsWith("/pool/");
+ return url.getPath().startsWith("/pools/")
+ || url.getPath().startsWith("/pool/show/");
}
- private boolean isSearch(URL url) {
- return url.getPath().startsWith("/post/index/")
- || (url.getPath().equals("/post") && url.getQuery().startsWith(
- "tags="));
+ // set will be renamed into search by canonical url
+ private boolean isSearchOrSet(URL url) {
+ return
+ // search:
+ (url.getPath().equals("/posts") && url.getQuery().contains("tags="))
+ // or set:
+ || isSetOriginalUrl(url);
}
}