X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Ffanfix%2Fsupported%2FE621.java;h=c0125fb3f107087f170f8d2d8e6c718cb5253e87;hb=4d9e85d00d491422eb436d05b5de5af0cb326cdb;hp=316ac2c2bcfbad54411467af2f34713e37772d8d;hpb=8ac3d09905c102fc1d4e6acb07bb3e0676c18931;p=fanfix.git diff --git a/src/be/nikiroo/fanfix/supported/E621.java b/src/be/nikiroo/fanfix/supported/E621.java index 316ac2c..c0125fb 100644 --- a/src/be/nikiroo/fanfix/supported/E621.java +++ b/src/be/nikiroo/fanfix/supported/E621.java @@ -3,6 +3,7 @@ package be.nikiroo.fanfix.supported; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.util.AbstractMap; @@ -16,6 +17,7 @@ import java.util.Map.Entry; import org.jsoup.helper.DataUtil; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.data.MetaData; @@ -99,16 +101,25 @@ class E621 extends BasicSupport { @Override protected List> getChapters(Progress pg) throws IOException { + List> chapters = new LinkedList>(); + if (isPool(getSource())) { String baseUrl = "https://e621.net/" + getSource().getPath() + "?page="; - return getChapters(getSource(), pg, baseUrl, ""); + chapters = getChapters(getSource(), pg, baseUrl, ""); } else if (isSearchOrSet(getSource())) { String baseUrl = "https://e621.net/posts/?page="; String search = "&tags=" + getTagsFromUrl(getSource()); - return getChapters(getSource(), pg, baseUrl, search); + + chapters = getChapters(getSource(), pg, + baseUrl, search); } - return new LinkedList>(); + // sets and some pools are sorted in reverse order on the website + if (getSource().getPath().startsWith("/posts")) { + Collections.reverse(chapters); + } + + return chapters; } private List> getChapters(URL source, Progress pg, String baseUrl, String parameters) @@ -122,7 +133,7 @@ class E621 extends BasicSupport { for (int i = 1; true; i++) { URL url = new URL(baseUrl + i + parameters); try { - InputStream pageI = Instance.getCache().open(url, this, false); + InputStream pageI = Instance.getInstance().getCache().open(url, this, false); try { if (IOUtils.readSmallStream(pageI).contains("Nobody here but us chickens!")) { break; @@ -136,8 +147,6 @@ class E621 extends BasicSupport { } } - // They are sorted in reverse order on the website - Collections.reverse(urls); return urls; } @@ -145,7 +154,15 @@ class E621 extends BasicSupport { protected String getChapterContent(URL chapUrl, int number, Progress pg) throws IOException { StringBuilder builder = new StringBuilder(); Document chapterNode = loadDocument(chapUrl); - for (Element el : chapterNode.getElementsByTag("article")) { + + Elements articles = chapterNode.getElementsByTag("article"); + + // sets and some pools are sorted in reverse order on the website + if (getSource().getPath().startsWith("/posts")) { + Collections.reverse(articles); + } + + for (Element el : articles) { builder.append("["); builder.append(el.attr("data-file-url")); builder.append("]
"); @@ -156,9 +173,28 @@ class E621 extends BasicSupport { @Override protected URL getCanonicalUrl(URL source) { + // Convert search-pools into proper pools + if (source.getPath().equals("/posts") && source.getQuery() != null + && source.getQuery().startsWith("tags=pool%3A")) { + String poolNumber = source.getQuery() + .substring("tags=pool%3A".length()); + try { + Integer.parseInt(poolNumber); + String base = source.getProtocol() + "://" + source.getHost(); + if (source.getPort() != -1) { + base = base + ":" + source.getPort(); + } + source = new URL(base + "/pools/" + poolNumber); + } catch (NumberFormatException e) { + // Not a simple pool, skip + } catch (MalformedURLException e) { + // Cannot happen + } + } + if (isSetOriginalUrl(source)) { try { - Document doc = DataUtil.load(Instance.getCache().open(source, this, false), "UTF-8", source.toString()); + Document doc = DataUtil.load(Instance.getInstance().getCache().open(source, this, false), "UTF-8", source.toString()); for (Element shortname : doc.getElementsByClass("set-shortname")) { for (Element el : shortname.getElementsByTag("a")) { if (!el.attr("href").isEmpty()) @@ -166,7 +202,14 @@ class E621 extends BasicSupport { } } } catch (IOException e) { - Instance.getTraceHandler().error(e); + Instance.getInstance().getTraceHandler().error(e); + } + } + + if (isPool(source)) { + try { + return new URL(source.toString().replace("/pool/show/", "/pools/")); + } catch (MalformedURLException e) { } } @@ -204,19 +247,19 @@ class E621 extends BasicSupport { title = el.text().trim(); } - for (String s : new String[] { "e621", "-", "e621" }) { + for (String s : new String[] { "e621", "-", "e621", "Pool", "-" }) { if (title.startsWith(s)) { title = title.substring(s.length()).trim(); } if (title.endsWith(s)) { title = title.substring(0, title.length() - s.length()).trim(); } - } if (isSearchOrSet(getSource())) { title = title.isEmpty() ? "e621" : "[e621] " + title; } + return title; } @@ -249,6 +292,22 @@ class E621 extends BasicSupport { builder.append(" ").append(tab[i]); } } + + if (builder.length() == 0) { + String url = "https://e621.net/" + getSource().getPath() + + "?page=1"; + Document page1 = DataUtil.load(Instance.getInstance().getCache() + .open(getSource(), this, false), "UTF-8", + url.toString()); + for (Element el : page1.getElementsByClass("search-tag")) { + if (el.attr("itemprop").equals("author")) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(el.text().trim()); + } + } + } } return builder.toString(); @@ -274,8 +333,12 @@ class E621 extends BasicSupport { Image image = null; List> chapters = getChapters(null); if (!chapters.isEmpty()) { - URL url = chapters.get(0).getValue(); - image = bsImages.getImage(this, url); + URL chap1Url = chapters.get(0).getValue(); + String imgsChap1 = getChapterContent(chap1Url, 1, null); + if (!imgsChap1.isEmpty()) { + imgsChap1 = imgsChap1.split("]")[0].substring(1).trim(); + image = bsImages.getImage(this, new URL(imgsChap1)); + } } return image; @@ -287,7 +350,7 @@ class E621 extends BasicSupport { } private boolean isPool(URL url) { - return url.getPath().startsWith("/pools/"); + return url.getPath().startsWith("/pools/") || url.getPath().startsWith("/pool/show/"); } // set will be renamed into search by canonical url