X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Ffanfix%2Fsupported%2FE621.java;h=c57f8fd2cfbe0c565f15904cc5873a085acbb8b3;hb=b56c9d60fe4ed9470881ebf6221acb0a52997291;hp=2455c8753e822910785cbc1a83a150bee1d35d37;hpb=08fe2e33007063e30fe22dc1d290f8afaa18eb1d;p=fanfix.git diff --git a/src/be/nikiroo/fanfix/supported/E621.java b/src/be/nikiroo/fanfix/supported/E621.java index 2455c87..c57f8fd 100644 --- a/src/be/nikiroo/fanfix/supported/E621.java +++ b/src/be/nikiroo/fanfix/supported/E621.java @@ -2,15 +2,24 @@ package be.nikiroo.fanfix.supported; import java.io.IOException; import java.io.InputStream; +import java.io.UnsupportedEncodingException; import java.net.URL; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.util.AbstractMap; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; +import java.util.AbstractMap.SimpleEntry; import java.util.Map.Entry; import java.util.Scanner; import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.data.Chapter; +import be.nikiroo.fanfix.data.MetaData; import be.nikiroo.fanfix.data.Story; +import be.nikiroo.utils.Image; +import be.nikiroo.utils.Progress; import be.nikiroo.utils.StringUtils; /** @@ -23,21 +32,77 @@ import be.nikiroo.utils.StringUtils; * * @author niki */ -class E621 extends BasicSupport { +class E621 extends BasicSupport_Deprecated { @Override public String getSourceName() { return "e621.net"; } @Override - public boolean isImageDocument(URL source, InputStream in) { - return true; + protected MetaData getMeta(URL source, InputStream in) throws IOException { + MetaData meta = new MetaData(); + + meta.setTitle(getTitle(reset(in))); + meta.setAuthor(getAuthor(source, reset(in))); + meta.setDate(""); + meta.setTags(getTags(source, reset(in), false)); + meta.setSource(getSourceName()); + meta.setUrl(source.toString()); + meta.setPublisher(getSourceName()); + meta.setUuid(source.toString()); + meta.setLuid(""); + meta.setLang("en"); + meta.setSubject("Furry"); + meta.setType(getType().toString()); + meta.setImageDocument(true); + meta.setCover(getCover(source, reset(in))); + meta.setFakeCover(true); + + System.out.println("Meta from e621: "+meta); + + return meta; + } + + private List getTags(URL source, InputStream in, boolean authors) { + List tags = new ArrayList(); + + if (isSearch(source)) { + String tagLine = getLine(in, "id=\"tag-sidebar\"", 1); + if (tagLine != null) { + String key = "href=\""; + for (int pos = tagLine.indexOf(key); pos >= 0; pos = tagLine + .indexOf(key, pos + 1)) { + int end = tagLine.indexOf("\"", pos + key.length()); + if (end >= 0) { + String href = tagLine.substring(pos, end); + String subkey; + if (authors) + subkey = "?name="; + else + subkey = "?title="; + if (href.contains(subkey)) { + String tag = href.substring(href.indexOf(subkey) + + subkey.length()); + try { + tags.add(URLDecoder.decode(tag, "UTF-8")); + } catch (UnsupportedEncodingException e) { + // supported JVMs must have UTF-8 support + e.printStackTrace(); + } + } + } + } + + } + } + + return tags; } @Override - public Story process(URL url) throws IOException { + public Story process(URL url, Progress pg) throws IOException { // There is no chapters on e621, just pagination... - Story story = super.process(url); + Story story = super.process(url, pg); Chapter only = new Chapter(1, null); for (Chapter chap : story) { @@ -58,7 +123,7 @@ class E621 extends BasicSupport { } return ("e621.net".equals(host) || "e926.net".equals(host)) - && url.getPath().startsWith("/pool/"); + && (isPool(url) || isSearch(url)); } @Override @@ -66,8 +131,36 @@ class E621 extends BasicSupport { return true; } - @Override - protected String getAuthor(URL source, InputStream in) throws IOException { + private Image getCover(URL source, InputStream in) throws IOException { + URL urlForCover = source; + if (isPool(source)) { + urlForCover = new URL(source.toString() + "?page=1"); + } + + String images = getChapterContent(urlForCover, in, 1, null); + if (!images.isEmpty()) { + int pos = images.indexOf("
"); + if (pos >= 0) { + images = images.substring(1, pos - 1); + return getImage(this, null, images); + } + } + + return null; + } + + private String getAuthor(URL source, InputStream in) { + if (isSearch(source)) { + StringBuilder builder = new StringBuilder(); + for (String author : getTags(source, in, true)) { + if (builder.length() > 0) + builder.append(", "); + builder.append(author); + } + + return builder.toString(); + } + String author = getLine(in, "href=\"/post/show/", 0); if (author != null) { String key = "href=\""; @@ -79,24 +172,28 @@ class E621 extends BasicSupport { author = author.substring(0, pos - 1); String page = source.getProtocol() + "://" + source.getHost() + author; - InputStream pageIn = Instance.getCache().open( - new URL(page), this, false); try { - key = "class=\"tag-type-artist\""; - author = getLine(pageIn, key, 0); - if (author != null) { - pos = author.indexOf("= 0) { - author = author.substring(pos); - pos = author.indexOf(""); + InputStream pageIn = Instance.getCache().open( + new URL(page), this, false); + try { + key = "class=\"tag-type-artist\""; + author = getLine(pageIn, key, 0); + if (author != null) { + pos = author.indexOf("= 0) { - author = author.substring(0, pos); - return StringUtils.unhtml(author); + author = author.substring(pos); + pos = author.indexOf(""); + if (pos >= 0) { + author = author.substring(0, pos); + return StringUtils.unhtml(author); + } } } + } finally { + pageIn.close(); } - } finally { - pageIn.close(); + } catch (Exception e) { + // No author found } } } @@ -105,23 +202,7 @@ class E621 extends BasicSupport { return null; } - @Override - protected String getDate(URL source, InputStream in) throws IOException { - return null; - } - - @Override - protected String getSubject(URL source, InputStream in) throws IOException { - return null; - } - - @Override - protected URL getCover(URL source, InputStream in) throws IOException { - return null; - } - - @Override - protected String getTitle(URL source, InputStream in) throws IOException { + private String getTitle(InputStream in) { String title = getLine(in, "", 0); if (title != null) { int pos = title.indexOf('>'); @@ -137,7 +218,7 @@ class E621 extends BasicSupport { title = title.substring("Pool:".length()); } - title = title.trim(); + title = StringUtils.unhtml(title).trim(); } return title; @@ -168,8 +249,58 @@ class E621 extends BasicSupport { } @Override - protected List<Entry<String, URL>> getChapters(URL source, InputStream in) - throws IOException { + protected List<Entry<String, URL>> getChapters(URL source, InputStream in, + Progress pg) throws IOException { + if (isPool(source)) { + return getChaptersPool(source, in, pg); + } else if (isSearch(source)) { + return getChaptersSearch(source, in, pg); + } + + return new LinkedList<Entry<String, URL>>(); + } + + private List<Entry<String, URL>> getChaptersSearch(URL source, + InputStream in, Progress pg) throws IOException { + List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>(); + + String search = source.getPath(); + if (search.endsWith("/")) { + search = search.substring(0, search.length() - 1); + } + + int pos = search.lastIndexOf('/'); + if (pos >= 0) { + search = search.substring(pos + 1); + } + + String baseUrl = "https://e621.net/post/index/"; + if (source.getHost().contains("e926")) { + baseUrl = baseUrl.replace("e621", "e926"); + } + + for (int i = 1; true; i++) { + URL url = new URL(baseUrl + i + "/" + search + "/"); + try { + InputStream pageI = Instance.getCache().open(url, this, false); + try { + if (getLine(pageI, "No posts matched your search.", 0) != null) + break; + urls.add(new AbstractMap.SimpleEntry<String, URL>(Integer + .toString(i), url)); + } finally { + pageI.close(); + } + } catch (Exception e) { + break; + } + } + + return urls; + } + + private List<Entry<String, URL>> getChaptersPool(URL source, + InputStream in, Progress pg) throws IOException { List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>(); int last = 1; // no pool/show when only one page @@ -196,29 +327,16 @@ class E621 extends BasicSupport { } for (int i = 1; i <= last; i++) { - final String key = Integer.toString(i); - final URL value = new URL(source.toString() + "?page=" + i); - urls.add(new Entry<String, URL>() { - public URL setValue(URL value) { - return null; - } - - public URL getValue() { - return value; - } - - public String getKey() { - return key; - } - }); + urls.add(new AbstractMap.SimpleEntry<String, URL>(Integer + .toString(i), new URL(source.toString() + "?page=" + i))); } return urls; } @Override - protected String getChapterContent(URL source, InputStream in, int number) - throws IOException { + protected String getChapterContent(URL source, InputStream in, int number, + Progress pg) throws IOException { StringBuilder builder = new StringBuilder(); String staticSite = "https://static1.e621.net"; if (source.getHost().contains("e926")) { @@ -232,7 +350,7 @@ class E621 extends BasicSupport { scan.useDelimiter("\\n"); while (scan.hasNext()) { String line = scan.next(); - if (line.contains("class=\"preview\"")) { + if (line.contains("class=\"preview")) { for (int pos = line.indexOf(key); pos >= 0; pos = line.indexOf( key, pos + key.length())) { int endPos = line.indexOf("\"", pos); @@ -245,7 +363,7 @@ class E621 extends BasicSupport { id = id.substring(0, dotPos); builder.append("["); builder.append(id); - builder.append("]\n"); + builder.append("]<br/>"); } } } @@ -254,4 +372,12 @@ class E621 extends BasicSupport { return builder.toString(); } + + private boolean isPool(URL url) { + return url.getPath().startsWith("/pool/"); + } + + private boolean isSearch(URL url) { + return url.getPath().startsWith("/post/index/"); + } }