From b5e9855b962842b306c17688c1c793dc1378ea62 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Fri, 10 Aug 2018 15:26:19 +0200 Subject: [PATCH] e621: supports searches --- changelog-fr.md | 1 + changelog.md | 1 + src/be/nikiroo/fanfix/data/MetaData.java | 55 ++++++++ src/be/nikiroo/fanfix/data/Story.java | 50 +------ .../supported/BasicSupport_Deprecated.java | 4 +- src/be/nikiroo/fanfix/supported/E621.java | 123 +++++++++++++++++- 6 files changed, 180 insertions(+), 54 deletions(-) diff --git a/changelog-fr.md b/changelog-fr.md index 08a32f2..d5a7d59 100644 --- a/changelog-fr.md +++ b/changelog-fr.md @@ -3,6 +3,7 @@ # Version WIP - FimfictionAPI: les noms des chapitres sont maintenant triés correctement +- e621: supporte aussi les recherches (/post/) # Version 1.7.1 diff --git a/changelog.md b/changelog.md index 661fc7c..87213a5 100644 --- a/changelog.md +++ b/changelog.md @@ -3,6 +3,7 @@ # Version WIP - FimfictionAPI: chapter names are now correctly ordered +- e621: now supports searches (/post/) # Version 1.7.1 diff --git a/src/be/nikiroo/fanfix/data/MetaData.java b/src/be/nikiroo/fanfix/data/MetaData.java index b2217ae..4b8e65d 100644 --- a/src/be/nikiroo/fanfix/data/MetaData.java +++ b/src/be/nikiroo/fanfix/data/MetaData.java @@ -431,4 +431,59 @@ public class MetaData implements Cloneable, Comparable { return meta; } + + /** + * Display a DEBUG {@link String} representation of this object. + *

+ * This is not efficient, nor intended to be. + */ + @Override + public String toString() { + String title = ""; + if (getTitle() != null) { + title = getTitle(); + } + + StringBuilder tags = new StringBuilder(); + if (getTags() != null) { + for (String tag : getTags()) { + if (tags.length() > 0) { + tags.append(", "); + } + tags.append(tag); + } + } + + String resume = ""; + if (getResume() != null) { + for (Paragraph para : getResume()) { + resume += "\n\t"; + resume += para.toString().substring(0, + Math.min(para.toString().length(), 120)); + } + resume += "\n"; + } + + String cover = "none"; + if (getCover() != null) { + cover = " bytes"; + + int size = getCover().getData().length; + if (size > 1000) { + size /= 1000; + cover = " kb"; + if (size > 1000) { + size /= 1000; + cover = " mb"; + } + } + + cover = size + cover; + } + + return String.format( + "Title: [%s]\nAuthor: [%s]\nDate: [%s]\nTags: [%s]\n" + + "Resume: [%s]\nCover: [%s]", title, getAuthor(), + getDate(), tags.toString(), resume, cover); + } } diff --git a/src/be/nikiroo/fanfix/data/Story.java b/src/be/nikiroo/fanfix/data/Story.java index f4b653a..0e0279f 100644 --- a/src/be/nikiroo/fanfix/data/Story.java +++ b/src/be/nikiroo/fanfix/data/Story.java @@ -67,53 +67,9 @@ public class Story implements Iterable, Cloneable { */ @Override public String toString() { - String title = ""; - if (meta != null && meta.getTitle() != null) { - title = meta.getTitle(); - } - - StringBuilder tags = new StringBuilder(); - if (meta != null && meta.getTags() != null) { - for (String tag : meta.getTags()) { - if (tags.length() > 0) { - tags.append(", "); - } - tags.append(tag); - } - } - - String resume = ""; - if (meta != null && meta.getResume() != null) { - for (Paragraph para : meta.getResume()) { - resume += "\n\t"; - resume += para.toString().substring(0, - Math.min(para.toString().length(), 120)); - } - resume += "\n"; - } - - String cover = "none"; - if (meta != null && meta.getCover() != null) { - cover = " bytes"; - - int size = meta.getCover().getData().length; - if (size > 1000) { - size /= 1000; - cover = " kb"; - if (size > 1000) { - size /= 1000; - cover = " mb"; - } - } - - cover = size + cover; - } - - return String.format( - "Title: [%s]\nAuthor: [%s]\nDate: [%s]\nTags: [%s]\n" - + "Resume: [%s]\nCover: [%s]", title, meta == null ? "" - : meta.getAuthor(), meta == null ? "" : meta.getDate(), - tags.toString(), resume, cover); + if (getMeta() != null) + return "Story: [\n" + getMeta().toString() + "\n]"; + return "Story: [ no metadata found ]"; } @Override diff --git a/src/be/nikiroo/fanfix/supported/BasicSupport_Deprecated.java b/src/be/nikiroo/fanfix/supported/BasicSupport_Deprecated.java index e22724a..ba47484 100644 --- a/src/be/nikiroo/fanfix/supported/BasicSupport_Deprecated.java +++ b/src/be/nikiroo/fanfix/supported/BasicSupport_Deprecated.java @@ -1128,7 +1128,7 @@ public abstract class BasicSupport_Deprecated extends BasicSupport { * the line to return based upon the target line position (-1 = * the line before, 0 = the target line...) * - * @return the line + * @return the line, or NULL if not found */ static protected String getLine(InputStream in, String needle, int relativeLine) { @@ -1152,7 +1152,7 @@ public abstract class BasicSupport_Deprecated extends BasicSupport { * takes the first result (as opposed to the last one, which will * also always spend the input) * - * @return the line + * @return the line, or NULL if not found */ static protected String getLine(InputStream in, String needle, int relativeLine, boolean first) { diff --git a/src/be/nikiroo/fanfix/supported/E621.java b/src/be/nikiroo/fanfix/supported/E621.java index f299ee7..44621b9 100644 --- a/src/be/nikiroo/fanfix/supported/E621.java +++ b/src/be/nikiroo/fanfix/supported/E621.java @@ -2,10 +2,15 @@ package be.nikiroo.fanfix.supported; import java.io.IOException; import java.io.InputStream; +import java.io.UnsupportedEncodingException; import java.net.URL; +import java.net.URLDecoder; +import java.net.URLEncoder; import java.util.AbstractMap; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; +import java.util.AbstractMap.SimpleEntry; import java.util.Map.Entry; import java.util.Scanner; @@ -40,7 +45,7 @@ class E621 extends BasicSupport_Deprecated { meta.setTitle(getTitle(reset(in))); meta.setAuthor(getAuthor(source, reset(in))); meta.setDate(""); - meta.setTags(new ArrayList()); // TODDO ??? + meta.setTags(getTags(source, reset(in), false)); meta.setSource(getSourceName()); meta.setUrl(source.toString()); meta.setPublisher(getSourceName()); @@ -50,12 +55,48 @@ class E621 extends BasicSupport_Deprecated { meta.setSubject("Furry"); meta.setType(getType().toString()); meta.setImageDocument(true); - meta.setCover(getCover(source)); + meta.setCover(getCover(source, reset(in))); meta.setFakeCover(true); return meta; } + private List getTags(URL source, InputStream in, boolean authors) { + List tags = new ArrayList(); + + if (isSearch(source)) { + String tagLine = getLine(in, "id=\"tag-sidebar\"", 1); + if (tagLine != null) { + String key = "href=\""; + for (int pos = tagLine.indexOf(key); pos >= 0; pos = tagLine + .indexOf(key, pos + 1)) { + int end = tagLine.indexOf("\"", pos + key.length()); + if (end >= 0) { + String href = tagLine.substring(pos, end); + String subkey; + if (authors) + subkey = "?name="; + else + subkey = "?title="; + if (href.contains(subkey)) { + String tag = href.substring(href.indexOf(subkey) + + subkey.length()); + try { + tags.add(URLDecoder.decode(tag, "UTF-8")); + } catch (UnsupportedEncodingException e) { + // supported JVMs must have UTF-8 support + e.printStackTrace(); + } + } + } + } + + } + } + + return tags; + } + @Override public Story process(URL url, Progress pg) throws IOException { // There is no chapters on e621, just pagination... @@ -80,7 +121,7 @@ class E621 extends BasicSupport_Deprecated { } return ("e621.net".equals(host) || "e926.net".equals(host)) - && url.getPath().startsWith("/pool/"); + && (isPool(url) || isSearch(url)); } @Override @@ -88,8 +129,11 @@ class E621 extends BasicSupport_Deprecated { return true; } - private Image getCover(URL source) throws IOException { - InputStream in = Instance.getCache().open(source, this, true); + private Image getCover(URL source, InputStream in) throws IOException { + // No cover on searches (/post/) + if (isSearch(source)) + return null; + String images = getChapterContent(new URL(source.toString() + "?page=" + 1), in, 1, null); if (!images.isEmpty()) { @@ -104,6 +148,17 @@ class E621 extends BasicSupport_Deprecated { } private String getAuthor(URL source, InputStream in) { + if (isSearch(source)) { + StringBuilder builder = new StringBuilder(); + for (String author : getTags(source, in, true)) { + if (builder.length() > 0) + builder.append(", "); + builder.append(author); + } + + return builder.toString(); + } + String author = getLine(in, "href=\"/post/show/", 0); if (author != null) { String key = "href=\""; @@ -194,6 +249,56 @@ class E621 extends BasicSupport_Deprecated { @Override protected List> getChapters(URL source, InputStream in, Progress pg) throws IOException { + if (isPool(source)) { + return getChaptersPool(source, in, pg); + } else if (isSearch(source)) { + return getChaptersSearch(source, in, pg); + } + + return new LinkedList>(); + } + + private List> getChaptersSearch(URL source, + InputStream in, Progress pg) throws IOException { + List> urls = new ArrayList>(); + + String search = source.getPath(); + if (search.endsWith("/")) { + search = search.substring(0, search.length() - 1); + } + + int pos = search.lastIndexOf('/'); + if (pos >= 0) { + search = search.substring(pos + 1); + } + + String baseUrl = "https://e621.net/post/index/"; + if (source.getHost().contains("e926")) { + baseUrl = baseUrl.replace("e621", "e926"); + } + + for (int i = 1; true; i++) { + URL url = new URL(baseUrl + i + "/" + search + "/"); + try { + InputStream pageI = Instance.getCache().open(url, this, false); + try { + if (getLine(pageI, "No posts matched your search.", 0) != null) + break; + urls.add(new AbstractMap.SimpleEntry(Integer + .toString(i), url)); + } finally { + pageI.close(); + } + } catch (Exception e) { + break; + } + } + + return urls; + } + + private List> getChaptersPool(URL source, + InputStream in, Progress pg) throws IOException { List> urls = new ArrayList>(); int last = 1; // no pool/show when only one page @@ -265,4 +370,12 @@ class E621 extends BasicSupport_Deprecated { return builder.toString(); } + + private boolean isPool(URL url) { + return url.getPath().startsWith("/pool/"); + } + + private boolean isSearch(URL url) { + return url.getPath().startsWith("/post/index/"); + } } -- 2.27.0