From 826e45696764433b7b46e83143c2259f416c3b61 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Sat, 25 Aug 2018 08:35:43 +0200 Subject: [PATCH] FimfictionApi: do not BasicSupport_Deprecated --- src/be/nikiroo/fanfix/Main.java | 2 + .../fanfix/supported/BasicSupport.java | 37 ++--- .../fanfix/supported/FimfictionApi.java | 132 +++++++++++++++--- 3 files changed, 121 insertions(+), 50 deletions(-) diff --git a/src/be/nikiroo/fanfix/Main.java b/src/be/nikiroo/fanfix/Main.java index 1e4daa8..53c4a86 100644 --- a/src/be/nikiroo/fanfix/Main.java +++ b/src/be/nikiroo/fanfix/Main.java @@ -527,6 +527,7 @@ public class Main { String target, boolean infoCover, Progress pg) { int exitCode = 0; + Instance.getTraceHandler().trace("Convert: " + urlString); String sourceName = urlString; try { URL source = BasicReader.getUrl(urlString); @@ -547,6 +548,7 @@ public class Main { BasicSupport support = BasicSupport.getSupport(source); if (support != null) { + Instance.getTraceHandler().trace("Support found: " + support.getClass()); Progress pgIn = new Progress(); Progress pgOut = new Progress(); if (pg != null) { diff --git a/src/be/nikiroo/fanfix/supported/BasicSupport.java b/src/be/nikiroo/fanfix/supported/BasicSupport.java index 527f4de..6843a02 100644 --- a/src/be/nikiroo/fanfix/supported/BasicSupport.java +++ b/src/be/nikiroo/fanfix/supported/BasicSupport.java @@ -84,7 +84,7 @@ public abstract class BasicSupport { protected abstract String getDesc() throws IOException; /** - * Return the list of chapters (name and resource). * + * Return the list of chapters (name and resource). *

* Can be NULL if this {@link BasicSupport} do no use chapters. * @@ -247,27 +247,6 @@ public abstract class BasicSupport { setCurrentReferer(null); } - /** - * Process the given story resource into a partially filled {@link Story} - * object containing the name and metadata, except for the description. - * - * @return the {@link Story} - * - * @throws IOException - * in case of I/O error - */ - public final Story processMeta() throws IOException { - Story story = null; - - try { - story = processMeta(false, null); - } finally { - close(); - } - - return story; - } - /** * Process the given story resource into a partially filled {@link Story} * object containing the name and metadata. @@ -321,9 +300,8 @@ public abstract class BasicSupport { } /** - * Actual processing step, without the calls to other methods. - *

- * Will convert the story resource into a fully filled {@link Story} object. + * Process the given story resource into a fully filled {@link Story} + * object. * * @param pg * the optional progress reporter @@ -333,7 +311,7 @@ public abstract class BasicSupport { * @throws IOException * in case of I/O error */ - // TODO: add final + // ADD final when BasicSupport_Deprecated is gone public Story process(Progress pg) throws IOException { setCurrentReferer(source); login(); @@ -347,8 +325,9 @@ public abstract class BasicSupport { } /** - * Process the given story resource into a fully filled {@link Story} - * object. + * Actual processing step, without the calls to other methods. + *

+ * Will convert the story resource into a fully filled {@link Story} object. * * @param pg * the optional progress reporter @@ -358,7 +337,7 @@ public abstract class BasicSupport { * @throws IOException * in case of I/O error */ - public Story doProcess(Progress pg) throws IOException { + protected Story doProcess(Progress pg) throws IOException { if (pg == null) { pg = new Progress(); } else { diff --git a/src/be/nikiroo/fanfix/supported/FimfictionApi.java b/src/be/nikiroo/fanfix/supported/FimfictionApi.java index 52678e4..f31f856 100644 --- a/src/be/nikiroo/fanfix/supported/FimfictionApi.java +++ b/src/be/nikiroo/fanfix/supported/FimfictionApi.java @@ -11,10 +11,14 @@ import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; +import org.jsoup.nodes.Document; + import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.data.MetaData; +import be.nikiroo.fanfix.data.Story; import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.Image; import be.nikiroo.utils.Progress; /** @@ -25,9 +29,8 @@ import be.nikiroo.utils.Progress; * * @author niki */ -class FimfictionApi extends BasicSupport_Deprecated { +class FimfictionApi extends BasicSupport { private String oauth; - private String storyId; private String json; private Map chapterNames; @@ -65,6 +68,12 @@ class FimfictionApi extends BasicSupport_Deprecated { this.oauth = oauth; } + @Override + protected Document loadDocument(URL source) throws IOException { + json = getJsonData(); + return null; + } + @Override public String getOAuth() { return oauth; @@ -80,11 +89,19 @@ class FimfictionApi extends BasicSupport_Deprecated { return "FimFiction.net"; } - @Override - protected void preprocess(URL source, InputStream in) throws IOException { + /** + * Extract the full JSON data we will later use to build the {@link Story}. + * + * @return the data in a JSON format + * + * @throws IOException + * in case of I/O error + */ + private String getJsonData() throws IOException { // extract the ID from: // https://www.fimfiction.net/story/123456/name-of-story - storyId = getKeyText(source.toString(), "/story/", null, "/"); + String storyId = getKeyText(getSource().toString(), "/story/", null, + "/"); // Selectors, so to download all I need and only what I need String storyContent = "fields[story]=title,description,date_published,cover_image"; @@ -106,19 +123,14 @@ class FimfictionApi extends BasicSupport_Deprecated { URL url = new URL(urlString); InputStream jsonIn = Instance.getCache().open(url, this, false); try { - json = IOUtils.readSmallStream(jsonIn); + return IOUtils.readSmallStream(jsonIn); } finally { jsonIn.close(); } } @Override - protected InputStream openInput(URL source) throws IOException { - return null; - } - - @Override - protected MetaData getMeta(URL source, InputStream in) throws IOException { + protected MetaData getMeta() throws IOException { MetaData meta = new MetaData(); meta.setTitle(getKeyJson(json, 0, "type", "story", "title")); @@ -126,9 +138,9 @@ class FimfictionApi extends BasicSupport_Deprecated { meta.setDate(getKeyJson(json, 0, "type", "story", "date_published")); meta.setTags(getTags()); meta.setSource(getSourceName()); - meta.setUrl(source.toString()); + meta.setUrl(getSource().toString()); meta.setPublisher(getSourceName()); - meta.setUuid(source.toString()); + meta.setUuid(getSource().toString()); meta.setLuid(""); meta.setLang("en"); meta.setSubject("MLP"); @@ -138,7 +150,14 @@ class FimfictionApi extends BasicSupport_Deprecated { String coverImageLink = getKeyJson(json, 0, "type", "story", "cover_image", "full"); if (!coverImageLink.trim().isEmpty()) { - meta.setCover(getImage(this, null, coverImageLink.trim())); + InputStream in = null; + try { + URL coverImageUrl = new URL(coverImageLink.trim()); + in = Instance.getCache().open(coverImageUrl, this, true); + meta.setCover(new Image(in)); + } finally { + in.close(); + } } return meta; @@ -160,14 +179,13 @@ class FimfictionApi extends BasicSupport_Deprecated { } @Override - protected String getDesc(URL source, InputStream in) { + protected String getDesc() { String desc = getKeyJson(json, 0, "type", "story", "description"); return unbbcode(desc); } @Override - protected List> getChapters(URL source, InputStream in, - Progress pg) { + protected List> getChapters(Progress pg) { chapterNames = new TreeMap(); chapterContents = new TreeMap(); @@ -201,8 +219,7 @@ class FimfictionApi extends BasicSupport_Deprecated { } @Override - protected String getChapterContent(URL source, InputStream in, int number, - Progress pg) { + protected String getChapterContent(URL source, int number, Progress pg) { return chapterContents.get(number); } @@ -248,8 +265,8 @@ class FimfictionApi extends BasicSupport_Deprecated { // access_token = "xxxxxxxxxxxxxx" // } - String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\""); String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\""); + String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\""); return tokenType + " " + token; } @@ -328,4 +345,77 @@ class FimfictionApi extends BasicSupport_Deprecated { .replaceAll("\\[[^\\]]*\\]", ""); return text; } + + /** + * Return the text between the key and the endKey (and optional subKey can + * be passed, in this case we will look for the key first, then take the + * text between the subKey and the endKey). + * + * @param in + * the input + * @param key + * the key to match (also supports "^" at start to say + * "only if it starts with" the key) + * @param subKey + * the sub key or NULL if none + * @param endKey + * the end key or NULL for "up to the end" + * @return the text or NULL if not found + */ + static private String getKeyText(String in, String key, String subKey, + String endKey) { + String result = null; + + String line = in; + if (line != null && line.contains(key)) { + line = line.substring(line.indexOf(key) + key.length()); + if (subKey == null || subKey.isEmpty() || line.contains(subKey)) { + if (subKey != null) { + line = line.substring(line.indexOf(subKey) + + subKey.length()); + } + if (endKey == null || line.contains(endKey)) { + if (endKey != null) { + line = line.substring(0, line.indexOf(endKey)); + result = line; + } + } + } + } + + return result; + } + + /** + * Return the first index after all the given "afters" have been found in + * the {@link String}, or -1 if it was not possible. + * + * @param in + * the input + * @param startAt + * start at this position in the string + * @param afters + * the sub-keys to find before checking for key/endKey + * + * @return the text or NULL if not found + */ + static private int indexOfAfter(String in, int startAt, String... afters) { + int pos = -1; + if (in != null && !in.isEmpty()) { + pos = startAt; + if (afters != null) { + for (int i = 0; pos >= 0 && i < afters.length; i++) { + String subKey = afters[i]; + if (!subKey.isEmpty()) { + pos = in.indexOf(subKey, pos); + if (pos >= 0) { + pos += subKey.length(); + } + } + } + } + } + + return pos; + } } -- 2.27.0