From: Niki Roo Date: Wed, 12 Jul 2017 19:52:17 +0000 (+0200) Subject: New FimFiction.net API downloading: X-Git-Tag: fanfix-1.6.0~15 X-Git-Url: http://git.nikiroo.be/?p=fanfix.git;a=commitdiff_plain;h=315f14ae3752d90c683a07fa20f1aa53f6010d6d New FimFiction.net API downloading: - the new class FimfictionApi can download stories from the new API - it will only do so if a token (or a client key/pass) is conigured - Cache was adapted to use directories (shorter filenames) - Cache now supports GET/POST parameters - Cache now supports OAuth2 (enough for Fimfiction new API) - Debug mode will now output cache hit/miss on stderr --- diff --git a/changelog.md b/changelog.md index cfcfb82..3747022 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,8 @@ - A server option to offer stories on the network - A remote library to get said stories from the network - Update to latest version of nikiroo-utils +- New option to download from FimFiction via the new beta API +- Cache update (you may want to clear your current cache) ## Version 1.5.3 diff --git a/libs/nikiroo-utils-2.0.0-sources.jar b/libs/nikiroo-utils-2.1.0-sources.jar similarity index 76% rename from libs/nikiroo-utils-2.0.0-sources.jar rename to libs/nikiroo-utils-2.1.0-sources.jar index 26cad68..b6143e8 100644 Binary files a/libs/nikiroo-utils-2.0.0-sources.jar and b/libs/nikiroo-utils-2.1.0-sources.jar differ diff --git a/src/be/nikiroo/fanfix/Cache.java b/src/be/nikiroo/fanfix/Cache.java index 40ce15e..9983d78 100644 --- a/src/be/nikiroo/fanfix/Cache.java +++ b/src/be/nikiroo/fanfix/Cache.java @@ -141,7 +141,13 @@ public class Cache { // MUST NOT return null try { InputStream in = load(originalUrl, false, stable); + if (Instance.isDebug()) { + System.err.println("Cache " + (in != null ? "hit" : "miss") + + ": " + url); + } + if (in == null) { + try { save(url, support, originalUrl); } catch (IOException e) { @@ -176,7 +182,7 @@ public class Cache { */ public InputStream openNoCache(URL url, BasicSupport support) throws IOException { - return openNoCache(url, support, url, null); + return openNoCache(url, support, url, null, null, null); } /** @@ -189,6 +195,10 @@ public class Cache { * the {@link BasicSupport} used for the cookies * @param postParams * the POST parameters + * @param getParams + * the GET parameters (priority over POST) + * @param oauth + * OAuth authorization (aka, "bearer XXXXXXX") * * @return the {@link InputStream} of the opened page * @@ -196,8 +206,9 @@ public class Cache { * in case of I/O error */ public InputStream openNoCache(URL url, BasicSupport support, - Map postParams) throws IOException { - return openNoCache(url, support, url, postParams); + Map postParams, Map getParams, + String oauth) throws IOException { + return openNoCache(url, support, url, postParams, getParams, oauth); } /** @@ -212,40 +223,75 @@ public class Cache { * the original {@link URL} before any redirection occurs * @param postParams * the POST parameters - * + * @param getParams + * the GET parameters (priority over POST) + * @param oauth + * OAuth authorization (aka, "bearer XXXXXXX") * @return the {@link InputStream} of the opened page * * @throws IOException * in case of I/O error */ private InputStream openNoCache(URL url, BasicSupport support, - final URL originalUrl, Map postParams) - throws IOException { + final URL originalUrl, Map postParams, + Map getParams, String oauth) throws IOException { + + if (Instance.isDebug()) { + System.err.println("Open no cache: " + url); + } URLConnection conn = openConnectionWithCookies(url, support); - if (postParams != null && conn instanceof HttpURLConnection) { - StringBuilder postData = new StringBuilder(); - for (Map.Entry param : postParams.entrySet()) { - if (postData.length() != 0) - postData.append('&'); - postData.append(URLEncoder.encode(param.getKey(), "UTF-8")); - postData.append('='); - postData.append(URLEncoder.encode( - String.valueOf(param.getValue()), "UTF-8")); + if (support != null) { + // priority: arguments + if (oauth == null) { + oauth = support.getOAuth(); + } + } + + // Priority: GET over POST + Map params = getParams; + if (getParams == null) { + params = postParams; + } + + if ((params != null || oauth != null) + && conn instanceof HttpURLConnection) { + StringBuilder requestData = null; + if (params != null) { + requestData = new StringBuilder(); + for (Map.Entry param : params.entrySet()) { + if (requestData.length() != 0) + requestData.append('&'); + requestData.append(URLEncoder.encode(param.getKey(), + "UTF-8")); + requestData.append('='); + requestData.append(URLEncoder.encode( + String.valueOf(param.getValue()), "UTF-8")); + } + + conn.setDoOutput(true); + + if (getParams == null && postParams != null) { + ((HttpURLConnection) conn).setRequestMethod("POST"); + } + + conn.setRequestProperty("Content-Type", + "application/x-www-form-urlencoded"); + conn.setRequestProperty("charset", "utf-8"); } - conn.setDoOutput(true); - ((HttpURLConnection) conn).setRequestMethod("POST"); - conn.setRequestProperty("Content-Type", - "application/x-www-form-urlencoded"); - conn.setRequestProperty("charset", "utf-8"); + if (oauth != null) { + conn.setRequestProperty("Authorization", oauth); + } - OutputStreamWriter writer = new OutputStreamWriter( - conn.getOutputStream()); + if (requestData != null) { + OutputStreamWriter writer = new OutputStreamWriter( + conn.getOutputStream()); - writer.write(postData.toString()); - writer.flush(); - writer.close(); + writer.write(requestData.toString()); + writer.flush(); + writer.close(); + } } conn.connect(); @@ -255,7 +301,7 @@ public class Cache { && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) { String newUrl = conn.getHeaderField("Location"); return openNoCache(new URL(newUrl), support, originalUrl, - postParams); + postParams, getParams, oauth); } InputStream in = conn.getInputStream(); @@ -343,6 +389,9 @@ public class Cache { */ public File addToCache(InputStream in, String uniqueID) throws IOException { File file = getCached(uniqueID); + File subdir = new File(file.getParentFile(), "_"); + file = new File(subdir, file.getName()); + subdir.mkdir(); IOUtils.write(in, file); return file; } @@ -358,6 +407,8 @@ public class Cache { */ public InputStream getFromCache(String uniqueID) { File file = getCached(uniqueID); + File subdir = new File(file.getParentFile(), "_"); + file = new File(subdir, file.getName()); if (file.exists()) { try { return new MarkableFileInputStream(new FileInputStream(file)); @@ -377,17 +428,36 @@ public class Cache { * @return the number of cleaned items */ public int cleanCache(boolean onlyOld) { + return cleanCache(onlyOld, dir); + } + + /** + * Clean the cache (delete the cached items) in the given cache directory. + * + * @param onlyOld + * only clean the files that are considered too old + * @param cacheDir + * the cache directory to clean + * + * @return the number of cleaned items + */ + private int cleanCache(boolean onlyOld, File cacheDir) { int num = 0; - for (File file : dir.listFiles()) { - if (!onlyOld || isOld(file, true)) { - if (file.delete()) { - num++; - } else { - System.err.println("Cannot delete temporary file: " - + file.getAbsolutePath()); + for (File file : cacheDir.listFiles()) { + if (file.isDirectory()) { + num += cleanCache(onlyOld, file); + } else { + if (!onlyOld || isOld(file, true)) { + if (file.delete()) { + num++; + } else { + System.err.println("Cannot delete temporary file: " + + file.getAbsolutePath()); + } } } } + return num; } @@ -432,7 +502,8 @@ public class Cache { */ private void save(URL url, BasicSupport support, URL originalUrl) throws IOException { - InputStream in = openNoCache(url, support, originalUrl, null); + InputStream in = openNoCache(url, support, originalUrl, null, null, + null); try { File cached = getCached(originalUrl); BufferedOutputStream out = new BufferedOutputStream( @@ -513,28 +584,39 @@ public class Cache { } /** - * Get the cache resource from the cache if it is present for this - * {@link URL}. + * Return the associated cache {@link File} from this {@link URL}. * * @param url * the url * - * @return the cached version if present, NULL if not + * @return the cached {@link File} version of this {@link URL} */ private File getCached(URL url) { + File subdir = null; + String name = url.getHost(); if (name == null || name.isEmpty()) { name = url.getFile(); } else { - name = url.toString(); + File cacheDir = getCached(".").getParentFile(); + File subsubDir = new File(cacheDir, allowedChars(url.getHost())); + subdir = new File(subsubDir, "_" + allowedChars(url.getPath())); + name = allowedChars("_" + url.getQuery()); } - return getCached(name); + File cacheFile = getCached(name); + if (subdir != null) { + cacheFile = new File(subdir, cacheFile.getName()); + subdir.mkdirs(); + } + + return cacheFile; } /** - * Get the cache resource from the cache if it is present for this unique - * ID. + * Get the basic cache resource file corresponding to this unique ID. + *

+ * Note that you may need to add a sub-directory in some cases. * * @param uniqueID * the id @@ -542,10 +624,19 @@ public class Cache { * @return the cached version if present, NULL if not */ private File getCached(String uniqueID) { - uniqueID = uniqueID.replace('/', '_').replace(':', '_') - .replace("\\", "_"); + return new File(dir, allowedChars(uniqueID)); + } - return new File(dir, uniqueID); + /** + * Replace not allowed chars (in a {@link File}) by "_". + * + * @param raw + * the raw {@link String} + * + * @return the sanitised {@link String} + */ + private String allowedChars(String raw) { + return raw.replace('/', '_').replace(':', '_').replace("\\", "_"); } /** @@ -566,18 +657,14 @@ public class Cache { } if (support != null) { - try { - for (Map.Entry set : support.getCookies() - .entrySet()) { - if (builder.length() > 0) { - builder.append(';'); - } - builder.append(set.getKey()); - builder.append('='); - builder.append(set.getValue()); + for (Map.Entry set : support.getCookies() + .entrySet()) { + if (builder.length() > 0) { + builder.append(';'); } - } catch (IOException e) { - Instance.syserr(e); + builder.append(set.getKey()); + builder.append('='); + builder.append(set.getValue()); } } diff --git a/src/be/nikiroo/fanfix/Instance.java b/src/be/nikiroo/fanfix/Instance.java index 2231414..9b6448e 100644 --- a/src/be/nikiroo/fanfix/Instance.java +++ b/src/be/nikiroo/fanfix/Instance.java @@ -262,6 +262,15 @@ public class Instance { } } + /** + * The program is in DEBUG mode (more verbose). + * + * @return TRUE if it is + */ + public static boolean isDebug() { + return debug; + } + /** * Return a path, but support the special $HOME variable. * diff --git a/src/be/nikiroo/fanfix/bundles/Config.java b/src/be/nikiroo/fanfix/bundles/Config.java index 63e6465..ae60b75 100644 --- a/src/be/nikiroo/fanfix/bundles/Config.java +++ b/src/be/nikiroo/fanfix/bundles/Config.java @@ -54,5 +54,13 @@ public enum Config { @Meta(description = "Login information (password) for YiffStar to have access to all the stories (should not be necessary anymore)", format = Format.PASSWORD) LOGIN_YIFFSTAR_PASS, // @Meta(description = "If the last update check was done at least that many days, check for updates at startup (-1 for 'no checks' -- default is 1 day)", format = Format.INT) - UPDATE_INTERVAL, + UPDATE_INTERVAL, // + @Meta(description = "An API key required to create a token from FimFiction", format = Format.STRING) + LOGIN_FIMFICTION_APIKEY_CLIENT_ID, // + @Meta(description = "An API key required to create a token from FimFiction", format = Format.PASSWORD) + LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET, // + @Meta(description = "Do not use the new API, even if we have a token, and force HTML scraping", format = Format.BOOLEAN) + LOGIN_FIMFICTION_APIKEY_FORCE_HTML, // + @Meta(description = "A token is required to use the beta APIv2 from FimFiction (see APIKEY_CLIENT_*)", format = Format.PASSWORD) + LOGIN_FIMFICTION_APIKEY_TOKEN, // } diff --git a/src/be/nikiroo/fanfix/bundles/config.properties b/src/be/nikiroo/fanfix/bundles/config.properties index 061b0fb..937c421 100644 --- a/src/be/nikiroo/fanfix/bundles/config.properties +++ b/src/be/nikiroo/fanfix/bundles/config.properties @@ -65,3 +65,9 @@ LOGIN_YIFFSTAR_PASS = # If the last update check was done at least that many days, check for updates at startup (-1 for 'no checks' -- default is 1 day) # (FORMAT: INT) UPDATE_INTERVAL = +# An API key required to use the beta APIv2 from FimFiction +# (FORMAT: PASSWORD) +LOGIN_FIMFICTION_APIKEY = +# Do not use the new API, even if we have an API key, and force HTML scraping +# (FORMAT: BOOLEAN) +LOGIN_FIMFICTION_FORCE_HTML = diff --git a/src/be/nikiroo/fanfix/supported/BasicSupport.java b/src/be/nikiroo/fanfix/supported/BasicSupport.java index d56e52d..e6089eb 100644 --- a/src/be/nikiroo/fanfix/supported/BasicSupport.java +++ b/src/be/nikiroo/fanfix/supported/BasicSupport.java @@ -271,8 +271,8 @@ public abstract class BasicSupport { * @throws IOException * in case of I/O error */ + @SuppressWarnings("unused") public void login() throws IOException { - } /** @@ -283,14 +283,20 @@ public abstract class BasicSupport { * it. * * @return the cookies - * - * @throws IOException - * in case of I/O error */ - public Map getCookies() throws IOException { + public Map getCookies() { return new HashMap(); } + /** + * OAuth authorisation (aka, "bearer XXXXXXX"). + * + * @return the OAuth string + */ + public String getOAuth() { + return null; + } + /** * Return the canonical form of the main {@link URL}. * @@ -302,6 +308,7 @@ public abstract class BasicSupport { * @throws IOException * in case of I/O error */ + @SuppressWarnings("unused") public URL getCanonicalUrl(URL source) throws IOException { return source; } @@ -355,11 +362,7 @@ public abstract class BasicSupport { setCurrentReferer(url); - in = openInput(url); - if (in == null) { - return null; - } - + in = openInput(url); // NULL allowed here try { preprocess(url, getInput()); pg.setProgress(30); @@ -465,9 +468,12 @@ public abstract class BasicSupport { int i = 1; for (Entry chap : chapters) { pgChaps.setName("Extracting chapter " + i); - setCurrentReferer(chap.getValue()); - InputStream chapIn = Instance.getCache().open( - chap.getValue(), this, true); + InputStream chapIn = null; + if (chap.getValue() != null) { + setCurrentReferer(chap.getValue()); + chapIn = Instance.getCache().open(chap.getValue(), + this, true); + } pgChaps.setProgress(i * 100); try { Progress pgGetChapterContent = new Progress(); @@ -494,7 +500,9 @@ public abstract class BasicSupport { story.getMeta().setWords(words); } } finally { - chapIn.close(); + if (chapIn != null) { + chapIn.close(); + } } i++; @@ -586,6 +594,7 @@ public abstract class BasicSupport { * @throws IOException * on I/O error */ + @SuppressWarnings("unused") protected void close() throws IOException { } @@ -972,6 +981,9 @@ public abstract class BasicSupport { /** * Open the input file that will be used through the support. + *

+ * Can return NULL, in which case you are supposed to work without an + * {@link InputStream}. * * @param source * the source {@link URL} @@ -985,22 +997,6 @@ public abstract class BasicSupport { return Instance.getCache().open(source, this, false); } - /** - * Reset the given {@link InputStream} and return it. - * - * @param in - * the {@link InputStream} to reset - * - * @return the same {@link InputStream} after reset - */ - protected InputStream reset(InputStream in) { - try { - in.reset(); - } catch (IOException e) { - } - return in; - } - /** * Reset then return {@link BasicSupport#in}. * @@ -1400,7 +1396,12 @@ public abstract class BasicSupport { case INFO_TEXT: return new InfoText().setType(type); case FIMFICTION: - return new Fimfiction().setType(type); + try { + // Can fail if no client key or NO in options + return new FimfictionApi().setType(type); + } catch (IOException e) { + return new Fimfiction().setType(type); + } case FANFICTION: return new Fanfiction().setType(type); case TEXT: @@ -1422,6 +1423,25 @@ public abstract class BasicSupport { return null; } + /** + * Reset the given {@link InputStream} and return it. + * + * @param in + * the {@link InputStream} to reset + * + * @return the same {@link InputStream} after reset + */ + static protected InputStream reset(InputStream in) { + try { + if (in != null) { + in.reset(); + } + } catch (IOException e) { + } + + return in; + } + /** * Return the first line from the given input which correspond to the given * selectors. @@ -1438,7 +1458,8 @@ public abstract class BasicSupport { * * @return the line */ - static String getLine(InputStream in, String needle, int relativeLine) { + static protected String getLine(InputStream in, String needle, + int relativeLine) { return getLine(in, needle, relativeLine, true); } @@ -1461,15 +1482,11 @@ public abstract class BasicSupport { * * @return the line */ - static String getLine(InputStream in, String needle, int relativeLine, - boolean first) { + static protected String getLine(InputStream in, String needle, + int relativeLine, boolean first) { String rep = null; - try { - in.reset(); - } catch (IOException e) { - Instance.syserr(e); - } + reset(in); List lines = new ArrayList(); @SuppressWarnings("resource") @@ -1524,11 +1541,32 @@ public abstract class BasicSupport { * the end key or NULL for "up to the end" * @return the text or NULL if not found */ - static String getKeyLine(InputStream in, String key, String subKey, + static protected String getKeyLine(InputStream in, String key, + String subKey, String endKey) { + return getKeyText(getLine(in, key, 0), key, subKey, endKey); + } + + /** + * Return the text between the key and the endKey (and optional subKey can + * be passed, in this case we will look for the key first, then take the + * text between the subKey and the endKey). + * + * @param in + * the input + * @param key + * the key to match (also supports "^" at start to say + * "only if it starts with" the key) + * @param subKey + * the sub key or NULL if none + * @param endKey + * the end key or NULL for "up to the end" + * @return the text or NULL if not found + */ + static protected String getKeyText(String in, String key, String subKey, String endKey) { String result = null; - String line = getLine(in, key, 0); + String line = in; if (line != null && line.contains(key)) { line = line.substring(line.indexOf(key) + key.length()); if (subKey == null || subKey.isEmpty() || line.contains(subKey)) { @@ -1547,4 +1585,68 @@ public abstract class BasicSupport { return result; } + + /** + * Return the text between the key and the endKey (optional subKeys can be + * passed, in this case we will look for the subKeys first, then take the + * text between the key and the endKey). + * + * @param in + * the input + * @param key + * the key to match + * @param endKey + * the end key or NULL for "up to the end" + * @param afters + * the sub-keys to find before checking for key/endKey + * + * @return the text or NULL if not found + */ + static protected String getKeyTextAfter(String in, String key, + String endKey, String... afters) { + + if (in != null && !in.isEmpty()) { + int pos = indexOfAfter(in, 0, afters); + if (pos < 0) { + return null; + } + + in = in.substring(pos); + } + + return getKeyText(in, key, null, endKey); + } + + /** + * Return the first index after all the given "afters" have been found in + * the {@link String}, or -1 if it was not possible. + * + * @param in + * the input + * @param startAt + * start at this position in the string + * @param afters + * the sub-keys to find before checking for key/endKey + * + * @return the text or NULL if not found + */ + static protected int indexOfAfter(String in, int startAt, String... afters) { + int pos = -1; + if (in != null && !in.isEmpty()) { + pos = startAt; + if (afters != null) { + for (int i = 0; pos >= 0 && i < afters.length; i++) { + String subKey = afters[i]; + if (!subKey.isEmpty()) { + pos = in.indexOf(subKey, pos); + if (pos >= 0) { + pos += subKey.length(); + } + } + } + } + } + + return pos; + } } diff --git a/src/be/nikiroo/fanfix/supported/FimfictionApi.java b/src/be/nikiroo/fanfix/supported/FimfictionApi.java new file mode 100644 index 0000000..591bbb0 --- /dev/null +++ b/src/be/nikiroo/fanfix/supported/FimfictionApi.java @@ -0,0 +1,286 @@ +package be.nikiroo.fanfix.supported; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import be.nikiroo.fanfix.Instance; +import be.nikiroo.fanfix.bundles.Config; +import be.nikiroo.fanfix.data.MetaData; +import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.Progress; + +/** + * Support class for FimFiction.net + * stories, a website dedicated to My Little Pony. + *

+ * This version uses the new, official API of FimFiction. + * + * @author niki + */ +class FimfictionApi extends BasicSupport { + private String oauth; + private String storyId; + private String json; + + private Map chapterNames; + private Map chapterContents; + + public FimfictionApi() throws IOException { + if (Instance.getConfig().getBoolean( + Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) { + throw new IOException( + "Configuration is set to force HTML scrapping"); + } + + String oauth = Instance.getConfig().getString( + Config.LOGIN_FIMFICTION_APIKEY_TOKEN); + + if (oauth == null || oauth.isEmpty()) { + String clientId = Instance.getConfig().getString( + Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID) + + ""; + String clientSecret = Instance.getConfig().getString( + Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) + + ""; + + if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) { + throw new IOException("API key required for the beta API v2"); + } + + oauth = generateOAuth(clientId, clientSecret); + + Instance.getConfig().setString( + Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth); + Instance.getConfig().updateFile(); + } + + this.oauth = oauth; + } + + @Override + public String getOAuth() { + return oauth; + } + + @Override + protected boolean isHtml() { + return true; + } + + @Override + public String getSourceName() { + return "FimFiction.net"; + } + + @Override + protected void preprocess(URL source, InputStream in) throws IOException { + // extract the ID from: + // https://www.fimfiction.net/story/123456/name-of-story + storyId = getKeyText(source.toString(), "/story/", null, "/"); + + // Selectors, so to download all I need and only what I need + String storyContent = "fields[story]=title,description,date_published,cover_image"; + String authorContent = "fields[author]=name"; + String chapterContent = "fields[chapter]=chapter_number,title,content,authors_note"; + String contentContent = "fields[content]=html"; + String authorsNoteContent = "fields[authors_note]=html"; + String includes = "author,chapters,tags"; + + String urlString = String.format( + "https://www.fimfiction.net/api/v2/stories/%s?" // + + "%s&%s&"// + + "%s&%s&%s&" // + + "include=%s", // + storyId, // + storyContent, authorContent, // + chapterContent, contentContent, authorsNoteContent,// + includes); + + // URL params must be URL-encoded: "[ ]" <-> "%5B %5D" + urlString = urlString.replace("[", "%5B").replace("]", "%5D"); + + URL url = new URL(urlString); + InputStream jsonIn = Instance.getCache().open(url, this, false); + try { + json = IOUtils.readSmallStream(jsonIn); + } finally { + jsonIn.close(); + } + } + + @Override + protected InputStream openInput(URL source) throws IOException { + return null; + } + + @Override + protected MetaData getMeta(URL source, InputStream in) throws IOException { + MetaData meta = new MetaData(); + + meta.setTitle(getKeyJson(json, 0, "type", "story", "title")); + meta.setAuthor(getKeyJson(json, 0, "type", "user", "name")); + meta.setDate(getKeyJson(json, 0, "type", "story", "date_published")); + meta.setTags(getTags()); + meta.setSource(getSourceName()); + meta.setUrl(source.toString()); + meta.setPublisher(getSourceName()); + meta.setUuid(source.toString()); + meta.setLuid(""); + meta.setLang("EN"); + meta.setSubject("MLP"); + meta.setType(getType().toString()); + meta.setImageDocument(false); + meta.setCover(getImage(this, null, + getKeyJson(json, 0, "type", "story", "cover_image", "full"))); + + return meta; + } + + private List getTags() { + List tags = new ArrayList(); + tags.add("MLP"); + + int pos = 0; + while (pos >= 0) { + pos = indexOfJsonAfter(json, pos, "type", "story_tag"); + if (pos >= 0) { + tags.add(getKeyJson(json, pos, "name")); + } + } + + return tags; + } + + @Override + protected String getDesc(URL source, InputStream in) { + return getKeyJson(json, 0, "type", "story", "description"); + } + + @Override + protected List> getChapters(URL source, InputStream in, + Progress pg) { + List> urls = new ArrayList>(); + + chapterNames = new HashMap(); + chapterContents = new HashMap(); + + int pos = 0; + while (pos >= 0) { + pos = indexOfJsonAfter(json, pos, "type", "chapter"); + if (pos >= 0) { + int posNumber = indexOfJsonAfter(json, pos, "chapter_number"); + int posComa = json.indexOf(",", posNumber); + final int number = Integer.parseInt(json.substring(posNumber, + posComa).trim()); + final String title = getKeyJson(json, pos, "title"); + String notes = getKeyJson(json, pos, "authors_note", "html"); + String content = getKeyJson(json, pos, "content", "html"); + + chapterNames.put(number, title); + chapterContents + .put(number, content + "
* * *
" + notes); + + urls.add(new Entry() { + @Override + public URL setValue(URL value) { + return null; + } + + @Override + public String getKey() { + return title; + } + + @Override + public URL getValue() { + return null; + } + }); + } + } + + return urls; + } + + @Override + protected String getChapterContent(URL source, InputStream in, int number, + Progress pg) { + return chapterContents.get(number); + } + + @Override + protected boolean supports(URL url) { + return "fimfiction.net".equals(url.getHost()) + || "www.fimfiction.net".equals(url.getHost()); + } + + static private String generateOAuth(String clientId, String clientSecret) + throws IOException { + URL url = new URL("https://www.fimfiction.net/api/v2/token"); + Map params = new HashMap(); + params.put("client_id", clientId); + params.put("client_secret", clientSecret); + params.put("grant_type", "client_credentials"); + InputStream in = Instance.getCache().openNoCache(url, null, params, + null, null); + + String jsonToken = IOUtils.readSmallStream(in); + + // Extract token type and token from: { + // token_type = "bearer", + // access_token = "xxxxxxxxxxxxxx" + // } + + String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\""); + String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\""); + + // TODO: remove this once the bug is fixed on the server side + if ("bearer".equals(tokenType)) { + tokenType = "Bearer"; + } + + return tokenType + " " + token; + } + + // afters: [name, value] pairs (or "" for any of them), can end without + // value + static private int indexOfJsonAfter(String json, int startAt, + String... afterKeys) { + ArrayList afters = new ArrayList(); + boolean name = true; + for (String key : afterKeys) { + if (key != null && !key.isEmpty()) { + afters.add("\"" + key + "\""); + } else { + afters.add("\""); + afters.add("\""); + } + + if (name) { + afters.add(":"); + } + + name = !name; + } + + return indexOfAfter(json, startAt, afters.toArray(new String[] {})); + } + + // afters: [name, value] pairs (or "" for any of them), can end without + // value + static private String getKeyJson(String json, int startAt, + String... afterKeys) { + int pos = indexOfJsonAfter(json, startAt, afterKeys); + if (pos < 0) { + return null; + } + + return getKeyText(json.substring(pos), "\"", null, "\""); + } +} diff --git a/src/be/nikiroo/fanfix/supported/YiffStar.java b/src/be/nikiroo/fanfix/supported/YiffStar.java index ba24e50..94c310d 100644 --- a/src/be/nikiroo/fanfix/supported/YiffStar.java +++ b/src/be/nikiroo/fanfix/supported/YiffStar.java @@ -88,7 +88,7 @@ class YiffStar extends BasicSupport { // logged in Instance.getCache() .openNoCache(new URL("https://www.sofurry.com/user/login"), - this, post).close(); + this, post, null, null).close(); } }