X-Git-Url: http://git.nikiroo.be/?p=fanfix.git;a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Ffanfix%2Fsupported%2FFimfictionApi.java;h=e6dd6118721b93aec36c0d7c734faad1c6d9a95b;hp=591bbb08736c921333b885c19548c51e7d5c78c2;hb=0a264fbe3d5a43516006052574a5f322d9d38897;hpb=315f14ae3752d90c683a07fa20f1aa53f6010d6d diff --git a/src/be/nikiroo/fanfix/supported/FimfictionApi.java b/src/be/nikiroo/fanfix/supported/FimfictionApi.java index 591bbb0..e6dd611 100644 --- a/src/be/nikiroo/fanfix/supported/FimfictionApi.java +++ b/src/be/nikiroo/fanfix/supported/FimfictionApi.java @@ -3,16 +3,22 @@ package be.nikiroo.fanfix.supported; import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.TreeMap; + +import org.jsoup.nodes.Document; import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.data.MetaData; +import be.nikiroo.fanfix.data.Story; import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.Image; import be.nikiroo.utils.Progress; /** @@ -25,29 +31,23 @@ import be.nikiroo.utils.Progress; */ class FimfictionApi extends BasicSupport { private String oauth; - private String storyId; private String json; private Map chapterNames; private Map chapterContents; public FimfictionApi() throws IOException { - if (Instance.getConfig().getBoolean( - Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) { - throw new IOException( - "Configuration is set to force HTML scrapping"); + if (Instance.getInstance().getConfig().getBoolean(Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) { + throw new IOException("Configuration is set to force HTML scrapping"); } - String oauth = Instance.getConfig().getString( - Config.LOGIN_FIMFICTION_APIKEY_TOKEN); + String oauth = Instance.getInstance().getConfig().getString(Config.LOGIN_FIMFICTION_APIKEY_TOKEN); if (oauth == null || oauth.isEmpty()) { - String clientId = Instance.getConfig().getString( - Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID) - + ""; - String clientSecret = Instance.getConfig().getString( - Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) + String clientId = Instance.getInstance().getConfig().getString(Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID) + ""; + String clientSecret = Instance.getInstance().getConfig() + .getString(Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) + ""; if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) { throw new IOException("API key required for the beta API v2"); @@ -55,14 +55,19 @@ class FimfictionApi extends BasicSupport { oauth = generateOAuth(clientId, clientSecret); - Instance.getConfig().setString( - Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth); - Instance.getConfig().updateFile(); + Instance.getInstance().getConfig().setString(Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth); + Instance.getInstance().getConfig().updateFile(); } this.oauth = oauth; } + @Override + protected Document loadDocument(URL source) throws IOException { + json = getJsonData(); + return null; + } + @Override public String getOAuth() { return oauth; @@ -73,71 +78,90 @@ class FimfictionApi extends BasicSupport { return true; } - @Override - public String getSourceName() { - return "FimFiction.net"; - } - - @Override - protected void preprocess(URL source, InputStream in) throws IOException { + /** + * Extract the full JSON data we will later use to build the {@link Story}. + * + * @return the data in a JSON format + * + * @throws IOException + * in case of I/O error + */ + private String getJsonData() throws IOException { // extract the ID from: // https://www.fimfiction.net/story/123456/name-of-story - storyId = getKeyText(source.toString(), "/story/", null, "/"); + String storyId = getKeyText(getSource().toString(), "/story/", null, + "/"); // Selectors, so to download all I need and only what I need String storyContent = "fields[story]=title,description,date_published,cover_image"; String authorContent = "fields[author]=name"; - String chapterContent = "fields[chapter]=chapter_number,title,content,authors_note"; - String contentContent = "fields[content]=html"; - String authorsNoteContent = "fields[authors_note]=html"; + String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html"; String includes = "author,chapters,tags"; String urlString = String.format( "https://www.fimfiction.net/api/v2/stories/%s?" // - + "%s&%s&"// + "%s&%s&%s&" // + "include=%s", // storyId, // - storyContent, authorContent, // - chapterContent, contentContent, authorsNoteContent,// + storyContent, authorContent, chapterContent,// includes); // URL params must be URL-encoded: "[ ]" <-> "%5B %5D" urlString = urlString.replace("[", "%5B").replace("]", "%5D"); URL url = new URL(urlString); - InputStream jsonIn = Instance.getCache().open(url, this, false); + InputStream jsonIn = Instance.getInstance().getCache().open(url, this, false); try { - json = IOUtils.readSmallStream(jsonIn); + return IOUtils.readSmallStream(jsonIn); } finally { jsonIn.close(); } } @Override - protected InputStream openInput(URL source) throws IOException { - return null; - } - - @Override - protected MetaData getMeta(URL source, InputStream in) throws IOException { + protected MetaData getMeta() throws IOException { MetaData meta = new MetaData(); meta.setTitle(getKeyJson(json, 0, "type", "story", "title")); meta.setAuthor(getKeyJson(json, 0, "type", "user", "name")); - meta.setDate(getKeyJson(json, 0, "type", "story", "date_published")); + meta.setDate(bsHelper.formatDate( + getKeyJson(json, 0, "type", "story", "date_published"))); meta.setTags(getTags()); - meta.setSource(getSourceName()); - meta.setUrl(source.toString()); - meta.setPublisher(getSourceName()); - meta.setUuid(source.toString()); + meta.setSource(getType().getSourceName()); + meta.setUrl(getSource().toString()); + meta.setPublisher(getType().getSourceName()); + meta.setUuid(getSource().toString()); meta.setLuid(""); - meta.setLang("EN"); + meta.setLang("en"); meta.setSubject("MLP"); meta.setType(getType().toString()); meta.setImageDocument(false); - meta.setCover(getImage(this, null, - getKeyJson(json, 0, "type", "story", "cover_image", "full"))); + + String coverImageLink = getKeyJson(json, 0, "type", "story", + "cover_image", "full"); + if (!coverImageLink.trim().isEmpty()) { + URL coverImageUrl = new URL(coverImageLink.trim()); + + // No need to use the oauth, cookies... for the cover + // Plus: it crashes on Android because of the referer + try { + InputStream in = Instance.getInstance().getCache().open(coverImageUrl, null, true); + try { + Image img = new Image(in); + if (img.getSize() == 0) { + img.close(); + throw new IOException( + "Empty image not accepted"); + } + meta.setCover(img); + } finally { + in.close(); + } + } catch (IOException e) { + Instance.getInstance().getTraceHandler() + .error(new IOException("Cannot get the story cover, ignoring...", e)); + } + } return meta; } @@ -150,7 +174,7 @@ class FimfictionApi extends BasicSupport { while (pos >= 0) { pos = indexOfJsonAfter(json, pos, "type", "story_tag"); if (pos >= 0) { - tags.add(getKeyJson(json, pos, "name")); + tags.add(getKeyJson(json, pos, "name").trim()); } } @@ -158,17 +182,15 @@ class FimfictionApi extends BasicSupport { } @Override - protected String getDesc(URL source, InputStream in) { - return getKeyJson(json, 0, "type", "story", "description"); + protected String getDesc() { + String desc = getKeyJson(json, 0, "type", "story", "description"); + return unbbcode(desc); } @Override - protected List> getChapters(URL source, InputStream in, - Progress pg) { - List> urls = new ArrayList>(); - - chapterNames = new HashMap(); - chapterContents = new HashMap(); + protected List> getChapters(Progress pg) { + chapterNames = new TreeMap(); + chapterContents = new TreeMap(); int pos = 0; while (pos >= 0) { @@ -179,38 +201,28 @@ class FimfictionApi extends BasicSupport { final int number = Integer.parseInt(json.substring(posNumber, posComa).trim()); final String title = getKeyJson(json, pos, "title"); - String notes = getKeyJson(json, pos, "authors_note", "html"); - String content = getKeyJson(json, pos, "content", "html"); - - chapterNames.put(number, title); - chapterContents - .put(number, content + "
* * *
" + notes); - - urls.add(new Entry() { - @Override - public URL setValue(URL value) { - return null; - } + String notes = getKeyJson(json, pos, "authors_note_html"); + String content = getKeyJson(json, pos, "content_html"); - @Override - public String getKey() { - return title; - } + if (!notes.trim().isEmpty()) { + notes = "
* * *
" + notes; + } - @Override - public URL getValue() { - return null; - } - }); + chapterNames.put(number, title); + chapterContents.put(number, content + notes); } } + List> urls = new ArrayList>(); + for (String title : chapterNames.values()) { + urls.add(new AbstractMap.SimpleEntry(title, null)); + } + return urls; } @Override - protected String getChapterContent(URL source, InputStream in, int number, - Progress pg) { + protected String getChapterContent(URL source, int number, Progress pg) { return chapterContents.get(number); } @@ -220,6 +232,24 @@ class FimfictionApi extends BasicSupport { || "www.fimfiction.net".equals(url.getHost()); } + /** + * Generate a new token from the client ID and secret. + *

+ * Note that those tokens are long-lived, and it would be badly seen to + * create a lot of them without due cause. + *

+ * So, please cache and re-use them. + * + * @param clientId + * the client ID offered on FimFiction + * @param clientSecret + * the client secret that goes with it + * + * @return a new generated token linked to that client ID + * + * @throws IOException + * in case of I/O errors + */ static private String generateOAuth(String clientId, String clientSecret) throws IOException { URL url = new URL("https://www.fimfiction.net/api/v2/token"); @@ -227,23 +257,18 @@ class FimfictionApi extends BasicSupport { params.put("client_id", clientId); params.put("client_secret", clientSecret); params.put("grant_type", "client_credentials"); - InputStream in = Instance.getCache().openNoCache(url, null, params, - null, null); + InputStream in = Instance.getInstance().getCache().openNoCache(url, null, params, null, null); String jsonToken = IOUtils.readSmallStream(in); + in.close(); // Extract token type and token from: { - // token_type = "bearer", + // token_type = "Bearer", // access_token = "xxxxxxxxxxxxxx" // } - String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\""); String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\""); - - // TODO: remove this once the bug is fixed on the server side - if ("bearer".equals(tokenType)) { - tokenType = "Bearer"; - } + String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\""); return tokenType + " " + token; } @@ -273,14 +298,126 @@ class FimfictionApi extends BasicSupport { } // afters: [name, value] pairs (or "" for any of them), can end without - // value + // value but will then be empty, not NULL static private String getKeyJson(String json, int startAt, String... afterKeys) { int pos = indexOfJsonAfter(json, startAt, afterKeys); if (pos < 0) { - return null; + return ""; + } + + String result = ""; + String wip = json.substring(pos); + + pos = nextUnescapedQuote(wip, 0); + if (pos >= 0) { + wip = wip.substring(pos + 1); + pos = nextUnescapedQuote(wip, 0); + if (pos >= 0) { + result = wip.substring(0, pos); + } + } + + result = result.replace("\\t", "\t").replace("\\\"", "\""); + + return result; + } + + // next " but don't take \" into account + static private int nextUnescapedQuote(String result, int pos) { + while (pos >= 0) { + pos = result.indexOf("\"", pos); + if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) { + break; + } + + if (pos < result.length()) { + pos++; + } + } + + return pos; + } + + // quick & dirty filter + static private String unbbcode(String bbcode) { + String text = bbcode.replace("\\r\\n", "
") // + .replace("[i]", "_").replace("[/i]", "_") // + .replace("[b]", "*").replace("[/b]", "*") // + .replaceAll("\\[[^\\]]*\\]", ""); + return text; + } + + /** + * Return the text between the key and the endKey (and optional subKey can + * be passed, in this case we will look for the key first, then take the + * text between the subKey and the endKey). + * + * @param in + * the input + * @param key + * the key to match (also supports "^" at start to say + * "only if it starts with" the key) + * @param subKey + * the sub key or NULL if none + * @param endKey + * the end key or NULL for "up to the end" + * @return the text or NULL if not found + */ + static private String getKeyText(String in, String key, String subKey, + String endKey) { + String result = null; + + String line = in; + if (line != null && line.contains(key)) { + line = line.substring(line.indexOf(key) + key.length()); + if (subKey == null || subKey.isEmpty() || line.contains(subKey)) { + if (subKey != null) { + line = line.substring(line.indexOf(subKey) + + subKey.length()); + } + if (endKey == null || line.contains(endKey)) { + if (endKey != null) { + line = line.substring(0, line.indexOf(endKey)); + result = line; + } + } + } + } + + return result; + } + + /** + * Return the first index after all the given "afters" have been found in + * the {@link String}, or -1 if it was not possible. + * + * @param in + * the input + * @param startAt + * start at this position in the string + * @param afters + * the sub-keys to find before checking for key/endKey + * + * @return the text or NULL if not found + */ + static private int indexOfAfter(String in, int startAt, String... afters) { + int pos = -1; + if (in != null && !in.isEmpty()) { + pos = startAt; + if (afters != null) { + for (int i = 0; pos >= 0 && i < afters.length; i++) { + String subKey = afters[i]; + if (!subKey.isEmpty()) { + pos = in.indexOf(subKey, pos); + if (pos >= 0) { + pos += subKey.length(); + } + } + } + } } - return getKeyText(json.substring(pos), "\"", null, "\""); + return pos; } }