X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Ffanfix%2Fsupported%2FFimfictionApi.java;h=43d01d19494f616c6811e5757da4a6f2e14c83cc;hb=98b95fb81566ca8b04c8d891a02c8019d8bed63d;hp=887f5f9c121397a4b8e90d7023c8d1ffac073f9b;hpb=727108fef9dcc661d45fa69ebf8b76f5128a2b6f;p=fanfix.git diff --git a/src/be/nikiroo/fanfix/supported/FimfictionApi.java b/src/be/nikiroo/fanfix/supported/FimfictionApi.java deleted file mode 100644 index 887f5f9..0000000 --- a/src/be/nikiroo/fanfix/supported/FimfictionApi.java +++ /dev/null @@ -1,416 +0,0 @@ -package be.nikiroo.fanfix.supported; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; - -import org.jsoup.nodes.Document; - -import be.nikiroo.fanfix.Instance; -import be.nikiroo.fanfix.bundles.Config; -import be.nikiroo.fanfix.data.MetaData; -import be.nikiroo.fanfix.data.Story; -import be.nikiroo.utils.IOUtils; -import be.nikiroo.utils.Image; -import be.nikiroo.utils.Progress; - -/** - * Support class for FimFiction.net - * stories, a website dedicated to My Little Pony. - *

- * This version uses the new, official API of FimFiction. - * - * @author niki - */ -class FimfictionApi extends BasicSupport { - private String oauth; - private String json; - - private Map chapterNames; - private Map chapterContents; - - public FimfictionApi() throws IOException { - if (Instance.getConfig().getBoolean( - Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) { - throw new IOException( - "Configuration is set to force HTML scrapping"); - } - - String oauth = Instance.getConfig().getString( - Config.LOGIN_FIMFICTION_APIKEY_TOKEN); - - if (oauth == null || oauth.isEmpty()) { - String clientId = Instance.getConfig().getString( - Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID) - + ""; - String clientSecret = Instance.getConfig().getString( - Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) - + ""; - - if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) { - throw new IOException("API key required for the beta API v2"); - } - - oauth = generateOAuth(clientId, clientSecret); - - Instance.getConfig().setString( - Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth); - Instance.getConfig().updateFile(); - } - - this.oauth = oauth; - } - - @Override - protected Document loadDocument(URL source) throws IOException { - json = getJsonData(); - return null; - } - - @Override - public String getOAuth() { - return oauth; - } - - @Override - protected boolean isHtml() { - return true; - } - - /** - * Extract the full JSON data we will later use to build the {@link Story}. - * - * @return the data in a JSON format - * - * @throws IOException - * in case of I/O error - */ - private String getJsonData() throws IOException { - // extract the ID from: - // https://www.fimfiction.net/story/123456/name-of-story - String storyId = getKeyText(getSource().toString(), "/story/", null, - "/"); - - // Selectors, so to download all I need and only what I need - String storyContent = "fields[story]=title,description,date_published,cover_image"; - String authorContent = "fields[author]=name"; - String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html"; - String includes = "author,chapters,tags"; - - String urlString = String.format( - "https://www.fimfiction.net/api/v2/stories/%s?" // - + "%s&%s&%s&" // - + "include=%s", // - storyId, // - storyContent, authorContent, chapterContent,// - includes); - - // URL params must be URL-encoded: "[ ]" <-> "%5B %5D" - urlString = urlString.replace("[", "%5B").replace("]", "%5D"); - - URL url = new URL(urlString); - InputStream jsonIn = Instance.getCache().open(url, this, false); - try { - return IOUtils.readSmallStream(jsonIn); - } finally { - jsonIn.close(); - } - } - - @Override - protected MetaData getMeta() throws IOException { - MetaData meta = new MetaData(); - - meta.setTitle(getKeyJson(json, 0, "type", "story", "title")); - meta.setAuthor(getKeyJson(json, 0, "type", "user", "name")); - meta.setDate(getKeyJson(json, 0, "type", "story", "date_published")); - meta.setTags(getTags()); - meta.setSource(getType().getSourceName()); - meta.setUrl(getSource().toString()); - meta.setPublisher(getType().getSourceName()); - meta.setUuid(getSource().toString()); - meta.setLuid(""); - meta.setLang("en"); - meta.setSubject("MLP"); - meta.setType(getType().toString()); - meta.setImageDocument(false); - - String coverImageLink = getKeyJson(json, 0, "type", "story", - "cover_image", "full"); - if (!coverImageLink.trim().isEmpty()) { - InputStream in = null; - try { - URL coverImageUrl = new URL(coverImageLink.trim()); - in = Instance.getCache().open(coverImageUrl, this, true); - meta.setCover(new Image(in)); - } finally { - in.close(); - } - } - - return meta; - } - - private List getTags() { - List tags = new ArrayList(); - tags.add("MLP"); - - int pos = 0; - while (pos >= 0) { - pos = indexOfJsonAfter(json, pos, "type", "story_tag"); - if (pos >= 0) { - tags.add(getKeyJson(json, pos, "name").trim()); - } - } - - return tags; - } - - @Override - protected String getDesc() { - String desc = getKeyJson(json, 0, "type", "story", "description"); - return unbbcode(desc); - } - - @Override - protected List> getChapters(Progress pg) { - chapterNames = new TreeMap(); - chapterContents = new TreeMap(); - - int pos = 0; - while (pos >= 0) { - pos = indexOfJsonAfter(json, pos, "type", "chapter"); - if (pos >= 0) { - int posNumber = indexOfJsonAfter(json, pos, "chapter_number"); - int posComa = json.indexOf(",", posNumber); - final int number = Integer.parseInt(json.substring(posNumber, - posComa).trim()); - final String title = getKeyJson(json, pos, "title"); - String notes = getKeyJson(json, pos, "authors_note_html"); - String content = getKeyJson(json, pos, "content_html"); - - if (!notes.trim().isEmpty()) { - notes = "
* * *
" + notes; - } - - chapterNames.put(number, title); - chapterContents.put(number, content + notes); - } - } - - List> urls = new ArrayList>(); - for (String title : chapterNames.values()) { - urls.add(new AbstractMap.SimpleEntry(title, null)); - } - - return urls; - } - - @Override - protected String getChapterContent(URL source, int number, Progress pg) { - return chapterContents.get(number); - } - - @Override - protected boolean supports(URL url) { - return "fimfiction.net".equals(url.getHost()) - || "www.fimfiction.net".equals(url.getHost()); - } - - /** - * Generate a new token from the client ID and secret. - *

- * Note that those tokens are long-lived, and it would be badly seen to - * create a lot of them without due cause. - *

- * So, please cache and re-use them. - * - * @param clientId - * the client ID offered on FimFiction - * @param clientSecret - * the client secret that goes with it - * - * @return a new generated token linked to that client ID - * - * @throws IOException - * in case of I/O errors - */ - static private String generateOAuth(String clientId, String clientSecret) - throws IOException { - URL url = new URL("https://www.fimfiction.net/api/v2/token"); - Map params = new HashMap(); - params.put("client_id", clientId); - params.put("client_secret", clientSecret); - params.put("grant_type", "client_credentials"); - InputStream in = Instance.getCache().openNoCache(url, null, params, - null, null); - - String jsonToken = IOUtils.readSmallStream(in); - in.close(); - - // Extract token type and token from: { - // token_type = "Bearer", - // access_token = "xxxxxxxxxxxxxx" - // } - - String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\""); - String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\""); - - return tokenType + " " + token; - } - - // afters: [name, value] pairs (or "" for any of them), can end without - // value - static private int indexOfJsonAfter(String json, int startAt, - String... afterKeys) { - ArrayList afters = new ArrayList(); - boolean name = true; - for (String key : afterKeys) { - if (key != null && !key.isEmpty()) { - afters.add("\"" + key + "\""); - } else { - afters.add("\""); - afters.add("\""); - } - - if (name) { - afters.add(":"); - } - - name = !name; - } - - return indexOfAfter(json, startAt, afters.toArray(new String[] {})); - } - - // afters: [name, value] pairs (or "" for any of them), can end without - // value but will then be empty, not NULL - static private String getKeyJson(String json, int startAt, - String... afterKeys) { - int pos = indexOfJsonAfter(json, startAt, afterKeys); - if (pos < 0) { - return ""; - } - - String result = ""; - String wip = json.substring(pos); - - pos = nextUnescapedQuote(wip, 0); - if (pos >= 0) { - wip = wip.substring(pos + 1); - pos = nextUnescapedQuote(wip, 0); - if (pos >= 0) { - result = wip.substring(0, pos); - } - } - - result = result.replace("\\t", "\t").replace("\\\"", "\""); - - return result; - } - - // next " but don't take \" into account - static private int nextUnescapedQuote(String result, int pos) { - while (pos >= 0) { - pos = result.indexOf("\"", pos); - if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) { - break; - } - - if (pos < result.length()) { - pos++; - } - } - - return pos; - } - - // quick & dirty filter - static private String unbbcode(String bbcode) { - String text = bbcode.replace("\\r\\n", "
") // - .replace("[i]", "_").replace("[/i]", "_") // - .replace("[b]", "*").replace("[/b]", "*") // - .replaceAll("\\[[^\\]]*\\]", ""); - return text; - } - - /** - * Return the text between the key and the endKey (and optional subKey can - * be passed, in this case we will look for the key first, then take the - * text between the subKey and the endKey). - * - * @param in - * the input - * @param key - * the key to match (also supports "^" at start to say - * "only if it starts with" the key) - * @param subKey - * the sub key or NULL if none - * @param endKey - * the end key or NULL for "up to the end" - * @return the text or NULL if not found - */ - static private String getKeyText(String in, String key, String subKey, - String endKey) { - String result = null; - - String line = in; - if (line != null && line.contains(key)) { - line = line.substring(line.indexOf(key) + key.length()); - if (subKey == null || subKey.isEmpty() || line.contains(subKey)) { - if (subKey != null) { - line = line.substring(line.indexOf(subKey) - + subKey.length()); - } - if (endKey == null || line.contains(endKey)) { - if (endKey != null) { - line = line.substring(0, line.indexOf(endKey)); - result = line; - } - } - } - } - - return result; - } - - /** - * Return the first index after all the given "afters" have been found in - * the {@link String}, or -1 if it was not possible. - * - * @param in - * the input - * @param startAt - * start at this position in the string - * @param afters - * the sub-keys to find before checking for key/endKey - * - * @return the text or NULL if not found - */ - static private int indexOfAfter(String in, int startAt, String... afters) { - int pos = -1; - if (in != null && !in.isEmpty()) { - pos = startAt; - if (afters != null) { - for (int i = 0; pos >= 0 && i < afters.length; i++) { - String subKey = afters[i]; - if (!subKey.isEmpty()) { - pos = in.indexOf(subKey, pos); - if (pos >= 0) { - pos += subKey.length(); - } - } - } - } - } - - return pos; - } -}