X-Git-Url: http://git.nikiroo.be/?p=fanfix.git;a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Ffanfix%2Fsupported%2FFimfictionApi.java;h=e6dd6118721b93aec36c0d7c734faad1c6d9a95b;hp=231405b96f9678e7c233fb097ff6b7a51012d424;hb=0a264fbe3d5a43516006052574a5f322d9d38897;hpb=0ffa47548f474c1330d8d723300d9aa7a4894736 diff --git a/src/be/nikiroo/fanfix/supported/FimfictionApi.java b/src/be/nikiroo/fanfix/supported/FimfictionApi.java index 231405b..e6dd611 100644 --- a/src/be/nikiroo/fanfix/supported/FimfictionApi.java +++ b/src/be/nikiroo/fanfix/supported/FimfictionApi.java @@ -3,16 +3,22 @@ package be.nikiroo.fanfix.supported; import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.TreeMap; + +import org.jsoup.nodes.Document; import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.data.MetaData; +import be.nikiroo.fanfix.data.Story; import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.Image; import be.nikiroo.utils.Progress; /** @@ -23,31 +29,25 @@ import be.nikiroo.utils.Progress; * * @author niki */ -class FimfictionApi extends BasicSupport_Deprecated { +class FimfictionApi extends BasicSupport { private String oauth; - private String storyId; private String json; private Map chapterNames; private Map chapterContents; public FimfictionApi() throws IOException { - if (Instance.getConfig().getBoolean( - Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) { - throw new IOException( - "Configuration is set to force HTML scrapping"); + if (Instance.getInstance().getConfig().getBoolean(Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) { + throw new IOException("Configuration is set to force HTML scrapping"); } - String oauth = Instance.getConfig().getString( - Config.LOGIN_FIMFICTION_APIKEY_TOKEN); + String oauth = Instance.getInstance().getConfig().getString(Config.LOGIN_FIMFICTION_APIKEY_TOKEN); if (oauth == null || oauth.isEmpty()) { - String clientId = Instance.getConfig().getString( - Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID) - + ""; - String clientSecret = Instance.getConfig().getString( - Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) + String clientId = Instance.getInstance().getConfig().getString(Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID) + ""; + String clientSecret = Instance.getInstance().getConfig() + .getString(Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) + ""; if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) { throw new IOException("API key required for the beta API v2"); @@ -55,14 +55,19 @@ class FimfictionApi extends BasicSupport_Deprecated { oauth = generateOAuth(clientId, clientSecret); - Instance.getConfig().setString( - Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth); - Instance.getConfig().updateFile(); + Instance.getInstance().getConfig().setString(Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth); + Instance.getInstance().getConfig().updateFile(); } this.oauth = oauth; } + @Override + protected Document loadDocument(URL source) throws IOException { + json = getJsonData(); + return null; + } + @Override public String getOAuth() { return oauth; @@ -73,16 +78,19 @@ class FimfictionApi extends BasicSupport_Deprecated { return true; } - @Override - public String getSourceName() { - return "FimFiction.net"; - } - - @Override - protected void preprocess(URL source, InputStream in) throws IOException { + /** + * Extract the full JSON data we will later use to build the {@link Story}. + * + * @return the data in a JSON format + * + * @throws IOException + * in case of I/O error + */ + private String getJsonData() throws IOException { // extract the ID from: // https://www.fimfiction.net/story/123456/name-of-story - storyId = getKeyText(source.toString(), "/story/", null, "/"); + String storyId = getKeyText(getSource().toString(), "/story/", null, + "/"); // Selectors, so to download all I need and only what I need String storyContent = "fields[story]=title,description,date_published,cover_image"; @@ -102,41 +110,57 @@ class FimfictionApi extends BasicSupport_Deprecated { urlString = urlString.replace("[", "%5B").replace("]", "%5D"); URL url = new URL(urlString); - InputStream jsonIn = Instance.getCache().open(url, this, false); + InputStream jsonIn = Instance.getInstance().getCache().open(url, this, false); try { - json = IOUtils.readSmallStream(jsonIn); + return IOUtils.readSmallStream(jsonIn); } finally { jsonIn.close(); } } @Override - protected InputStream openInput(URL source) throws IOException { - return null; - } - - @Override - protected MetaData getMeta(URL source, InputStream in) throws IOException { + protected MetaData getMeta() throws IOException { MetaData meta = new MetaData(); meta.setTitle(getKeyJson(json, 0, "type", "story", "title")); meta.setAuthor(getKeyJson(json, 0, "type", "user", "name")); - meta.setDate(getKeyJson(json, 0, "type", "story", "date_published")); + meta.setDate(bsHelper.formatDate( + getKeyJson(json, 0, "type", "story", "date_published"))); meta.setTags(getTags()); - meta.setSource(getSourceName()); - meta.setUrl(source.toString()); - meta.setPublisher(getSourceName()); - meta.setUuid(source.toString()); + meta.setSource(getType().getSourceName()); + meta.setUrl(getSource().toString()); + meta.setPublisher(getType().getSourceName()); + meta.setUuid(getSource().toString()); meta.setLuid(""); - meta.setLang("EN"); + meta.setLang("en"); meta.setSubject("MLP"); meta.setType(getType().toString()); meta.setImageDocument(false); - - String coverImageLink = - getKeyJson(json, 0, "type", "story", "cover_image", "full"); + + String coverImageLink = getKeyJson(json, 0, "type", "story", + "cover_image", "full"); if (!coverImageLink.trim().isEmpty()) { - meta.setCover(getImage(this, null, coverImageLink.trim())); + URL coverImageUrl = new URL(coverImageLink.trim()); + + // No need to use the oauth, cookies... for the cover + // Plus: it crashes on Android because of the referer + try { + InputStream in = Instance.getInstance().getCache().open(coverImageUrl, null, true); + try { + Image img = new Image(in); + if (img.getSize() == 0) { + img.close(); + throw new IOException( + "Empty image not accepted"); + } + meta.setCover(img); + } finally { + in.close(); + } + } catch (IOException e) { + Instance.getInstance().getTraceHandler() + .error(new IOException("Cannot get the story cover, ignoring...", e)); + } } return meta; @@ -158,18 +182,15 @@ class FimfictionApi extends BasicSupport_Deprecated { } @Override - protected String getDesc(URL source, InputStream in) { + protected String getDesc() { String desc = getKeyJson(json, 0, "type", "story", "description"); return unbbcode(desc); } @Override - protected List> getChapters(URL source, InputStream in, - Progress pg) { - List> urls = new ArrayList>(); - - chapterNames = new HashMap(); - chapterContents = new HashMap(); + protected List> getChapters(Progress pg) { + chapterNames = new TreeMap(); + chapterContents = new TreeMap(); int pos = 0; while (pos >= 0) { @@ -182,40 +203,26 @@ class FimfictionApi extends BasicSupport_Deprecated { final String title = getKeyJson(json, pos, "title"); String notes = getKeyJson(json, pos, "authors_note_html"); String content = getKeyJson(json, pos, "content_html"); - + if (!notes.trim().isEmpty()) { notes = "
* * *
" + notes; } - - chapterNames.put(number, title); - chapterContents - .put(number, content + notes); - - urls.add(new Entry() { - @Override - public URL setValue(URL value) { - return null; - } - - @Override - public String getKey() { - return title; - } - @Override - public URL getValue() { - return null; - } - }); + chapterNames.put(number, title); + chapterContents.put(number, content + notes); } } + List> urls = new ArrayList>(); + for (String title : chapterNames.values()) { + urls.add(new AbstractMap.SimpleEntry(title, null)); + } + return urls; } @Override - protected String getChapterContent(URL source, InputStream in, int number, - Progress pg) { + protected String getChapterContent(URL source, int number, Progress pg) { return chapterContents.get(number); } @@ -250,8 +257,7 @@ class FimfictionApi extends BasicSupport_Deprecated { params.put("client_id", clientId); params.put("client_secret", clientSecret); params.put("grant_type", "client_credentials"); - InputStream in = Instance.getCache().openNoCache(url, null, params, - null, null); + InputStream in = Instance.getInstance().getCache().openNoCache(url, null, params, null, null); String jsonToken = IOUtils.readSmallStream(in); in.close(); @@ -261,8 +267,8 @@ class FimfictionApi extends BasicSupport_Deprecated { // access_token = "xxxxxxxxxxxxxx" // } - String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\""); String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\""); + String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\""); return tokenType + " " + token; } @@ -311,10 +317,9 @@ class FimfictionApi extends BasicSupport_Deprecated { result = wip.substring(0, pos); } } - - result = result.replace("\\t", "\t") - .replace("\\\"", "\""); - + + result = result.replace("\\t", "\t").replace("\\\"", "\""); + return result; } @@ -342,4 +347,77 @@ class FimfictionApi extends BasicSupport_Deprecated { .replaceAll("\\[[^\\]]*\\]", ""); return text; } + + /** + * Return the text between the key and the endKey (and optional subKey can + * be passed, in this case we will look for the key first, then take the + * text between the subKey and the endKey). + * + * @param in + * the input + * @param key + * the key to match (also supports "^" at start to say + * "only if it starts with" the key) + * @param subKey + * the sub key or NULL if none + * @param endKey + * the end key or NULL for "up to the end" + * @return the text or NULL if not found + */ + static private String getKeyText(String in, String key, String subKey, + String endKey) { + String result = null; + + String line = in; + if (line != null && line.contains(key)) { + line = line.substring(line.indexOf(key) + key.length()); + if (subKey == null || subKey.isEmpty() || line.contains(subKey)) { + if (subKey != null) { + line = line.substring(line.indexOf(subKey) + + subKey.length()); + } + if (endKey == null || line.contains(endKey)) { + if (endKey != null) { + line = line.substring(0, line.indexOf(endKey)); + result = line; + } + } + } + } + + return result; + } + + /** + * Return the first index after all the given "afters" have been found in + * the {@link String}, or -1 if it was not possible. + * + * @param in + * the input + * @param startAt + * start at this position in the string + * @param afters + * the sub-keys to find before checking for key/endKey + * + * @return the text or NULL if not found + */ + static private int indexOfAfter(String in, int startAt, String... afters) { + int pos = -1; + if (in != null && !in.isEmpty()) { + pos = startAt; + if (afters != null) { + for (int i = 0; pos >= 0 && i < afters.length; i++) { + String subKey = afters[i]; + if (!subKey.isEmpty()) { + pos = in.indexOf(subKey, pos); + if (pos >= 0) { + pos += subKey.length(); + } + } + } + } + } + + return pos; + } }