import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
+import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.jsoup.nodes.Document;
import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.bundles.Config;
import be.nikiroo.fanfix.data.MetaData;
+import be.nikiroo.fanfix.data.Story;
import be.nikiroo.utils.IOUtils;
+import be.nikiroo.utils.Image;
import be.nikiroo.utils.Progress;
/**
*/
class FimfictionApi extends BasicSupport {
private String oauth;
- private String storyId;
private String json;
private Map<Integer, String> chapterNames;
private Map<Integer, String> chapterContents;
public FimfictionApi() throws IOException {
- if (Instance.getConfig().getBoolean(
- Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) {
- throw new IOException(
- "Configuration is set to force HTML scrapping");
+ if (Instance.getInstance().getConfig().getBoolean(Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) {
+ throw new IOException("Configuration is set to force HTML scrapping");
}
- String oauth = Instance.getConfig().getString(
- Config.LOGIN_FIMFICTION_APIKEY_TOKEN);
+ String oauth = Instance.getInstance().getConfig().getString(Config.LOGIN_FIMFICTION_APIKEY_TOKEN);
if (oauth == null || oauth.isEmpty()) {
- String clientId = Instance.getConfig().getString(
- Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID)
- + "";
- String clientSecret = Instance.getConfig().getString(
- Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET)
+ String clientId = Instance.getInstance().getConfig().getString(Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID)
+ "";
+ String clientSecret = Instance.getInstance().getConfig()
+ .getString(Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) + "";
if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) {
throw new IOException("API key required for the beta API v2");
oauth = generateOAuth(clientId, clientSecret);
- Instance.getConfig().setString(
- Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth);
- Instance.getConfig().updateFile();
+ Instance.getInstance().getConfig().setString(Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth);
+ Instance.getInstance().getConfig().updateFile();
}
this.oauth = oauth;
}
+ @Override
+ protected Document loadDocument(URL source) throws IOException {
+ json = getJsonData();
+ return null;
+ }
+
@Override
public String getOAuth() {
return oauth;
return true;
}
- @Override
- public String getSourceName() {
- return "FimFiction.net";
- }
-
- @Override
- protected void preprocess(URL source, InputStream in) throws IOException {
+ /**
+ * Extract the full JSON data we will later use to build the {@link Story}.
+ *
+ * @return the data in a JSON format
+ *
+ * @throws IOException
+ * in case of I/O error
+ */
+ private String getJsonData() throws IOException {
// extract the ID from:
// https://www.fimfiction.net/story/123456/name-of-story
- storyId = getKeyText(source.toString(), "/story/", null, "/");
+ String storyId = getKeyText(getSource().toString(), "/story/", null,
+ "/");
// Selectors, so to download all I need and only what I need
String storyContent = "fields[story]=title,description,date_published,cover_image";
String authorContent = "fields[author]=name";
- String chapterContent = "fields[chapter]=chapter_number,title,content,authors_note";
- String contentContent = "fields[content]=html";
- String authorsNoteContent = "fields[authors_note]=html";
+ String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html";
String includes = "author,chapters,tags";
String urlString = String.format(
"https://www.fimfiction.net/api/v2/stories/%s?" //
- + "%s&%s&"//
+ "%s&%s&%s&" //
+ "include=%s", //
storyId, //
- storyContent, authorContent, //
- chapterContent, contentContent, authorsNoteContent,//
+ storyContent, authorContent, chapterContent,//
includes);
// URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
urlString = urlString.replace("[", "%5B").replace("]", "%5D");
URL url = new URL(urlString);
- InputStream jsonIn = Instance.getCache().open(url, this, false);
+ InputStream jsonIn = Instance.getInstance().getCache().open(url, this, false);
try {
- json = IOUtils.readSmallStream(jsonIn);
+ return IOUtils.readSmallStream(jsonIn);
} finally {
jsonIn.close();
}
}
@Override
- protected InputStream openInput(URL source) throws IOException {
- return null;
- }
-
- @Override
- protected MetaData getMeta(URL source, InputStream in) throws IOException {
+ protected MetaData getMeta() throws IOException {
MetaData meta = new MetaData();
meta.setTitle(getKeyJson(json, 0, "type", "story", "title"));
meta.setAuthor(getKeyJson(json, 0, "type", "user", "name"));
- meta.setDate(getKeyJson(json, 0, "type", "story", "date_published"));
+ meta.setDate(bsHelper.formatDate(
+ getKeyJson(json, 0, "type", "story", "date_published")));
meta.setTags(getTags());
- meta.setSource(getSourceName());
- meta.setUrl(source.toString());
- meta.setPublisher(getSourceName());
- meta.setUuid(source.toString());
+ meta.setSource(getType().getSourceName());
+ meta.setUrl(getSource().toString());
+ meta.setPublisher(getType().getSourceName());
+ meta.setUuid(getSource().toString());
meta.setLuid("");
- meta.setLang("EN");
+ meta.setLang("en");
meta.setSubject("MLP");
meta.setType(getType().toString());
meta.setImageDocument(false);
- meta.setCover(getImage(this, null,
- getKeyJson(json, 0, "type", "story", "cover_image", "full")));
+
+ String coverImageLink = getKeyJson(json, 0, "type", "story",
+ "cover_image", "full");
+ if (!coverImageLink.trim().isEmpty()) {
+ URL coverImageUrl = new URL(coverImageLink.trim());
+
+ // No need to use the oauth, cookies... for the cover
+ // Plus: it crashes on Android because of the referer
+ try {
+ InputStream in = Instance.getInstance().getCache().open(coverImageUrl, null, true);
+ try {
+ Image img = new Image(in);
+ if (img.getSize() == 0) {
+ img.close();
+ throw new IOException(
+ "Empty image not accepted");
+ }
+ meta.setCover(img);
+ } finally {
+ in.close();
+ }
+ } catch (IOException e) {
+ Instance.getInstance().getTraceHandler()
+ .error(new IOException("Cannot get the story cover, ignoring...", e));
+ }
+ }
return meta;
}
while (pos >= 0) {
pos = indexOfJsonAfter(json, pos, "type", "story_tag");
if (pos >= 0) {
- tags.add(getKeyJson(json, pos, "name"));
+ tags.add(getKeyJson(json, pos, "name").trim());
}
}
}
@Override
- protected String getDesc(URL source, InputStream in) {
+ protected String getDesc() {
String desc = getKeyJson(json, 0, "type", "story", "description");
return unbbcode(desc);
}
@Override
- protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
- Progress pg) {
- List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
-
- chapterNames = new HashMap<Integer, String>();
- chapterContents = new HashMap<Integer, String>();
+ protected List<Entry<String, URL>> getChapters(Progress pg) {
+ chapterNames = new TreeMap<Integer, String>();
+ chapterContents = new TreeMap<Integer, String>();
int pos = 0;
while (pos >= 0) {
final int number = Integer.parseInt(json.substring(posNumber,
posComa).trim());
final String title = getKeyJson(json, pos, "title");
- String notes = getKeyJson(json, pos, "authors_note", "html");
- String content = getKeyJson(json, pos, "content", "html");
-
- chapterNames.put(number, title);
- chapterContents
- .put(number, content + "<br/>* * *<br/>" + notes);
-
- urls.add(new Entry<String, URL>() {
- @Override
- public URL setValue(URL value) {
- return null;
- }
+ String notes = getKeyJson(json, pos, "authors_note_html");
+ String content = getKeyJson(json, pos, "content_html");
- @Override
- public String getKey() {
- return title;
- }
+ if (!notes.trim().isEmpty()) {
+ notes = "<br/>* * *<br/>" + notes;
+ }
- @Override
- public URL getValue() {
- return null;
- }
- });
+ chapterNames.put(number, title);
+ chapterContents.put(number, content + notes);
}
}
+ List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
+ for (String title : chapterNames.values()) {
+ urls.add(new AbstractMap.SimpleEntry<String, URL>(title, null));
+ }
+
return urls;
}
@Override
- protected String getChapterContent(URL source, InputStream in, int number,
- Progress pg) {
+ protected String getChapterContent(URL source, int number, Progress pg) {
return chapterContents.get(number);
}
params.put("client_id", clientId);
params.put("client_secret", clientSecret);
params.put("grant_type", "client_credentials");
- InputStream in = Instance.getCache().openNoCache(url, null, params,
- null, null);
+ InputStream in = Instance.getInstance().getCache().openNoCache(url, null, params, null, null);
String jsonToken = IOUtils.readSmallStream(in);
+ in.close();
// Extract token type and token from: {
// token_type = "Bearer",
// access_token = "xxxxxxxxxxxxxx"
// }
- String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\"");
String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\"");
+ String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\"");
return tokenType + " " + token;
}
}
// afters: [name, value] pairs (or "" for any of them), can end without
- // value
+ // value but will then be empty, not NULL
static private String getKeyJson(String json, int startAt,
String... afterKeys) {
int pos = indexOfJsonAfter(json, startAt, afterKeys);
if (pos < 0) {
- return null;
+ return "";
}
- String result = null;
+ String result = "";
String wip = json.substring(pos);
pos = nextUnescapedQuote(wip, 0);
}
}
+ result = result.replace("\\t", "\t").replace("\\\"", "\"");
+
return result;
}
.replaceAll("\\[[^\\]]*\\]", "");
return text;
}
+
+ /**
+ * Return the text between the key and the endKey (and optional subKey can
+ * be passed, in this case we will look for the key first, then take the
+ * text between the subKey and the endKey).
+ *
+ * @param in
+ * the input
+ * @param key
+ * the key to match (also supports "^" at start to say
+ * "only if it starts with" the key)
+ * @param subKey
+ * the sub key or NULL if none
+ * @param endKey
+ * the end key or NULL for "up to the end"
+ * @return the text or NULL if not found
+ */
+ static private String getKeyText(String in, String key, String subKey,
+ String endKey) {
+ String result = null;
+
+ String line = in;
+ if (line != null && line.contains(key)) {
+ line = line.substring(line.indexOf(key) + key.length());
+ if (subKey == null || subKey.isEmpty() || line.contains(subKey)) {
+ if (subKey != null) {
+ line = line.substring(line.indexOf(subKey)
+ + subKey.length());
+ }
+ if (endKey == null || line.contains(endKey)) {
+ if (endKey != null) {
+ line = line.substring(0, line.indexOf(endKey));
+ result = line;
+ }
+ }
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * Return the first index after all the given "afters" have been found in
+ * the {@link String}, or -1 if it was not possible.
+ *
+ * @param in
+ * the input
+ * @param startAt
+ * start at this position in the string
+ * @param afters
+ * the sub-keys to find before checking for key/endKey
+ *
+ * @return the text or NULL if not found
+ */
+ static private int indexOfAfter(String in, int startAt, String... afters) {
+ int pos = -1;
+ if (in != null && !in.isEmpty()) {
+ pos = startAt;
+ if (afters != null) {
+ for (int i = 0; pos >= 0 && i < afters.length; i++) {
+ String subKey = afters[i];
+ if (!subKey.isEmpty()) {
+ pos = in.indexOf(subKey, pos);
+ if (pos >= 0) {
+ pos += subKey.length();
+ }
+ }
+ }
+ }
+ }
+
+ return pos;
+ }
}
git://git.nikiroo.be / fanfix.git / blobdiff