package be.nikiroo.fanfix.supported;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import org.jsoup.nodes.Document;
import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.bundles.Config;
import be.nikiroo.fanfix.data.MetaData;
import be.nikiroo.fanfix.data.Story;
import be.nikiroo.utils.IOUtils;
import be.nikiroo.utils.Image;
import be.nikiroo.utils.Progress;
/**
* Support class for FimFiction.net
* stories, a website dedicated to My Little Pony.
*
* This version uses the new, official API of FimFiction.
*
* @author niki
*/
class FimfictionApi extends BasicSupport {
private String oauth;
private String json;
private Map chapterNames;
private Map chapterContents;
public FimfictionApi() throws IOException {
if (Instance.getConfig().getBoolean(
Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) {
throw new IOException(
"Configuration is set to force HTML scrapping");
}
String oauth = Instance.getConfig().getString(
Config.LOGIN_FIMFICTION_APIKEY_TOKEN);
if (oauth == null || oauth.isEmpty()) {
String clientId = Instance.getConfig().getString(
Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID)
+ "";
String clientSecret = Instance.getConfig().getString(
Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET)
+ "";
if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) {
throw new IOException("API key required for the beta API v2");
}
oauth = generateOAuth(clientId, clientSecret);
Instance.getConfig().setString(
Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth);
Instance.getConfig().updateFile();
}
this.oauth = oauth;
}
@Override
protected Document loadDocument(URL source) throws IOException {
json = getJsonData();
return null;
}
@Override
public String getOAuth() {
return oauth;
}
@Override
protected boolean isHtml() {
return true;
}
@Override
public String getSourceName() {
return "FimFiction.net";
}
/**
* Extract the full JSON data we will later use to build the {@link Story}.
*
* @return the data in a JSON format
*
* @throws IOException
* in case of I/O error
*/
private String getJsonData() throws IOException {
// extract the ID from:
// https://www.fimfiction.net/story/123456/name-of-story
String storyId = getKeyText(getSource().toString(), "/story/", null,
"/");
// Selectors, so to download all I need and only what I need
String storyContent = "fields[story]=title,description,date_published,cover_image";
String authorContent = "fields[author]=name";
String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html";
String includes = "author,chapters,tags";
String urlString = String.format(
"https://www.fimfiction.net/api/v2/stories/%s?" //
+ "%s&%s&%s&" //
+ "include=%s", //
storyId, //
storyContent, authorContent, chapterContent,//
includes);
// URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
urlString = urlString.replace("[", "%5B").replace("]", "%5D");
URL url = new URL(urlString);
InputStream jsonIn = Instance.getCache().open(url, this, false);
try {
return IOUtils.readSmallStream(jsonIn);
} finally {
jsonIn.close();
}
}
@Override
protected MetaData getMeta() throws IOException {
MetaData meta = new MetaData();
meta.setTitle(getKeyJson(json, 0, "type", "story", "title"));
meta.setAuthor(getKeyJson(json, 0, "type", "user", "name"));
meta.setDate(getKeyJson(json, 0, "type", "story", "date_published"));
meta.setTags(getTags());
meta.setSource(getSourceName());
meta.setUrl(getSource().toString());
meta.setPublisher(getSourceName());
meta.setUuid(getSource().toString());
meta.setLuid("");
meta.setLang("en");
meta.setSubject("MLP");
meta.setType(getType().toString());
meta.setImageDocument(false);
String coverImageLink = getKeyJson(json, 0, "type", "story",
"cover_image", "full");
if (!coverImageLink.trim().isEmpty()) {
InputStream in = null;
try {
URL coverImageUrl = new URL(coverImageLink.trim());
in = Instance.getCache().open(coverImageUrl, this, true);
meta.setCover(new Image(in));
} finally {
in.close();
}
}
return meta;
}
private List getTags() {
List tags = new ArrayList();
tags.add("MLP");
int pos = 0;
while (pos >= 0) {
pos = indexOfJsonAfter(json, pos, "type", "story_tag");
if (pos >= 0) {
tags.add(getKeyJson(json, pos, "name").trim());
}
}
return tags;
}
@Override
protected String getDesc() {
String desc = getKeyJson(json, 0, "type", "story", "description");
return unbbcode(desc);
}
@Override
protected List> getChapters(Progress pg) {
chapterNames = new TreeMap();
chapterContents = new TreeMap();
int pos = 0;
while (pos >= 0) {
pos = indexOfJsonAfter(json, pos, "type", "chapter");
if (pos >= 0) {
int posNumber = indexOfJsonAfter(json, pos, "chapter_number");
int posComa = json.indexOf(",", posNumber);
final int number = Integer.parseInt(json.substring(posNumber,
posComa).trim());
final String title = getKeyJson(json, pos, "title");
String notes = getKeyJson(json, pos, "authors_note_html");
String content = getKeyJson(json, pos, "content_html");
if (!notes.trim().isEmpty()) {
notes = "
* * *
" + notes;
}
chapterNames.put(number, title);
chapterContents.put(number, content + notes);
}
}
List> urls = new ArrayList>();
for (String title : chapterNames.values()) {
urls.add(new AbstractMap.SimpleEntry(title, null));
}
return urls;
}
@Override
protected String getChapterContent(URL source, int number, Progress pg) {
return chapterContents.get(number);
}
@Override
protected boolean supports(URL url) {
return "fimfiction.net".equals(url.getHost())
|| "www.fimfiction.net".equals(url.getHost());
}
/**
* Generate a new token from the client ID and secret.
*
* Note that those tokens are long-lived, and it would be badly seen to
* create a lot of them without due cause.
*
* So, please cache and re-use them.
*
* @param clientId
* the client ID offered on FimFiction
* @param clientSecret
* the client secret that goes with it
*
* @return a new generated token linked to that client ID
*
* @throws IOException
* in case of I/O errors
*/
static private String generateOAuth(String clientId, String clientSecret)
throws IOException {
URL url = new URL("https://www.fimfiction.net/api/v2/token");
Map params = new HashMap();
params.put("client_id", clientId);
params.put("client_secret", clientSecret);
params.put("grant_type", "client_credentials");
InputStream in = Instance.getCache().openNoCache(url, null, params,
null, null);
String jsonToken = IOUtils.readSmallStream(in);
in.close();
// Extract token type and token from: {
// token_type = "Bearer",
// access_token = "xxxxxxxxxxxxxx"
// }
String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\"");
String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\"");
return tokenType + " " + token;
}
// afters: [name, value] pairs (or "" for any of them), can end without
// value
static private int indexOfJsonAfter(String json, int startAt,
String... afterKeys) {
ArrayList afters = new ArrayList();
boolean name = true;
for (String key : afterKeys) {
if (key != null && !key.isEmpty()) {
afters.add("\"" + key + "\"");
} else {
afters.add("\"");
afters.add("\"");
}
if (name) {
afters.add(":");
}
name = !name;
}
return indexOfAfter(json, startAt, afters.toArray(new String[] {}));
}
// afters: [name, value] pairs (or "" for any of them), can end without
// value but will then be empty, not NULL
static private String getKeyJson(String json, int startAt,
String... afterKeys) {
int pos = indexOfJsonAfter(json, startAt, afterKeys);
if (pos < 0) {
return "";
}
String result = "";
String wip = json.substring(pos);
pos = nextUnescapedQuote(wip, 0);
if (pos >= 0) {
wip = wip.substring(pos + 1);
pos = nextUnescapedQuote(wip, 0);
if (pos >= 0) {
result = wip.substring(0, pos);
}
}
result = result.replace("\\t", "\t").replace("\\\"", "\"");
return result;
}
// next " but don't take \" into account
static private int nextUnescapedQuote(String result, int pos) {
while (pos >= 0) {
pos = result.indexOf("\"", pos);
if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) {
break;
}
if (pos < result.length()) {
pos++;
}
}
return pos;
}
// quick & dirty filter
static private String unbbcode(String bbcode) {
String text = bbcode.replace("\\r\\n", "
") //
.replace("[i]", "_").replace("[/i]", "_") //
.replace("[b]", "*").replace("[/b]", "*") //
.replaceAll("\\[[^\\]]*\\]", "");
return text;
}
/**
* Return the text between the key and the endKey (and optional subKey can
* be passed, in this case we will look for the key first, then take the
* text between the subKey and the endKey).
*
* @param in
* the input
* @param key
* the key to match (also supports "^" at start to say
* "only if it starts with" the key)
* @param subKey
* the sub key or NULL if none
* @param endKey
* the end key or NULL for "up to the end"
* @return the text or NULL if not found
*/
static private String getKeyText(String in, String key, String subKey,
String endKey) {
String result = null;
String line = in;
if (line != null && line.contains(key)) {
line = line.substring(line.indexOf(key) + key.length());
if (subKey == null || subKey.isEmpty() || line.contains(subKey)) {
if (subKey != null) {
line = line.substring(line.indexOf(subKey)
+ subKey.length());
}
if (endKey == null || line.contains(endKey)) {
if (endKey != null) {
line = line.substring(0, line.indexOf(endKey));
result = line;
}
}
}
}
return result;
}
/**
* Return the first index after all the given "afters" have been found in
* the {@link String}, or -1 if it was not possible.
*
* @param in
* the input
* @param startAt
* start at this position in the string
* @param afters
* the sub-keys to find before checking for key/endKey
*
* @return the text or NULL if not found
*/
static private int indexOfAfter(String in, int startAt, String... afters) {
int pos = -1;
if (in != null && !in.isEmpty()) {
pos = startAt;
if (afters != null) {
for (int i = 0; pos >= 0 && i < afters.length; i++) {
String subKey = afters[i];
if (!subKey.isEmpty()) {
pos = in.indexOf(subKey, pos);
if (pos >= 0) {
pos += subKey.length();
}
}
}
}
}
return pos;
}
}