Merge commit 'edcd53bbbba9f94e21f43fd03d3a2febcc2b1564'
[nikiroo-utils.git] / src / be / nikiroo / fanfix / supported / FimfictionApi.java
1 package be.nikiroo.fanfix.supported;
2
3 import java.io.IOException;
4 import java.io.InputStream;
5 import java.net.URL;
6 import java.util.AbstractMap;
7 import java.util.ArrayList;
8 import java.util.HashMap;
9 import java.util.List;
10 import java.util.Map;
11 import java.util.Map.Entry;
12 import java.util.TreeMap;
13
14 import org.jsoup.nodes.Document;
15
16 import be.nikiroo.fanfix.Instance;
17 import be.nikiroo.fanfix.bundles.Config;
18 import be.nikiroo.fanfix.data.MetaData;
19 import be.nikiroo.fanfix.data.Story;
20 import be.nikiroo.utils.IOUtils;
21 import be.nikiroo.utils.Image;
22 import be.nikiroo.utils.Progress;
23
24 /**
25 * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a>
26 * stories, a website dedicated to My Little Pony.
27 * <p>
28 * This version uses the new, official API of FimFiction.
29 *
30 * @author niki
31 */
32 class FimfictionApi extends BasicSupport {
33 private String oauth;
34 private String json;
35
36 private Map<Integer, String> chapterNames;
37 private Map<Integer, String> chapterContents;
38
39 public FimfictionApi() throws IOException {
40 if (Instance.getConfig().getBoolean(
41 Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) {
42 throw new IOException(
43 "Configuration is set to force HTML scrapping");
44 }
45
46 String oauth = Instance.getConfig().getString(
47 Config.LOGIN_FIMFICTION_APIKEY_TOKEN);
48
49 if (oauth == null || oauth.isEmpty()) {
50 String clientId = Instance.getConfig().getString(
51 Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID)
52 + "";
53 String clientSecret = Instance.getConfig().getString(
54 Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET)
55 + "";
56
57 if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) {
58 throw new IOException("API key required for the beta API v2");
59 }
60
61 oauth = generateOAuth(clientId, clientSecret);
62
63 Instance.getConfig().setString(
64 Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth);
65 Instance.getConfig().updateFile();
66 }
67
68 this.oauth = oauth;
69 }
70
71 @Override
72 protected Document loadDocument(URL source) throws IOException {
73 json = getJsonData();
74 return null;
75 }
76
77 @Override
78 public String getOAuth() {
79 return oauth;
80 }
81
82 @Override
83 protected boolean isHtml() {
84 return true;
85 }
86
87 /**
88 * Extract the full JSON data we will later use to build the {@link Story}.
89 *
90 * @return the data in a JSON format
91 *
92 * @throws IOException
93 * in case of I/O error
94 */
95 private String getJsonData() throws IOException {
96 // extract the ID from:
97 // https://www.fimfiction.net/story/123456/name-of-story
98 String storyId = getKeyText(getSource().toString(), "/story/", null,
99 "/");
100
101 // Selectors, so to download all I need and only what I need
102 String storyContent = "fields[story]=title,description,date_published,cover_image";
103 String authorContent = "fields[author]=name";
104 String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html";
105 String includes = "author,chapters,tags";
106
107 String urlString = String.format(
108 "https://www.fimfiction.net/api/v2/stories/%s?" //
109 + "%s&%s&%s&" //
110 + "include=%s", //
111 storyId, //
112 storyContent, authorContent, chapterContent,//
113 includes);
114
115 // URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
116 urlString = urlString.replace("[", "%5B").replace("]", "%5D");
117
118 URL url = new URL(urlString);
119 InputStream jsonIn = Instance.getCache().open(url, this, false);
120 try {
121 return IOUtils.readSmallStream(jsonIn);
122 } finally {
123 jsonIn.close();
124 }
125 }
126
127 @Override
128 protected MetaData getMeta() throws IOException {
129 MetaData meta = new MetaData();
130
131 meta.setTitle(getKeyJson(json, 0, "type", "story", "title"));
132 meta.setAuthor(getKeyJson(json, 0, "type", "user", "name"));
133 meta.setDate(getKeyJson(json, 0, "type", "story", "date_published"));
134 meta.setTags(getTags());
135 meta.setSource(getType().getSourceName());
136 meta.setUrl(getSource().toString());
137 meta.setPublisher(getType().getSourceName());
138 meta.setUuid(getSource().toString());
139 meta.setLuid("");
140 meta.setLang("en");
141 meta.setSubject("MLP");
142 meta.setType(getType().toString());
143 meta.setImageDocument(false);
144
145 String coverImageLink = getKeyJson(json, 0, "type", "story",
146 "cover_image", "full");
147 if (!coverImageLink.trim().isEmpty()) {
148 URL coverImageUrl = new URL(coverImageLink.trim());
149
150 // No need to use the oauth, cookies... for the cover
151 // Plus: it crashes on Android because of the referer
152 try {
153 InputStream in = Instance.getCache().open(coverImageUrl, null,
154 true);
155 try {
156 meta.setCover(new Image(in));
157 } finally {
158 in.close();
159 }
160 } catch (IOException e) {
161 Instance.getTraceHandler().error(
162 new IOException(
163 "Cannot get the story cover, ignoring...", e));
164 }
165 }
166
167 return meta;
168 }
169
170 private List<String> getTags() {
171 List<String> tags = new ArrayList<String>();
172 tags.add("MLP");
173
174 int pos = 0;
175 while (pos >= 0) {
176 pos = indexOfJsonAfter(json, pos, "type", "story_tag");
177 if (pos >= 0) {
178 tags.add(getKeyJson(json, pos, "name").trim());
179 }
180 }
181
182 return tags;
183 }
184
185 @Override
186 protected String getDesc() {
187 String desc = getKeyJson(json, 0, "type", "story", "description");
188 return unbbcode(desc);
189 }
190
191 @Override
192 protected List<Entry<String, URL>> getChapters(Progress pg) {
193 chapterNames = new TreeMap<Integer, String>();
194 chapterContents = new TreeMap<Integer, String>();
195
196 int pos = 0;
197 while (pos >= 0) {
198 pos = indexOfJsonAfter(json, pos, "type", "chapter");
199 if (pos >= 0) {
200 int posNumber = indexOfJsonAfter(json, pos, "chapter_number");
201 int posComa = json.indexOf(",", posNumber);
202 final int number = Integer.parseInt(json.substring(posNumber,
203 posComa).trim());
204 final String title = getKeyJson(json, pos, "title");
205 String notes = getKeyJson(json, pos, "authors_note_html");
206 String content = getKeyJson(json, pos, "content_html");
207
208 if (!notes.trim().isEmpty()) {
209 notes = "<br/>* * *<br/>" + notes;
210 }
211
212 chapterNames.put(number, title);
213 chapterContents.put(number, content + notes);
214 }
215 }
216
217 List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
218 for (String title : chapterNames.values()) {
219 urls.add(new AbstractMap.SimpleEntry<String, URL>(title, null));
220 }
221
222 return urls;
223 }
224
225 @Override
226 protected String getChapterContent(URL source, int number, Progress pg) {
227 return chapterContents.get(number);
228 }
229
230 @Override
231 protected boolean supports(URL url) {
232 return "fimfiction.net".equals(url.getHost())
233 || "www.fimfiction.net".equals(url.getHost());
234 }
235
236 /**
237 * Generate a new token from the client ID and secret.
238 * <p>
239 * Note that those tokens are long-lived, and it would be badly seen to
240 * create a lot of them without due cause.
241 * <p>
242 * So, please cache and re-use them.
243 *
244 * @param clientId
245 * the client ID offered on FimFiction
246 * @param clientSecret
247 * the client secret that goes with it
248 *
249 * @return a new generated token linked to that client ID
250 *
251 * @throws IOException
252 * in case of I/O errors
253 */
254 static private String generateOAuth(String clientId, String clientSecret)
255 throws IOException {
256 URL url = new URL("https://www.fimfiction.net/api/v2/token");
257 Map<String, String> params = new HashMap<String, String>();
258 params.put("client_id", clientId);
259 params.put("client_secret", clientSecret);
260 params.put("grant_type", "client_credentials");
261 InputStream in = Instance.getCache().openNoCache(url, null, params,
262 null, null);
263
264 String jsonToken = IOUtils.readSmallStream(in);
265 in.close();
266
267 // Extract token type and token from: {
268 // token_type = "Bearer",
269 // access_token = "xxxxxxxxxxxxxx"
270 // }
271
272 String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\"");
273 String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\"");
274
275 return tokenType + " " + token;
276 }
277
278 // afters: [name, value] pairs (or "" for any of them), can end without
279 // value
280 static private int indexOfJsonAfter(String json, int startAt,
281 String... afterKeys) {
282 ArrayList<String> afters = new ArrayList<String>();
283 boolean name = true;
284 for (String key : afterKeys) {
285 if (key != null && !key.isEmpty()) {
286 afters.add("\"" + key + "\"");
287 } else {
288 afters.add("\"");
289 afters.add("\"");
290 }
291
292 if (name) {
293 afters.add(":");
294 }
295
296 name = !name;
297 }
298
299 return indexOfAfter(json, startAt, afters.toArray(new String[] {}));
300 }
301
302 // afters: [name, value] pairs (or "" for any of them), can end without
303 // value but will then be empty, not NULL
304 static private String getKeyJson(String json, int startAt,
305 String... afterKeys) {
306 int pos = indexOfJsonAfter(json, startAt, afterKeys);
307 if (pos < 0) {
308 return "";
309 }
310
311 String result = "";
312 String wip = json.substring(pos);
313
314 pos = nextUnescapedQuote(wip, 0);
315 if (pos >= 0) {
316 wip = wip.substring(pos + 1);
317 pos = nextUnescapedQuote(wip, 0);
318 if (pos >= 0) {
319 result = wip.substring(0, pos);
320 }
321 }
322
323 result = result.replace("\\t", "\t").replace("\\\"", "\"");
324
325 return result;
326 }
327
328 // next " but don't take \" into account
329 static private int nextUnescapedQuote(String result, int pos) {
330 while (pos >= 0) {
331 pos = result.indexOf("\"", pos);
332 if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) {
333 break;
334 }
335
336 if (pos < result.length()) {
337 pos++;
338 }
339 }
340
341 return pos;
342 }
343
344 // quick & dirty filter
345 static private String unbbcode(String bbcode) {
346 String text = bbcode.replace("\\r\\n", "<br/>") //
347 .replace("[i]", "_").replace("[/i]", "_") //
348 .replace("[b]", "*").replace("[/b]", "*") //
349 .replaceAll("\\[[^\\]]*\\]", "");
350 return text;
351 }
352
353 /**
354 * Return the text between the key and the endKey (and optional subKey can
355 * be passed, in this case we will look for the key first, then take the
356 * text between the subKey and the endKey).
357 *
358 * @param in
359 * the input
360 * @param key
361 * the key to match (also supports "^" at start to say
362 * "only if it starts with" the key)
363 * @param subKey
364 * the sub key or NULL if none
365 * @param endKey
366 * the end key or NULL for "up to the end"
367 * @return the text or NULL if not found
368 */
369 static private String getKeyText(String in, String key, String subKey,
370 String endKey) {
371 String result = null;
372
373 String line = in;
374 if (line != null && line.contains(key)) {
375 line = line.substring(line.indexOf(key) + key.length());
376 if (subKey == null || subKey.isEmpty() || line.contains(subKey)) {
377 if (subKey != null) {
378 line = line.substring(line.indexOf(subKey)
379 + subKey.length());
380 }
381 if (endKey == null || line.contains(endKey)) {
382 if (endKey != null) {
383 line = line.substring(0, line.indexOf(endKey));
384 result = line;
385 }
386 }
387 }
388 }
389
390 return result;
391 }
392
393 /**
394 * Return the first index after all the given "afters" have been found in
395 * the {@link String}, or -1 if it was not possible.
396 *
397 * @param in
398 * the input
399 * @param startAt
400 * start at this position in the string
401 * @param afters
402 * the sub-keys to find before checking for key/endKey
403 *
404 * @return the text or NULL if not found
405 */
406 static private int indexOfAfter(String in, int startAt, String... afters) {
407 int pos = -1;
408 if (in != null && !in.isEmpty()) {
409 pos = startAt;
410 if (afters != null) {
411 for (int i = 0; pos >= 0 && i < afters.length; i++) {
412 String subKey = afters[i];
413 if (!subKey.isEmpty()) {
414 pos = in.indexOf(subKey, pos);
415 if (pos >= 0) {
416 pos += subKey.length();
417 }
418 }
419 }
420 }
421 }
422
423 return pos;
424 }
425 }