remote: fix 30s timeout bug
[nikiroo-utils.git] / src / be / nikiroo / fanfix / supported / FimfictionApi.java
... / ...
CommitLineData
1package be.nikiroo.fanfix.supported;
2
3import java.io.IOException;
4import java.io.InputStream;
5import java.net.URL;
6import java.util.AbstractMap;
7import java.util.ArrayList;
8import java.util.HashMap;
9import java.util.List;
10import java.util.Map;
11import java.util.Map.Entry;
12import java.util.TreeMap;
13
14import org.jsoup.nodes.Document;
15
16import be.nikiroo.fanfix.Instance;
17import be.nikiroo.fanfix.bundles.Config;
18import be.nikiroo.fanfix.data.MetaData;
19import be.nikiroo.fanfix.data.Story;
20import be.nikiroo.utils.IOUtils;
21import be.nikiroo.utils.Image;
22import be.nikiroo.utils.Progress;
23
24/**
25 * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a>
26 * stories, a website dedicated to My Little Pony.
27 * <p>
28 * This version uses the new, official API of FimFiction.
29 *
30 * @author niki
31 */
32class FimfictionApi extends BasicSupport {
33 private String oauth;
34 private String json;
35
36 private Map<Integer, String> chapterNames;
37 private Map<Integer, String> chapterContents;
38
39 public FimfictionApi() throws IOException {
40 if (Instance.getConfig().getBoolean(
41 Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) {
42 throw new IOException(
43 "Configuration is set to force HTML scrapping");
44 }
45
46 String oauth = Instance.getConfig().getString(
47 Config.LOGIN_FIMFICTION_APIKEY_TOKEN);
48
49 if (oauth == null || oauth.isEmpty()) {
50 String clientId = Instance.getConfig().getString(
51 Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID)
52 + "";
53 String clientSecret = Instance.getConfig().getString(
54 Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET)
55 + "";
56
57 if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) {
58 throw new IOException("API key required for the beta API v2");
59 }
60
61 oauth = generateOAuth(clientId, clientSecret);
62
63 Instance.getConfig().setString(
64 Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth);
65 Instance.getConfig().updateFile();
66 }
67
68 this.oauth = oauth;
69 }
70
71 @Override
72 protected Document loadDocument(URL source) throws IOException {
73 json = getJsonData();
74 return null;
75 }
76
77 @Override
78 public String getOAuth() {
79 return oauth;
80 }
81
82 @Override
83 protected boolean isHtml() {
84 return true;
85 }
86
87 @Override
88 public String getSourceName() {
89 return "FimFiction.net";
90 }
91
92 /**
93 * Extract the full JSON data we will later use to build the {@link Story}.
94 *
95 * @return the data in a JSON format
96 *
97 * @throws IOException
98 * in case of I/O error
99 */
100 private String getJsonData() throws IOException {
101 // extract the ID from:
102 // https://www.fimfiction.net/story/123456/name-of-story
103 String storyId = getKeyText(getSource().toString(), "/story/", null,
104 "/");
105
106 // Selectors, so to download all I need and only what I need
107 String storyContent = "fields[story]=title,description,date_published,cover_image";
108 String authorContent = "fields[author]=name";
109 String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html";
110 String includes = "author,chapters,tags";
111
112 String urlString = String.format(
113 "https://www.fimfiction.net/api/v2/stories/%s?" //
114 + "%s&%s&%s&" //
115 + "include=%s", //
116 storyId, //
117 storyContent, authorContent, chapterContent,//
118 includes);
119
120 // URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
121 urlString = urlString.replace("[", "%5B").replace("]", "%5D");
122
123 URL url = new URL(urlString);
124 InputStream jsonIn = Instance.getCache().open(url, this, false);
125 try {
126 return IOUtils.readSmallStream(jsonIn);
127 } finally {
128 jsonIn.close();
129 }
130 }
131
132 @Override
133 protected MetaData getMeta() throws IOException {
134 MetaData meta = new MetaData();
135
136 meta.setTitle(getKeyJson(json, 0, "type", "story", "title"));
137 meta.setAuthor(getKeyJson(json, 0, "type", "user", "name"));
138 meta.setDate(getKeyJson(json, 0, "type", "story", "date_published"));
139 meta.setTags(getTags());
140 meta.setSource(getSourceName());
141 meta.setUrl(getSource().toString());
142 meta.setPublisher(getSourceName());
143 meta.setUuid(getSource().toString());
144 meta.setLuid("");
145 meta.setLang("en");
146 meta.setSubject("MLP");
147 meta.setType(getType().toString());
148 meta.setImageDocument(false);
149
150 String coverImageLink = getKeyJson(json, 0, "type", "story",
151 "cover_image", "full");
152 if (!coverImageLink.trim().isEmpty()) {
153 InputStream in = null;
154 try {
155 URL coverImageUrl = new URL(coverImageLink.trim());
156 in = Instance.getCache().open(coverImageUrl, this, true);
157 meta.setCover(new Image(in));
158 } finally {
159 in.close();
160 }
161 }
162
163 return meta;
164 }
165
166 private List<String> getTags() {
167 List<String> tags = new ArrayList<String>();
168 tags.add("MLP");
169
170 int pos = 0;
171 while (pos >= 0) {
172 pos = indexOfJsonAfter(json, pos, "type", "story_tag");
173 if (pos >= 0) {
174 tags.add(getKeyJson(json, pos, "name").trim());
175 }
176 }
177
178 return tags;
179 }
180
181 @Override
182 protected String getDesc() {
183 String desc = getKeyJson(json, 0, "type", "story", "description");
184 return unbbcode(desc);
185 }
186
187 @Override
188 protected List<Entry<String, URL>> getChapters(Progress pg) {
189 chapterNames = new TreeMap<Integer, String>();
190 chapterContents = new TreeMap<Integer, String>();
191
192 int pos = 0;
193 while (pos >= 0) {
194 pos = indexOfJsonAfter(json, pos, "type", "chapter");
195 if (pos >= 0) {
196 int posNumber = indexOfJsonAfter(json, pos, "chapter_number");
197 int posComa = json.indexOf(",", posNumber);
198 final int number = Integer.parseInt(json.substring(posNumber,
199 posComa).trim());
200 final String title = getKeyJson(json, pos, "title");
201 String notes = getKeyJson(json, pos, "authors_note_html");
202 String content = getKeyJson(json, pos, "content_html");
203
204 if (!notes.trim().isEmpty()) {
205 notes = "<br/>* * *<br/>" + notes;
206 }
207
208 chapterNames.put(number, title);
209 chapterContents.put(number, content + notes);
210 }
211 }
212
213 List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
214 for (String title : chapterNames.values()) {
215 urls.add(new AbstractMap.SimpleEntry(title, null));
216 }
217
218 return urls;
219 }
220
221 @Override
222 protected String getChapterContent(URL source, int number, Progress pg) {
223 return chapterContents.get(number);
224 }
225
226 @Override
227 protected boolean supports(URL url) {
228 return "fimfiction.net".equals(url.getHost())
229 || "www.fimfiction.net".equals(url.getHost());
230 }
231
232 /**
233 * Generate a new token from the client ID and secret.
234 * <p>
235 * Note that those tokens are long-lived, and it would be badly seen to
236 * create a lot of them without due cause.
237 * <p>
238 * So, please cache and re-use them.
239 *
240 * @param clientId
241 * the client ID offered on FimFiction
242 * @param clientSecret
243 * the client secret that goes with it
244 *
245 * @return a new generated token linked to that client ID
246 *
247 * @throws IOException
248 * in case of I/O errors
249 */
250 static private String generateOAuth(String clientId, String clientSecret)
251 throws IOException {
252 URL url = new URL("https://www.fimfiction.net/api/v2/token");
253 Map<String, String> params = new HashMap<String, String>();
254 params.put("client_id", clientId);
255 params.put("client_secret", clientSecret);
256 params.put("grant_type", "client_credentials");
257 InputStream in = Instance.getCache().openNoCache(url, null, params,
258 null, null);
259
260 String jsonToken = IOUtils.readSmallStream(in);
261 in.close();
262
263 // Extract token type and token from: {
264 // token_type = "Bearer",
265 // access_token = "xxxxxxxxxxxxxx"
266 // }
267
268 String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\"");
269 String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\"");
270
271 return tokenType + " " + token;
272 }
273
274 // afters: [name, value] pairs (or "" for any of them), can end without
275 // value
276 static private int indexOfJsonAfter(String json, int startAt,
277 String... afterKeys) {
278 ArrayList<String> afters = new ArrayList<String>();
279 boolean name = true;
280 for (String key : afterKeys) {
281 if (key != null && !key.isEmpty()) {
282 afters.add("\"" + key + "\"");
283 } else {
284 afters.add("\"");
285 afters.add("\"");
286 }
287
288 if (name) {
289 afters.add(":");
290 }
291
292 name = !name;
293 }
294
295 return indexOfAfter(json, startAt, afters.toArray(new String[] {}));
296 }
297
298 // afters: [name, value] pairs (or "" for any of them), can end without
299 // value but will then be empty, not NULL
300 static private String getKeyJson(String json, int startAt,
301 String... afterKeys) {
302 int pos = indexOfJsonAfter(json, startAt, afterKeys);
303 if (pos < 0) {
304 return "";
305 }
306
307 String result = "";
308 String wip = json.substring(pos);
309
310 pos = nextUnescapedQuote(wip, 0);
311 if (pos >= 0) {
312 wip = wip.substring(pos + 1);
313 pos = nextUnescapedQuote(wip, 0);
314 if (pos >= 0) {
315 result = wip.substring(0, pos);
316 }
317 }
318
319 result = result.replace("\\t", "\t").replace("\\\"", "\"");
320
321 return result;
322 }
323
324 // next " but don't take \" into account
325 static private int nextUnescapedQuote(String result, int pos) {
326 while (pos >= 0) {
327 pos = result.indexOf("\"", pos);
328 if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) {
329 break;
330 }
331
332 if (pos < result.length()) {
333 pos++;
334 }
335 }
336
337 return pos;
338 }
339
340 // quick & dirty filter
341 static private String unbbcode(String bbcode) {
342 String text = bbcode.replace("\\r\\n", "<br/>") //
343 .replace("[i]", "_").replace("[/i]", "_") //
344 .replace("[b]", "*").replace("[/b]", "*") //
345 .replaceAll("\\[[^\\]]*\\]", "");
346 return text;
347 }
348
349 /**
350 * Return the text between the key and the endKey (and optional subKey can
351 * be passed, in this case we will look for the key first, then take the
352 * text between the subKey and the endKey).
353 *
354 * @param in
355 * the input
356 * @param key
357 * the key to match (also supports "^" at start to say
358 * "only if it starts with" the key)
359 * @param subKey
360 * the sub key or NULL if none
361 * @param endKey
362 * the end key or NULL for "up to the end"
363 * @return the text or NULL if not found
364 */
365 static private String getKeyText(String in, String key, String subKey,
366 String endKey) {
367 String result = null;
368
369 String line = in;
370 if (line != null && line.contains(key)) {
371 line = line.substring(line.indexOf(key) + key.length());
372 if (subKey == null || subKey.isEmpty() || line.contains(subKey)) {
373 if (subKey != null) {
374 line = line.substring(line.indexOf(subKey)
375 + subKey.length());
376 }
377 if (endKey == null || line.contains(endKey)) {
378 if (endKey != null) {
379 line = line.substring(0, line.indexOf(endKey));
380 result = line;
381 }
382 }
383 }
384 }
385
386 return result;
387 }
388
389 /**
390 * Return the first index after all the given "afters" have been found in
391 * the {@link String}, or -1 if it was not possible.
392 *
393 * @param in
394 * the input
395 * @param startAt
396 * start at this position in the string
397 * @param afters
398 * the sub-keys to find before checking for key/endKey
399 *
400 * @return the text or NULL if not found
401 */
402 static private int indexOfAfter(String in, int startAt, String... afters) {
403 int pos = -1;
404 if (in != null && !in.isEmpty()) {
405 pos = startAt;
406 if (afters != null) {
407 for (int i = 0; pos >= 0 && i < afters.length; i++) {
408 String subKey = afters[i];
409 if (!subKey.isEmpty()) {
410 pos = in.indexOf(subKey, pos);
411 if (pos >= 0) {
412 pos += subKey.length();
413 }
414 }
415 }
416 }
417 }
418
419 return pos;
420 }
421}