JsonIO: progress can have a key 'meta'
[fanfix.git] / src / be / nikiroo / fanfix / supported / FimfictionApi.java
1 package be.nikiroo.fanfix.supported;
2
3 import java.io.IOException;
4 import java.io.InputStream;
5 import java.net.URL;
6 import java.util.AbstractMap;
7 import java.util.ArrayList;
8 import java.util.HashMap;
9 import java.util.List;
10 import java.util.Map;
11 import java.util.Map.Entry;
12 import java.util.TreeMap;
13
14 import org.jsoup.nodes.Document;
15
16 import be.nikiroo.fanfix.Instance;
17 import be.nikiroo.fanfix.bundles.Config;
18 import be.nikiroo.fanfix.data.MetaData;
19 import be.nikiroo.fanfix.data.Story;
20 import be.nikiroo.utils.IOUtils;
21 import be.nikiroo.utils.Image;
22 import be.nikiroo.utils.Progress;
23
24 /**
25 * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a>
26 * stories, a website dedicated to My Little Pony.
27 * <p>
28 * This version uses the new, official API of FimFiction.
29 *
30 * @author niki
31 */
32 class FimfictionApi extends BasicSupport {
33 private String oauth;
34 private String json;
35
36 private Map<Integer, String> chapterNames;
37 private Map<Integer, String> chapterContents;
38
39 public FimfictionApi() throws IOException {
40 if (Instance.getInstance().getConfig().getBoolean(Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) {
41 throw new IOException("Configuration is set to force HTML scrapping");
42 }
43
44 String oauth = Instance.getInstance().getConfig().getString(Config.LOGIN_FIMFICTION_APIKEY_TOKEN);
45
46 if (oauth == null || oauth.isEmpty()) {
47 String clientId = Instance.getInstance().getConfig().getString(Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID)
48 + "";
49 String clientSecret = Instance.getInstance().getConfig()
50 .getString(Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) + "";
51
52 if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) {
53 throw new IOException("API key required for the beta API v2");
54 }
55
56 oauth = generateOAuth(clientId, clientSecret);
57
58 Instance.getInstance().getConfig().setString(Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth);
59 Instance.getInstance().getConfig().updateFile();
60 }
61
62 this.oauth = oauth;
63 }
64
65 @Override
66 protected Document loadDocument(URL source) throws IOException {
67 json = getJsonData();
68 return null;
69 }
70
71 @Override
72 public String getOAuth() {
73 return oauth;
74 }
75
76 @Override
77 protected boolean isHtml() {
78 return true;
79 }
80
81 /**
82 * Extract the full JSON data we will later use to build the {@link Story}.
83 *
84 * @return the data in a JSON format
85 *
86 * @throws IOException
87 * in case of I/O error
88 */
89 private String getJsonData() throws IOException {
90 // extract the ID from:
91 // https://www.fimfiction.net/story/123456/name-of-story
92 String storyId = getKeyText(getSource().toString(), "/story/", null,
93 "/");
94
95 // Selectors, so to download all I need and only what I need
96 String storyContent = "fields[story]=title,description,date_published,cover_image";
97 String authorContent = "fields[author]=name";
98 String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html";
99 String includes = "author,chapters,tags";
100
101 String urlString = String.format(
102 "https://www.fimfiction.net/api/v2/stories/%s?" //
103 + "%s&%s&%s&" //
104 + "include=%s", //
105 storyId, //
106 storyContent, authorContent, chapterContent,//
107 includes);
108
109 // URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
110 urlString = urlString.replace("[", "%5B").replace("]", "%5D");
111
112 URL url = new URL(urlString);
113 InputStream jsonIn = Instance.getInstance().getCache().open(url, this, false);
114 try {
115 return IOUtils.readSmallStream(jsonIn);
116 } finally {
117 jsonIn.close();
118 }
119 }
120
121 @Override
122 protected MetaData getMeta() throws IOException {
123 MetaData meta = new MetaData();
124
125 meta.setTitle(getKeyJson(json, 0, "type", "story", "title"));
126 meta.setAuthor(getKeyJson(json, 0, "type", "user", "name"));
127 meta.setDate(bsHelper.formatDate(
128 getKeyJson(json, 0, "type", "story", "date_published")));
129 meta.setTags(getTags());
130 meta.setSource(getType().getSourceName());
131 meta.setUrl(getSource().toString());
132 meta.setPublisher(getType().getSourceName());
133 meta.setUuid(getSource().toString());
134 meta.setLuid("");
135 meta.setLang("en");
136 meta.setSubject("MLP");
137 meta.setType(getType().toString());
138 meta.setImageDocument(false);
139
140 String coverImageLink = getKeyJson(json, 0, "type", "story",
141 "cover_image", "full");
142 if (!coverImageLink.trim().isEmpty()) {
143 URL coverImageUrl = new URL(coverImageLink.trim());
144
145 // No need to use the oauth, cookies... for the cover
146 // Plus: it crashes on Android because of the referer
147 try {
148 InputStream in = Instance.getInstance().getCache().open(coverImageUrl, null, true);
149 try {
150 meta.setCover(new Image(in));
151 } finally {
152 in.close();
153 }
154 } catch (IOException e) {
155 Instance.getInstance().getTraceHandler()
156 .error(new IOException("Cannot get the story cover, ignoring...", e));
157 }
158 }
159
160 return meta;
161 }
162
163 private List<String> getTags() {
164 List<String> tags = new ArrayList<String>();
165 tags.add("MLP");
166
167 int pos = 0;
168 while (pos >= 0) {
169 pos = indexOfJsonAfter(json, pos, "type", "story_tag");
170 if (pos >= 0) {
171 tags.add(getKeyJson(json, pos, "name").trim());
172 }
173 }
174
175 return tags;
176 }
177
178 @Override
179 protected String getDesc() {
180 String desc = getKeyJson(json, 0, "type", "story", "description");
181 return unbbcode(desc);
182 }
183
184 @Override
185 protected List<Entry<String, URL>> getChapters(Progress pg) {
186 chapterNames = new TreeMap<Integer, String>();
187 chapterContents = new TreeMap<Integer, String>();
188
189 int pos = 0;
190 while (pos >= 0) {
191 pos = indexOfJsonAfter(json, pos, "type", "chapter");
192 if (pos >= 0) {
193 int posNumber = indexOfJsonAfter(json, pos, "chapter_number");
194 int posComa = json.indexOf(",", posNumber);
195 final int number = Integer.parseInt(json.substring(posNumber,
196 posComa).trim());
197 final String title = getKeyJson(json, pos, "title");
198 String notes = getKeyJson(json, pos, "authors_note_html");
199 String content = getKeyJson(json, pos, "content_html");
200
201 if (!notes.trim().isEmpty()) {
202 notes = "<br/>* * *<br/>" + notes;
203 }
204
205 chapterNames.put(number, title);
206 chapterContents.put(number, content + notes);
207 }
208 }
209
210 List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
211 for (String title : chapterNames.values()) {
212 urls.add(new AbstractMap.SimpleEntry<String, URL>(title, null));
213 }
214
215 return urls;
216 }
217
218 @Override
219 protected String getChapterContent(URL source, int number, Progress pg) {
220 return chapterContents.get(number);
221 }
222
223 @Override
224 protected boolean supports(URL url) {
225 return "fimfiction.net".equals(url.getHost())
226 || "www.fimfiction.net".equals(url.getHost());
227 }
228
229 /**
230 * Generate a new token from the client ID and secret.
231 * <p>
232 * Note that those tokens are long-lived, and it would be badly seen to
233 * create a lot of them without due cause.
234 * <p>
235 * So, please cache and re-use them.
236 *
237 * @param clientId
238 * the client ID offered on FimFiction
239 * @param clientSecret
240 * the client secret that goes with it
241 *
242 * @return a new generated token linked to that client ID
243 *
244 * @throws IOException
245 * in case of I/O errors
246 */
247 static private String generateOAuth(String clientId, String clientSecret)
248 throws IOException {
249 URL url = new URL("https://www.fimfiction.net/api/v2/token");
250 Map<String, String> params = new HashMap<String, String>();
251 params.put("client_id", clientId);
252 params.put("client_secret", clientSecret);
253 params.put("grant_type", "client_credentials");
254 InputStream in = Instance.getInstance().getCache().openNoCache(url, null, params, null, null);
255
256 String jsonToken = IOUtils.readSmallStream(in);
257 in.close();
258
259 // Extract token type and token from: {
260 // token_type = "Bearer",
261 // access_token = "xxxxxxxxxxxxxx"
262 // }
263
264 String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\"");
265 String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\"");
266
267 return tokenType + " " + token;
268 }
269
270 // afters: [name, value] pairs (or "" for any of them), can end without
271 // value
272 static private int indexOfJsonAfter(String json, int startAt,
273 String... afterKeys) {
274 ArrayList<String> afters = new ArrayList<String>();
275 boolean name = true;
276 for (String key : afterKeys) {
277 if (key != null && !key.isEmpty()) {
278 afters.add("\"" + key + "\"");
279 } else {
280 afters.add("\"");
281 afters.add("\"");
282 }
283
284 if (name) {
285 afters.add(":");
286 }
287
288 name = !name;
289 }
290
291 return indexOfAfter(json, startAt, afters.toArray(new String[] {}));
292 }
293
294 // afters: [name, value] pairs (or "" for any of them), can end without
295 // value but will then be empty, not NULL
296 static private String getKeyJson(String json, int startAt,
297 String... afterKeys) {
298 int pos = indexOfJsonAfter(json, startAt, afterKeys);
299 if (pos < 0) {
300 return "";
301 }
302
303 String result = "";
304 String wip = json.substring(pos);
305
306 pos = nextUnescapedQuote(wip, 0);
307 if (pos >= 0) {
308 wip = wip.substring(pos + 1);
309 pos = nextUnescapedQuote(wip, 0);
310 if (pos >= 0) {
311 result = wip.substring(0, pos);
312 }
313 }
314
315 result = result.replace("\\t", "\t").replace("\\\"", "\"");
316
317 return result;
318 }
319
320 // next " but don't take \" into account
321 static private int nextUnescapedQuote(String result, int pos) {
322 while (pos >= 0) {
323 pos = result.indexOf("\"", pos);
324 if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) {
325 break;
326 }
327
328 if (pos < result.length()) {
329 pos++;
330 }
331 }
332
333 return pos;
334 }
335
336 // quick & dirty filter
337 static private String unbbcode(String bbcode) {
338 String text = bbcode.replace("\\r\\n", "<br/>") //
339 .replace("[i]", "_").replace("[/i]", "_") //
340 .replace("[b]", "*").replace("[/b]", "*") //
341 .replaceAll("\\[[^\\]]*\\]", "");
342 return text;
343 }
344
345 /**
346 * Return the text between the key and the endKey (and optional subKey can
347 * be passed, in this case we will look for the key first, then take the
348 * text between the subKey and the endKey).
349 *
350 * @param in
351 * the input
352 * @param key
353 * the key to match (also supports "^" at start to say
354 * "only if it starts with" the key)
355 * @param subKey
356 * the sub key or NULL if none
357 * @param endKey
358 * the end key or NULL for "up to the end"
359 * @return the text or NULL if not found
360 */
361 static private String getKeyText(String in, String key, String subKey,
362 String endKey) {
363 String result = null;
364
365 String line = in;
366 if (line != null && line.contains(key)) {
367 line = line.substring(line.indexOf(key) + key.length());
368 if (subKey == null || subKey.isEmpty() || line.contains(subKey)) {
369 if (subKey != null) {
370 line = line.substring(line.indexOf(subKey)
371 + subKey.length());
372 }
373 if (endKey == null || line.contains(endKey)) {
374 if (endKey != null) {
375 line = line.substring(0, line.indexOf(endKey));
376 result = line;
377 }
378 }
379 }
380 }
381
382 return result;
383 }
384
385 /**
386 * Return the first index after all the given "afters" have been found in
387 * the {@link String}, or -1 if it was not possible.
388 *
389 * @param in
390 * the input
391 * @param startAt
392 * start at this position in the string
393 * @param afters
394 * the sub-keys to find before checking for key/endKey
395 *
396 * @return the text or NULL if not found
397 */
398 static private int indexOfAfter(String in, int startAt, String... afters) {
399 int pos = -1;
400 if (in != null && !in.isEmpty()) {
401 pos = startAt;
402 if (afters != null) {
403 for (int i = 0; pos >= 0 && i < afters.length; i++) {
404 String subKey = afters[i];
405 if (!subKey.isEmpty()) {
406 pos = in.indexOf(subKey, pos);
407 if (pos >= 0) {
408 pos += subKey.length();
409 }
410 }
411 }
412 }
413 }
414
415 return pos;
416 }
417 }