doc
[nikiroo-utils.git] / supported / FimfictionApi.java
1 package be.nikiroo.fanfix.supported;
2
3 import java.io.IOException;
4 import java.io.InputStream;
5 import java.net.URL;
6 import java.util.ArrayList;
7 import java.util.HashMap;
8 import java.util.List;
9 import java.util.Map;
10 import java.util.Map.Entry;
11
12 import be.nikiroo.fanfix.Instance;
13 import be.nikiroo.fanfix.bundles.Config;
14 import be.nikiroo.fanfix.data.MetaData;
15 import be.nikiroo.utils.IOUtils;
16 import be.nikiroo.utils.Progress;
17
18 /**
19 * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a>
20 * stories, a website dedicated to My Little Pony.
21 * <p>
22 * This version uses the new, official API of FimFiction.
23 *
24 * @author niki
25 */
26 class FimfictionApi extends BasicSupport {
27 private String oauth;
28 private String storyId;
29 private String json;
30
31 private Map<Integer, String> chapterNames;
32 private Map<Integer, String> chapterContents;
33
34 public FimfictionApi() throws IOException {
35 if (Instance.getConfig().getBoolean(
36 Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) {
37 throw new IOException(
38 "Configuration is set to force HTML scrapping");
39 }
40
41 String oauth = Instance.getConfig().getString(
42 Config.LOGIN_FIMFICTION_APIKEY_TOKEN);
43
44 if (oauth == null || oauth.isEmpty()) {
45 String clientId = Instance.getConfig().getString(
46 Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID)
47 + "";
48 String clientSecret = Instance.getConfig().getString(
49 Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET)
50 + "";
51
52 if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) {
53 throw new IOException("API key required for the beta API v2");
54 }
55
56 oauth = generateOAuth(clientId, clientSecret);
57
58 Instance.getConfig().setString(
59 Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth);
60 Instance.getConfig().updateFile();
61 }
62
63 this.oauth = oauth;
64 }
65
66 @Override
67 public String getOAuth() {
68 return oauth;
69 }
70
71 @Override
72 protected boolean isHtml() {
73 return true;
74 }
75
76 @Override
77 public String getSourceName() {
78 return "FimFiction.net";
79 }
80
81 @Override
82 protected void preprocess(URL source, InputStream in) throws IOException {
83 // extract the ID from:
84 // https://www.fimfiction.net/story/123456/name-of-story
85 storyId = getKeyText(source.toString(), "/story/", null, "/");
86
87 // Selectors, so to download all I need and only what I need
88 String storyContent = "fields[story]=title,description,date_published,cover_image";
89 String authorContent = "fields[author]=name";
90 String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html";
91 String includes = "author,chapters,tags";
92
93 String urlString = String.format(
94 "https://www.fimfiction.net/api/v2/stories/%s?" //
95 + "%s&%s&%s&" //
96 + "include=%s", //
97 storyId, //
98 storyContent, authorContent, chapterContent,//
99 includes);
100
101 // URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
102 urlString = urlString.replace("[", "%5B").replace("]", "%5D");
103
104 URL url = new URL(urlString);
105 InputStream jsonIn = Instance.getCache().open(url, this, false);
106 try {
107 json = IOUtils.readSmallStream(jsonIn);
108 } finally {
109 jsonIn.close();
110 }
111 }
112
113 @Override
114 protected InputStream openInput(URL source) throws IOException {
115 return null;
116 }
117
118 @Override
119 protected MetaData getMeta(URL source, InputStream in) throws IOException {
120 MetaData meta = new MetaData();
121
122 meta.setTitle(getKeyJson(json, 0, "type", "story", "title"));
123 meta.setAuthor(getKeyJson(json, 0, "type", "user", "name"));
124 meta.setDate(getKeyJson(json, 0, "type", "story", "date_published"));
125 meta.setTags(getTags());
126 meta.setSource(getSourceName());
127 meta.setUrl(source.toString());
128 meta.setPublisher(getSourceName());
129 meta.setUuid(source.toString());
130 meta.setLuid("");
131 meta.setLang("EN");
132 meta.setSubject("MLP");
133 meta.setType(getType().toString());
134 meta.setImageDocument(false);
135
136 String coverImageLink =
137 getKeyJson(json, 0, "type", "story", "cover_image", "full");
138 if (!coverImageLink.trim().isEmpty()) {
139 meta.setCover(getImage(this, null, coverImageLink.trim()));
140 }
141
142 return meta;
143 }
144
145 private List<String> getTags() {
146 List<String> tags = new ArrayList<String>();
147 tags.add("MLP");
148
149 int pos = 0;
150 while (pos >= 0) {
151 pos = indexOfJsonAfter(json, pos, "type", "story_tag");
152 if (pos >= 0) {
153 tags.add(getKeyJson(json, pos, "name").trim());
154 }
155 }
156
157 return tags;
158 }
159
160 @Override
161 protected String getDesc(URL source, InputStream in) {
162 String desc = getKeyJson(json, 0, "type", "story", "description");
163 return unbbcode(desc);
164 }
165
166 @Override
167 protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
168 Progress pg) {
169 List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
170
171 chapterNames = new HashMap<Integer, String>();
172 chapterContents = new HashMap<Integer, String>();
173
174 int pos = 0;
175 while (pos >= 0) {
176 pos = indexOfJsonAfter(json, pos, "type", "chapter");
177 if (pos >= 0) {
178 int posNumber = indexOfJsonAfter(json, pos, "chapter_number");
179 int posComa = json.indexOf(",", posNumber);
180 final int number = Integer.parseInt(json.substring(posNumber,
181 posComa).trim());
182 final String title = getKeyJson(json, pos, "title");
183 String notes = getKeyJson(json, pos, "authors_note_html");
184 String content = getKeyJson(json, pos, "content_html");
185
186 if (!notes.trim().isEmpty()) {
187 notes = "<br/>* * *<br/>" + notes;
188 }
189
190 chapterNames.put(number, title);
191 chapterContents
192 .put(number, content + notes);
193
194 urls.add(new Entry<String, URL>() {
195 @Override
196 public URL setValue(URL value) {
197 return null;
198 }
199
200 @Override
201 public String getKey() {
202 return title;
203 }
204
205 @Override
206 public URL getValue() {
207 return null;
208 }
209 });
210 }
211 }
212
213 return urls;
214 }
215
216 @Override
217 protected String getChapterContent(URL source, InputStream in, int number,
218 Progress pg) {
219 return chapterContents.get(number);
220 }
221
222 @Override
223 protected boolean supports(URL url) {
224 return "fimfiction.net".equals(url.getHost())
225 || "www.fimfiction.net".equals(url.getHost());
226 }
227
228 /**
229 * Generate a new token from the client ID and secret.
230 * <p>
231 * Note that those tokens are long-lived, and it would be badly seen to
232 * create a lot of them without due cause.
233 * <p>
234 * So, please cache and re-use them.
235 *
236 * @param clientId
237 * the client ID offered on FimFiction
238 * @param clientSecret
239 * the client secret that goes with it
240 *
241 * @return a new generated token linked to that client ID
242 *
243 * @throws IOException
244 * in case of I/O errors
245 */
246 static private String generateOAuth(String clientId, String clientSecret)
247 throws IOException {
248 URL url = new URL("https://www.fimfiction.net/api/v2/token");
249 Map<String, String> params = new HashMap<String, String>();
250 params.put("client_id", clientId);
251 params.put("client_secret", clientSecret);
252 params.put("grant_type", "client_credentials");
253 InputStream in = Instance.getCache().openNoCache(url, null, params,
254 null, null);
255
256 String jsonToken = IOUtils.readSmallStream(in);
257
258 // Extract token type and token from: {
259 // token_type = "Bearer",
260 // access_token = "xxxxxxxxxxxxxx"
261 // }
262
263 String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\"");
264 String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\"");
265
266 return tokenType + " " + token;
267 }
268
269 // afters: [name, value] pairs (or "" for any of them), can end without
270 // value
271 static private int indexOfJsonAfter(String json, int startAt,
272 String... afterKeys) {
273 ArrayList<String> afters = new ArrayList<String>();
274 boolean name = true;
275 for (String key : afterKeys) {
276 if (key != null && !key.isEmpty()) {
277 afters.add("\"" + key + "\"");
278 } else {
279 afters.add("\"");
280 afters.add("\"");
281 }
282
283 if (name) {
284 afters.add(":");
285 }
286
287 name = !name;
288 }
289
290 return indexOfAfter(json, startAt, afters.toArray(new String[] {}));
291 }
292
293 // afters: [name, value] pairs (or "" for any of them), can end without
294 // value but will then be empty, not NULL
295 static private String getKeyJson(String json, int startAt,
296 String... afterKeys) {
297 int pos = indexOfJsonAfter(json, startAt, afterKeys);
298 if (pos < 0) {
299 return "";
300 }
301
302 String result = "";
303 String wip = json.substring(pos);
304
305 pos = nextUnescapedQuote(wip, 0);
306 if (pos >= 0) {
307 wip = wip.substring(pos + 1);
308 pos = nextUnescapedQuote(wip, 0);
309 if (pos >= 0) {
310 result = wip.substring(0, pos);
311 }
312 }
313
314 result = result.replace("\\t", "\t")
315 .replace("\\\"", "\"");
316
317 return result;
318 }
319
320 // next " but don't take \" into account
321 static private int nextUnescapedQuote(String result, int pos) {
322 while (pos >= 0) {
323 pos = result.indexOf("\"", pos);
324 if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) {
325 break;
326 }
327
328 if (pos < result.length()) {
329 pos++;
330 }
331 }
332
333 return pos;
334 }
335
336 // quick & dirty filter
337 static private String unbbcode(String bbcode) {
338 String text = bbcode.replace("\\r\\n", "<br/>") //
339 .replace("[i]", "_").replace("[/i]", "_") //
340 .replace("[b]", "*").replace("[/b]", "*") //
341 .replaceAll("\\[[^\\]]*\\]", "");
342 return text;
343 }
344 }