fix changelog headers
[fanfix.git] / FimfictionApi.java
... / ...
CommitLineData
1package be.nikiroo.fanfix.supported;
2
3import java.io.IOException;
4import java.io.InputStream;
5import java.net.URL;
6import java.util.ArrayList;
7import java.util.HashMap;
8import java.util.List;
9import java.util.Map;
10import java.util.Map.Entry;
11
12import be.nikiroo.fanfix.Instance;
13import be.nikiroo.fanfix.bundles.Config;
14import be.nikiroo.fanfix.data.MetaData;
15import be.nikiroo.utils.IOUtils;
16import be.nikiroo.utils.Progress;
17
18/**
19 * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a>
20 * stories, a website dedicated to My Little Pony.
21 * <p>
22 * This version uses the new, official API of FimFiction.
23 *
24 * @author niki
25 */
26class FimfictionApi extends BasicSupport {
27 private String oauth;
28 private String storyId;
29 private String json;
30
31 private Map<Integer, String> chapterNames;
32 private Map<Integer, String> chapterContents;
33
34 public FimfictionApi() throws IOException {
35 if (Instance.getConfig().getBoolean(
36 Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) {
37 throw new IOException(
38 "Configuration is set to force HTML scrapping");
39 }
40
41 String oauth = Instance.getConfig().getString(
42 Config.LOGIN_FIMFICTION_APIKEY_TOKEN);
43
44 if (oauth == null || oauth.isEmpty()) {
45 String clientId = Instance.getConfig().getString(
46 Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID)
47 + "";
48 String clientSecret = Instance.getConfig().getString(
49 Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET)
50 + "";
51
52 if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) {
53 throw new IOException("API key required for the beta API v2");
54 }
55
56 oauth = generateOAuth(clientId, clientSecret);
57
58 Instance.getConfig().setString(
59 Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth);
60 Instance.getConfig().updateFile();
61 }
62
63 this.oauth = oauth;
64 }
65
66 @Override
67 public String getOAuth() {
68 return oauth;
69 }
70
71 @Override
72 protected boolean isHtml() {
73 return true;
74 }
75
76 @Override
77 public String getSourceName() {
78 return "FimFiction.net";
79 }
80
81 @Override
82 protected void preprocess(URL source, InputStream in) throws IOException {
83 // extract the ID from:
84 // https://www.fimfiction.net/story/123456/name-of-story
85 storyId = getKeyText(source.toString(), "/story/", null, "/");
86
87 // Selectors, so to download all I need and only what I need
88 String storyContent = "fields[story]=title,description,date_published,cover_image";
89 String authorContent = "fields[author]=name";
90 String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html";
91 String includes = "author,chapters,tags";
92
93 String urlString = String.format(
94 "https://www.fimfiction.net/api/v2/stories/%s?" //
95 + "%s&%s&%s&" //
96 + "include=%s", //
97 storyId, //
98 storyContent, authorContent, chapterContent,//
99 includes);
100
101 // URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
102 urlString = urlString.replace("[", "%5B").replace("]", "%5D");
103
104 URL url = new URL(urlString);
105 InputStream jsonIn = Instance.getCache().open(url, this, false);
106 try {
107 json = IOUtils.readSmallStream(jsonIn);
108 } finally {
109 jsonIn.close();
110 }
111 }
112
113 @Override
114 protected InputStream openInput(URL source) throws IOException {
115 return null;
116 }
117
118 @Override
119 protected MetaData getMeta(URL source, InputStream in) throws IOException {
120 MetaData meta = new MetaData();
121
122 meta.setTitle(getKeyJson(json, 0, "type", "story", "title"));
123 meta.setAuthor(getKeyJson(json, 0, "type", "user", "name"));
124 meta.setDate(getKeyJson(json, 0, "type", "story", "date_published"));
125 meta.setTags(getTags());
126 meta.setSource(getSourceName());
127 meta.setUrl(source.toString());
128 meta.setPublisher(getSourceName());
129 meta.setUuid(source.toString());
130 meta.setLuid("");
131 meta.setLang("EN");
132 meta.setSubject("MLP");
133 meta.setType(getType().toString());
134 meta.setImageDocument(false);
135
136 String coverImageLink =
137 getKeyJson(json, 0, "type", "story", "cover_image", "full");
138 if (!coverImageLink.trim().isEmpty()) {
139 meta.setCover(getImage(this, null, coverImageLink.trim()));
140 }
141
142 return meta;
143 }
144
145 private List<String> getTags() {
146 List<String> tags = new ArrayList<String>();
147 tags.add("MLP");
148
149 int pos = 0;
150 while (pos >= 0) {
151 pos = indexOfJsonAfter(json, pos, "type", "story_tag");
152 if (pos >= 0) {
153 tags.add(getKeyJson(json, pos, "name").trim());
154 }
155 }
156
157 return tags;
158 }
159
160 @Override
161 protected String getDesc(URL source, InputStream in) {
162 String desc = getKeyJson(json, 0, "type", "story", "description");
163 return unbbcode(desc);
164 }
165
166 @Override
167 protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
168 Progress pg) {
169 List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
170
171 chapterNames = new HashMap<Integer, String>();
172 chapterContents = new HashMap<Integer, String>();
173
174 int pos = 0;
175 while (pos >= 0) {
176 pos = indexOfJsonAfter(json, pos, "type", "chapter");
177 if (pos >= 0) {
178 int posNumber = indexOfJsonAfter(json, pos, "chapter_number");
179 int posComa = json.indexOf(",", posNumber);
180 final int number = Integer.parseInt(json.substring(posNumber,
181 posComa).trim());
182 final String title = getKeyJson(json, pos, "title");
183 String notes = getKeyJson(json, pos, "authors_note_html");
184 String content = getKeyJson(json, pos, "content_html");
185
186 if (!notes.trim().isEmpty()) {
187 notes = "<br/>* * *<br/>" + notes;
188 }
189
190 chapterNames.put(number, title);
191 chapterContents
192 .put(number, content + notes);
193
194 urls.add(new Entry<String, URL>() {
195 @Override
196 public URL setValue(URL value) {
197 return null;
198 }
199
200 @Override
201 public String getKey() {
202 return title;
203 }
204
205 @Override
206 public URL getValue() {
207 return null;
208 }
209 });
210 }
211 }
212
213 return urls;
214 }
215
216 @Override
217 protected String getChapterContent(URL source, InputStream in, int number,
218 Progress pg) {
219 return chapterContents.get(number);
220 }
221
222 @Override
223 protected boolean supports(URL url) {
224 return "fimfiction.net".equals(url.getHost())
225 || "www.fimfiction.net".equals(url.getHost());
226 }
227
228 /**
229 * Generate a new token from the client ID and secret.
230 * <p>
231 * Note that those tokens are long-lived, and it would be badly seen to
232 * create a lot of them without due cause.
233 * <p>
234 * So, please cache and re-use them.
235 *
236 * @param clientId
237 * the client ID offered on FimFiction
238 * @param clientSecret
239 * the client secret that goes with it
240 *
241 * @return a new generated token linked to that client ID
242 *
243 * @throws IOException
244 * in case of I/O errors
245 */
246 static private String generateOAuth(String clientId, String clientSecret)
247 throws IOException {
248 URL url = new URL("https://www.fimfiction.net/api/v2/token");
249 Map<String, String> params = new HashMap<String, String>();
250 params.put("client_id", clientId);
251 params.put("client_secret", clientSecret);
252 params.put("grant_type", "client_credentials");
253 InputStream in = Instance.getCache().openNoCache(url, null, params,
254 null, null);
255
256 String jsonToken = IOUtils.readSmallStream(in);
257 in.close();
258
259 // Extract token type and token from: {
260 // token_type = "Bearer",
261 // access_token = "xxxxxxxxxxxxxx"
262 // }
263
264 String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\"");
265 String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\"");
266
267 return tokenType + " " + token;
268 }
269
270 // afters: [name, value] pairs (or "" for any of them), can end without
271 // value
272 static private int indexOfJsonAfter(String json, int startAt,
273 String... afterKeys) {
274 ArrayList<String> afters = new ArrayList<String>();
275 boolean name = true;
276 for (String key : afterKeys) {
277 if (key != null && !key.isEmpty()) {
278 afters.add("\"" + key + "\"");
279 } else {
280 afters.add("\"");
281 afters.add("\"");
282 }
283
284 if (name) {
285 afters.add(":");
286 }
287
288 name = !name;
289 }
290
291 return indexOfAfter(json, startAt, afters.toArray(new String[] {}));
292 }
293
294 // afters: [name, value] pairs (or "" for any of them), can end without
295 // value but will then be empty, not NULL
296 static private String getKeyJson(String json, int startAt,
297 String... afterKeys) {
298 int pos = indexOfJsonAfter(json, startAt, afterKeys);
299 if (pos < 0) {
300 return "";
301 }
302
303 String result = "";
304 String wip = json.substring(pos);
305
306 pos = nextUnescapedQuote(wip, 0);
307 if (pos >= 0) {
308 wip = wip.substring(pos + 1);
309 pos = nextUnescapedQuote(wip, 0);
310 if (pos >= 0) {
311 result = wip.substring(0, pos);
312 }
313 }
314
315 result = result.replace("\\t", "\t")
316 .replace("\\\"", "\"");
317
318 return result;
319 }
320
321 // next " but don't take \" into account
322 static private int nextUnescapedQuote(String result, int pos) {
323 while (pos >= 0) {
324 pos = result.indexOf("\"", pos);
325 if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) {
326 break;
327 }
328
329 if (pos < result.length()) {
330 pos++;
331 }
332 }
333
334 return pos;
335 }
336
337 // quick & dirty filter
338 static private String unbbcode(String bbcode) {
339 String text = bbcode.replace("\\r\\n", "<br/>") //
340 .replace("[i]", "_").replace("[/i]", "_") //
341 .replace("[b]", "*").replace("[/b]", "*") //
342 .replaceAll("\\[[^\\]]*\\]", "");
343 return text;
344 }
345}