Commit | Line | Data |
---|---|---|
315f14ae NR |
1 | package be.nikiroo.fanfix.supported; |
2 | ||
3 | import java.io.IOException; | |
4 | import java.io.InputStream; | |
5 | import java.net.URL; | |
ce297a79 | 6 | import java.util.AbstractMap; |
315f14ae NR |
7 | import java.util.ArrayList; |
8 | import java.util.HashMap; | |
9 | import java.util.List; | |
10 | import java.util.Map; | |
11 | import java.util.Map.Entry; | |
ce297a79 | 12 | import java.util.TreeMap; |
315f14ae NR |
13 | |
14 | import be.nikiroo.fanfix.Instance; | |
15 | import be.nikiroo.fanfix.bundles.Config; | |
16 | import be.nikiroo.fanfix.data.MetaData; | |
17 | import be.nikiroo.utils.IOUtils; | |
18 | import be.nikiroo.utils.Progress; | |
19 | ||
20 | /** | |
21 | * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a> | |
22 | * stories, a website dedicated to My Little Pony. | |
23 | * <p> | |
24 | * This version uses the new, official API of FimFiction. | |
25 | * | |
26 | * @author niki | |
27 | */ | |
0ffa4754 | 28 | class FimfictionApi extends BasicSupport_Deprecated { |
315f14ae NR |
29 | private String oauth; |
30 | private String storyId; | |
31 | private String json; | |
32 | ||
33 | private Map<Integer, String> chapterNames; | |
34 | private Map<Integer, String> chapterContents; | |
35 | ||
36 | public FimfictionApi() throws IOException { | |
37 | if (Instance.getConfig().getBoolean( | |
38 | Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) { | |
39 | throw new IOException( | |
40 | "Configuration is set to force HTML scrapping"); | |
41 | } | |
42 | ||
43 | String oauth = Instance.getConfig().getString( | |
44 | Config.LOGIN_FIMFICTION_APIKEY_TOKEN); | |
45 | ||
46 | if (oauth == null || oauth.isEmpty()) { | |
47 | String clientId = Instance.getConfig().getString( | |
48 | Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID) | |
49 | + ""; | |
50 | String clientSecret = Instance.getConfig().getString( | |
51 | Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) | |
52 | + ""; | |
53 | ||
54 | if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) { | |
55 | throw new IOException("API key required for the beta API v2"); | |
56 | } | |
57 | ||
58 | oauth = generateOAuth(clientId, clientSecret); | |
59 | ||
60 | Instance.getConfig().setString( | |
61 | Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth); | |
62 | Instance.getConfig().updateFile(); | |
63 | } | |
64 | ||
65 | this.oauth = oauth; | |
66 | } | |
67 | ||
68 | @Override | |
69 | public String getOAuth() { | |
70 | return oauth; | |
71 | } | |
72 | ||
73 | @Override | |
74 | protected boolean isHtml() { | |
75 | return true; | |
76 | } | |
77 | ||
78 | @Override | |
79 | public String getSourceName() { | |
80 | return "FimFiction.net"; | |
81 | } | |
82 | ||
83 | @Override | |
84 | protected void preprocess(URL source, InputStream in) throws IOException { | |
85 | // extract the ID from: | |
86 | // https://www.fimfiction.net/story/123456/name-of-story | |
87 | storyId = getKeyText(source.toString(), "/story/", null, "/"); | |
88 | ||
89 | // Selectors, so to download all I need and only what I need | |
90 | String storyContent = "fields[story]=title,description,date_published,cover_image"; | |
91 | String authorContent = "fields[author]=name"; | |
77e28d38 | 92 | String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html"; |
315f14ae NR |
93 | String includes = "author,chapters,tags"; |
94 | ||
95 | String urlString = String.format( | |
96 | "https://www.fimfiction.net/api/v2/stories/%s?" // | |
315f14ae NR |
97 | + "%s&%s&%s&" // |
98 | + "include=%s", // | |
99 | storyId, // | |
77e28d38 | 100 | storyContent, authorContent, chapterContent,// |
315f14ae NR |
101 | includes); |
102 | ||
103 | // URL params must be URL-encoded: "[ ]" <-> "%5B %5D" | |
104 | urlString = urlString.replace("[", "%5B").replace("]", "%5D"); | |
105 | ||
106 | URL url = new URL(urlString); | |
107 | InputStream jsonIn = Instance.getCache().open(url, this, false); | |
108 | try { | |
109 | json = IOUtils.readSmallStream(jsonIn); | |
110 | } finally { | |
111 | jsonIn.close(); | |
112 | } | |
113 | } | |
114 | ||
115 | @Override | |
116 | protected InputStream openInput(URL source) throws IOException { | |
117 | return null; | |
118 | } | |
119 | ||
120 | @Override | |
121 | protected MetaData getMeta(URL source, InputStream in) throws IOException { | |
122 | MetaData meta = new MetaData(); | |
123 | ||
124 | meta.setTitle(getKeyJson(json, 0, "type", "story", "title")); | |
125 | meta.setAuthor(getKeyJson(json, 0, "type", "user", "name")); | |
126 | meta.setDate(getKeyJson(json, 0, "type", "story", "date_published")); | |
127 | meta.setTags(getTags()); | |
128 | meta.setSource(getSourceName()); | |
129 | meta.setUrl(source.toString()); | |
130 | meta.setPublisher(getSourceName()); | |
131 | meta.setUuid(source.toString()); | |
132 | meta.setLuid(""); | |
276f95c6 | 133 | meta.setLang("en"); |
315f14ae NR |
134 | meta.setSubject("MLP"); |
135 | meta.setType(getType().toString()); | |
136 | meta.setImageDocument(false); | |
ce297a79 NR |
137 | |
138 | String coverImageLink = getKeyJson(json, 0, "type", "story", | |
139 | "cover_image", "full"); | |
fce43164 NR |
140 | if (!coverImageLink.trim().isEmpty()) { |
141 | meta.setCover(getImage(this, null, coverImageLink.trim())); | |
142 | } | |
315f14ae NR |
143 | |
144 | return meta; | |
145 | } | |
146 | ||
147 | private List<String> getTags() { | |
148 | List<String> tags = new ArrayList<String>(); | |
149 | tags.add("MLP"); | |
150 | ||
151 | int pos = 0; | |
152 | while (pos >= 0) { | |
153 | pos = indexOfJsonAfter(json, pos, "type", "story_tag"); | |
154 | if (pos >= 0) { | |
fce43164 | 155 | tags.add(getKeyJson(json, pos, "name").trim()); |
315f14ae NR |
156 | } |
157 | } | |
158 | ||
159 | return tags; | |
160 | } | |
161 | ||
162 | @Override | |
163 | protected String getDesc(URL source, InputStream in) { | |
d9a94285 | 164 | String desc = getKeyJson(json, 0, "type", "story", "description"); |
a8209dd0 | 165 | return unbbcode(desc); |
315f14ae NR |
166 | } |
167 | ||
168 | @Override | |
169 | protected List<Entry<String, URL>> getChapters(URL source, InputStream in, | |
170 | Progress pg) { | |
ce297a79 NR |
171 | chapterNames = new TreeMap<Integer, String>(); |
172 | chapterContents = new TreeMap<Integer, String>(); | |
315f14ae NR |
173 | |
174 | int pos = 0; | |
175 | while (pos >= 0) { | |
176 | pos = indexOfJsonAfter(json, pos, "type", "chapter"); | |
177 | if (pos >= 0) { | |
178 | int posNumber = indexOfJsonAfter(json, pos, "chapter_number"); | |
179 | int posComa = json.indexOf(",", posNumber); | |
180 | final int number = Integer.parseInt(json.substring(posNumber, | |
181 | posComa).trim()); | |
182 | final String title = getKeyJson(json, pos, "title"); | |
77e28d38 NR |
183 | String notes = getKeyJson(json, pos, "authors_note_html"); |
184 | String content = getKeyJson(json, pos, "content_html"); | |
ce297a79 | 185 | |
fce43164 NR |
186 | if (!notes.trim().isEmpty()) { |
187 | notes = "<br/>* * *<br/>" + notes; | |
188 | } | |
ce297a79 | 189 | |
315f14ae | 190 | chapterNames.put(number, title); |
ce297a79 | 191 | chapterContents.put(number, content + notes); |
315f14ae NR |
192 | } |
193 | } | |
194 | ||
ce297a79 NR |
195 | List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>(); |
196 | for (String title : chapterNames.values()) { | |
197 | urls.add(new AbstractMap.SimpleEntry(title, null)); | |
198 | } | |
199 | ||
315f14ae NR |
200 | return urls; |
201 | } | |
202 | ||
203 | @Override | |
204 | protected String getChapterContent(URL source, InputStream in, int number, | |
205 | Progress pg) { | |
206 | return chapterContents.get(number); | |
207 | } | |
208 | ||
209 | @Override | |
210 | protected boolean supports(URL url) { | |
211 | return "fimfiction.net".equals(url.getHost()) | |
212 | || "www.fimfiction.net".equals(url.getHost()); | |
213 | } | |
214 | ||
a8209dd0 NR |
215 | /** |
216 | * Generate a new token from the client ID and secret. | |
217 | * <p> | |
218 | * Note that those tokens are long-lived, and it would be badly seen to | |
219 | * create a lot of them without due cause. | |
220 | * <p> | |
221 | * So, please cache and re-use them. | |
222 | * | |
223 | * @param clientId | |
224 | * the client ID offered on FimFiction | |
225 | * @param clientSecret | |
226 | * the client secret that goes with it | |
227 | * | |
228 | * @return a new generated token linked to that client ID | |
229 | * | |
230 | * @throws IOException | |
231 | * in case of I/O errors | |
232 | */ | |
315f14ae NR |
233 | static private String generateOAuth(String clientId, String clientSecret) |
234 | throws IOException { | |
235 | URL url = new URL("https://www.fimfiction.net/api/v2/token"); | |
236 | Map<String, String> params = new HashMap<String, String>(); | |
237 | params.put("client_id", clientId); | |
238 | params.put("client_secret", clientSecret); | |
239 | params.put("grant_type", "client_credentials"); | |
240 | InputStream in = Instance.getCache().openNoCache(url, null, params, | |
241 | null, null); | |
242 | ||
243 | String jsonToken = IOUtils.readSmallStream(in); | |
581d42c0 | 244 | in.close(); |
315f14ae NR |
245 | |
246 | // Extract token type and token from: { | |
a8209dd0 | 247 | // token_type = "Bearer", |
315f14ae NR |
248 | // access_token = "xxxxxxxxxxxxxx" |
249 | // } | |
250 | ||
251 | String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\""); | |
252 | String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\""); | |
253 | ||
315f14ae NR |
254 | return tokenType + " " + token; |
255 | } | |
256 | ||
257 | // afters: [name, value] pairs (or "" for any of them), can end without | |
258 | // value | |
259 | static private int indexOfJsonAfter(String json, int startAt, | |
260 | String... afterKeys) { | |
261 | ArrayList<String> afters = new ArrayList<String>(); | |
262 | boolean name = true; | |
263 | for (String key : afterKeys) { | |
264 | if (key != null && !key.isEmpty()) { | |
265 | afters.add("\"" + key + "\""); | |
266 | } else { | |
267 | afters.add("\""); | |
268 | afters.add("\""); | |
269 | } | |
270 | ||
271 | if (name) { | |
272 | afters.add(":"); | |
273 | } | |
274 | ||
275 | name = !name; | |
276 | } | |
277 | ||
278 | return indexOfAfter(json, startAt, afters.toArray(new String[] {})); | |
279 | } | |
280 | ||
281 | // afters: [name, value] pairs (or "" for any of them), can end without | |
fce43164 | 282 | // value but will then be empty, not NULL |
315f14ae NR |
283 | static private String getKeyJson(String json, int startAt, |
284 | String... afterKeys) { | |
285 | int pos = indexOfJsonAfter(json, startAt, afterKeys); | |
286 | if (pos < 0) { | |
fce43164 | 287 | return ""; |
315f14ae NR |
288 | } |
289 | ||
fce43164 | 290 | String result = ""; |
37fdbdef NR |
291 | String wip = json.substring(pos); |
292 | ||
293 | pos = nextUnescapedQuote(wip, 0); | |
294 | if (pos >= 0) { | |
295 | wip = wip.substring(pos + 1); | |
296 | pos = nextUnescapedQuote(wip, 0); | |
297 | if (pos >= 0) { | |
298 | result = wip.substring(0, pos); | |
299 | } | |
300 | } | |
ce297a79 NR |
301 | |
302 | result = result.replace("\\t", "\t").replace("\\\"", "\""); | |
303 | ||
37fdbdef NR |
304 | return result; |
305 | } | |
306 | ||
307 | // next " but don't take \" into account | |
308 | static private int nextUnescapedQuote(String result, int pos) { | |
309 | while (pos >= 0) { | |
310 | pos = result.indexOf("\"", pos); | |
311 | if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) { | |
312 | break; | |
313 | } | |
314 | ||
315 | if (pos < result.length()) { | |
316 | pos++; | |
317 | } | |
318 | } | |
319 | ||
320 | return pos; | |
315f14ae | 321 | } |
a8209dd0 NR |
322 | |
323 | // quick & dirty filter | |
324 | static private String unbbcode(String bbcode) { | |
325 | String text = bbcode.replace("\\r\\n", "<br/>") // | |
326 | .replace("[i]", "_").replace("[/i]", "_") // | |
327 | .replace("[b]", "*").replace("[/b]", "*") // | |
328 | .replaceAll("\\[[^\\]]*\\]", ""); | |
329 | return text; | |
330 | } | |
315f14ae | 331 | } |