1 package be
.nikiroo
.fanfix
.supported
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.util
.AbstractMap
;
7 import java
.util
.ArrayList
;
8 import java
.util
.HashMap
;
11 import java
.util
.Map
.Entry
;
12 import java
.util
.TreeMap
;
14 import org
.jsoup
.nodes
.Document
;
16 import be
.nikiroo
.fanfix
.Instance
;
17 import be
.nikiroo
.fanfix
.bundles
.Config
;
18 import be
.nikiroo
.fanfix
.data
.MetaData
;
19 import be
.nikiroo
.fanfix
.data
.Story
;
20 import be
.nikiroo
.utils
.IOUtils
;
21 import be
.nikiroo
.utils
.Image
;
22 import be
.nikiroo
.utils
.Progress
;
25 * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a>
26 * stories, a website dedicated to My Little Pony.
28 * This version uses the new, official API of FimFiction.
32 class FimfictionApi
extends BasicSupport
{
36 private Map
<Integer
, String
> chapterNames
;
37 private Map
<Integer
, String
> chapterContents
;
39 public FimfictionApi() throws IOException
{
40 if (Instance
.getConfig().getBoolean(
41 Config
.LOGIN_FIMFICTION_APIKEY_FORCE_HTML
, false)) {
42 throw new IOException(
43 "Configuration is set to force HTML scrapping");
46 String oauth
= Instance
.getConfig().getString(
47 Config
.LOGIN_FIMFICTION_APIKEY_TOKEN
);
49 if (oauth
== null || oauth
.isEmpty()) {
50 String clientId
= Instance
.getConfig().getString(
51 Config
.LOGIN_FIMFICTION_APIKEY_CLIENT_ID
)
53 String clientSecret
= Instance
.getConfig().getString(
54 Config
.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET
)
57 if (clientId
.trim().isEmpty() || clientSecret
.trim().isEmpty()) {
58 throw new IOException("API key required for the beta API v2");
61 oauth
= generateOAuth(clientId
, clientSecret
);
63 Instance
.getConfig().setString(
64 Config
.LOGIN_FIMFICTION_APIKEY_TOKEN
, oauth
);
65 Instance
.getConfig().updateFile();
72 protected Document
loadDocument(URL source
) throws IOException
{
78 public String
getOAuth() {
83 protected boolean isHtml() {
88 public String
getSourceName() {
89 return "FimFiction.net";
93 * Extract the full JSON data we will later use to build the {@link Story}.
95 * @return the data in a JSON format
98 * in case of I/O error
100 private String
getJsonData() throws IOException
{
101 // extract the ID from:
102 // https://www.fimfiction.net/story/123456/name-of-story
103 String storyId
= getKeyText(getSource().toString(), "/story/", null,
106 // Selectors, so to download all I need and only what I need
107 String storyContent
= "fields[story]=title,description,date_published,cover_image";
108 String authorContent
= "fields[author]=name";
109 String chapterContent
= "fields[chapter]=chapter_number,title,content_html,authors_note_html";
110 String includes
= "author,chapters,tags";
112 String urlString
= String
.format(
113 "https://www.fimfiction.net/api/v2/stories/%s?" //
117 storyContent
, authorContent
, chapterContent
,//
120 // URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
121 urlString
= urlString
.replace("[", "%5B").replace("]", "%5D");
123 URL url
= new URL(urlString
);
124 InputStream jsonIn
= Instance
.getCache().open(url
, this, false);
126 return IOUtils
.readSmallStream(jsonIn
);
133 protected MetaData
getMeta() throws IOException
{
134 MetaData meta
= new MetaData();
136 meta
.setTitle(getKeyJson(json
, 0, "type", "story", "title"));
137 meta
.setAuthor(getKeyJson(json
, 0, "type", "user", "name"));
138 meta
.setDate(getKeyJson(json
, 0, "type", "story", "date_published"));
139 meta
.setTags(getTags());
140 meta
.setSource(getSourceName());
141 meta
.setUrl(getSource().toString());
142 meta
.setPublisher(getSourceName());
143 meta
.setUuid(getSource().toString());
146 meta
.setSubject("MLP");
147 meta
.setType(getType().toString());
148 meta
.setImageDocument(false);
150 String coverImageLink
= getKeyJson(json
, 0, "type", "story",
151 "cover_image", "full");
152 if (!coverImageLink
.trim().isEmpty()) {
153 InputStream in
= null;
155 URL coverImageUrl
= new URL(coverImageLink
.trim());
156 in
= Instance
.getCache().open(coverImageUrl
, this, true);
157 meta
.setCover(new Image(in
));
166 private List
<String
> getTags() {
167 List
<String
> tags
= new ArrayList
<String
>();
172 pos
= indexOfJsonAfter(json
, pos
, "type", "story_tag");
174 tags
.add(getKeyJson(json
, pos
, "name").trim());
182 protected String
getDesc() {
183 String desc
= getKeyJson(json
, 0, "type", "story", "description");
184 return unbbcode(desc
);
188 protected List
<Entry
<String
, URL
>> getChapters(Progress pg
) {
189 chapterNames
= new TreeMap
<Integer
, String
>();
190 chapterContents
= new TreeMap
<Integer
, String
>();
194 pos
= indexOfJsonAfter(json
, pos
, "type", "chapter");
196 int posNumber
= indexOfJsonAfter(json
, pos
, "chapter_number");
197 int posComa
= json
.indexOf(",", posNumber
);
198 final int number
= Integer
.parseInt(json
.substring(posNumber
,
200 final String title
= getKeyJson(json
, pos
, "title");
201 String notes
= getKeyJson(json
, pos
, "authors_note_html");
202 String content
= getKeyJson(json
, pos
, "content_html");
204 if (!notes
.trim().isEmpty()) {
205 notes
= "<br/>* * *<br/>" + notes
;
208 chapterNames
.put(number
, title
);
209 chapterContents
.put(number
, content
+ notes
);
213 List
<Entry
<String
, URL
>> urls
= new ArrayList
<Entry
<String
, URL
>>();
214 for (String title
: chapterNames
.values()) {
215 urls
.add(new AbstractMap
.SimpleEntry
<String
, URL
>(title
, null));
222 protected String
getChapterContent(URL source
, int number
, Progress pg
) {
223 return chapterContents
.get(number
);
227 protected boolean supports(URL url
) {
228 return "fimfiction.net".equals(url
.getHost())
229 || "www.fimfiction.net".equals(url
.getHost());
233 * Generate a new token from the client ID and secret.
235 * Note that those tokens are long-lived, and it would be badly seen to
236 * create a lot of them without due cause.
238 * So, please cache and re-use them.
241 * the client ID offered on FimFiction
242 * @param clientSecret
243 * the client secret that goes with it
245 * @return a new generated token linked to that client ID
247 * @throws IOException
248 * in case of I/O errors
250 static private String
generateOAuth(String clientId
, String clientSecret
)
252 URL url
= new URL("https://www.fimfiction.net/api/v2/token");
253 Map
<String
, String
> params
= new HashMap
<String
, String
>();
254 params
.put("client_id", clientId
);
255 params
.put("client_secret", clientSecret
);
256 params
.put("grant_type", "client_credentials");
257 InputStream in
= Instance
.getCache().openNoCache(url
, null, params
,
260 String jsonToken
= IOUtils
.readSmallStream(in
);
263 // Extract token type and token from: {
264 // token_type = "Bearer",
265 // access_token = "xxxxxxxxxxxxxx"
268 String tokenType
= getKeyText(jsonToken
, "\"token_type\"", "\"", "\"");
269 String token
= getKeyText(jsonToken
, "\"access_token\"", "\"", "\"");
271 return tokenType
+ " " + token
;
274 // afters: [name, value] pairs (or "" for any of them), can end without
276 static private int indexOfJsonAfter(String json
, int startAt
,
277 String
... afterKeys
) {
278 ArrayList
<String
> afters
= new ArrayList
<String
>();
280 for (String key
: afterKeys
) {
281 if (key
!= null && !key
.isEmpty()) {
282 afters
.add("\"" + key
+ "\"");
295 return indexOfAfter(json
, startAt
, afters
.toArray(new String
[] {}));
298 // afters: [name, value] pairs (or "" for any of them), can end without
299 // value but will then be empty, not NULL
300 static private String
getKeyJson(String json
, int startAt
,
301 String
... afterKeys
) {
302 int pos
= indexOfJsonAfter(json
, startAt
, afterKeys
);
308 String wip
= json
.substring(pos
);
310 pos
= nextUnescapedQuote(wip
, 0);
312 wip
= wip
.substring(pos
+ 1);
313 pos
= nextUnescapedQuote(wip
, 0);
315 result
= wip
.substring(0, pos
);
319 result
= result
.replace("\\t", "\t").replace("\\\"", "\"");
324 // next " but don't take \" into account
325 static private int nextUnescapedQuote(String result
, int pos
) {
327 pos
= result
.indexOf("\"", pos
);
328 if (pos
== 0 || (pos
> 0 && result
.charAt(pos
- 1) != '\\')) {
332 if (pos
< result
.length()) {
340 // quick & dirty filter
341 static private String
unbbcode(String bbcode
) {
342 String text
= bbcode
.replace("\\r\\n", "<br/>") //
343 .replace("[i]", "_").replace("[/i]", "_") //
344 .replace("[b]", "*").replace("[/b]", "*") //
345 .replaceAll("\\[[^\\]]*\\]", "");
350 * Return the text between the key and the endKey (and optional subKey can
351 * be passed, in this case we will look for the key first, then take the
352 * text between the subKey and the endKey).
357 * the key to match (also supports "^" at start to say
358 * "only if it starts with" the key)
360 * the sub key or NULL if none
362 * the end key or NULL for "up to the end"
363 * @return the text or NULL if not found
365 static private String
getKeyText(String in
, String key
, String subKey
,
367 String result
= null;
370 if (line
!= null && line
.contains(key
)) {
371 line
= line
.substring(line
.indexOf(key
) + key
.length());
372 if (subKey
== null || subKey
.isEmpty() || line
.contains(subKey
)) {
373 if (subKey
!= null) {
374 line
= line
.substring(line
.indexOf(subKey
)
377 if (endKey
== null || line
.contains(endKey
)) {
378 if (endKey
!= null) {
379 line
= line
.substring(0, line
.indexOf(endKey
));
390 * Return the first index after all the given "afters" have been found in
391 * the {@link String}, or -1 if it was not possible.
396 * start at this position in the string
398 * the sub-keys to find before checking for key/endKey
400 * @return the text or NULL if not found
402 static private int indexOfAfter(String in
, int startAt
, String
... afters
) {
404 if (in
!= null && !in
.isEmpty()) {
406 if (afters
!= null) {
407 for (int i
= 0; pos
>= 0 && i
< afters
.length
; i
++) {
408 String subKey
= afters
[i
];
409 if (!subKey
.isEmpty()) {
410 pos
= in
.indexOf(subKey
, pos
);
412 pos
+= subKey
.length();