1 package be
.nikiroo
.fanfix
.supported
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.util
.AbstractMap
;
7 import java
.util
.ArrayList
;
8 import java
.util
.HashMap
;
11 import java
.util
.Map
.Entry
;
12 import java
.util
.TreeMap
;
14 import be
.nikiroo
.fanfix
.Instance
;
15 import be
.nikiroo
.fanfix
.bundles
.Config
;
16 import be
.nikiroo
.fanfix
.data
.MetaData
;
17 import be
.nikiroo
.utils
.IOUtils
;
18 import be
.nikiroo
.utils
.Progress
;
21 * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a>
22 * stories, a website dedicated to My Little Pony.
24 * This version uses the new, official API of FimFiction.
28 class FimfictionApi
extends BasicSupport_Deprecated
{
30 private String storyId
;
33 private Map
<Integer
, String
> chapterNames
;
34 private Map
<Integer
, String
> chapterContents
;
36 public FimfictionApi() throws IOException
{
37 if (Instance
.getConfig().getBoolean(
38 Config
.LOGIN_FIMFICTION_APIKEY_FORCE_HTML
, false)) {
39 throw new IOException(
40 "Configuration is set to force HTML scrapping");
43 String oauth
= Instance
.getConfig().getString(
44 Config
.LOGIN_FIMFICTION_APIKEY_TOKEN
);
46 if (oauth
== null || oauth
.isEmpty()) {
47 String clientId
= Instance
.getConfig().getString(
48 Config
.LOGIN_FIMFICTION_APIKEY_CLIENT_ID
)
50 String clientSecret
= Instance
.getConfig().getString(
51 Config
.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET
)
54 if (clientId
.trim().isEmpty() || clientSecret
.trim().isEmpty()) {
55 throw new IOException("API key required for the beta API v2");
58 oauth
= generateOAuth(clientId
, clientSecret
);
60 Instance
.getConfig().setString(
61 Config
.LOGIN_FIMFICTION_APIKEY_TOKEN
, oauth
);
62 Instance
.getConfig().updateFile();
69 public String
getOAuth() {
74 protected boolean isHtml() {
79 public String
getSourceName() {
80 return "FimFiction.net";
84 protected void preprocess(URL source
, InputStream in
) throws IOException
{
85 // extract the ID from:
86 // https://www.fimfiction.net/story/123456/name-of-story
87 storyId
= getKeyText(source
.toString(), "/story/", null, "/");
89 // Selectors, so to download all I need and only what I need
90 String storyContent
= "fields[story]=title,description,date_published,cover_image";
91 String authorContent
= "fields[author]=name";
92 String chapterContent
= "fields[chapter]=chapter_number,title,content_html,authors_note_html";
93 String includes
= "author,chapters,tags";
95 String urlString
= String
.format(
96 "https://www.fimfiction.net/api/v2/stories/%s?" //
100 storyContent
, authorContent
, chapterContent
,//
103 // URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
104 urlString
= urlString
.replace("[", "%5B").replace("]", "%5D");
106 URL url
= new URL(urlString
);
107 InputStream jsonIn
= Instance
.getCache().open(url
, this, false);
109 json
= IOUtils
.readSmallStream(jsonIn
);
116 protected InputStream
openInput(URL source
) throws IOException
{
121 protected MetaData
getMeta(URL source
, InputStream in
) throws IOException
{
122 MetaData meta
= new MetaData();
124 meta
.setTitle(getKeyJson(json
, 0, "type", "story", "title"));
125 meta
.setAuthor(getKeyJson(json
, 0, "type", "user", "name"));
126 meta
.setDate(getKeyJson(json
, 0, "type", "story", "date_published"));
127 meta
.setTags(getTags());
128 meta
.setSource(getSourceName());
129 meta
.setUrl(source
.toString());
130 meta
.setPublisher(getSourceName());
131 meta
.setUuid(source
.toString());
134 meta
.setSubject("MLP");
135 meta
.setType(getType().toString());
136 meta
.setImageDocument(false);
138 String coverImageLink
= getKeyJson(json
, 0, "type", "story",
139 "cover_image", "full");
140 if (!coverImageLink
.trim().isEmpty()) {
141 meta
.setCover(getImage(this, null, coverImageLink
.trim()));
147 private List
<String
> getTags() {
148 List
<String
> tags
= new ArrayList
<String
>();
153 pos
= indexOfJsonAfter(json
, pos
, "type", "story_tag");
155 tags
.add(getKeyJson(json
, pos
, "name").trim());
163 protected String
getDesc(URL source
, InputStream in
) {
164 String desc
= getKeyJson(json
, 0, "type", "story", "description");
165 return unbbcode(desc
);
169 protected List
<Entry
<String
, URL
>> getChapters(URL source
, InputStream in
,
171 chapterNames
= new TreeMap
<Integer
, String
>();
172 chapterContents
= new TreeMap
<Integer
, String
>();
176 pos
= indexOfJsonAfter(json
, pos
, "type", "chapter");
178 int posNumber
= indexOfJsonAfter(json
, pos
, "chapter_number");
179 int posComa
= json
.indexOf(",", posNumber
);
180 final int number
= Integer
.parseInt(json
.substring(posNumber
,
182 final String title
= getKeyJson(json
, pos
, "title");
183 String notes
= getKeyJson(json
, pos
, "authors_note_html");
184 String content
= getKeyJson(json
, pos
, "content_html");
186 if (!notes
.trim().isEmpty()) {
187 notes
= "<br/>* * *<br/>" + notes
;
190 chapterNames
.put(number
, title
);
191 chapterContents
.put(number
, content
+ notes
);
195 List
<Entry
<String
, URL
>> urls
= new ArrayList
<Entry
<String
, URL
>>();
196 for (String title
: chapterNames
.values()) {
197 urls
.add(new AbstractMap
.SimpleEntry(title
, null));
204 protected String
getChapterContent(URL source
, InputStream in
, int number
,
206 return chapterContents
.get(number
);
210 protected boolean supports(URL url
) {
211 return "fimfiction.net".equals(url
.getHost())
212 || "www.fimfiction.net".equals(url
.getHost());
216 * Generate a new token from the client ID and secret.
218 * Note that those tokens are long-lived, and it would be badly seen to
219 * create a lot of them without due cause.
221 * So, please cache and re-use them.
224 * the client ID offered on FimFiction
225 * @param clientSecret
226 * the client secret that goes with it
228 * @return a new generated token linked to that client ID
230 * @throws IOException
231 * in case of I/O errors
233 static private String
generateOAuth(String clientId
, String clientSecret
)
235 URL url
= new URL("https://www.fimfiction.net/api/v2/token");
236 Map
<String
, String
> params
= new HashMap
<String
, String
>();
237 params
.put("client_id", clientId
);
238 params
.put("client_secret", clientSecret
);
239 params
.put("grant_type", "client_credentials");
240 InputStream in
= Instance
.getCache().openNoCache(url
, null, params
,
243 String jsonToken
= IOUtils
.readSmallStream(in
);
246 // Extract token type and token from: {
247 // token_type = "Bearer",
248 // access_token = "xxxxxxxxxxxxxx"
251 String token
= getKeyText(jsonToken
, "\"access_token\"", "\"", "\"");
252 String tokenType
= getKeyText(jsonToken
, "\"token_type\"", "\"", "\"");
254 return tokenType
+ " " + token
;
257 // afters: [name, value] pairs (or "" for any of them), can end without
259 static private int indexOfJsonAfter(String json
, int startAt
,
260 String
... afterKeys
) {
261 ArrayList
<String
> afters
= new ArrayList
<String
>();
263 for (String key
: afterKeys
) {
264 if (key
!= null && !key
.isEmpty()) {
265 afters
.add("\"" + key
+ "\"");
278 return indexOfAfter(json
, startAt
, afters
.toArray(new String
[] {}));
281 // afters: [name, value] pairs (or "" for any of them), can end without
282 // value but will then be empty, not NULL
283 static private String
getKeyJson(String json
, int startAt
,
284 String
... afterKeys
) {
285 int pos
= indexOfJsonAfter(json
, startAt
, afterKeys
);
291 String wip
= json
.substring(pos
);
293 pos
= nextUnescapedQuote(wip
, 0);
295 wip
= wip
.substring(pos
+ 1);
296 pos
= nextUnescapedQuote(wip
, 0);
298 result
= wip
.substring(0, pos
);
302 result
= result
.replace("\\t", "\t").replace("\\\"", "\"");
307 // next " but don't take \" into account
308 static private int nextUnescapedQuote(String result
, int pos
) {
310 pos
= result
.indexOf("\"", pos
);
311 if (pos
== 0 || (pos
> 0 && result
.charAt(pos
- 1) != '\\')) {
315 if (pos
< result
.length()) {
323 // quick & dirty filter
324 static private String
unbbcode(String bbcode
) {
325 String text
= bbcode
.replace("\\r\\n", "<br/>") //
326 .replace("[i]", "_").replace("[/i]", "_") //
327 .replace("[b]", "*").replace("[/b]", "*") //
328 .replaceAll("\\[[^\\]]*\\]", "");