src/be/nikiroo/fanfix/supported/FimfictionApi.java

   1 package be.nikiroo.fanfix.supported;
   2
   3 import java.io.IOException;
   4 import java.io.InputStream;
   5 import java.net.URL;
   6 import java.util.AbstractMap;
   7 import java.util.ArrayList;
   8 import java.util.HashMap;
   9 import java.util.List;
  10 import java.util.Map;
  11 import java.util.Map.Entry;
  12 import java.util.TreeMap;
  13
  14 import org.jsoup.nodes.Document;
  15
  16 import be.nikiroo.fanfix.Instance;
  17 import be.nikiroo.fanfix.bundles.Config;
  18 import be.nikiroo.fanfix.data.MetaData;
  19 import be.nikiroo.fanfix.data.Story;
  20 import be.nikiroo.utils.IOUtils;
  21 import be.nikiroo.utils.Image;
  22 import be.nikiroo.utils.Progress;
  23
  24 /**
  25  * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a>
  26  * stories, a website dedicated to My Little Pony.
  27  * <p>
  28  * This version uses the new, official API of FimFiction.
  29  *
  30  * @author niki
  31  */
  32 class FimfictionApi extends BasicSupport {
  33         private String oauth;
  34         private String json;
  35
  36         private Map<Integer, String> chapterNames;
  37         private Map<Integer, String> chapterContents;
  38
  39         public FimfictionApi() throws IOException {
  40                 if (Instance.getInstance().getConfig().getBoolean(Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) {
  41                         throw new IOException("Configuration is set to force HTML scrapping");
  42                 }
  43
  44                 String oauth = Instance.getInstance().getConfig().getString(Config.LOGIN_FIMFICTION_APIKEY_TOKEN);
  45
  46                 if (oauth == null || oauth.isEmpty()) {
  47                         String clientId = Instance.getInstance().getConfig().getString(Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID)
  48                                         + "";
  49                         String clientSecret = Instance.getInstance().getConfig()
  50                                         .getString(Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) + "";
  51
  52                         if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) {
  53                                 throw new IOException("API key required for the beta API v2");
  54                         }
  55
  56                         oauth = generateOAuth(clientId, clientSecret);
  57
  58                         Instance.getInstance().getConfig().setString(Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth);
  59                         Instance.getInstance().getConfig().updateFile();
  60                 }
  61
  62                 this.oauth = oauth;
  63         }
  64
  65         @Override
  66         protected Document loadDocument(URL source) throws IOException {
  67                 json = getJsonData();
  68                 return null;
  69         }
  70
  71         @Override
  72         public String getOAuth() {
  73                 return oauth;
  74         }
  75
  76         @Override
  77         protected boolean isHtml() {
  78                 return true;
  79         }
  80
  81         /**
  82          * Extract the full JSON data we will later use to build the {@link Story}.
  83          *
  84          * @return the data in a JSON format
  85          *
  86          * @throws IOException
  87          *             in case of I/O error
  88          */
  89         private String getJsonData() throws IOException {
  90                 // extract the ID from:
  91                 // https://www.fimfiction.net/story/123456/name-of-story
  92                 String storyId = getKeyText(getSource().toString(), "/story/", null,
  93                                 "/");
  94
  95                 // Selectors, so to download all I need and only what I need
  96                 String storyContent = "fields[story]=title,description,date_published,cover_image";
  97                 String authorContent = "fields[author]=name";
  98                 String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html";
  99                 String includes = "author,chapters,tags";
 100
 101                 String urlString = String.format(
 102                                 "https://www.fimfiction.net/api/v2/stories/%s?" //
 103                                                 + "%s&%s&%s&" //
 104                                                 + "include=%s", //
 105                                 storyId, //
 106                                 storyContent, authorContent, chapterContent,//
 107                                 includes);
 108
 109                 // URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
 110                 urlString = urlString.replace("[", "%5B").replace("]", "%5D");
 111
 112                 URL url = new URL(urlString);
 113                 InputStream jsonIn = Instance.getInstance().getCache().open(url, this, false);
 114                 try {
 115                         return IOUtils.readSmallStream(jsonIn);
 116                 } finally {
 117                         jsonIn.close();
 118                 }
 119         }
 120
 121         @Override
 122         protected MetaData getMeta() throws IOException {
 123                 MetaData meta = new MetaData();
 124
 125                 meta.setTitle(getKeyJson(json, 0, "type", "story", "title"));
 126                 meta.setAuthor(getKeyJson(json, 0, "type", "user", "name"));
 127                 meta.setDate(bsHelper.formatDate(
 128                                 getKeyJson(json, 0, "type", "story", "date_published")));
 129                 meta.setTags(getTags());
 130                 meta.setSource(getType().getSourceName());
 131                 meta.setUrl(getSource().toString());
 132                 meta.setPublisher(getType().getSourceName());
 133                 meta.setUuid(getSource().toString());
 134                 meta.setLuid("");
 135                 meta.setLang("en");
 136                 meta.setSubject("MLP");
 137                 meta.setType(getType().toString());
 138                 meta.setImageDocument(false);
 139
 140                 String coverImageLink = getKeyJson(json, 0, "type", "story",
 141                                 "cover_image", "full");
 142                 if (!coverImageLink.trim().isEmpty()) {
 143                         URL coverImageUrl = new URL(coverImageLink.trim());
 144
 145                         // No need to use the oauth, cookies... for the cover
 146                         // Plus: it crashes on Android because of the referer
 147                         try {
 148                                 InputStream in = Instance.getInstance().getCache().open(coverImageUrl, null, true);
 149                                 try {
 150                                         Image img = new Image(in);
 151                                         if (img.getSize() == 0) {
 152                                                 img.close();
 153                                                 throw new IOException(
 154                                                                 "Empty image not accepted");
 155                                         }
 156                                         meta.setCover(img);
 157                                 } finally {
 158                                         in.close();
 159                                 }
 160                         } catch (IOException e) {
 161                                 Instance.getInstance().getTraceHandler()
 162                                                 .error(new IOException("Cannot get the story cover, ignoring...", e));
 163                         }
 164                 }
 165
 166                 return meta;
 167         }
 168
 169         private List<String> getTags() {
 170                 List<String> tags = new ArrayList<String>();
 171                 tags.add("MLP");
 172
 173                 int pos = 0;
 174                 while (pos >= 0) {
 175                         pos = indexOfJsonAfter(json, pos, "type", "story_tag");
 176                         if (pos >= 0) {
 177                                 tags.add(getKeyJson(json, pos, "name").trim());
 178                         }
 179                 }
 180
 181                 return tags;
 182         }
 183
 184         @Override
 185         protected String getDesc() {
 186                 String desc = getKeyJson(json, 0, "type", "story", "description");
 187                 return unbbcode(desc);
 188         }
 189
 190         @Override
 191         protected List<Entry<String, URL>> getChapters(Progress pg) {
 192                 chapterNames = new TreeMap<Integer, String>();
 193                 chapterContents = new TreeMap<Integer, String>();
 194
 195                 int pos = 0;
 196                 while (pos >= 0) {
 197                         pos = indexOfJsonAfter(json, pos, "type", "chapter");
 198                         if (pos >= 0) {
 199                                 int posNumber = indexOfJsonAfter(json, pos, "chapter_number");
 200                                 int posComa = json.indexOf(",", posNumber);
 201                                 final int number = Integer.parseInt(json.substring(posNumber,
 202                                                 posComa).trim());
 203                                 final String title = getKeyJson(json, pos, "title");
 204                                 String notes = getKeyJson(json, pos, "authors_note_html");
 205                                 String content = getKeyJson(json, pos, "content_html");
 206
 207                                 if (!notes.trim().isEmpty()) {
 208                                         notes = "<br/>* * *<br/>" + notes;
 209                                 }
 210
 211                                 chapterNames.put(number, title);
 212                                 chapterContents.put(number, content + notes);
 213                         }
 214                 }
 215
 216                 List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
 217                 for (String title : chapterNames.values()) {
 218                         urls.add(new AbstractMap.SimpleEntry<String, URL>(title, null));
 219                 }
 220
 221                 return urls;
 222         }
 223
 224         @Override
 225         protected String getChapterContent(URL source, int number, Progress pg) {
 226                 return chapterContents.get(number);
 227         }
 228
 229         @Override
 230         protected boolean supports(URL url) {
 231                 return "fimfiction.net".equals(url.getHost())
 232                                 || "www.fimfiction.net".equals(url.getHost());
 233         }
 234
 235         /**
 236          * Generate a new token from the client ID and secret.
 237          * <p>
 238          * Note that those tokens are long-lived, and it would be badly seen to
 239          * create a lot of them without due cause.
 240          * <p>
 241          * So, please cache and re-use them.
 242          *
 243          * @param clientId
 244          *            the client ID offered on FimFiction
 245          * @param clientSecret
 246          *            the client secret that goes with it
 247          *
 248          * @return a new generated token linked to that client ID
 249          *
 250          * @throws IOException
 251          *             in case of I/O errors
 252          */
 253         static private String generateOAuth(String clientId, String clientSecret)
 254                         throws IOException {
 255                 URL url = new URL("https://www.fimfiction.net/api/v2/token");
 256                 Map<String, String> params = new HashMap<String, String>();
 257                 params.put("client_id", clientId);
 258                 params.put("client_secret", clientSecret);
 259                 params.put("grant_type", "client_credentials");
 260                 InputStream in = Instance.getInstance().getCache().openNoCache(url, null, params, null, null);
 261
 262                 String jsonToken = IOUtils.readSmallStream(in);
 263                 in.close();
 264
 265                 // Extract token type and token from: {
 266                 // token_type = "Bearer",
 267                 // access_token = "xxxxxxxxxxxxxx"
 268                 // }
 269
 270                 String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\"");
 271                 String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\"");
 272
 273                 return tokenType + " " + token;
 274         }
 275
 276         // afters: [name, value] pairs (or "" for any of them), can end without
 277         // value
 278         static private int indexOfJsonAfter(String json, int startAt,
 279                         String... afterKeys) {
 280                 ArrayList<String> afters = new ArrayList<String>();
 281                 boolean name = true;
 282                 for (String key : afterKeys) {
 283                         if (key != null && !key.isEmpty()) {
 284                                 afters.add("\"" + key + "\"");
 285                         } else {
 286                                 afters.add("\"");
 287                                 afters.add("\"");
 288                         }
 289
 290                         if (name) {
 291                                 afters.add(":");
 292                         }
 293
 294                         name = !name;
 295                 }
 296
 297                 return indexOfAfter(json, startAt, afters.toArray(new String[] {}));
 298         }
 299
 300         // afters: [name, value] pairs (or "" for any of them), can end without
 301         // value but will then be empty, not NULL
 302         static private String getKeyJson(String json, int startAt,
 303                         String... afterKeys) {
 304                 int pos = indexOfJsonAfter(json, startAt, afterKeys);
 305                 if (pos < 0) {
 306                         return "";
 307                 }
 308
 309                 String result = "";
 310                 String wip = json.substring(pos);
 311
 312                 pos = nextUnescapedQuote(wip, 0);
 313                 if (pos >= 0) {
 314                         wip = wip.substring(pos + 1);
 315                         pos = nextUnescapedQuote(wip, 0);
 316                         if (pos >= 0) {
 317                                 result = wip.substring(0, pos);
 318                         }
 319                 }
 320
 321                 result = result.replace("\\t", "\t").replace("\\\"", "\"");
 322
 323                 return result;
 324         }
 325
 326         // next " but don't take \" into account
 327         static private int nextUnescapedQuote(String result, int pos) {
 328                 while (pos >= 0) {
 329                         pos = result.indexOf("\"", pos);
 330                         if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) {
 331                                 break;
 332                         }
 333
 334                         if (pos < result.length()) {
 335                                 pos++;
 336                         }
 337                 }
 338
 339                 return pos;
 340         }
 341
 342         // quick & dirty filter
 343         static private String unbbcode(String bbcode) {
 344                 String text = bbcode.replace("\\r\\n", "<br/>") //
 345                                 .replace("[i]", "_").replace("[/i]", "_") //
 346                                 .replace("[b]", "*").replace("[/b]", "*") //
 347                                 .replaceAll("\\[[^\\]]*\\]", "");
 348                 return text;
 349         }
 350
 351         /**
 352          * Return the text between the key and the endKey (and optional subKey can
 353          * be passed, in this case we will look for the key first, then take the
 354          * text between the subKey and the endKey).
 355          *
 356          * @param in
 357          *            the input
 358          * @param key
 359          *            the key to match (also supports "^" at start to say
 360          *            "only if it starts with" the key)
 361          * @param subKey
 362          *            the sub key or NULL if none
 363          * @param endKey
 364          *            the end key or NULL for "up to the end"
 365          * @return the text or NULL if not found
 366          */
 367         static private String getKeyText(String in, String key, String subKey,
 368                         String endKey) {
 369                 String result = null;
 370
 371                 String line = in;
 372                 if (line != null && line.contains(key)) {
 373                         line = line.substring(line.indexOf(key) + key.length());
 374                         if (subKey == null || subKey.isEmpty() || line.contains(subKey)) {
 375                                 if (subKey != null) {
 376                                         line = line.substring(line.indexOf(subKey)
 377                                                         + subKey.length());
 378                                 }
 379                                 if (endKey == null || line.contains(endKey)) {
 380                                         if (endKey != null) {
 381                                                 line = line.substring(0, line.indexOf(endKey));
 382                                                 result = line;
 383                                         }
 384                                 }
 385                         }
 386                 }
 387
 388                 return result;
 389         }
 390
 391         /**
 392          * Return the first index after all the given "afters" have been found in
 393          * the {@link String}, or -1 if it was not possible.
 394          *
 395          * @param in
 396          *            the input
 397          * @param startAt
 398          *            start at this position in the string
 399          * @param afters
 400          *            the sub-keys to find before checking for key/endKey
 401          *
 402          * @return the text or NULL if not found
 403          */
 404         static private int indexOfAfter(String in, int startAt, String... afters) {
 405                 int pos = -1;
 406                 if (in != null && !in.isEmpty()) {
 407                         pos = startAt;
 408                         if (afters != null) {
 409                                 for (int i = 0; pos >= 0 && i < afters.length; i++) {
 410                                         String subKey = afters[i];
 411                                         if (!subKey.isEmpty()) {
 412                                                 pos = in.indexOf(subKey, pos);
 413                                                 if (pos >= 0) {
 414                                                         pos += subKey.length();
 415                                                 }
 416                                         }
 417                                 }
 418                         }
 419                 }
 420
 421                 return pos;
 422         }
 423 }