src/be/nikiroo/fanfix/supported/FimfictionApi.java

   1 package be.nikiroo.fanfix.supported;
   2
   3 import java.io.IOException;
   4 import java.io.InputStream;
   5 import java.net.URL;
   6 import java.util.ArrayList;
   7 import java.util.HashMap;
   8 import java.util.List;
   9 import java.util.Map;
  10 import java.util.Map.Entry;
  11
  12 import be.nikiroo.fanfix.Instance;
  13 import be.nikiroo.fanfix.bundles.Config;
  14 import be.nikiroo.fanfix.data.MetaData;
  15 import be.nikiroo.utils.IOUtils;
  16 import be.nikiroo.utils.Progress;
  17
  18 /**
  19  * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a>
  20  * stories, a website dedicated to My Little Pony.
  21  * <p>
  22  * This version uses the new, official API of FimFiction.
  23  *
  24  * @author niki
  25  */
  26 class FimfictionApi extends BasicSupport {
  27         private String oauth;
  28         private String storyId;
  29         private String json;
  30
  31         private Map<Integer, String> chapterNames;
  32         private Map<Integer, String> chapterContents;
  33
  34         public FimfictionApi() throws IOException {
  35                 if (Instance.getConfig().getBoolean(
  36                                 Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) {
  37                         throw new IOException(
  38                                         "Configuration is set to force HTML scrapping");
  39                 }
  40
  41                 String oauth = Instance.getConfig().getString(
  42                                 Config.LOGIN_FIMFICTION_APIKEY_TOKEN);
  43
  44                 if (oauth == null || oauth.isEmpty()) {
  45                         String clientId = Instance.getConfig().getString(
  46                                         Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID)
  47                                         + "";
  48                         String clientSecret = Instance.getConfig().getString(
  49                                         Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET)
  50                                         + "";
  51
  52                         if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) {
  53                                 throw new IOException("API key required for the beta API v2");
  54                         }
  55
  56                         oauth = generateOAuth(clientId, clientSecret);
  57
  58                         Instance.getConfig().setString(
  59                                         Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth);
  60                         Instance.getConfig().updateFile();
  61                 }
  62
  63                 this.oauth = oauth;
  64         }
  65
  66         @Override
  67         public String getOAuth() {
  68                 return oauth;
  69         }
  70
  71         @Override
  72         protected boolean isHtml() {
  73                 return true;
  74         }
  75
  76         @Override
  77         public String getSourceName() {
  78                 return "FimFiction.net";
  79         }
  80
  81         @Override
  82         protected void preprocess(URL source, InputStream in) throws IOException {
  83                 // extract the ID from:
  84                 // https://www.fimfiction.net/story/123456/name-of-story
  85                 storyId = getKeyText(source.toString(), "/story/", null, "/");
  86
  87                 // Selectors, so to download all I need and only what I need
  88                 String storyContent = "fields[story]=title,description,date_published,cover_image";
  89                 String authorContent = "fields[author]=name";
  90                 String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html";
  91                 String includes = "author,chapters,tags";
  92
  93                 String urlString = String.format(
  94                                 "https://www.fimfiction.net/api/v2/stories/%s?" //
  95                                                 + "%s&%s&%s&" //
  96                                                 + "include=%s", //
  97                                 storyId, //
  98                                 storyContent, authorContent, chapterContent,//
  99                                 includes);
 100
 101                 // URL params must be URL-encoded: "[ ]" <-> "%5B %5D"
 102                 urlString = urlString.replace("[", "%5B").replace("]", "%5D");
 103
 104                 URL url = new URL(urlString);
 105                 InputStream jsonIn = Instance.getCache().open(url, this, false);
 106                 try {
 107                         json = IOUtils.readSmallStream(jsonIn);
 108                 } finally {
 109                         jsonIn.close();
 110                 }
 111         }
 112
 113         @Override
 114         protected InputStream openInput(URL source) throws IOException {
 115                 return null;
 116         }
 117
 118         @Override
 119         protected MetaData getMeta(URL source, InputStream in) throws IOException {
 120                 MetaData meta = new MetaData();
 121
 122                 meta.setTitle(getKeyJson(json, 0, "type", "story", "title"));
 123                 meta.setAuthor(getKeyJson(json, 0, "type", "user", "name"));
 124                 meta.setDate(getKeyJson(json, 0, "type", "story", "date_published"));
 125                 meta.setTags(getTags());
 126                 meta.setSource(getSourceName());
 127                 meta.setUrl(source.toString());
 128                 meta.setPublisher(getSourceName());
 129                 meta.setUuid(source.toString());
 130                 meta.setLuid("");
 131                 meta.setLang("EN");
 132                 meta.setSubject("MLP");
 133                 meta.setType(getType().toString());
 134                 meta.setImageDocument(false);
 135
 136                 String coverImageLink =
 137                                 getKeyJson(json, 0, "type", "story", "cover_image", "full");
 138                 if (!coverImageLink.trim().isEmpty()) {
 139                         meta.setCover(getImage(this, null, coverImageLink.trim()));
 140                 }
 141
 142                 return meta;
 143         }
 144
 145         private List<String> getTags() {
 146                 List<String> tags = new ArrayList<String>();
 147                 tags.add("MLP");
 148
 149                 int pos = 0;
 150                 while (pos >= 0) {
 151                         pos = indexOfJsonAfter(json, pos, "type", "story_tag");
 152                         if (pos >= 0) {
 153                                 tags.add(getKeyJson(json, pos, "name").trim());
 154                         }
 155                 }
 156
 157                 return tags;
 158         }
 159
 160         @Override
 161         protected String getDesc(URL source, InputStream in) {
 162                 String desc = getKeyJson(json, 0, "type", "story", "description");
 163                 return unbbcode(desc);
 164         }
 165
 166         @Override
 167         protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
 168                         Progress pg) {
 169                 List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
 170
 171                 chapterNames = new HashMap<Integer, String>();
 172                 chapterContents = new HashMap<Integer, String>();
 173
 174                 int pos = 0;
 175                 while (pos >= 0) {
 176                         pos = indexOfJsonAfter(json, pos, "type", "chapter");
 177                         if (pos >= 0) {
 178                                 int posNumber = indexOfJsonAfter(json, pos, "chapter_number");
 179                                 int posComa = json.indexOf(",", posNumber);
 180                                 final int number = Integer.parseInt(json.substring(posNumber,
 181                                                 posComa).trim());
 182                                 final String title = getKeyJson(json, pos, "title");
 183                                 String notes = getKeyJson(json, pos, "authors_note_html");
 184                                 String content = getKeyJson(json, pos, "content_html");
 185
 186                                 if (!notes.trim().isEmpty()) {
 187                                         notes = "<br/>* * *<br/>" + notes;
 188                                 }
 189
 190                                 chapterNames.put(number, title);
 191                                 chapterContents
 192                                                 .put(number, content + notes);
 193
 194                                 urls.add(new Entry<String, URL>() {
 195                                         @Override
 196                                         public URL setValue(URL value) {
 197                                                 return null;
 198                                         }
 199
 200                                         @Override
 201                                         public String getKey() {
 202                                                 return title;
 203                                         }
 204
 205                                         @Override
 206                                         public URL getValue() {
 207                                                 return null;
 208                                         }
 209                                 });
 210                         }
 211                 }
 212
 213                 return urls;
 214         }
 215
 216         @Override
 217         protected String getChapterContent(URL source, InputStream in, int number,
 218                         Progress pg) {
 219                 return chapterContents.get(number);
 220         }
 221
 222         @Override
 223         protected boolean supports(URL url) {
 224                 return "fimfiction.net".equals(url.getHost())
 225                                 || "www.fimfiction.net".equals(url.getHost());
 226         }
 227
 228         /**
 229          * Generate a new token from the client ID and secret.
 230          * <p>
 231          * Note that those tokens are long-lived, and it would be badly seen to
 232          * create a lot of them without due cause.
 233          * <p>
 234          * So, please cache and re-use them.
 235          *
 236          * @param clientId
 237          *            the client ID offered on FimFiction
 238          * @param clientSecret
 239          *            the client secret that goes with it
 240          *
 241          * @return a new generated token linked to that client ID
 242          *
 243          * @throws IOException
 244          *             in case of I/O errors
 245          */
 246         static private String generateOAuth(String clientId, String clientSecret)
 247                         throws IOException {
 248                 URL url = new URL("https://www.fimfiction.net/api/v2/token");
 249                 Map<String, String> params = new HashMap<String, String>();
 250                 params.put("client_id", clientId);
 251                 params.put("client_secret", clientSecret);
 252                 params.put("grant_type", "client_credentials");
 253                 InputStream in = Instance.getCache().openNoCache(url, null, params,
 254                                 null, null);
 255
 256                 String jsonToken = IOUtils.readSmallStream(in);
 257
 258                 // Extract token type and token from: {
 259                 // token_type = "Bearer",
 260                 // access_token = "xxxxxxxxxxxxxx"
 261                 // }
 262
 263                 String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\"");
 264                 String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\"");
 265
 266                 return tokenType + " " + token;
 267         }
 268
 269         // afters: [name, value] pairs (or "" for any of them), can end without
 270         // value
 271         static private int indexOfJsonAfter(String json, int startAt,
 272                         String... afterKeys) {
 273                 ArrayList<String> afters = new ArrayList<String>();
 274                 boolean name = true;
 275                 for (String key : afterKeys) {
 276                         if (key != null && !key.isEmpty()) {
 277                                 afters.add("\"" + key + "\"");
 278                         } else {
 279                                 afters.add("\"");
 280                                 afters.add("\"");
 281                         }
 282
 283                         if (name) {
 284                                 afters.add(":");
 285                         }
 286
 287                         name = !name;
 288                 }
 289
 290                 return indexOfAfter(json, startAt, afters.toArray(new String[] {}));
 291         }
 292
 293         // afters: [name, value] pairs (or "" for any of them), can end without
 294         // value but will then be empty, not NULL
 295         static private String getKeyJson(String json, int startAt,
 296                         String... afterKeys) {
 297                 int pos = indexOfJsonAfter(json, startAt, afterKeys);
 298                 if (pos < 0) {
 299                         return "";
 300                 }
 301
 302                 String result = "";
 303                 String wip = json.substring(pos);
 304
 305                 pos = nextUnescapedQuote(wip, 0);
 306                 if (pos >= 0) {
 307                         wip = wip.substring(pos + 1);
 308                         pos = nextUnescapedQuote(wip, 0);
 309                         if (pos >= 0) {
 310                                 result = wip.substring(0, pos);
 311                         }
 312                 }
 313
 314                 result = result.replace("\\t", "\t")
 315                         .replace("\\\"", "\"");
 316
 317                 return result;
 318         }
 319
 320         // next " but don't take \" into account
 321         static private int nextUnescapedQuote(String result, int pos) {
 322                 while (pos >= 0) {
 323                         pos = result.indexOf("\"", pos);
 324                         if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) {
 325                                 break;
 326                         }
 327
 328                         if (pos < result.length()) {
 329                                 pos++;
 330                         }
 331                 }
 332
 333                 return pos;
 334         }
 335
 336         // quick & dirty filter
 337         static private String unbbcode(String bbcode) {
 338                 String text = bbcode.replace("\\r\\n", "<br/>") //
 339                                 .replace("[i]", "_").replace("[/i]", "_") //
 340                                 .replace("[b]", "*").replace("[/b]", "*") //
 341                                 .replaceAll("\\[[^\\]]*\\]", "");
 342                 return text;
 343         }
 344 }