| 1 | package be.nikiroo.fanfix.supported; |
| 2 | |
| 3 | import java.io.IOException; |
| 4 | import java.io.InputStream; |
| 5 | import java.net.URL; |
| 6 | import java.util.ArrayList; |
| 7 | import java.util.HashMap; |
| 8 | import java.util.List; |
| 9 | import java.util.Map; |
| 10 | import java.util.Map.Entry; |
| 11 | |
| 12 | import be.nikiroo.fanfix.Instance; |
| 13 | import be.nikiroo.fanfix.bundles.Config; |
| 14 | import be.nikiroo.fanfix.data.MetaData; |
| 15 | import be.nikiroo.utils.IOUtils; |
| 16 | import be.nikiroo.utils.Progress; |
| 17 | |
| 18 | /** |
| 19 | * Support class for <a href="http://www.fimfiction.net/">FimFiction.net</a> |
| 20 | * stories, a website dedicated to My Little Pony. |
| 21 | * <p> |
| 22 | * This version uses the new, official API of FimFiction. |
| 23 | * |
| 24 | * @author niki |
| 25 | */ |
| 26 | class FimfictionApi extends BasicSupport { |
| 27 | private String oauth; |
| 28 | private String storyId; |
| 29 | private String json; |
| 30 | |
| 31 | private Map<Integer, String> chapterNames; |
| 32 | private Map<Integer, String> chapterContents; |
| 33 | |
| 34 | public FimfictionApi() throws IOException { |
| 35 | if (Instance.getConfig().getBoolean( |
| 36 | Config.LOGIN_FIMFICTION_APIKEY_FORCE_HTML, false)) { |
| 37 | throw new IOException( |
| 38 | "Configuration is set to force HTML scrapping"); |
| 39 | } |
| 40 | |
| 41 | String oauth = Instance.getConfig().getString( |
| 42 | Config.LOGIN_FIMFICTION_APIKEY_TOKEN); |
| 43 | |
| 44 | if (oauth == null || oauth.isEmpty()) { |
| 45 | String clientId = Instance.getConfig().getString( |
| 46 | Config.LOGIN_FIMFICTION_APIKEY_CLIENT_ID) |
| 47 | + ""; |
| 48 | String clientSecret = Instance.getConfig().getString( |
| 49 | Config.LOGIN_FIMFICTION_APIKEY_CLIENT_SECRET) |
| 50 | + ""; |
| 51 | |
| 52 | if (clientId.trim().isEmpty() || clientSecret.trim().isEmpty()) { |
| 53 | throw new IOException("API key required for the beta API v2"); |
| 54 | } |
| 55 | |
| 56 | oauth = generateOAuth(clientId, clientSecret); |
| 57 | |
| 58 | Instance.getConfig().setString( |
| 59 | Config.LOGIN_FIMFICTION_APIKEY_TOKEN, oauth); |
| 60 | Instance.getConfig().updateFile(); |
| 61 | } |
| 62 | |
| 63 | this.oauth = oauth; |
| 64 | } |
| 65 | |
| 66 | @Override |
| 67 | public String getOAuth() { |
| 68 | return oauth; |
| 69 | } |
| 70 | |
| 71 | @Override |
| 72 | protected boolean isHtml() { |
| 73 | return true; |
| 74 | } |
| 75 | |
| 76 | @Override |
| 77 | public String getSourceName() { |
| 78 | return "FimFiction.net"; |
| 79 | } |
| 80 | |
| 81 | @Override |
| 82 | protected void preprocess(URL source, InputStream in) throws IOException { |
| 83 | // extract the ID from: |
| 84 | // https://www.fimfiction.net/story/123456/name-of-story |
| 85 | storyId = getKeyText(source.toString(), "/story/", null, "/"); |
| 86 | |
| 87 | // Selectors, so to download all I need and only what I need |
| 88 | String storyContent = "fields[story]=title,description,date_published,cover_image"; |
| 89 | String authorContent = "fields[author]=name"; |
| 90 | String chapterContent = "fields[chapter]=chapter_number,title,content_html,authors_note_html"; |
| 91 | String includes = "author,chapters,tags"; |
| 92 | |
| 93 | String urlString = String.format( |
| 94 | "https://www.fimfiction.net/api/v2/stories/%s?" // |
| 95 | + "%s&%s&%s&" // |
| 96 | + "include=%s", // |
| 97 | storyId, // |
| 98 | storyContent, authorContent, chapterContent,// |
| 99 | includes); |
| 100 | |
| 101 | // URL params must be URL-encoded: "[ ]" <-> "%5B %5D" |
| 102 | urlString = urlString.replace("[", "%5B").replace("]", "%5D"); |
| 103 | |
| 104 | URL url = new URL(urlString); |
| 105 | InputStream jsonIn = Instance.getCache().open(url, this, false); |
| 106 | try { |
| 107 | json = IOUtils.readSmallStream(jsonIn); |
| 108 | } finally { |
| 109 | jsonIn.close(); |
| 110 | } |
| 111 | } |
| 112 | |
| 113 | @Override |
| 114 | protected InputStream openInput(URL source) throws IOException { |
| 115 | return null; |
| 116 | } |
| 117 | |
| 118 | @Override |
| 119 | protected MetaData getMeta(URL source, InputStream in) throws IOException { |
| 120 | MetaData meta = new MetaData(); |
| 121 | |
| 122 | meta.setTitle(getKeyJson(json, 0, "type", "story", "title")); |
| 123 | meta.setAuthor(getKeyJson(json, 0, "type", "user", "name")); |
| 124 | meta.setDate(getKeyJson(json, 0, "type", "story", "date_published")); |
| 125 | meta.setTags(getTags()); |
| 126 | meta.setSource(getSourceName()); |
| 127 | meta.setUrl(source.toString()); |
| 128 | meta.setPublisher(getSourceName()); |
| 129 | meta.setUuid(source.toString()); |
| 130 | meta.setLuid(""); |
| 131 | meta.setLang("EN"); |
| 132 | meta.setSubject("MLP"); |
| 133 | meta.setType(getType().toString()); |
| 134 | meta.setImageDocument(false); |
| 135 | meta.setCover(getImage(this, null, |
| 136 | getKeyJson(json, 0, "type", "story", "cover_image", "full"))); |
| 137 | |
| 138 | return meta; |
| 139 | } |
| 140 | |
| 141 | private List<String> getTags() { |
| 142 | List<String> tags = new ArrayList<String>(); |
| 143 | tags.add("MLP"); |
| 144 | |
| 145 | int pos = 0; |
| 146 | while (pos >= 0) { |
| 147 | pos = indexOfJsonAfter(json, pos, "type", "story_tag"); |
| 148 | if (pos >= 0) { |
| 149 | tags.add(getKeyJson(json, pos, "name")); |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | return tags; |
| 154 | } |
| 155 | |
| 156 | @Override |
| 157 | protected String getDesc(URL source, InputStream in) { |
| 158 | String desc = getKeyJson(json, 0, "type", "story", "description"); |
| 159 | return unbbcode(desc); |
| 160 | } |
| 161 | |
| 162 | @Override |
| 163 | protected List<Entry<String, URL>> getChapters(URL source, InputStream in, |
| 164 | Progress pg) { |
| 165 | List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>(); |
| 166 | |
| 167 | chapterNames = new HashMap<Integer, String>(); |
| 168 | chapterContents = new HashMap<Integer, String>(); |
| 169 | |
| 170 | int pos = 0; |
| 171 | while (pos >= 0) { |
| 172 | pos = indexOfJsonAfter(json, pos, "type", "chapter"); |
| 173 | if (pos >= 0) { |
| 174 | int posNumber = indexOfJsonAfter(json, pos, "chapter_number"); |
| 175 | int posComa = json.indexOf(",", posNumber); |
| 176 | final int number = Integer.parseInt(json.substring(posNumber, |
| 177 | posComa).trim()); |
| 178 | final String title = getKeyJson(json, pos, "title"); |
| 179 | String notes = getKeyJson(json, pos, "authors_note_html"); |
| 180 | String content = getKeyJson(json, pos, "content_html"); |
| 181 | |
| 182 | chapterNames.put(number, title); |
| 183 | chapterContents |
| 184 | .put(number, content + "<br/>* * *<br/>" + notes); |
| 185 | |
| 186 | urls.add(new Entry<String, URL>() { |
| 187 | @Override |
| 188 | public URL setValue(URL value) { |
| 189 | return null; |
| 190 | } |
| 191 | |
| 192 | @Override |
| 193 | public String getKey() { |
| 194 | return title; |
| 195 | } |
| 196 | |
| 197 | @Override |
| 198 | public URL getValue() { |
| 199 | return null; |
| 200 | } |
| 201 | }); |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | return urls; |
| 206 | } |
| 207 | |
| 208 | @Override |
| 209 | protected String getChapterContent(URL source, InputStream in, int number, |
| 210 | Progress pg) { |
| 211 | return chapterContents.get(number); |
| 212 | } |
| 213 | |
| 214 | @Override |
| 215 | protected boolean supports(URL url) { |
| 216 | return "fimfiction.net".equals(url.getHost()) |
| 217 | || "www.fimfiction.net".equals(url.getHost()); |
| 218 | } |
| 219 | |
| 220 | /** |
| 221 | * Generate a new token from the client ID and secret. |
| 222 | * <p> |
| 223 | * Note that those tokens are long-lived, and it would be badly seen to |
| 224 | * create a lot of them without due cause. |
| 225 | * <p> |
| 226 | * So, please cache and re-use them. |
| 227 | * |
| 228 | * @param clientId |
| 229 | * the client ID offered on FimFiction |
| 230 | * @param clientSecret |
| 231 | * the client secret that goes with it |
| 232 | * |
| 233 | * @return a new generated token linked to that client ID |
| 234 | * |
| 235 | * @throws IOException |
| 236 | * in case of I/O errors |
| 237 | */ |
| 238 | static private String generateOAuth(String clientId, String clientSecret) |
| 239 | throws IOException { |
| 240 | URL url = new URL("https://www.fimfiction.net/api/v2/token"); |
| 241 | Map<String, String> params = new HashMap<String, String>(); |
| 242 | params.put("client_id", clientId); |
| 243 | params.put("client_secret", clientSecret); |
| 244 | params.put("grant_type", "client_credentials"); |
| 245 | InputStream in = Instance.getCache().openNoCache(url, null, params, |
| 246 | null, null); |
| 247 | |
| 248 | String jsonToken = IOUtils.readSmallStream(in); |
| 249 | |
| 250 | // Extract token type and token from: { |
| 251 | // token_type = "Bearer", |
| 252 | // access_token = "xxxxxxxxxxxxxx" |
| 253 | // } |
| 254 | |
| 255 | String token = getKeyText(jsonToken, "\"access_token\"", "\"", "\""); |
| 256 | String tokenType = getKeyText(jsonToken, "\"token_type\"", "\"", "\""); |
| 257 | |
| 258 | return tokenType + " " + token; |
| 259 | } |
| 260 | |
| 261 | // afters: [name, value] pairs (or "" for any of them), can end without |
| 262 | // value |
| 263 | static private int indexOfJsonAfter(String json, int startAt, |
| 264 | String... afterKeys) { |
| 265 | ArrayList<String> afters = new ArrayList<String>(); |
| 266 | boolean name = true; |
| 267 | for (String key : afterKeys) { |
| 268 | if (key != null && !key.isEmpty()) { |
| 269 | afters.add("\"" + key + "\""); |
| 270 | } else { |
| 271 | afters.add("\""); |
| 272 | afters.add("\""); |
| 273 | } |
| 274 | |
| 275 | if (name) { |
| 276 | afters.add(":"); |
| 277 | } |
| 278 | |
| 279 | name = !name; |
| 280 | } |
| 281 | |
| 282 | return indexOfAfter(json, startAt, afters.toArray(new String[] {})); |
| 283 | } |
| 284 | |
| 285 | // afters: [name, value] pairs (or "" for any of them), can end without |
| 286 | // value |
| 287 | static private String getKeyJson(String json, int startAt, |
| 288 | String... afterKeys) { |
| 289 | int pos = indexOfJsonAfter(json, startAt, afterKeys); |
| 290 | if (pos < 0) { |
| 291 | return null; |
| 292 | } |
| 293 | |
| 294 | String result = null; |
| 295 | String wip = json.substring(pos); |
| 296 | |
| 297 | pos = nextUnescapedQuote(wip, 0); |
| 298 | if (pos >= 0) { |
| 299 | wip = wip.substring(pos + 1); |
| 300 | pos = nextUnescapedQuote(wip, 0); |
| 301 | if (pos >= 0) { |
| 302 | result = wip.substring(0, pos); |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | return result; |
| 307 | } |
| 308 | |
| 309 | // next " but don't take \" into account |
| 310 | static private int nextUnescapedQuote(String result, int pos) { |
| 311 | while (pos >= 0) { |
| 312 | pos = result.indexOf("\"", pos); |
| 313 | if (pos == 0 || (pos > 0 && result.charAt(pos - 1) != '\\')) { |
| 314 | break; |
| 315 | } |
| 316 | |
| 317 | if (pos < result.length()) { |
| 318 | pos++; |
| 319 | } |
| 320 | } |
| 321 | |
| 322 | return pos; |
| 323 | } |
| 324 | |
| 325 | // quick & dirty filter |
| 326 | static private String unbbcode(String bbcode) { |
| 327 | String text = bbcode.replace("\\r\\n", "<br/>") // |
| 328 | .replace("[i]", "_").replace("[/i]", "_") // |
| 329 | .replace("[b]", "*").replace("[/b]", "*") // |
| 330 | .replaceAll("\\[[^\\]]*\\]", ""); |
| 331 | return text; |
| 332 | } |
| 333 | } |