Commit | Line | Data |
---|---|---|
08fe2e33 NR |
1 | package be.nikiroo.fanfix.supported; |
2 | ||
3 | import java.io.IOException; | |
b5e9855b | 4 | import java.io.UnsupportedEncodingException; |
c4b18c94 | 5 | import java.net.MalformedURLException; |
08fe2e33 | 6 | import java.net.URL; |
b5e9855b | 7 | import java.net.URLDecoder; |
ce297a79 | 8 | import java.util.AbstractMap; |
08fe2e33 | 9 | import java.util.ArrayList; |
8ac3d099 | 10 | import java.util.Date; |
b5e9855b | 11 | import java.util.LinkedList; |
08fe2e33 NR |
12 | import java.util.List; |
13 | import java.util.Map.Entry; | |
8ac3d099 | 14 | |
5cf61f35 NR |
15 | import org.json.JSONArray; |
16 | import org.json.JSONException; | |
17 | import org.json.JSONObject; | |
8ac3d099 NR |
18 | import org.jsoup.helper.DataUtil; |
19 | import org.jsoup.nodes.Document; | |
20 | import org.jsoup.nodes.Element; | |
08fe2e33 NR |
21 | |
22 | import be.nikiroo.fanfix.Instance; | |
5cf61f35 | 23 | import be.nikiroo.fanfix.bundles.Config; |
68686a37 | 24 | import be.nikiroo.fanfix.data.MetaData; |
16a81ef7 | 25 | import be.nikiroo.utils.Image; |
3b2b638f | 26 | import be.nikiroo.utils.Progress; |
08fe2e33 | 27 | import be.nikiroo.utils.StringUtils; |
5cf61f35 | 28 | import be.nikiroo.utils.Version; |
08fe2e33 NR |
29 | |
30 | /** | |
8ac3d099 NR |
31 | * Support class for <a href="http://e621.net/">e621.net</a> and |
32 | * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics, | |
08fe2e33 NR |
33 | * including some of MLP. |
34 | * <p> | |
35 | * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and | |
36 | * comics, but it can be difficult to browse. | |
37 | * | |
38 | * @author niki | |
39 | */ | |
8ac3d099 | 40 | class E621 extends BasicSupport { |
08fe2e33 NR |
41 | @Override |
42 | protected boolean supports(URL url) { | |
43 | String host = url.getHost(); | |
44 | if (host.startsWith("www.")) { | |
45 | host = host.substring("www.".length()); | |
46 | } | |
47 | ||
5cf61f35 NR |
48 | return ("e621.net".equals(host) || "e926.net".equals(host)) |
49 | && (isPool(url) || isSearchOrSet(url)); | |
08fe2e33 NR |
50 | } |
51 | ||
52 | @Override | |
53 | protected boolean isHtml() { | |
54 | return true; | |
55 | } | |
56 | ||
8ac3d099 NR |
57 | @Override |
58 | protected MetaData getMeta() throws IOException { | |
59 | MetaData meta = new MetaData(); | |
b5e9855b | 60 | |
8ac3d099 NR |
61 | meta.setTitle(getTitle()); |
62 | meta.setAuthor(getAuthor()); | |
bff19b54 | 63 | meta.setDate(bsHelper.formatDate(getDate())); |
8ac3d099 NR |
64 | meta.setTags(getTags()); |
65 | meta.setSource(getType().getSourceName()); | |
66 | meta.setUrl(getSource().toString()); | |
67 | meta.setPublisher(getType().getSourceName()); | |
68 | meta.setUuid(getSource().toString()); | |
69 | meta.setLuid(""); | |
70 | meta.setLang("en"); | |
71 | meta.setSubject("Furry"); | |
72 | meta.setType(getType().toString()); | |
73 | meta.setImageDocument(true); | |
74 | meta.setCover(getCover()); | |
75 | meta.setFakeCover(true); | |
595dfa7a | 76 | |
8ac3d099 | 77 | return meta; |
595dfa7a NR |
78 | } |
79 | ||
8ac3d099 NR |
80 | @Override |
81 | protected String getDesc() throws IOException { | |
82 | if (isSearchOrSet(getSource())) { | |
b5e9855b | 83 | StringBuilder builder = new StringBuilder(); |
a8a7222f | 84 | builder.append("<div>"); |
5cf61f35 | 85 | builder.append("A collection of images from ") |
a8a7222f NR |
86 | .append(getSource().getHost()) // |
87 | .append("<br/>\n") // | |
88 | .append(" Time of creation: " | |
5cf61f35 | 89 | + StringUtils.fromTime(new Date().getTime())) |
a8a7222f NR |
90 | .append("<br/>\n") // |
91 | .append(" tTags: ");// | |
8ac3d099 | 92 | for (String tag : getTags()) { |
a8a7222f NR |
93 | builder.append( |
94 | "\n<br/> ") | |
95 | .append(tag); | |
b5e9855b | 96 | } |
a8a7222f | 97 | builder.append("\n</div>"); |
b5e9855b NR |
98 | |
99 | return builder.toString(); | |
100 | } | |
101 | ||
8ac3d099 NR |
102 | if (isPool(getSource())) { |
103 | Element el = getSourceNode().getElementById("description"); | |
104 | if (el != null) { | |
a8a7222f | 105 | return el.html(); |
08fe2e33 NR |
106 | } |
107 | } | |
108 | ||
109 | return null; | |
110 | } | |
111 | ||
08fe2e33 | 112 | @Override |
5cf61f35 | 113 | protected List<Entry<String, URL>> getChapters(Progress pg) |
8ac3d099 | 114 | throws IOException { |
5cf61f35 NR |
115 | int i = 1; |
116 | String jsonUrl = getJsonUrl(); | |
117 | if (jsonUrl != null) { | |
118 | for (i = 1; true; i++) { | |
119 | if (i > 1) { | |
120 | try { | |
121 | // The API does not accept more than 2 request per sec, | |
122 | // and asks us to limit at one per sec when possible | |
123 | Thread.sleep(1000); | |
124 | } catch (InterruptedException e) { | |
125 | } | |
126 | } | |
b5e9855b | 127 | |
b5e9855b | 128 | try { |
5cf61f35 NR |
129 | JSONObject json = getJson(jsonUrl + "&page=" + i, false); |
130 | if (!json.has("posts")) | |
b5e9855b | 131 | break; |
5cf61f35 NR |
132 | JSONArray posts = json.getJSONArray("posts"); |
133 | if (posts.isEmpty()) | |
134 | break; | |
135 | } catch (Exception e) { | |
136 | e.printStackTrace(); | |
b5e9855b | 137 | } |
b5e9855b | 138 | } |
5cf61f35 NR |
139 | |
140 | // The last page was empty: | |
141 | i--; | |
142 | } | |
143 | ||
144 | // The pages and images are in reverse order on /posts/ | |
145 | List<Entry<String, URL>> chapters = new LinkedList<Entry<String, URL>>(); | |
146 | for (int page = i; page > 0; page--) { | |
147 | chapters.add(new AbstractMap.SimpleEntry<String, URL>( | |
148 | "Page " + Integer.toString(i - page + 1), | |
149 | new URL(jsonUrl + "&page=" + page))); | |
b5e9855b NR |
150 | } |
151 | ||
5cf61f35 | 152 | return chapters; |
b5e9855b NR |
153 | } |
154 | ||
8ac3d099 | 155 | @Override |
5cf61f35 NR |
156 | protected String getChapterContent(URL chapUrl, int number, Progress pg) |
157 | throws IOException { | |
8ac3d099 | 158 | StringBuilder builder = new StringBuilder(); |
5cf61f35 NR |
159 | |
160 | JSONObject json = getJson(chapUrl, false); | |
161 | JSONArray postsArr = json.getJSONArray("posts"); | |
162 | ||
163 | // The pages and images are in reverse order on /posts/ | |
164 | List<JSONObject> posts = new ArrayList<JSONObject>(postsArr.length()); | |
165 | for (int i = postsArr.length() - 1; i >= 0; i--) { | |
166 | Object o = postsArr.get(i); | |
167 | if (o instanceof JSONObject) | |
168 | posts.add((JSONObject) o); | |
75002fcc | 169 | } |
5cf61f35 NR |
170 | |
171 | for (JSONObject post : posts) { | |
172 | if (!post.has("file")) | |
173 | continue; | |
174 | JSONObject file = post.getJSONObject("file"); | |
175 | if (!file.has("url")) | |
176 | continue; | |
177 | ||
178 | try { | |
179 | String url = file.getString("url"); | |
180 | builder.append("["); | |
181 | builder.append(url); | |
182 | builder.append("]<br/>"); | |
183 | } catch (JSONException e) { | |
184 | // Can be NULL if filtered | |
185 | // When the value is NULL, we get an exception | |
186 | // but the "has" method still returns true | |
a3d0728c NR |
187 | Instance.getInstance().getTraceHandler() |
188 | .error("Cannot get image for chapter " + number + " of " | |
189 | + getSource()); | |
5cf61f35 | 190 | } |
8ac3d099 NR |
191 | } |
192 | ||
193 | return builder.toString(); | |
194 | } | |
195 | ||
196 | @Override | |
197 | protected URL getCanonicalUrl(URL source) { | |
8fbfa934 NR |
198 | // Convert search-pools into proper pools |
199 | if (source.getPath().equals("/posts") && source.getQuery() != null | |
200 | && source.getQuery().startsWith("tags=pool%3A")) { | |
201 | String poolNumber = source.getQuery() | |
202 | .substring("tags=pool%3A".length()); | |
203 | try { | |
204 | Integer.parseInt(poolNumber); | |
205 | String base = source.getProtocol() + "://" + source.getHost(); | |
206 | if (source.getPort() != -1) { | |
207 | base = base + ":" + source.getPort(); | |
208 | } | |
42cdf6f0 | 209 | source = new URL(base + "/pools/" + poolNumber); |
8fbfa934 | 210 | } catch (NumberFormatException e) { |
36c35b92 | 211 | // Not a simple pool, skip |
8fbfa934 NR |
212 | } catch (MalformedURLException e) { |
213 | // Cannot happen | |
214 | } | |
215 | } | |
5cf61f35 | 216 | |
8ac3d099 NR |
217 | if (isSetOriginalUrl(source)) { |
218 | try { | |
5cf61f35 NR |
219 | Document doc = DataUtil.load(Instance.getInstance().getCache() |
220 | .open(source, this, false), "UTF-8", source.toString()); | |
221 | for (Element shortname : doc | |
222 | .getElementsByClass("set-shortname")) { | |
8ac3d099 NR |
223 | for (Element el : shortname.getElementsByTag("a")) { |
224 | if (!el.attr("href").isEmpty()) | |
225 | return new URL(el.absUrl("href")); | |
08fe2e33 NR |
226 | } |
227 | } | |
8ac3d099 | 228 | } catch (IOException e) { |
d66deb8d | 229 | Instance.getInstance().getTraceHandler().error(e); |
08fe2e33 NR |
230 | } |
231 | } | |
232 | ||
c4b18c94 NR |
233 | if (isPool(source)) { |
234 | try { | |
5cf61f35 NR |
235 | return new URL( |
236 | source.toString().replace("/pool/show/", "/pools/")); | |
c4b18c94 NR |
237 | } catch (MalformedURLException e) { |
238 | } | |
239 | } | |
240 | ||
8ac3d099 NR |
241 | return super.getCanonicalUrl(source); |
242 | } | |
243 | ||
8ac3d099 NR |
244 | private String getTitle() { |
245 | String title = ""; | |
246 | ||
247 | Element el = getSourceNode().getElementsByTag("title").first(); | |
248 | if (el != null) { | |
249 | title = el.text().trim(); | |
08fe2e33 NR |
250 | } |
251 | ||
36c35b92 | 252 | for (String s : new String[] { "e621", "-", "e621", "Pool", "-" }) { |
8ac3d099 NR |
253 | if (title.startsWith(s)) { |
254 | title = title.substring(s.length()).trim(); | |
08fe2e33 | 255 | } |
8ac3d099 NR |
256 | if (title.endsWith(s)) { |
257 | title = title.substring(0, title.length() - s.length()).trim(); | |
258 | } | |
08fe2e33 NR |
259 | } |
260 | ||
8ac3d099 NR |
261 | if (isSearchOrSet(getSource())) { |
262 | title = title.isEmpty() ? "e621" : "[e621] " + title; | |
263 | } | |
5cf61f35 | 264 | |
8ac3d099 | 265 | return title; |
08fe2e33 | 266 | } |
b5e9855b | 267 | |
5cf61f35 NR |
268 | private String getAuthor() { |
269 | List<String> list = new ArrayList<String>(); | |
270 | String jsonUrl = getJsonUrl(); | |
271 | if (jsonUrl != null) { | |
272 | try { | |
273 | JSONObject json = getJson(jsonUrl, false); | |
274 | JSONArray posts = json.getJSONArray("posts"); | |
275 | for (Object obj : posts) { | |
276 | if (!(obj instanceof JSONObject)) | |
277 | continue; | |
278 | ||
279 | JSONObject post = (JSONObject) obj; | |
280 | if (!post.has("tags")) | |
281 | continue; | |
282 | ||
283 | JSONObject tags = post.getJSONObject("tags"); | |
284 | if (!tags.has("artist")) | |
285 | continue; | |
286 | ||
287 | JSONArray artists = tags.getJSONArray("artist"); | |
288 | for (Object artist : artists) { | |
289 | if (list.contains(artist.toString())) | |
290 | continue; | |
291 | ||
292 | list.add(artist.toString()); | |
8ac3d099 | 293 | } |
9948521d | 294 | } |
5cf61f35 NR |
295 | } catch (Exception e) { |
296 | e.printStackTrace(); | |
8ac3d099 NR |
297 | } |
298 | } | |
299 | ||
5cf61f35 NR |
300 | StringBuilder builder = new StringBuilder(); |
301 | for (String artist : list) { | |
302 | if (builder.length() > 0) { | |
303 | builder.append(", "); | |
304 | } | |
305 | builder.append(artist); | |
306 | } | |
8ac3d099 | 307 | |
5cf61f35 NR |
308 | return builder.toString(); |
309 | } | |
8ac3d099 | 310 | |
5cf61f35 NR |
311 | private String getDate() { |
312 | String jsonUrl = getJsonUrl(); | |
313 | if (jsonUrl != null) { | |
314 | try { | |
315 | JSONObject json = getJson(jsonUrl, false); | |
316 | JSONArray posts = json.getJSONArray("posts"); | |
317 | for (Object obj : posts) { | |
318 | if (!(obj instanceof JSONObject)) | |
319 | continue; | |
8d1a4fd2 | 320 | |
5cf61f35 NR |
321 | JSONObject post = (JSONObject) obj; |
322 | if (!post.has("created_at")) | |
323 | continue; | |
324 | ||
325 | return post.getString("created_at"); | |
8d1a4fd2 | 326 | } |
5cf61f35 NR |
327 | } catch (Exception e) { |
328 | e.printStackTrace(); | |
8d1a4fd2 | 329 | } |
8ac3d099 | 330 | } |
9948521d | 331 | |
5cf61f35 | 332 | return ""; |
8ac3d099 NR |
333 | } |
334 | ||
335 | // no tags for pools | |
336 | private List<String> getTags() { | |
337 | List<String> tags = new ArrayList<String>(); | |
338 | if (isSearchOrSet(getSource())) { | |
339 | String str = getTagsFromUrl(getSource()); | |
340 | for (String tag : str.split("\\+")) { | |
9b863b20 | 341 | try { |
8ac3d099 NR |
342 | tags.add(URLDecoder.decode(tag.trim(), "UTF-8").trim()); |
343 | } catch (UnsupportedEncodingException e) { | |
9b863b20 NR |
344 | } |
345 | } | |
346 | } | |
9948521d | 347 | |
8ac3d099 NR |
348 | return tags; |
349 | } | |
350 | ||
5cf61f35 NR |
351 | // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query |
352 | private String getTagsFromUrl(URL url) { | |
353 | String tags = url == null ? "" : url.getQuery(); | |
354 | int pos = tags.indexOf("tags="); | |
355 | ||
356 | if (pos >= 0) { | |
357 | tags = tags.substring(pos).substring("tags=".length()); | |
358 | } else { | |
359 | return ""; | |
360 | } | |
361 | ||
362 | pos = tags.indexOf('&'); | |
363 | if (pos > 0) { | |
364 | tags = tags.substring(0, pos); | |
365 | } | |
366 | pos = tags.indexOf('/'); | |
367 | if (pos > 0) { | |
368 | tags = tags.substring(0, pos); | |
369 | } | |
370 | ||
371 | return tags; | |
372 | } | |
373 | ||
8ac3d099 NR |
374 | private Image getCover() throws IOException { |
375 | Image image = null; | |
376 | List<Entry<String, URL>> chapters = getChapters(null); | |
377 | if (!chapters.isEmpty()) { | |
12c180fc NR |
378 | URL chap1Url = chapters.get(0).getValue(); |
379 | String imgsChap1 = getChapterContent(chap1Url, 1, null); | |
380 | if (!imgsChap1.isEmpty()) { | |
381 | imgsChap1 = imgsChap1.split("]")[0].substring(1).trim(); | |
382 | image = bsImages.getImage(this, new URL(imgsChap1)); | |
383 | } | |
8ac3d099 NR |
384 | } |
385 | ||
386 | return image; | |
387 | } | |
388 | ||
5cf61f35 NR |
389 | // always /posts.json/ url |
390 | private String getJsonUrl() { | |
391 | String url = null; | |
392 | if (isSearchOrSet(getSource())) { | |
393 | url = getSource().toString().replace("/posts", "/posts.json"); | |
394 | } | |
395 | ||
396 | if (isPool(getSource())) { | |
397 | String poolNumber = getSource().getPath() | |
398 | .substring("/pools/".length()); | |
399 | url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber; | |
400 | } | |
401 | ||
402 | if (url != null) { | |
403 | // Note: one way to override the blacklist | |
404 | String login = Instance.getInstance().getConfig() | |
405 | .getString(Config.LOGIN_E621_LOGIN); | |
406 | String apk = Instance.getInstance().getConfig() | |
407 | .getString(Config.LOGIN_E621_APIKEY); | |
408 | ||
409 | if (login != null && !login.isEmpty() && apk != null | |
410 | && !apk.isEmpty()) { | |
411 | url = String.format("%s&login=%s&api_key=%s&_client=%s", url, | |
412 | login, apk, "fanfix-" + Version.getCurrentVersion()); | |
413 | } | |
414 | } | |
415 | ||
416 | return url; | |
417 | } | |
418 | ||
8ac3d099 NR |
419 | // note: will be removed at getCanonicalUrl() |
420 | private boolean isSetOriginalUrl(URL originalUrl) { | |
421 | return originalUrl.getPath().startsWith("/post_sets/"); | |
9b863b20 NR |
422 | } |
423 | ||
b5e9855b | 424 | private boolean isPool(URL url) { |
5cf61f35 NR |
425 | return url.getPath().startsWith("/pools/") |
426 | || url.getPath().startsWith("/pool/show/"); | |
b5e9855b NR |
427 | } |
428 | ||
8ac3d099 NR |
429 | // set will be renamed into search by canonical url |
430 | private boolean isSearchOrSet(URL url) { | |
431 | return | |
432 | // search: | |
433 | (url.getPath().equals("/posts") && url.getQuery().contains("tags=")) | |
434 | // or set: | |
435 | || isSetOriginalUrl(url); | |
b5e9855b | 436 | } |
08fe2e33 | 437 | } |