1 package be
.nikiroo
.fanfix
.supported
;
3 import java
.io
.IOException
;
4 import java
.io
.UnsupportedEncodingException
;
5 import java
.net
.MalformedURLException
;
7 import java
.net
.URLDecoder
;
8 import java
.util
.AbstractMap
;
9 import java
.util
.ArrayList
;
10 import java
.util
.Date
;
11 import java
.util
.LinkedList
;
12 import java
.util
.List
;
13 import java
.util
.Map
.Entry
;
15 import org
.json
.JSONArray
;
16 import org
.json
.JSONException
;
17 import org
.json
.JSONObject
;
18 import org
.jsoup
.helper
.DataUtil
;
19 import org
.jsoup
.nodes
.Document
;
20 import org
.jsoup
.nodes
.Element
;
22 import be
.nikiroo
.fanfix
.Instance
;
23 import be
.nikiroo
.fanfix
.bundles
.Config
;
24 import be
.nikiroo
.fanfix
.data
.MetaData
;
25 import be
.nikiroo
.utils
.Image
;
26 import be
.nikiroo
.utils
.Progress
;
27 import be
.nikiroo
.utils
.StringUtils
;
28 import be
.nikiroo
.utils
.Version
;
31 * Support class for <a href="http://e621.net/">e621.net</a> and
32 * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
33 * including some of MLP.
35 * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
36 * comics, but it can be difficult to browse.
40 class E621
extends BasicSupport
{
42 protected boolean supports(URL url
) {
43 String host
= url
.getHost();
44 if (host
.startsWith("www.")) {
45 host
= host
.substring("www.".length());
48 return ("e621.net".equals(host
) || "e926.net".equals(host
))
49 && (isPool(url
) || isSearchOrSet(url
));
53 protected boolean isHtml() {
58 protected MetaData
getMeta() throws IOException
{
59 MetaData meta
= new MetaData();
61 meta
.setTitle(getTitle());
62 meta
.setAuthor(getAuthor());
63 meta
.setDate(bsHelper
.formatDate(getDate()));
64 meta
.setTags(getTags());
65 meta
.setSource(getType().getSourceName());
66 meta
.setUrl(getSource().toString());
67 meta
.setPublisher(getType().getSourceName());
68 meta
.setUuid(getSource().toString());
71 meta
.setSubject("Furry");
72 meta
.setType(getType().toString());
73 meta
.setImageDocument(true);
74 meta
.setCover(getCover());
75 meta
.setFakeCover(true);
81 protected String
getDesc() throws IOException
{
82 if (isSearchOrSet(getSource())) {
83 StringBuilder builder
= new StringBuilder();
84 builder
.append("<div>");
85 builder
.append("A collection of images from ")
86 .append(getSource().getHost()) //
88 .append(" Time of creation: "
89 + StringUtils
.fromTime(new Date().getTime()))
91 .append(" tTags: ");//
92 for (String tag
: getTags()) {
94 "\n<br/> ")
97 builder
.append("\n</div>");
99 return builder
.toString();
102 if (isPool(getSource())) {
103 Element el
= getSourceNode().getElementById("description");
113 protected List
<Entry
<String
, URL
>> getChapters(Progress pg
)
116 String jsonUrl
= getJsonUrl();
117 if (jsonUrl
!= null) {
118 for (i
= 1; true; i
++) {
121 // The API does not accept more than 2 request per sec,
122 // and asks us to limit at one per sec when possible
124 } catch (InterruptedException e
) {
129 JSONObject json
= getJson(jsonUrl
+ "&page=" + i
, false);
130 if (!json
.has("posts"))
132 JSONArray posts
= json
.getJSONArray("posts");
135 } catch (Exception e
) {
140 // The last page was empty:
144 // The pages and images are in reverse order on /posts/
145 List
<Entry
<String
, URL
>> chapters
= new LinkedList
<Entry
<String
, URL
>>();
146 for (int page
= i
; page
> 0; page
--) {
147 chapters
.add(new AbstractMap
.SimpleEntry
<String
, URL
>(
148 "Page " + Integer
.toString(i
- page
+ 1),
149 new URL(jsonUrl
+ "&page=" + page
)));
156 protected String
getChapterContent(URL chapUrl
, int number
, Progress pg
)
158 StringBuilder builder
= new StringBuilder();
160 JSONObject json
= getJson(chapUrl
, false);
161 JSONArray postsArr
= json
.getJSONArray("posts");
163 // The pages and images are in reverse order on /posts/
164 List
<JSONObject
> posts
= new ArrayList
<JSONObject
>(postsArr
.length());
165 for (int i
= postsArr
.length() - 1; i
>= 0; i
--) {
166 Object o
= postsArr
.get(i
);
167 if (o
instanceof JSONObject
)
168 posts
.add((JSONObject
) o
);
171 for (JSONObject post
: posts
) {
172 if (!post
.has("file"))
174 JSONObject file
= post
.getJSONObject("file");
175 if (!file
.has("url"))
179 String url
= file
.getString("url");
182 builder
.append("]<br/>");
183 } catch (JSONException e
) {
184 // Can be NULL if filtered
185 // When the value is NULL, we get an exception
186 // but the "has" method still returns true
187 Instance
.getInstance().getTraceHandler()
188 .error("Cannot get image for chapter " + number
+ " of "
193 return builder
.toString();
197 protected URL
getCanonicalUrl(URL source
) {
198 // Convert search-pools into proper pools
199 if (source
.getPath().equals("/posts") && source
.getQuery() != null
200 && source
.getQuery().startsWith("tags=pool%3A")) {
201 String poolNumber
= source
.getQuery()
202 .substring("tags=pool%3A".length());
204 Integer
.parseInt(poolNumber
);
205 String base
= source
.getProtocol() + "://" + source
.getHost();
206 if (source
.getPort() != -1) {
207 base
= base
+ ":" + source
.getPort();
209 source
= new URL(base
+ "/pools/" + poolNumber
);
210 } catch (NumberFormatException e
) {
211 // Not a simple pool, skip
212 } catch (MalformedURLException e
) {
217 if (isSetOriginalUrl(source
)) {
219 Document doc
= DataUtil
.load(Instance
.getInstance().getCache()
220 .open(source
, this, false), "UTF-8", source
.toString());
221 for (Element shortname
: doc
222 .getElementsByClass("set-shortname")) {
223 for (Element el
: shortname
.getElementsByTag("a")) {
224 if (!el
.attr("href").isEmpty())
225 return new URL(el
.absUrl("href"));
228 } catch (IOException e
) {
229 Instance
.getInstance().getTraceHandler().error(e
);
233 if (isPool(source
)) {
236 source
.toString().replace("/pool/show/", "/pools/"));
237 } catch (MalformedURLException e
) {
241 return super.getCanonicalUrl(source
);
244 private String
getTitle() {
247 Element el
= getSourceNode().getElementsByTag("title").first();
249 title
= el
.text().trim();
252 for (String s
: new String
[] { "e621", "-", "e621", "Pool", "-" }) {
253 if (title
.startsWith(s
)) {
254 title
= title
.substring(s
.length()).trim();
256 if (title
.endsWith(s
)) {
257 title
= title
.substring(0, title
.length() - s
.length()).trim();
261 if (isSearchOrSet(getSource())) {
262 title
= title
.isEmpty() ?
"e621" : "[e621] " + title
;
268 private String
getAuthor() {
269 List
<String
> list
= new ArrayList
<String
>();
270 String jsonUrl
= getJsonUrl();
271 if (jsonUrl
!= null) {
273 JSONObject json
= getJson(jsonUrl
, false);
274 JSONArray posts
= json
.getJSONArray("posts");
275 for (Object obj
: posts
) {
276 if (!(obj
instanceof JSONObject
))
279 JSONObject post
= (JSONObject
) obj
;
280 if (!post
.has("tags"))
283 JSONObject tags
= post
.getJSONObject("tags");
284 if (!tags
.has("artist"))
287 JSONArray artists
= tags
.getJSONArray("artist");
288 for (Object artist
: artists
) {
289 if (list
.contains(artist
.toString()))
292 list
.add(artist
.toString());
295 } catch (Exception e
) {
300 StringBuilder builder
= new StringBuilder();
301 for (String artist
: list
) {
302 if (builder
.length() > 0) {
303 builder
.append(", ");
305 builder
.append(artist
);
308 return builder
.toString();
311 private String
getDate() {
312 String jsonUrl
= getJsonUrl();
313 if (jsonUrl
!= null) {
315 JSONObject json
= getJson(jsonUrl
, false);
316 JSONArray posts
= json
.getJSONArray("posts");
317 for (Object obj
: posts
) {
318 if (!(obj
instanceof JSONObject
))
321 JSONObject post
= (JSONObject
) obj
;
322 if (!post
.has("created_at"))
325 return post
.getString("created_at");
327 } catch (Exception e
) {
336 private List
<String
> getTags() {
337 List
<String
> tags
= new ArrayList
<String
>();
338 if (isSearchOrSet(getSource())) {
339 String str
= getTagsFromUrl(getSource());
340 for (String tag
: str
.split("\\+")) {
342 tags
.add(URLDecoder
.decode(tag
.trim(), "UTF-8").trim());
343 } catch (UnsupportedEncodingException e
) {
351 // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
352 private String
getTagsFromUrl(URL url
) {
353 String tags
= url
== null ?
"" : url
.getQuery();
354 int pos
= tags
.indexOf("tags=");
357 tags
= tags
.substring(pos
).substring("tags=".length());
362 pos
= tags
.indexOf('&');
364 tags
= tags
.substring(0, pos
);
366 pos
= tags
.indexOf('/');
368 tags
= tags
.substring(0, pos
);
374 private Image
getCover() throws IOException
{
376 List
<Entry
<String
, URL
>> chapters
= getChapters(null);
377 if (!chapters
.isEmpty()) {
378 URL chap1Url
= chapters
.get(0).getValue();
379 String imgsChap1
= getChapterContent(chap1Url
, 1, null);
380 if (!imgsChap1
.isEmpty()) {
381 imgsChap1
= imgsChap1
.split("]")[0].substring(1).trim();
382 image
= bsImages
.getImage(this, new URL(imgsChap1
));
389 // always /posts.json/ url
390 private String
getJsonUrl() {
392 if (isSearchOrSet(getSource())) {
393 url
= getSource().toString().replace("/posts", "/posts.json");
396 if (isPool(getSource())) {
397 String poolNumber
= getSource().getPath()
398 .substring("/pools/".length());
399 url
= "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber
;
403 // Note: one way to override the blacklist
404 String login
= Instance
.getInstance().getConfig()
405 .getString(Config
.LOGIN_E621_LOGIN
);
406 String apk
= Instance
.getInstance().getConfig()
407 .getString(Config
.LOGIN_E621_APIKEY
);
409 if (login
!= null && !login
.isEmpty() && apk
!= null
411 url
= String
.format("%s&login=%s&api_key=%s&_client=%s", url
,
412 login
, apk
, "fanfix-" + Version
.getCurrentVersion());
419 // note: will be removed at getCanonicalUrl()
420 private boolean isSetOriginalUrl(URL originalUrl
) {
421 return originalUrl
.getPath().startsWith("/post_sets/");
424 private boolean isPool(URL url
) {
425 return url
.getPath().startsWith("/pools/")
426 || url
.getPath().startsWith("/pool/show/");
429 // set will be renamed into search by canonical url
430 private boolean isSearchOrSet(URL url
) {
433 (url
.getPath().equals("/posts") && url
.getQuery().contains("tags="))
435 || isSetOriginalUrl(url
);