1 package be
.nikiroo
.fanfix
.supported
;
3 import java
.io
.IOException
;
4 import java
.io
.UnsupportedEncodingException
;
5 import java
.net
.MalformedURLException
;
7 import java
.net
.URLDecoder
;
8 import java
.util
.AbstractMap
;
9 import java
.util
.ArrayList
;
10 import java
.util
.Collections
;
11 import java
.util
.Date
;
12 import java
.util
.LinkedList
;
13 import java
.util
.List
;
14 import java
.util
.Map
.Entry
;
16 import org
.json
.JSONArray
;
17 import org
.json
.JSONException
;
18 import org
.json
.JSONObject
;
19 import org
.jsoup
.helper
.DataUtil
;
20 import org
.jsoup
.nodes
.Document
;
21 import org
.jsoup
.nodes
.Element
;
23 import be
.nikiroo
.fanfix
.Instance
;
24 import be
.nikiroo
.fanfix
.bundles
.Config
;
25 import be
.nikiroo
.fanfix
.data
.MetaData
;
26 import be
.nikiroo
.utils
.Image
;
27 import be
.nikiroo
.utils
.Progress
;
28 import be
.nikiroo
.utils
.StringUtils
;
29 import be
.nikiroo
.utils
.Version
;
32 * Support class for <a href="http://e621.net/">e621.net</a> and
33 * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
34 * including some of MLP.
36 * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
37 * comics, but it can be difficult to browse.
41 class E621
extends BasicSupport
{
43 protected boolean supports(URL url
) {
44 String host
= url
.getHost();
45 if (host
.startsWith("www.")) {
46 host
= host
.substring("www.".length());
49 return ("e621.net".equals(host
) || "e926.net".equals(host
))
50 && (isPool(url
) || isSearchOrSet(url
));
54 protected boolean isHtml() {
59 protected MetaData
getMeta() throws IOException
{
60 MetaData meta
= new MetaData();
62 meta
.setTitle(getTitle());
63 meta
.setAuthor(getAuthor());
64 meta
.setDate(getDate());
65 meta
.setTags(getTags());
66 meta
.setSource(getType().getSourceName());
67 meta
.setUrl(getSource().toString());
68 meta
.setPublisher(getType().getSourceName());
69 meta
.setUuid(getSource().toString());
72 meta
.setSubject("Furry");
73 meta
.setType(getType().toString());
74 meta
.setImageDocument(true);
75 meta
.setCover(getCover());
76 meta
.setFakeCover(true);
82 protected String
getDesc() throws IOException
{
83 if (isSearchOrSet(getSource())) {
84 StringBuilder builder
= new StringBuilder();
85 builder
.append("A collection of images from ")
86 .append(getSource().getHost()).append("\n") //
87 .append("\tTime of creation: "
88 + StringUtils
.fromTime(new Date().getTime()))
90 .append("\tTags: ");//
91 for (String tag
: getTags()) {
92 builder
.append("\t\t").append(tag
);
95 return builder
.toString();
98 if (isPool(getSource())) {
99 Element el
= getSourceNode().getElementById("description");
109 protected List
<Entry
<String
, URL
>> getChapters(Progress pg
)
112 String jsonUrl
= getJsonUrl();
113 if (jsonUrl
!= null) {
114 for (i
= 1; true; i
++) {
117 // The API does not accept more than 2 request per sec,
118 // and asks us to limit at one per sec when possible
120 } catch (InterruptedException e
) {
125 JSONObject json
= getJson(jsonUrl
+ "&page=" + i
, false);
126 if (!json
.has("posts"))
128 JSONArray posts
= json
.getJSONArray("posts");
131 } catch (Exception e
) {
136 // The last page was empty:
140 // The pages and images are in reverse order on /posts/
141 List
<Entry
<String
, URL
>> chapters
= new LinkedList
<Entry
<String
, URL
>>();
142 for (int page
= i
; page
> 0; page
--) {
143 chapters
.add(new AbstractMap
.SimpleEntry
<String
, URL
>(
144 "Page " + Integer
.toString(i
- page
+ 1),
145 new URL(jsonUrl
+ "&page=" + page
)));
152 protected String
getChapterContent(URL chapUrl
, int number
, Progress pg
)
154 StringBuilder builder
= new StringBuilder();
156 JSONObject json
= getJson(chapUrl
, false);
157 JSONArray postsArr
= json
.getJSONArray("posts");
159 // The pages and images are in reverse order on /posts/
160 List
<JSONObject
> posts
= new ArrayList
<JSONObject
>(postsArr
.length());
161 for (int i
= postsArr
.length() - 1; i
>= 0; i
--) {
162 Object o
= postsArr
.get(i
);
163 if (o
instanceof JSONObject
)
164 posts
.add((JSONObject
) o
);
167 for (JSONObject post
: posts
) {
168 if (!post
.has("file"))
170 JSONObject file
= post
.getJSONObject("file");
171 if (!file
.has("url"))
175 String url
= file
.getString("url");
178 builder
.append("]<br/>");
179 } catch (JSONException e
) {
180 // Can be NULL if filtered
181 // When the value is NULL, we get an exception
182 // but the "has" method still returns true
186 return builder
.toString();
190 protected URL
getCanonicalUrl(URL source
) {
191 // Convert search-pools into proper pools
192 if (source
.getPath().equals("/posts") && source
.getQuery() != null
193 && source
.getQuery().startsWith("tags=pool%3A")) {
194 String poolNumber
= source
.getQuery()
195 .substring("tags=pool%3A".length());
197 Integer
.parseInt(poolNumber
);
198 String base
= source
.getProtocol() + "://" + source
.getHost();
199 if (source
.getPort() != -1) {
200 base
= base
+ ":" + source
.getPort();
202 source
= new URL(base
+ "/pools/" + poolNumber
);
203 } catch (NumberFormatException e
) {
204 // Not a simple pool, skip
205 } catch (MalformedURLException e
) {
210 if (isSetOriginalUrl(source
)) {
212 Document doc
= DataUtil
.load(Instance
.getInstance().getCache()
213 .open(source
, this, false), "UTF-8", source
.toString());
214 for (Element shortname
: doc
215 .getElementsByClass("set-shortname")) {
216 for (Element el
: shortname
.getElementsByTag("a")) {
217 if (!el
.attr("href").isEmpty())
218 return new URL(el
.absUrl("href"));
221 } catch (IOException e
) {
222 Instance
.getInstance().getTraceHandler().error(e
);
226 if (isPool(source
)) {
229 source
.toString().replace("/pool/show/", "/pools/"));
230 } catch (MalformedURLException e
) {
234 return super.getCanonicalUrl(source
);
237 private String
getTitle() {
240 Element el
= getSourceNode().getElementsByTag("title").first();
242 title
= el
.text().trim();
245 for (String s
: new String
[] { "e621", "-", "e621", "Pool", "-" }) {
246 if (title
.startsWith(s
)) {
247 title
= title
.substring(s
.length()).trim();
249 if (title
.endsWith(s
)) {
250 title
= title
.substring(0, title
.length() - s
.length()).trim();
254 if (isSearchOrSet(getSource())) {
255 title
= title
.isEmpty() ?
"e621" : "[e621] " + title
;
261 private String
getAuthor() {
262 List
<String
> list
= new ArrayList
<String
>();
263 String jsonUrl
= getJsonUrl();
264 if (jsonUrl
!= null) {
266 JSONObject json
= getJson(jsonUrl
, false);
267 JSONArray posts
= json
.getJSONArray("posts");
268 for (Object obj
: posts
) {
269 if (!(obj
instanceof JSONObject
))
272 JSONObject post
= (JSONObject
) obj
;
273 if (!post
.has("tags"))
276 JSONObject tags
= post
.getJSONObject("tags");
277 if (!tags
.has("artist"))
280 JSONArray artists
= tags
.getJSONArray("artist");
281 for (Object artist
: artists
) {
282 if (list
.contains(artist
.toString()))
285 list
.add(artist
.toString());
288 } catch (Exception e
) {
293 StringBuilder builder
= new StringBuilder();
294 for (String artist
: list
) {
295 if (builder
.length() > 0) {
296 builder
.append(", ");
298 builder
.append(artist
);
301 return builder
.toString();
304 private String
getDate() {
305 String jsonUrl
= getJsonUrl();
306 if (jsonUrl
!= null) {
308 JSONObject json
= getJson(jsonUrl
, false);
309 JSONArray posts
= json
.getJSONArray("posts");
310 for (Object obj
: posts
) {
311 if (!(obj
instanceof JSONObject
))
314 JSONObject post
= (JSONObject
) obj
;
315 if (!post
.has("created_at"))
318 return post
.getString("created_at");
320 } catch (Exception e
) {
329 private List
<String
> getTags() {
330 List
<String
> tags
= new ArrayList
<String
>();
331 if (isSearchOrSet(getSource())) {
332 String str
= getTagsFromUrl(getSource());
333 for (String tag
: str
.split("\\+")) {
335 tags
.add(URLDecoder
.decode(tag
.trim(), "UTF-8").trim());
336 } catch (UnsupportedEncodingException e
) {
344 // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
345 private String
getTagsFromUrl(URL url
) {
346 String tags
= url
== null ?
"" : url
.getQuery();
347 int pos
= tags
.indexOf("tags=");
350 tags
= tags
.substring(pos
).substring("tags=".length());
355 pos
= tags
.indexOf('&');
357 tags
= tags
.substring(0, pos
);
359 pos
= tags
.indexOf('/');
361 tags
= tags
.substring(0, pos
);
367 private Image
getCover() throws IOException
{
369 List
<Entry
<String
, URL
>> chapters
= getChapters(null);
370 if (!chapters
.isEmpty()) {
371 URL chap1Url
= chapters
.get(0).getValue();
372 String imgsChap1
= getChapterContent(chap1Url
, 1, null);
373 if (!imgsChap1
.isEmpty()) {
374 imgsChap1
= imgsChap1
.split("]")[0].substring(1).trim();
375 image
= bsImages
.getImage(this, new URL(imgsChap1
));
382 // always /posts.json/ url
383 private String
getJsonUrl() {
385 if (isSearchOrSet(getSource())) {
386 url
= getSource().toString().replace("/posts", "/posts.json");
389 if (isPool(getSource())) {
390 String poolNumber
= getSource().getPath()
391 .substring("/pools/".length());
392 url
= "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber
;
396 // Note: one way to override the blacklist
397 String login
= Instance
.getInstance().getConfig()
398 .getString(Config
.LOGIN_E621_LOGIN
);
399 String apk
= Instance
.getInstance().getConfig()
400 .getString(Config
.LOGIN_E621_APIKEY
);
402 if (login
!= null && !login
.isEmpty() && apk
!= null
404 url
= String
.format("%s&login=%s&api_key=%s&_client=%s", url
,
405 login
, apk
, "fanfix-" + Version
.getCurrentVersion());
412 // note: will be removed at getCanonicalUrl()
413 private boolean isSetOriginalUrl(URL originalUrl
) {
414 return originalUrl
.getPath().startsWith("/post_sets/");
417 private boolean isPool(URL url
) {
418 return url
.getPath().startsWith("/pools/")
419 || url
.getPath().startsWith("/pool/show/");
422 // set will be renamed into search by canonical url
423 private boolean isSearchOrSet(URL url
) {
426 (url
.getPath().equals("/posts") && url
.getQuery().contains("tags="))
428 || isSetOriginalUrl(url
);