1 package be
.nikiroo
.fanfix
.supported
;
3 import java
.io
.IOException
;
4 import java
.io
.UnsupportedEncodingException
;
5 import java
.net
.MalformedURLException
;
7 import java
.net
.URLDecoder
;
8 import java
.util
.AbstractMap
;
9 import java
.util
.ArrayList
;
10 import java
.util
.Collections
;
11 import java
.util
.Date
;
12 import java
.util
.LinkedList
;
13 import java
.util
.List
;
14 import java
.util
.Map
.Entry
;
16 import org
.json
.JSONArray
;
17 import org
.json
.JSONException
;
18 import org
.json
.JSONObject
;
19 import org
.jsoup
.helper
.DataUtil
;
20 import org
.jsoup
.nodes
.Document
;
21 import org
.jsoup
.nodes
.Element
;
23 import be
.nikiroo
.fanfix
.Instance
;
24 import be
.nikiroo
.fanfix
.bundles
.Config
;
25 import be
.nikiroo
.fanfix
.data
.MetaData
;
26 import be
.nikiroo
.utils
.Image
;
27 import be
.nikiroo
.utils
.Progress
;
28 import be
.nikiroo
.utils
.StringUtils
;
29 import be
.nikiroo
.utils
.Version
;
32 * Support class for <a href="http://e621.net/">e621.net</a> and
33 * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
34 * including some of MLP.
36 * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
37 * comics, but it can be difficult to browse.
41 class E621
extends BasicSupport
{
43 protected boolean supports(URL url
) {
44 String host
= url
.getHost();
45 if (host
.startsWith("www.")) {
46 host
= host
.substring("www.".length());
49 return ("e621.net".equals(host
) || "e926.net".equals(host
))
50 && (isPool(url
) || isSearchOrSet(url
));
54 protected boolean isHtml() {
59 protected MetaData
getMeta() throws IOException
{
60 MetaData meta
= new MetaData();
62 meta
.setTitle(getTitle());
63 meta
.setAuthor(getAuthor());
64 meta
.setDate(bsHelper
.formatDate(getDate()));
65 meta
.setTags(getTags());
66 meta
.setSource(getType().getSourceName());
67 meta
.setUrl(getSource().toString());
68 meta
.setPublisher(getType().getSourceName());
69 meta
.setUuid(getSource().toString());
72 meta
.setSubject("Furry");
73 meta
.setType(getType().toString());
74 meta
.setImageDocument(true);
75 meta
.setCover(getCover());
76 meta
.setFakeCover(true);
82 protected String
getDesc() throws IOException
{
83 if (isSearchOrSet(getSource())) {
84 StringBuilder builder
= new StringBuilder();
85 builder
.append("A collection of images from ")
86 .append(getSource().getHost()).append("\n") //
87 .append("\tTime of creation: "
88 + StringUtils
.fromTime(new Date().getTime()))
90 .append("\tTags: ");//
91 for (String tag
: getTags()) {
92 builder
.append("\t\t").append(tag
);
95 return builder
.toString();
98 if (isPool(getSource())) {
99 Element el
= getSourceNode().getElementById("description");
109 protected List
<Entry
<String
, URL
>> getChapters(Progress pg
)
112 String jsonUrl
= getJsonUrl();
113 if (jsonUrl
!= null) {
114 for (i
= 1; true; i
++) {
117 // The API does not accept more than 2 request per sec,
118 // and asks us to limit at one per sec when possible
120 } catch (InterruptedException e
) {
125 JSONObject json
= getJson(jsonUrl
+ "&page=" + i
, false);
126 if (!json
.has("posts"))
128 JSONArray posts
= json
.getJSONArray("posts");
131 } catch (Exception e
) {
136 // The last page was empty:
140 // The pages and images are in reverse order on /posts/
141 List
<Entry
<String
, URL
>> chapters
= new LinkedList
<Entry
<String
, URL
>>();
142 for (int page
= i
; page
> 0; page
--) {
143 chapters
.add(new AbstractMap
.SimpleEntry
<String
, URL
>(
144 "Page " + Integer
.toString(i
- page
+ 1),
145 new URL(jsonUrl
+ "&page=" + page
)));
152 protected String
getChapterContent(URL chapUrl
, int number
, Progress pg
)
154 StringBuilder builder
= new StringBuilder();
156 JSONObject json
= getJson(chapUrl
, false);
157 JSONArray postsArr
= json
.getJSONArray("posts");
159 // The pages and images are in reverse order on /posts/
160 List
<JSONObject
> posts
= new ArrayList
<JSONObject
>(postsArr
.length());
161 for (int i
= postsArr
.length() - 1; i
>= 0; i
--) {
162 Object o
= postsArr
.get(i
);
163 if (o
instanceof JSONObject
)
164 posts
.add((JSONObject
) o
);
167 for (JSONObject post
: posts
) {
168 if (!post
.has("file"))
170 JSONObject file
= post
.getJSONObject("file");
171 if (!file
.has("url"))
175 String url
= file
.getString("url");
178 builder
.append("]<br/>");
179 } catch (JSONException e
) {
180 // Can be NULL if filtered
181 // When the value is NULL, we get an exception
182 // but the "has" method still returns true
183 Instance
.getInstance().getTraceHandler()
184 .error("Cannot get image for chapter " + number
+ " of "
189 return builder
.toString();
193 protected URL
getCanonicalUrl(URL source
) {
194 // Convert search-pools into proper pools
195 if (source
.getPath().equals("/posts") && source
.getQuery() != null
196 && source
.getQuery().startsWith("tags=pool%3A")) {
197 String poolNumber
= source
.getQuery()
198 .substring("tags=pool%3A".length());
200 Integer
.parseInt(poolNumber
);
201 String base
= source
.getProtocol() + "://" + source
.getHost();
202 if (source
.getPort() != -1) {
203 base
= base
+ ":" + source
.getPort();
205 source
= new URL(base
+ "/pools/" + poolNumber
);
206 } catch (NumberFormatException e
) {
207 // Not a simple pool, skip
208 } catch (MalformedURLException e
) {
213 if (isSetOriginalUrl(source
)) {
215 Document doc
= DataUtil
.load(Instance
.getInstance().getCache()
216 .open(source
, this, false), "UTF-8", source
.toString());
217 for (Element shortname
: doc
218 .getElementsByClass("set-shortname")) {
219 for (Element el
: shortname
.getElementsByTag("a")) {
220 if (!el
.attr("href").isEmpty())
221 return new URL(el
.absUrl("href"));
224 } catch (IOException e
) {
225 Instance
.getInstance().getTraceHandler().error(e
);
229 if (isPool(source
)) {
232 source
.toString().replace("/pool/show/", "/pools/"));
233 } catch (MalformedURLException e
) {
237 return super.getCanonicalUrl(source
);
240 private String
getTitle() {
243 Element el
= getSourceNode().getElementsByTag("title").first();
245 title
= el
.text().trim();
248 for (String s
: new String
[] { "e621", "-", "e621", "Pool", "-" }) {
249 if (title
.startsWith(s
)) {
250 title
= title
.substring(s
.length()).trim();
252 if (title
.endsWith(s
)) {
253 title
= title
.substring(0, title
.length() - s
.length()).trim();
257 if (isSearchOrSet(getSource())) {
258 title
= title
.isEmpty() ?
"e621" : "[e621] " + title
;
264 private String
getAuthor() {
265 List
<String
> list
= new ArrayList
<String
>();
266 String jsonUrl
= getJsonUrl();
267 if (jsonUrl
!= null) {
269 JSONObject json
= getJson(jsonUrl
, false);
270 JSONArray posts
= json
.getJSONArray("posts");
271 for (Object obj
: posts
) {
272 if (!(obj
instanceof JSONObject
))
275 JSONObject post
= (JSONObject
) obj
;
276 if (!post
.has("tags"))
279 JSONObject tags
= post
.getJSONObject("tags");
280 if (!tags
.has("artist"))
283 JSONArray artists
= tags
.getJSONArray("artist");
284 for (Object artist
: artists
) {
285 if (list
.contains(artist
.toString()))
288 list
.add(artist
.toString());
291 } catch (Exception e
) {
296 StringBuilder builder
= new StringBuilder();
297 for (String artist
: list
) {
298 if (builder
.length() > 0) {
299 builder
.append(", ");
301 builder
.append(artist
);
304 return builder
.toString();
307 private String
getDate() {
308 String jsonUrl
= getJsonUrl();
309 if (jsonUrl
!= null) {
311 JSONObject json
= getJson(jsonUrl
, false);
312 JSONArray posts
= json
.getJSONArray("posts");
313 for (Object obj
: posts
) {
314 if (!(obj
instanceof JSONObject
))
317 JSONObject post
= (JSONObject
) obj
;
318 if (!post
.has("created_at"))
321 return post
.getString("created_at");
323 } catch (Exception e
) {
332 private List
<String
> getTags() {
333 List
<String
> tags
= new ArrayList
<String
>();
334 if (isSearchOrSet(getSource())) {
335 String str
= getTagsFromUrl(getSource());
336 for (String tag
: str
.split("\\+")) {
338 tags
.add(URLDecoder
.decode(tag
.trim(), "UTF-8").trim());
339 } catch (UnsupportedEncodingException e
) {
347 // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
348 private String
getTagsFromUrl(URL url
) {
349 String tags
= url
== null ?
"" : url
.getQuery();
350 int pos
= tags
.indexOf("tags=");
353 tags
= tags
.substring(pos
).substring("tags=".length());
358 pos
= tags
.indexOf('&');
360 tags
= tags
.substring(0, pos
);
362 pos
= tags
.indexOf('/');
364 tags
= tags
.substring(0, pos
);
370 private Image
getCover() throws IOException
{
372 List
<Entry
<String
, URL
>> chapters
= getChapters(null);
373 if (!chapters
.isEmpty()) {
374 URL chap1Url
= chapters
.get(0).getValue();
375 String imgsChap1
= getChapterContent(chap1Url
, 1, null);
376 if (!imgsChap1
.isEmpty()) {
377 imgsChap1
= imgsChap1
.split("]")[0].substring(1).trim();
378 image
= bsImages
.getImage(this, new URL(imgsChap1
));
385 // always /posts.json/ url
386 private String
getJsonUrl() {
388 if (isSearchOrSet(getSource())) {
389 url
= getSource().toString().replace("/posts", "/posts.json");
392 if (isPool(getSource())) {
393 String poolNumber
= getSource().getPath()
394 .substring("/pools/".length());
395 url
= "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber
;
399 // Note: one way to override the blacklist
400 String login
= Instance
.getInstance().getConfig()
401 .getString(Config
.LOGIN_E621_LOGIN
);
402 String apk
= Instance
.getInstance().getConfig()
403 .getString(Config
.LOGIN_E621_APIKEY
);
405 if (login
!= null && !login
.isEmpty() && apk
!= null
407 url
= String
.format("%s&login=%s&api_key=%s&_client=%s", url
,
408 login
, apk
, "fanfix-" + Version
.getCurrentVersion());
415 // note: will be removed at getCanonicalUrl()
416 private boolean isSetOriginalUrl(URL originalUrl
) {
417 return originalUrl
.getPath().startsWith("/post_sets/");
420 private boolean isPool(URL url
) {
421 return url
.getPath().startsWith("/pools/")
422 || url
.getPath().startsWith("/pool/show/");
425 // set will be renamed into search by canonical url
426 private boolean isSearchOrSet(URL url
) {
429 (url
.getPath().equals("/posts") && url
.getQuery().contains("tags="))
431 || isSetOriginalUrl(url
);