1 package be
.nikiroo
.fanfix
.supported
;
3 import java
.io
.IOException
;
4 import java
.io
.UnsupportedEncodingException
;
5 import java
.net
.MalformedURLException
;
7 import java
.net
.URLDecoder
;
8 import java
.util
.AbstractMap
;
9 import java
.util
.ArrayList
;
10 import java
.util
.Date
;
11 import java
.util
.LinkedList
;
12 import java
.util
.List
;
13 import java
.util
.Map
.Entry
;
15 import org
.json
.JSONArray
;
16 import org
.json
.JSONException
;
17 import org
.json
.JSONObject
;
18 import org
.jsoup
.helper
.DataUtil
;
19 import org
.jsoup
.nodes
.Document
;
20 import org
.jsoup
.nodes
.Element
;
22 import be
.nikiroo
.fanfix
.Instance
;
23 import be
.nikiroo
.fanfix
.bundles
.Config
;
24 import be
.nikiroo
.fanfix
.data
.MetaData
;
25 import be
.nikiroo
.utils
.Image
;
26 import be
.nikiroo
.utils
.Progress
;
27 import be
.nikiroo
.utils
.StringUtils
;
28 import be
.nikiroo
.utils
.Version
;
31 * Support class for <a href="http://e621.net/">e621.net</a> and
32 * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
33 * including some of MLP.
35 * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
36 * comics, but it can be difficult to browse.
40 class E621
extends BasicSupport
{
42 protected boolean supports(URL url
) {
43 String host
= url
.getHost();
44 if (host
.startsWith("www.")) {
45 host
= host
.substring("www.".length());
48 return ("e621.net".equals(host
) || "e926.net".equals(host
))
49 && (isPool(url
) || isSearchOrSet(url
));
53 protected boolean isHtml() {
58 protected MetaData
getMeta() throws IOException
{
59 MetaData meta
= new MetaData();
61 meta
.setTitle(getTitle());
62 meta
.setAuthor(getAuthor());
63 meta
.setDate(bsHelper
.formatDate(getDate()));
64 meta
.setTags(getTags());
65 meta
.setUrl(getSource().toString());
66 meta
.setUuid(getSource().toString());
69 meta
.setSubject("Furry");
70 meta
.setImageDocument(true);
71 meta
.setCover(getCover());
72 meta
.setFakeCover(true);
78 protected String
getDesc() throws IOException
{
79 if (isSearchOrSet(getSource())) {
80 StringBuilder builder
= new StringBuilder();
81 builder
.append("<div>");
82 builder
.append("A collection of images from ")
83 .append(getSource().getHost()) //
85 .append(" Time of creation: "
86 + StringUtils
.fromTime(new Date().getTime()))
88 .append(" tTags: ");//
89 for (String tag
: getTags()) {
91 "\n<br/> ")
94 builder
.append("\n</div>");
96 return builder
.toString();
99 if (isPool(getSource())) {
100 Element el
= getSourceNode().getElementById("description");
110 protected List
<Entry
<String
, URL
>> getChapters(Progress pg
)
113 String jsonUrl
= getJsonUrl();
114 if (jsonUrl
!= null) {
115 for (i
= 1; true; i
++) {
118 // The API does not accept more than 2 request per sec,
119 // and asks us to limit at one per sec when possible
121 } catch (InterruptedException e
) {
126 JSONObject json
= getJson(jsonUrl
+ "&page=" + i
, false);
127 if (!json
.has("posts"))
129 JSONArray posts
= json
.getJSONArray("posts");
132 } catch (Exception e
) {
137 // The last page was empty:
141 // The pages and images are in reverse order on /posts/
142 List
<Entry
<String
, URL
>> chapters
= new LinkedList
<Entry
<String
, URL
>>();
143 for (int page
= i
; page
> 0; page
--) {
144 chapters
.add(new AbstractMap
.SimpleEntry
<String
, URL
>(
145 "Page " + Integer
.toString(i
- page
+ 1),
146 new URL(jsonUrl
+ "&page=" + page
)));
153 protected String
getChapterContent(URL chapUrl
, int number
, Progress pg
)
155 StringBuilder builder
= new StringBuilder();
157 JSONObject json
= getJson(chapUrl
, false);
158 JSONArray postsArr
= json
.getJSONArray("posts");
160 // The pages and images are in reverse order on /posts/
161 List
<JSONObject
> posts
= new ArrayList
<JSONObject
>(postsArr
.length());
162 for (int i
= postsArr
.length() - 1; i
>= 0; i
--) {
163 Object o
= postsArr
.get(i
);
164 if (o
instanceof JSONObject
)
165 posts
.add((JSONObject
) o
);
168 for (JSONObject post
: posts
) {
169 if (!post
.has("file"))
171 JSONObject file
= post
.getJSONObject("file");
172 if (!file
.has("url"))
176 String url
= file
.getString("url");
179 builder
.append("]<br/>");
180 } catch (JSONException e
) {
181 // Can be NULL if filtered
182 // When the value is NULL, we get an exception
183 // but the "has" method still returns true
184 Instance
.getInstance().getTraceHandler()
185 .error("Cannot get image for chapter " + number
+ " of "
190 return builder
.toString();
194 protected URL
getCanonicalUrl(URL source
) {
195 // Convert search-pools into proper pools
196 if (source
.getPath().equals("/posts") && source
.getQuery() != null
197 && source
.getQuery().startsWith("tags=pool%3A")) {
198 String poolNumber
= source
.getQuery()
199 .substring("tags=pool%3A".length());
201 Integer
.parseInt(poolNumber
);
202 String base
= source
.getProtocol() + "://" + source
.getHost();
203 if (source
.getPort() != -1) {
204 base
= base
+ ":" + source
.getPort();
206 source
= new URL(base
+ "/pools/" + poolNumber
);
207 } catch (NumberFormatException e
) {
208 // Not a simple pool, skip
209 } catch (MalformedURLException e
) {
214 if (isSetOriginalUrl(source
)) {
216 Document doc
= DataUtil
.load(Instance
.getInstance().getCache()
217 .open(source
, this, false), "UTF-8", source
.toString());
218 for (Element shortname
: doc
219 .getElementsByClass("set-shortname")) {
220 for (Element el
: shortname
.getElementsByTag("a")) {
221 if (!el
.attr("href").isEmpty())
222 return new URL(el
.absUrl("href"));
225 } catch (IOException e
) {
226 Instance
.getInstance().getTraceHandler().error(e
);
230 if (isPool(source
)) {
233 source
.toString().replace("/pool/show/", "/pools/"));
234 } catch (MalformedURLException e
) {
238 return super.getCanonicalUrl(source
);
241 private String
getTitle() {
244 Element el
= getSourceNode().getElementsByTag("title").first();
246 title
= el
.text().trim();
249 for (String s
: new String
[] { "e621", "-", "e621", "Pool", "-" }) {
250 if (title
.startsWith(s
)) {
251 title
= title
.substring(s
.length()).trim();
253 if (title
.endsWith(s
)) {
254 title
= title
.substring(0, title
.length() - s
.length()).trim();
258 if (isSearchOrSet(getSource())) {
259 title
= title
.isEmpty() ?
"e621" : "[e621] " + title
;
265 private String
getAuthor() {
266 List
<String
> list
= new ArrayList
<String
>();
267 String jsonUrl
= getJsonUrl();
268 if (jsonUrl
!= null) {
270 JSONObject json
= getJson(jsonUrl
, false);
271 JSONArray posts
= json
.getJSONArray("posts");
272 for (Object obj
: posts
) {
273 if (!(obj
instanceof JSONObject
))
276 JSONObject post
= (JSONObject
) obj
;
277 if (!post
.has("tags"))
280 JSONObject tags
= post
.getJSONObject("tags");
281 if (!tags
.has("artist"))
284 JSONArray artists
= tags
.getJSONArray("artist");
285 for (Object artist
: artists
) {
286 if (list
.contains(artist
.toString()))
289 list
.add(artist
.toString());
292 } catch (Exception e
) {
297 StringBuilder builder
= new StringBuilder();
298 for (String artist
: list
) {
299 if (builder
.length() > 0) {
300 builder
.append(", ");
302 builder
.append(artist
);
305 return builder
.toString();
308 private String
getDate() {
309 String jsonUrl
= getJsonUrl();
310 if (jsonUrl
!= null) {
312 JSONObject json
= getJson(jsonUrl
, false);
313 JSONArray posts
= json
.getJSONArray("posts");
314 for (Object obj
: posts
) {
315 if (!(obj
instanceof JSONObject
))
318 JSONObject post
= (JSONObject
) obj
;
319 if (!post
.has("created_at"))
322 return post
.getString("created_at");
324 } catch (Exception e
) {
333 private List
<String
> getTags() {
334 List
<String
> tags
= new ArrayList
<String
>();
335 if (isSearchOrSet(getSource())) {
336 String str
= getTagsFromUrl(getSource());
337 for (String tag
: str
.split("\\+")) {
339 tags
.add(URLDecoder
.decode(tag
.trim(), "UTF-8").trim());
340 } catch (UnsupportedEncodingException e
) {
348 // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
349 private String
getTagsFromUrl(URL url
) {
350 String tags
= url
== null ?
"" : url
.getQuery();
351 int pos
= tags
.indexOf("tags=");
354 tags
= tags
.substring(pos
).substring("tags=".length());
359 pos
= tags
.indexOf('&');
361 tags
= tags
.substring(0, pos
);
363 pos
= tags
.indexOf('/');
365 tags
= tags
.substring(0, pos
);
371 private Image
getCover() throws IOException
{
373 List
<Entry
<String
, URL
>> chapters
= getChapters(null);
374 if (!chapters
.isEmpty()) {
375 URL chap1Url
= chapters
.get(0).getValue();
376 String imgsChap1
= getChapterContent(chap1Url
, 1, null);
377 if (!imgsChap1
.isEmpty()) {
378 imgsChap1
= imgsChap1
.split("]")[0].substring(1).trim();
379 image
= bsImages
.getImage(this, new URL(imgsChap1
));
386 // always /posts.json/ url
387 private String
getJsonUrl() {
389 if (isSearchOrSet(getSource())) {
390 url
= getSource().toString().replace("/posts", "/posts.json");
393 if (isPool(getSource())) {
394 String poolNumber
= getSource().getPath()
395 .substring("/pools/".length());
396 url
= "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber
;
400 // Note: one way to override the blacklist
401 String login
= Instance
.getInstance().getConfig()
402 .getString(Config
.LOGIN_E621_LOGIN
);
403 String apk
= Instance
.getInstance().getConfig()
404 .getString(Config
.LOGIN_E621_APIKEY
);
406 if (login
!= null && !login
.isEmpty() && apk
!= null
408 url
= String
.format("%s&login=%s&api_key=%s&_client=%s", url
,
409 login
, apk
, "fanfix-" + Version
.getCurrentVersion());
416 // note: will be removed at getCanonicalUrl()
417 private boolean isSetOriginalUrl(URL originalUrl
) {
418 return originalUrl
.getPath().startsWith("/post_sets/");
421 private boolean isPool(URL url
) {
422 return url
.getPath().startsWith("/pools/")
423 || url
.getPath().startsWith("/pool/show/");
426 // set will be renamed into search by canonical url
427 private boolean isSearchOrSet(URL url
) {
430 (url
.getPath().equals("/posts") && url
.getQuery().contains("tags="))
432 || isSetOriginalUrl(url
);