merge from master
[nikiroo-utils.git] / supported / E621.java
CommitLineData
08fe2e33
NR
1package be.nikiroo.fanfix.supported;
2
3import java.io.IOException;
b5e9855b 4import java.io.UnsupportedEncodingException;
c4b18c94 5import java.net.MalformedURLException;
08fe2e33 6import java.net.URL;
b5e9855b 7import java.net.URLDecoder;
ce297a79 8import java.util.AbstractMap;
08fe2e33 9import java.util.ArrayList;
8ac3d099 10import java.util.Date;
b5e9855b 11import java.util.LinkedList;
08fe2e33
NR
12import java.util.List;
13import java.util.Map.Entry;
8ac3d099 14
5cf61f35
NR
15import org.json.JSONArray;
16import org.json.JSONException;
17import org.json.JSONObject;
8ac3d099
NR
18import org.jsoup.helper.DataUtil;
19import org.jsoup.nodes.Document;
20import org.jsoup.nodes.Element;
08fe2e33
NR
21
22import be.nikiroo.fanfix.Instance;
5cf61f35 23import be.nikiroo.fanfix.bundles.Config;
68686a37 24import be.nikiroo.fanfix.data.MetaData;
16a81ef7 25import be.nikiroo.utils.Image;
3b2b638f 26import be.nikiroo.utils.Progress;
08fe2e33 27import be.nikiroo.utils.StringUtils;
5cf61f35 28import be.nikiroo.utils.Version;
08fe2e33
NR
29
30/**
8ac3d099
NR
31 * Support class for <a href="http://e621.net/">e621.net</a> and
32 * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
08fe2e33
NR
33 * including some of MLP.
34 * <p>
35 * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
36 * comics, but it can be difficult to browse.
37 *
38 * @author niki
39 */
8ac3d099 40class E621 extends BasicSupport {
08fe2e33
NR
41 @Override
42 protected boolean supports(URL url) {
43 String host = url.getHost();
44 if (host.startsWith("www.")) {
45 host = host.substring("www.".length());
46 }
47
5cf61f35
NR
48 return ("e621.net".equals(host) || "e926.net".equals(host))
49 && (isPool(url) || isSearchOrSet(url));
08fe2e33
NR
50 }
51
52 @Override
53 protected boolean isHtml() {
54 return true;
55 }
56
8ac3d099
NR
57 @Override
58 protected MetaData getMeta() throws IOException {
59 MetaData meta = new MetaData();
b5e9855b 60
8ac3d099
NR
61 meta.setTitle(getTitle());
62 meta.setAuthor(getAuthor());
bff19b54 63 meta.setDate(bsHelper.formatDate(getDate()));
8ac3d099
NR
64 meta.setTags(getTags());
65 meta.setSource(getType().getSourceName());
66 meta.setUrl(getSource().toString());
67 meta.setPublisher(getType().getSourceName());
68 meta.setUuid(getSource().toString());
69 meta.setLuid("");
70 meta.setLang("en");
71 meta.setSubject("Furry");
72 meta.setType(getType().toString());
73 meta.setImageDocument(true);
74 meta.setCover(getCover());
75 meta.setFakeCover(true);
595dfa7a 76
8ac3d099 77 return meta;
595dfa7a
NR
78 }
79
8ac3d099
NR
80 @Override
81 protected String getDesc() throws IOException {
82 if (isSearchOrSet(getSource())) {
b5e9855b 83 StringBuilder builder = new StringBuilder();
5cf61f35
NR
84 builder.append("A collection of images from ")
85 .append(getSource().getHost()).append("\n") //
86 .append("\tTime of creation: "
87 + StringUtils.fromTime(new Date().getTime()))
88 .append("\n") //
8ac3d099
NR
89 .append("\tTags: ");//
90 for (String tag : getTags()) {
91 builder.append("\t\t").append(tag);
b5e9855b
NR
92 }
93
94 return builder.toString();
95 }
96
8ac3d099
NR
97 if (isPool(getSource())) {
98 Element el = getSourceNode().getElementById("description");
99 if (el != null) {
100 return el.text();
08fe2e33
NR
101 }
102 }
103
104 return null;
105 }
106
08fe2e33 107 @Override
5cf61f35 108 protected List<Entry<String, URL>> getChapters(Progress pg)
8ac3d099 109 throws IOException {
5cf61f35
NR
110 int i = 1;
111 String jsonUrl = getJsonUrl();
112 if (jsonUrl != null) {
113 for (i = 1; true; i++) {
114 if (i > 1) {
115 try {
116 // The API does not accept more than 2 request per sec,
117 // and asks us to limit at one per sec when possible
118 Thread.sleep(1000);
119 } catch (InterruptedException e) {
120 }
121 }
b5e9855b 122
b5e9855b 123 try {
5cf61f35
NR
124 JSONObject json = getJson(jsonUrl + "&page=" + i, false);
125 if (!json.has("posts"))
b5e9855b 126 break;
5cf61f35
NR
127 JSONArray posts = json.getJSONArray("posts");
128 if (posts.isEmpty())
129 break;
130 } catch (Exception e) {
131 e.printStackTrace();
b5e9855b 132 }
b5e9855b 133 }
5cf61f35
NR
134
135 // The last page was empty:
136 i--;
137 }
138
139 // The pages and images are in reverse order on /posts/
140 List<Entry<String, URL>> chapters = new LinkedList<Entry<String, URL>>();
141 for (int page = i; page > 0; page--) {
142 chapters.add(new AbstractMap.SimpleEntry<String, URL>(
143 "Page " + Integer.toString(i - page + 1),
144 new URL(jsonUrl + "&page=" + page)));
b5e9855b
NR
145 }
146
5cf61f35 147 return chapters;
b5e9855b
NR
148 }
149
8ac3d099 150 @Override
5cf61f35
NR
151 protected String getChapterContent(URL chapUrl, int number, Progress pg)
152 throws IOException {
8ac3d099 153 StringBuilder builder = new StringBuilder();
5cf61f35
NR
154
155 JSONObject json = getJson(chapUrl, false);
156 JSONArray postsArr = json.getJSONArray("posts");
157
158 // The pages and images are in reverse order on /posts/
159 List<JSONObject> posts = new ArrayList<JSONObject>(postsArr.length());
160 for (int i = postsArr.length() - 1; i >= 0; i--) {
161 Object o = postsArr.get(i);
162 if (o instanceof JSONObject)
163 posts.add((JSONObject) o);
75002fcc 164 }
5cf61f35
NR
165
166 for (JSONObject post : posts) {
167 if (!post.has("file"))
168 continue;
169 JSONObject file = post.getJSONObject("file");
170 if (!file.has("url"))
171 continue;
172
173 try {
174 String url = file.getString("url");
175 builder.append("[");
176 builder.append(url);
177 builder.append("]<br/>");
178 } catch (JSONException e) {
179 // Can be NULL if filtered
180 // When the value is NULL, we get an exception
181 // but the "has" method still returns true
a3d0728c
NR
182 Instance.getInstance().getTraceHandler()
183 .error("Cannot get image for chapter " + number + " of "
184 + getSource());
5cf61f35 185 }
8ac3d099
NR
186 }
187
188 return builder.toString();
189 }
190
191 @Override
192 protected URL getCanonicalUrl(URL source) {
8fbfa934
NR
193 // Convert search-pools into proper pools
194 if (source.getPath().equals("/posts") && source.getQuery() != null
195 && source.getQuery().startsWith("tags=pool%3A")) {
196 String poolNumber = source.getQuery()
197 .substring("tags=pool%3A".length());
198 try {
199 Integer.parseInt(poolNumber);
200 String base = source.getProtocol() + "://" + source.getHost();
201 if (source.getPort() != -1) {
202 base = base + ":" + source.getPort();
203 }
42cdf6f0 204 source = new URL(base + "/pools/" + poolNumber);
8fbfa934 205 } catch (NumberFormatException e) {
36c35b92 206 // Not a simple pool, skip
8fbfa934
NR
207 } catch (MalformedURLException e) {
208 // Cannot happen
209 }
210 }
5cf61f35 211
8ac3d099
NR
212 if (isSetOriginalUrl(source)) {
213 try {
5cf61f35
NR
214 Document doc = DataUtil.load(Instance.getInstance().getCache()
215 .open(source, this, false), "UTF-8", source.toString());
216 for (Element shortname : doc
217 .getElementsByClass("set-shortname")) {
8ac3d099
NR
218 for (Element el : shortname.getElementsByTag("a")) {
219 if (!el.attr("href").isEmpty())
220 return new URL(el.absUrl("href"));
08fe2e33
NR
221 }
222 }
8ac3d099 223 } catch (IOException e) {
d66deb8d 224 Instance.getInstance().getTraceHandler().error(e);
08fe2e33
NR
225 }
226 }
227
c4b18c94
NR
228 if (isPool(source)) {
229 try {
5cf61f35
NR
230 return new URL(
231 source.toString().replace("/pool/show/", "/pools/"));
c4b18c94
NR
232 } catch (MalformedURLException e) {
233 }
234 }
235
8ac3d099
NR
236 return super.getCanonicalUrl(source);
237 }
238
8ac3d099
NR
239 private String getTitle() {
240 String title = "";
241
242 Element el = getSourceNode().getElementsByTag("title").first();
243 if (el != null) {
244 title = el.text().trim();
08fe2e33
NR
245 }
246
36c35b92 247 for (String s : new String[] { "e621", "-", "e621", "Pool", "-" }) {
8ac3d099
NR
248 if (title.startsWith(s)) {
249 title = title.substring(s.length()).trim();
08fe2e33 250 }
8ac3d099
NR
251 if (title.endsWith(s)) {
252 title = title.substring(0, title.length() - s.length()).trim();
253 }
08fe2e33
NR
254 }
255
8ac3d099
NR
256 if (isSearchOrSet(getSource())) {
257 title = title.isEmpty() ? "e621" : "[e621] " + title;
258 }
5cf61f35 259
8ac3d099 260 return title;
08fe2e33 261 }
b5e9855b 262
5cf61f35
NR
263 private String getAuthor() {
264 List<String> list = new ArrayList<String>();
265 String jsonUrl = getJsonUrl();
266 if (jsonUrl != null) {
267 try {
268 JSONObject json = getJson(jsonUrl, false);
269 JSONArray posts = json.getJSONArray("posts");
270 for (Object obj : posts) {
271 if (!(obj instanceof JSONObject))
272 continue;
273
274 JSONObject post = (JSONObject) obj;
275 if (!post.has("tags"))
276 continue;
277
278 JSONObject tags = post.getJSONObject("tags");
279 if (!tags.has("artist"))
280 continue;
281
282 JSONArray artists = tags.getJSONArray("artist");
283 for (Object artist : artists) {
284 if (list.contains(artist.toString()))
285 continue;
286
287 list.add(artist.toString());
8ac3d099 288 }
9948521d 289 }
5cf61f35
NR
290 } catch (Exception e) {
291 e.printStackTrace();
8ac3d099
NR
292 }
293 }
294
5cf61f35
NR
295 StringBuilder builder = new StringBuilder();
296 for (String artist : list) {
297 if (builder.length() > 0) {
298 builder.append(", ");
299 }
300 builder.append(artist);
301 }
8ac3d099 302
5cf61f35
NR
303 return builder.toString();
304 }
8ac3d099 305
5cf61f35
NR
306 private String getDate() {
307 String jsonUrl = getJsonUrl();
308 if (jsonUrl != null) {
309 try {
310 JSONObject json = getJson(jsonUrl, false);
311 JSONArray posts = json.getJSONArray("posts");
312 for (Object obj : posts) {
313 if (!(obj instanceof JSONObject))
314 continue;
8d1a4fd2 315
5cf61f35
NR
316 JSONObject post = (JSONObject) obj;
317 if (!post.has("created_at"))
318 continue;
319
320 return post.getString("created_at");
8d1a4fd2 321 }
5cf61f35
NR
322 } catch (Exception e) {
323 e.printStackTrace();
8d1a4fd2 324 }
8ac3d099 325 }
9948521d 326
5cf61f35 327 return "";
8ac3d099
NR
328 }
329
330 // no tags for pools
331 private List<String> getTags() {
332 List<String> tags = new ArrayList<String>();
333 if (isSearchOrSet(getSource())) {
334 String str = getTagsFromUrl(getSource());
335 for (String tag : str.split("\\+")) {
9b863b20 336 try {
8ac3d099
NR
337 tags.add(URLDecoder.decode(tag.trim(), "UTF-8").trim());
338 } catch (UnsupportedEncodingException e) {
9b863b20
NR
339 }
340 }
341 }
9948521d 342
8ac3d099
NR
343 return tags;
344 }
345
5cf61f35
NR
346 // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
347 private String getTagsFromUrl(URL url) {
348 String tags = url == null ? "" : url.getQuery();
349 int pos = tags.indexOf("tags=");
350
351 if (pos >= 0) {
352 tags = tags.substring(pos).substring("tags=".length());
353 } else {
354 return "";
355 }
356
357 pos = tags.indexOf('&');
358 if (pos > 0) {
359 tags = tags.substring(0, pos);
360 }
361 pos = tags.indexOf('/');
362 if (pos > 0) {
363 tags = tags.substring(0, pos);
364 }
365
366 return tags;
367 }
368
8ac3d099
NR
369 private Image getCover() throws IOException {
370 Image image = null;
371 List<Entry<String, URL>> chapters = getChapters(null);
372 if (!chapters.isEmpty()) {
12c180fc
NR
373 URL chap1Url = chapters.get(0).getValue();
374 String imgsChap1 = getChapterContent(chap1Url, 1, null);
375 if (!imgsChap1.isEmpty()) {
376 imgsChap1 = imgsChap1.split("]")[0].substring(1).trim();
377 image = bsImages.getImage(this, new URL(imgsChap1));
378 }
8ac3d099
NR
379 }
380
381 return image;
382 }
383
5cf61f35
NR
384 // always /posts.json/ url
385 private String getJsonUrl() {
386 String url = null;
387 if (isSearchOrSet(getSource())) {
388 url = getSource().toString().replace("/posts", "/posts.json");
389 }
390
391 if (isPool(getSource())) {
392 String poolNumber = getSource().getPath()
393 .substring("/pools/".length());
394 url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber;
395 }
396
397 if (url != null) {
398 // Note: one way to override the blacklist
399 String login = Instance.getInstance().getConfig()
400 .getString(Config.LOGIN_E621_LOGIN);
401 String apk = Instance.getInstance().getConfig()
402 .getString(Config.LOGIN_E621_APIKEY);
403
404 if (login != null && !login.isEmpty() && apk != null
405 && !apk.isEmpty()) {
406 url = String.format("%s&login=%s&api_key=%s&_client=%s", url,
407 login, apk, "fanfix-" + Version.getCurrentVersion());
408 }
409 }
410
411 return url;
412 }
413
8ac3d099
NR
414 // note: will be removed at getCanonicalUrl()
415 private boolean isSetOriginalUrl(URL originalUrl) {
416 return originalUrl.getPath().startsWith("/post_sets/");
9b863b20
NR
417 }
418
b5e9855b 419 private boolean isPool(URL url) {
5cf61f35
NR
420 return url.getPath().startsWith("/pools/")
421 || url.getPath().startsWith("/pool/show/");
b5e9855b
NR
422 }
423
8ac3d099
NR
424 // set will be renamed into search by canonical url
425 private boolean isSearchOrSet(URL url) {
426 return
427 // search:
428 (url.getPath().equals("/posts") && url.getQuery().contains("tags="))
429 // or set:
430 || isSetOriginalUrl(url);
b5e9855b 431 }
08fe2e33 432}