e621: use api key
[fanfix.git] / supported / E621.java
... / ...
CommitLineData
1package be.nikiroo.fanfix.supported;
2
3import java.io.IOException;
4import java.io.UnsupportedEncodingException;
5import java.net.MalformedURLException;
6import java.net.URL;
7import java.net.URLDecoder;
8import java.util.AbstractMap;
9import java.util.ArrayList;
10import java.util.Collections;
11import java.util.Date;
12import java.util.LinkedList;
13import java.util.List;
14import java.util.Map.Entry;
15
16import org.json.JSONArray;
17import org.json.JSONException;
18import org.json.JSONObject;
19import org.jsoup.helper.DataUtil;
20import org.jsoup.nodes.Document;
21import org.jsoup.nodes.Element;
22
23import be.nikiroo.fanfix.Instance;
24import be.nikiroo.fanfix.bundles.Config;
25import be.nikiroo.fanfix.data.MetaData;
26import be.nikiroo.utils.Image;
27import be.nikiroo.utils.Progress;
28import be.nikiroo.utils.StringUtils;
29import be.nikiroo.utils.Version;
30
31/**
32 * Support class for <a href="http://e621.net/">e621.net</a> and
33 * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
34 * including some of MLP.
35 * <p>
36 * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
37 * comics, but it can be difficult to browse.
38 *
39 * @author niki
40 */
41class E621 extends BasicSupport {
42 @Override
43 protected boolean supports(URL url) {
44 String host = url.getHost();
45 if (host.startsWith("www.")) {
46 host = host.substring("www.".length());
47 }
48
49 return ("e621.net".equals(host) || "e926.net".equals(host))
50 && (isPool(url) || isSearchOrSet(url));
51 }
52
53 @Override
54 protected boolean isHtml() {
55 return true;
56 }
57
58 @Override
59 protected MetaData getMeta() throws IOException {
60 MetaData meta = new MetaData();
61
62 meta.setTitle(getTitle());
63 meta.setAuthor(getAuthor());
64 meta.setDate(getDate());
65 meta.setTags(getTags());
66 meta.setSource(getType().getSourceName());
67 meta.setUrl(getSource().toString());
68 meta.setPublisher(getType().getSourceName());
69 meta.setUuid(getSource().toString());
70 meta.setLuid("");
71 meta.setLang("en");
72 meta.setSubject("Furry");
73 meta.setType(getType().toString());
74 meta.setImageDocument(true);
75 meta.setCover(getCover());
76 meta.setFakeCover(true);
77
78 return meta;
79 }
80
81 @Override
82 protected String getDesc() throws IOException {
83 if (isSearchOrSet(getSource())) {
84 StringBuilder builder = new StringBuilder();
85 builder.append("A collection of images from ")
86 .append(getSource().getHost()).append("\n") //
87 .append("\tTime of creation: "
88 + StringUtils.fromTime(new Date().getTime()))
89 .append("\n") //
90 .append("\tTags: ");//
91 for (String tag : getTags()) {
92 builder.append("\t\t").append(tag);
93 }
94
95 return builder.toString();
96 }
97
98 if (isPool(getSource())) {
99 Element el = getSourceNode().getElementById("description");
100 if (el != null) {
101 return el.text();
102 }
103 }
104
105 return null;
106 }
107
108 @Override
109 protected List<Entry<String, URL>> getChapters(Progress pg)
110 throws IOException {
111 int i = 1;
112 String jsonUrl = getJsonUrl();
113 if (jsonUrl != null) {
114 for (i = 1; true; i++) {
115 if (i > 1) {
116 try {
117 // The API does not accept more than 2 request per sec,
118 // and asks us to limit at one per sec when possible
119 Thread.sleep(1000);
120 } catch (InterruptedException e) {
121 }
122 }
123
124 try {
125 JSONObject json = getJson(jsonUrl + "&page=" + i, false);
126 if (!json.has("posts"))
127 break;
128 JSONArray posts = json.getJSONArray("posts");
129 if (posts.isEmpty())
130 break;
131 } catch (Exception e) {
132 e.printStackTrace();
133 }
134 }
135
136 // The last page was empty:
137 i--;
138 }
139
140 // The pages and images are in reverse order on /posts/
141 List<Entry<String, URL>> chapters = new LinkedList<Entry<String, URL>>();
142 for (int page = i; page > 0; page--) {
143 chapters.add(new AbstractMap.SimpleEntry<String, URL>(
144 "Page " + Integer.toString(i - page + 1),
145 new URL(jsonUrl + "&page=" + page)));
146 }
147
148 return chapters;
149 }
150
151 @Override
152 protected String getChapterContent(URL chapUrl, int number, Progress pg)
153 throws IOException {
154 StringBuilder builder = new StringBuilder();
155
156 JSONObject json = getJson(chapUrl, false);
157 JSONArray postsArr = json.getJSONArray("posts");
158
159 // The pages and images are in reverse order on /posts/
160 List<JSONObject> posts = new ArrayList<JSONObject>(postsArr.length());
161 for (int i = postsArr.length() - 1; i >= 0; i--) {
162 Object o = postsArr.get(i);
163 if (o instanceof JSONObject)
164 posts.add((JSONObject) o);
165 }
166
167 for (JSONObject post : posts) {
168 if (!post.has("file"))
169 continue;
170 JSONObject file = post.getJSONObject("file");
171 if (!file.has("url"))
172 continue;
173
174 try {
175 String url = file.getString("url");
176 builder.append("[");
177 builder.append(url);
178 builder.append("]<br/>");
179 } catch (JSONException e) {
180 // Can be NULL if filtered
181 // When the value is NULL, we get an exception
182 // but the "has" method still returns true
183 }
184 }
185
186 return builder.toString();
187 }
188
189 @Override
190 protected URL getCanonicalUrl(URL source) {
191 // Convert search-pools into proper pools
192 if (source.getPath().equals("/posts") && source.getQuery() != null
193 && source.getQuery().startsWith("tags=pool%3A")) {
194 String poolNumber = source.getQuery()
195 .substring("tags=pool%3A".length());
196 try {
197 Integer.parseInt(poolNumber);
198 String base = source.getProtocol() + "://" + source.getHost();
199 if (source.getPort() != -1) {
200 base = base + ":" + source.getPort();
201 }
202 source = new URL(base + "/pools/" + poolNumber);
203 } catch (NumberFormatException e) {
204 // Not a simple pool, skip
205 } catch (MalformedURLException e) {
206 // Cannot happen
207 }
208 }
209
210 if (isSetOriginalUrl(source)) {
211 try {
212 Document doc = DataUtil.load(Instance.getInstance().getCache()
213 .open(source, this, false), "UTF-8", source.toString());
214 for (Element shortname : doc
215 .getElementsByClass("set-shortname")) {
216 for (Element el : shortname.getElementsByTag("a")) {
217 if (!el.attr("href").isEmpty())
218 return new URL(el.absUrl("href"));
219 }
220 }
221 } catch (IOException e) {
222 Instance.getInstance().getTraceHandler().error(e);
223 }
224 }
225
226 if (isPool(source)) {
227 try {
228 return new URL(
229 source.toString().replace("/pool/show/", "/pools/"));
230 } catch (MalformedURLException e) {
231 }
232 }
233
234 return super.getCanonicalUrl(source);
235 }
236
237 private String getTitle() {
238 String title = "";
239
240 Element el = getSourceNode().getElementsByTag("title").first();
241 if (el != null) {
242 title = el.text().trim();
243 }
244
245 for (String s : new String[] { "e621", "-", "e621", "Pool", "-" }) {
246 if (title.startsWith(s)) {
247 title = title.substring(s.length()).trim();
248 }
249 if (title.endsWith(s)) {
250 title = title.substring(0, title.length() - s.length()).trim();
251 }
252 }
253
254 if (isSearchOrSet(getSource())) {
255 title = title.isEmpty() ? "e621" : "[e621] " + title;
256 }
257
258 return title;
259 }
260
261 private String getAuthor() {
262 List<String> list = new ArrayList<String>();
263 String jsonUrl = getJsonUrl();
264 if (jsonUrl != null) {
265 try {
266 JSONObject json = getJson(jsonUrl, false);
267 JSONArray posts = json.getJSONArray("posts");
268 for (Object obj : posts) {
269 if (!(obj instanceof JSONObject))
270 continue;
271
272 JSONObject post = (JSONObject) obj;
273 if (!post.has("tags"))
274 continue;
275
276 JSONObject tags = post.getJSONObject("tags");
277 if (!tags.has("artist"))
278 continue;
279
280 JSONArray artists = tags.getJSONArray("artist");
281 for (Object artist : artists) {
282 if (list.contains(artist.toString()))
283 continue;
284
285 list.add(artist.toString());
286 }
287 }
288 } catch (Exception e) {
289 e.printStackTrace();
290 }
291 }
292
293 StringBuilder builder = new StringBuilder();
294 for (String artist : list) {
295 if (builder.length() > 0) {
296 builder.append(", ");
297 }
298 builder.append(artist);
299 }
300
301 return builder.toString();
302 }
303
304 private String getDate() {
305 String jsonUrl = getJsonUrl();
306 if (jsonUrl != null) {
307 try {
308 JSONObject json = getJson(jsonUrl, false);
309 JSONArray posts = json.getJSONArray("posts");
310 for (Object obj : posts) {
311 if (!(obj instanceof JSONObject))
312 continue;
313
314 JSONObject post = (JSONObject) obj;
315 if (!post.has("created_at"))
316 continue;
317
318 return post.getString("created_at");
319 }
320 } catch (Exception e) {
321 e.printStackTrace();
322 }
323 }
324
325 return "";
326 }
327
328 // no tags for pools
329 private List<String> getTags() {
330 List<String> tags = new ArrayList<String>();
331 if (isSearchOrSet(getSource())) {
332 String str = getTagsFromUrl(getSource());
333 for (String tag : str.split("\\+")) {
334 try {
335 tags.add(URLDecoder.decode(tag.trim(), "UTF-8").trim());
336 } catch (UnsupportedEncodingException e) {
337 }
338 }
339 }
340
341 return tags;
342 }
343
344 // returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
345 private String getTagsFromUrl(URL url) {
346 String tags = url == null ? "" : url.getQuery();
347 int pos = tags.indexOf("tags=");
348
349 if (pos >= 0) {
350 tags = tags.substring(pos).substring("tags=".length());
351 } else {
352 return "";
353 }
354
355 pos = tags.indexOf('&');
356 if (pos > 0) {
357 tags = tags.substring(0, pos);
358 }
359 pos = tags.indexOf('/');
360 if (pos > 0) {
361 tags = tags.substring(0, pos);
362 }
363
364 return tags;
365 }
366
367 private Image getCover() throws IOException {
368 Image image = null;
369 List<Entry<String, URL>> chapters = getChapters(null);
370 if (!chapters.isEmpty()) {
371 URL chap1Url = chapters.get(0).getValue();
372 String imgsChap1 = getChapterContent(chap1Url, 1, null);
373 if (!imgsChap1.isEmpty()) {
374 imgsChap1 = imgsChap1.split("]")[0].substring(1).trim();
375 image = bsImages.getImage(this, new URL(imgsChap1));
376 }
377 }
378
379 return image;
380 }
381
382 // always /posts.json/ url
383 private String getJsonUrl() {
384 String url = null;
385 if (isSearchOrSet(getSource())) {
386 url = getSource().toString().replace("/posts", "/posts.json");
387 }
388
389 if (isPool(getSource())) {
390 String poolNumber = getSource().getPath()
391 .substring("/pools/".length());
392 url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber;
393 }
394
395 if (url != null) {
396 // Note: one way to override the blacklist
397 String login = Instance.getInstance().getConfig()
398 .getString(Config.LOGIN_E621_LOGIN);
399 String apk = Instance.getInstance().getConfig()
400 .getString(Config.LOGIN_E621_APIKEY);
401
402 if (login != null && !login.isEmpty() && apk != null
403 && !apk.isEmpty()) {
404 url = String.format("%s&login=%s&api_key=%s&_client=%s", url,
405 login, apk, "fanfix-" + Version.getCurrentVersion());
406 }
407 }
408
409 return url;
410 }
411
412 // note: will be removed at getCanonicalUrl()
413 private boolean isSetOriginalUrl(URL originalUrl) {
414 return originalUrl.getPath().startsWith("/post_sets/");
415 }
416
417 private boolean isPool(URL url) {
418 return url.getPath().startsWith("/pools/")
419 || url.getPath().startsWith("/pool/show/");
420 }
421
422 // set will be renamed into search by canonical url
423 private boolean isSearchOrSet(URL url) {
424 return
425 // search:
426 (url.getPath().equals("/posts") && url.getQuery().contains("tags="))
427 // or set:
428 || isSetOriginalUrl(url);
429 }
430}