Update nikiroo-utils, update Library
[fanfix.git] / src / be / nikiroo / fanfix / supported / E621.java
1 package be.nikiroo.fanfix.supported;
2
3 import java.awt.image.BufferedImage;
4 import java.io.IOException;
5 import java.io.InputStream;
6 import java.net.URL;
7 import java.util.ArrayList;
8 import java.util.List;
9 import java.util.Map.Entry;
10 import java.util.Scanner;
11
12 import be.nikiroo.fanfix.Instance;
13 import be.nikiroo.fanfix.data.Chapter;
14 import be.nikiroo.fanfix.data.MetaData;
15 import be.nikiroo.fanfix.data.Story;
16 import be.nikiroo.utils.Progress;
17 import be.nikiroo.utils.StringUtils;
18
19 /**
20 * Support class for <a href="http://e621.net/">e621.net</a> and <a
21 * href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
22 * including some of MLP.
23 * <p>
24 * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
25 * comics, but it can be difficult to browse.
26 *
27 * @author niki
28 */
29 class E621 extends BasicSupport {
30 @Override
31 public String getSourceName() {
32 return "e621.net";
33 }
34
35 @Override
36 protected MetaData getMeta(URL source, InputStream in) throws IOException {
37 MetaData meta = new MetaData();
38
39 meta.setTitle(getTitle(reset(in)));
40 meta.setAuthor(getAuthor(source, reset(in)));
41 meta.setDate("");
42 meta.setTags(new ArrayList<String>()); // TODDO ???
43 meta.setSource(getSourceName());
44 meta.setUrl(source.toString());
45 meta.setPublisher(getSourceName());
46 meta.setUuid(source.toString());
47 meta.setLuid("");
48 meta.setLang("EN");
49 meta.setSubject("Furry");
50 meta.setType(getType().toString());
51 meta.setImageDocument(true);
52 meta.setCover(getCover(source));
53 meta.setFakeCover(true);
54
55 return meta;
56 }
57
58 @Override
59 public Story process(URL url, Progress pg) throws IOException {
60 // There is no chapters on e621, just pagination...
61 Story story = super.process(url, pg);
62
63 Chapter only = new Chapter(1, null);
64 for (Chapter chap : story) {
65 only.getParagraphs().addAll(chap.getParagraphs());
66 }
67
68 story.getChapters().clear();
69 story.getChapters().add(only);
70
71 return story;
72 }
73
74 @Override
75 protected boolean supports(URL url) {
76 String host = url.getHost();
77 if (host.startsWith("www.")) {
78 host = host.substring("www.".length());
79 }
80
81 return ("e621.net".equals(host) || "e926.net".equals(host))
82 && url.getPath().startsWith("/pool/");
83 }
84
85 @Override
86 protected boolean isHtml() {
87 return true;
88 }
89
90 private BufferedImage getCover(URL source) throws IOException {
91 InputStream in = Instance.getCache().open(source, this, true);
92 String images = getChapterContent(new URL(source.toString() + "?page="
93 + 1), in, 1, null);
94 if (!images.isEmpty()) {
95 int pos = images.indexOf("<br/>");
96 if (pos >= 0) {
97 images = images.substring(1, pos - 1);
98 return getImage(this, null, images);
99 }
100 }
101
102 return null;
103 }
104
105 private String getAuthor(URL source, InputStream in) throws IOException {
106 String author = getLine(in, "href=\"/post/show/", 0);
107 if (author != null) {
108 String key = "href=\"";
109 int pos = author.indexOf(key);
110 if (pos >= 0) {
111 author = author.substring(pos + key.length());
112 pos = author.indexOf("\"");
113 if (pos >= 0) {
114 author = author.substring(0, pos - 1);
115 String page = source.getProtocol() + "://"
116 + source.getHost() + author;
117 try {
118 InputStream pageIn = Instance.getCache().open(
119 new URL(page), this, false);
120 try {
121 key = "class=\"tag-type-artist\"";
122 author = getLine(pageIn, key, 0);
123 if (author != null) {
124 pos = author.indexOf("<a href=\"");
125 if (pos >= 0) {
126 author = author.substring(pos);
127 pos = author.indexOf("</a>");
128 if (pos >= 0) {
129 author = author.substring(0, pos);
130 return StringUtils.unhtml(author);
131 }
132 }
133 }
134 } finally {
135 pageIn.close();
136 }
137 } catch (Exception e) {
138 // No author found
139 }
140 }
141 }
142 }
143
144 return null;
145 }
146
147 private String getTitle(InputStream in) throws IOException {
148 String title = getLine(in, "<title>", 0);
149 if (title != null) {
150 int pos = title.indexOf('>');
151 if (pos >= 0) {
152 title = title.substring(pos + 1);
153 pos = title.indexOf('<');
154 if (pos >= 0) {
155 title = title.substring(0, pos);
156 }
157 }
158
159 if (title.startsWith("Pool:")) {
160 title = title.substring("Pool:".length());
161 }
162
163 title = StringUtils.unhtml(title).trim();
164 }
165
166 return title;
167 }
168
169 @Override
170 protected String getDesc(URL source, InputStream in) throws IOException {
171 String desc = getLine(in, "margin-bottom: 2em;", 0);
172
173 if (desc != null) {
174 StringBuilder builder = new StringBuilder();
175
176 boolean inTags = false;
177 for (char car : desc.toCharArray()) {
178 if ((inTags && car == '>') || (!inTags && car == '<')) {
179 inTags = !inTags;
180 }
181
182 if (inTags) {
183 builder.append(car);
184 }
185 }
186
187 return builder.toString().trim();
188 }
189
190 return null;
191 }
192
193 @Override
194 protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
195 Progress pg) throws IOException {
196 List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
197 int last = 1; // no pool/show when only one page
198
199 @SuppressWarnings("resource")
200 Scanner scan = new Scanner(in, "UTF-8");
201 scan.useDelimiter("\\n");
202 while (scan.hasNext()) {
203 String line = scan.next();
204 for (int pos = line.indexOf(source.getPath()); pos >= 0; pos = line
205 .indexOf(source.getPath(), pos + source.getPath().length())) {
206 int equalPos = line.indexOf("=", pos);
207 int quotePos = line.indexOf("\"", pos);
208 if (equalPos >= 0 && quotePos > equalPos) {
209 String snum = line.substring(equalPos + 1, quotePos);
210 try {
211 int num = Integer.parseInt(snum);
212 if (num > last) {
213 last = num;
214 }
215 } catch (NumberFormatException e) {
216 }
217 }
218 }
219 }
220
221 for (int i = 1; i <= last; i++) {
222 final String key = Integer.toString(i);
223 final URL value = new URL(source.toString() + "?page=" + i);
224 urls.add(new Entry<String, URL>() {
225 public URL setValue(URL value) {
226 return null;
227 }
228
229 public URL getValue() {
230 return value;
231 }
232
233 public String getKey() {
234 return key;
235 }
236 });
237 }
238
239 return urls;
240 }
241
242 @Override
243 protected String getChapterContent(URL source, InputStream in, int number,
244 Progress pg) throws IOException {
245 StringBuilder builder = new StringBuilder();
246 String staticSite = "https://static1.e621.net";
247 if (source.getHost().contains("e926")) {
248 staticSite = staticSite.replace("e621", "e926");
249 }
250
251 String key = staticSite + "/data/preview/";
252
253 @SuppressWarnings("resource")
254 Scanner scan = new Scanner(in, "UTF-8");
255 scan.useDelimiter("\\n");
256 while (scan.hasNext()) {
257 String line = scan.next();
258 if (line.contains("class=\"preview")) {
259 for (int pos = line.indexOf(key); pos >= 0; pos = line.indexOf(
260 key, pos + key.length())) {
261 int endPos = line.indexOf("\"", pos);
262 if (endPos >= 0) {
263 String id = line.substring(pos + key.length(), endPos);
264 id = staticSite + "/data/" + id;
265
266 int dotPos = id.lastIndexOf(".");
267 if (dotPos >= 0) {
268 id = id.substring(0, dotPos);
269 builder.append("[");
270 builder.append(id);
271 builder.append("]<br/>");
272 }
273 }
274 }
275 }
276 }
277
278 return builder.toString();
279 }
280 }