X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Ffanfix%2Fsupported%2FE621.java;h=f1660e18cb34a9da6373ac03e6918ccc630ff25f;hb=d66deb8d8b30cff6b54db352eef34a3508939f84;hp=05d122e68b89efc71587fda628f9b17e720736d7;hpb=406447a4c017f2a4592429a67d546a6984460617;p=nikiroo-utils.git

diff --git a/src/be/nikiroo/fanfix/supported/E621.java b/src/be/nikiroo/fanfix/supported/E621.java
index 05d122e..f1660e1 100644
--- a/src/be/nikiroo/fanfix/supported/E621.java
+++ b/src/be/nikiroo/fanfix/supported/E621.java
@@ -1,24 +1,33 @@
 package be.nikiroo.fanfix.supported;
 
-import java.awt.image.BufferedImage;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.MalformedURLException;
 import java.net.URL;
+import java.net.URLDecoder;
+import java.util.AbstractMap;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map.Entry;
-import java.util.Scanner;
+
+import org.jsoup.helper.DataUtil;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
 
 import be.nikiroo.fanfix.Instance;
-import be.nikiroo.fanfix.data.Chapter;
 import be.nikiroo.fanfix.data.MetaData;
-import be.nikiroo.fanfix.data.Story;
+import be.nikiroo.utils.IOUtils;
+import be.nikiroo.utils.Image;
 import be.nikiroo.utils.Progress;
 import be.nikiroo.utils.StringUtils;
 
 /**
- * Support class for <a href="http://e621.net/">e621.net</a> and <a
- * href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
+ * Support class for <a href="http://e621.net/">e621.net</a> and
+ * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
  * including some of MLP.
  * <p>
  * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
@@ -28,253 +37,277 @@ import be.nikiroo.utils.StringUtils;
  */
 class E621 extends BasicSupport {
 	@Override
-	public String getSourceName() {
-		return "e621.net";
+	protected boolean supports(URL url) {
+		String host = url.getHost();
+		if (host.startsWith("www.")) {
+			host = host.substring("www.".length());
+		}
+
+		return ("e621.net".equals(host) || "e926.net".equals(host)) && (isPool(url) || isSearchOrSet(url));
+	}
+
+	@Override
+	protected boolean isHtml() {
+		return true;
 	}
 
 	@Override
-	protected MetaData getMeta(URL source, InputStream in) throws IOException {
+	protected MetaData getMeta() throws IOException {
 		MetaData meta = new MetaData();
 
-		meta.setTitle(getTitle(reset(in)));
-		meta.setAuthor(getAuthor(source, reset(in)));
+		meta.setTitle(getTitle());
+		meta.setAuthor(getAuthor());
 		meta.setDate("");
-		meta.setTags(new ArrayList<String>()); // TODDO ???
-		meta.setSource(getSourceName());
-		meta.setUrl(source.toString());
-		meta.setPublisher(getSourceName());
-		meta.setUuid(source.toString());
+		meta.setTags(getTags());
+		meta.setSource(getType().getSourceName());
+		meta.setUrl(getSource().toString());
+		meta.setPublisher(getType().getSourceName());
+		meta.setUuid(getSource().toString());
 		meta.setLuid("");
-		meta.setLang("EN");
+		meta.setLang("en");
 		meta.setSubject("Furry");
 		meta.setType(getType().toString());
 		meta.setImageDocument(true);
-		meta.setCover(getCover(source));
+		meta.setCover(getCover());
 		meta.setFakeCover(true);
 
 		return meta;
 	}
 
 	@Override
-	public Story process(URL url, Progress pg) throws IOException {
-		// There is no chapters on e621, just pagination...
-		Story story = super.process(url, pg);
+	protected String getDesc() throws IOException {
+		if (isSearchOrSet(getSource())) {
+			StringBuilder builder = new StringBuilder();
+			builder.append("A collection of images from ").append(getSource().getHost()).append("\n") //
+					.append("\tTime of creation: " + StringUtils.fromTime(new Date().getTime())).append("\n") //
+					.append("\tTags: ");//
+			for (String tag : getTags()) {
+				builder.append("\t\t").append(tag);
+			}
 
-		Chapter only = new Chapter(1, null);
-		for (Chapter chap : story) {
-			only.getParagraphs().addAll(chap.getParagraphs());
+			return builder.toString();
 		}
 
-		story.getChapters().clear();
-		story.getChapters().add(only);
+		if (isPool(getSource())) {
+			Element el = getSourceNode().getElementById("description");
+			if (el != null) {
+				return el.text();
+			}
+		}
 
-		return story;
+		return null;
 	}
 
 	@Override
-	protected boolean supports(URL url) {
-		String host = url.getHost();
-		if (host.startsWith("www.")) {
-			host = host.substring("www.".length());
+	protected List<Entry<String, URL>> getChapters(Progress pg) throws IOException {
+		if (isPool(getSource())) {
+			String baseUrl = "https://e621.net/" + getSource().getPath() + "?page=";
+			return getChapters(getSource(), pg, baseUrl, "");
+		} else if (isSearchOrSet(getSource())) {
+			String baseUrl = "https://e621.net/posts/?page=";
+			String search = "&tags=" + getTagsFromUrl(getSource());
+			return getChapters(getSource(), pg, baseUrl, search);
 		}
 
-		return ("e621.net".equals(host) || "e926.net".equals(host))
-				&& url.getPath().startsWith("/pool/");
+		return new LinkedList<Entry<String, URL>>();
 	}
 
-	@Override
-	protected boolean isHtml() {
-		return true;
-	}
+	private List<Entry<String, URL>> getChapters(URL source, Progress pg, String baseUrl, String parameters)
+			throws IOException {
+		List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
 
-	private BufferedImage getCover(URL source) throws IOException {
-		InputStream in = Instance.getCache().open(source, this, true);
-		String images = getChapterContent(new URL(source.toString() + "?page="
-				+ 1), in, 1);
-		if (!images.isEmpty()) {
-			int pos = images.indexOf("<br/>");
-			if (pos >= 0) {
-				images = images.substring(1, pos - 1);
-				return getImage(this, null, images);
-			}
+		if (source.getHost().contains("e926")) {
+			baseUrl = baseUrl.replace("e621", "e926");
 		}
 
-		return null;
-	}
-
-	private String getAuthor(URL source, InputStream in) throws IOException {
-		String author = getLine(in, "href=\"/post/show/", 0);
-		if (author != null) {
-			String key = "href=\"";
-			int pos = author.indexOf(key);
-			if (pos >= 0) {
-				author = author.substring(pos + key.length());
-				pos = author.indexOf("\"");
-				if (pos >= 0) {
-					author = author.substring(0, pos - 1);
-					String page = source.getProtocol() + "://"
-							+ source.getHost() + author;
-					try {
-						InputStream pageIn = Instance.getCache().open(
-								new URL(page), this, false);
-						try {
-							key = "class=\"tag-type-artist\"";
-							author = getLine(pageIn, key, 0);
-							if (author != null) {
-								pos = author.indexOf("<a href=\"");
-								if (pos >= 0) {
-									author = author.substring(pos);
-									pos = author.indexOf("</a>");
-									if (pos >= 0) {
-										author = author.substring(0, pos);
-										return StringUtils.unhtml(author);
-									}
-								}
-							}
-						} finally {
-							pageIn.close();
-						}
-					} catch (Exception e) {
-						// No author found
+		for (int i = 1; true; i++) {
+			URL url = new URL(baseUrl + i + parameters);
+			try {
+				InputStream pageI = Instance.getInstance().getCache().open(url, this, false);
+				try {
+					if (IOUtils.readSmallStream(pageI).contains("Nobody here but us chickens!")) {
+						break;
 					}
+					urls.add(new AbstractMap.SimpleEntry<String, URL>("Page " + Integer.toString(i), url));
+				} finally {
+					pageI.close();
 				}
+			} catch (Exception e) {
+				break;
 			}
 		}
 
-		return null;
+		// They are sorted in reverse order on the website
+		Collections.reverse(urls);
+		return urls;
 	}
 
-	private String getTitle(InputStream in) throws IOException {
-		String title = getLine(in, "<title>", 0);
-		if (title != null) {
-			int pos = title.indexOf('>');
-			if (pos >= 0) {
-				title = title.substring(pos + 1);
-				pos = title.indexOf('<');
-				if (pos >= 0) {
-					title = title.substring(0, pos);
+	@Override
+	protected String getChapterContent(URL chapUrl, int number, Progress pg) throws IOException {
+		StringBuilder builder = new StringBuilder();
+		Document chapterNode = loadDocument(chapUrl);
+		for (Element el : chapterNode.getElementsByTag("article")) {
+			builder.append("[");
+			builder.append(el.attr("data-file-url"));
+			builder.append("]<br/>");
+		}
+
+		return builder.toString();
+	}
+
+	@Override
+	protected URL getCanonicalUrl(URL source) {
+		if (isSetOriginalUrl(source)) {
+			try {
+				Document doc = DataUtil.load(Instance.getInstance().getCache().open(source, this, false), "UTF-8", source.toString());
+				for (Element shortname : doc.getElementsByClass("set-shortname")) {
+					for (Element el : shortname.getElementsByTag("a")) {
+						if (!el.attr("href").isEmpty())
+							return new URL(el.absUrl("href"));
+					}
 				}
+			} catch (IOException e) {
+				Instance.getInstance().getTraceHandler().error(e);
 			}
+		}
 
-			if (title.startsWith("Pool:")) {
-				title = title.substring("Pool:".length());
+		if (isPool(source)) {
+			try {
+				return new URL(source.toString().replace("/pool/show/", "/pools/"));
+			} catch (MalformedURLException e) {
 			}
-
-			title = StringUtils.unhtml(title).trim();
 		}
 
-		return title;
+		return super.getCanonicalUrl(source);
 	}
 
-	@Override
-	protected String getDesc(URL source, InputStream in) throws IOException {
-		String desc = getLine(in, "margin-bottom: 2em;", 0);
+	// returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
+	private String getTagsFromUrl(URL url) {
+		String tags = url == null ? "" : url.getQuery();
+		int pos = tags.indexOf("tags=");
 
-		if (desc != null) {
-			StringBuilder builder = new StringBuilder();
+		if (pos >= 0) {
+			tags = tags.substring(pos).substring("tags=".length());
+		} else {
+			return "";
+		}
 
-			boolean inTags = false;
-			for (char car : desc.toCharArray()) {
-				if ((inTags && car == '>') || (!inTags && car == '<')) {
-					inTags = !inTags;
-				}
+		pos = tags.indexOf('&');
+		if (pos > 0) {
+			tags = tags.substring(0, pos);
+		}
+		pos = tags.indexOf('/');
+		if (pos > 0) {
+			tags = tags.substring(0, pos);
+		}
 
-				if (inTags) {
-					builder.append(car);
-				}
+		return tags;
+	}
+
+	private String getTitle() {
+		String title = "";
+
+		Element el = getSourceNode().getElementsByTag("title").first();
+		if (el != null) {
+			title = el.text().trim();
+		}
+
+		for (String s : new String[] { "e621", "-", "e621" }) {
+			if (title.startsWith(s)) {
+				title = title.substring(s.length()).trim();
+			}
+			if (title.endsWith(s)) {
+				title = title.substring(0, title.length() - s.length()).trim();
 			}
 
-			return builder.toString().trim();
 		}
 
-		return null;
+		if (isSearchOrSet(getSource())) {
+			title = title.isEmpty() ? "e621" : "[e621] " + title;
+		}
+		return title;
 	}
 
-	@Override
-	protected List<Entry<String, URL>> getChapters(URL source, InputStream in)
-			throws IOException {
-		List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
-		int last = 1; // no pool/show when only one page
-
-		@SuppressWarnings("resource")
-		Scanner scan = new Scanner(in, "UTF-8");
-		scan.useDelimiter("\\n");
-		while (scan.hasNext()) {
-			String line = scan.next();
-			for (int pos = line.indexOf(source.getPath()); pos >= 0; pos = line
-					.indexOf(source.getPath(), pos + source.getPath().length())) {
-				int equalPos = line.indexOf("=", pos);
-				int quotePos = line.indexOf("\"", pos);
-				if (equalPos >= 0 && quotePos > equalPos) {
-					String snum = line.substring(equalPos + 1, quotePos);
-					try {
-						int num = Integer.parseInt(snum);
-						if (num > last) {
-							last = num;
-						}
-					} catch (NumberFormatException e) {
+	private String getAuthor() throws IOException {
+		StringBuilder builder = new StringBuilder();
+
+		if (isSearchOrSet(getSource())) {
+			for (Element el : getSourceNode().getElementsByClass("search-tag")) {
+				if (el.attr("itemprop").equals("author")) {
+					if (builder.length() > 0) {
+						builder.append(", ");
 					}
+					builder.append(el.text().trim());
 				}
 			}
 		}
 
-		for (int i = 1; i <= last; i++) {
-			final String key = Integer.toString(i);
-			final URL value = new URL(source.toString() + "?page=" + i);
-			urls.add(new Entry<String, URL>() {
-				public URL setValue(URL value) {
-					return null;
-				}
+		if (isPool(getSource())) {
+			String desc = getDesc();
+			String descL = desc.toLowerCase();
 
-				public URL getValue() {
-					return value;
-				}
+			if (descL.startsWith("by:") || descL.startsWith("by ")) {
+				desc = desc.substring(3).trim();
+				desc = desc.split("\n")[0];
 
-				public String getKey() {
-					return key;
+				String tab[] = desc.split(" ");
+				for (int i = 0; i < Math.min(tab.length, 5); i++) {
+					if (tab[i].startsWith("http"))
+						break;
+					builder.append(" ").append(tab[i]);
 				}
-			});
+			}
 		}
 
-		return urls;
+		return builder.toString();
 	}
 
-	@Override
-	protected String getChapterContent(URL source, InputStream in, int number)
-			throws IOException {
-		StringBuilder builder = new StringBuilder();
-		String staticSite = "https://static1.e621.net";
-		if (source.getHost().contains("e926")) {
-			staticSite = staticSite.replace("e621", "e926");
+	// no tags for pools
+	private List<String> getTags() {
+		List<String> tags = new ArrayList<String>();
+		if (isSearchOrSet(getSource())) {
+			String str = getTagsFromUrl(getSource());
+			for (String tag : str.split("\\+")) {
+				try {
+					tags.add(URLDecoder.decode(tag.trim(), "UTF-8").trim());
+				} catch (UnsupportedEncodingException e) {
+				}
+			}
 		}
 
-		String key = staticSite + "/data/preview/";
-
-		@SuppressWarnings("resource")
-		Scanner scan = new Scanner(in, "UTF-8");
-		scan.useDelimiter("\\n");
-		while (scan.hasNext()) {
-			String line = scan.next();
-			if (line.contains("class=\"preview")) {
-				for (int pos = line.indexOf(key); pos >= 0; pos = line.indexOf(
-						key, pos + key.length())) {
-					int endPos = line.indexOf("\"", pos);
-					if (endPos >= 0) {
-						String id = line.substring(pos + key.length(), endPos);
-						id = staticSite + "/data/" + id;
-
-						int dotPos = id.lastIndexOf(".");
-						if (dotPos >= 0) {
-							id = id.substring(0, dotPos);
-							builder.append("[");
-							builder.append(id);
-							builder.append("]<br/>");
-						}
-					}
-				}
+		return tags;
+	}
+
+	private Image getCover() throws IOException {
+		Image image = null;
+		List<Entry<String, URL>> chapters = getChapters(null);
+		if (!chapters.isEmpty()) {
+			URL chap1Url = chapters.get(0).getValue();
+			String imgsChap1 = getChapterContent(chap1Url, 1, null);
+			if (!imgsChap1.isEmpty()) {
+				imgsChap1 = imgsChap1.split("]")[0].substring(1).trim();
+				image = bsImages.getImage(this, new URL(imgsChap1));
 			}
 		}
 
-		return builder.toString();
+		return image;
+	}
+
+	// note: will be removed at getCanonicalUrl()
+	private boolean isSetOriginalUrl(URL originalUrl) {
+		return originalUrl.getPath().startsWith("/post_sets/");
+	}
+
+	private boolean isPool(URL url) {
+		return url.getPath().startsWith("/pools/") || url.getPath().startsWith("/pool/show/");
+	}
+
+	// set will be renamed into search by canonical url
+	private boolean isSearchOrSet(URL url) {
+		return
+		// search:
+		(url.getPath().equals("/posts") && url.getQuery().contains("tags="))
+				// or set:
+				|| isSetOriginalUrl(url);
 	}
 }