merge from master

[nikiroo-utils.git] / supported / E621.java

package be.nikiroo.fanfix.supported;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.helper.DataUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.bundles.Config;
import be.nikiroo.fanfix.data.MetaData;
import be.nikiroo.utils.Image;
import be.nikiroo.utils.Progress;
import be.nikiroo.utils.StringUtils;
import be.nikiroo.utils.Version;

/**
 * Support class for <a href="http://e621.net/">e621.net</a> and
 * <a href="http://e926.net/">e926.net</a>, a Furry website supporting comics,
 * including some of MLP.
 * <p>
 * <a href="http://e926.net/">e926.net</a> only shows the "clean" images and
 * comics, but it can be difficult to browse.
 * 
 * @author niki
 */
class E621 extends BasicSupport {
	@Override
	protected boolean supports(URL url) {
		String host = url.getHost();
		if (host.startsWith("www.")) {
			host = host.substring("www.".length());
		}

		return ("e621.net".equals(host) || "e926.net".equals(host))
				&& (isPool(url) || isSearchOrSet(url));
	}

	@Override
	protected boolean isHtml() {
		return true;
	}

	@Override
	protected MetaData getMeta() throws IOException {
		MetaData meta = new MetaData();

		meta.setTitle(getTitle());
		meta.setAuthor(getAuthor());
		meta.setDate(bsHelper.formatDate(getDate()));
		meta.setTags(getTags());
		meta.setSource(getType().getSourceName());
		meta.setUrl(getSource().toString());
		meta.setPublisher(getType().getSourceName());
		meta.setUuid(getSource().toString());
		meta.setLuid("");
		meta.setLang("en");
		meta.setSubject("Furry");
		meta.setType(getType().toString());
		meta.setImageDocument(true);
		meta.setCover(getCover());
		meta.setFakeCover(true);

		return meta;
	}

	@Override
	protected String getDesc() throws IOException {
		if (isSearchOrSet(getSource())) {
			StringBuilder builder = new StringBuilder();
			builder.append("A collection of images from ")
					.append(getSource().getHost()).append("\n") //
					.append("\tTime of creation: "
							+ StringUtils.fromTime(new Date().getTime()))
					.append("\n") //
					.append("\tTags: ");//
			for (String tag : getTags()) {
				builder.append("\t\t").append(tag);
			}

			return builder.toString();
		}

		if (isPool(getSource())) {
			Element el = getSourceNode().getElementById("description");
			if (el != null) {
				return el.text();
			}
		}

		return null;
	}

	@Override
	protected List<Entry<String, URL>> getChapters(Progress pg)
			throws IOException {
		int i = 1;
		String jsonUrl = getJsonUrl();
		if (jsonUrl != null) {
			for (i = 1; true; i++) {
				if (i > 1) {
					try {
						// The API does not accept more than 2 request per sec,
						// and asks us to limit at one per sec when possible
						Thread.sleep(1000);
					} catch (InterruptedException e) {
					}
				}

				try {
					JSONObject json = getJson(jsonUrl + "&page=" + i, false);
					if (!json.has("posts"))
						break;
					JSONArray posts = json.getJSONArray("posts");
					if (posts.isEmpty())
						break;
				} catch (Exception e) {
					e.printStackTrace();
				}
			}

			// The last page was empty:
			i--;
		}

		// The pages and images are in reverse order on /posts/
		List<Entry<String, URL>> chapters = new LinkedList<Entry<String, URL>>();
		for (int page = i; page > 0; page--) {
			chapters.add(new AbstractMap.SimpleEntry<String, URL>(
					"Page " + Integer.toString(i - page + 1),
					new URL(jsonUrl + "&page=" + page)));
		}

		return chapters;
	}

	@Override
	protected String getChapterContent(URL chapUrl, int number, Progress pg)
			throws IOException {
		StringBuilder builder = new StringBuilder();

		JSONObject json = getJson(chapUrl, false);
		JSONArray postsArr = json.getJSONArray("posts");

		// The pages and images are in reverse order on /posts/
		List<JSONObject> posts = new ArrayList<JSONObject>(postsArr.length());
		for (int i = postsArr.length() - 1; i >= 0; i--) {
			Object o = postsArr.get(i);
			if (o instanceof JSONObject)
				posts.add((JSONObject) o);
		}

		for (JSONObject post : posts) {
			if (!post.has("file"))
				continue;
			JSONObject file = post.getJSONObject("file");
			if (!file.has("url"))
				continue;

			try {
				String url = file.getString("url");
				builder.append("[");
				builder.append(url);
				builder.append("]<br/>");
			} catch (JSONException e) {
				// Can be NULL if filtered
				// When the value is NULL, we get an exception
				// but the "has" method still returns true
				Instance.getInstance().getTraceHandler()
						.error("Cannot get image for chapter " + number + " of "
								+ getSource());
			}
		}

		return builder.toString();
	}

	@Override
	protected URL getCanonicalUrl(URL source) {
		// Convert search-pools into proper pools
		if (source.getPath().equals("/posts") && source.getQuery() != null
				&& source.getQuery().startsWith("tags=pool%3A")) {
			String poolNumber = source.getQuery()
					.substring("tags=pool%3A".length());
			try {
				Integer.parseInt(poolNumber);
				String base = source.getProtocol() + "://" + source.getHost();
				if (source.getPort() != -1) {
					base = base + ":" + source.getPort();
				}
				source = new URL(base + "/pools/" + poolNumber);
			} catch (NumberFormatException e) {
				// Not a simple pool, skip
			} catch (MalformedURLException e) {
				// Cannot happen
			}
		}

		if (isSetOriginalUrl(source)) {
			try {
				Document doc = DataUtil.load(Instance.getInstance().getCache()
						.open(source, this, false), "UTF-8", source.toString());
				for (Element shortname : doc
						.getElementsByClass("set-shortname")) {
					for (Element el : shortname.getElementsByTag("a")) {
						if (!el.attr("href").isEmpty())
							return new URL(el.absUrl("href"));
					}
				}
			} catch (IOException e) {
				Instance.getInstance().getTraceHandler().error(e);
			}
		}

		if (isPool(source)) {
			try {
				return new URL(
						source.toString().replace("/pool/show/", "/pools/"));
			} catch (MalformedURLException e) {
			}
		}

		return super.getCanonicalUrl(source);
	}

	private String getTitle() {
		String title = "";

		Element el = getSourceNode().getElementsByTag("title").first();
		if (el != null) {
			title = el.text().trim();
		}

		for (String s : new String[] { "e621", "-", "e621", "Pool", "-" }) {
			if (title.startsWith(s)) {
				title = title.substring(s.length()).trim();
			}
			if (title.endsWith(s)) {
				title = title.substring(0, title.length() - s.length()).trim();
			}
		}

		if (isSearchOrSet(getSource())) {
			title = title.isEmpty() ? "e621" : "[e621] " + title;
		}

		return title;
	}

	private String getAuthor() {
		List<String> list = new ArrayList<String>();
		String jsonUrl = getJsonUrl();
		if (jsonUrl != null) {
			try {
				JSONObject json = getJson(jsonUrl, false);
				JSONArray posts = json.getJSONArray("posts");
				for (Object obj : posts) {
					if (!(obj instanceof JSONObject))
						continue;

					JSONObject post = (JSONObject) obj;
					if (!post.has("tags"))
						continue;

					JSONObject tags = post.getJSONObject("tags");
					if (!tags.has("artist"))
						continue;

					JSONArray artists = tags.getJSONArray("artist");
					for (Object artist : artists) {
						if (list.contains(artist.toString()))
							continue;

						list.add(artist.toString());
					}
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}

		StringBuilder builder = new StringBuilder();
		for (String artist : list) {
			if (builder.length() > 0) {
				builder.append(", ");
			}
			builder.append(artist);
		}

		return builder.toString();
	}

	private String getDate() {
		String jsonUrl = getJsonUrl();
		if (jsonUrl != null) {
			try {
				JSONObject json = getJson(jsonUrl, false);
				JSONArray posts = json.getJSONArray("posts");
				for (Object obj : posts) {
					if (!(obj instanceof JSONObject))
						continue;

					JSONObject post = (JSONObject) obj;
					if (!post.has("created_at"))
						continue;

					return post.getString("created_at");
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}

		return "";
	}

	// no tags for pools
	private List<String> getTags() {
		List<String> tags = new ArrayList<String>();
		if (isSearchOrSet(getSource())) {
			String str = getTagsFromUrl(getSource());
			for (String tag : str.split("\\+")) {
				try {
					tags.add(URLDecoder.decode(tag.trim(), "UTF-8").trim());
				} catch (UnsupportedEncodingException e) {
				}
			}
		}

		return tags;
	}

	// returns "xxx+ddd+ggg" if "tags=xxx+ddd+ggg" was present in the query
	private String getTagsFromUrl(URL url) {
		String tags = url == null ? "" : url.getQuery();
		int pos = tags.indexOf("tags=");

		if (pos >= 0) {
			tags = tags.substring(pos).substring("tags=".length());
		} else {
			return "";
		}

		pos = tags.indexOf('&');
		if (pos > 0) {
			tags = tags.substring(0, pos);
		}
		pos = tags.indexOf('/');
		if (pos > 0) {
			tags = tags.substring(0, pos);
		}

		return tags;
	}

	private Image getCover() throws IOException {
		Image image = null;
		List<Entry<String, URL>> chapters = getChapters(null);
		if (!chapters.isEmpty()) {
			URL chap1Url = chapters.get(0).getValue();
			String imgsChap1 = getChapterContent(chap1Url, 1, null);
			if (!imgsChap1.isEmpty()) {
				imgsChap1 = imgsChap1.split("]")[0].substring(1).trim();
				image = bsImages.getImage(this, new URL(imgsChap1));
			}
		}

		return image;
	}

	// always /posts.json/ url
	private String getJsonUrl() {
		String url = null;
		if (isSearchOrSet(getSource())) {
			url = getSource().toString().replace("/posts", "/posts.json");
		}

		if (isPool(getSource())) {
			String poolNumber = getSource().getPath()
					.substring("/pools/".length());
			url = "https://e621.net/posts.json" + "?tags=pool%3A" + poolNumber;
		}

		if (url != null) {
			// Note: one way to override the blacklist
			String login = Instance.getInstance().getConfig()
					.getString(Config.LOGIN_E621_LOGIN);
			String apk = Instance.getInstance().getConfig()
					.getString(Config.LOGIN_E621_APIKEY);

			if (login != null && !login.isEmpty() && apk != null
					&& !apk.isEmpty()) {
				url = String.format("%s&login=%s&api_key=%s&_client=%s", url,
						login, apk, "fanfix-" + Version.getCurrentVersion());
			}
		}

		return url;
	}

	// note: will be removed at getCanonicalUrl()
	private boolean isSetOriginalUrl(URL originalUrl) {
		return originalUrl.getPath().startsWith("/post_sets/");
	}

	private boolean isPool(URL url) {
		return url.getPath().startsWith("/pools/")
				|| url.getPath().startsWith("/pool/show/");
	}

	// set will be renamed into search by canonical url
	private boolean isSearchOrSet(URL url) {
		return
		// search:
		(url.getPath().equals("/posts") && url.getQuery().contains("tags="))
				// or set:
				|| isSetOriginalUrl(url);
	}
}