[nikiroo-utils.git] / supported / Fanfiction.java

package be.nikiroo.fanfix.supported;

import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map.Entry;
import java.util.Scanner;

import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.bundles.Config;
import be.nikiroo.fanfix.data.MetaData;
import be.nikiroo.utils.Image;
import be.nikiroo.utils.Progress;
import be.nikiroo.utils.StringUtils;

/**
 * Support class for <a href="http://www.fanfiction.net/">Faniction.net</a>
 * stories, a website dedicated to fanfictions of many, many different
 * universes, from TV shows to novels to games.
 * 
 * @author niki
 */
class Fanfiction extends BasicSupport_Deprecated {
	@Override
	protected boolean isHtml() {
		return true;
	}

	@Override
	protected MetaData getMeta(URL source, InputStream in) throws IOException {
		MetaData meta = new MetaData();

		meta.setTitle(getTitle(reset(in)));
		meta.setAuthor(getAuthor(reset(in)));
		meta.setDate(getDate(reset(in)));
		meta.setTags(getTags(reset(in)));
		meta.setSource(getType().getSourceName());
		meta.setUrl(source.toString());
		meta.setPublisher(getType().getSourceName());
		meta.setUuid(source.toString());
		meta.setLuid("");
		meta.setLang("en"); // TODO!
		meta.setSubject(getSubject(reset(in)));
		meta.setType(getType().toString());
		meta.setImageDocument(false);
		meta.setCover(getCover(source, reset(in)));

		return meta;
	}

	private String getSubject(InputStream in) {
		String line = getLine(in, "id=pre_story_links", 0);
		if (line != null) {
			int pos = line.lastIndexOf('"');
			if (pos >= 1) {
				line = line.substring(pos + 1);
				pos = line.indexOf('<');
				if (pos >= 0) {
					return StringUtils.unhtml(line.substring(0, pos)).trim();
				}
			}
		}

		return null;
	}

	private List<String> getTags(InputStream in) {
		List<String> tags = new ArrayList<String>();

		String key = "title=\"Send Private Message\"";
		String line = getLine(in, key, 2);
		if (line != null) {
			key = "Rated:";
			int pos = line.indexOf(key);
			if (pos >= 0) {
				line = line.substring(pos + key.length());
				key = "Chapters:";
				pos = line.indexOf(key);
				if (pos >= 0) {
					line = line.substring(0, pos);
					line = StringUtils.unhtml(line).trim();
					if (line.endsWith("-")) {
						line = line.substring(0, line.length() - 1);
					}

					for (String tag : line.split("-")) {
						tags.add(StringUtils.unhtml(tag).trim());
					}
				}
			}
		}

		return tags;
	}

	private String getTitle(InputStream in) {
		int i = 0;
		@SuppressWarnings("resource")
		Scanner scan = new Scanner(in, "UTF-8");
		scan.useDelimiter("\\n");
		while (scan.hasNext()) {
			String line = scan.next();
			if (line.contains("xcontrast_txt")) {
				if ((++i) == 2) {
					line = StringUtils.unhtml(line).trim();
					if (line.startsWith("Follow/Fav")) {
						line = line.substring("Follow/Fav".length()).trim();
					}

					return StringUtils.unhtml(line).trim();
				}
			}
		}

		return "";
	}

	private String getAuthor(InputStream in) {
		String author = null;

		int i = 0;
		@SuppressWarnings("resource")
		Scanner scan = new Scanner(in, "UTF-8");
		scan.useDelimiter("\\n");
		while (scan.hasNext()) {
			String line = scan.next();
			if (line.contains("xcontrast_txt")) {
				if ((++i) == 3) {
					author = StringUtils.unhtml(line).trim();
					break;
				}
			}
		}

		return BasicSupportHelper.fixAuthor(author);
	}

	private String getDate(InputStream in) {
		String key = "Published: <span data-xutime='";
		String line = getLine(in, key, 0);
		if (line != null) {
			int pos = line.indexOf(key);
			if (pos >= 0) {
				line = line.substring(pos + key.length());
				pos = line.indexOf('\'');
				if (pos >= 0) {
					line = line.substring(0, pos).trim();
					try {
						SimpleDateFormat sdf = new SimpleDateFormat(
								"YYYY-MM-dd");
						return sdf
								.format(new Date(1000 * Long.parseLong(line)));
					} catch (NumberFormatException e) {
						Instance.getTraceHandler().error(
								new IOException(
										"Cannot convert publication date: "
												+ line, e));
					}
				}
			}
		}

		return null;
	}

	@Override
	protected String getDesc(URL source, InputStream in) {
		return getLine(in, "title=\"Send Private Message\"", 1);
	}

	private Image getCover(URL url, InputStream in) {
		String key = "class='cimage";
		String line = getLine(in, key, 0);
		if (line != null) {
			int pos = line.indexOf(key);
			if (pos >= 0) {
				line = line.substring(pos + key.length());
				key = "src='";
				pos = line.indexOf(key);
				if (pos >= 0) {
					line = line.substring(pos + key.length());
					pos = line.indexOf('\'');
					if (pos >= 0) {
						line = line.substring(0, pos);
						if (line.startsWith("//")) {
							line = url.getProtocol() + "://"
									+ line.substring(2);
						} else if (line.startsWith("//")) {
							line = url.getProtocol() + "://" + url.getHost()
									+ "/" + line.substring(1);
						} else {
							line = url.getProtocol() + "://" + url.getHost()
									+ "/" + url.getPath() + "/" + line;
						}

						return getImage(this, null, line);
					}
				}
			}
		}

		return null;
	}

	@Override
	protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
			Progress pg) {
		List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();

		String base = source.toString();
		int pos = base.lastIndexOf('/');
		String suffix = base.substring(pos); // including '/' at start
		base = base.substring(0, pos);
		if (base.endsWith("/1")) {
			base = base.substring(0, base.length() - 1); // including '/' at end
		}

		String line = getLine(in, "id=chap_select", 0);
		String key = "<option  value=";
		int i = 1;

		if (line != null) {
			for (pos = line.indexOf(key); pos >= 0; pos = line
					.indexOf(key, pos), i++) {
				pos = line.indexOf('>', pos);
				if (pos >= 0) {
					int endOfName = line.indexOf('<', pos);
					if (endOfName >= 0) {
						String name = line.substring(pos + 1, endOfName);
						String chapNum = i + ".";
						if (name.startsWith(chapNum)) {
							name = name.substring(chapNum.length(),
									name.length());
						}

						try {
							urls.add(new AbstractMap.SimpleEntry<String, URL>(
									name.trim(), new URL(base + i + suffix)));
						} catch (MalformedURLException e) {
							Instance.getTraceHandler()
									.error(new IOException(
											"Cannot parse chapter " + i
													+ " url: "
													+ (base + i + suffix), e));
						}
					}
				}
			}
		} else {
			// only one chapter:
			final String chapName = getTitle(reset(in));
			final URL chapURL = source;
			urls.add(new Entry<String, URL>() {
				@Override
				public URL setValue(URL value) {
					return null;
				}

				@Override
				public URL getValue() {
					return chapURL;
				}

				@Override
				public String getKey() {
					return chapName;
				}
			});
		}

		return urls;
	}

	@Override
	protected String getChapterContent(URL source, InputStream in, int number,
			Progress pg) {
		StringBuilder builder = new StringBuilder();
		String startAt = "class='storytext ";
		String endAt1 = "function review_init";
		String endAt2 = "id=chap_select";
		boolean ok = false;

		@SuppressWarnings("resource")
		Scanner scan = new Scanner(in, "UTF-8");
		scan.useDelimiter("\\n");
		while (scan.hasNext()) {
			String line = scan.next();
			if (!ok && line.contains(startAt)) {
				ok = true;
			} else if (ok && (line.contains(endAt1) || line.contains(endAt2))) {
				ok = false;
				break;
			}

			if (ok) {
				// First line may contain the title and chap name again
				if (builder.length() == 0) {
					int pos = line.indexOf("<hr");
					if (pos >= 0) {
						boolean chaptered = false;
						for (String lang : Instance.getConfig().getList(
								Config.CONF_CHAPTER)) {
							String chapterWord = Instance.getConfig()
									.getStringX(Config.CONF_CHAPTER, lang);
							int posChap = line.indexOf(chapterWord + " ");
							if (posChap < pos) {
								chaptered = true;
								break;
							}
						}

						if (chaptered) {
							line = line.substring(pos);
						}
					}
				}

				builder.append(line);
				builder.append(' ');
			}
		}

		return builder.toString();
	}

	@Override
	protected boolean supports(URL url) {
		return "fanfiction.net".equals(url.getHost())
				|| "www.fanfiction.net".equals(url.getHost());
	}
}
Commit	Line	Data
	1	package be.nikiroo.fanfix.supported;
	2
	3	import java.io.IOException;
	4	import java.io.InputStream;
	5	import java.net.MalformedURLException;
	6	import java.net.URL;
	7	import java.text.SimpleDateFormat;
	8	import java.util.AbstractMap;
	9	import java.util.ArrayList;
	10	import java.util.Date;
	11	import java.util.List;
	12	import java.util.Map.Entry;
	13	import java.util.Scanner;
	14
	15	import be.nikiroo.fanfix.Instance;
	16	import be.nikiroo.fanfix.bundles.Config;
	17	import be.nikiroo.fanfix.data.MetaData;
	18	import be.nikiroo.utils.Image;
	19	import be.nikiroo.utils.Progress;
	20	import be.nikiroo.utils.StringUtils;
	21
	22	/**
	23	* Support class for <a href="http://www.fanfiction.net/">Faniction.net</a>
	24	* stories, a website dedicated to fanfictions of many, many different
	25	* universes, from TV shows to novels to games.
	26	*
	27	* @author niki
	28	*/
	29	class Fanfiction extends BasicSupport_Deprecated {
	30	@Override
	31	protected boolean isHtml() {
	32	return true;
	33	}
	34
	35	@Override
	36	protected MetaData getMeta(URL source, InputStream in) throws IOException {
	37	MetaData meta = new MetaData();
	38
	39	meta.setTitle(getTitle(reset(in)));
	40	meta.setAuthor(getAuthor(reset(in)));
	41	meta.setDate(getDate(reset(in)));
	42	meta.setTags(getTags(reset(in)));
	43	meta.setSource(getType().getSourceName());
	44	meta.setUrl(source.toString());
	45	meta.setPublisher(getType().getSourceName());
	46	meta.setUuid(source.toString());
	47	meta.setLuid("");
	48	meta.setLang("en"); // TODO!
	49	meta.setSubject(getSubject(reset(in)));
	50	meta.setType(getType().toString());
	51	meta.setImageDocument(false);
	52	meta.setCover(getCover(source, reset(in)));
	53
	54	return meta;
	55	}
	56
	57	private String getSubject(InputStream in) {
	58	String line = getLine(in, "id=pre_story_links", 0);
	59	if (line != null) {
	60	int pos = line.lastIndexOf('"');
	61	if (pos >= 1) {
	62	line = line.substring(pos + 1);
	63	pos = line.indexOf('<');
	64	if (pos >= 0) {
	65	return StringUtils.unhtml(line.substring(0, pos)).trim();
	66	}
	67	}
	68	}
	69
	70	return null;
	71	}
	72
	73	private List<String> getTags(InputStream in) {
	74	List<String> tags = new ArrayList<String>();
	75
	76	String key = "title=\"Send Private Message\"";
	77	String line = getLine(in, key, 2);
	78	if (line != null) {
	79	key = "Rated:";
	80	int pos = line.indexOf(key);
	81	if (pos >= 0) {
	82	line = line.substring(pos + key.length());
	83	key = "Chapters:";
	84	pos = line.indexOf(key);
	85	if (pos >= 0) {
	86	line = line.substring(0, pos);
	87	line = StringUtils.unhtml(line).trim();
	88	if (line.endsWith("-")) {
	89	line = line.substring(0, line.length() - 1);
	90	}
	91
	92	for (String tag : line.split("-")) {
	93	tags.add(StringUtils.unhtml(tag).trim());
	94	}
	95	}
	96	}
	97	}
	98
	99	return tags;
	100	}
	101
	102	private String getTitle(InputStream in) {
	103	int i = 0;
	104	@SuppressWarnings("resource")
	105	Scanner scan = new Scanner(in, "UTF-8");
	106	scan.useDelimiter("\\n");
	107	while (scan.hasNext()) {
	108	String line = scan.next();
	109	if (line.contains("xcontrast_txt")) {
	110	if ((++i) == 2) {
	111	line = StringUtils.unhtml(line).trim();
	112	if (line.startsWith("Follow/Fav")) {
	113	line = line.substring("Follow/Fav".length()).trim();
	114	}
	115
	116	return StringUtils.unhtml(line).trim();
	117	}
	118	}
	119	}
	120
	121	return "";
	122	}
	123
	124	private String getAuthor(InputStream in) {
	125	String author = null;
	126
	127	int i = 0;
	128	@SuppressWarnings("resource")
	129	Scanner scan = new Scanner(in, "UTF-8");
	130	scan.useDelimiter("\\n");
	131	while (scan.hasNext()) {
	132	String line = scan.next();
	133	if (line.contains("xcontrast_txt")) {
	134	if ((++i) == 3) {
	135	author = StringUtils.unhtml(line).trim();
	136	break;
	137	}
	138	}
	139	}
	140
	141	return BasicSupportHelper.fixAuthor(author);
	142	}
	143
	144	private String getDate(InputStream in) {
	145	String key = "Published: <span data-xutime='";
	146	String line = getLine(in, key, 0);
	147	if (line != null) {
	148	int pos = line.indexOf(key);
	149	if (pos >= 0) {
	150	line = line.substring(pos + key.length());
	151	pos = line.indexOf('\'');
	152	if (pos >= 0) {
	153	line = line.substring(0, pos).trim();
	154	try {
	155	SimpleDateFormat sdf = new SimpleDateFormat(
	156	"YYYY-MM-dd");
	157	return sdf
	158	.format(new Date(1000 * Long.parseLong(line)));
	159	} catch (NumberFormatException e) {
	160	Instance.getTraceHandler().error(
	161	new IOException(
	162	"Cannot convert publication date: "
	163	+ line, e));
	164	}
	165	}
	166	}
	167	}
	168
	169	return null;
	170	}
	171
	172	@Override
	173	protected String getDesc(URL source, InputStream in) {
	174	return getLine(in, "title=\"Send Private Message\"", 1);
	175	}
	176
	177	private Image getCover(URL url, InputStream in) {
	178	String key = "class='cimage";
	179	String line = getLine(in, key, 0);
	180	if (line != null) {
	181	int pos = line.indexOf(key);
	182	if (pos >= 0) {
	183	line = line.substring(pos + key.length());
	184	key = "src='";
	185	pos = line.indexOf(key);
	186	if (pos >= 0) {
	187	line = line.substring(pos + key.length());
	188	pos = line.indexOf('\'');
	189	if (pos >= 0) {
	190	line = line.substring(0, pos);
	191	if (line.startsWith("//")) {
	192	line = url.getProtocol() + "://"
	193	+ line.substring(2);
	194	} else if (line.startsWith("//")) {
	195	line = url.getProtocol() + "://" + url.getHost()
	196	+ "/" + line.substring(1);
	197	} else {
	198	line = url.getProtocol() + "://" + url.getHost()
	199	+ "/" + url.getPath() + "/" + line;
	200	}
	201
	202	return getImage(this, null, line);
	203	}
	204	}
	205	}
	206	}
	207
	208	return null;
	209	}
	210
	211	@Override
	212	protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
	213	Progress pg) {
	214	List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
	215
	216	String base = source.toString();
	217	int pos = base.lastIndexOf('/');
	218	String suffix = base.substring(pos); // including '/' at start
	219	base = base.substring(0, pos);
	220	if (base.endsWith("/1")) {
	221	base = base.substring(0, base.length() - 1); // including '/' at end
	222	}
	223
	224	String line = getLine(in, "id=chap_select", 0);
	225	String key = "<option value=";
	226	int i = 1;
	227
	228	if (line != null) {
	229	for (pos = line.indexOf(key); pos >= 0; pos = line
	230	.indexOf(key, pos), i++) {
	231	pos = line.indexOf('>', pos);
	232	if (pos >= 0) {
	233	int endOfName = line.indexOf('<', pos);
	234	if (endOfName >= 0) {
	235	String name = line.substring(pos + 1, endOfName);
	236	String chapNum = i + ".";
	237	if (name.startsWith(chapNum)) {
	238	name = name.substring(chapNum.length(),
	239	name.length());
	240	}
	241
	242	try {
	243	urls.add(new AbstractMap.SimpleEntry<String, URL>(
	244	name.trim(), new URL(base + i + suffix)));
	245	} catch (MalformedURLException e) {
	246	Instance.getTraceHandler()
	247	.error(new IOException(
	248	"Cannot parse chapter " + i
	249	+ " url: "
	250	+ (base + i + suffix), e));
	251	}
	252	}
	253	}
	254	}
	255	} else {
	256	// only one chapter:
	257	final String chapName = getTitle(reset(in));
	258	final URL chapURL = source;
	259	urls.add(new Entry<String, URL>() {
	260	@Override
	261	public URL setValue(URL value) {
	262	return null;
	263	}
	264
	265	@Override
	266	public URL getValue() {
	267	return chapURL;
	268	}
	269
	270	@Override
	271	public String getKey() {
	272	return chapName;
	273	}
	274	});
	275	}
	276
	277	return urls;
	278	}
	279
	280	@Override
	281	protected String getChapterContent(URL source, InputStream in, int number,
	282	Progress pg) {
	283	StringBuilder builder = new StringBuilder();
	284	String startAt = "class='storytext ";
	285	String endAt1 = "function review_init";
	286	String endAt2 = "id=chap_select";
	287	boolean ok = false;
	288
	289	@SuppressWarnings("resource")
	290	Scanner scan = new Scanner(in, "UTF-8");
	291	scan.useDelimiter("\\n");
	292	while (scan.hasNext()) {
	293	String line = scan.next();
	294	if (!ok && line.contains(startAt)) {
	295	ok = true;
	296	} else if (ok && (line.contains(endAt1) \|\| line.contains(endAt2))) {
	297	ok = false;
	298	break;
	299	}
	300
	301	if (ok) {
	302	// First line may contain the title and chap name again
	303	if (builder.length() == 0) {
	304	int pos = line.indexOf("<hr");
	305	if (pos >= 0) {
	306	boolean chaptered = false;
	307	for (String lang : Instance.getConfig().getList(
	308	Config.CONF_CHAPTER)) {
	309	String chapterWord = Instance.getConfig()
	310	.getStringX(Config.CONF_CHAPTER, lang);
	311	int posChap = line.indexOf(chapterWord + " ");
	312	if (posChap < pos) {
	313	chaptered = true;
	314	break;
	315	}
	316	}
	317
	318	if (chaptered) {
	319	line = line.substring(pos);
	320	}
	321	}
	322	}
	323
	324	builder.append(line);
	325	builder.append(' ');
	326	}
	327	}
	328
	329	return builder.toString();
	330	}
	331
	332	@Override
	333	protected boolean supports(URL url) {
	334	return "fanfiction.net".equals(url.getHost())
	335	\|\| "www.fanfiction.net".equals(url.getHost());
	336	}
	337	}