[nikiroo-utils.git] / src / be / nikiroo / fanfix / supported / Fanfiction.java

package be.nikiroo.fanfix.supported;

import java.awt.image.BufferedImage;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map.Entry;
import java.util.Scanner;

import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.bundles.Config;
import be.nikiroo.fanfix.data.MetaData;
import be.nikiroo.utils.StringUtils;

/**
 * Support class for <a href="http://www.fanfiction.net/">Faniction.net</a>
 * stories, a website dedicated to fanfictions of many, many different
 * universes, from TV shows to novels to games.
 * 
 * @author niki
 */
class Fanfiction extends BasicSupport {
	@Override
	protected boolean isHtml() {
		return true;
	}

	@Override
	public String getSourceName() {
		return "Fanfiction.net";
	}

	@Override
	protected MetaData getMeta(URL source, InputStream in) throws IOException {
		MetaData meta = new MetaData();

		meta.setTitle(getTitle(reset(in)));
		meta.setAuthor(getAuthor(reset(in)));
		meta.setDate(getDate(reset(in)));
		meta.setTags(getTags(reset(in)));
		meta.setSource(getSourceName());
		meta.setUrl(source.toString());
		meta.setPublisher(getSourceName());
		meta.setUuid(source.toString());
		meta.setLuid("");
		meta.setLang("EN");
		meta.setSubject(getSubject(reset(in)));
		meta.setType(getType().toString());
		meta.setImageDocument(false);
		meta.setCover(getCover(source, reset(in)));

		return meta;
	}

	private String getSubject(InputStream in) {
		String line = getLine(in, "id=pre_story_links", 0);
		if (line != null) {
			int pos = line.lastIndexOf('"');
			if (pos >= 1) {
				line = line.substring(pos + 1);
				pos = line.indexOf('<');
				if (pos >= 0) {
					return StringUtils.unhtml(line.substring(0, pos)).trim();
				}
			}
		}

		return null;
	}

	private List<String> getTags(InputStream in) throws IOException {
		List<String> tags = new ArrayList<String>();

		String key = "title=\"Send Private Message\"";
		String line = getLine(in, key, 2);
		if (line != null) {
			key = "Rated:";
			int pos = line.indexOf(key);
			if (pos >= 0) {
				line = line.substring(pos + key.length());
				key = "Chapters:";
				pos = line.indexOf(key);
				if (pos >= 0) {
					line = line.substring(0, pos);
					line = StringUtils.unhtml(line).trim();
					if (line.endsWith("-")) {
						line = line.substring(0, line.length() - 1);
					}

					for (String tag : line.split("-")) {
						tags.add(StringUtils.unhtml(tag).trim());
					}
				}
			}
		}

		return tags;
	}

	private String getTitle(InputStream in) {
		int i = 0;
		@SuppressWarnings("resource")
		Scanner scan = new Scanner(in, "UTF-8");
		scan.useDelimiter("\\n");
		while (scan.hasNext()) {
			String line = scan.next();
			if (line.contains("xcontrast_txt")) {
				if ((++i) == 2) {
					line = StringUtils.unhtml(line).trim();
					if (line.startsWith("Follow/Fav")) {
						line = line.substring("Follow/Fav".length()).trim();
					}

					return StringUtils.unhtml(line).trim();
				}
			}
		}

		return null;
	}

	private String getAuthor(InputStream in) {
		String author = null;

		int i = 0;
		@SuppressWarnings("resource")
		Scanner scan = new Scanner(in, "UTF-8");
		scan.useDelimiter("\\n");
		while (scan.hasNext()) {
			String line = scan.next();
			if (line.contains("xcontrast_txt")) {
				if ((++i) == 3) {
					author = StringUtils.unhtml(line).trim();
					break;
				}
			}
		}

		return fixAuthor(author);
	}

	private String getDate(InputStream in) {
		String key = "Published: <span data-xutime='";
		String line = getLine(in, key, 0);
		if (line != null) {
			int pos = line.indexOf(key);
			if (pos >= 0) {
				line = line.substring(pos + key.length());
				pos = line.indexOf('\'');
				if (pos >= 0) {
					line = line.substring(0, pos).trim();
					try {
						SimpleDateFormat sdf = new SimpleDateFormat(
								"YYYY-MM-dd");
						return sdf
								.format(new Date(1000 * Long.parseLong(line)));
					} catch (NumberFormatException e) {
						Instance.syserr(new IOException(
								"Cannot convert publication date: " + line, e));
					}
				}
			}
		}

		return null;
	}

	@Override
	protected String getDesc(URL source, InputStream in) {
		return getLine(in, "title=\"Send Private Message\"", 1);
	}

	private BufferedImage getCover(URL url, InputStream in) {
		String key = "class='cimage";
		String line = getLine(in, key, 0);
		if (line != null) {
			int pos = line.indexOf(key);
			if (pos >= 0) {
				line = line.substring(pos + key.length());
				key = "src='";
				pos = line.indexOf(key);
				if (pos >= 0) {
					line = line.substring(pos + key.length());
					pos = line.indexOf('\'');
					if (pos >= 0) {
						line = line.substring(0, pos);
						if (line.startsWith("//")) {
							line = url.getProtocol() + "://"
									+ line.substring(2);
						} else if (line.startsWith("//")) {
							line = url.getProtocol() + "://" + url.getHost()
									+ "/" + line.substring(1);
						} else {
							line = url.getProtocol() + "://" + url.getHost()
									+ "/" + url.getPath() + "/" + line;
						}

						return getImage(this, null, line);
					}
				}
			}
		}

		return null;
	}

	@Override
	protected List<Entry<String, URL>> getChapters(URL source, InputStream in) {
		List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();

		String base = source.toString();
		int pos = base.lastIndexOf('/');
		String suffix = base.substring(pos); // including '/' at start
		base = base.substring(0, pos);
		if (base.endsWith("/1")) {
			base = base.substring(0, base.length() - 1); // including '/' at end
		}

		String line = getLine(in, "id=chap_select", 0);
		String key = "<option  value=";
		int i = 1;

		if (line != null) {
			for (pos = line.indexOf(key); pos >= 0; pos = line
					.indexOf(key, pos), i++) {
				pos = line.indexOf('>', pos);
				if (pos >= 0) {
					int endOfName = line.indexOf('<', pos);
					if (endOfName >= 0) {
						String name = line.substring(pos + 1, endOfName);
						String chapNum = i + ".";
						if (name.startsWith(chapNum)) {
							name = name.substring(chapNum.length(),
									name.length());
						}

						try {
							final String chapName = name.trim();
							final URL chapURL = new URL(base + i + suffix);
							urls.add(new Entry<String, URL>() {
								public URL setValue(URL value) {
									return null;
								}

								public URL getValue() {
									return chapURL;
								}

								public String getKey() {
									return chapName;
								}
							});
						} catch (MalformedURLException e) {
							Instance.syserr(new IOException(
									"Cannot parse chapter " + i + " url: "
											+ (base + i + suffix), e));
						}
					}
				}
			}
		} else {
			// only one chapter:
			final String chapName = getTitle(reset(in));
			final URL chapURL = source;
			urls.add(new Entry<String, URL>() {
				public URL setValue(URL value) {
					return null;
				}

				public URL getValue() {
					return chapURL;
				}

				public String getKey() {
					return chapName;
				}
			});
		}

		return urls;
	}

	@Override
	protected String getChapterContent(URL source, InputStream in, int number) {
		StringBuilder builder = new StringBuilder();
		String startAt = "class='storytext ";
		String endAt1 = "function review_init";
		String endAt2 = "id=chap_select";
		boolean ok = false;

		@SuppressWarnings("resource")
		Scanner scan = new Scanner(in, "UTF-8");
		scan.useDelimiter("\\n");
		while (scan.hasNext()) {
			String line = scan.next();
			if (!ok && line.contains(startAt)) {
				ok = true;
			} else if (ok && (line.contains(endAt1) || line.contains(endAt2))) {
				ok = false;
				break;
			}

			if (ok) {
				// First line may contain the title and chap name again
				if (builder.length() == 0) {
					int pos = line.indexOf("<hr");
					if (pos >= 0) {
						boolean chaptered = false;
						for (String lang : Instance.getConfig()
								.getString(Config.CHAPTER).split(",")) {
							String chapterWord = Instance.getConfig()
									.getStringX(Config.CHAPTER, lang);
							int posChap = line.indexOf(chapterWord + " ");
							if (posChap < pos) {
								chaptered = true;
								break;
							}
						}

						if (chaptered) {
							line = line.substring(pos);
						}
					}
				}

				builder.append(line);
				builder.append('\n');
			}
		}

		return builder.toString();
	}

	@Override
	protected boolean supports(URL url) {
		return "fanfiction.net".equals(url.getHost())
				|| "www.fanfiction.net".equals(url.getHost());
	}
}
Commit	Line	Data
08fe2e33 NR	1	package be.nikiroo.fanfix.supported;
08fe2e33 NR	2
68686a37	3	import java.awt.image.BufferedImage;
08fe2e33 NR	4	import java.io.IOException;
	5	import java.io.InputStream;
	6	import java.net.MalformedURLException;
	7	import java.net.URL;
	8	import java.text.SimpleDateFormat;
	9	import java.util.ArrayList;
	10	import java.util.Date;
	11	import java.util.List;
	12	import java.util.Map.Entry;
	13	import java.util.Scanner;
	14
	15	import be.nikiroo.fanfix.Instance;
22848428	16	import be.nikiroo.fanfix.bundles.Config;
68686a37	17	import be.nikiroo.fanfix.data.MetaData;
08fe2e33 NR	18	import be.nikiroo.utils.StringUtils;
	19
	20	/**
	21	* Support class for <a href="http://www.fanfiction.net/">Faniction.net</a>
	22	* stories, a website dedicated to fanfictions of many, many different
	23	* universes, from TV shows to novels to games.
	24	*
	25	* @author niki
	26	*/
	27	class Fanfiction extends BasicSupport {
	28	@Override
	29	protected boolean isHtml() {
	30	return true;
	31	}
	32
	33	@Override
	34	public String getSourceName() {
	35	return "Fanfiction.net";
	36	}
	37
	38	@Override
68686a37 NR	39	protected MetaData getMeta(URL source, InputStream in) throws IOException {
	40	MetaData meta = new MetaData();
	41
	42	meta.setTitle(getTitle(reset(in)));
	43	meta.setAuthor(getAuthor(reset(in)));
	44	meta.setDate(getDate(reset(in)));
	45	meta.setTags(getTags(reset(in)));
	46	meta.setSource(getSourceName());
2206ef66	47	meta.setUrl(source.toString());
68686a37 NR	48	meta.setPublisher(getSourceName());
	49	meta.setUuid(source.toString());
	50	meta.setLuid("");
	51	meta.setLang("EN");
	52	meta.setSubject(getSubject(reset(in)));
	53	meta.setType(getType().toString());
	54	meta.setImageDocument(false);
	55	meta.setCover(getCover(source, reset(in)));
	56
	57	return meta;
	58	}
	59
	60	private String getSubject(InputStream in) {
08fe2e33 NR	61	String line = getLine(in, "id=pre_story_links", 0);
	62	if (line != null) {
	63	int pos = line.lastIndexOf('"');
	64	if (pos >= 1) {
	65	line = line.substring(pos + 1);
	66	pos = line.indexOf('<');
	67	if (pos >= 0) {
68686a37	68	return StringUtils.unhtml(line.substring(0, pos)).trim();
08fe2e33 NR	69	}
	70	}
	71	}
	72
	73	return null;
	74	}
	75
68686a37 NR	76	private List<String> getTags(InputStream in) throws IOException {
68686a37 NR	77	List<String> tags = new ArrayList<String>();
08fe2e33 NR	78
	79	String key = "title=\"Send Private Message\"";
	80	String line = getLine(in, key, 2);
	81	if (line != null) {
	82	key = "Rated:";
	83	int pos = line.indexOf(key);
	84	if (pos >= 0) {
	85	line = line.substring(pos + key.length());
	86	key = "Chapters:";
	87	pos = line.indexOf(key);
	88	if (pos >= 0) {
	89	line = line.substring(0, pos);
	90	line = StringUtils.unhtml(line).trim();
	91	if (line.endsWith("-")) {
	92	line = line.substring(0, line.length() - 1);
	93	}
	94
	95	for (String tag : line.split("-")) {
68686a37	96	tags.add(StringUtils.unhtml(tag).trim());
08fe2e33 NR	97	}
	98	}
	99	}
	100	}
	101
	102	return tags;
	103	}
	104
68686a37	105	private String getTitle(InputStream in) {
08fe2e33 NR	106	int i = 0;
	107	@SuppressWarnings("resource")
	108	Scanner scan = new Scanner(in, "UTF-8");
	109	scan.useDelimiter("\\n");
	110	while (scan.hasNext()) {
	111	String line = scan.next();
	112	if (line.contains("xcontrast_txt")) {
	113	if ((++i) == 2) {
	114	line = StringUtils.unhtml(line).trim();
	115	if (line.startsWith("Follow/Fav")) {
	116	line = line.substring("Follow/Fav".length()).trim();
	117	}
	118
68686a37	119	return StringUtils.unhtml(line).trim();
08fe2e33 NR	120	}
	121	}
	122	}
	123
	124	return null;
	125	}
	126
68686a37	127	private String getAuthor(InputStream in) {
b4dc6ab5 NR	128	String author = null;
b4dc6ab5 NR	129
08fe2e33 NR	130	int i = 0;
	131	@SuppressWarnings("resource")
	132	Scanner scan = new Scanner(in, "UTF-8");
	133	scan.useDelimiter("\\n");
	134	while (scan.hasNext()) {
	135	String line = scan.next();
	136	if (line.contains("xcontrast_txt")) {
	137	if ((++i) == 3) {
b4dc6ab5 NR	138	author = StringUtils.unhtml(line).trim();
b4dc6ab5 NR	139	break;
08fe2e33 NR	140	}
	141	}
	142	}
	143
b4dc6ab5	144	return fixAuthor(author);
08fe2e33 NR	145	}
08fe2e33 NR	146
68686a37	147	private String getDate(InputStream in) {
08fe2e33 NR	148	String key = "Published: <span data-xutime='";
	149	String line = getLine(in, key, 0);
	150	if (line != null) {
	151	int pos = line.indexOf(key);
	152	if (pos >= 0) {
	153	line = line.substring(pos + key.length());
	154	pos = line.indexOf('\'');
	155	if (pos >= 0) {
	156	line = line.substring(0, pos).trim();
	157	try {
	158	SimpleDateFormat sdf = new SimpleDateFormat(
	159	"YYYY-MM-dd");
	160	return sdf
	161	.format(new Date(1000 * Long.parseLong(line)));
	162	} catch (NumberFormatException e) {
	163	Instance.syserr(new IOException(
	164	"Cannot convert publication date: " + line, e));
	165	}
	166	}
	167	}
	168	}
	169
	170	return null;
	171	}
	172
	173	@Override
	174	protected String getDesc(URL source, InputStream in) {
	175	return getLine(in, "title=\"Send Private Message\"", 1);
	176	}
	177
68686a37	178	private BufferedImage getCover(URL url, InputStream in) {
08fe2e33 NR	179	String key = "class='cimage";
	180	String line = getLine(in, key, 0);
	181	if (line != null) {
	182	int pos = line.indexOf(key);
	183	if (pos >= 0) {
	184	line = line.substring(pos + key.length());
	185	key = "src='";
	186	pos = line.indexOf(key);
	187	if (pos >= 0) {
	188	line = line.substring(pos + key.length());
	189	pos = line.indexOf('\'');
	190	if (pos >= 0) {
	191	line = line.substring(0, pos);
	192	if (line.startsWith("//")) {
	193	line = url.getProtocol() + "://"
	194	+ line.substring(2);
	195	} else if (line.startsWith("//")) {
	196	line = url.getProtocol() + "://" + url.getHost()
	197	+ "/" + line.substring(1);
	198	} else {
	199	line = url.getProtocol() + "://" + url.getHost()
	200	+ "/" + url.getPath() + "/" + line;
	201	}
	202
333f0e7b	203	return getImage(this, null, line);
08fe2e33 NR	204	}
	205	}
	206	}
	207	}
	208
	209	return null;
	210	}
	211
	212	@Override
	213	protected List<Entry<String, URL>> getChapters(URL source, InputStream in) {
	214	List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
	215
	216	String base = source.toString();
	217	int pos = base.lastIndexOf('/');
	218	String suffix = base.substring(pos); // including '/' at start
	219	base = base.substring(0, pos);
	220	if (base.endsWith("/1")) {
	221	base = base.substring(0, base.length() - 1); // including '/' at end
	222	}
	223
	224	String line = getLine(in, "id=chap_select", 0);
	225	String key = "<option value=";
	226	int i = 1;
08fe2e33	227
333f0e7b NR	228	if (line != null) {
	229	for (pos = line.indexOf(key); pos >= 0; pos = line
	230	.indexOf(key, pos), i++) {
	231	pos = line.indexOf('>', pos);
	232	if (pos >= 0) {
	233	int endOfName = line.indexOf('<', pos);
	234	if (endOfName >= 0) {
	235	String name = line.substring(pos + 1, endOfName);
	236	String chapNum = i + ".";
	237	if (name.startsWith(chapNum)) {
	238	name = name.substring(chapNum.length(),
	239	name.length());
	240	}
	241
	242	try {
	243	final String chapName = name.trim();
	244	final URL chapURL = new URL(base + i + suffix);
	245	urls.add(new Entry<String, URL>() {
	246	public URL setValue(URL value) {
	247	return null;
	248	}
	249
	250	public URL getValue() {
	251	return chapURL;
	252	}
	253
	254	public String getKey() {
	255	return chapName;
	256	}
	257	});
	258	} catch (MalformedURLException e) {
	259	Instance.syserr(new IOException(
	260	"Cannot parse chapter " + i + " url: "
	261	+ (base + i + suffix), e));
	262	}
08fe2e33 NR	263	}
	264	}
	265	}
333f0e7b NR	266	} else {
	267	// only one chapter:
	268	final String chapName = getTitle(reset(in));
	269	final URL chapURL = source;
	270	urls.add(new Entry<String, URL>() {
	271	public URL setValue(URL value) {
	272	return null;
	273	}
	274
	275	public URL getValue() {
	276	return chapURL;
	277	}
	278
	279	public String getKey() {
	280	return chapName;
	281	}
	282	});
08fe2e33 NR	283	}
	284
	285	return urls;
	286	}
	287
	288	@Override
	289	protected String getChapterContent(URL source, InputStream in, int number) {
	290	StringBuilder builder = new StringBuilder();
	291	String startAt = "class='storytext ";
	292	String endAt1 = "function review_init";
	293	String endAt2 = "id=chap_select";
	294	boolean ok = false;
	295
	296	@SuppressWarnings("resource")
	297	Scanner scan = new Scanner(in, "UTF-8");
	298	scan.useDelimiter("\\n");
	299	while (scan.hasNext()) {
	300	String line = scan.next();
	301	if (!ok && line.contains(startAt)) {
	302	ok = true;
	303	} else if (ok && (line.contains(endAt1) \|\| line.contains(endAt2))) {
	304	ok = false;
	305	break;
	306	}
	307
	308	if (ok) {
	309	// First line may contain the title and chap name again
	310	if (builder.length() == 0) {
	311	int pos = line.indexOf("<hr");
	312	if (pos >= 0) {
22848428 NR	313	boolean chaptered = false;
	314	for (String lang : Instance.getConfig()
	315	.getString(Config.CHAPTER).split(",")) {
	316	String chapterWord = Instance.getConfig()
	317	.getStringX(Config.CHAPTER, lang);
	318	int posChap = line.indexOf(chapterWord + " ");
	319	if (posChap < pos) {
	320	chaptered = true;
	321	break;
	322	}
	323	}
	324
	325	if (chaptered) {
	326	line = line.substring(pos);
	327	}
08fe2e33 NR	328	}
	329	}
	330
	331	builder.append(line);
a4143cd7	332	builder.append('\n');
08fe2e33 NR	333	}
	334	}
	335
	336	return builder.toString();
	337	}
	338
	339	@Override
	340	protected boolean supports(URL url) {
	341	return "fanfiction.net".equals(url.getHost())
	342	\|\| "www.fanfiction.net".equals(url.getHost());
	343	}
	344	}