search: .equals
[nikiroo-utils.git] / src / be / nikiroo / fanfix / searchable / Fanfiction.java
CommitLineData
158b372d
NR
1package be.nikiroo.fanfix.searchable;
2
3import java.io.IOException;
e66c9078 4import java.io.InputStream;
158b372d 5import java.net.URL;
e66c9078
NR
6import java.net.URLEncoder;
7import java.text.SimpleDateFormat;
158b372d 8import java.util.ArrayList;
e66c9078 9import java.util.Date;
158b372d
NR
10import java.util.HashMap;
11import java.util.List;
12import java.util.Map;
13
14import org.jsoup.nodes.Document;
15import org.jsoup.nodes.Element;
16import org.jsoup.select.Elements;
17
18import be.nikiroo.fanfix.Instance;
19import be.nikiroo.fanfix.bundles.StringId;
20import be.nikiroo.fanfix.data.MetaData;
21import be.nikiroo.fanfix.supported.SupportType;
e66c9078 22import be.nikiroo.utils.Image;
596ed3d6 23import be.nikiroo.utils.StringUtils;
158b372d
NR
24
25/**
26 * A {@link BasicSearchable} for Fanfiction.NET.
27 *
28 * @author niki
29 */
30class Fanfiction extends BasicSearchable {
76ec935e
NR
31 static private String BASE_URL = "http://fanfiction.net/";
32
158b372d
NR
33 /**
34 * Create a new {@link Fanfiction}.
35 *
36 * @param type
37 * {@link SupportType#FANFICTION}
38 */
39 public Fanfiction(SupportType type) {
40 super(type);
41 }
42
43 @Override
44 public List<SearchableTag> getTags() throws IOException {
45 String storiesName = null;
46 String crossoversName = null;
47 Map<String, String> stories = new HashMap<String, String>();
48 Map<String, String> crossovers = new HashMap<String, String>();
49
76ec935e 50 Document mainPage = load(BASE_URL, true);
158b372d
NR
51 Element menu = mainPage.getElementsByClass("dropdown").first();
52 if (menu != null) {
53 Element ul = menu.getElementsByClass("dropdown-menu").first();
54 if (ul != null) {
55 Map<String, String> currentList = null;
56 for (Element li : ul.getElementsByTag("li")) {
57 if (li.hasClass("disabled")) {
58 if (storiesName == null) {
59 storiesName = li.text();
60 currentList = stories;
61 } else {
62 crossoversName = li.text();
63 currentList = crossovers;
64 }
65 } else if (currentList != null) {
66 Element a = li.getElementsByTag("a").first();
67 if (a != null) {
68 currentList.put(a.absUrl("href"), a.text());
69 }
70 }
71 }
72 }
73 }
74
75 List<SearchableTag> tags = new ArrayList<SearchableTag>();
76
77 if (storiesName != null) {
76ec935e 78 SearchableTag tag = new SearchableTag(null, storiesName, false);
158b372d 79 for (String id : stories.keySet()) {
b3b9a1cd 80 tag.add(new SearchableTag(id, stories.get(id), false, false));
158b372d
NR
81 }
82 tags.add(tag);
83 }
84
85 if (crossoversName != null) {
76ec935e 86 SearchableTag tag = new SearchableTag(null, crossoversName, false);
158b372d 87 for (String id : crossovers.keySet()) {
76ec935e 88 tag.add(new SearchableTag(id, crossovers.get(id), false, false));
158b372d
NR
89 }
90 tags.add(tag);
91 }
92
93 return tags;
94 }
95
96 @Override
91b82a5c 97 public void fillTag(SearchableTag tag) throws IOException {
158b372d
NR
98 if (tag.getId() == null || tag.isComplete()) {
99 return;
100 }
101
76ec935e 102 Document doc = load(tag.getId(), false);
158b372d
NR
103 Element list = doc.getElementById("list_output");
104 if (list != null) {
105 Element table = list.getElementsByTag("table").first();
106 if (table != null) {
107 for (Element div : table.getElementsByTag("div")) {
108 Element a = div.getElementsByTag("a").first();
109 Element span = div.getElementsByTag("span").first();
110
111 if (a != null) {
e66c9078
NR
112 String subid = a.absUrl("href");
113 boolean crossoverSubtag = subid
114 .contains("/crossovers/");
115
116 SearchableTag subtag = new SearchableTag(subid,
117 a.text(), !crossoverSubtag, !crossoverSubtag);
118
158b372d
NR
119 tag.add(subtag);
120 if (span != null) {
121 String nr = span.text();
122 if (nr.startsWith("(")) {
123 nr = nr.substring(1);
124 }
125 if (nr.endsWith(")")) {
126 nr = nr.substring(0, nr.length() - 1);
127 }
128 nr = nr.trim();
74a43961
NR
129
130 // TODO: fix toNumber/fromNumber
b3b9a1cd 131 nr = nr.replaceAll("\\.[0-9]*", "");
74a43961
NR
132
133 subtag.setCount(StringUtils.toNumber(nr));
158b372d
NR
134 }
135 }
136 }
137 }
138 }
139
140 tag.setComplete(true);
141 }
142
143 @Override
8ffc8b73 144 public List<MetaData> search(String search, int page) throws IOException {
e66c9078 145 String encoded = URLEncoder.encode(search.toLowerCase(), "utf-8");
124442f1
NR
146 String url = BASE_URL + "search/?ready=1&type=story&keywords="
147 + encoded + "&ppage=" + page;
148
149 return getStories(url, null, null);
150 }
151
158b372d 152 @Override
e66c9078
NR
153 public List<MetaData> search(SearchableTag tag, int page)
154 throws IOException {
158b372d
NR
155 List<MetaData> metas = new ArrayList<MetaData>();
156
e66c9078
NR
157 String url = tag.getId();
158 if (url != null) {
159 if (page > 1) {
160 int pos = url.indexOf("&p=");
161 if (pos >= 0) {
162 url = url.replaceAll("(.*\\&p=)[0-9]*(.*)", "$1\\" + page
163 + "$2");
164 } else {
165 url += "&p=" + page;
166 }
167 }
74a43961 168
e66c9078
NR
169 Document doc = load(url, false);
170
171 // Update the pages number if needed
b3b9a1cd 172 if (tag.getPages() < 0 && tag.isLeaf()) {
e66c9078
NR
173 tag.setPages(getPages(doc));
174 }
175
176 // Find out the full subjects (including parents)
177 String subjects = "";
178 for (SearchableTag t = tag; t != null; t = t.getParent()) {
179 if (!subjects.isEmpty()) {
180 subjects += ", ";
181 }
182 subjects += t.getName();
183 }
184
185 metas = getStories(url, doc, subjects);
186 }
187
188 return metas;
189 }
190
81acd363
NR
191 @Override
192 public int searchPages(String search) throws IOException {
193 String encoded = URLEncoder.encode(search.toLowerCase(), "utf-8");
194 String url = BASE_URL + "search/?ready=1&type=story&keywords="
195 + encoded;
196
197 return getPages(load(url, false));
198 }
199
200 @Override
201 public int searchPages(SearchableTag tag) throws IOException {
202 if (tag.isLeaf()) {
203 String url = tag.getId();
204 return getPages(load(url, false));
205 }
206
207 return 0;
208 }
209
e66c9078
NR
210 /**
211 * Return the number of pages in this stories result listing.
212 *
213 * @param doc
214 * the document
215 *
216 * @return the number of pages or -1 if unknown
e66c9078 217 */
8ffc8b73 218 private int getPages(Document doc) {
e66c9078 219 int pages = -1;
76ec935e 220
e66c9078 221 if (doc != null) {
76ec935e
NR
222 Element center = doc.getElementsByTag("center").first();
223 if (center != null) {
76ec935e
NR
224 for (Element a : center.getElementsByTag("a")) {
225 if (a.absUrl("href").contains("&p=")) {
226 int thisLinkPages = -1;
227 try {
228 String[] tab = a.absUrl("href").split("=");
229 tab = tab[tab.length - 1].split("&");
230 thisLinkPages = Integer
231 .parseInt(tab[tab.length - 1]);
232 } catch (Exception e) {
233 }
234
235 pages = Math.max(pages, thisLinkPages);
236 }
237 }
76ec935e 238 }
e66c9078
NR
239 }
240
241 return pages;
242 }
243
244 /**
245 * Fetch the stories from the given page.
246 *
247 * @param sourceUrl
248 * the url of the document
249 * @param doc
250 * the document to use (if NULL, will be loaded from
251 * <tt>sourceUrl</tt>)
252 * @param mainSubject
253 * the main subject (the anime/book/movie item related to the
254 * stories, like "MLP" or "Doctor Who"), or NULL if none
255 *
256 * @return the stories found in it
257 *
258 * @throws IOException
259 * in case of I/O errors
260 */
261 private List<MetaData> getStories(String sourceUrl, Document doc,
262 String mainSubject) throws IOException {
263 List<MetaData> metas = new ArrayList<MetaData>();
264
265 if (doc == null) {
266 doc = load(sourceUrl, false);
267 }
76ec935e 268
e66c9078
NR
269 for (Element story : doc.getElementsByClass("z-list")) {
270 MetaData meta = new MetaData();
271 meta.setImageDocument(false);
272 meta.setSource(getType().getSourceName());
d16065ec
NR
273 meta.setPublisher(getType().getSourceName());
274 meta.setType(getType().toString());
e66c9078 275
596ed3d6 276 // Title, URL, Cover
e66c9078
NR
277 Element stitle = story.getElementsByClass("stitle").first();
278 if (stitle != null) {
279 meta.setTitle(stitle.text());
280 meta.setUrl(stitle.absUrl("href"));
d16065ec 281 meta.setUuid(meta.getUrl());
e66c9078
NR
282 Element cover = stitle.getElementsByTag("img").first();
283 if (cover != null) {
284 // note: see data-original if needed?
285 String coverUrl = cover.absUrl("src");
286
287 try {
288 InputStream in = Instance.getCache().open(
289 new URL(coverUrl), getSupport(), true);
290 try {
291 meta.setCover(new Image(in));
292 } finally {
293 in.close();
294 }
295 } catch (Exception e) {
12443642
NR
296 // Should not happen on Fanfiction.net
297 Instance.getTraceHandler().error(
298 new Exception(
299 "Cannot download cover for Fanfiction story in search mode: "
300 + meta.getTitle(), e));
158b372d
NR
301 }
302 }
e66c9078 303 }
158b372d 304
596ed3d6 305 // Author
e66c9078
NR
306 Elements as = story.getElementsByTag("a");
307 if (as.size() > 1) {
308 meta.setAuthor(as.get(1).text());
309 }
158b372d 310
596ed3d6 311 // Tags (concatenated text), published date, updated date, Resume
e66c9078 312 String tags = "";
596ed3d6 313 List<String> tagList = new ArrayList<String>();
e66c9078
NR
314 Elements divs = story.getElementsByTag("div");
315 if (divs.size() > 1 && divs.get(1).childNodeSize() > 0) {
316 String resume = divs.get(1).text();
317 if (divs.size() > 2) {
318 tags = divs.get(2).text();
319 resume = resume.substring(0,
320 resume.length() - tags.length()).trim();
158b372d 321
e66c9078
NR
322 for (Element d : divs.get(2).getElementsByAttribute(
323 "data-xutime")) {
324 String secs = d.attr("data-xutime");
325 try {
326 String date = new SimpleDateFormat("yyyy-MM-dd")
327 .format(new Date(
328 Long.parseLong(secs) * 1000));
329 // (updated, ) published
330 if (meta.getDate() != null) {
331 tagList.add("Updated: " + meta.getDate());
332 }
333 meta.setDate(date);
334 } catch (Exception e) {
335 }
158b372d
NR
336 }
337 }
338
e66c9078
NR
339 meta.setResume(getSupport().makeChapter(new URL(sourceUrl), 0,
340 Instance.getTrans().getString(StringId.DESCRIPTION),
158b372d 341 resume));
158b372d 342 }
158b372d 343
e66c9078
NR
344 // How are the tags ordered?
345 // We have "Rated: xx", then the language, then all other tags
346 // If the subject(s) is/are present, they are before "Rated: xx"
347
596ed3d6 348 // ////////////
e66c9078 349 // Examples: //
596ed3d6 350 // ////////////
e66c9078
NR
351
352 // Search (Luna) Tags: [Harry Potter, Rated: T, English, Chapters:
353 // 1, Words: 270, Reviews: 2, Published: 2/19/2013, Luna L.]
354
355 // Normal (MLP) Tags: [Rated: T, Spanish, Drama/Suspense, Chapters:
356 // 2, Words: 8,686, Reviews: 1, Favs: 1, Follows: 1, Updated: 4/7,
596ed3d6 357 // Published: 4/2]
e66c9078
NR
358
359 // Crossover (MLP/Who) Tags: [Rated: K+, English, Adventure/Romance,
360 // Chapters: 8, Words: 7,788, Reviews: 2, Favs: 2, Follows: 1,
596ed3d6 361 // Published: 9/1/2016]
e66c9078
NR
362
363 boolean rated = false;
364 boolean isLang = false;
596ed3d6 365 String subject = mainSubject == null ? "" : mainSubject;
e66c9078
NR
366 String[] tab = tags.split(" *- *");
367 for (int i = 0; i < tab.length; i++) {
368 String tag = tab[i];
369 if (tag.startsWith("Rated: ")) {
370 rated = true;
371 }
158b372d 372
e66c9078
NR
373 if (!rated) {
374 if (!subject.isEmpty()) {
375 subject += ", ";
376 }
377 subject += tag;
378 } else if (isLang) {
379 meta.setLang(tag);
380 isLang = false;
381 } else {
382 if (tag.contains(":")) {
383 // Handle special tags:
384 if (tag.startsWith("Words: ")) {
385 try {
386 meta.setWords(Long.parseLong(tag
387 .substring("Words: ".length())
388 .replace(",", "").trim()));
389 } catch (Exception e) {
390 }
391 } else if (tag.startsWith("Rated: ")) {
392 tagList.add(tag);
393 }
394 } else {
596ed3d6 395 // Normal tags are "/"-separated
e66c9078
NR
396 for (String t : tag.split("/")) {
397 tagList.add(t);
398 }
399 }
158b372d 400
e66c9078
NR
401 if (tag.startsWith("Rated: ")) {
402 isLang = true;
403 }
404 }
405 }
158b372d 406
e66c9078
NR
407 meta.setSubject(subject);
408 meta.setTags(tagList);
158b372d 409
e66c9078
NR
410 metas.add(meta);
411 }
158b372d 412
e66c9078 413 return metas;
158b372d
NR
414 }
415}