1 package be
.nikiroo
.fanfix
.searchable
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.net
.URLEncoder
;
7 import java
.text
.SimpleDateFormat
;
8 import java
.util
.ArrayList
;
10 import java
.util
.HashMap
;
11 import java
.util
.List
;
14 import org
.jsoup
.nodes
.Document
;
15 import org
.jsoup
.nodes
.Element
;
16 import org
.jsoup
.select
.Elements
;
18 import be
.nikiroo
.fanfix
.Instance
;
19 import be
.nikiroo
.fanfix
.bundles
.StringId
;
20 import be
.nikiroo
.fanfix
.data
.MetaData
;
21 import be
.nikiroo
.fanfix
.supported
.SupportType
;
22 import be
.nikiroo
.utils
.Image
;
23 import be
.nikiroo
.utils
.StringUtils
;
26 * A {@link BasicSearchable} for Fanfiction.NET.
30 class Fanfiction
extends BasicSearchable
{
31 static private String BASE_URL
= "http://fanfiction.net/";
34 * Create a new {@link Fanfiction}.
37 * {@link SupportType#FANFICTION}
39 public Fanfiction(SupportType type
) {
44 public List
<SearchableTag
> getTags() throws IOException
{
45 String storiesName
= null;
46 String crossoversName
= null;
47 Map
<String
, String
> stories
= new HashMap
<String
, String
>();
48 Map
<String
, String
> crossovers
= new HashMap
<String
, String
>();
50 Document mainPage
= load(BASE_URL
, true);
51 Element menu
= mainPage
.getElementsByClass("dropdown").first();
53 Element ul
= menu
.getElementsByClass("dropdown-menu").first();
55 Map
<String
, String
> currentList
= null;
56 for (Element li
: ul
.getElementsByTag("li")) {
57 if (li
.hasClass("disabled")) {
58 if (storiesName
== null) {
59 storiesName
= li
.text();
60 currentList
= stories
;
62 crossoversName
= li
.text();
63 currentList
= crossovers
;
65 } else if (currentList
!= null) {
66 Element a
= li
.getElementsByTag("a").first();
68 currentList
.put(a
.absUrl("href"), a
.text());
75 List
<SearchableTag
> tags
= new ArrayList
<SearchableTag
>();
77 if (storiesName
!= null) {
78 SearchableTag tag
= new SearchableTag(null, storiesName
, false);
79 for (String id
: stories
.keySet()) {
80 tag
.add(new SearchableTag(id
, stories
.get(id
), false, false));
85 if (crossoversName
!= null) {
86 SearchableTag tag
= new SearchableTag(null, crossoversName
, false);
87 for (String id
: crossovers
.keySet()) {
88 tag
.add(new SearchableTag(id
, crossovers
.get(id
), false, false));
97 public void fillTag(SearchableTag tag
) throws IOException
{
98 if (tag
.getId() == null || tag
.isComplete()) {
102 Document doc
= load(tag
.getId(), false);
103 Element list
= doc
.getElementById("list_output");
105 Element table
= list
.getElementsByTag("table").first();
107 for (Element div
: table
.getElementsByTag("div")) {
108 Element a
= div
.getElementsByTag("a").first();
109 Element span
= div
.getElementsByTag("span").first();
112 String subid
= a
.absUrl("href");
113 boolean crossoverSubtag
= subid
114 .contains("/crossovers/");
116 SearchableTag subtag
= new SearchableTag(subid
,
117 a
.text(), !crossoverSubtag
, !crossoverSubtag
);
121 String nr
= span
.text();
122 if (nr
.startsWith("(")) {
123 nr
= nr
.substring(1);
125 if (nr
.endsWith(")")) {
126 nr
= nr
.substring(0, nr
.length() - 1);
130 // TODO: fix toNumber/fromNumber
131 nr
= nr
.replaceAll("\\.[0-9]*", "");
133 subtag
.setCount(StringUtils
.toNumber(nr
));
140 tag
.setComplete(true);
144 public List
<MetaData
> search(String search
, int page
) throws IOException
{
145 String encoded
= URLEncoder
.encode(search
.toLowerCase(), "utf-8");
146 return getStories(BASE_URL
+ "search/?ready=1&type=story&keywords="
147 + encoded
+ "&ppage=" + page
, null, null);
151 public List
<MetaData
> search(SearchableTag tag
, int page
)
153 List
<MetaData
> metas
= new ArrayList
<MetaData
>();
155 String url
= tag
.getId();
158 int pos
= url
.indexOf("&p=");
160 url
= url
.replaceAll("(.*\\&p=)[0-9]*(.*)", "$1\\" + page
167 Document doc
= load(url
, false);
169 // Update the pages number if needed
170 if (tag
.getPages() < 0 && tag
.isLeaf()) {
171 tag
.setPages(getPages(doc
));
174 // Find out the full subjects (including parents)
175 String subjects
= "";
176 for (SearchableTag t
= tag
; t
!= null; t
= t
.getParent()) {
177 if (!subjects
.isEmpty()) {
180 subjects
+= t
.getName();
183 metas
= getStories(url
, doc
, subjects
);
190 * Return the number of pages in this stories result listing.
195 * @return the number of pages or -1 if unknown
197 private int getPages(Document doc
) {
201 Element center
= doc
.getElementsByTag("center").first();
202 if (center
!= null) {
203 for (Element a
: center
.getElementsByTag("a")) {
204 if (a
.absUrl("href").contains("&p=")) {
205 int thisLinkPages
= -1;
207 String
[] tab
= a
.absUrl("href").split("=");
208 tab
= tab
[tab
.length
- 1].split("&");
209 thisLinkPages
= Integer
210 .parseInt(tab
[tab
.length
- 1]);
211 } catch (Exception e
) {
214 pages
= Math
.max(pages
, thisLinkPages
);
224 * Fetch the stories from the given page.
227 * the url of the document
229 * the document to use (if NULL, will be loaded from
230 * <tt>sourceUrl</tt>)
232 * the main subject (the anime/book/movie item related to the
233 * stories, like "MLP" or "Doctor Who"), or NULL if none
235 * @return the stories found in it
237 * @throws IOException
238 * in case of I/O errors
240 private List
<MetaData
> getStories(String sourceUrl
, Document doc
,
241 String mainSubject
) throws IOException
{
242 List
<MetaData
> metas
= new ArrayList
<MetaData
>();
245 doc
= load(sourceUrl
, false);
248 for (Element story
: doc
.getElementsByClass("z-list")) {
249 MetaData meta
= new MetaData();
250 meta
.setImageDocument(false);
251 meta
.setSource(getType().getSourceName());
254 Element stitle
= story
.getElementsByClass("stitle").first();
255 if (stitle
!= null) {
256 meta
.setTitle(stitle
.text());
257 meta
.setUrl(stitle
.absUrl("href"));
258 Element cover
= stitle
.getElementsByTag("img").first();
260 // note: see data-original if needed?
261 String coverUrl
= cover
.absUrl("src");
264 InputStream in
= Instance
.getCache().open(
265 new URL(coverUrl
), getSupport(), true);
267 meta
.setCover(new Image(in
));
271 } catch (Exception e
) {
272 Instance
.getTraceHandler()
273 .error(new Exception(
274 "Cannot download cover for Fanfiction story in search mode",
281 Elements as
= story
.getElementsByTag("a");
283 meta
.setAuthor(as
.get(1).text());
286 // Tags (concatenated text), published date, updated date, Resume
288 List
<String
> tagList
= new ArrayList
<String
>();
289 Elements divs
= story
.getElementsByTag("div");
290 if (divs
.size() > 1 && divs
.get(1).childNodeSize() > 0) {
291 String resume
= divs
.get(1).text();
292 if (divs
.size() > 2) {
293 tags
= divs
.get(2).text();
294 resume
= resume
.substring(0,
295 resume
.length() - tags
.length()).trim();
297 for (Element d
: divs
.get(2).getElementsByAttribute(
299 String secs
= d
.attr("data-xutime");
301 String date
= new SimpleDateFormat("yyyy-MM-dd")
303 Long
.parseLong(secs
) * 1000));
304 // (updated, ) published
305 if (meta
.getDate() != null) {
306 tagList
.add("Updated: " + meta
.getDate());
309 } catch (Exception e
) {
314 meta
.setResume(getSupport().makeChapter(new URL(sourceUrl
), 0,
315 Instance
.getTrans().getString(StringId
.DESCRIPTION
),
319 // How are the tags ordered?
320 // We have "Rated: xx", then the language, then all other tags
321 // If the subject(s) is/are present, they are before "Rated: xx"
327 // Search (Luna) Tags: [Harry Potter, Rated: T, English, Chapters:
328 // 1, Words: 270, Reviews: 2, Published: 2/19/2013, Luna L.]
330 // Normal (MLP) Tags: [Rated: T, Spanish, Drama/Suspense, Chapters:
331 // 2, Words: 8,686, Reviews: 1, Favs: 1, Follows: 1, Updated: 4/7,
334 // Crossover (MLP/Who) Tags: [Rated: K+, English, Adventure/Romance,
335 // Chapters: 8, Words: 7,788, Reviews: 2, Favs: 2, Follows: 1,
336 // Published: 9/1/2016]
338 boolean rated
= false;
339 boolean isLang
= false;
340 String subject
= mainSubject
== null ?
"" : mainSubject
;
341 String
[] tab
= tags
.split(" *- *");
342 for (int i
= 0; i
< tab
.length
; i
++) {
344 if (tag
.startsWith("Rated: ")) {
349 if (!subject
.isEmpty()) {
357 if (tag
.contains(":")) {
358 // Handle special tags:
359 if (tag
.startsWith("Words: ")) {
361 meta
.setWords(Long
.parseLong(tag
362 .substring("Words: ".length())
363 .replace(",", "").trim()));
364 } catch (Exception e
) {
366 } else if (tag
.startsWith("Rated: ")) {
370 // Normal tags are "/"-separated
371 for (String t
: tag
.split("/")) {
376 if (tag
.startsWith("Rated: ")) {
382 meta
.setSubject(subject
);
383 meta
.setTags(tagList
);