1 package be
.nikiroo
.gofetch
.support
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.text
.ParseException
;
7 import java
.text
.SimpleDateFormat
;
8 import java
.util
.ArrayList
;
9 import java
.util
.Arrays
;
10 import java
.util
.Date
;
11 import java
.util
.List
;
12 import java
.util
.Map
.Entry
;
14 import org
.jsoup
.helper
.DataUtil
;
15 import org
.jsoup
.helper
.StringUtil
;
16 import org
.jsoup
.nodes
.Document
;
17 import org
.jsoup
.nodes
.Element
;
18 import org
.jsoup
.nodes
.Node
;
19 import org
.jsoup
.nodes
.TextNode
;
20 import org
.jsoup
.select
.NodeTraversor
;
21 import org
.jsoup
.select
.NodeVisitor
;
23 import be
.nikiroo
.gofetch
.data
.Comment
;
24 import be
.nikiroo
.gofetch
.data
.Story
;
25 import be
.nikiroo
.utils
.Downloader
;
26 import be
.nikiroo
.utils
.StringUtils
;
29 * Base class for website support.
33 public abstract class BasicSupport
{
34 /** The downloader to use for all websites. */
35 static protected Downloader downloader
= new Downloader("gofetcher");
37 static private String preselector
;
42 * The website textual description, to add in the dispatcher page.
46 * @return the description
48 abstract public String
getDescription();
51 * The gopher "selector" to use for output.
53 * A kind of "URL path", like "/news/" or "/misc/news/" or...
55 * @return the selector
57 public String
getSelector() {
58 return getSelector(type
);
66 public Type
getType() {
71 * List all the recent items, but only assure the ID and internal URL to
72 * fetch it later on (until it has been fetched, the rest of the
73 * {@link Story} is not confirmed).
75 * @return the list of new stories
80 public List
<Story
> list() throws IOException
{
81 List
<Story
> list
= new ArrayList
<Story
>();
83 for (Entry
<URL
, String
> entry
: getUrls()) {
84 URL url
= entry
.getKey();
85 String defaultCateg
= entry
.getValue();
86 if (defaultCateg
== null) {
90 InputStream in
= downloader
.open(url
);
91 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
92 List
<Element
> articles
= getArticles(doc
);
93 for (Element article
: articles
) {
94 String id
= getArticleId(doc
, article
).trim();
95 String title
= getArticleTitle(doc
, article
).trim();
96 String author
= getArticleAuthor(doc
, article
).trim();
97 String date
= getArticleDate(doc
, article
).trim();
98 String categ
= getArticleCategory(doc
, article
, defaultCateg
)
100 String details
= getArticleDetails(doc
, article
).trim();
101 String intUrl
= getArticleIntUrl(doc
, article
).trim();
102 String extUrl
= getArticleExtUrl(doc
, article
).trim();
103 String content
= getArticleContent(doc
, article
).trim();
105 if (id
.isEmpty() && date
.isEmpty()) {
110 while (id
.length() < 10) {
114 id
= date
.replace(":", "_").replace("+", "_");
119 list
.add(new Story(getType(), id
, title
, author
, date
, categ
,
120 details
, intUrl
, extUrl
, content
));
128 * The {@link URL}s to process for this website.
130 * @return the list of {@link URL}s
132 * @throws IOException
133 * in case of I/O error
135 abstract protected List
<Entry
<URL
, String
>> getUrls() throws IOException
;
138 * The article {@link Element}s of this document.
141 * the main document for the current category
143 * @return the articles
145 abstract protected List
<Element
> getArticles(Document doc
);
148 * The ID of the article (defaults to the date element if empty).
151 * the main document for the current category
153 * the article to look into
157 abstract protected String
getArticleId(Document doc
, Element article
);
160 * The article title to display.
163 * the main document for the current category
165 * the article to look into
169 abstract protected String
getArticleTitle(Document doc
, Element article
);
172 * The optional article author.
175 * the main document for the current category
177 * the article to look into
181 abstract protected String
getArticleAuthor(Document doc
, Element article
);
184 * The optional article date.
187 * the main document for the current category
189 * the article to look into
193 abstract protected String
getArticleDate(Document doc
, Element article
);
196 * the optional article category.
199 * the main document for the current category
201 * the article to look into
202 * @param currentCategory
203 * the currently listed category if any (can be NULL)
205 * @return the category
207 abstract protected String
getArticleCategory(Document doc
, Element article
,
208 String currentCategory
);
211 * the optional details of the article (can replace the date, author and
212 * category, for instance).
215 * the main document for the current category
217 * the article to look into
219 * @return the details
221 abstract protected String
getArticleDetails(Document doc
, Element article
);
224 * The (required) {@link URL} that points to the news page on the supported
228 * the main document for the current category
230 * the article to look into
232 * @return the internal {@link URL}
234 abstract protected String
getArticleIntUrl(Document doc
, Element article
);
237 * the optional {@link URL} that points to an external website for more
241 * the main document for the current category
243 * the article to look into
245 * @return the external {@link URL}
247 abstract protected String
getArticleExtUrl(Document doc
, Element article
);
250 * The optional article short-content (not the full content, that will be
251 * fetched by {@link BasicSupport#fetch(Story)}).
254 * the main document for the current category
256 * the article to look into
258 * @return the short content
260 abstract protected String
getArticleContent(Document doc
, Element article
);
263 * Fetch the full article content as well as all the comments associated to
264 * this {@link Story}, if any (can be empty, but not NULL).
267 * the story to fetch the comments of
269 * @throws IOException
270 * in case of I/O error
272 public void fetch(Story story
) throws IOException
{
273 String fullContent
= "";
275 URL url
= new URL(story
.getUrlInternal());
276 InputStream in
= downloader
.open(url
);
278 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
279 Element article
= getFullArticle(doc
);
280 if (article
!= null) {
281 StringBuilder builder
= new StringBuilder();
282 ElementProcessor eProc
= getElementProcessorFullArticle();
284 for (String line
: toLines(article
, eProc
)) {
285 builder
.append(line
+ "\n");
288 builder
.append(article
.text());
291 // Content is too tight with a single break per line:
292 fullContent
= builder
.toString().replace("\n", "\n\n") //
293 .replace("\n\n\n\n", "\n\n") //
294 .replace("\n\n\n\n", "\n\n") //
298 if (fullContent
.isEmpty()) {
299 fullContent
= story
.getContent();
302 story
.setFullContent(fullContent
);
303 story
.setComments(getComments(doc
,
304 getFullArticleCommentPosts(doc
, url
)));
313 * Return the full article if available.
316 * the (full article) document to work on
318 * @return the article or NULL
320 abstract protected Element
getFullArticle(Document doc
);
323 * Return the list of comment {@link Element}s from this optional container
324 * -- must <b>NOT</b> return the "container" as a comment {@link Element}.
327 * the (full article) document to work on
329 * the internal {@link URL} this article wa taken from (the
330 * {@link URL} from the supported website)
332 * @return the list of comment posts
334 abstract protected List
<Element
> getFullArticleCommentPosts(Document doc
,
338 * The {@link ElementProcessor} to use to convert the main article element
339 * (see {@link BasicSupport#getFullArticle(Document)}) into text.
341 * See {@link BasicElementProcessor} for a working, basic implementation.
343 * Can be NULL to simply use {@link Element#text()}.
345 * @return the processor, or NULL
347 abstract protected ElementProcessor
getElementProcessorFullArticle();
350 * Convert the comment elements into {@link Comment}s
353 * the document we work on
355 * the comment elements
357 * @return the converted {@link Comment}s
359 private List
<Comment
> getComments(Document doc
, List
<Element
> posts
) {
360 List
<Comment
> comments
= new ArrayList
<Comment
>();
362 for (Element post
: posts
) {
363 String id
= getCommentId(post
).trim();
364 String author
= getCommentAuthor(post
).trim();
365 String title
= getCommentTitle(post
).trim();
366 String date
= getCommentDate(post
).trim();
368 List
<String
> content
= new ArrayList
<String
>();
376 Element contentE
= getCommentContentElement(post
);
377 if (contentE
!= null) {
378 ElementProcessor eProc
= getElementProcessorComment();
380 for (String line
: toLines(contentE
, eProc
)) {
384 content
= Arrays
.asList(contentE
.text().split("\n"));
388 Comment comment
= new Comment(id
, author
, title
, date
, content
);
389 comment
.addAll(getComments(doc
,
390 getCommentCommentPosts(doc
, post
)));
392 if (!comment
.isEmpty()) {
393 comments
.add(comment
);
402 * Return the list of subcomment {@link Element}s from this comment element
403 * -- must <b>NOT</b> return the "container" as a comment {@link Element}.
406 * the (full article) document to work on
408 * the container (a comment {@link Element})
410 * @return the list of comment posts
412 abstract protected List
<Element
> getCommentCommentPosts(Document doc
,
416 * Compute the ID of the given comment element.
419 * the comment element
423 abstract protected String
getCommentId(Element post
);
426 * Compute the author of the given comment element.
429 * the comment element
433 abstract protected String
getCommentAuthor(Element post
);
436 * Compute the title of the given comment element.
439 * the comment element
443 abstract protected String
getCommentTitle(Element post
);
446 * Compute the date of the given comment element.
449 * the comment element
453 abstract protected String
getCommentDate(Element post
);
456 * Get the main of the given comment element, which can be NULL.
459 * the comment element
461 * @return the element
463 abstract protected Element
getCommentContentElement(Element post
);
466 * The {@link ElementProcessor} to use to convert the main comment element
467 * (see {@link BasicSupport#getCommentContentElement(Element)}) into text.
469 * See {@link BasicElementProcessor} for a working, basic implementation.
471 * Can be NULL to simply use {@link Element#text()}.
473 * @return the processor
475 abstract protected ElementProcessor
getElementProcessorComment();
483 protected void setType(Type type
) {
488 * The {@link String} to append to the selector (the selector will be
489 * constructed as "this string" then "/type/".
492 * the preselector to set
494 static public void setPreselector(String preselector
) {
495 BasicSupport
.preselector
= preselector
;
499 * Return a {@link BasicSupport} that is compatible with the given
500 * {@link Type} if it exists (or NULL if not).
505 * @return a compatible {@link BasicSupport} if it exists (or NULL if not)
507 static public BasicSupport
getSupport(Type type
) {
508 BasicSupport support
= null;
513 support
= new Slashdot();
516 support
= new Pipedot();
522 support
= new LeMonde();
525 support
= new TheRegister();
528 support
= new TooLinux();
531 support
= new EreNumerique();
534 support
= new Phoronix();
538 if (support
!= null) {
539 support
.setType(type
);
547 * The gopher "selector" to use for output for this type, using the
550 * A kind of "URL path", like "/news/" or "/misc/news/" or...
553 * the type to get the selector of
555 * @return the selector
557 static public String
getSelector(Type type
) {
558 return preselector
+ "/" + type
+ "/";
562 * Process the given element into text (each line is a text paragraph and
563 * can be prepended with ">" signs to indicate a quote or sub-quote or
567 * the element to process
568 * @param elementProcessor
569 * the element processor, must not be NULL
571 * @return text lines, each line is a paragraph
573 static protected List
<String
> toLines(Element element
,
574 final ElementProcessor elementProcessor
) {
575 final List
<String
> lines
= new ArrayList
<String
>();
576 final StringBuilder currentLine
= new StringBuilder();
577 final List
<Integer
> quoted
= new ArrayList
<Integer
>();
578 final List
<Node
> ignoredNodes
= new ArrayList
<Node
>();
579 final List
<String
> footnotes
= new ArrayList
<String
>();
581 if (element
!= null) {
582 new NodeTraversor(new NodeVisitor() {
584 public void head(Node node
, int depth
) {
585 String manual
= null;
586 boolean ignore
= elementProcessor
.ignoreNode(node
)
587 || ignoredNodes
.contains(node
.parentNode());
590 manual
= elementProcessor
.manualProcessing(node
);
591 if (manual
!= null) {
592 currentLine
.append(manual
);
599 String subtitle
= elementProcessor
.isSubtitle(node
);
600 if (subtitle
!= null) {
601 subtitle
= subtitle
.trim();
602 currentLine
.append("\n[ " + subtitle
+ " ]\n");
609 if (node
instanceof Element
) {
610 Element el
= (Element
) node
;
611 if ("pre".equals(el
.tagName())) {
612 currentLine
.append(StringUtils
613 .unhtml(el
.text()).trim());
620 ignoredNodes
.add(node
);
625 for (int i
= 0; i
< quoted
.size(); i
++) {
630 boolean enterQuote
= elementProcessor
.detectQuote(node
);
631 boolean leaveQuote
= quoted
.contains(depth
);
638 quoted
.remove(Integer
.valueOf(depth
));
641 if (enterQuote
|| leaveQuote
) {
642 if (currentLine
.length() > 0) {
643 if (currentLine
.charAt(currentLine
.length() - 1) == '\n') {
644 currentLine
.setLength(currentLine
.length() - 1);
646 for (String l
: currentLine
.toString().split("\n")) {
650 currentLine
.setLength(0);
653 if (node
instanceof Element
) {
654 Element element
= (Element
) node
;
655 boolean block
= element
.isBlock()
656 || element
.tagName().equalsIgnoreCase("br");
657 if (block
&& currentLine
.length() > 0) {
658 currentLine
.append("\n");
661 if (!element
.absUrl("href").trim().isEmpty()) {
662 footnotes
.add(element
.absUrl("href"));
663 currentLine
.append("[" + footnotes
.size() + "]");
665 } else if (node
instanceof TextNode
) {
666 TextNode textNode
= (TextNode
) node
;
667 String line
= StringUtil
.normaliseWhitespace(textNode
670 currentLine
.append(elementProcessor
.processText(line
));
671 currentLine
.append(" ");
676 public void tail(Node node
, int depth
) {
678 }).traverse(element
);
681 if (currentLine
.length() > 0) {
683 for (int i
= 0; i
< quoted
.size(); i
++) {
687 if (currentLine
.length() > 0) {
688 if (currentLine
.charAt(currentLine
.length() - 1) == '\n') {
689 currentLine
.setLength(currentLine
.length() - 1);
691 for (String l
: currentLine
.toString().split("\n")) {
697 // Fix spaces and nbsp, remove multiple following blank lines
698 List
<String
> linesCopy
= new ArrayList
<String
>(lines
.size());
700 for (int i
= 0; i
< lines
.size(); i
++) {
701 String line
= lines
.get(i
).replace(" ", " ") // nbsp -> space
702 .replace(" ", " ").trim();
703 if (line
.isEmpty()) {
714 // Footnotes insertion
715 if (footnotes
.size() > 0) {
720 for (int i
= 0; i
< footnotes
.size(); i
++) {
721 linesCopy
.add("[" + (i
+ 1) + "] " + footnotes
.get(i
));
729 * Reformat the date if possible.
734 * @return the reformated date, or the same value if it was not parsable
736 static private String
date(String date
) {
737 SimpleDateFormat out
= new SimpleDateFormat("yyyy/MM/dd");
741 epoch
= Long
.parseLong(date
.trim());
742 } catch (Exception e
) {
747 return out
.format(new Date(1000 * epoch
));
751 Date dat
= new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")
753 return out
.format(dat
);
754 } catch (ParseException e
) {