1 package be
.nikiroo
.gofetch
.support
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.text
.ParseException
;
7 import java
.text
.SimpleDateFormat
;
8 import java
.util
.ArrayList
;
9 import java
.util
.Arrays
;
10 import java
.util
.Date
;
11 import java
.util
.List
;
12 import java
.util
.Map
.Entry
;
14 import org
.jsoup
.helper
.DataUtil
;
15 import org
.jsoup
.helper
.StringUtil
;
16 import org
.jsoup
.nodes
.Document
;
17 import org
.jsoup
.nodes
.Element
;
18 import org
.jsoup
.nodes
.Node
;
19 import org
.jsoup
.nodes
.TextNode
;
20 import org
.jsoup
.select
.NodeTraversor
;
21 import org
.jsoup
.select
.NodeVisitor
;
23 import be
.nikiroo
.gofetch
.data
.Comment
;
24 import be
.nikiroo
.gofetch
.data
.Story
;
25 import be
.nikiroo
.utils
.Downloader
;
26 import be
.nikiroo
.utils
.StringUtils
;
29 * Base class for website support.
33 public abstract class BasicSupport
{
34 /** The downloader to use for all websites. */
35 protected static Downloader downloader
= new Downloader("gofetcher");
37 static private String preselector
;
42 * The website textual description, to add in the dispatcher page.
46 * @return the description
48 abstract public String
getDescription();
51 * The gopher "selector" to use for output.
53 * A kind of "URL path", like "/news/" or "/misc/news/" or...
55 * @return the selector
57 public String
getSelector() {
58 return getSelector(type
);
66 public Type
getType() {
71 * List all the recent items, but only assure the ID and internal URL to
72 * fetch it later on (until it has been fetched, the rest of the
73 * {@link Story} is not confirmed).
75 * @return the list of new stories
80 public List
<Story
> list() throws IOException
{
81 List
<Story
> list
= new ArrayList
<Story
>();
83 for (Entry
<URL
, String
> entry
: getUrls()) {
84 URL url
= entry
.getKey();
85 String defaultCateg
= entry
.getValue();
86 if (defaultCateg
== null) {
90 InputStream in
= downloader
.open(url
);
91 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
92 List
<Element
> articles
= getArticles(doc
);
93 for (Element article
: articles
) {
94 String id
= getArticleId(doc
, article
).trim();
95 String title
= getArticleTitle(doc
, article
).trim();
96 String author
= getArticleAuthor(doc
, article
).trim();
97 String date
= getArticleDate(doc
, article
).trim();
98 String categ
= getArticleCategory(doc
, article
, defaultCateg
)
100 String details
= getArticleDetails(doc
, article
).trim();
101 String intUrl
= getArticleIntUrl(doc
, article
).trim();
102 String extUrl
= getArticleExtUrl(doc
, article
).trim();
103 String content
= getArticleContent(doc
, article
).trim();
105 if (id
.isEmpty() && date
.isEmpty()) {
110 id
= date
.replace(":", "_").replace("+", "_");
115 list
.add(new Story(getType(), id
, title
, author
, date
, categ
,
116 details
, intUrl
, extUrl
, content
));
124 * The {@link URL}s to process for this website.
126 * @return the list of {@link URL}s
128 * @throws IOException
129 * in case of I/O error
131 abstract protected List
<Entry
<URL
, String
>> getUrls() throws IOException
;
134 * The article {@link Element}s of this document.
137 * the main document for the current category
139 * @return the articles
141 abstract protected List
<Element
> getArticles(Document doc
);
144 * The ID of the article (defaults to the date element if empty).
147 * the main document for the current category
149 * the article to look into
153 abstract protected String
getArticleId(Document doc
, Element article
);
156 * The article title to display.
159 * the main document for the current category
161 * the article to look into
165 abstract protected String
getArticleTitle(Document doc
, Element article
);
168 * The optional article author.
171 * the main document for the current category
173 * the article to look into
177 abstract protected String
getArticleAuthor(Document doc
, Element article
);
180 * The optional article date.
183 * the main document for the current category
185 * the article to look into
189 abstract protected String
getArticleDate(Document doc
, Element article
);
192 * the optional article category.
195 * the main document for the current category
197 * the article to look into
198 * @param currentCategory
199 * the currently listed category if any (can be NULL)
201 * @return the category
203 abstract protected String
getArticleCategory(Document doc
, Element article
,
204 String currentCategory
);
207 * the optional details of the article (can replace the date, author and
208 * category, for instance).
211 * the main document for the current category
213 * the article to look into
215 * @return the details
217 abstract protected String
getArticleDetails(Document doc
, Element article
);
220 * The (required) {@link URL} that points to the news page on the supported
224 * the main document for the current category
226 * the article to look into
228 * @return the internal {@link URL}
230 abstract protected String
getArticleIntUrl(Document doc
, Element article
);
233 * the optional {@link URL} that points to an external website for more
237 * the main document for the current category
239 * the article to look into
241 * @return the external {@link URL}
243 abstract protected String
getArticleExtUrl(Document doc
, Element article
);
246 * The optional article short-content (not the full content, that will be
247 * fetched by {@link BasicSupport#fetch(Story)}).
250 * the main document for the current category
252 * the article to look into
254 * @return the short content
256 abstract protected String
getArticleContent(Document doc
, Element article
);
259 * Fetch the full article content as well as all the comments associated to
260 * this {@link Story}, if any (can be empty, but not NULL).
263 * the story to fetch the comments of
265 * @throws IOException
266 * in case of I/O error
268 public void fetch(Story story
) throws IOException
{
269 String fullContent
= "";
271 URL url
= new URL(story
.getUrlInternal());
272 InputStream in
= downloader
.open(url
);
274 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
275 Element article
= getFullArticle(doc
);
276 if (article
!= null) {
277 StringBuilder builder
= new StringBuilder();
278 ElementProcessor eProc
= getElementProcessorFullArticle();
280 for (String line
: toLines(article
, eProc
)) {
281 builder
.append(line
+ "\n");
284 builder
.append(article
.text());
287 // Content is too tight with a single break per line:
288 fullContent
= builder
.toString().replace("\n", "\n\n") //
289 .replace("\n\n\n\n", "\n\n") //
290 .replace("\n\n\n\n", "\n\n") //
294 if (fullContent
.isEmpty()) {
295 fullContent
= story
.getContent();
298 story
.setFullContent(fullContent
);
299 story
.setComments(getComments(doc
,
300 getFullArticleCommentPosts(doc
, url
)));
309 * Return the full article if available.
312 * the (full article) document to work on
314 * @return the article or NULL
316 abstract protected Element
getFullArticle(Document doc
);
319 * Return the list of comment {@link Element}s from this optional container
320 * -- must <b>NOT</b> return the "container" as a comment {@link Element}.
323 * the (full article) document to work on
325 * the internal {@link URL} this article wa taken from (the
326 * {@link URL} from the supported website)
328 * @return the list of comment posts
330 abstract protected List
<Element
> getFullArticleCommentPosts(Document doc
,
334 * The {@link ElementProcessor} to use to convert the main article element
335 * (see {@link BasicSupport#getFullArticle(Document)}) into text.
337 * See {@link BasicElementProcessor} for a working, basic implementation.
339 * Can be NULL to simply use {@link Element#text()}.
341 * @return the processor, or NULL
343 abstract protected ElementProcessor
getElementProcessorFullArticle();
346 * Convert the comment elements into {@link Comment}s
349 * the document we work on
351 * the comment elements
353 * @return the converted {@link Comment}s
355 private List
<Comment
> getComments(Document doc
, List
<Element
> posts
) {
356 List
<Comment
> comments
= new ArrayList
<Comment
>();
358 for (Element post
: posts
) {
359 String id
= getCommentId(post
).trim();
360 String author
= getCommentAuthor(post
).trim();
361 String title
= getCommentTitle(post
).trim();
362 String date
= getCommentDate(post
).trim();
364 List
<String
> content
= new ArrayList
<String
>();
372 Element contentE
= getCommentContentElement(post
);
373 if (contentE
!= null) {
374 ElementProcessor eProc
= getElementProcessorComment();
376 for (String line
: toLines(contentE
, eProc
)) {
380 content
= Arrays
.asList(contentE
.text().split("\n"));
384 Comment comment
= new Comment(id
, author
, title
, date
, content
);
385 comment
.addAll(getComments(doc
,
386 getCommentCommentPosts(doc
, post
)));
388 if (!comment
.isEmpty()) {
389 comments
.add(comment
);
398 * Return the list of subcomment {@link Element}s from this comment element
399 * -- must <b>NOT</b> return the "container" as a comment {@link Element}.
402 * the (full article) document to work on
404 * the container (a comment {@link Element})
406 * @return the list of comment posts
408 abstract protected List
<Element
> getCommentCommentPosts(Document doc
,
412 * Compute the ID of the given comment element.
415 * the comment element
419 abstract protected String
getCommentId(Element post
);
422 * Compute the author of the given comment element.
425 * the comment element
429 abstract protected String
getCommentAuthor(Element post
);
432 * Compute the title of the given comment element.
435 * the comment element
439 abstract protected String
getCommentTitle(Element post
);
442 * Compute the date of the given comment element.
445 * the comment element
449 abstract protected String
getCommentDate(Element post
);
452 * Get the main of the given comment element, which can be NULL.
455 * the comment element
457 * @return the element
459 abstract protected Element
getCommentContentElement(Element post
);
462 * The {@link ElementProcessor} to use to convert the main comment element
463 * (see {@link BasicSupport#getCommentContentElement(Element)}) into text.
465 * See {@link BasicElementProcessor} for a working, basic implementation.
467 * Can be NULL to simply use {@link Element#text()}.
469 * @return the processor
471 abstract protected ElementProcessor
getElementProcessorComment();
479 protected void setType(Type type
) {
484 * The {@link String} to append to the selector (the selector will be
485 * constructed as "this string" then "/type/".
488 * the preselector to set
490 static public void setPreselector(String preselector
) {
491 BasicSupport
.preselector
= preselector
;
495 * Return a {@link BasicSupport} that is compatible with the given
496 * {@link Type} if it exists (or NULL if not).
501 * @return a compatible {@link BasicSupport} if it exists (or NULL if not)
503 static public BasicSupport
getSupport(Type type
) {
504 BasicSupport support
= null;
509 support
= new Slashdot();
512 support
= new Pipedot();
518 support
= new LeMonde();
521 support
= new TheRegister();
524 support
= new TooLinux();
527 support
= new EreNumerique();
530 support
= new Phoronix();
534 if (support
!= null) {
535 support
.setType(type
);
543 * The gopher "selector" to use for output for this type, using the
546 * A kind of "URL path", like "/news/" or "/misc/news/" or...
549 * the type to get the selector of
551 * @return the selector
553 static public String
getSelector(Type type
) {
554 return preselector
+ "/" + type
+ "/";
558 * Process the given element into text (each line is a text paragraph and
559 * can be prepended with ">" signs to indicate a quote or sub-quote or
563 * the element to process
564 * @param elementProcessor
565 * the element processor, must not be NULL
567 * @return text lines, each line is a paragraph
569 static protected List
<String
> toLines(Element element
,
570 final ElementProcessor elementProcessor
) {
571 final List
<String
> lines
= new ArrayList
<String
>();
572 final StringBuilder currentLine
= new StringBuilder();
573 final List
<Integer
> quoted
= new ArrayList
<Integer
>();
574 final List
<Node
> ignoredNodes
= new ArrayList
<Node
>();
575 final List
<String
> footnotes
= new ArrayList
<String
>();
577 if (element
!= null) {
578 new NodeTraversor(new NodeVisitor() {
580 public void head(Node node
, int depth
) {
581 String manual
= null;
582 boolean ignore
= elementProcessor
.ignoreNode(node
)
583 || ignoredNodes
.contains(node
.parentNode());
586 manual
= elementProcessor
.manualProcessing(node
);
587 if (manual
!= null) {
588 currentLine
.append(manual
);
595 String subtitle
= elementProcessor
.isSubtitle(node
);
596 if (subtitle
!= null) {
597 subtitle
= subtitle
.trim();
598 currentLine
.append("\n[ " + subtitle
+ " ]\n");
605 if (node
instanceof Element
) {
606 Element el
= (Element
) node
;
607 if ("pre".equals(el
.tagName())) {
608 currentLine
.append(StringUtils
609 .unhtml(el
.text()).trim());
616 ignoredNodes
.add(node
);
621 for (int i
= 0; i
< quoted
.size(); i
++) {
626 boolean enterQuote
= elementProcessor
.detectQuote(node
);
627 boolean leaveQuote
= quoted
.contains(depth
);
634 quoted
.remove(Integer
.valueOf(depth
));
637 if (enterQuote
|| leaveQuote
) {
638 if (currentLine
.length() > 0) {
639 if (currentLine
.charAt(currentLine
.length() - 1) == '\n') {
640 currentLine
.setLength(currentLine
.length() - 1);
642 for (String l
: currentLine
.toString().split("\n")) {
646 currentLine
.setLength(0);
649 if (node
instanceof Element
) {
650 Element element
= (Element
) node
;
651 boolean block
= element
.isBlock()
652 || element
.tagName().equalsIgnoreCase("br");
653 if (block
&& currentLine
.length() > 0) {
654 currentLine
.append("\n");
657 if (!element
.absUrl("href").trim().isEmpty()) {
658 footnotes
.add(element
.absUrl("href"));
659 currentLine
.append("[" + footnotes
.size() + "]");
661 } else if (node
instanceof TextNode
) {
662 TextNode textNode
= (TextNode
) node
;
663 String line
= StringUtil
.normaliseWhitespace(textNode
666 currentLine
.append(elementProcessor
.processText(line
));
667 currentLine
.append(" ");
672 public void tail(Node node
, int depth
) {
674 }).traverse(element
);
677 if (currentLine
.length() > 0) {
679 for (int i
= 0; i
< quoted
.size(); i
++) {
683 if (currentLine
.length() > 0) {
684 if (currentLine
.charAt(currentLine
.length() - 1) == '\n') {
685 currentLine
.setLength(currentLine
.length() - 1);
687 for (String l
: currentLine
.toString().split("\n")) {
693 // Fix spaces and nbsp, remove multiple following blank lines
694 List
<String
> linesCopy
= new ArrayList
<String
>(lines
.size());
696 for (int i
= 0; i
< lines
.size(); i
++) {
697 String line
= lines
.get(i
).replace(" ", " ") // nbsp -> space
698 .replace(" ", " ").trim();
699 if (line
.isEmpty()) {
710 // Footnotes insertion
711 if (footnotes
.size() > 0) {
716 for (int i
= 0; i
< footnotes
.size(); i
++) {
717 linesCopy
.add("[" + (i
+ 1) + "] " + footnotes
.get(i
));
725 * Reformat the date if possible.
730 * @return the reformated date, or the same value if it was not parsable
732 static private String
date(String date
) {
733 SimpleDateFormat out
= new SimpleDateFormat("yyyy/MM/dd");
737 epoch
= Long
.parseLong(date
.trim());
738 } catch (Exception e
) {
743 return out
.format(new Date(1000 * epoch
));
747 Date dat
= new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")
749 return out
.format(dat
);
750 } catch (ParseException e
) {