1 package be
.nikiroo
.gofetch
.support
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.text
.SimpleDateFormat
;
7 import java
.util
.ArrayList
;
8 import java
.util
.Arrays
;
10 import java
.util
.HashMap
;
11 import java
.util
.List
;
13 import java
.util
.Map
.Entry
;
15 import org
.jsoup
.helper
.DataUtil
;
16 import org
.jsoup
.helper
.StringUtil
;
17 import org
.jsoup
.nodes
.Document
;
18 import org
.jsoup
.nodes
.Element
;
19 import org
.jsoup
.nodes
.Node
;
20 import org
.jsoup
.nodes
.TextNode
;
21 import org
.jsoup
.select
.NodeTraversor
;
22 import org
.jsoup
.select
.NodeVisitor
;
24 import be
.nikiroo
.gofetch
.data
.Comment
;
25 import be
.nikiroo
.gofetch
.data
.Story
;
26 import be
.nikiroo
.utils
.Downloader
;
27 import be
.nikiroo
.utils
.StringUtils
;
30 * Base class for website support.
34 public abstract class BasicSupport
{
36 * The {@link Downloader} to use for all web sites via
37 * {@link BasicSupport#open(URL)}
39 static private Downloader downloader
= new Downloader("gofetcher");
41 static private String preselector
;
44 * The optional cookies to use to get the site data.
46 private Map
<String
, String
> cookies
= new HashMap
<String
, String
>();
51 * Login on the web site (this method does nothing by default, but can be
52 * overridden if needed).
55 * in case of I/O error
58 @SuppressWarnings("unused")
59 public void login() throws IOException
{
63 * The website textual description, to add in the dispatcher page.
67 * @return the description
69 abstract public String
getDescription();
72 * The gopher "selector" to use for output.
74 * A kind of "URL path", like "/news/" or "/misc/news/" or...
76 * @return the selector
78 public String
getSelector() {
79 return getSelector(getType());
87 public Type
getType() {
92 * List all the recent items, but only assure the ID and internal URL to
93 * fetch it later on (until it has been fetched, the rest of the
94 * {@link Story} is not confirmed).
96 * @return the list of new stories
101 public List
<Story
> list() throws IOException
{
102 List
<Story
> list
= new ArrayList
<Story
>();
105 for (Entry
<URL
, String
> entry
: getUrls()) {
106 URL url
= entry
.getKey();
107 String defaultCateg
= entry
.getValue();
108 if (defaultCateg
== null) {
112 InputStream in
= open(url
);
113 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
114 List
<Element
> articles
= getArticles(doc
);
115 for (Element article
: articles
) {
116 String id
= getArticleId(doc
, article
).trim();
117 String title
= getArticleTitle(doc
, article
).trim();
118 String author
= getArticleAuthor(doc
, article
).trim();
119 String date
= getArticleDate(doc
, article
).trim();
120 String categ
= getArticleCategory(doc
, article
, defaultCateg
)
122 String details
= getArticleDetails(doc
, article
).trim();
123 String intUrl
= getArticleIntUrl(doc
, article
).trim();
124 String extUrl
= getArticleExtUrl(doc
, article
).trim();
125 String content
= getArticleContent(doc
, article
).trim();
127 if (id
.isEmpty() && date
.isEmpty()) {
132 while (id
.length() < 10) {
136 id
= date
.replace(":", "_").replace("+", "_")
142 list
.add(new Story(getType(), id
, title
, author
, date
, categ
,
143 details
, intUrl
, extUrl
, content
));
151 * The {@link URL}s to process for this website.
153 * @return the list of {@link URL}s
155 * @throws IOException
156 * in case of I/O error
158 abstract protected List
<Entry
<URL
, String
>> getUrls() throws IOException
;
161 * The article {@link Element}s of this document.
164 * the main document for the current category
166 * @return the articles
168 abstract protected List
<Element
> getArticles(Document doc
);
171 * The ID of the article (defaults to the date element if empty).
174 * the main document for the current category
176 * the article to look into
180 abstract protected String
getArticleId(Document doc
, Element article
);
183 * The article title to display.
186 * the main document for the current category
188 * the article to look into
192 abstract protected String
getArticleTitle(Document doc
, Element article
);
195 * The optional article author.
198 * the main document for the current category
200 * the article to look into
204 abstract protected String
getArticleAuthor(Document doc
, Element article
);
207 * The optional article date.
210 * the main document for the current category
212 * the article to look into
216 abstract protected String
getArticleDate(Document doc
, Element article
);
219 * the optional article category.
222 * the main document for the current category
224 * the article to look into
225 * @param currentCategory
226 * the currently listed category if any (can be NULL)
228 * @return the category
230 abstract protected String
getArticleCategory(Document doc
, Element article
,
231 String currentCategory
);
234 * the optional details of the article (can replace the date, author and
235 * category, for instance).
238 * the main document for the current category
240 * the article to look into
242 * @return the details
244 abstract protected String
getArticleDetails(Document doc
, Element article
);
247 * The (required) {@link URL} that points to the news page on the supported
251 * the main document for the current category
253 * the article to look into
255 * @return the internal {@link URL}
257 abstract protected String
getArticleIntUrl(Document doc
, Element article
);
260 * the optional {@link URL} that points to an external website for more
264 * the main document for the current category
266 * the article to look into
268 * @return the external {@link URL}
270 abstract protected String
getArticleExtUrl(Document doc
, Element article
);
273 * The optional article short-content (not the full content, that will be
274 * fetched by {@link BasicSupport#fetch(Story)}).
277 * the main document for the current category
279 * the article to look into
281 * @return the short content
283 abstract protected String
getArticleContent(Document doc
, Element article
);
286 * Fetch the full article content as well as all the comments associated to
287 * this {@link Story}, if any (can be empty, but not NULL).
290 * the story to fetch the comments of
292 * @throws IOException
293 * in case of I/O error
295 public void fetch(Story story
) throws IOException
{
296 String fullContent
= "";
298 URL url
= new URL(story
.getUrlInternal());
299 InputStream in
= open(url
);
301 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
302 Element article
= getFullArticle(doc
);
303 if (article
!= null) {
304 fullContent
= getArticleText(article
);
307 if (fullContent
.isEmpty()) {
308 fullContent
= story
.getContent();
311 story
.setFullContent(fullContent
);
312 story
.setComments(getComments(doc
,
313 getFullArticleCommentPosts(doc
, url
)));
322 * Return the text from this {@link Element}, using the
323 * {@link BasicSupport#getElementProcessorFullArticle()} processor logic.
326 * the element to extract the text from
330 protected String
getArticleText(Element article
) {
331 StringBuilder builder
= new StringBuilder();
332 ElementProcessor eProc
= getElementProcessorFullArticle();
334 for (String line
: toLines(article
, eProc
)) {
335 builder
.append(line
+ "\n");
338 builder
.append(article
.text());
341 // Content is too tight with a single break per line:
342 return builder
.toString().replace("\n", "\n\n") //
343 .replace("\n\n\n\n", "\n\n") //
344 .replace("\n\n\n\n", "\n\n") //
349 * Return the full article if available (this is the article to retrieve
350 * from the newly downloaded page at {@link Story#getUrlInternal()}).
353 * the (full article) document to work on
355 * @return the article or NULL
357 abstract protected Element
getFullArticle(Document doc
);
360 * Return the list of comment {@link Element}s from this optional container
361 * -- must <b>NOT</b> return the "container" as a comment {@link Element}.
364 * the (full article) document to work on
366 * the internal {@link URL} this article wa taken from (the
367 * {@link URL} from the supported website)
369 * @return the list of comment posts
371 abstract protected List
<Element
> getFullArticleCommentPosts(Document doc
,
375 * The {@link ElementProcessor} to use to convert the main article element
376 * (see {@link BasicSupport#getFullArticle(Document)}) into text.
378 * See {@link BasicElementProcessor} for a working, basic implementation.
380 * Can be NULL to simply use {@link Element#text()}.
382 * @return the processor, or NULL
384 abstract protected ElementProcessor
getElementProcessorFullArticle();
387 * Open a network resource.
389 * You need to close the returned {@link InputStream} when done.
394 * @return the content
396 * @throws IOException
397 * in case of I/O error
399 protected InputStream
open(URL url
) throws IOException
{
400 return downloader
.open(url
, url
, cookies
, null, null, null);
404 * Convert the comment elements into {@link Comment}s
407 * the document we work on
409 * the comment elements
411 * @return the converted {@link Comment}s
413 private List
<Comment
> getComments(Document doc
, List
<Element
> posts
) {
414 List
<Comment
> comments
= new ArrayList
<Comment
>();
416 for (Element post
: posts
) {
417 String id
= getCommentId(post
).trim();
418 String author
= getCommentAuthor(post
).trim();
419 String title
= getCommentTitle(post
).trim();
420 String date
= getCommentDate(post
).trim();
422 List
<String
> content
= new ArrayList
<String
>();
430 Element contentE
= getCommentContentElement(post
);
431 if (contentE
!= null) {
432 ElementProcessor eProc
= getElementProcessorComment();
434 for (String line
: toLines(contentE
, eProc
)) {
438 content
= Arrays
.asList(contentE
.text().split("\n"));
442 Comment comment
= new Comment(id
, author
, title
, date
, content
);
443 comment
.addAll(getComments(doc
,
444 getCommentCommentPosts(doc
, post
)));
446 if (!comment
.isEmpty()) {
447 comments
.add(comment
);
456 * Return the list of subcomment {@link Element}s from this comment element
457 * -- must <b>NOT</b> return the "container" as a comment {@link Element}.
460 * the (full article) document to work on
462 * the container (a comment {@link Element})
464 * @return the list of comment posts
466 abstract protected List
<Element
> getCommentCommentPosts(Document doc
,
470 * Compute the ID of the given comment element.
473 * the comment element
477 abstract protected String
getCommentId(Element post
);
480 * Compute the author of the given comment element.
483 * the comment element
487 abstract protected String
getCommentAuthor(Element post
);
490 * Compute the title of the given comment element.
493 * the comment element
497 abstract protected String
getCommentTitle(Element post
);
500 * Compute the date of the given comment element.
503 * the comment element
507 abstract protected String
getCommentDate(Element post
);
510 * Get the main of the given comment element, which can be NULL.
513 * the comment element
515 * @return the element
517 abstract protected Element
getCommentContentElement(Element post
);
520 * The {@link ElementProcessor} to use to convert the main comment element
521 * (see {@link BasicSupport#getCommentContentElement(Element)}) into text.
523 * See {@link BasicElementProcessor} for a working, basic implementation.
525 * Can be NULL to simply use {@link Element#text()}.
527 * @return the processor
529 abstract protected ElementProcessor
getElementProcessorComment();
537 protected void setType(Type type
) {
542 * Add a cookie for all site connections.
549 protected void addCookie(String name
, String value
) {
550 cookies
.put(name
, value
);
554 * The {@link String} to append to the selector (the selector will be
555 * constructed as "this string" then "/type/".
558 * the preselector to set
560 static public void setPreselector(String preselector
) {
561 BasicSupport
.preselector
= preselector
;
565 * Return a {@link BasicSupport} that is compatible with the given
566 * {@link Type} if it exists (or NULL if not).
571 * @return a compatible {@link BasicSupport} if it exists (or NULL if not)
573 static public BasicSupport
getSupport(Type type
) {
574 BasicSupport support
= null;
579 support
= new Slashdot();
585 support
= new LeMonde();
588 support
= new TheRegister();
591 support
= new TooLinux();
594 support
= new EreNumerique();
597 support
= new Phoronix();
600 support
= new SeptSurSept();
603 support
= new Reddit();
607 if (support
!= null) {
608 support
.setType(type
);
616 * The gopher "selector" to use for output for this type, using the
619 * A kind of "URL path", like "/news/" or "/misc/news/" or...
622 * the type to get the selector of
624 * @return the selector
626 static public String
getSelector(Type type
) {
627 return preselector
+ "/" + type
+ "/";
631 * Process the given element into text (each line is a text paragraph and
632 * can be prepended with ">" signs to indicate a quote or sub-quote or
636 * the element to process
637 * @param elementProcessor
638 * the element processor, must not be NULL
640 * @return text lines, each line is a paragraph
642 static protected List
<String
> toLines(Element element
,
643 final ElementProcessor elementProcessor
) {
644 final List
<String
> lines
= new ArrayList
<String
>();
645 final StringBuilder currentLine
= new StringBuilder();
646 final List
<Integer
> quoted
= new ArrayList
<Integer
>();
647 final List
<Node
> ignoredNodes
= new ArrayList
<Node
>();
648 final List
<String
> footnotes
= new ArrayList
<String
>();
650 if (element
!= null) {
651 new NodeTraversor(new NodeVisitor() {
653 public void head(Node node
, int depth
) {
654 String manual
= null;
655 boolean ignore
= elementProcessor
.ignoreNode(node
)
656 || ignoredNodes
.contains(node
.parentNode());
659 manual
= elementProcessor
.manualProcessing(node
);
660 if (manual
!= null) {
661 currentLine
.append(manual
);
668 String subtitle
= elementProcessor
.isSubtitle(node
);
669 if (subtitle
!= null) {
670 subtitle
= subtitle
.trim();
671 currentLine
.append("\n[ " + subtitle
+ " ]\n");
678 if (node
instanceof Element
) {
679 Element el
= (Element
) node
;
680 if ("pre".equals(el
.tagName())) {
681 currentLine
.append(StringUtils
682 .unhtml(el
.text()).trim());
689 ignoredNodes
.add(node
);
694 for (int i
= 0; i
< quoted
.size(); i
++) {
699 boolean enterQuote
= elementProcessor
.detectQuote(node
);
700 boolean leaveQuote
= quoted
.contains(depth
);
707 quoted
.remove(Integer
.valueOf(depth
));
710 if (enterQuote
|| leaveQuote
) {
711 if (currentLine
.length() > 0) {
712 if (currentLine
.charAt(currentLine
.length() - 1) == '\n') {
713 currentLine
.setLength(currentLine
.length() - 1);
715 for (String l
: currentLine
.toString().split("\n")) {
719 currentLine
.setLength(0);
722 if (node
instanceof Element
) {
723 Element element
= (Element
) node
;
724 boolean block
= element
.isBlock()
725 || element
.tagName().equalsIgnoreCase("br");
726 if (block
&& currentLine
.length() > 0) {
727 currentLine
.append("\n");
730 if (!element
.absUrl("href").trim().isEmpty()) {
731 footnotes
.add(element
.absUrl("href"));
732 currentLine
.append("[" + footnotes
.size() + "]");
734 } else if (node
instanceof TextNode
) {
735 TextNode textNode
= (TextNode
) node
;
736 String line
= StringUtil
.normaliseWhitespace(textNode
739 currentLine
.append(elementProcessor
.processText(line
));
740 currentLine
.append(" ");
745 public void tail(Node node
, int depth
) {
747 }).traverse(element
);
750 if (currentLine
.length() > 0) {
752 for (int i
= 0; i
< quoted
.size(); i
++) {
756 if (currentLine
.length() > 0) {
757 if (currentLine
.charAt(currentLine
.length() - 1) == '\n') {
758 currentLine
.setLength(currentLine
.length() - 1);
760 for (String l
: currentLine
.toString().split("\n")) {
766 // Fix spaces and nbsp, remove multiple following blank lines
767 List
<String
> linesCopy
= new ArrayList
<String
>(lines
.size());
769 for (int i
= 0; i
< lines
.size(); i
++) {
770 String line
= lines
.get(i
).replace(" ", " ") // nbsp -> space
771 .replace(" ", " ").trim();
772 if (line
.isEmpty()) {
783 // Footnotes insertion
784 if (footnotes
.size() > 0) {
789 for (int i
= 0; i
< footnotes
.size(); i
++) {
790 linesCopy
.add("[" + (i
+ 1) + "] " + footnotes
.get(i
));
798 * Reformat the date if possible.
803 * @return the reformated date, or the same value if it was not parsable
805 static private String
date(String date
) {
806 SimpleDateFormat out
= new SimpleDateFormat("yyyy/MM/dd");
810 epoch
= Long
.parseLong(date
.trim());
811 } catch (Exception e
) {
816 return out
.format(new Date(1000 * epoch
));
820 Date dat
= new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")
822 return out
.format(dat
);
823 } catch (Exception e
) {