1 package be
.nikiroo
.gofetch
.support
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.text
.ParseException
;
7 import java
.text
.SimpleDateFormat
;
8 import java
.util
.ArrayList
;
9 import java
.util
.Arrays
;
10 import java
.util
.Date
;
11 import java
.util
.HashMap
;
12 import java
.util
.List
;
14 import java
.util
.Map
.Entry
;
16 import org
.jsoup
.helper
.DataUtil
;
17 import org
.jsoup
.helper
.StringUtil
;
18 import org
.jsoup
.nodes
.Document
;
19 import org
.jsoup
.nodes
.Element
;
20 import org
.jsoup
.nodes
.Node
;
21 import org
.jsoup
.nodes
.TextNode
;
22 import org
.jsoup
.select
.NodeTraversor
;
23 import org
.jsoup
.select
.NodeVisitor
;
25 import be
.nikiroo
.gofetch
.data
.Comment
;
26 import be
.nikiroo
.gofetch
.data
.Story
;
27 import be
.nikiroo
.utils
.Downloader
;
28 import be
.nikiroo
.utils
.StringUtils
;
31 * Base class for website support.
35 public abstract class BasicSupport
{
37 * The downloader to use for all web sites via
38 * {@link BasicSupport#open(URL)}
40 static private Downloader downloader
= new Downloader("gofetcher");
42 static private String preselector
;
45 * The optional cookies to use to get the site data.
47 private Map
<String
, String
> cookies
= new HashMap
<String
, String
>();
52 * Login on the web site (this method does nothing by default, but can be
53 * overridden if needed).
56 * in case of I/O error
59 public void login() throws IOException
{
63 * The website textual description, to add in the dispatcher page.
67 * @return the description
69 abstract public String
getDescription();
72 * The gopher "selector" to use for output.
74 * A kind of "URL path", like "/news/" or "/misc/news/" or...
76 * @return the selector
78 public String
getSelector() {
79 return getSelector(getType());
87 public Type
getType() {
92 * List all the recent items, but only assure the ID and internal URL to
93 * fetch it later on (until it has been fetched, the rest of the
94 * {@link Story} is not confirmed).
96 * @return the list of new stories
101 public List
<Story
> list() throws IOException
{
102 List
<Story
> list
= new ArrayList
<Story
>();
105 for (Entry
<URL
, String
> entry
: getUrls()) {
106 URL url
= entry
.getKey();
107 String defaultCateg
= entry
.getValue();
108 if (defaultCateg
== null) {
112 InputStream in
= open(url
);
113 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
114 List
<Element
> articles
= getArticles(doc
);
115 for (Element article
: articles
) {
116 String id
= getArticleId(doc
, article
).trim();
117 String title
= getArticleTitle(doc
, article
).trim();
118 String author
= getArticleAuthor(doc
, article
).trim();
119 String date
= getArticleDate(doc
, article
).trim();
120 String categ
= getArticleCategory(doc
, article
, defaultCateg
)
122 String details
= getArticleDetails(doc
, article
).trim();
123 String intUrl
= getArticleIntUrl(doc
, article
).trim();
124 String extUrl
= getArticleExtUrl(doc
, article
).trim();
125 String content
= getArticleContent(doc
, article
).trim();
127 if (id
.isEmpty() && date
.isEmpty()) {
132 while (id
.length() < 10) {
136 id
= date
.replace(":", "_").replace("+", "_");
141 list
.add(new Story(getType(), id
, title
, author
, date
, categ
,
142 details
, intUrl
, extUrl
, content
));
150 * The {@link URL}s to process for this website.
152 * @return the list of {@link URL}s
154 * @throws IOException
155 * in case of I/O error
157 abstract protected List
<Entry
<URL
, String
>> getUrls() throws IOException
;
160 * The article {@link Element}s of this document.
163 * the main document for the current category
165 * @return the articles
167 abstract protected List
<Element
> getArticles(Document doc
);
170 * The ID of the article (defaults to the date element if empty).
173 * the main document for the current category
175 * the article to look into
179 abstract protected String
getArticleId(Document doc
, Element article
);
182 * The article title to display.
185 * the main document for the current category
187 * the article to look into
191 abstract protected String
getArticleTitle(Document doc
, Element article
);
194 * The optional article author.
197 * the main document for the current category
199 * the article to look into
203 abstract protected String
getArticleAuthor(Document doc
, Element article
);
206 * The optional article date.
209 * the main document for the current category
211 * the article to look into
215 abstract protected String
getArticleDate(Document doc
, Element article
);
218 * the optional article category.
221 * the main document for the current category
223 * the article to look into
224 * @param currentCategory
225 * the currently listed category if any (can be NULL)
227 * @return the category
229 abstract protected String
getArticleCategory(Document doc
, Element article
,
230 String currentCategory
);
233 * the optional details of the article (can replace the date, author and
234 * category, for instance).
237 * the main document for the current category
239 * the article to look into
241 * @return the details
243 abstract protected String
getArticleDetails(Document doc
, Element article
);
246 * The (required) {@link URL} that points to the news page on the supported
250 * the main document for the current category
252 * the article to look into
254 * @return the internal {@link URL}
256 abstract protected String
getArticleIntUrl(Document doc
, Element article
);
259 * the optional {@link URL} that points to an external website for more
263 * the main document for the current category
265 * the article to look into
267 * @return the external {@link URL}
269 abstract protected String
getArticleExtUrl(Document doc
, Element article
);
272 * The optional article short-content (not the full content, that will be
273 * fetched by {@link BasicSupport#fetch(Story)}).
276 * the main document for the current category
278 * the article to look into
280 * @return the short content
282 abstract protected String
getArticleContent(Document doc
, Element article
);
285 * Fetch the full article content as well as all the comments associated to
286 * this {@link Story}, if any (can be empty, but not NULL).
289 * the story to fetch the comments of
291 * @throws IOException
292 * in case of I/O error
294 public void fetch(Story story
) throws IOException
{
295 String fullContent
= "";
297 URL url
= new URL(story
.getUrlInternal());
298 InputStream in
= open(url
);
300 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
301 Element article
= getFullArticle(doc
);
302 if (article
!= null) {
303 StringBuilder builder
= new StringBuilder();
304 ElementProcessor eProc
= getElementProcessorFullArticle();
306 for (String line
: toLines(article
, eProc
)) {
307 builder
.append(line
+ "\n");
310 builder
.append(article
.text());
313 // Content is too tight with a single break per line:
314 fullContent
= builder
.toString().replace("\n", "\n\n") //
315 .replace("\n\n\n\n", "\n\n") //
316 .replace("\n\n\n\n", "\n\n") //
320 if (fullContent
.isEmpty()) {
321 fullContent
= story
.getContent();
324 story
.setFullContent(fullContent
);
325 story
.setComments(getComments(doc
,
326 getFullArticleCommentPosts(doc
, url
)));
335 * Return the full article if available (this is the article to retrieve
336 * from the newly downloaded page at {@link Story#getUrlInternal()}).
339 * the (full article) document to work on
341 * @return the article or NULL
343 abstract protected Element
getFullArticle(Document doc
);
346 * Return the list of comment {@link Element}s from this optional container
347 * -- must <b>NOT</b> return the "container" as a comment {@link Element}.
350 * the (full article) document to work on
352 * the internal {@link URL} this article wa taken from (the
353 * {@link URL} from the supported website)
355 * @return the list of comment posts
357 abstract protected List
<Element
> getFullArticleCommentPosts(Document doc
,
361 * The {@link ElementProcessor} to use to convert the main article element
362 * (see {@link BasicSupport#getFullArticle(Document)}) into text.
364 * See {@link BasicElementProcessor} for a working, basic implementation.
366 * Can be NULL to simply use {@link Element#text()}.
368 * @return the processor, or NULL
370 abstract protected ElementProcessor
getElementProcessorFullArticle();
373 * Open a network resource.
375 * You need to close the returned {@link InputStream} when done.
380 * @return the content
382 * @throws IOException
383 * in case of I/O error
385 protected InputStream
open(URL url
) throws IOException
{
386 return downloader
.open(url
, url
, cookies
, null, null, null);
390 * Convert the comment elements into {@link Comment}s
393 * the document we work on
395 * the comment elements
397 * @return the converted {@link Comment}s
399 private List
<Comment
> getComments(Document doc
, List
<Element
> posts
) {
400 List
<Comment
> comments
= new ArrayList
<Comment
>();
402 for (Element post
: posts
) {
403 String id
= getCommentId(post
).trim();
404 String author
= getCommentAuthor(post
).trim();
405 String title
= getCommentTitle(post
).trim();
406 String date
= getCommentDate(post
).trim();
408 List
<String
> content
= new ArrayList
<String
>();
416 Element contentE
= getCommentContentElement(post
);
417 if (contentE
!= null) {
418 ElementProcessor eProc
= getElementProcessorComment();
420 for (String line
: toLines(contentE
, eProc
)) {
424 content
= Arrays
.asList(contentE
.text().split("\n"));
428 Comment comment
= new Comment(id
, author
, title
, date
, content
);
429 comment
.addAll(getComments(doc
,
430 getCommentCommentPosts(doc
, post
)));
432 if (!comment
.isEmpty()) {
433 comments
.add(comment
);
442 * Return the list of subcomment {@link Element}s from this comment element
443 * -- must <b>NOT</b> return the "container" as a comment {@link Element}.
446 * the (full article) document to work on
448 * the container (a comment {@link Element})
450 * @return the list of comment posts
452 abstract protected List
<Element
> getCommentCommentPosts(Document doc
,
456 * Compute the ID of the given comment element.
459 * the comment element
463 abstract protected String
getCommentId(Element post
);
466 * Compute the author of the given comment element.
469 * the comment element
473 abstract protected String
getCommentAuthor(Element post
);
476 * Compute the title of the given comment element.
479 * the comment element
483 abstract protected String
getCommentTitle(Element post
);
486 * Compute the date of the given comment element.
489 * the comment element
493 abstract protected String
getCommentDate(Element post
);
496 * Get the main of the given comment element, which can be NULL.
499 * the comment element
501 * @return the element
503 abstract protected Element
getCommentContentElement(Element post
);
506 * The {@link ElementProcessor} to use to convert the main comment element
507 * (see {@link BasicSupport#getCommentContentElement(Element)}) into text.
509 * See {@link BasicElementProcessor} for a working, basic implementation.
511 * Can be NULL to simply use {@link Element#text()}.
513 * @return the processor
515 abstract protected ElementProcessor
getElementProcessorComment();
523 protected void setType(Type type
) {
528 * Add a cookie for all site connections.
535 protected void addCookie(String name
, String value
) {
536 cookies
.put(name
, value
);
540 * The {@link String} to append to the selector (the selector will be
541 * constructed as "this string" then "/type/".
544 * the preselector to set
546 static public void setPreselector(String preselector
) {
547 BasicSupport
.preselector
= preselector
;
551 * Return a {@link BasicSupport} that is compatible with the given
552 * {@link Type} if it exists (or NULL if not).
557 * @return a compatible {@link BasicSupport} if it exists (or NULL if not)
559 static public BasicSupport
getSupport(Type type
) {
560 BasicSupport support
= null;
565 support
= new Slashdot();
568 support
= new Pipedot();
574 support
= new LeMonde();
577 support
= new TheRegister();
580 support
= new TooLinux();
583 support
= new EreNumerique();
586 support
= new Phoronix();
589 support
= new SeptSurSept();
593 if (support
!= null) {
594 support
.setType(type
);
602 * The gopher "selector" to use for output for this type, using the
605 * A kind of "URL path", like "/news/" or "/misc/news/" or...
608 * the type to get the selector of
610 * @return the selector
612 static public String
getSelector(Type type
) {
613 return preselector
+ "/" + type
+ "/";
617 * Process the given element into text (each line is a text paragraph and
618 * can be prepended with ">" signs to indicate a quote or sub-quote or
622 * the element to process
623 * @param elementProcessor
624 * the element processor, must not be NULL
626 * @return text lines, each line is a paragraph
628 static protected List
<String
> toLines(Element element
,
629 final ElementProcessor elementProcessor
) {
630 final List
<String
> lines
= new ArrayList
<String
>();
631 final StringBuilder currentLine
= new StringBuilder();
632 final List
<Integer
> quoted
= new ArrayList
<Integer
>();
633 final List
<Node
> ignoredNodes
= new ArrayList
<Node
>();
634 final List
<String
> footnotes
= new ArrayList
<String
>();
636 if (element
!= null) {
637 new NodeTraversor(new NodeVisitor() {
639 public void head(Node node
, int depth
) {
640 String manual
= null;
641 boolean ignore
= elementProcessor
.ignoreNode(node
)
642 || ignoredNodes
.contains(node
.parentNode());
645 manual
= elementProcessor
.manualProcessing(node
);
646 if (manual
!= null) {
647 currentLine
.append(manual
);
654 String subtitle
= elementProcessor
.isSubtitle(node
);
655 if (subtitle
!= null) {
656 subtitle
= subtitle
.trim();
657 currentLine
.append("\n[ " + subtitle
+ " ]\n");
664 if (node
instanceof Element
) {
665 Element el
= (Element
) node
;
666 if ("pre".equals(el
.tagName())) {
667 currentLine
.append(StringUtils
668 .unhtml(el
.text()).trim());
675 ignoredNodes
.add(node
);
680 for (int i
= 0; i
< quoted
.size(); i
++) {
685 boolean enterQuote
= elementProcessor
.detectQuote(node
);
686 boolean leaveQuote
= quoted
.contains(depth
);
693 quoted
.remove(Integer
.valueOf(depth
));
696 if (enterQuote
|| leaveQuote
) {
697 if (currentLine
.length() > 0) {
698 if (currentLine
.charAt(currentLine
.length() - 1) == '\n') {
699 currentLine
.setLength(currentLine
.length() - 1);
701 for (String l
: currentLine
.toString().split("\n")) {
705 currentLine
.setLength(0);
708 if (node
instanceof Element
) {
709 Element element
= (Element
) node
;
710 boolean block
= element
.isBlock()
711 || element
.tagName().equalsIgnoreCase("br");
712 if (block
&& currentLine
.length() > 0) {
713 currentLine
.append("\n");
716 if (!element
.absUrl("href").trim().isEmpty()) {
717 footnotes
.add(element
.absUrl("href"));
718 currentLine
.append("[" + footnotes
.size() + "]");
720 } else if (node
instanceof TextNode
) {
721 TextNode textNode
= (TextNode
) node
;
722 String line
= StringUtil
.normaliseWhitespace(textNode
725 currentLine
.append(elementProcessor
.processText(line
));
726 currentLine
.append(" ");
731 public void tail(Node node
, int depth
) {
733 }).traverse(element
);
736 if (currentLine
.length() > 0) {
738 for (int i
= 0; i
< quoted
.size(); i
++) {
742 if (currentLine
.length() > 0) {
743 if (currentLine
.charAt(currentLine
.length() - 1) == '\n') {
744 currentLine
.setLength(currentLine
.length() - 1);
746 for (String l
: currentLine
.toString().split("\n")) {
752 // Fix spaces and nbsp, remove multiple following blank lines
753 List
<String
> linesCopy
= new ArrayList
<String
>(lines
.size());
755 for (int i
= 0; i
< lines
.size(); i
++) {
756 String line
= lines
.get(i
).replace(" ", " ") // nbsp -> space
757 .replace(" ", " ").trim();
758 if (line
.isEmpty()) {
769 // Footnotes insertion
770 if (footnotes
.size() > 0) {
775 for (int i
= 0; i
< footnotes
.size(); i
++) {
776 linesCopy
.add("[" + (i
+ 1) + "] " + footnotes
.get(i
));
784 * Reformat the date if possible.
789 * @return the reformated date, or the same value if it was not parsable
791 static private String
date(String date
) {
792 SimpleDateFormat out
= new SimpleDateFormat("yyyy/MM/dd");
796 epoch
= Long
.parseLong(date
.trim());
797 } catch (Exception e
) {
802 return out
.format(new Date(1000 * epoch
));
806 Date dat
= new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")
808 return out
.format(dat
);
809 } catch (ParseException e
) {