1 package be
.nikiroo
.gofetch
.support
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.util
.ArrayList
;
9 import org
.jsoup
.helper
.DataUtil
;
10 import org
.jsoup
.nodes
.Document
;
11 import org
.jsoup
.nodes
.Element
;
12 import org
.jsoup
.nodes
.Node
;
13 import org
.jsoup
.select
.Elements
;
15 import be
.nikiroo
.gofetch
.data
.Comment
;
16 import be
.nikiroo
.gofetch
.data
.Story
;
19 * Support <a href='https://slashdot.org/'>https://slashdot.org/</a>.
23 public class Slashdot
extends BasicSupport
{
25 public String
getDescription() {
26 return "Slashdot: News for nerds, stuff that matters!";
30 public List
<Story
> list() throws IOException
{
31 List
<Story
> list
= new ArrayList
<Story
>();
33 URL url
= new URL("https://slashdot.org/");
34 InputStream in
= open(url
);
35 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
36 Elements articles
= doc
.getElementsByTag("header");
37 for (Element article
: articles
) {
38 Elements titles
= article
.getElementsByClass("story-title");
39 if (titles
.size() == 0) {
43 Element title
= titles
.get(0);
45 String id
= "" + title
.attr("id");
46 if (id
.startsWith("title-")) {
47 id
= id
.substring("title-".length());
50 Elements links
= title
.getElementsByTag("a");
53 if (links
.size() > 0) {
54 intUrl
= links
.get(0).absUrl("href");
56 if (links
.size() > 1) {
57 extUrl
= links
.get(1).absUrl("href");
61 Elements detailsElements
= article
.getElementsByClass("details");
62 if (detailsElements
.size() > 0) {
63 details
= detailsElements
.get(0).text();
67 Element bodyElement
= doc
.getElementById("text-" + id
);
68 if (bodyElement
!= null) {
69 body
= bodyElement
.text();
72 list
.add(new Story(getType(), id
, title
.text(), details
, intUrl
,
80 public void fetch(Story story
) throws IOException
{
81 List
<Comment
> comments
= new ArrayList
<Comment
>();
83 URL url
= new URL(story
.getUrlInternal());
84 InputStream in
= open(url
);
85 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
86 Element listing
= doc
.getElementById("commentlisting");
87 if (listing
!= null) {
88 comments
.addAll(getComments(listing
));
91 story
.setComments(comments
);
94 private List
<Comment
> getComments(Element listing
) {
95 List
<Comment
> comments
= new ArrayList
<Comment
>();
96 Comment lastComment
= null;
97 for (Element commentElement
: listing
.children()) {
98 if (commentElement
.hasClass("comment")) {
99 if (!commentElement
.hasClass("hidden")) {
100 lastComment
= getComment(commentElement
);
101 comments
.add(lastComment
);
104 List
<Comment
> subComments
= new ArrayList
<Comment
>();
105 for (Element child
: commentElement
.children()) {
106 if (child
.id().contains("commtree_")) {
107 subComments
.addAll(getComments(child
));
111 if (lastComment
== null) {
112 comments
.addAll(subComments
);
114 lastComment
.addAll(subComments
);
123 * Get a comment from the given element.
125 * @param commentElement
126 * the element to get the comment of.
128 * @return the comment, <b>NOT</b> including sub-comments
130 private Comment
getComment(Element commentElement
) {
131 String title
= firstOrEmpty(commentElement
, "title").text();
132 String author
= firstOrEmpty(commentElement
, "by").text();
133 String date
= firstOrEmpty(commentElement
, "otherdetails").text();
134 Element content
= firstOrEmpty(commentElement
, "commentBody");
136 return new Comment(commentElement
.id(), author
, title
, date
,
140 private List
<String
> toLines(Element element
) {
141 return toLines(element
, new QuoteProcessor() {
143 public String
processText(String text
) {
144 while (text
.startsWith(">")) { // comment in one-liners
145 text
= text
.substring(1).trim();
152 public boolean detectQuote(Node node
) {
153 if (node
instanceof Element
) {
154 Element elementNode
= (Element
) node
;
155 if (elementNode
.tagName().equals("blockquote")
156 || elementNode
.hasClass("quote")
157 || (elementNode
.tagName().equals("p")
158 && elementNode
.textNodes().size() == 1 && elementNode
159 .textNodes().get(0).getWholeText()
169 public boolean ignoreNode(Node node
) {
174 public String
manualProcessing(Node node
) {