1 package be
.nikiroo
.gofetch
.support
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.util
.ArrayList
;
9 import org
.jsoup
.helper
.DataUtil
;
10 import org
.jsoup
.nodes
.Document
;
11 import org
.jsoup
.nodes
.Element
;
12 import org
.jsoup
.nodes
.Node
;
13 import org
.jsoup
.select
.Elements
;
15 import be
.nikiroo
.gofetch
.data
.Comment
;
16 import be
.nikiroo
.gofetch
.data
.Story
;
19 * Support <a href='https://pipedot.org/'>https://pipedot.org/</a>.
23 public class Pipedot
extends BasicSupport
{
25 public String
getDescription() {
26 return "Pipedot: News for nerds, without the corporate slant";
30 public List
<Story
> list() throws IOException
{
31 List
<Story
> list
= new ArrayList
<Story
>();
33 URL url
= new URL("https://pipedot.org/");
34 InputStream in
= open(url
);
35 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
36 Elements articles
= doc
.getElementsByClass("story");
37 for (Element article
: articles
) {
38 Elements titles
= article
.getElementsByTag("h1");
39 if (titles
.size() == 0) {
43 Element title
= titles
.get(0);
46 for (Element idElem
: article
.getElementsByTag("a")) {
47 if (idElem
.attr("href").startsWith("/pipe/")) {
48 id
= idElem
.attr("href").substring("/pipe/".length());
56 Elements links
= article
.getElementsByTag("a");
57 if (links
.size() > 0) {
58 intUrl
= links
.get(0).absUrl("href");
61 // Take first ext URL as original source
62 for (Element link
: links
) {
63 String uuu
= link
.absUrl("href");
64 if (!uuu
.isEmpty() && !uuu
.contains("pipedot.org/")) {
71 Elements detailsElements
= article
.getElementsByTag("div");
72 if (detailsElements
.size() > 0) {
73 details
= detailsElements
.get(0).text();
77 for (Element elem
: article
.children()) {
78 String tag
= elem
.tag().toString();
79 if (!tag
.equals("header") && !tag
.equals("footer")) {
85 list
.add(new Story(getType(), id
, title
.text(), details
, intUrl
,
93 public void fetch(Story story
) throws IOException
{
94 List
<Comment
> comments
= new ArrayList
<Comment
>();
96 URL url
= new URL(story
.getUrlInternal());
97 InputStream in
= open(url
);
98 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
99 Elements listing
= doc
.getElementsByTag("main");
100 if (listing
.size() > 0) {
101 comments
.addAll(getComments(listing
.get(0)));
104 story
.setComments(comments
);
107 private List
<Comment
> getComments(Element listing
) {
108 List
<Comment
> comments
= new ArrayList
<Comment
>();
109 for (Element commentElement
: listing
.children()) {
110 if (commentElement
.hasClass("comment")) {
111 Comment comment
= getComment(commentElement
);
112 if (!comment
.isEmpty()) {
113 comments
.add(comment
);
120 private Comment
getComment(Element commentElement
) {
121 String title
= firstOrEmptyTag(commentElement
, "h3").text();
122 String author
= firstOrEmpty(commentElement
, "h4").text();
123 Element content
= firstOrEmpty(commentElement
, "comment-body");
126 int pos
= author
.lastIndexOf(" on ");
128 date
= author
.substring(pos
+ " on ".length()).trim();
129 author
= author
.substring(0, pos
).trim();
132 Comment comment
= new Comment(commentElement
.id(), author
, title
, date
,
135 Elements commentOutline
= commentElement
136 .getElementsByClass("comment-outline");
137 if (commentOutline
.size() > 0) {
138 comment
.addAll(getComments(commentOutline
.get(0)));
144 private List
<String
> toLines(Element element
) {
145 return toLines(element
, new QuoteProcessor() {
147 public String
processText(String text
) {
152 public boolean detectQuote(Node node
) {
153 if (node
instanceof Element
) {
154 Element elementNode
= (Element
) node
;
155 if (elementNode
.tagName().equals("blockquote")
156 || elementNode
.hasClass("quote")) {
165 public boolean ignoreNode(Node node
) {
170 public String
manualProcessing(Node node
) {