Commit | Line | Data |
---|---|---|
2d95a873 NR |
1 | package be.nikiroo.gofetch.support; |
2 | ||
3 | import java.io.IOException; | |
4 | import java.io.InputStream; | |
5 | import java.net.URL; | |
6 | import java.util.ArrayList; | |
7 | import java.util.List; | |
8 | ||
9 | import org.jsoup.helper.DataUtil; | |
10 | import org.jsoup.nodes.Document; | |
11 | import org.jsoup.nodes.Element; | |
27008a87 | 12 | import org.jsoup.nodes.Node; |
2d95a873 NR |
13 | import org.jsoup.select.Elements; |
14 | ||
15 | import be.nikiroo.gofetch.data.Comment; | |
16 | import be.nikiroo.gofetch.data.Story; | |
17 | ||
18 | /** | |
19 | * Support <a href='https://pipedot.org/'>https://pipedot.org/</a>. | |
20 | * | |
21 | * @author niki | |
22 | */ | |
23 | public class Pipedot extends BasicSupport { | |
24 | @Override | |
25 | public String getDescription() { | |
26 | return "Pipedot: News for nerds, without the corporate slant"; | |
27 | } | |
28 | ||
29 | @Override | |
30 | public List<Story> list() throws IOException { | |
31 | List<Story> list = new ArrayList<Story>(); | |
32 | ||
33 | URL url = new URL("https://pipedot.org/"); | |
34 | InputStream in = open(url); | |
35 | Document doc = DataUtil.load(in, "UTF-8", url.toString()); | |
100a8395 NR |
36 | Elements articles = doc.getElementsByClass("story"); |
37 | for (Element article : articles) { | |
38 | Elements titles = article.getElementsByTag("h1"); | |
2d95a873 NR |
39 | if (titles.size() == 0) { |
40 | continue; | |
41 | } | |
42 | ||
43 | Element title = titles.get(0); | |
44 | ||
45 | String id = ""; | |
100a8395 | 46 | for (Element idElem : article.getElementsByTag("a")) { |
2d95a873 NR |
47 | if (idElem.attr("href").startsWith("/pipe/")) { |
48 | id = idElem.attr("href").substring("/pipe/".length()); | |
49 | break; | |
50 | } | |
51 | } | |
52 | ||
53 | String intUrl = null; | |
54 | String extUrl = null; | |
55 | ||
100a8395 | 56 | Elements links = article.getElementsByTag("a"); |
2d95a873 NR |
57 | if (links.size() > 0) { |
58 | intUrl = links.get(0).absUrl("href"); | |
59 | } | |
60 | ||
61 | // Take first ext URL as original source | |
62 | for (Element link : links) { | |
63 | String uuu = link.absUrl("href"); | |
64 | if (!uuu.isEmpty() && !uuu.contains("pipedot.org/")) { | |
65 | extUrl = uuu; | |
66 | break; | |
67 | } | |
68 | } | |
69 | ||
70 | String details = ""; | |
100a8395 | 71 | Elements detailsElements = article.getElementsByTag("div"); |
2d95a873 NR |
72 | if (detailsElements.size() > 0) { |
73 | details = detailsElements.get(0).text(); | |
74 | } | |
75 | ||
76 | String body = ""; | |
100a8395 | 77 | for (Element elem : article.children()) { |
2d95a873 NR |
78 | String tag = elem.tag().toString(); |
79 | if (!tag.equals("header") && !tag.equals("footer")) { | |
80 | body = elem.text(); | |
81 | break; | |
82 | } | |
83 | } | |
84 | ||
85 | list.add(new Story(getType(), id, title.text(), details, intUrl, | |
86 | extUrl, body)); | |
87 | } | |
88 | ||
89 | return list; | |
90 | } | |
91 | ||
92 | @Override | |
5c056aad | 93 | public void fetch(Story story) throws IOException { |
2d95a873 NR |
94 | List<Comment> comments = new ArrayList<Comment>(); |
95 | ||
96 | URL url = new URL(story.getUrlInternal()); | |
97 | InputStream in = open(url); | |
98 | Document doc = DataUtil.load(in, "UTF-8", url.toString()); | |
99 | Elements listing = doc.getElementsByTag("main"); | |
100 | if (listing.size() > 0) { | |
101 | comments.addAll(getComments(listing.get(0))); | |
102 | } | |
103 | ||
5c056aad | 104 | story.setComments(comments); |
2d95a873 NR |
105 | } |
106 | ||
107 | private List<Comment> getComments(Element listing) { | |
108 | List<Comment> comments = new ArrayList<Comment>(); | |
109 | for (Element commentElement : listing.children()) { | |
110 | if (commentElement.hasClass("comment")) { | |
111 | Comment comment = getComment(commentElement); | |
112 | if (!comment.isEmpty()) { | |
113 | comments.add(comment); | |
114 | } | |
115 | } | |
116 | } | |
117 | return comments; | |
118 | } | |
119 | ||
120 | private Comment getComment(Element commentElement) { | |
27008a87 NR |
121 | String title = firstOrEmptyTag(commentElement, "h3").text(); |
122 | String author = firstOrEmpty(commentElement, "h4").text(); | |
123 | Element content = firstOrEmpty(commentElement, "comment-body"); | |
2d95a873 NR |
124 | |
125 | String date = ""; | |
126 | int pos = author.lastIndexOf(" on "); | |
127 | if (pos >= 0) { | |
128 | date = author.substring(pos + " on ".length()).trim(); | |
129 | author = author.substring(0, pos).trim(); | |
130 | } | |
131 | ||
132 | Comment comment = new Comment(commentElement.id(), author, title, date, | |
27008a87 | 133 | toLines(content)); |
2d95a873 NR |
134 | |
135 | Elements commentOutline = commentElement | |
136 | .getElementsByClass("comment-outline"); | |
137 | if (commentOutline.size() > 0) { | |
138 | comment.addAll(getComments(commentOutline.get(0))); | |
139 | } | |
140 | ||
141 | return comment; | |
142 | } | |
143 | ||
27008a87 NR |
144 | private List<String> toLines(Element element) { |
145 | return toLines(element, new QuoteProcessor() { | |
146 | @Override | |
147 | public String processText(String text) { | |
148 | return text; | |
149 | } | |
2d95a873 | 150 | |
27008a87 NR |
151 | @Override |
152 | public boolean detectQuote(Node node) { | |
153 | if (node instanceof Element) { | |
154 | Element elementNode = (Element) node; | |
155 | if (elementNode.tagName().equals("blockquote") | |
156 | || elementNode.hasClass("quote")) { | |
157 | return true; | |
158 | } | |
159 | } | |
2d95a873 | 160 | |
27008a87 NR |
161 | return false; |
162 | } | |
2d95a873 | 163 | |
27008a87 NR |
164 | @Override |
165 | public boolean ignoreNode(Node node) { | |
166 | return false; | |
167 | } | |
100a8395 NR |
168 | |
169 | @Override | |
170 | public String manualProcessing(Node node) { | |
171 | return null; | |
172 | } | |
27008a87 | 173 | }); |
2d95a873 NR |
174 | } |
175 | } |