Commit | Line | Data |
---|---|---|
eaaeae39 NR |
1 | package be.nikiroo.gofetch.support; |
2 | ||
3 | import java.io.IOException; | |
4 | import java.io.InputStream; | |
5 | import java.net.URL; | |
6 | import java.util.ArrayList; | |
7 | import java.util.List; | |
8 | ||
9 | import org.jsoup.helper.DataUtil; | |
10 | import org.jsoup.nodes.Document; | |
11 | import org.jsoup.nodes.Element; | |
bb0d9eb2 | 12 | import org.jsoup.nodes.Node; |
eaaeae39 NR |
13 | import org.jsoup.select.Elements; |
14 | ||
15 | import be.nikiroo.gofetch.data.Comment; | |
16 | import be.nikiroo.gofetch.data.Story; | |
17 | ||
18 | /** | |
19 | * Support <a href='https://lwn.net/'>https://lwn.net/</a>. | |
20 | * | |
21 | * @author niki | |
22 | */ | |
23 | public class LWN extends BasicSupport { | |
24 | @Override | |
25 | public String getDescription() { | |
26 | return "LWN: Linux Weekly Newsletter"; | |
27 | } | |
28 | ||
29 | @Override | |
30 | public List<Story> list() throws IOException { | |
eaaeae39 NR |
31 | List<Story> list = new ArrayList<Story>(); |
32 | ||
33 | URL url = new URL("https://lwn.net/"); | |
34 | InputStream in = open(url); | |
35 | Document doc = DataUtil.load(in, "UTF-8", url.toString()); | |
36 | Elements stories = doc.getElementsByClass("pure-u-1"); | |
37 | for (Element story : stories) { | |
38 | Elements titles = story.getElementsByClass("Headline"); | |
39 | Elements listings = story.getElementsByClass("BlurbListing"); | |
40 | if (titles.size() == 0) { | |
41 | continue; | |
42 | } | |
43 | if (listings.size() == 0) { | |
44 | continue; | |
45 | } | |
5c056aad | 46 | |
eaaeae39 NR |
47 | Element listing = listings.get(0); |
48 | if (listing.children().size() < 2) { | |
49 | continue; | |
50 | } | |
eaaeae39 NR |
51 | |
52 | String title = titles.get(0).text(); | |
53 | String details = listing.children().get(0).text(); | |
25271075 NR |
54 | String body = ""; |
55 | // All but the first and two last children | |
5c056aad | 56 | for (int i = 1; i < listing.children().size() - 2; i++) { |
25271075 NR |
57 | Element e = listing.children().get(i); |
58 | body = body.trim() + " " + e.text().trim(); | |
59 | } | |
60 | body = body.trim(); | |
5c056aad | 61 | |
eaaeae39 NR |
62 | String author = ""; |
63 | int pos = details.indexOf(" by "); | |
64 | if (pos >= 0) { | |
65 | author = details.substring(pos + " by ".length()).trim(); | |
66 | } | |
5c056aad | 67 | |
eaaeae39 NR |
68 | String date = ""; |
69 | pos = details.indexOf(" Posted "); | |
70 | if (pos >= 0) { | |
71 | date = details.substring(pos + " Posted ".length()).trim(); | |
72 | } | |
eaaeae39 NR |
73 | |
74 | String id = ""; | |
75 | String intUrl = ""; | |
76 | String extUrl = ""; | |
77 | for (Element idElem : story.getElementsByTag("a")) { | |
78 | // Last link is the story link | |
79 | intUrl = idElem.absUrl("href"); | |
80 | pos = intUrl.indexOf("#Comments"); | |
81 | if (pos >= 0) { | |
5c056aad | 82 | intUrl = intUrl.substring(0, pos - 1); |
eaaeae39 NR |
83 | } |
84 | id = intUrl.replaceAll("[^0-9]", ""); | |
85 | } | |
86 | ||
5c056aad NR |
87 | list.add(new Story(getType(), id, title, details, intUrl, extUrl, |
88 | body)); | |
eaaeae39 NR |
89 | } |
90 | ||
91 | return list; | |
92 | } | |
93 | ||
94 | @Override | |
5c056aad | 95 | public void fetch(Story story) throws IOException { |
bb0d9eb2 NR |
96 | List<Comment> comments = new ArrayList<Comment>(); |
97 | String fullContent = story.getContent(); | |
98 | ||
99 | // Do not try the paid-for stories... | |
100 | if (!story.getTitle().startsWith("[$]")) { | |
101 | URL url = new URL(story.getUrlInternal()); | |
102 | InputStream in = open(url); | |
103 | Document doc = DataUtil.load(in, "UTF-8", url.toString()); | |
104 | Elements fullContentElements = doc | |
105 | .getElementsByClass("ArticleText"); | |
106 | if (fullContentElements.size() > 0) { | |
107 | // comments.addAll(getComments(listing.get(0))); | |
108 | fullContent = fullContentElements.get(0).text(); | |
109 | } | |
110 | ||
111 | Elements listing = doc.getElementsByClass("lwn-u-1"); | |
112 | if (listing.size() > 0) { | |
113 | comments.addAll(getComments(listing.get(0))); | |
114 | } | |
115 | } else { | |
116 | fullContent = "[$] Sorry, this article is currently available to LWN suscribers only [https://lwn.net/subscribe/]."; | |
117 | } | |
118 | ||
119 | story.setFullContent(fullContent); | |
120 | story.setComments(comments); | |
eaaeae39 NR |
121 | } |
122 | ||
123 | private List<Comment> getComments(Element listing) { | |
124 | List<Comment> comments = new ArrayList<Comment>(); | |
125 | for (Element commentElement : listing.children()) { | |
bb0d9eb2 | 126 | if (commentElement.hasClass("CommentBox")) { |
eaaeae39 NR |
127 | Comment comment = getComment(commentElement); |
128 | if (!comment.isEmpty()) { | |
129 | comments.add(comment); | |
130 | } | |
bb0d9eb2 NR |
131 | } else if (commentElement.hasClass("Comment")) { |
132 | if (comments.size() > 0) { | |
133 | comments.get(comments.size() - 1).addAll( | |
134 | getComments(commentElement)); | |
135 | } | |
eaaeae39 NR |
136 | } |
137 | } | |
138 | return comments; | |
139 | } | |
140 | ||
141 | private Comment getComment(Element commentElement) { | |
bb0d9eb2 NR |
142 | String title = firstOrEmpty(commentElement, "CommentTitle"); |
143 | String author = firstOrEmpty(commentElement, "CommentPoster"); | |
eaaeae39 NR |
144 | |
145 | String date = ""; | |
bb0d9eb2 | 146 | int pos = author.lastIndexOf(" by "); |
eaaeae39 | 147 | if (pos >= 0) { |
bb0d9eb2 NR |
148 | date = author.substring(0, pos).trim(); |
149 | author = author.substring(pos + " by ".length()).trim(); | |
150 | ||
151 | if (author.startsWith("Posted ")) { | |
152 | author = author.substring("Posted ".length()).trim(); | |
153 | } | |
eaaeae39 NR |
154 | } |
155 | ||
bb0d9eb2 NR |
156 | String content = ""; |
157 | Elements commentBodyElements = commentElement | |
158 | .getElementsByClass("CommentBody"); | |
159 | if (commentBodyElements.size() > 0) { | |
160 | for (Node contentNode : commentBodyElements.get(0).childNodes()) { | |
161 | if (contentNode instanceof Element) { | |
162 | Element contentElement = (Element) contentNode; | |
163 | if (!contentElement.hasClass("CommentPoster")) { | |
164 | content = content.trim() + " " | |
165 | + contentElement.text().trim(); | |
166 | } | |
167 | } else { | |
168 | content = content.trim() + " " | |
169 | + contentNode.outerHtml().trim(); | |
170 | } | |
eaaeae39 | 171 | |
bb0d9eb2 NR |
172 | } |
173 | content = content.trim(); | |
eaaeae39 NR |
174 | } |
175 | ||
bb0d9eb2 NR |
176 | Comment comment = new Comment(commentElement.id(), author, title, date, |
177 | content); | |
178 | ||
eaaeae39 NR |
179 | return comment; |
180 | } | |
181 | ||
182 | /** | |
183 | * Get the first element of the given class, or an empty {@link String} if | |
184 | * none found. | |
185 | * | |
186 | * @param element | |
187 | * the element to look in | |
188 | * @param className | |
189 | * the class to look for | |
190 | * | |
191 | * @return the value or an empty {@link String} | |
192 | */ | |
193 | private String firstOrEmpty(Element element, String className) { | |
194 | Elements subElements = element.getElementsByClass(className); | |
195 | if (subElements.size() > 0) { | |
196 | return subElements.get(0).text(); | |
197 | } | |
198 | ||
199 | return ""; | |
200 | } | |
201 | ||
202 | /** | |
203 | * Get the first element of the given tag, or an empty {@link String} if | |
204 | * none found. | |
205 | * | |
206 | * @param element | |
207 | * the element to look in | |
208 | * @param tagName | |
209 | * the tag to look for | |
210 | * | |
211 | * @return the value or an empty {@link String} | |
212 | */ | |
213 | private String firstOrEmptyTag(Element element, String tagName) { | |
214 | Elements subElements = element.getElementsByTag(tagName); | |
215 | if (subElements.size() > 0) { | |
216 | return subElements.get(0).text(); | |
217 | } | |
218 | ||
219 | return ""; | |
220 | } | |
221 | } |