1 package be
.nikiroo
.gofetch
.support
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.util
.ArrayList
;
9 import org
.jsoup
.helper
.DataUtil
;
10 import org
.jsoup
.nodes
.Document
;
11 import org
.jsoup
.nodes
.Element
;
12 import org
.jsoup
.select
.Elements
;
14 import be
.nikiroo
.gofetch
.data
.Comment
;
15 import be
.nikiroo
.gofetch
.data
.Story
;
18 * Support <a href='https://lwn.net/'>https://lwn.net/</a>.
22 public class LWN
extends BasicSupport
{
24 public String
getDescription() {
25 return "LWN: Linux Weekly Newsletter";
29 public List
<Story
> list() throws IOException
{
30 // TODO: comments + do not get comment for [$] stories
31 // + update body on getComment (global change, also LinuxToday)
33 List
<Story
> list
= new ArrayList
<Story
>();
35 URL url
= new URL("https://lwn.net/");
36 InputStream in
= open(url
);
37 Document doc
= DataUtil
.load(in
, "UTF-8", url
.toString());
38 Elements stories
= doc
.getElementsByClass("pure-u-1");
39 for (Element story
: stories
) {
40 Elements titles
= story
.getElementsByClass("Headline");
41 Elements listings
= story
.getElementsByClass("BlurbListing");
42 if (titles
.size() == 0) {
45 if (listings
.size() == 0) {
49 Element listing
= listings
.get(0);
50 if (listing
.children().size() < 2) {
55 String title
= titles
.get(0).text();
56 String details
= listing
.children().get(0).text();
58 // All but the first and two last children
59 for (int i
= 1 ; i
< listing
.children().size() - 2; i
++) {
60 Element e
= listing
.children().get(i
);
61 body
= body
.trim() + " " + e
.text().trim();
66 int pos
= details
.indexOf(" by ");
68 author
= details
.substring(pos
+ " by ".length()).trim();
72 pos
= details
.indexOf(" Posted ");
74 date
= details
.substring(pos
+ " Posted ".length()).trim();
81 for (Element idElem
: story
.getElementsByTag("a")) {
82 // Last link is the story link
83 intUrl
= idElem
.absUrl("href");
84 pos
= intUrl
.indexOf("#Comments");
86 intUrl
= intUrl
.substring(0, pos
-1);
88 id
= intUrl
.replaceAll("[^0-9]", "");
91 list
.add(new Story(getType(), id
, title
, details
, intUrl
, extUrl
, body
));
98 public List
<Comment
> getComments(Story story
) throws IOException
{
99 List
<Comment
> comments
= new ArrayList
<Comment
>();
102 URL url = new URL(story.getUrlInternal());
103 InputStream in = open(url);
104 Document doc = DataUtil.load(in, "UTF-8", url.toString());
105 Elements listing = doc.getElementsByTag("main");
106 if (listing.size() > 0) {
107 comments.addAll(getComments(listing.get(0)));
114 private List
<Comment
> getComments(Element listing
) {
115 List
<Comment
> comments
= new ArrayList
<Comment
>();
116 for (Element commentElement
: listing
.children()) {
117 if (commentElement
.hasClass("comment")) {
118 Comment comment
= getComment(commentElement
);
119 if (!comment
.isEmpty()) {
120 comments
.add(comment
);
127 private Comment
getComment(Element commentElement
) {
128 String title
= firstOrEmptyTag(commentElement
, "h3");
129 String author
= firstOrEmpty(commentElement
, "h4");
130 String content
= firstOrEmpty(commentElement
, "comment-body");
133 int pos
= author
.lastIndexOf(" on ");
135 date
= author
.substring(pos
+ " on ".length()).trim();
136 author
= author
.substring(0, pos
).trim();
139 Comment comment
= new Comment(commentElement
.id(), author
, title
, date
,
142 Elements commentOutline
= commentElement
143 .getElementsByClass("comment-outline");
144 if (commentOutline
.size() > 0) {
145 comment
.addAll(getComments(commentOutline
.get(0)));
152 * Get the first element of the given class, or an empty {@link String} if
156 * the element to look in
158 * the class to look for
160 * @return the value or an empty {@link String}
162 private String
firstOrEmpty(Element element
, String className
) {
163 Elements subElements
= element
.getElementsByClass(className
);
164 if (subElements
.size() > 0) {
165 return subElements
.get(0).text();
172 * Get the first element of the given tag, or an empty {@link String} if
176 * the element to look in
178 * the tag to look for
180 * @return the value or an empty {@link String}
182 private String
firstOrEmptyTag(Element element
, String tagName
) {
183 Elements subElements
= element
.getElementsByTag(tagName
);
184 if (subElements
.size() > 0) {
185 return subElements
.get(0).text();