X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Fgofetch%2Fsupport%2FLWN.java;h=aa2a653d905510e21bff24358b0deb16e2ce5d82;hb=3367f6256b5143b7cba2a61de36e74f389a5f379;hp=e25bc92d519ed0d340b2d99695bfc167164ff5bf;hpb=2527107526aea628df09c3ad53432e9a5480fcd7;p=gofetch.git diff --git a/src/be/nikiroo/gofetch/support/LWN.java b/src/be/nikiroo/gofetch/support/LWN.java index e25bc92..aa2a653 100644 --- a/src/be/nikiroo/gofetch/support/LWN.java +++ b/src/be/nikiroo/gofetch/support/LWN.java @@ -1,15 +1,16 @@ package be.nikiroo.gofetch.support; import java.io.IOException; -import java.io.InputStream; import java.net.URL; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.List; +import java.util.Map.Entry; -import org.jsoup.helper.DataUtil; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; +import org.jsoup.nodes.Node; +import org.jsoup.nodes.TextNode; import be.nikiroo.gofetch.data.Comment; import be.nikiroo.gofetch.data.Story; @@ -26,163 +27,277 @@ public class LWN extends BasicSupport { } @Override - public List list() throws IOException { - // TODO: comments + do not get comment for [$] stories - // + update body on getComment (global change, also LinuxToday) - - List list = new ArrayList(); + public void fetch(Story story) throws IOException { + // Do not try the paid-for stories... + if (!story.getTitle().startsWith("[$]")) { + super.fetch(story); + } else { + String fullContent = "[$] Sorry, this article is currently available to LWN suscribers only [https://lwn.net/subscribe/]."; + story.setFullContent(fullContent); + story.setComments(new ArrayList()); + } + } - URL url = new URL("https://lwn.net/"); - InputStream in = open(url); - Document doc = DataUtil.load(in, "UTF-8", url.toString()); - Elements stories = doc.getElementsByClass("pure-u-1"); - for (Element story : stories) { - Elements titles = story.getElementsByClass("Headline"); - Elements listings = story.getElementsByClass("BlurbListing"); - if (titles.size() == 0) { - continue; - } - if (listings.size() == 0) { - continue; - } - - Element listing = listings.get(0); - if (listing.children().size() < 2) { - continue; - } - + @Override + protected List> getUrls() throws IOException { + List> urls = new ArrayList>(); + urls.add(new AbstractMap.SimpleEntry(new URL( + "https://lwn.net/"), "")); + return urls; + } - String title = titles.get(0).text(); - String details = listing.children().get(0).text(); - String body = ""; - // All but the first and two last children - for (int i = 1 ; i < listing.children().size() - 2; i++) { - Element e = listing.children().get(i); - body = body.trim() + " " + e.text().trim(); - } - body = body.trim(); - - String author = ""; - int pos = details.indexOf(" by "); + @Override + protected List getArticles(Document doc) { + return doc.getElementsByClass("pure-u-1"); + } + + @Override + protected String getArticleId(Document doc, Element article) { + return getArticleIntUrl(doc, article).replaceAll("[^0-9]", ""); + } + + @Override + protected String getArticleTitle(Document doc, Element article) { + Element title = article.getElementsByClass("Headline").first(); + if (title != null) { + return title.text(); + } + + return ""; + } + + @Override + protected String getArticleAuthor(Document doc, Element article) { + String author = ""; + String details = getArticleDetailsReal(article); + int pos = details.indexOf(" by "); + if (pos >= 0) { + author = details.substring(pos + " by ".length()).trim(); + } + + return author; + } + + @Override + protected String getArticleDate(Document doc, Element article) { + String date = ""; + String details = getArticleDetailsReal(article); + int pos = details.indexOf(" Posted "); + if (pos >= 0) { + date = details.substring(pos + " Posted ".length()).trim(); + pos = date.indexOf(" by "); if (pos >= 0) { - author = details.substring(pos + " by ".length()).trim(); + date = date.substring(0, pos).trim(); } - - String date = ""; - pos = details.indexOf(" Posted "); + } + + return date; + } + + @Override + protected String getArticleCategory(Document doc, Element article, + String currentCategory) { + String categ = ""; + String details = getArticleDetailsReal(article); + int pos = details.indexOf("]"); + if (pos >= 0) { + categ = details.substring(1, pos).trim(); + } + + return categ; + } + + @Override + protected String getArticleDetails(Document doc, Element article) { + return ""; // We actually extract all the values + } + + @Override + protected String getArticleIntUrl(Document doc, Element article) { + String intUrl = ""; + for (Element idElem : article.getElementsByTag("a")) { + // Last link is the story link + intUrl = idElem.absUrl("href"); + int pos = intUrl.indexOf("#Comments"); if (pos >= 0) { - date = details.substring(pos + " Posted ".length()).trim(); + intUrl = intUrl.substring(0, pos - 1); } - - - String id = ""; - String intUrl = ""; - String extUrl = ""; - for (Element idElem : story.getElementsByTag("a")) { - // Last link is the story link - intUrl = idElem.absUrl("href"); - pos = intUrl.indexOf("#Comments"); - if (pos >= 0) { - intUrl = intUrl.substring(0, pos -1); - } - id = intUrl.replaceAll("[^0-9]", ""); + } + + return intUrl; + } + + @Override + protected String getArticleExtUrl(Document doc, Element article) { + return ""; + } + + @Override + protected String getArticleContent(Document doc, Element article) { + Element listing = article.getElementsByClass("BlurbListing").first(); + if (listing != null && listing.children().size() >= 2) { + String content = ""; + + // All but the first and two last children + for (int i = 1; i < listing.children().size() - 2; i++) { + Element e = listing.children().get(i); + content = content.trim() + " " + getArticleText(e); } - list.add(new Story(getType(), id, title, details, intUrl, extUrl, body)); + return content; } - return list; + return ""; + } + + @Override + protected Element getFullArticle(Document doc) { + return doc.getElementsByClass("ArticleText").first(); } @Override - public List getComments(Story story) throws IOException { - List comments = new ArrayList(); + protected List getFullArticleCommentPosts(Document doc, URL intUrl) { + return doc.getElementsByClass("lwn-u-1"); + } - /* - URL url = new URL(story.getUrlInternal()); - InputStream in = open(url); - Document doc = DataUtil.load(in, "UTF-8", url.toString()); - Elements listing = doc.getElementsByTag("main"); - if (listing.size() > 0) { - comments.addAll(getComments(listing.get(0))); - } - */ + @Override + protected ElementProcessor getElementProcessorFullArticle() { + return new BasicElementProcessor() { + @Override + public boolean ignoreNode(Node node) { + if (node instanceof Element) { + Element el = (Element) node; + if ("Log in".equals(el.text().trim())) { + return true; + } + } else if (node instanceof TextNode) { + TextNode text = (TextNode) node; + String t = text.text().trim(); + if (t.equals("(") || t.equals("to post comments)")) { + return true; + } + } - return comments; + return false; + } + }; } - private List getComments(Element listing) { - List comments = new ArrayList(); - for (Element commentElement : listing.children()) { - if (commentElement.hasClass("comment")) { - Comment comment = getComment(commentElement); - if (!comment.isEmpty()) { - comments.add(comment); + @Override + protected List getCommentCommentPosts(Document doc, + Element container) { + List commentElements = new ArrayList(); + if (container != null) { + for (Element possibleCommentElement : container.children()) { + if (possibleCommentElement.hasClass("CommentBox")) { + commentElements.add(possibleCommentElement); + } else if (possibleCommentElement.hasClass("Comment")) { + commentElements.add(possibleCommentElement); } } } - return comments; + + return commentElements; } - private Comment getComment(Element commentElement) { - String title = firstOrEmptyTag(commentElement, "h3"); - String author = firstOrEmpty(commentElement, "h4"); - String content = firstOrEmpty(commentElement, "comment-body"); + @Override + protected String getCommentId(Element post) { + return post.id(); + } - String date = ""; - int pos = author.lastIndexOf(" on "); - if (pos >= 0) { - date = author.substring(pos + " on ".length()).trim(); - author = author.substring(0, pos).trim(); + @Override + protected String getCommentAuthor(Element post) { + Element detailsE = post.getElementsByClass("CommentPoster").first(); + if (detailsE != null) { + String details = detailsE.text(); + + int pos = details.lastIndexOf(" by "); + if (pos >= 0) { + details = details.substring(pos + " by ".length()).trim(); + + if (details.startsWith("Posted ")) { + return details.substring("Posted ".length()).trim(); + } + } } - Comment comment = new Comment(commentElement.id(), author, title, date, - content); + return ""; + } - Elements commentOutline = commentElement - .getElementsByClass("comment-outline"); - if (commentOutline.size() > 0) { - comment.addAll(getComments(commentOutline.get(0))); + @Override + protected String getCommentTitle(Element post) { + Element title = post.getElementsByClass("CommentTitle").first(); + if (title != null) { + return title.text(); } - return comment; - } - - /** - * Get the first element of the given class, or an empty {@link String} if - * none found. - * - * @param element - * the element to look in - * @param className - * the class to look for - * - * @return the value or an empty {@link String} - */ - private String firstOrEmpty(Element element, String className) { - Elements subElements = element.getElementsByClass(className); - if (subElements.size() > 0) { - return subElements.get(0).text(); + return ""; + } + + @Override + protected String getCommentDate(Element post) { + Element detailsE = post.getElementsByClass("CommentPoster").first(); + if (detailsE != null) { + String details = detailsE.text(); + + int pos = details.lastIndexOf(" by "); + if (pos >= 0) { + return details.substring(0, pos).trim(); + } } return ""; } - /** - * Get the first element of the given tag, or an empty {@link String} if - * none found. - * - * @param element - * the element to look in - * @param tagName - * the tag to look for - * - * @return the value or an empty {@link String} - */ - private String firstOrEmptyTag(Element element, String tagName) { - Elements subElements = element.getElementsByTag(tagName); - if (subElements.size() > 0) { - return subElements.get(0).text(); + @Override + protected Element getCommentContentElement(Element post) { + return post.getElementsByClass("CommentBody").first(); + } + + @Override + protected ElementProcessor getElementProcessorComment() { + return new BasicElementProcessor() { + @Override + public String processText(String text) { + while (text.startsWith(">")) { // comments + text = text.substring(1).trim(); + } + + return text; + } + + @Override + public boolean detectQuote(Node node) { + if (node instanceof Element) { + Element elementNode = (Element) node; + if (elementNode.tagName().equals("blockquote") + || elementNode.hasClass("QuotedText")) { + return true; + } + } + + return false; + } + + @Override + public boolean ignoreNode(Node node) { + if (node instanceof Element) { + Element elementNode = (Element) node; + if (elementNode.hasClass("CommentPoster")) { + return true; + } + } + + return false; + } + }; + } + + private String getArticleDetailsReal(Element article) { + Element listing = article.getElementsByClass("BlurbListing").first(); + // Valid articles have 2+ listings + if (listing != null && listing.children().size() >= 2) { + return listing.children().get(0).text(); } return "";