From 21f1a997d7addd93226243b0d6d80ce8afcde153 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Wed, 26 Dec 2018 13:08:00 +0100 Subject: [PATCH] Remove dead website PipeDot --- README.md | 1 - .../nikiroo/gofetch/support/BasicSupport.java | 12 +- src/be/nikiroo/gofetch/support/Pipedot.java | 263 ------------------ src/be/nikiroo/gofetch/support/Type.java | 2 - 4 files changed, 5 insertions(+), 273 deletions(-) delete mode 100644 src/be/nikiroo/gofetch/support/Pipedot.java diff --git a/README.md b/README.md index 7d0d650..786c9e4 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,6 @@ The program will also helpfully create a ```gophermap``` and an ```index.html``` ### Supported websites - Slashdot: News for nerds, stuff that matters! -- Pipedot: News for nerds, without the corporate slant - LWN: Linux Weekly Newsletter - Le Monde: Actualités et Infos en France et dans le monde - The Register: Biting the hand that feeds IT diff --git a/src/be/nikiroo/gofetch/support/BasicSupport.java b/src/be/nikiroo/gofetch/support/BasicSupport.java index 17a3c15..9555b9d 100644 --- a/src/be/nikiroo/gofetch/support/BasicSupport.java +++ b/src/be/nikiroo/gofetch/support/BasicSupport.java @@ -3,7 +3,6 @@ package be.nikiroo.gofetch.support; import java.io.IOException; import java.io.InputStream; import java.net.URL; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -34,7 +33,7 @@ import be.nikiroo.utils.StringUtils; */ public abstract class BasicSupport { /** - * The downloader to use for all web sites via + * The {@link Downloader} to use for all web sites via * {@link BasicSupport#open(URL)} */ static private Downloader downloader = new Downloader("gofetcher"); @@ -56,6 +55,7 @@ public abstract class BasicSupport { * in case of I/O error * */ + @SuppressWarnings("unused") public void login() throws IOException { } @@ -133,9 +133,10 @@ public abstract class BasicSupport { id = "0" + id; } } else { - id = date.replace(":", "_").replace("+", "_").replace("/", "-"); + id = date.replace(":", "_").replace("+", "_") + .replace("/", "-"); } - + date = date(date); list.add(new Story(getType(), id, title, author, date, categ, @@ -577,9 +578,6 @@ public abstract class BasicSupport { case SLASHDOT: support = new Slashdot(); break; - case PIPEDOT: - support = new Pipedot(); - break; case LWN: support = new LWN(); break; diff --git a/src/be/nikiroo/gofetch/support/Pipedot.java b/src/be/nikiroo/gofetch/support/Pipedot.java deleted file mode 100644 index 0d044e5..0000000 --- a/src/be/nikiroo/gofetch/support/Pipedot.java +++ /dev/null @@ -1,263 +0,0 @@ -package be.nikiroo.gofetch.support; - -import java.io.IOException; -import java.net.URL; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.List; -import java.util.Map.Entry; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.nodes.Node; -import org.jsoup.select.Elements; - -/** - * Support https://pipedot.org/. - * - * @author niki - */ -public class Pipedot extends BasicSupport { - @Override - public String getDescription() { - return "Pipedot: News for nerds, without the corporate slant"; - } - - @Override - protected List> getUrls() throws IOException { - List> urls = new ArrayList>(); - urls.add(new AbstractMap.SimpleEntry(new URL( - "https://pipedot.org/"), "")); - return urls; - } - - @Override - protected List getArticles(Document doc) { - return doc.getElementsByClass("story"); - } - - @Override - protected String getArticleId(Document doc, Element article) { - // Don't try on bad articles - if (getArticleTitle(doc, article).isEmpty()) { - return ""; - } - - for (Element idElem : article.getElementsByTag("a")) { - if (idElem.attr("href").startsWith("/pipe/")) { - return idElem.attr("href").substring("/pipe/".length()); - } - } - - return ""; - } - - @Override - protected String getArticleTitle(Document doc, Element article) { - Element title = article.getElementsByTag("h1").first(); - if (title != null) { - return title.text(); - } - - return ""; - } - - @Override - protected String getArticleAuthor(Document doc, Element article) { - String value = getArticleDetailsReal(article); - int pos = value.indexOf("by "); - if (pos >= 0) { - value = value.substring(pos + "by ".length()).trim(); - pos = value.indexOf(" in "); - if (pos >= 0) { - value = value.substring(0, pos).trim(); - } - - return value; - } - - return ""; - } - - @Override - protected String getArticleDate(Document doc, Element article) { - Element dateElement = article.getElementsByTag("time").first(); - if (dateElement != null) { - return dateElement.attr("datetime"); - } - - return ""; - } - - @Override - protected String getArticleCategory(Document doc, Element article, - String currentCategory) { - String value = getArticleDetailsReal(article); - int pos = value.indexOf(" in "); - if (pos >= 0) { - value = value.substring(pos + " in ".length()).trim(); - pos = value.indexOf(" on "); - if (pos >= 0) { - value = value.substring(0, pos).trim(); - } - - return value; - } - - return ""; - } - - @Override - protected String getArticleDetails(Document doc, Element article) { - return ""; // We alrady extracted all the info - } - - @Override - protected String getArticleIntUrl(Document doc, Element article) { - Element link = article.getElementsByTag("a").first(); - if (link != null) { - return link.absUrl("href"); - } - - return ""; - } - - @Override - protected String getArticleExtUrl(Document doc, Element article) { - Element link = article.getElementsByTag("a").first(); - if (link != null) { - String possibleExtLink = link.absUrl("href").trim(); - if (!possibleExtLink.isEmpty() - && !possibleExtLink.contains("pipedot.org/")) { - return possibleExtLink; - } - } - - return ""; - } - - @Override - protected String getArticleContent(Document doc, Element article) { - for (Element elem : article.children()) { - String tag = elem.tagName(); - if (!tag.equals("header") && !tag.equals("footer")) { - return getArticleText(elem); - } - } - - return ""; - } - - @Override - protected Element getFullArticle(Document doc) { - return null; - } - - @Override - protected List getFullArticleCommentPosts(Document doc, URL intUrl) { - return getCommentElements(doc.getElementsByTag("main").first()); - } - - @Override - protected ElementProcessor getElementProcessorFullArticle() { - return new BasicElementProcessor(); - } - - @Override - protected List getCommentCommentPosts(Document doc, - Element container) { - - if (container != null) { - container = container.getElementsByClass("comment-outline").first(); - } - - return getCommentElements(container); - } - - @Override - protected String getCommentId(Element post) { - return post.id(); - } - - @Override - protected String getCommentAuthor(Element post) { - Element authorDateE = post.getElementsByTag("h3").first(); - if (authorDateE != null) { - String authorDate = authorDateE.text(); - int pos = authorDate.lastIndexOf(" on "); - if (pos >= 0) { - return authorDate.substring(0, pos).trim(); - } - } - - return ""; - } - - @Override - protected String getCommentTitle(Element post) { - Element title = post.getElementsByTag("h3").first(); - if (title != null) { - return title.text(); - } - - return ""; - } - - @Override - protected String getCommentDate(Element post) { - Element authorDateE = post.getElementsByTag("h3").first(); - if (authorDateE != null) { - String authorDate = authorDateE.text(); - int pos = authorDate.lastIndexOf(" on "); - if (pos >= 0) { - return authorDate.substring(pos + " on ".length()).trim(); - } - } - - return ""; - } - - @Override - protected Element getCommentContentElement(Element post) { - return post.getElementsByClass("comment-body").first(); - } - - @Override - protected ElementProcessor getElementProcessorComment() { - return new BasicElementProcessor() { - @Override - public boolean detectQuote(Node node) { - if (node instanceof Element) { - Element elementNode = (Element) node; - if (elementNode.tagName().equals("blockquote") - || elementNode.hasClass("quote")) { - return true; - } - } - - return false; - } - }; - } - - private String getArticleDetailsReal(Element article) { - Elements detailsElements = article.getElementsByTag("div"); - if (detailsElements.size() > 0) { - return detailsElements.get(0).text().trim(); - } - - return ""; - } - - private List getCommentElements(Element container) { - List commentElements = new ArrayList(); - if (container != null) { - for (Element commentElement : container.children()) { - if (commentElement.hasClass("comment")) { - commentElements.add(commentElement); - } - } - } - return commentElements; - } -} diff --git a/src/be/nikiroo/gofetch/support/Type.java b/src/be/nikiroo/gofetch/support/Type.java index a90cb44..cc26428 100644 --- a/src/be/nikiroo/gofetch/support/Type.java +++ b/src/be/nikiroo/gofetch/support/Type.java @@ -8,8 +8,6 @@ package be.nikiroo.gofetch.support; public enum Type { /** EN: Any, but mostly IT/Sci */ SLASHDOT, - /** EN: Clone of Slashdot, mostly abandoned */ - PIPEDOT, /** EN: Linux */ LWN, /** FR: Any */ -- 2.27.0