From: Niki Roo Date: Sun, 18 Mar 2018 10:54:52 +0000 (+0100) Subject: Update nikiroo-utils, use it, fix 2 bugs: X-Git-Url: https://git.nikiroo.be/?a=commitdiff_plain;h=136ab80122a17caa0720116d6d2552521239fbb3;p=gofetch.git Update nikiroo-utils, use it, fix 2 bugs: -fix missing file causes exception (e.g., when old html files are deleted but not gopher files) - (hopefully) fix slashdot connection error via new nikiroo-utils Downloader --- diff --git a/libs/nikiroo-utils-2.1.0-sources.jar b/libs/nikiroo-utils-4.1.0-sources.jar similarity index 72% rename from libs/nikiroo-utils-2.1.0-sources.jar rename to libs/nikiroo-utils-4.1.0-sources.jar index e7975f4..7b91d06 100644 Binary files a/libs/nikiroo-utils-2.1.0-sources.jar and b/libs/nikiroo-utils-4.1.0-sources.jar differ diff --git a/src/be/nikiroo/gofetch/Fetcher.java b/src/be/nikiroo/gofetch/Fetcher.java index e53bfaf..5f81eda 100644 --- a/src/be/nikiroo/gofetch/Fetcher.java +++ b/src/be/nikiroo/gofetch/Fetcher.java @@ -93,8 +93,8 @@ public class Fetcher { } ref = "../" + ref + "/index.html"; - htmlBuilder.append(getLink(support.getDescription(), - ref, true, true)); + htmlBuilder.append(getLink(support.getDescription(), ref, true, + true)); } File gopherCache = new File(dir, preselector); @@ -188,28 +188,23 @@ public class Fetcher { List gopherLines = new ArrayList(); List htmlLines = new ArrayList(); for (i = 0; i < headers.length; i++) { - gopherLines - .add(IOUtils.readSmallFile(new File(varDir, headers[i]))); - htmlLines.add(IOUtils.readSmallFile(new File(varDir, headers[i] - + ".html"))); + File gopherFile = new File(varDir, headers[i]); + File htmlFile = new File(varDir, headers[i] + ".html"); + + if (gopherFile.exists()) + gopherLines.add(IOUtils.readSmallFile(gopherFile)); + if (htmlFile.exists()) + htmlLines.add(IOUtils.readSmallFile(htmlFile)); boolean enoughStories = (i > 0 && i % maxStories == 0); boolean last = i == headers.length - 1; if (enoughStories || last) { if (!last) { - gopherLines.add(getLink("More", - support.getSelector() - + "gophermap_" - + (page + 1), - true, - false)); - - htmlLines.add(getLink("More", - "index_" - + (page + 1) - + ".html", - true, - true)); + gopherLines.add(getLink("More", support.getSelector() + + "gophermap_" + (page + 1), true, false)); + + htmlLines.add(getLink("More", "index_" + (page + 1) + + ".html", true, true)); } write(gopherLines, varDir, "gophermap", "", page); @@ -237,13 +232,18 @@ public class Fetcher { } /** + * Create a link. * * @param name + * the link name (what the user will see) * @param ref + * the actual link reference (the target) * @param menu - * menu (gophermap, i) mode + * menu (gophermap, i) mode -- not used in html mode * @param html - * @return + * TRUE for html mode, FALSE for gopher mode + * + * @return the ready-to-use link in a {@link String} */ private String getLink(String name, String ref, boolean menu, boolean html) { if (!html) { diff --git a/src/be/nikiroo/gofetch/support/BasicSupport.java b/src/be/nikiroo/gofetch/support/BasicSupport.java index f3348e3..615c72d 100644 --- a/src/be/nikiroo/gofetch/support/BasicSupport.java +++ b/src/be/nikiroo/gofetch/support/BasicSupport.java @@ -1,12 +1,8 @@ package be.nikiroo.gofetch.support; import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.net.URLConnection; import java.util.ArrayList; import java.util.List; -import java.util.zip.GZIPInputStream; import org.jsoup.helper.StringUtil; import org.jsoup.nodes.Element; @@ -17,8 +13,11 @@ import org.jsoup.select.NodeTraversor; import org.jsoup.select.NodeVisitor; import be.nikiroo.gofetch.data.Story; +import be.nikiroo.utils.Downloader; public abstract class BasicSupport { + protected static Downloader downloader = new Downloader("gofetcher"); + public enum Type { SLASHDOT, PIPEDOT, LWN, LEMONDE, } @@ -193,18 +192,6 @@ public abstract class BasicSupport { return preselector + "/" + type + "/"; } - // TODO: check Downloader.java? - static protected InputStream open(URL url) throws IOException { - URLConnection conn = url.openConnection(); - conn.connect(); - InputStream in = conn.getInputStream(); - if ("gzip".equals(conn.getContentEncoding())) { - in = new GZIPInputStream(in); - } - - return in; - } - /** * Get the first {@link Element} of the given class, or an empty span * {@link Element} if none found. diff --git a/src/be/nikiroo/gofetch/support/LWN.java b/src/be/nikiroo/gofetch/support/LWN.java index c48ed2a..27b539c 100644 --- a/src/be/nikiroo/gofetch/support/LWN.java +++ b/src/be/nikiroo/gofetch/support/LWN.java @@ -31,7 +31,7 @@ public class LWN extends BasicSupport { List list = new ArrayList(); URL url = new URL("https://lwn.net/"); - InputStream in = open(url); + InputStream in = downloader.open(url); Document doc = DataUtil.load(in, "UTF-8", url.toString()); Elements articles = doc.getElementsByClass("pure-u-1"); for (Element article : articles) { @@ -99,7 +99,7 @@ public class LWN extends BasicSupport { // Do not try the paid-for stories... if (!story.getTitle().startsWith("[$]")) { URL url = new URL(story.getUrlInternal()); - InputStream in = open(url); + InputStream in = downloader.open(url); Document doc = DataUtil.load(in, "UTF-8", url.toString()); Elements fullContentElements = doc .getElementsByClass("ArticleText"); diff --git a/src/be/nikiroo/gofetch/support/LeMonde.java b/src/be/nikiroo/gofetch/support/LeMonde.java index c83dc14..d11ba79 100644 --- a/src/be/nikiroo/gofetch/support/LeMonde.java +++ b/src/be/nikiroo/gofetch/support/LeMonde.java @@ -28,7 +28,7 @@ public class LeMonde extends BasicSupport { for (String topic : new String[] { "international", "politique", "societe", "sciences" }) { URL url = new URL("http://www.lemonde.fr/" + topic + "/1.html"); - InputStream in = open(url); + InputStream in = downloader.open(url); Document doc = DataUtil.load(in, "UTF-8", url.toString()); Elements articles = doc.getElementsByTag("article"); for (Element article : articles) { @@ -74,7 +74,7 @@ public class LeMonde extends BasicSupport { // some javascript, I need to check...) URL url = new URL(story.getUrlInternal()); - InputStream in = open(url); + InputStream in = downloader.open(url); Document doc = DataUtil.load(in, "UTF-8", url.toString()); Element article = doc.getElementById("articleBody"); if (article != null) { diff --git a/src/be/nikiroo/gofetch/support/Pipedot.java b/src/be/nikiroo/gofetch/support/Pipedot.java index 8db4749..17388b2 100644 --- a/src/be/nikiroo/gofetch/support/Pipedot.java +++ b/src/be/nikiroo/gofetch/support/Pipedot.java @@ -31,7 +31,7 @@ public class Pipedot extends BasicSupport { List list = new ArrayList(); URL url = new URL("https://pipedot.org/"); - InputStream in = open(url); + InputStream in = downloader.open(url); Document doc = DataUtil.load(in, "UTF-8", url.toString()); Elements articles = doc.getElementsByClass("story"); for (Element article : articles) { @@ -94,7 +94,7 @@ public class Pipedot extends BasicSupport { List comments = new ArrayList(); URL url = new URL(story.getUrlInternal()); - InputStream in = open(url); + InputStream in = downloader.open(url); Document doc = DataUtil.load(in, "UTF-8", url.toString()); Elements listing = doc.getElementsByTag("main"); if (listing.size() > 0) { diff --git a/src/be/nikiroo/gofetch/support/Slashdot.java b/src/be/nikiroo/gofetch/support/Slashdot.java index 1581d23..43d35f4 100644 --- a/src/be/nikiroo/gofetch/support/Slashdot.java +++ b/src/be/nikiroo/gofetch/support/Slashdot.java @@ -31,7 +31,7 @@ public class Slashdot extends BasicSupport { List list = new ArrayList(); URL url = new URL("https://slashdot.org/"); - InputStream in = open(url); + InputStream in = downloader.open(url); Document doc = DataUtil.load(in, "UTF-8", url.toString()); Elements articles = doc.getElementsByTag("header"); for (Element article : articles) { @@ -81,7 +81,7 @@ public class Slashdot extends BasicSupport { List comments = new ArrayList(); URL url = new URL(story.getUrlInternal()); - InputStream in = open(url); + InputStream in = downloader.open(url); Document doc = DataUtil.load(in, "UTF-8", url.toString()); Element listing = doc.getElementById("commentlisting"); if (listing != null) {