Update nikiroo-utils, use it, fix 2 bugs:

-fix missing file causes exception (e.g., when old html files are deleted but not gopher files)
- (hopefully) fix slashdot connection error via new nikiroo-utils Downloader

diff --git a/libs/nikiroo-utils-2.1.0-sources.jar b/libs/nikiroo-utils-4.1.0-sources.jar
similarity index 72%
rename from libs/nikiroo-utils-2.1.0-sources.jar
rename to libs/nikiroo-utils-4.1.0-sources.jar
index e7975f4beb5459e99d9ff8e19261d5f5ba2ce7fa..7b91d065bc016fa704802546daaef5e66a296456 100644 (file)
Binary files a/libs/nikiroo-utils-2.1.0-sources.jar and b/libs/nikiroo-utils-4.1.0-sources.jar differ

diff --git a/src/be/nikiroo/gofetch/Fetcher.java b/src/be/nikiroo/gofetch/Fetcher.java
index e53bfaf3c41d72bd566dd47ef9ec9fdd40b9d015..5f81edabcb887cdb9e211c558331bab8a78339ec 100644 (file)
--- a/src/be/nikiroo/gofetch/Fetcher.java
+++ b/src/be/nikiroo/gofetch/Fetcher.java
@@ -93,8 +93,8 @@ public class Fetcher {
                        }
                        ref = "../" + ref + "/index.html";
 
-                       htmlBuilder.append(getLink(support.getDescription(), 
-                                       ref, true, true));
+                       htmlBuilder.append(getLink(support.getDescription(), ref, true,
+                                       true));
                }
 
                File gopherCache = new File(dir, preselector);
@@ -188,28 +188,23 @@ public class Fetcher {
                List<String> gopherLines = new ArrayList<String>();
                List<String> htmlLines = new ArrayList<String>();
                for (i = 0; i < headers.length; i++) {
-                       gopherLines
-                                       .add(IOUtils.readSmallFile(new File(varDir, headers[i])));
-                       htmlLines.add(IOUtils.readSmallFile(new File(varDir, headers[i]
-                                       + ".html")));
+                       File gopherFile = new File(varDir, headers[i]);
+                       File htmlFile = new File(varDir, headers[i] + ".html");
+
+                       if (gopherFile.exists())
+                               gopherLines.add(IOUtils.readSmallFile(gopherFile));
+                       if (htmlFile.exists())
+                               htmlLines.add(IOUtils.readSmallFile(htmlFile));
 
                        boolean enoughStories = (i > 0 && i % maxStories == 0);
                        boolean last = i == headers.length - 1;
                        if (enoughStories || last) {
                                if (!last) {
-                                       gopherLines.add(getLink("More", 
-                                               support.getSelector() 
-                                                       + "gophermap_" 
-                                                       + (page + 1),
-                                               true, 
-                                               false));
-                                       
-                                       htmlLines.add(getLink("More", 
-                                               "index_" 
-                                                       + (page + 1)
-                                                       + ".html",
-                                               true, 
-                                               true));
+                                       gopherLines.add(getLink("More", support.getSelector()
+                                                       + "gophermap_" + (page + 1), true, false));
+
+                                       htmlLines.add(getLink("More", "index_" + (page + 1)
+                                                       + ".html", true, true));
                                }
 
                                write(gopherLines, varDir, "gophermap", "", page);
@@ -237,13 +232,18 @@ public class Fetcher {
        }
 
        /**
+        * Create a link.
         * 
         * @param name
+        *            the link name (what the user will see)
         * @param ref
+        *            the actual link reference (the target)
         * @param menu
-        *            menu (gophermap, i) mode
+        *            menu (gophermap, i) mode -- not used in html mode
         * @param html
-        * @return
+        *            TRUE for html mode, FALSE for gopher mode
+        * 
+        * @return the ready-to-use link in a {@link String}
         */
        private String getLink(String name, String ref, boolean menu, boolean html) {
                if (!html) {

diff --git a/src/be/nikiroo/gofetch/support/BasicSupport.java b/src/be/nikiroo/gofetch/support/BasicSupport.java
index f3348e366f1ceeece3c6bec39ea01f031ffe8c95..615c72d6b0cbd81c4f59d8472878ea07ef4b730a 100644 (file)
--- a/src/be/nikiroo/gofetch/support/BasicSupport.java
+++ b/src/be/nikiroo/gofetch/support/BasicSupport.java
@@ -1,12 +1,8 @@
 package be.nikiroo.gofetch.support;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.net.URLConnection;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.zip.GZIPInputStream;
 
 import org.jsoup.helper.StringUtil;
 import org.jsoup.nodes.Element;
@@ -17,8 +13,11 @@ import org.jsoup.select.NodeTraversor;
 import org.jsoup.select.NodeVisitor;
 
 import be.nikiroo.gofetch.data.Story;
+import be.nikiroo.utils.Downloader;
 
 public abstract class BasicSupport {
+       protected static Downloader downloader = new Downloader("gofetcher");
+
        public enum Type {
                SLASHDOT, PIPEDOT, LWN, LEMONDE,
        }
@@ -193,18 +192,6 @@ public abstract class BasicSupport {
                return preselector + "/" + type + "/";
        }
 
-       // TODO: check Downloader.java?
-       static protected InputStream open(URL url) throws IOException {
-               URLConnection conn = url.openConnection();
-               conn.connect();
-               InputStream in = conn.getInputStream();
-               if ("gzip".equals(conn.getContentEncoding())) {
-                       in = new GZIPInputStream(in);
-               }
-
-               return in;
-       }
-
        /**
         * Get the first {@link Element} of the given class, or an empty span
         * {@link Element} if none found.

diff --git a/src/be/nikiroo/gofetch/support/LWN.java b/src/be/nikiroo/gofetch/support/LWN.java
index c48ed2a2130b63b5bc81ed3389612e4efdb72ec0..27b539c5e42ffa6bec08c5c4e689377bf3742068 100644 (file)
--- a/src/be/nikiroo/gofetch/support/LWN.java
+++ b/src/be/nikiroo/gofetch/support/LWN.java
@@ -31,7 +31,7 @@ public class LWN extends BasicSupport {
                List<Story> list = new ArrayList<Story>();
 
                URL url = new URL("https://lwn.net/");
-               InputStream in = open(url);
+               InputStream in = downloader.open(url);
                Document doc = DataUtil.load(in, "UTF-8", url.toString());
                Elements articles = doc.getElementsByClass("pure-u-1");
                for (Element article : articles) {
@@ -99,7 +99,7 @@ public class LWN extends BasicSupport {
                // Do not try the paid-for stories...
                if (!story.getTitle().startsWith("[$]")) {
                        URL url = new URL(story.getUrlInternal());
-                       InputStream in = open(url);
+                       InputStream in = downloader.open(url);
                        Document doc = DataUtil.load(in, "UTF-8", url.toString());
                        Elements fullContentElements = doc
                                        .getElementsByClass("ArticleText");

diff --git a/src/be/nikiroo/gofetch/support/LeMonde.java b/src/be/nikiroo/gofetch/support/LeMonde.java
index c83dc14b8d0198c89ac46bf5ac7980c86eaf2da6..d11ba797f7767cb89e84793fb2234de2b5417b6d 100644 (file)
--- a/src/be/nikiroo/gofetch/support/LeMonde.java
+++ b/src/be/nikiroo/gofetch/support/LeMonde.java
@@ -28,7 +28,7 @@ public class LeMonde extends BasicSupport {
                for (String topic : new String[] { "international", "politique",
                                "societe", "sciences" }) {
                        URL url = new URL("http://www.lemonde.fr/" + topic + "/1.html");
-                       InputStream in = open(url);
+                       InputStream in = downloader.open(url);
                        Document doc = DataUtil.load(in, "UTF-8", url.toString());
                        Elements articles = doc.getElementsByTag("article");
                        for (Element article : articles) {
@@ -74,7 +74,7 @@ public class LeMonde extends BasicSupport {
                // some javascript, I need to check...)
 
                URL url = new URL(story.getUrlInternal());
-               InputStream in = open(url);
+               InputStream in = downloader.open(url);
                Document doc = DataUtil.load(in, "UTF-8", url.toString());
                Element article = doc.getElementById("articleBody");
                if (article != null) {

diff --git a/src/be/nikiroo/gofetch/support/Pipedot.java b/src/be/nikiroo/gofetch/support/Pipedot.java
index 8db4749d71855287d644936ab716c1ae2b19159e..17388b21030f5cbdbbd12518b9b0b1023594fff5 100644 (file)
--- a/src/be/nikiroo/gofetch/support/Pipedot.java
+++ b/src/be/nikiroo/gofetch/support/Pipedot.java
@@ -31,7 +31,7 @@ public class Pipedot extends BasicSupport {
                List<Story> list = new ArrayList<Story>();
 
                URL url = new URL("https://pipedot.org/");
-               InputStream in = open(url);
+               InputStream in = downloader.open(url);
                Document doc = DataUtil.load(in, "UTF-8", url.toString());
                Elements articles = doc.getElementsByClass("story");
                for (Element article : articles) {
@@ -94,7 +94,7 @@ public class Pipedot extends BasicSupport {
                List<Comment> comments = new ArrayList<Comment>();
 
                URL url = new URL(story.getUrlInternal());
-               InputStream in = open(url);
+               InputStream in = downloader.open(url);
                Document doc = DataUtil.load(in, "UTF-8", url.toString());
                Elements listing = doc.getElementsByTag("main");
                if (listing.size() > 0) {

diff --git a/src/be/nikiroo/gofetch/support/Slashdot.java b/src/be/nikiroo/gofetch/support/Slashdot.java
index 1581d23cb2361f8fa55c912ff1e2516c8fc91d9d..43d35f4afa13281d8565da54e113fab82529f35f 100644 (file)
--- a/src/be/nikiroo/gofetch/support/Slashdot.java
+++ b/src/be/nikiroo/gofetch/support/Slashdot.java
@@ -31,7 +31,7 @@ public class Slashdot extends BasicSupport {
                List<Story> list = new ArrayList<Story>();
 
                URL url = new URL("https://slashdot.org/");
-               InputStream in = open(url);
+               InputStream in = downloader.open(url);
                Document doc = DataUtil.load(in, "UTF-8", url.toString());
                Elements articles = doc.getElementsByTag("header");
                for (Element article : articles) {
@@ -81,7 +81,7 @@ public class Slashdot extends BasicSupport {
                List<Comment> comments = new ArrayList<Comment>();
 
                URL url = new URL(story.getUrlInternal());
-               InputStream in = open(url);
+               InputStream in = downloader.open(url);
                Document doc = DataUtil.load(in, "UTF-8", url.toString());
                Element listing = doc.getElementById("commentlisting");
                if (listing != null) {