Skip bad stories instead of aborting the process

[gofetch.git] / src / be / nikiroo / gofetch / Fetcher.java

diff --git a/src/be/nikiroo/gofetch/Fetcher.java b/src/be/nikiroo/gofetch/Fetcher.java
index 7490037c201929aba38297d809e7f9c917af1e95..258f1961e58b8746b8286f7131726d2e9de659d3 100644 (file)
--- a/src/be/nikiroo/gofetch/Fetcher.java
+++ b/src/be/nikiroo/gofetch/Fetcher.java
@@ -42,8 +42,8 @@ public class Fetcher {
         *            the sub directory and (pre-)selector to use for the resources
         *            (<b>will</b> have an impact on the files' content)
         * @param type
-        *            the type of news to get (or the special keyword ALL to get all
-        *            of the supported sources)
+        *            the type of news to get (or NULL to get all of the supported
+        *            sources)
         * @param maxStories
         *            the maximum number of stories to show on the resume page
         * @param hostname
@@ -81,7 +81,12 @@ public class Fetcher {
                        BasicSupport support = BasicSupport.getSupport(type);
 
                        if (type == this.type || this.type == null) {
-                               list(support);
+                               try {
+                                       list(support);
+                               } catch (Exception e) {
+                                       new Exception("Failed to process support: " + type, e)
+                                                       .printStackTrace();
+                               }
                        }
 
                        gopherBuilder.append(getLink(support.getDescription(),
@@ -93,14 +98,14 @@ public class Fetcher {
                        }
                        ref = "../" + ref + "/index.html";
 
-                       htmlBuilder.append(getLink(support.getDescription(), ref, false,
+                       htmlBuilder.append(getLink(support.getDescription(), ref, true,
                                        true));
                }
 
                File gopherCache = new File(dir, preselector);
                gopherCache.mkdirs();
                File htmlIndex = new File(gopherCache, "index.html");
-               gopherCache = new File(gopherCache, ".cache");
+               gopherCache = new File(gopherCache, "gophermap");
 
                Output gopher = new Gopher(null, hostname, preselector, port);
                Output html = new Html(null, hostname, preselector, port);
@@ -142,12 +147,20 @@ public class Fetcher {
 
                // Get comments (and update stories if needed):
                int i = 1;
+               List<Story> fetchedStories = new ArrayList<Story>(stories.size());
                for (Story story : stories) {
-                       System.err.println(String.format("%02d/%02d", i, stories.size())
+                       System.err.print(String.format("%02d/%02d", i, stories.size())
                                        + " Fetching full story " + story.getId() + "...");
-                       support.fetch(story);
+                       try {
+                               support.fetch(story);
+                               fetchedStories.add(story);
+                               System.err.println();
+                       } catch (IOException e) {
+                               System.err.println(" Failed to get story!");
+                       }
                        i++;
                }
+               stories = fetchedStories;
 
                Output gopher = new Gopher(support.getType(), hostname, preselector,
                                port);
@@ -188,22 +201,26 @@ public class Fetcher {
                List<String> gopherLines = new ArrayList<String>();
                List<String> htmlLines = new ArrayList<String>();
                for (i = 0; i < headers.length; i++) {
-                       gopherLines
-                                       .add(IOUtils.readSmallFile(new File(varDir, headers[i])));
-                       htmlLines.add(IOUtils.readSmallFile(new File(varDir, headers[i]
-                                       + ".html")));
+                       File gopherFile = new File(varDir, headers[i]);
+                       File htmlFile = new File(varDir, headers[i] + ".html");
+
+                       if (gopherFile.exists())
+                               gopherLines.add(IOUtils.readSmallFile(gopherFile));
+                       if (htmlFile.exists())
+                               htmlLines.add(IOUtils.readSmallFile(htmlFile));
 
                        boolean enoughStories = (i > 0 && i % maxStories == 0);
                        boolean last = i == headers.length - 1;
                        if (enoughStories || last) {
                                if (!last) {
                                        gopherLines.add(getLink("More", support.getSelector()
-                                                       + ".cache_" + (page + 1), false, false));
+                                                       + "gophermap_" + (page + 1), true, false));
+
                                        htmlLines.add(getLink("More", "index_" + (page + 1)
-                                                       + ".html", false, true));
+                                                       + ".html", true, true));
                                }
 
-                               write(gopherLines, varDir, ".cache", "", page);
+                               write(gopherLines, varDir, "gophermap", "", page);
                                write(htmlLines, varDir, "index", ".html", page);
                                gopherLines = new ArrayList<String>();
                                htmlLines = new ArrayList<String>();
@@ -227,10 +244,24 @@ public class Fetcher {
                }
        }
 
-       private String getLink(String name, String ref, boolean index, boolean html) {
+       /**
+        * Create a link.
+        * 
+        * @param name
+        *            the link name (what the user will see)
+        * @param ref
+        *            the actual link reference (the target)
+        * @param menu
+        *            menu (gophermap, i) mode -- not used in html mode
+        * @param html
+        *            TRUE for html mode, FALSE for gopher mode
+        * 
+        * @return the ready-to-use link in a {@link String}
+        */
+       private String getLink(String name, String ref, boolean menu, boolean html) {
                if (!html) {
-                       return new StringBuilder().append("1" + name).append("\t")
-                                       .append((index ? "1" : "0") + ref) //
+                       return new StringBuilder().append((menu ? "1" : "0") + name)
+                                       .append("\t").append(ref) //
                                        .append("\t").append(hostname) //
                                        .append("\t").append(Integer.toString(port)) //
                                        .append("\r\n").toString();