X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Fgofetch%2FFetcher.java;h=bd5a4543fda92a92da61d665205fb5744364b217;hb=e570bb42dfb90f4c31429069ac7095514ff3ee2f;hp=5a9c774aa05fb109ed4a5a2d61017ee2a45c78e3;hpb=737852686d8897331706ed4b902dbd9d5038cb53;p=gofetch.git diff --git a/src/be/nikiroo/gofetch/Fetcher.java b/src/be/nikiroo/gofetch/Fetcher.java index 5a9c774..bd5a454 100644 --- a/src/be/nikiroo/gofetch/Fetcher.java +++ b/src/be/nikiroo/gofetch/Fetcher.java @@ -4,10 +4,11 @@ import java.io.File; import java.io.FileWriter; import java.io.FilenameFilter; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; -import be.nikiroo.gofetch.data.Comment; import be.nikiroo.gofetch.data.Story; import be.nikiroo.gofetch.output.Gopher; import be.nikiroo.gofetch.output.Html; @@ -41,8 +42,8 @@ public class Fetcher { * the sub directory and (pre-)selector to use for the resources * (will have an impact on the files' content) * @param type - * the type of news to get (or the special keyword ALL to get all - * of the supported sources) + * the type of news to get (or NULL to get all of the supported + * sources) * @param maxStories * the maximum number of stories to show on the resume page * @param hostname @@ -72,51 +73,57 @@ public class Fetcher { * in case of I/O error */ public void start() throws IOException { - File cache = new File(dir, preselector); - cache.mkdirs(); - File cacheHtml = new File(cache, "index.html"); - cache = new File(cache, ".cache"); + StringBuilder gopherBuilder = new StringBuilder(); + StringBuilder htmlBuilder = new StringBuilder(); - Output gopher = new Gopher(null, hostname, port); - Output html = new Html(null); + BasicSupport.setPreselector(preselector); + for (Type type : Type.values()) { + BasicSupport support = BasicSupport.getSupport(type); - FileWriter writer = new FileWriter(cache); - try { - FileWriter writerHtml = new FileWriter(cacheHtml); - try { - writer.append(gopher.getIndexHeader()); - writerHtml.append(html.getIndexHeader()); - - Type types[]; - if (type == null) { - types = Type.values(); - } else { - types = new Type[] { type }; - } - - BasicSupport.setPreselector(preselector); - for (Type type : types) { - BasicSupport support = BasicSupport.getSupport(type); + if (type == this.type || this.type == null) { + try { list(support); - - writer.append("1" + support.getDescription()).append("\t") - .append("1" + support.getSelector()) // - .append("\t").append(hostname) // - .append("\t").append(Integer.toString(port)) // - .append("\r\n"); - String ref = support.getSelector(); - while (ref.startsWith("/")) { - ref = ref.substring(1); - } - writerHtml.append("
" + support.getDescription() + "
"); + } catch (Exception e) { + new Exception("Failed to process support: " + type, e) + .printStackTrace(); } + } + + gopherBuilder.append(getLink(support.getDescription(), + support.getSelector(), true, false)); - writer.append(gopher.getIndexFooter()); - writerHtml.append(html.getIndexFooter()); - } finally { - writerHtml.close(); + String ref = support.getSelector(); + while (ref.startsWith("/")) { + ref = ref.substring(1); } + ref = "../" + ref + "/index.html"; + + htmlBuilder.append(getLink(support.getDescription(), ref, true, + true)); + } + + File gopherCache = new File(dir, preselector); + gopherCache.mkdirs(); + File htmlIndex = new File(gopherCache, "index.html"); + gopherCache = new File(gopherCache, "gophermap"); + + Output gopher = new Gopher(null, hostname, preselector, port); + Output html = new Html(null, hostname, preselector, port); + + FileWriter writer = new FileWriter(gopherCache); + try { + writer.append(gopher.getIndexHeader()); + writer.append(gopherBuilder.toString()); + writer.append(gopher.getIndexFooter()); + } finally { + writer.close(); + } + + try { + writer = new FileWriter(htmlIndex); + writer.append(html.getIndexHeader()); + writer.append(htmlBuilder.toString()); + writer.append(html.getIndexFooter()); } finally { writer.close(); } @@ -132,34 +139,40 @@ public class Fetcher { * in case of I/O error **/ private void list(BasicSupport support) throws IOException { - Output gopher = new Gopher(support.getType(), hostname, port); - Output html = new Html(support.getType()); - - new File(dir, support.getSelector()).mkdirs(); - + // Get stories: System.err .print("Listing recent news for " + support.getType() + "..."); List stories = support.list(); System.err.println(" " + stories.size() + " stories found!"); + + // Get comments (and update stories if needed): int i = 1; + for (Story story : stories) { + System.err.println(String.format("%02d/%02d", i, stories.size()) + + " Fetching full story " + story.getId() + "..."); + support.fetch(story); + i++; + } + + Output gopher = new Gopher(support.getType(), hostname, preselector, + port); + Output html = new Html(support.getType(), hostname, preselector, port); + + new File(dir, support.getSelector()).mkdirs(); + for (Story story : stories) { IOUtils.writeSmallFile(dir, story.getSelector() + ".header", - gopher.export(story)); + gopher.exportHeader(story)); IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html", - html.export(story)); - - System.err.println(String.format("%02d/%02d", i, stories.size()) - + " Fetching comments for story " + story.getId() + "..."); - List comments = support.getComments(story); + html.exportHeader(story)); IOUtils.writeSmallFile(dir, story.getSelector(), - gopher.export(story, comments)); + gopher.export(story)); IOUtils.writeSmallFile(dir, story.getSelector() + ".html", - html.export(story, comments)); - - i++; + html.export(story)); } + // Finding headers of all stories in cache: File varDir = new File(dir, support.getSelector()); String[] headers = varDir.list(new FilenameFilter() { @Override @@ -168,32 +181,86 @@ public class Fetcher { } }); - File cache = new File(varDir, ".cache"); - File cacheHtml = new File(varDir, "index.html"); - FileWriter writer = new FileWriter(cache); - try { - FileWriter writerHtml = new FileWriter(cacheHtml); - try { - if (headers.length > 0) { - Arrays.sort(headers); - int from = headers.length - 1; - int to = headers.length - maxStories; - if (to < 0) { - to = 0; - } - for (i = from; i >= to; i--) { - writer.append(IOUtils.readSmallFile(new File(varDir, - headers[i]))); - - writerHtml.append(IOUtils.readSmallFile(new File( - varDir, headers[i] + ".html"))); - } + // Reverse sort: + Arrays.sort(headers); + List tmp = Arrays.asList(headers); + Collections.reverse(tmp); + headers = tmp.toArray(new String[] {}); + // + + // Write the index (with "MORE" links if needed) + int page = 0; + List gopherLines = new ArrayList(); + List htmlLines = new ArrayList(); + for (i = 0; i < headers.length; i++) { + File gopherFile = new File(varDir, headers[i]); + File htmlFile = new File(varDir, headers[i] + ".html"); + + if (gopherFile.exists()) + gopherLines.add(IOUtils.readSmallFile(gopherFile)); + if (htmlFile.exists()) + htmlLines.add(IOUtils.readSmallFile(htmlFile)); + + boolean enoughStories = (i > 0 && i % maxStories == 0); + boolean last = i == headers.length - 1; + if (enoughStories || last) { + if (!last) { + gopherLines.add(getLink("More", support.getSelector() + + "gophermap_" + (page + 1), true, false)); + + htmlLines.add(getLink("More", "index_" + (page + 1) + + ".html", true, true)); } - } finally { - writerHtml.close(); + + write(gopherLines, varDir, "gophermap", "", page); + write(htmlLines, varDir, "index", ".html", page); + gopherLines = new ArrayList(); + htmlLines = new ArrayList(); + page++; + } + } + } + + private void write(List lines, File varDir, String basename, + String ext, int page) throws IOException { + File file = new File(varDir, basename + (page > 0 ? "_" + page : "") + + ext); + + FileWriter writer = new FileWriter(file); + try { + for (String line : lines) { + writer.append(line).append("\r\n"); } } finally { writer.close(); } } + + /** + * Create a link. + * + * @param name + * the link name (what the user will see) + * @param ref + * the actual link reference (the target) + * @param menu + * menu (gophermap, i) mode -- not used in html mode + * @param html + * TRUE for html mode, FALSE for gopher mode + * + * @return the ready-to-use link in a {@link String} + */ + private String getLink(String name, String ref, boolean menu, boolean html) { + if (!html) { + return new StringBuilder().append((menu ? "1" : "0") + name) + .append("\t").append(ref) // + .append("\t").append(hostname) // + .append("\t").append(Integer.toString(port)) // + .append("\r\n").toString(); + } + + return new StringBuilder().append( + "
" + name + + "
\n").toString(); + } }