X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Fgofetch%2FFetcher.java;h=920b48af504db25f0f61e946c48ce27d931bea99;hb=029feaedb06a22016a2c20df90d6d57d2e5e3b55;hp=5a9c774aa05fb109ed4a5a2d61017ee2a45c78e3;hpb=737852686d8897331706ed4b902dbd9d5038cb53;p=gofetch.git diff --git a/src/be/nikiroo/gofetch/Fetcher.java b/src/be/nikiroo/gofetch/Fetcher.java index 5a9c774..920b48a 100644 --- a/src/be/nikiroo/gofetch/Fetcher.java +++ b/src/be/nikiroo/gofetch/Fetcher.java @@ -4,10 +4,11 @@ import java.io.File; import java.io.FileWriter; import java.io.FilenameFilter; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; -import be.nikiroo.gofetch.data.Comment; import be.nikiroo.gofetch.data.Story; import be.nikiroo.gofetch.output.Gopher; import be.nikiroo.gofetch.output.Html; @@ -72,51 +73,52 @@ public class Fetcher { * in case of I/O error */ public void start() throws IOException { - File cache = new File(dir, preselector); - cache.mkdirs(); - File cacheHtml = new File(cache, "index.html"); - cache = new File(cache, ".cache"); + StringBuilder gopherBuilder = new StringBuilder(); + StringBuilder htmlBuilder = new StringBuilder(); - Output gopher = new Gopher(null, hostname, port); - Output html = new Html(null); + BasicSupport.setPreselector(preselector); + for (Type type : Type.values()) { + BasicSupport support = BasicSupport.getSupport(type); - FileWriter writer = new FileWriter(cache); - try { - FileWriter writerHtml = new FileWriter(cacheHtml); - try { - writer.append(gopher.getIndexHeader()); - writerHtml.append(html.getIndexHeader()); - - Type types[]; - if (type == null) { - types = Type.values(); - } else { - types = new Type[] { type }; - } + if (type == this.type || this.type == null) { + list(support); + } - BasicSupport.setPreselector(preselector); - for (Type type : types) { - BasicSupport support = BasicSupport.getSupport(type); - list(support); - - writer.append("1" + support.getDescription()).append("\t") - .append("1" + support.getSelector()) // - .append("\t").append(hostname) // - .append("\t").append(Integer.toString(port)) // - .append("\r\n"); - String ref = support.getSelector(); - while (ref.startsWith("/")) { - ref = ref.substring(1); - } - writerHtml.append("
" + support.getDescription() + "
"); - } + gopherBuilder.append(getLink(support.getDescription(), + support.getSelector(), true, false)); - writer.append(gopher.getIndexFooter()); - writerHtml.append(html.getIndexFooter()); - } finally { - writerHtml.close(); + String ref = support.getSelector(); + while (ref.startsWith("/")) { + ref = ref.substring(1); } + ref = "../" + ref + "/index.html"; + + htmlBuilder.append(getLink(support.getDescription(), ref, false, + true)); + } + + File gopherCache = new File(dir, preselector); + gopherCache.mkdirs(); + File htmlIndex = new File(gopherCache, "index.html"); + gopherCache = new File(gopherCache, "gophermap"); + + Output gopher = new Gopher(null, hostname, preselector, port); + Output html = new Html(null, hostname, preselector, port); + + FileWriter writer = new FileWriter(gopherCache); + try { + writer.append(gopher.getIndexHeader()); + writer.append(gopherBuilder.toString()); + writer.append(gopher.getIndexFooter()); + } finally { + writer.close(); + } + + try { + writer = new FileWriter(htmlIndex); + writer.append(html.getIndexHeader()); + writer.append(htmlBuilder.toString()); + writer.append(html.getIndexFooter()); } finally { writer.close(); } @@ -132,34 +134,40 @@ public class Fetcher { * in case of I/O error **/ private void list(BasicSupport support) throws IOException { - Output gopher = new Gopher(support.getType(), hostname, port); - Output html = new Html(support.getType()); - - new File(dir, support.getSelector()).mkdirs(); - + // Get stories: System.err .print("Listing recent news for " + support.getType() + "..."); List stories = support.list(); System.err.println(" " + stories.size() + " stories found!"); + + // Get comments (and update stories if needed): int i = 1; + for (Story story : stories) { + System.err.println(String.format("%02d/%02d", i, stories.size()) + + " Fetching full story " + story.getId() + "..."); + support.fetch(story); + i++; + } + + Output gopher = new Gopher(support.getType(), hostname, preselector, + port); + Output html = new Html(support.getType(), hostname, preselector, port); + + new File(dir, support.getSelector()).mkdirs(); + for (Story story : stories) { IOUtils.writeSmallFile(dir, story.getSelector() + ".header", - gopher.export(story)); + gopher.exportHeader(story)); IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html", - html.export(story)); - - System.err.println(String.format("%02d/%02d", i, stories.size()) - + " Fetching comments for story " + story.getId() + "..."); - List comments = support.getComments(story); + html.exportHeader(story)); IOUtils.writeSmallFile(dir, story.getSelector(), - gopher.export(story, comments)); + gopher.export(story)); IOUtils.writeSmallFile(dir, story.getSelector() + ".html", - html.export(story, comments)); - - i++; + html.export(story)); } + // Finding headers of all stories in cache: File varDir = new File(dir, support.getSelector()); String[] headers = varDir.list(new FilenameFilter() { @Override @@ -168,32 +176,77 @@ public class Fetcher { } }); - File cache = new File(varDir, ".cache"); - File cacheHtml = new File(varDir, "index.html"); - FileWriter writer = new FileWriter(cache); - try { - FileWriter writerHtml = new FileWriter(cacheHtml); - try { - if (headers.length > 0) { - Arrays.sort(headers); - int from = headers.length - 1; - int to = headers.length - maxStories; - if (to < 0) { - to = 0; - } - for (i = from; i >= to; i--) { - writer.append(IOUtils.readSmallFile(new File(varDir, - headers[i]))); - - writerHtml.append(IOUtils.readSmallFile(new File( - varDir, headers[i] + ".html"))); - } + // Reverse sort: + Arrays.sort(headers); + List tmp = Arrays.asList(headers); + Collections.reverse(tmp); + headers = tmp.toArray(new String[] {}); + // + + // Write the index (with "MORE" links if needed) + int page = 0; + List gopherLines = new ArrayList(); + List htmlLines = new ArrayList(); + for (i = 0; i < headers.length; i++) { + gopherLines + .add(IOUtils.readSmallFile(new File(varDir, headers[i]))); + htmlLines.add(IOUtils.readSmallFile(new File(varDir, headers[i] + + ".html"))); + + boolean enoughStories = (i > 0 && i % maxStories == 0); + boolean last = i == headers.length - 1; + if (enoughStories || last) { + if (!last) { + gopherLines.add(getLink("More", support.getSelector() + + "gophermap_" + (page + 1), false, false)); + htmlLines.add(getLink("More", "index_" + (page + 1) + + ".html", false, true)); } - } finally { - writerHtml.close(); + + write(gopherLines, varDir, "gophermap", "", page); + write(htmlLines, varDir, "index", ".html", page); + gopherLines = new ArrayList(); + htmlLines = new ArrayList(); + page++; + } + } + } + + private void write(List lines, File varDir, String basename, + String ext, int page) throws IOException { + File file = new File(varDir, basename + (page > 0 ? "_" + page : "") + + ext); + + FileWriter writer = new FileWriter(file); + try { + for (String line : lines) { + writer.append(line).append("\r\n"); } } finally { writer.close(); } } + + /** + * + * @param name + * @param ref + * @param index + * this link is in an index (and link to a NON index) + * @param html + * @return + */ + private String getLink(String name, String ref, boolean index, boolean html) { + if (!html) { + return new StringBuilder().append((index ? "0" : "1") + name) + .append("\t").append(ref) // + .append("\t").append(hostname) // + .append("\t").append(Integer.toString(port)) // + .append("\r\n").toString(); + } + + return new StringBuilder().append( + "
" + name + + "
\n").toString(); + } }