X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Fgofetch%2FFetcher.java;h=9dbf0e10bf60d52daad2081f9c615d27dc19973d;hb=6c13eccdd9953ff6d65f6fcad3a01fd2d57d68fd;hp=5a9c774aa05fb109ed4a5a2d61017ee2a45c78e3;hpb=737852686d8897331706ed4b902dbd9d5038cb53;p=gofetch.git
diff --git a/src/be/nikiroo/gofetch/Fetcher.java b/src/be/nikiroo/gofetch/Fetcher.java
index 5a9c774..9dbf0e1 100644
--- a/src/be/nikiroo/gofetch/Fetcher.java
+++ b/src/be/nikiroo/gofetch/Fetcher.java
@@ -4,10 +4,11 @@ import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
-import be.nikiroo.gofetch.data.Comment;
import be.nikiroo.gofetch.data.Story;
import be.nikiroo.gofetch.output.Gopher;
import be.nikiroo.gofetch.output.Html;
@@ -72,51 +73,52 @@ public class Fetcher {
* in case of I/O error
*/
public void start() throws IOException {
- File cache = new File(dir, preselector);
- cache.mkdirs();
- File cacheHtml = new File(cache, "index.html");
- cache = new File(cache, ".cache");
+ StringBuilder gopherBuilder = new StringBuilder();
+ StringBuilder htmlBuilder = new StringBuilder();
- Output gopher = new Gopher(null, hostname, port);
- Output html = new Html(null);
+ BasicSupport.setPreselector(preselector);
+ for (Type type : Type.values()) {
+ BasicSupport support = BasicSupport.getSupport(type);
- FileWriter writer = new FileWriter(cache);
- try {
- FileWriter writerHtml = new FileWriter(cacheHtml);
- try {
- writer.append(gopher.getIndexHeader());
- writerHtml.append(html.getIndexHeader());
-
- Type types[];
- if (type == null) {
- types = Type.values();
- } else {
- types = new Type[] { type };
- }
+ if (type == this.type || this.type == null) {
+ list(support);
+ }
- BasicSupport.setPreselector(preselector);
- for (Type type : types) {
- BasicSupport support = BasicSupport.getSupport(type);
- list(support);
-
- writer.append("1" + support.getDescription()).append("\t")
- .append("1" + support.getSelector()) //
- .append("\t").append(hostname) //
- .append("\t").append(Integer.toString(port)) //
- .append("\r\n");
- String ref = support.getSelector();
- while (ref.startsWith("/")) {
- ref = ref.substring(1);
- }
- writerHtml.append("
");
- }
+ gopherBuilder.append(getLink(support.getDescription(),
+ support.getSelector(), true, false));
- writer.append(gopher.getIndexFooter());
- writerHtml.append(html.getIndexFooter());
- } finally {
- writerHtml.close();
+ String ref = support.getSelector();
+ while (ref.startsWith("/")) {
+ ref = ref.substring(1);
}
+ ref = "../" + ref + "/index.html";
+
+ htmlBuilder.append(getLink(support.getDescription(), ref, true,
+ true));
+ }
+
+ File gopherCache = new File(dir, preselector);
+ gopherCache.mkdirs();
+ File htmlIndex = new File(gopherCache, "index.html");
+ gopherCache = new File(gopherCache, "gophermap");
+
+ Output gopher = new Gopher(null, hostname, preselector, port);
+ Output html = new Html(null, hostname, preselector, port);
+
+ FileWriter writer = new FileWriter(gopherCache);
+ try {
+ writer.append(gopher.getIndexHeader());
+ writer.append(gopherBuilder.toString());
+ writer.append(gopher.getIndexFooter());
+ } finally {
+ writer.close();
+ }
+
+ try {
+ writer = new FileWriter(htmlIndex);
+ writer.append(html.getIndexHeader());
+ writer.append(htmlBuilder.toString());
+ writer.append(html.getIndexFooter());
} finally {
writer.close();
}
@@ -132,34 +134,40 @@ public class Fetcher {
* in case of I/O error
**/
private void list(BasicSupport support) throws IOException {
- Output gopher = new Gopher(support.getType(), hostname, port);
- Output html = new Html(support.getType());
-
- new File(dir, support.getSelector()).mkdirs();
-
+ // Get stories:
System.err
.print("Listing recent news for " + support.getType() + "...");
List stories = support.list();
System.err.println(" " + stories.size() + " stories found!");
+
+ // Get comments (and update stories if needed):
int i = 1;
+ for (Story story : stories) {
+ System.err.println(String.format("%02d/%02d", i, stories.size())
+ + " Fetching full story " + story.getId() + "...");
+ support.fetch(story);
+ i++;
+ }
+
+ Output gopher = new Gopher(support.getType(), hostname, preselector,
+ port);
+ Output html = new Html(support.getType(), hostname, preselector, port);
+
+ new File(dir, support.getSelector()).mkdirs();
+
for (Story story : stories) {
IOUtils.writeSmallFile(dir, story.getSelector() + ".header",
- gopher.export(story));
+ gopher.exportHeader(story));
IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html",
- html.export(story));
-
- System.err.println(String.format("%02d/%02d", i, stories.size())
- + " Fetching comments for story " + story.getId() + "...");
- List comments = support.getComments(story);
+ html.exportHeader(story));
IOUtils.writeSmallFile(dir, story.getSelector(),
- gopher.export(story, comments));
+ gopher.export(story));
IOUtils.writeSmallFile(dir, story.getSelector() + ".html",
- html.export(story, comments));
-
- i++;
+ html.export(story));
}
+ // Finding headers of all stories in cache:
File varDir = new File(dir, support.getSelector());
String[] headers = varDir.list(new FilenameFilter() {
@Override
@@ -168,32 +176,77 @@ public class Fetcher {
}
});
- File cache = new File(varDir, ".cache");
- File cacheHtml = new File(varDir, "index.html");
- FileWriter writer = new FileWriter(cache);
- try {
- FileWriter writerHtml = new FileWriter(cacheHtml);
- try {
- if (headers.length > 0) {
- Arrays.sort(headers);
- int from = headers.length - 1;
- int to = headers.length - maxStories;
- if (to < 0) {
- to = 0;
- }
- for (i = from; i >= to; i--) {
- writer.append(IOUtils.readSmallFile(new File(varDir,
- headers[i])));
-
- writerHtml.append(IOUtils.readSmallFile(new File(
- varDir, headers[i] + ".html")));
- }
+ // Reverse sort:
+ Arrays.sort(headers);
+ List tmp = Arrays.asList(headers);
+ Collections.reverse(tmp);
+ headers = tmp.toArray(new String[] {});
+ //
+
+ // Write the index (with "MORE" links if needed)
+ int page = 0;
+ List gopherLines = new ArrayList();
+ List htmlLines = new ArrayList();
+ for (i = 0; i < headers.length; i++) {
+ gopherLines
+ .add(IOUtils.readSmallFile(new File(varDir, headers[i])));
+ htmlLines.add(IOUtils.readSmallFile(new File(varDir, headers[i]
+ + ".html")));
+
+ boolean enoughStories = (i > 0 && i % maxStories == 0);
+ boolean last = i == headers.length - 1;
+ if (enoughStories || last) {
+ if (!last) {
+ gopherLines.add(getLink("More", support.getSelector()
+ + "gophermap_" + (page + 1), false, false));
+ htmlLines.add(getLink("More", "index_" + (page + 1)
+ + ".html", false, true));
}
- } finally {
- writerHtml.close();
+
+ write(gopherLines, varDir, "gophermap", "", page);
+ write(htmlLines, varDir, "index", ".html", page);
+ gopherLines = new ArrayList();
+ htmlLines = new ArrayList();
+ page++;
+ }
+ }
+ }
+
+ private void write(List lines, File varDir, String basename,
+ String ext, int page) throws IOException {
+ File file = new File(varDir, basename + (page > 0 ? "_" + page : "")
+ + ext);
+
+ FileWriter writer = new FileWriter(file);
+ try {
+ for (String line : lines) {
+ writer.append(line).append("\r\n");
}
} finally {
writer.close();
}
}
+
+ /**
+ *
+ * @param name
+ * @param ref
+ * @param menu
+ * menu (gophermap, i) mode
+ * @param html
+ * @return
+ */
+ private String getLink(String name, String ref, boolean menu, boolean html) {
+ if (!html) {
+ return new StringBuilder().append((menu ? "1" : "0") + name)
+ .append("\t").append(ref) //
+ .append("\t").append(hostname) //
+ .append("\t").append(Integer.toString(port)) //
+ .append("\r\n").toString();
+ }
+
+ return new StringBuilder().append(
+ "\n").toString();
+ }
}