package be.nikiroo.gofetch; import java.io.File; import java.io.FileWriter; import java.io.FilenameFilter; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import be.nikiroo.gofetch.data.Story; import be.nikiroo.gofetch.output.Gopher; import be.nikiroo.gofetch.output.Html; import be.nikiroo.gofetch.output.Output; import be.nikiroo.gofetch.support.BasicSupport; import be.nikiroo.gofetch.support.Type; import be.nikiroo.utils.IOUtils; /** * The class that will manage the fetch operations. *
* It will scrap the required websites and process them to disk. * * @author niki */ public class Fetcher { private File dir; private String preselector; private int maxStories; private String hostname; private int port; private Type type; /** * Prepare a new {@link Fetcher}. * * @param dir * the target directory where to save the files (won't have * impact on the files' content) * @param preselector * the sub directory and (pre-)selector to use for the resources * (will have an impact on the files' content) * @param type * the type of news to get (or NULL to get all of the supported * sources) * @param maxStories * the maximum number of stories to show on the resume page * @param hostname * the gopher host to use (will have an impact on the * files' content) * @param port * the gopher port to use (will have an impact on the * files' content) */ public Fetcher(File dir, String preselector, Type type, int maxStories, String hostname, int port) { this.dir = dir; this.preselector = preselector; this.type = type; this.maxStories = maxStories; this.hostname = hostname; this.port = port; } /** * Start the fetching operation. *
* This method will handle the main pages itself, and will call
* {@link Fetcher#list(BasicSupport)} for the stories.
*
* @throws IOException
* in case of I/O error
*/
public void start() throws IOException {
StringBuilder gopherBuilder = new StringBuilder();
StringBuilder htmlBuilder = new StringBuilder();
BasicSupport.setPreselector(preselector);
for (Type type : Type.values()) {
BasicSupport support = BasicSupport.getSupport(type);
if (type == this.type || this.type == null) {
try {
list(support);
} catch (Exception e) {
new Exception("Failed to process support: " + type, e)
.printStackTrace();
}
}
gopherBuilder.append(getLink(support.getDescription(),
support.getSelector(), true, false));
String ref = support.getSelector();
while (ref.startsWith("/")) {
ref = ref.substring(1);
}
ref = "../" + ref + "/index.html";
htmlBuilder.append(getLink(support.getDescription(), ref, true,
true));
}
File gopherCache = new File(dir, preselector);
gopherCache.mkdirs();
File htmlIndex = new File(gopherCache, "index.html");
gopherCache = new File(gopherCache, "gophermap");
Output gopher = new Gopher(null, hostname, preselector, port);
Output html = new Html(null, hostname, preselector, port);
FileWriter writer = new FileWriter(gopherCache);
try {
writer.append(gopher.getMainIndexHeader());
writer.append(gopherBuilder.toString());
writer.append(gopher.getMainIndexFooter());
} finally {
writer.close();
}
try {
writer = new FileWriter(htmlIndex);
writer.append(html.getMainIndexHeader());
writer.append(htmlBuilder.toString());
writer.append(html.getMainIndexFooter());
} finally {
writer.close();
}
}
/**
* Process the stories for the given {@link BasicSupport} to disk.
*
* @param support
* the {@link BasicSupport} to download from
*
* @throws IOException
* in case of I/O error
**/
private void list(BasicSupport support) throws IOException {
// Get stories:
System.err
.print("Listing recent news for " + support.getType() + "...");
List