import java.util.Arrays;
import java.util.List;
-import be.nikiroo.gofetch.data.Comment;
import be.nikiroo.gofetch.data.Story;
import be.nikiroo.gofetch.output.Gopher;
import be.nikiroo.gofetch.output.Html;
* in case of I/O error
*/
public void start() throws IOException {
- File cache = new File(dir, preselector);
- cache.mkdirs();
- File cacheHtml = new File(cache, "index.html");
- cache = new File(cache, ".cache");
+ StringBuilder gopherBuilder = new StringBuilder();
+ StringBuilder htmlBuilder = new StringBuilder();
+
+ BasicSupport.setPreselector(preselector);
+ for (Type type : Type.values()) {
+ BasicSupport support = BasicSupport.getSupport(type);
+
+ if (type == this.type || this.type == null) {
+ list(support);
+ }
+
+ gopherBuilder.append("1" + support.getDescription()).append("\t")
+ .append("1" + support.getSelector()) //
+ .append("\t").append(hostname) //
+ .append("\t").append(Integer.toString(port)) //
+ .append("\r\n");
+
+ String ref = support.getSelector();
+ while (ref.startsWith("/")) {
+ ref = ref.substring(1);
+ }
+ htmlBuilder.append("<div class='site'><a href='../" + ref + "'>"
+ + support.getDescription() + "</a></div>\n");
+ }
+
+ File gopherCache = new File(dir, preselector);
+ gopherCache.mkdirs();
+ File htmlIndex = new File(gopherCache, "index.html");
+ gopherCache = new File(gopherCache, ".cache");
Output gopher = new Gopher(null, hostname, preselector, port);
Output html = new Html(null, hostname, preselector, port);
- FileWriter writer = new FileWriter(cache);
+ FileWriter writer = new FileWriter(gopherCache);
try {
- FileWriter writerHtml = new FileWriter(cacheHtml);
- try {
- writer.append(gopher.getIndexHeader());
- writerHtml.append(html.getIndexHeader());
-
- BasicSupport.setPreselector(preselector);
- for (Type type : Type.values()) {
- BasicSupport support = BasicSupport.getSupport(type);
-
- if (type == this.type || this.type == null) {
- list(support);
- }
-
- writer.append("1" + support.getDescription()).append("\t")
- .append("1" + support.getSelector()) //
- .append("\t").append(hostname) //
- .append("\t").append(Integer.toString(port)) //
- .append("\r\n");
- String ref = support.getSelector();
- while (ref.startsWith("/")) {
- ref = ref.substring(1);
- }
- writerHtml.append("<div class='site'><a href='../" + ref
- + "'>" + support.getDescription() + "</a></div>\n");
- }
-
- writer.append(gopher.getIndexFooter());
- writerHtml.append(html.getIndexFooter());
- } finally {
- writerHtml.close();
- }
+ writer.append(gopher.getIndexHeader());
+ writer.append(gopherBuilder.toString());
+ writer.append(gopher.getIndexFooter());
+ } finally {
+ writer.close();
+ }
+
+ try {
+ writer = new FileWriter(htmlIndex);
+ writer.append(html.getIndexHeader());
+ writer.append(htmlBuilder.toString());
+ writer.append(html.getIndexFooter());
} finally {
writer.close();
}
* in case of I/O error
**/
private void list(BasicSupport support) throws IOException {
+ // Get stories:
+ System.err
+ .print("Listing recent news for " + support.getType() + "...");
+ List<Story> stories = support.list();
+ System.err.println(" " + stories.size() + " stories found!");
+
+ // Get comments (and update stories if needed):
+ int i = 1;
+ for (Story story : stories) {
+ System.err.println(String.format("%02d/%02d", i, stories.size())
+ + " Fetching full story " + story.getId() + "...");
+ support.fetch(story);
+ i++;
+ }
+
Output gopher = new Gopher(support.getType(), hostname, preselector,
port);
Output html = new Html(support.getType(), hostname, preselector, port);
new File(dir, support.getSelector()).mkdirs();
- System.err
- .print("Listing recent news for " + support.getType() + "...");
- List<Story> stories = support.list();
- System.err.println(" " + stories.size() + " stories found!");
- int i = 1;
for (Story story : stories) {
IOUtils.writeSmallFile(dir, story.getSelector() + ".header",
- gopher.export(story));
+ gopher.exportHeader(story));
IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html",
- html.export(story));
-
- System.err.println(String.format("%02d/%02d", i, stories.size())
- + " Fetching comments for story " + story.getId() + "...");
- List<Comment> comments = support.getComments(story);
+ html.exportHeader(story));
IOUtils.writeSmallFile(dir, story.getSelector(),
- gopher.export(story, comments));
+ gopher.export(story));
IOUtils.writeSmallFile(dir, story.getSelector() + ".html",
- html.export(story, comments));
-
- i++;
+ html.export(story));
}
+ // Finding headers of all stories in cache:
File varDir = new File(dir, support.getSelector());
String[] headers = varDir.list(new FilenameFilter() {
@Override
}
});
- File cache = new File(varDir, ".cache");
- File cacheHtml = new File(varDir, "index.html");
- FileWriter writer = new FileWriter(cache);
+ // Finding which ones to show:
+ int from = 0;
+ int to = 0;
+ if (headers.length > 0) {
+ Arrays.sort(headers);
+ from = headers.length - 1;
+ to = headers.length - maxStories;
+ if (to < 0) {
+ to = 0;
+ }
+ }
+
+ // Writing the cache/index files with the stories:
+ File gopherCache = new File(varDir, ".cache");
+ FileWriter writer = new FileWriter(gopherCache);
+ try {
+ for (i = from; i >= to; i--) {
+ writer.append(IOUtils
+ .readSmallFile(new File(varDir, headers[i])));
+ }
+ } finally {
+ writer.close();
+ }
+
+ File htmlIndex = new File(varDir, "index.html");
+ writer = new FileWriter(htmlIndex);
try {
- FileWriter writerHtml = new FileWriter(cacheHtml);
- try {
- if (headers.length > 0) {
- Arrays.sort(headers);
- int from = headers.length - 1;
- int to = headers.length - maxStories;
- if (to < 0) {
- to = 0;
- }
- for (i = from; i >= to; i--) {
- writer.append(IOUtils.readSmallFile(new File(varDir,
- headers[i])));
-
- writerHtml.append(IOUtils.readSmallFile(new File(
- varDir, headers[i] + ".html")));
- }
- }
- } finally {
- writerHtml.close();
+ for (i = from; i >= to; i--) {
+ writer.append(IOUtils.readSmallFile(new File(varDir, headers[i]
+ + ".html")));
}
} finally {
writer.close();
package be.nikiroo.gofetch.data;
import java.net.URL;
+import java.util.List;
import be.nikiroo.gofetch.support.BasicSupport;
import be.nikiroo.gofetch.support.BasicSupport.Type;
private String urlExternal;
private String content;
+ private String fullContent;
+ private List<Comment> comments;
+
/**
* Create a news story.
*
this.urlInternal = urlInternal;
this.urlExternal = urlExternal;
this.content = content;
+
+ // Defaults fullContent to content
+ this.fullContent = content;
}
public String getSelector() {
public String getContent() {
return content;
}
+
+ /**
+ * @return the fullContent
+ */
+ public String getFullContent() {
+ return fullContent;
+ }
+
+ /**
+ * @param fullContent
+ * the fullContent to set
+ */
+ public void setFullContent(String fullContent) {
+ this.fullContent = fullContent;
+ }
+
+ /**
+ * @return the comments
+ */
+ public List<Comment> getComments() {
+ return comments;
+ }
+
+ /**
+ * @param comments
+ * the comments to set
+ */
+ public void setComments(List<Comment> comments) {
+ this.comments = comments;
+ }
}
\ No newline at end of file
package be.nikiroo.gofetch.output;
-import java.util.List;
-
import be.nikiroo.gofetch.StringJustifier;
import be.nikiroo.gofetch.data.Comment;
import be.nikiroo.gofetch.data.Story;
}
@Override
- public String export(Story story) {
+ public String exportHeader(Story story) {
return append(new StringBuilder(), story, false).append("i\r\ni\r\n")
.toString();
}
@Override
- public String export(Story story, List<Comment> comments) {
+ public String export(Story story) {
StringBuilder builder = new StringBuilder();
append(builder, story, true);
builder.append("i\r\n");
- if (comments != null) {
- for (Comment comment : comments) {
+ if (story.getComments() != null) {
+ for (Comment comment : story.getComments()) {
append(builder, comment, "");
}
}
}
private StringBuilder append(StringBuilder builder, Story story,
- boolean links) {
- if (links) {
+ boolean resume) {
+ if (!resume) {
appendCenter(builder, story.getTitle(), true);
builder.append("i\r\n");
appendLeft(builder, story.getDetails(), " ");
builder.append("i\r\n");
+
builder.append("i o News link: ").append(story.getUrlInternal())
.append("\r\n");
builder.append("i o Source link: ").append(story.getUrlExternal())
.append("\r\n");
builder.append("i\r\n");
+
+ builder.append("i\r\n");
+
+ appendLeft(builder, story.getFullContent(), " ");
} else {
builder.append('1').append(story.getTitle()) //
.append('\t').append("0").append(story.getSelector()) //
.append('\t').append(port) //
.append("\r\n");
appendLeft(builder, story.getDetails(), " ");
- }
-
- builder.append("i\r\n");
+ builder.append("i\r\n");
- appendLeft(builder, story.getContent(), " ");
+ appendLeft(builder, story.getContent(), " ");
+ }
builder.append("i\r\n");
package be.nikiroo.gofetch.output;
-import java.util.List;
-
import be.nikiroo.gofetch.data.Comment;
import be.nikiroo.gofetch.data.Story;
import be.nikiroo.gofetch.support.BasicSupport.Type;
if (!sel.isEmpty()) {
sel = "/1" + sel;
}
-
+
String gopherUrl = "gopher://" + hostname + sel + ":" + port;
return "<h1>News</h1>\n"//
}
@Override
- public String export(Story story) {
+ public String exportHeader(Story story) {
StringBuilder builder = new StringBuilder();
builder.append("<div class='story-header'>\n");
}
@Override
- public String export(Story story, List<Comment> comments) {
+ public String export(Story story) {
StringBuilder builder = new StringBuilder();
builder.append("<div class='story'>\n");
appendHtml(builder, story, false);
builder.append("<hr/>\n");
- if (comments != null) {
- for (Comment comment : comments) {
+ if (story.getComments() != null) {
+ for (Comment comment : story.getComments()) {
appendHtml(builder, comment, " ");
}
}
}
builder.append(" <div class='content'>\n");
- builder.append(" " + story.getContent() + "\n");
+ if (resume) {
+ builder.append(" " + story.getContent() + "\n");
+ } else {
+ builder.append(" " + story.getFullContent() + "\n");
+ }
builder.append(" </div>\n");
return builder;
package be.nikiroo.gofetch.output;
-import java.util.List;
-
-import be.nikiroo.gofetch.data.Comment;
import be.nikiroo.gofetch.data.Story;
import be.nikiroo.gofetch.support.BasicSupport.Type;
abstract public String getIndexFooter();
/**
- * Export a story (in resume mode).
+ * Export the header of a story (a <i>resume</i> mode).
*
* @param story
* the story
*
* @return the resume
*/
- abstract public String export(Story story);
+ abstract public String exportHeader(Story story);
/**
* Export a full story with comments.
*
* @param story
* the story
- * @param comments
- * the comments
*
* @return the story
*/
- abstract public String export(Story story, List<Comment> comments);
+ abstract public String export(Story story);
}
import java.util.List;
import java.util.zip.GZIPInputStream;
-import be.nikiroo.gofetch.data.Comment;
import be.nikiroo.gofetch.data.Story;
public abstract class BasicSupport {
abstract public List<Story> list() throws IOException;
- abstract public List<Comment> getComments(Story story) throws IOException;
+ /**
+ * Fetch the full article content as well as all the comments associated to
+ * this {@link Story}, if any (can be empty, but not NULL).
+ *
+ * @param story
+ * the story to fetch the comments of
+ *
+ * @throws IOException
+ * in case of I/O error
+ */
+ abstract public void fetch(Story story) throws IOException;
abstract public String getDescription();
@Override
public List<Story> list() throws IOException {
// TODO: comments + do not get comment for [$] stories
- // + update body on getComment (global change, also LinuxToday)
-
+
List<Story> list = new ArrayList<Story>();
URL url = new URL("https://lwn.net/");
if (listings.size() == 0) {
continue;
}
-
+
Element listing = listings.get(0);
if (listing.children().size() < 2) {
continue;
}
-
String title = titles.get(0).text();
String details = listing.children().get(0).text();
String body = "";
// All but the first and two last children
- for (int i = 1 ; i < listing.children().size() - 2; i++) {
+ for (int i = 1; i < listing.children().size() - 2; i++) {
Element e = listing.children().get(i);
body = body.trim() + " " + e.text().trim();
}
body = body.trim();
-
+
String author = "";
int pos = details.indexOf(" by ");
if (pos >= 0) {
author = details.substring(pos + " by ".length()).trim();
}
-
+
String date = "";
pos = details.indexOf(" Posted ");
if (pos >= 0) {
date = details.substring(pos + " Posted ".length()).trim();
}
-
String id = "";
String intUrl = "";
intUrl = idElem.absUrl("href");
pos = intUrl.indexOf("#Comments");
if (pos >= 0) {
- intUrl = intUrl.substring(0, pos -1);
+ intUrl = intUrl.substring(0, pos - 1);
}
id = intUrl.replaceAll("[^0-9]", "");
}
- list.add(new Story(getType(), id, title, details, intUrl, extUrl, body));
+ list.add(new Story(getType(), id, title, details, intUrl, extUrl,
+ body));
}
return list;
}
@Override
- public List<Comment> getComments(Story story) throws IOException {
- List<Comment> comments = new ArrayList<Comment>();
-
+ public void fetch(Story story) throws IOException {
/*
- URL url = new URL(story.getUrlInternal());
- InputStream in = open(url);
- Document doc = DataUtil.load(in, "UTF-8", url.toString());
- Elements listing = doc.getElementsByTag("main");
- if (listing.size() > 0) {
- comments.addAll(getComments(listing.get(0)));
- }
- */
-
- return comments;
+ * URL url = new URL(story.getUrlInternal()); InputStream in =
+ * open(url); Document doc = DataUtil.load(in, "UTF-8", url.toString());
+ * Elements listing = doc.getElementsByTag("main"); if (listing.size() >
+ * 0) { comments.addAll(getComments(listing.get(0))); }
+ */
}
private List<Comment> getComments(Element listing) {
}
@Override
- public List<Comment> getComments(Story story) throws IOException {
+ public void fetch(Story story) throws IOException {
List<Comment> comments = new ArrayList<Comment>();
URL url = new URL(story.getUrlInternal());
comments.addAll(getComments(listing.get(0)));
}
- return comments;
+ story.setComments(comments);
}
private List<Comment> getComments(Element listing) {
}
@Override
- public List<Comment> getComments(Story story) throws IOException {
+ public void fetch(Story story) throws IOException {
List<Comment> comments = new ArrayList<Comment>();
URL url = new URL(story.getUrlInternal());
comments.addAll(getComments(listing));
}
- return comments;
+ story.setComments(comments);
}
private List<Comment> getComments(Element listing) {
git://git.nikiroo.be / gofetch.git / commitdiff