--- /dev/null
+# Required parameters (the commented out ones are supposed to change per project):
+
+#MAIN = path to main java source to compile
+#MORE = path to supplementary needed resources not linked from MAIN
+#NAME = name of project (used for jar output file)
+#PREFIX = usually /usr/local (where to install the program)
+#TEST = path to main test source to compile
+#JAR_FLAGS += a list of things to pack, each usually prefixed with "-C bin/"
+#SJAR_FLAGS += a list of things to pack, each usually prefixed with "-C src/", for *-sources.jar files
+#TEST_PARAMS = any parameter to pass to the test runnable when "test-run"
+
+JAVAC = javac
+JAVAC_FLAGS += -encoding UTF-8 -d ./bin/ -cp ./src/
+JAVA = java
+JAVA_FLAGS += -cp ./bin/
+JAR = jar
+RJAR = java
+RJAR_FLAGS += -jar
+
+# Usual options:
+# make : to build the jar file
+# make libs : to update the libraries into src/
+# make build : to update the binaries (not the jar)
+# make test : to update the test binaries
+# make build jar : to update the binaries and jar file
+# make clean : to clean the directory of intermediate files
+# make mrpropre : to clean the directory of all outputs
+# make run : to run the program from the binaries
+# make run-test : to run the test program from the binaries
+# make jrun : to run the program from the jar file
+# make install : to install the application into $PREFIX
+
+# Note: build is actually slower than rebuild in most cases except when
+# small changes only are detected ; so we use rebuild by default
+
+all: build jar
+
+.PHONY: all clean mrproper mrpropre build run jrun jar resources test-resources install libs love
+
+bin:
+ @mkdir -p bin
+
+jar: $(NAME).jar
+
+build: resources
+ @echo Compiling program...
+ @echo " src/$(MAIN)"
+ @$(JAVAC) $(JAVAC_FLAGS) "src/$(MAIN).java"
+ @[ "$(MORE)" = "" ] || for sup in $(MORE); do \
+ echo " src/$$sup" ;\
+ $(JAVAC) $(JAVAC_FLAGS) "src/$$sup.java" ; \
+ done
+
+test: test-resources
+ @[ -e bin/$(MAIN).class ] || echo You need to build the sources
+ @[ -e bin/$(MAIN).class ]
+ @echo Compiling test program...
+ @[ "$(TEST)" != "" ] || echo No test sources defined.
+ @[ "$(TEST)" = "" ] || for sup in $(TEST); do \
+ echo " src/$$sup" ;\
+ $(JAVAC) $(JAVAC_FLAGS) "src/$$sup.java" ; \
+ done
+
+clean:
+ rm -rf bin/
+ @echo Removing sources taken from libs...
+ @for lib in libs/*-sources.jar libs/*-sources.patch.jar; do \
+ if [ "$$lib" != 'libs/*-sources.jar' -a "$$lib" != 'libs/*-sources.patch.jar' ]; then \
+ basename "$$lib"; \
+ jar tf "$$lib" | while read -r ln; do \
+ [ -f "src/$$ln" ] && rm "src/$$ln"; \
+ done; \
+ jar tf "$$lib" | tac | while read -r ln; do \
+ [ -d "src/$$ln" ] && rmdir "src/$$ln" 2>/dev/null || true; \
+ done; \
+ fi \
+ done
+
+mrproper: mrpropre
+
+mrpropre: clean
+ rm -f $(NAME).jar
+ rm -f $(NAME)-sources.jar
+ [ ! -e VERSION ] || rm -f "$(NAME)-`cat VERSION`.jar"
+ [ ! -e VERSION ] || rm -f "$(NAME)-`cat VERSION`-sources.jar"
+
+love:
+ @echo " ...not war."
+
+resources: libs
+ @echo Copying resources into bin/...
+ @cd src && find . | grep -v '\.java$$' | grep -v '/test/' | while read -r ln; do \
+ if [ -f "$$ln" ]; then \
+ dir="`dirname "$$ln"`"; \
+ mkdir -p "../bin/$$dir" ; \
+ cp "$$ln" "../bin/$$ln" ; \
+ fi ; \
+ done
+ @cp VERSION bin/
+
+test-resources: resources
+ @echo Copying test resources into bin/...
+ @cd src && find . | grep -v '\.java$$' | grep '/test/' | while read -r ln; do \
+ if [ -f "$$ln" ]; then \
+ dir="`dirname "$$ln"`"; \
+ mkdir -p "../bin/$$dir" ; \
+ cp "$$ln" "../bin/$$ln" ; \
+ fi ; \
+ done
+
+libs: bin
+ @[ -e bin/libs -o ! -d libs ] || echo Extracting sources from libs...
+ @[ -e bin/libs -o ! -d libs ] || (cd src && for lib in ../libs/*-sources.jar ../libs/*-sources.patch.jar; do \
+ if [ "$$lib" != '../libs/*-sources.jar' -a "$$lib" != '../libs/*-sources.patch.jar' ]; then \
+ basename "$$lib"; \
+ jar xf "$$lib"; \
+ fi \
+ done )
+ @[ ! -d libs ] || touch bin/libs
+
+$(NAME).jar: resources
+ @[ -e bin/$(MAIN).class ] || echo You need to build the sources
+ @[ -e bin/$(MAIN).class ]
+ @echo Making JAR file...
+ @echo > bin/manifest
+ @[ "$(SJAR_FLAGS)" = "" ] || echo Creating $(NAME)-sources.jar...
+ @[ "$(SJAR_FLAGS)" = "" ] || $(JAR) cfm $(NAME)-sources.jar bin/manifest $(SJAR_FLAGS)
+ @[ "$(SJAR_FLAGS)" = "" ] || [ ! -e VERSION ] || echo Copying to "$(NAME)-`cat VERSION`-sources.jar"...
+ @[ "$(SJAR_FLAGS)" = "" ] || [ ! -e VERSION ] || cp $(NAME)-sources.jar "$(NAME)-`cat VERSION`-sources.jar"
+ @echo "Main-Class: `echo "$(MAIN)" | sed 's:/:.:g'`" > bin/manifest
+ @echo >> bin/manifest
+ $(JAR) cfm $(NAME).jar bin/manifest $(JAR_FLAGS)
+ @[ ! -e VERSION ] || echo Copying to "$(NAME)-`cat VERSION`.jar"...
+ @[ ! -e VERSION ] || cp $(NAME).jar "$(NAME)-`cat VERSION`.jar"
+
+run:
+ @[ -e bin/$(MAIN).class ] || echo You need to build the sources
+ @[ -e bin/$(MAIN).class ]
+ @echo Running "$(NAME)"...
+ $(JAVA) $(JAVA_FLAGS) $(MAIN)
+
+jrun:
+ @[ -e $(NAME).jar ] || echo You need to build the jar
+ @[ -e $(NAME).jar ]
+ @echo Running "$(NAME).jar"...
+ $(RJAR) $(RJAR_FLAGS) $(NAME).jar
+
+run-test:
+ @[ "$(TEST)" = "" -o -e "bin/$(TEST).class" ] || echo You need to build the test sources
+ @[ "$(TEST)" = "" -o -e "bin/$(TEST).class" ]
+ @echo Running tests for "$(NAME)"...
+ @[ "$(TEST)" != "" ] || echo No test sources defined.
+ [ "$(TEST)" = "" ] || ( clear ; $(JAVA) $(JAVA_FLAGS) $(TEST) $(TEST_PARAMS) )
+
+install:
+ @[ -e $(NAME).jar ] || echo You need to build the jar
+ @[ -e $(NAME).jar ]
+ mkdir -p "$(PREFIX)/lib" "$(PREFIX)/bin"
+ cp $(NAME).jar "$(PREFIX)/lib/"
+ echo "#!/bin/sh" > "$(PREFIX)/bin/$(NAME)"
+ echo "$(RJAR) $(RJAR_FLAGS) \"$(PREFIX)/lib/$(NAME).jar\" \"\$$@\"" >> "$(PREFIX)/bin/$(NAME)"
+ chmod a+rx "$(PREFIX)/bin/$(NAME)"
+
--- /dev/null
+# GoFetch
+
+GoFetch is a simple web scrapper that outputs gopher-ready files.
+You point it to your gopher directory, you launch it, and you have a
+gopher view of the supported news sites.
+
+## Supported websites
+
+- Slashdot: News for nerds, stuff that matters!
+
+## Supported platforms
+
+Any platform with at lest Java 1.6 on it should be ok.
+
+## Usage
+
+```java -jar gofetch.jar [dir] [selector] [type] [max] [hostname] [port]```
+
+- dir: the target directory where to store the files
+- selector: the gopher selector to prepend (also a sub-directory in [dir])
+- max: the maximum number of stories to show on the main page
+- hostname: the gopher hostname
+- port: the gopher port
+
+## Compilation
+
+```./configure.sh && make```
+
+You can also import the java sources into, say, [Eclipse](https://eclipse.org/), and create a runnable JAR file from there.
+
+### Dependant libraries (included)
+
+- libs/nikiroo-utils-sources.jar: some shared utility functions I also use elsewhere
+- [libs/jsoup-sources.jar](https://jsoup.org/): a nice library to parse HTML
+
+Nothing else but Java 1.6+.
+
+Note that calling ```make libs``` will export the libraries into the src/ directory.
+
--- /dev/null
+# Gofetch
+
+## Version 0.0.1
+
+- First version
+- Slashdot supported
+
--- /dev/null
+#!/bin/sh
+
+# default:
+PREFIX=/usr/local
+PROGS="java javac jar make sed"
+
+valid=true
+while [ "$*" != "" ]; do
+ key=`echo "$1" | cut -f1 -d=`
+ val=`echo "$1" | cut -f2 -d=`
+ case "$key" in
+ --)
+ ;;
+ --help) # This help message
+ echo The following arguments can be used:
+ cat "$0" | grep '^\s*--' | grep '#' | while read ln; do
+ cmd=`echo "$ln" | cut -f1 -d')'`
+ msg=`echo "$ln" | cut -f2 -d'#'`
+ echo " $cmd$msg"
+ done
+ ;;
+ --prefix) #=PATH Change the prefix to the given path
+ PREFIX="$val"
+ ;;
+ *)
+ echo "Unsupported parameter: '$1'" >&2
+ echo >&2
+ sh "$0" --help >&2
+ valid=false
+ ;;
+ esac
+ shift
+done
+
+[ $valid = false ] && exit 1
+
+MESS="A required program cannot be found:"
+for prog in $PROGS; do
+ out="`whereis -b "$prog" 2>/dev/null`"
+ if [ "$out" = "$prog:" ]; then
+ echo "$MESS $prog" >&2
+ valid=false
+ fi
+done
+
+[ $valid = false ] && exit 2
+
+if [ "`whereis tput`" = "tput:" ]; then
+ ok='"[ ok ]"';
+ ko='"[ !! ]"';
+ cols=80;
+else
+ ok='"`tput bold`[`tput setf 2` OK `tput init``tput bold`]`tput init`"';
+ ko='"`tput bold`[`tput setf 4` !! `tput init``tput bold`]`tput init`"';
+ cols='"`tput cols`"';
+fi;
+
+echo "MAIN = be/nikiroo/gofetch/Main" > Makefile
+echo "MORE = " >> Makefile
+echo "TEST = " >> Makefile
+echo "TEST_PARAMS = $cols $ok $ko" >> Makefile
+echo "NAME = gofetch" >> Makefile
+echo "PREFIX = $PREFIX" >> Makefile
+echo "JAR_FLAGS += -C bin/ org -C bin/ be -C bin/ VERSION" >> Makefile
+#echo "SJAR_FLAGS += -C src/ org -C src/ jexer -C src/ be -C ./ LICENSE -C ./ README.md -C ./ VERSION" >> Makefile
+
+cat Makefile.base >> Makefile
+
--- /dev/null
+package be.nikiroo.gofetch;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import be.nikiroo.gofetch.data.Comment;
+import be.nikiroo.gofetch.data.Story;
+import be.nikiroo.gofetch.output.Gopher;
+import be.nikiroo.gofetch.output.Html;
+import be.nikiroo.gofetch.output.Output;
+import be.nikiroo.gofetch.support.BasicSupport;
+import be.nikiroo.gofetch.support.BasicSupport.Type;
+import be.nikiroo.utils.IOUtils;
+
+/**
+ * The class that will manage the fetch operations.
+ * <p>
+ * It will scrap the required websites and process them to disk.
+ *
+ * @author niki
+ */
+public class Fetcher {
+ private File dir;
+ private String preselector;
+ private int maxStories;
+ private String hostname;
+ private int port;
+ private Type type;
+
+ /**
+ * Prepare a new {@link Fetcher}.
+ *
+ * @param dir
+ * the target directory where to save the files (won't have
+ * impact on the files' content)
+ * @param preselector
+ * the sub directory and (pre-)selector to use for the resources
+ * (<b>will</b> have an impact on the files' content)
+ * @param type
+ * the type of news to get (or the special keyword ALL to get all
+ * of the supported sources)
+ * @param maxStories
+ * the maximum number of stories to show on the resume page
+ * @param hostname
+ * the gopher host to use (<b>will</b> have an impact on the
+ * files' content)
+ * @param port
+ * the gopher port to use (<b>will</b> have an impact on the
+ * files' content)
+ */
+ public Fetcher(File dir, String preselector, Type type, int maxStories,
+ String hostname, int port) {
+ this.dir = dir;
+ this.preselector = preselector;
+ this.type = type;
+ this.maxStories = maxStories;
+ this.hostname = hostname;
+ this.port = port;
+ }
+
+ /**
+ * Start the fetching operation.
+ * <p>
+ * This method will handle the main pages itself, and will call
+ * {@link Fetcher#list(BasicSupport)} for the stories.
+ *
+ * @throws IOException
+ * in case of I/O error
+ */
+ public void start() throws IOException {
+ File cache = new File(dir, preselector);
+ cache.mkdirs();
+ File cacheHtml = new File(cache, "index.html");
+ cache = new File(cache, ".cache");
+
+ Output gopher = new Gopher(null, hostname, port);
+ Output html = new Html(null);
+
+ FileWriter writer = new FileWriter(cache);
+ try {
+ FileWriter writerHtml = new FileWriter(cacheHtml);
+ try {
+ writer.append(gopher.getIndexHeader());
+ writerHtml.append(html.getIndexHeader());
+
+ Type types[];
+ if (type == null) {
+ types = Type.values();
+ } else {
+ types = new Type[] { type };
+ }
+
+ BasicSupport.setPreselector(preselector);
+ for (Type type : types) {
+ BasicSupport support = BasicSupport.getSupport(type);
+ list(support);
+
+ writer.append("1" + support.getDescription()).append("\t")
+ .append("1" + support.getSelector()) //
+ .append("\t").append(hostname) //
+ .append("\t").append(Integer.toString(port)) //
+ .append("\r\n");
+ String ref = support.getSelector();
+ while (ref.startsWith("/")) {
+ ref = ref.substring(1);
+ }
+ writerHtml.append("<div class='site'><a href='../" + ref
+ + "'>" + support.getDescription() + "</a></div>");
+ }
+
+ writer.append(gopher.getIndexFooter());
+ writerHtml.append(html.getIndexFooter());
+ } finally {
+ writerHtml.close();
+ }
+ } finally {
+ writer.close();
+ }
+ }
+
+ /**
+ * Process the stories for the given {@link BasicSupport} to disk.
+ *
+ * @param support
+ * the {@link BasicSupport} to download from
+ *
+ * @throws IOException
+ * in case of I/O error
+ **/
+ private void list(BasicSupport support) throws IOException {
+ Output gopher = new Gopher(support.getType(), hostname, port);
+ Output html = new Html(support.getType());
+
+ new File(dir, support.getSelector()).mkdirs();
+
+ System.err
+ .print("Listing recent news for " + support.getType() + "...");
+ List<Story> stories = support.list();
+ System.err.println(" " + stories.size() + " stories found!");
+ int i = 1;
+ for (Story story : stories) {
+ IOUtils.writeSmallFile(dir, story.getSelector() + ".header",
+ gopher.export(story));
+ IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html",
+ html.export(story));
+
+ System.err.println(String.format("%02d/%02d", i, stories.size())
+ + " Fetching comments for story " + story.getId() + "...");
+ List<Comment> comments = support.getComments(story);
+
+ IOUtils.writeSmallFile(dir, story.getSelector(),
+ gopher.export(story, comments));
+ IOUtils.writeSmallFile(dir, story.getSelector() + ".html",
+ html.export(story, comments));
+
+ i++;
+ }
+
+ File varDir = new File(dir, support.getSelector());
+ String[] headers = varDir.list(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".header");
+ }
+ });
+
+ File cache = new File(varDir, ".cache");
+ File cacheHtml = new File(varDir, "index.html");
+ FileWriter writer = new FileWriter(cache);
+ try {
+ FileWriter writerHtml = new FileWriter(cacheHtml);
+ try {
+ if (headers.length > 0) {
+ Arrays.sort(headers);
+ int from = headers.length - 1;
+ int to = headers.length - maxStories;
+ if (to < 0) {
+ to = 0;
+ }
+ for (i = from; i >= to; i--) {
+ writer.append(IOUtils.readSmallFile(new File(varDir,
+ headers[i])));
+
+ writerHtml.append(IOUtils.readSmallFile(new File(
+ varDir, headers[i] + ".html")));
+ }
+ }
+ } finally {
+ writerHtml.close();
+ }
+ } finally {
+ writer.close();
+ }
+ }
+}
--- /dev/null
+package be.nikiroo.gofetch;
+
+import java.io.File;
+import java.io.IOException;
+
+import be.nikiroo.gofetch.support.BasicSupport.Type;
+
+/**
+ * This class is tha main entry point of the program. It will parse the
+ * arguments, checks them (and warn-and-exit if they are invalid) then call
+ * {@link Fetcher#start()}.
+ *
+ * @author niki
+ */
+public class Main {
+ /**
+ * Main entry point.
+ *
+ * @param args
+ * save-to-dir selector-subdir type max hostname port
+ *
+ * @throws IOException
+ * in case of I/O error
+ */
+ public static void main(String[] args) throws IOException {
+ if (args.length < 6) {
+ System.err
+ .println("Syntax error: gofecth [target dir] [selector] [type or 'ALL'] [max stories] [hostname] [port]");
+ System.exit(1);
+ }
+
+ String dirStr = args[0];
+ String preselectorStr = args[1];
+ String typeStr = args[2];
+ String maxStoriesStr = args[3];
+ String hostnameStr = args[4];
+ String portStr = args[5];
+
+ // Dir
+ File dir = new File(dirStr);
+ dir.mkdirs();
+
+ if (!dir.exists()) {
+ System.err.println("Cannot open/create the root directory: "
+ + dirStr);
+ System.exit(1);
+ }
+
+ if (dir.isFile()) {
+ System.err
+ .println("Root directory exists and is a file: " + dirStr);
+ System.exit(1);
+ }
+
+ // Selector base (empty is ok, DO NOT end with /)
+ String preselector = "";
+ if (preselectorStr != null && !preselectorStr.startsWith("/")) {
+ preselector = "/" + preselectorStr;
+ }
+ while (preselector.endsWith("/")) {
+ preselector = preselector.substring(0, preselector.length() - 1);
+ }
+
+ // Type to download
+ Type type = null;
+ if (!"ALL".equals(typeStr)) {
+ try {
+ Type.valueOf(typeStr.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ System.err.println("Invalid type: " + typeStr);
+ System.exit(1);
+ }
+ }
+
+ // Max number of stories to display in the cache
+ int maxStories = 0;
+ try {
+ maxStories = Integer.parseInt(maxStoriesStr);
+ } catch (NumberFormatException e) {
+ System.err
+ .println("The maximum number of stories cannot be parsed: "
+ + maxStoriesStr);
+ System.exit(1);
+ }
+
+ //
+ String hostname = hostnameStr;
+
+ //
+ int port = 0;
+ try {
+ port = Integer.parseInt(portStr);
+ } catch (NumberFormatException e) {
+ System.err.println("The port cannot be parsed: " + portStr);
+ System.exit(1);
+ }
+
+ if (port < 0 || port > 65535) {
+ System.err.println("Invalid port number: " + portStr);
+ System.exit(1);
+ }
+
+ new Fetcher(dir, preselector, type, maxStories, hostname, port).start();
+ }
+}
\ No newline at end of file
--- /dev/null
+/*
+ * This file was taken from:
+ * Jexer - Java Text User Interface
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (C) 2017 Kevin Lamonte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * @author Kevin Lamonte [kevin.lamonte@gmail.com]
+ * @version 1
+ */
+package be.nikiroo.gofetch;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * StringJustifier contains methods to convert one or more long lines of strings
+ * into justified text paragraphs.
+ */
+public class StringJustifier {
+
+ /**
+ * Left-justify a string into a list of lines.
+ *
+ * @param str
+ * the string
+ * @param n
+ * the maximum number of characters in a line
+ * @return the list of lines
+ */
+ public static List<String> left(final String str, final int n) {
+ List<String> result = new LinkedList<String>();
+
+ /*
+ * General procedure:
+ *
+ * 1. Split on '\n' into paragraphs.
+ *
+ * 2. Scan each line, noting the position of the last
+ * beginning-of-a-word.
+ *
+ * 3. Chop at the last #2 if the next beginning-of-a-word exceeds n.
+ *
+ * 4. Return the lines.
+ */
+
+ String[] rawLines = str.split("\n");
+ for (int i = 0; i < rawLines.length; i++) {
+ StringBuilder line = new StringBuilder();
+ StringBuilder word = new StringBuilder();
+ boolean inWord = false;
+ for (int j = 0; j < rawLines[i].length(); j++) {
+ char ch = rawLines[i].charAt(j);
+ if ((ch == ' ') || (ch == '\t')) {
+ if (inWord == true) {
+ // We have just transitioned from a word to
+ // whitespace. See if we have enough space to add
+ // the word to the line.
+ if (word.length() + line.length() > n) {
+ // This word will exceed the line length. Wrap
+ // at it instead.
+ result.add(line.toString());
+ line = new StringBuilder();
+ }
+ if ((word.toString().startsWith(" "))
+ && (line.length() == 0)) {
+ line.append(word.substring(1));
+ } else {
+ line.append(word);
+ }
+ word = new StringBuilder();
+ word.append(ch);
+ inWord = false;
+ } else {
+ // We are in the whitespace before another word. Do
+ // nothing.
+ }
+ } else {
+ if (inWord == true) {
+ // We are appending to a word.
+ word.append(ch);
+ } else {
+ // We have transitioned from whitespace to a word.
+ word.append(ch);
+ inWord = true;
+ }
+ }
+ } // for (int j = 0; j < rawLines[i].length(); j++)
+
+ if (word.length() + line.length() > n) {
+ // This word will exceed the line length. Wrap at it
+ // instead.
+ result.add(line.toString());
+ line = new StringBuilder();
+ }
+ if ((word.toString().startsWith(" ")) && (line.length() == 0)) {
+ line.append(word.substring(1));
+ } else {
+ line.append(word);
+ }
+ result.add(line.toString());
+ } // for (int i = 0; i < rawLines.length; i++) {
+
+ return result;
+ }
+
+ /**
+ * Right-justify a string into a list of lines.
+ *
+ * @param str
+ * the string
+ * @param n
+ * the maximum number of characters in a line
+ * @return the list of lines
+ */
+ public static List<String> right(final String str, final int n) {
+ List<String> result = new LinkedList<String>();
+
+ /*
+ * Same as left(), but preceed each line with spaces to make it n chars
+ * long.
+ */
+ List<String> lines = left(str, n);
+ for (String line : lines) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < n - line.length(); i++) {
+ sb.append(' ');
+ }
+ sb.append(line);
+ result.add(sb.toString());
+ }
+
+ return result;
+ }
+
+ /**
+ * Center a string into a list of lines.
+ *
+ * @param str
+ * the string
+ * @param n
+ * the maximum number of characters in a line
+ * @return the list of lines
+ */
+ public static List<String> center(final String str, final int n) {
+ List<String> result = new LinkedList<String>();
+
+ /*
+ * Same as left(), but preceed/succeed each line with spaces to make it
+ * n chars long.
+ */
+ List<String> lines = left(str, n);
+ for (String line : lines) {
+ StringBuilder sb = new StringBuilder();
+ int l = (n - line.length()) / 2;
+ int r = n - line.length() - l;
+ for (int i = 0; i < l; i++) {
+ sb.append(' ');
+ }
+ sb.append(line);
+ for (int i = 0; i < r; i++) {
+ sb.append(' ');
+ }
+ result.add(sb.toString());
+ }
+
+ return result;
+ }
+
+ /**
+ * Fully-justify a string into a list of lines.
+ *
+ * @param str
+ * the string
+ * @param n
+ * the maximum number of characters in a line
+ * @return the list of lines
+ */
+ public static List<String> full(final String str, final int n) {
+ List<String> result = new LinkedList<String>();
+
+ /*
+ * Same as left(), but insert spaces between words to make each line n
+ * chars long. The "algorithm" here is pretty dumb: it performs a split
+ * on space and then re-inserts multiples of n between words.
+ */
+ List<String> lines = left(str, n);
+ for (int lineI = 0; lineI < lines.size() - 1; lineI++) {
+ String line = lines.get(lineI);
+ String[] words = line.split(" ");
+ if (words.length > 1) {
+ int charCount = 0;
+ for (int i = 0; i < words.length; i++) {
+ charCount += words[i].length();
+ }
+ int spaceCount = n - charCount;
+ int q = spaceCount / (words.length - 1);
+ int r = spaceCount % (words.length - 1);
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < words.length - 1; i++) {
+ sb.append(words[i]);
+ for (int j = 0; j < q; j++) {
+ sb.append(' ');
+ }
+ if (r > 0) {
+ sb.append(' ');
+ r--;
+ }
+ }
+ for (int j = 0; j < r; j++) {
+ sb.append(' ');
+ }
+ sb.append(words[words.length - 1]);
+ result.add(sb.toString());
+ } else {
+ result.add(line);
+ }
+ }
+ if (lines.size() > 0) {
+ result.add(lines.get(lines.size() - 1));
+ }
+
+ return result;
+ }
+}
--- /dev/null
+package be.nikiroo.gofetch.data;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class Comment implements Iterable<Comment> {
+ private String id;
+ private String author;
+ private String title;
+ private String date;
+ private String content;
+ private List<Comment> children;
+
+ public Comment(String id, String author, String title, String date,
+ String content) {
+ this.id = id;
+ this.author = author;
+ this.title = title;
+ this.date = date;
+ this.content = content;
+ this.children = new ArrayList<Comment>();
+ }
+
+ public void add(Comment comment) {
+ children.add(comment);
+ }
+
+ public void addAll(List<Comment> comments) {
+ children.addAll(comments);
+ }
+
+ /**
+ * @return the id
+ */
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * @return the author
+ */
+ public String getAuthor() {
+ return author;
+ }
+
+ /**
+ * @return the title
+ */
+ public String getTitle() {
+ return title;
+ }
+
+ /**
+ * @return the date
+ */
+ public String getDate() {
+ return date;
+ }
+
+ /**
+ * @return the content
+ */
+ public String getContent() {
+ return content;
+ }
+
+ public boolean isEmpty() {
+ return children.isEmpty()
+ && ("" + author + title + content).trim().isEmpty();
+ }
+
+ @Override
+ public Iterator<Comment> iterator() {
+ return children.iterator();
+ }
+}
--- /dev/null
+package be.nikiroo.gofetch.data;
+
+import java.net.URL;
+
+import be.nikiroo.gofetch.support.BasicSupport;
+import be.nikiroo.gofetch.support.BasicSupport.Type;
+
+/**
+ * A news story.
+ *
+ * @author niki
+ */
+public class Story {
+ private Type type;
+ private String id;
+ private String title;
+ private String details;
+ private String urlInternal;
+ private String urlExternal;
+ private String content;
+
+ /**
+ * Create a news story.
+ *
+ * @param type
+ * the source {@link Type}
+ * @param id
+ * the news ID
+ * @param title
+ * the news title
+ * @param details
+ * some details to add to the title
+ * @param urlInternal
+ * the {@link URL} to get this news on the associated news site
+ * @param urlExternal
+ * an external {@link URL} that serve as the news' source, if any
+ * @param content
+ * the story content
+ */
+ public Story(Type type, String id, String title, String details,
+ String urlInternal, String urlExternal, String content) {
+ this.type = type;
+ this.id = id;
+ this.title = title;
+ this.details = details;
+ this.urlInternal = urlInternal;
+ this.urlExternal = urlExternal;
+ this.content = content;
+ }
+
+ public String getSelector() {
+ return BasicSupport.getSelector(type) + id;
+ }
+
+ /**
+ * @return the id
+ */
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * @return the title
+ */
+ public String getTitle() {
+ return title;
+ }
+
+ /**
+ * @return the details
+ */
+ public String getDetails() {
+ return details;
+ }
+
+ /**
+ * @return the url
+ */
+ public String getUrlInternal() {
+ return urlInternal;
+ }
+
+ /**
+ * @return the urlExternal
+ */
+ public String getUrlExternal() {
+ return urlExternal;
+ }
+
+ /**
+ * @return the body
+ */
+ public String getContent() {
+ return content;
+ }
+}
\ No newline at end of file
--- /dev/null
+package be.nikiroo.gofetch.output;
+
+import java.util.List;
+
+import be.nikiroo.gofetch.StringJustifier;
+import be.nikiroo.gofetch.data.Comment;
+import be.nikiroo.gofetch.data.Story;
+import be.nikiroo.gofetch.support.BasicSupport.Type;
+
+public class Gopher extends Output {
+ static private final int LINE_SIZE = 70;
+
+ private String hostname;
+ private int port;
+
+ public Gopher(Type type, String hostname, int port) {
+ super(type);
+
+ this.hostname = hostname;
+ this.port = port;
+ }
+
+ @Override
+ public String getIndexHeader() {
+ return "iHello world!\r\niThis is my news site.\r\ni\r\n";
+ }
+
+ @Override
+ public String getIndexFooter() {
+ return "";
+ }
+
+ @Override
+ public String export(Story story) {
+ return append(new StringBuilder(), story, false).append("i\r\ni\r\n")
+ .toString();
+ }
+
+ @Override
+ public String export(Story story, List<Comment> comments) {
+ StringBuilder builder = new StringBuilder();
+ append(builder, story, true);
+
+ builder.append("i\r\n");
+
+ for (Comment comment : comments) {
+ append(builder, comment, "");
+ }
+
+ builder.append("i\r\n");
+
+ return builder.toString();
+ }
+
+ private StringBuilder append(StringBuilder builder, Comment comment,
+ String space) {
+
+ if (space.length() > LINE_SIZE - 20) {
+ space = space.substring(0, LINE_SIZE - 20);
+ }
+
+ appendLeft(builder, comment.getTitle(), ">> ", " ", space);
+ appendLeft(builder, "(" + comment.getAuthor() + ")", " ", " ",
+ space);
+
+ builder.append("i\r\n");
+
+ appendLeft(builder, comment.getContent(), " ", " ", space);
+
+ builder.append("i\r\n");
+ for (Comment subComment : comment) {
+ append(builder, subComment, space + " ");
+ builder.append("i\r\n");
+ }
+
+ return builder;
+ }
+
+ private StringBuilder append(StringBuilder builder, Story story,
+ boolean links) {
+ if (links) {
+ appendCenter(builder, story.getTitle(), true);
+ builder.append("i\r\n");
+ appendLeft(builder, story.getDetails(), "", "", " ");
+ builder.append("i\r\n");
+ builder.append("i o News link: ").append(story.getUrlInternal());
+ builder.append("i\r\n");
+ builder.append("i o Source link: ").append(story.getUrlExternal());
+ builder.append("i\r\n");
+ } else {
+ builder.append('1').append(story.getTitle()) //
+ .append('\t').append("0").append(story.getSelector()) //
+ .append('\t').append(hostname) //
+ .append('\t').append(port) //
+ .append("\r\n");
+ appendLeft(builder, story.getDetails(), "", "", " ");
+ }
+
+ builder.append("i\r\n");
+
+ appendLeft(builder, story.getContent(), "", "", " ");
+
+ builder.append("i\r\n");
+
+ return builder;
+ }
+
+ // note: adds "i"
+ private static void appendCenter(StringBuilder builder, String text,
+ boolean allCaps) {
+ if (allCaps) {
+ text = text.toUpperCase();
+ }
+
+ for (String line : StringJustifier.center(text, LINE_SIZE)) {
+ builder.append("i").append(line).append("\r\n");
+ }
+ }
+
+ // note: adds "i"
+ private static void appendLeft(StringBuilder builder, String text,
+ String prependFirst, String prependOthers, String space) {
+ String prepend = prependFirst;
+ for (String line : StringJustifier.left(text,
+ LINE_SIZE - space.length())) {
+ builder.append("i").append(space).append(prepend).append(line)
+ .append("\r\n");
+ prepend = prependOthers;
+ }
+ }
+}
--- /dev/null
+package be.nikiroo.gofetch.output;
+
+import java.util.List;
+
+import be.nikiroo.gofetch.data.Comment;
+import be.nikiroo.gofetch.data.Story;
+import be.nikiroo.gofetch.support.BasicSupport.Type;
+
+public class Html extends Output {
+ public Html(Type type) {
+ super(type);
+ }
+
+ @Override
+ public String getIndexHeader() {
+ return "<h1>Hello world!</h1><br/>TODO\n";
+ }
+
+ @Override
+ public String getIndexFooter() {
+ return "";
+ }
+
+ @Override
+ public String export(Story story) {
+ return appendHtml(new StringBuilder(), story, true).append("<hr/>\n")
+ .toString();
+ }
+
+ @Override
+ public String export(Story story, List<Comment> comments) {
+ StringBuilder builder = new StringBuilder();
+ appendHtml(builder, story, false);
+
+ // TODO: ext link and link
+
+ builder.append("<hr/>");
+ for (Comment comment : comments) {
+ appendHtml(builder, comment, " ");
+ }
+
+ return builder.toString();
+ }
+
+ private void appendHtml(StringBuilder builder, Comment comment, String space) {
+ builder.append(space).append(
+ "<div class='comment' style='display: block; margin-left: "
+ + (20 * space.length()) + "px'>");
+ builder.append(space).append(" <h2>").append(comment.getTitle())
+ .append("</h2>\n");
+ builder.append(space).append(" <div class='by'>")
+ .append(comment.getAuthor()).append("</div>\n");
+ builder.append(space).append(" <div class='comment_content'>")
+ .append(comment.getContent()).append("</div>\n");
+ for (Comment subComment : comment) {
+ appendHtml(builder, subComment, space + " ");
+ }
+ builder.append(space).append("</div>");
+ }
+
+ private StringBuilder appendHtml(StringBuilder builder, Story story,
+ boolean links) {
+ // TODO
+ builder.append("<div class='story'>");
+ if (links) {
+ builder.append(" <h1><a href='" + story.getId() + ".html'>"
+ + story.getTitle() + "</a></h1>");
+ } else {
+ builder.append(" <h1>" + story.getTitle() + "</h1>");
+ }
+ builder.append(" <div class='details'>(" + story.getDetails()
+ + ")</div>");
+ builder.append(" <br/>");
+ builder.append(" <div class='content'>");
+ builder.append(" " + story.getContent());
+ builder.append(" </div>");
+ builder.append("</div>");
+
+ return builder;
+ }
+}
--- /dev/null
+package be.nikiroo.gofetch.output;
+
+import java.util.List;
+
+import be.nikiroo.gofetch.data.Comment;
+import be.nikiroo.gofetch.data.Story;
+import be.nikiroo.gofetch.support.BasicSupport.Type;
+
+/**
+ * Base class for output operations.
+ *
+ * @author niki
+ */
+public abstract class Output {
+ /**
+ * The type of source, can be NULL for no-type.
+ */
+ protected Type type;
+
+ /**
+ * Create a new {@link Output} class for the given type (which can be NULL).
+ *
+ * @param type
+ * the type or NULL for no type
+ */
+ public Output(Type type) {
+ this.type = type;
+ }
+
+ /**
+ * Get the header to use in the index file.
+ *
+ * @return the header
+ */
+ abstract public String getIndexHeader();
+
+ /**
+ * Get the footer to use in the index file.
+ *
+ * @return the footer
+ */
+ abstract public String getIndexFooter();
+
+ /**
+ * Export a story (in resume mode).
+ *
+ * @param story
+ * the story
+ *
+ * @return the resume
+ */
+ abstract public String export(Story story);
+
+ /**
+ * Export a full story with comments.
+ *
+ * @param story
+ * the story
+ * @param comments
+ * the comments
+ *
+ * @return the story
+ */
+ abstract public String export(Story story, List<Comment> comments);
+}
--- /dev/null
+package be.nikiroo.gofetch.support;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.List;
+import java.util.zip.GZIPInputStream;
+
+import be.nikiroo.gofetch.data.Comment;
+import be.nikiroo.gofetch.data.Story;
+
+public abstract class BasicSupport {
+ public enum Type {
+ SLASHDOT,
+ }
+
+ static private String preselector;
+
+ private Type type;
+
+ abstract public List<Story> list() throws IOException;
+
+ abstract public List<Comment> getComments(Story story) throws IOException;
+
+ abstract public String getDescription();
+
+ public String getSelector() {
+ return getSelector(type);
+ }
+
+ public Type getType() {
+ return type;
+ }
+
+ protected void setType(Type type) {
+ this.type = type;
+ }
+
+ /**
+ * @param preselector
+ * the preselector to set
+ */
+ static public void setPreselector(String preselector) {
+ BasicSupport.preselector = preselector;
+ }
+
+ static public BasicSupport getSupport(Type type) {
+ BasicSupport support = null;
+
+ if (type != null) {
+ switch (type) {
+ case SLASHDOT:
+ support = new Slashdot();
+ break;
+ }
+
+ if (support != null) {
+ support.setType(type);
+ }
+ }
+
+ return support;
+ }
+
+ static public String getSelector(Type type) {
+ return preselector + "/" + type + "/";
+ }
+
+ // TODO: check Downloader.java?
+ static protected InputStream open(URL url) throws IOException {
+ URLConnection conn = url.openConnection();
+ conn.connect();
+ InputStream in = conn.getInputStream();
+ if ("gzip".equals(conn.getContentEncoding())) {
+ in = new GZIPInputStream(in);
+ }
+
+ return in;
+ }
+}
--- /dev/null
+package be.nikiroo.gofetch.support;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.jsoup.helper.DataUtil;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import be.nikiroo.gofetch.data.Comment;
+import be.nikiroo.gofetch.data.Story;
+
+public class Slashdot extends BasicSupport {
+ @Override
+ public String getDescription() {
+ return "Slashdot: News for nerds, stuff that matters!";
+ }
+
+ @Override
+ public List<Story> list() throws IOException {
+ List<Story> list = new ArrayList<Story>();
+
+ URL url = new URL("https://slashdot.org/");
+ InputStream in = open(url);
+ Document doc = DataUtil.load(in, "UTF-8", url.toString());
+ Elements stories = doc.getElementsByTag("header");
+ for (Element story : stories) {
+ Elements titles = story.getElementsByClass("story-title");
+ if (titles.size() == 0) {
+ continue;
+ }
+ Element title = titles.get(0);
+
+ String id = "" + title.attr("id");
+ if (id.startsWith("title-")) {
+ id = id.substring("title-".length());
+ }
+
+ Elements links = title.getElementsByTag("a");
+ String intUrl = null;
+ String extUrl = null;
+ if (links.size() > 0) {
+ intUrl = links.get(0).absUrl("href");
+ }
+ if (links.size() > 1) {
+ extUrl = links.get(1).absUrl("href");
+ }
+
+ String details = "";
+ Elements detailsElements = story.getElementsByClass("details");
+ if (detailsElements.size() > 0) {
+ details = detailsElements.get(0).text();
+ }
+
+ String body = "";
+ Element bodyElement = doc.getElementById("text-" + id);
+ if (bodyElement != null) {
+ body = bodyElement.text();
+ }
+
+ list.add(new Story(getType(), id, title.text(), details, intUrl,
+ extUrl, body));
+ }
+
+ return list;
+ }
+
+ @Override
+ public List<Comment> getComments(Story story) throws IOException {
+ List<Comment> comments = new ArrayList<Comment>();
+
+ URL url = new URL(story.getUrlInternal());
+ InputStream in = open(url);
+ Document doc = DataUtil.load(in, "UTF-8", url.toString());
+ Element listing = doc.getElementById("commentlisting");
+ if (listing != null) {
+ comments.addAll(getComments(listing));
+ }
+
+ return comments;
+ }
+
+ private List<Comment> getComments(Element listing) {
+ List<Comment> comments = new ArrayList<Comment>();
+ for (Element commentElement : listing.children()) {
+ if (commentElement.hasClass("comment")) {
+ Comment comment = getComment(commentElement);
+ if (!comment.isEmpty()) {
+ comments.add(comment);
+ }
+ }
+ }
+ return comments;
+ }
+
+ private Comment getComment(Element commentElement) {
+ String title = firstOrEmpty(commentElement, "title");
+ String author = firstOrEmpty(commentElement, "by");
+ String content = firstOrEmpty(commentElement, "commentBody");
+ String date = firstOrEmpty(commentElement, "otherdetails");
+
+ Comment comment = new Comment(commentElement.id(), author, title, date,
+ content);
+
+ for (Element child : commentElement.children()) {
+ if (child.id().contains("commtree_")) {
+ comment.addAll(getComments(child));
+ }
+ }
+
+ return comment;
+ }
+
+ private String firstOrEmpty(Element element, String className) {
+ Elements subElements = element.getElementsByClass(className);
+ if (subElements.size() > 0) {
+ return subElements.get(0).text();
+ }
+
+ return "";
+ }
+}
git://git.nikiroo.be / gofetch.git / commitdiff