X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Fgofetch%2Fsupport%2FBasicSupport.java;h=b15fac7e5e2598d0d67c3bcf493c6dae03a0a8a1;hb=b9afb12e17825f363d3679fcac75095fb1e9dc6d;hp=102023eb051a02b6c13a41e5ddc0d64e98743156;hpb=100a839503d23e324d2db3f6d3e47892def3bf81;p=gofetch.git
diff --git a/src/be/nikiroo/gofetch/support/BasicSupport.java b/src/be/nikiroo/gofetch/support/BasicSupport.java
index 102023e..b15fac7 100644
--- a/src/be/nikiroo/gofetch/support/BasicSupport.java
+++ b/src/be/nikiroo/gofetch/support/BasicSupport.java
@@ -1,12 +1,11 @@
package be.nikiroo.gofetch.support;
import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.net.URLConnection;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Date;
import java.util.List;
-import java.util.zip.GZIPInputStream;
import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Element;
@@ -17,29 +16,134 @@ import org.jsoup.select.NodeTraversor;
import org.jsoup.select.NodeVisitor;
import be.nikiroo.gofetch.data.Story;
+import be.nikiroo.utils.Downloader;
+/**
+ * Base class for website support.
+ *
+ * @author niki
+ */
public abstract class BasicSupport {
+ /** The downloader to use for all websites. */
+ protected static Downloader downloader = new Downloader("gofetcher");
+
+ /**
+ * The support type (each website we support has a single type).
+ *
+ * @author niki
+ */
public enum Type {
- SLASHDOT, PIPEDOT, LWN, LEMONDE,
+ /** EN: Any, but mostly IT/Sci */
+ SLASHDOT,
+ /** EN: Clone of Slashdot, mostly abandoned */
+ PIPEDOT,
+ /** EN: Linux */
+ LWN,
+ /** FR: Any */
+ LEMONDE,
+ /** EN: IT */
+ REGISTER,
+ /** FR: Linux */
+ TOO_LINUX,
+ /** FR: IT */
+ ERE_NUMERIQUE,
}
- public interface QuoteProcessor {
+ /**
+ * Used to process an element into lines.
+ *
+ * @author niki
+ */
+ public interface ElementProcessor {
+ /**
+ * Detect if this node is a quote and should be trated as such.
+ *
+ * @param node
+ * the node to check
+ * @return TRUE if it is
+ */
public boolean detectQuote(Node node);
+ /**
+ * Process text content (will be called on each text element, allowing
+ * you to modify it if needed).
+ *
+ * @param text
+ * the text to process
+ *
+ * @return the resulting text
+ */
public String processText(String text);
+ /**
+ * Ignore this node.
+ *
+ * @param node
+ * the node to ignore
+ * @return TRUE if it has to be ignored
+ */
public boolean ignoreNode(Node node);
/**
- * Manually process this node if so desired.
+ * Manually process this node (and return the manual processing value)
+ * if so desired.
+ *
+ * If the node is manually processed, it and its children will not be
+ * automatically processed.
*
* @param node
* the node to optionally process
*
- * @return NULL if not processed, a {@link String} (may be empty) if we
- * must not process it any further
+ * @return NULL if not processed (will thus be automatically processed
+ * as usual), a {@link String} (may be empty) if we process it
+ * manually -- the given {@link String} will be used instead of
+ * the usual automatic processing if not NULL
*/
public String manualProcessing(Node node);
+
+ /**
+ * This {@link Node} is a subtitle and should be treated as such
+ * (highlighted).
+ *
+ * @param node
+ * the node to check
+ *
+ * @return NULL if it is not a subtitle, the subtitle to use if it is
+ */
+ public String isSubtitle(Node node);
+ }
+
+ /**
+ * A default {@link ElementProcessor} (will not detect or process anything
+ * manually).
+ *
+ * @author niki
+ */
+ public class BasicElementProcessor implements ElementProcessor {
+ @Override
+ public boolean detectQuote(Node node) {
+ return false;
+ }
+
+ @Override
+ public String processText(String text) {
+ return text;
+ }
+
+ @Override
+ public boolean ignoreNode(Node node) {
+ return false;
+ }
+
+ @Override
+ public String manualProcessing(Node node) {
+ return null;
+ }
+
+ @Override
+ public String isSubtitle(Node node) {
+ return null;
+ }
}
static private String preselector;
@@ -70,21 +174,49 @@ public abstract class BasicSupport {
*/
abstract public void fetch(Story story) throws IOException;
+ /**
+ * The website textual description, to add in the dispatcher page.
+ *
+ * Should be short.
+ *
+ * @return the description
+ */
abstract public String getDescription();
+ /**
+ * The gopher "selector" to use for output.
+ *
+ * A kind of "URL path", like "/news/" or "/misc/news/" or...
+ *
+ * @return the selector
+ */
public String getSelector() {
return getSelector(type);
}
+ /**
+ * The support type.
+ *
+ * @return the type
+ */
public Type getType() {
return type;
}
+ /**
+ * The support type.
+ *
+ * @param type
+ * the new type
+ */
protected void setType(Type type) {
this.type = type;
}
/**
+ * The {@link String} to append to the selector (the selector will be
+ * constructed as "this string" then "/type/".
+ *
* @param preselector
* the preselector to set
*/
@@ -92,6 +224,15 @@ public abstract class BasicSupport {
BasicSupport.preselector = preselector;
}
+ /**
+ * Return a {@link BasicSupport} that is compatible with the given
+ * {@link Type} if it exists (or NULL if not).
+ *
+ * @param type
+ * the type
+ *
+ * @return a compatible {@link BasicSupport} if it exists (or NULL if not)
+ */
static public BasicSupport getSupport(Type type) {
BasicSupport support = null;
@@ -109,6 +250,15 @@ public abstract class BasicSupport {
case LEMONDE:
support = new LeMonde();
break;
+ case REGISTER:
+ support = new TheRegister();
+ break;
+ case TOO_LINUX:
+ support = new TooLinux();
+ break;
+ case ERE_NUMERIQUE:
+ support = new EreNumerique();
+ break;
}
if (support != null) {
@@ -119,22 +269,21 @@ public abstract class BasicSupport {
return support;
}
+ /**
+ * The gopher "selector" to use for output for this type, using the
+ * preselector.
+ *
+ * A kind of "URL path", like "/news/" or "/misc/news/" or...
+ *
+ * @param type
+ * the type to get the selector of
+ *
+ * @return the selector
+ */
static public String getSelector(Type type) {
return preselector + "/" + type + "/";
}
- // TODO: check Downloader.java?
- static protected InputStream open(URL url) throws IOException {
- URLConnection conn = url.openConnection();
- conn.connect();
- InputStream in = conn.getInputStream();
- if ("gzip".equals(conn.getContentEncoding())) {
- in = new GZIPInputStream(in);
- }
-
- return in;
- }
-
/**
* Get the first {@link Element} of the given class, or an empty span
* {@link Element} if none found.
@@ -175,8 +324,20 @@ public abstract class BasicSupport {
return new Element("span");
}
+ /**
+ * Process the given element into text (each line is a text paragraph and
+ * can be prepended with ">" signs to indicate a quote or sub-quote or
+ * sub-sub-quote...).
+ *
+ * @param element
+ * the element to process
+ * @param elementProcessor
+ * the element processor, must not be NULL
+ *
+ * @return text lines, each line is a paragraph
+ */
static protected List toLines(Element element,
- final QuoteProcessor quoteProcessor) {
+ final ElementProcessor elementProcessor) {
final List lines = new ArrayList();
final StringBuilder currentLine = new StringBuilder();
final List quoted = new ArrayList();
@@ -187,16 +348,27 @@ public abstract class BasicSupport {
@Override
public void head(Node node, int depth) {
String manual = null;
- boolean ignore = quoteProcessor.ignoreNode(node)
+ boolean ignore = elementProcessor.ignoreNode(node)
|| ignoredNodes.contains(node.parentNode());
+ // Manual processing
if (!ignore) {
- manual = quoteProcessor.manualProcessing(node);
+ manual = elementProcessor.manualProcessing(node);
if (manual != null) {
currentLine.append(manual);
ignore = true;
}
}
+ // Subtitle check
+ if (!ignore) {
+ String subtitle = elementProcessor.isSubtitle(node);
+ if (subtitle != null) {
+ subtitle = subtitle.trim();
+ currentLine.append("\n[ " + subtitle + " ]\n");
+ ignore = true;
+ }
+ }
+
if (ignore) {
ignoredNodes.add(node);
return;
@@ -208,7 +380,7 @@ public abstract class BasicSupport {
}
prep += " ";
- boolean enterQuote = quoteProcessor.detectQuote(node);
+ boolean enterQuote = elementProcessor.detectQuote(node);
boolean leaveQuote = quoted.contains(depth);
if (enterQuote) {
@@ -243,7 +415,7 @@ public abstract class BasicSupport {
String line = StringUtil.normaliseWhitespace(textNode
.getWholeText());
- currentLine.append(quoteProcessor.processText(line));
+ currentLine.append(elementProcessor.processText(line));
currentLine.append(" ");
}
}
@@ -276,4 +448,35 @@ public abstract class BasicSupport {
return lines;
}
+
+ /**
+ * Reformat the date if possible.
+ *
+ * @param date
+ * the input date
+ *
+ * @return the reformated date, or the same value if it was not parsable
+ */
+ static protected String date(String date) {
+ SimpleDateFormat out = new SimpleDateFormat("yyyy/MM/dd");
+
+ long epoch = 0;
+ try {
+ epoch = Long.parseLong(date.trim());
+ } catch (Exception e) {
+ epoch = 0;
+ }
+
+ if (epoch > 0) {
+ return out.format(new Date(1000 * epoch));
+ }
+
+ try {
+ Date dat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")
+ .parse(date.trim());
+ return out.format(dat);
+ } catch (ParseException e) {
+ return date;
+ }
+ }
}