X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Fgofetch%2Fsupport%2FBasicSupport.java;h=615c72d6b0cbd81c4f59d8472878ea07ef4b730a;hb=136ab80122a17caa0720116d6d2552521239fbb3;hp=1db066b3d107c8fefb26218675fa8c5c0767b815;hpb=27008a8782c0ed96e07c8dc39ff0ed1f5163a9d0;p=gofetch.git diff --git a/src/be/nikiroo/gofetch/support/BasicSupport.java b/src/be/nikiroo/gofetch/support/BasicSupport.java index 1db066b..615c72d 100644 --- a/src/be/nikiroo/gofetch/support/BasicSupport.java +++ b/src/be/nikiroo/gofetch/support/BasicSupport.java @@ -1,12 +1,8 @@ package be.nikiroo.gofetch.support; import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.net.URLConnection; import java.util.ArrayList; import java.util.List; -import java.util.zip.GZIPInputStream; import org.jsoup.helper.StringUtil; import org.jsoup.nodes.Element; @@ -17,24 +13,109 @@ import org.jsoup.select.NodeTraversor; import org.jsoup.select.NodeVisitor; import be.nikiroo.gofetch.data.Story; +import be.nikiroo.utils.Downloader; public abstract class BasicSupport { + protected static Downloader downloader = new Downloader("gofetcher"); + public enum Type { - SLASHDOT, PIPEDOT, LWN, + SLASHDOT, PIPEDOT, LWN, LEMONDE, } - public interface QuoteProcessor { + /** + * Used to process an element into lines. + * + * @author niki + */ + public interface ElementProcessor { + /** + * Detect if this node is a quote and should be trated as such. + * + * @param node + * the node to check + * @return TRUE if it is + */ public boolean detectQuote(Node node); + /** + * Process text content (will be called on each text element, allowing + * you to modify it if needed). + * + * @param text + * the text to process + * @return + */ public String processText(String text); + /** + * Ignore this node. + * + * @param node + * the node to ignore + * @return TRUE if it has to be ignored + */ public boolean ignoreNode(Node node); + + /** + * Manually process this node (and return the manual processing value) + * if so desired. + *

+ * If the node is manually processed, it and its children will not be + * automatically processed. + * + * @param node + * the node to optionally process + * + * @return NULL if not processed (will thus be automatically processed + * as usual), a {@link String} (may be empty) if we process it + * manually -- the given {@link String} will be used instead of + * the usual automatic processing if not NULL + */ + public String manualProcessing(Node node); + } + + /** + * A default {@link ElementProcessor} (will not detect or process anything + * manually). + * + * @author niki + */ + public class BasicElementProcessor implements ElementProcessor { + @Override + public boolean detectQuote(Node node) { + return false; + } + + @Override + public String processText(String text) { + return text; + } + + @Override + public boolean ignoreNode(Node node) { + return false; + } + + @Override + public String manualProcessing(Node node) { + return null; + } } static private String preselector; private Type type; + /** + * List all the recent items, but only assure the ID and internal URL to + * fetch it later on (until it has been fetched, the rest of the + * {@link Story} is not confirmed). + * + * @return the list of new stories + * + * @throws IOException + * in case of I/O + */ abstract public List list() throws IOException; /** @@ -71,6 +152,15 @@ public abstract class BasicSupport { BasicSupport.preselector = preselector; } + /** + * Return a {@link BasicSupport} that is compatible with the given + * {@link Type} if it exists (or NULL if not). + * + * @param type + * the type + * + * @return a compatible {@link BasicSupport} if it exists (or NULL if not) + */ static public BasicSupport getSupport(Type type) { BasicSupport support = null; @@ -85,6 +175,9 @@ public abstract class BasicSupport { case LWN: support = new LWN(); break; + case LEMONDE: + support = new LeMonde(); + break; } if (support != null) { @@ -99,18 +192,6 @@ public abstract class BasicSupport { return preselector + "/" + type + "/"; } - // TODO: check Downloader.java? - static protected InputStream open(URL url) throws IOException { - URLConnection conn = url.openConnection(); - conn.connect(); - InputStream in = conn.getInputStream(); - if ("gzip".equals(conn.getContentEncoding())) { - in = new GZIPInputStream(in); - } - - return in; - } - /** * Get the first {@link Element} of the given class, or an empty span * {@link Element} if none found. @@ -151,8 +232,20 @@ public abstract class BasicSupport { return new Element("span"); } + /** + * Process the given element into text (each line is a text paragraph and + * can be prepended with ">" signs to indicate a quote or sub-quote or + * sub-sub-quote...). + * + * @param element + * the element to process + * @param elementProcessor + * the element processor, must not be NULL + * + * @return text lines, each line is a paragraph + */ static protected List toLines(Element element, - final QuoteProcessor quoteProcessor) { + final ElementProcessor elementProcessor) { final List lines = new ArrayList(); final StringBuilder currentLine = new StringBuilder(); final List quoted = new ArrayList(); @@ -162,8 +255,18 @@ public abstract class BasicSupport { new NodeTraversor(new NodeVisitor() { @Override public void head(Node node, int depth) { - if (quoteProcessor.ignoreNode(node) - || ignoredNodes.contains(node.parentNode())) { + String manual = null; + boolean ignore = elementProcessor.ignoreNode(node) + || ignoredNodes.contains(node.parentNode()); + if (!ignore) { + manual = elementProcessor.manualProcessing(node); + if (manual != null) { + currentLine.append(manual); + ignore = true; + } + } + + if (ignore) { ignoredNodes.add(node); return; } @@ -174,7 +277,7 @@ public abstract class BasicSupport { } prep += " "; - boolean enterQuote = quoteProcessor.detectQuote(node); + boolean enterQuote = elementProcessor.detectQuote(node); boolean leaveQuote = quoted.contains(depth); if (enterQuote) { @@ -209,7 +312,7 @@ public abstract class BasicSupport { String line = StringUtil.normaliseWhitespace(textNode .getWholeText()); - currentLine.append(quoteProcessor.processText(line)); + currentLine.append(elementProcessor.processText(line)); currentLine.append(" "); } }