X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Fgofetch%2Fsupport%2FBasicSupport.java;h=ec19c3207fbbba7447da4ce5e419da9060f742b8;hb=b19b36322d5453c1b01761dd76190a67465e0d53;hp=80fe5d236d31c2379856a8245f09e80b4cfeab79;hpb=127e065f2ec5040ae8059cb3bca92c30ac2a0a47;p=gofetch.git

diff --git a/src/be/nikiroo/gofetch/support/BasicSupport.java b/src/be/nikiroo/gofetch/support/BasicSupport.java
index 80fe5d2..ec19c32 100644
--- a/src/be/nikiroo/gofetch/support/BasicSupport.java
+++ b/src/be/nikiroo/gofetch/support/BasicSupport.java
@@ -8,7 +8,9 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Map.Entry;
 
 import org.jsoup.helper.DataUtil;
@@ -31,13 +33,32 @@ import be.nikiroo.utils.StringUtils;
  * @author niki
  */
 public abstract class BasicSupport {
-	/** The downloader to use for all websites. */
-	protected static Downloader downloader = new Downloader("gofetcher");
+	/**
+	 * The downloader to use for all web sites via
+	 * {@link BasicSupport#open(URL)}
+	 */
+	static private Downloader downloader = new Downloader("gofetcher");
 
 	static private String preselector;
 
+	/**
+	 * The optional cookies to use to get the site data.
+	 */
+	private Map<String, String> cookies = new HashMap<String, String>();
+
 	private Type type;
 
+	/**
+	 * Login on the web site (this method does nothing by default, but can be
+	 * overridden if needed).
+	 * 
+	 * @throws IOException
+	 *             in case of I/O error
+	 * 
+	 */
+	public void login() throws IOException {
+	}
+
 	/**
 	 * The website textual description, to add in the dispatcher page.
 	 * <p>
@@ -55,7 +76,7 @@ public abstract class BasicSupport {
 	 * @return the selector
 	 */
 	public String getSelector() {
-		return getSelector(type);
+		return getSelector(getType());
 	}
 
 	/**
@@ -80,6 +101,7 @@ public abstract class BasicSupport {
 	public List<Story> list() throws IOException {
 		List<Story> list = new ArrayList<Story>();
 
+		login();
 		for (Entry<URL, String> entry : getUrls()) {
 			URL url = entry.getKey();
 			String defaultCateg = entry.getValue();
@@ -87,7 +109,7 @@ public abstract class BasicSupport {
 				defaultCateg = "";
 			}
 
-			InputStream in = downloader.open(url);
+			InputStream in = open(url);
 			Document doc = DataUtil.load(in, "UTF-8", url.toString());
 			List<Element> articles = getArticles(doc);
 			for (Element article : articles) {
@@ -106,7 +128,11 @@ public abstract class BasicSupport {
 					continue;
 				}
 
-				if (id.isEmpty()) {
+				if (!id.isEmpty()) {
+					while (id.length() < 10) {
+						id = "0" + id;
+					}
+				} else {
 					id = date.replace(":", "_").replace("+", "_");
 				}
 
@@ -269,26 +295,12 @@ public abstract class BasicSupport {
 		String fullContent = "";
 
 		URL url = new URL(story.getUrlInternal());
-		InputStream in = downloader.open(url);
+		InputStream in = open(url);
 		try {
 			Document doc = DataUtil.load(in, "UTF-8", url.toString());
 			Element article = getFullArticle(doc);
 			if (article != null) {
-				StringBuilder builder = new StringBuilder();
-				ElementProcessor eProc = getElementProcessorFullArticle();
-				if (eProc != null) {
-					for (String line : toLines(article, eProc)) {
-						builder.append(line + "\n");
-					}
-				} else {
-					builder.append(article.text());
-				}
-
-				// Content is too tight with a single break per line:
-				fullContent = builder.toString().replace("\n", "\n\n") //
-						.replace("\n\n\n\n", "\n\n") //
-						.replace("\n\n\n\n", "\n\n") //
-						.trim();
+				fullContent = getArticleText(article);
 			}
 
 			if (fullContent.isEmpty()) {
@@ -306,7 +318,35 @@ public abstract class BasicSupport {
 	}
 
 	/**
-	 * Return the full article if available.
+	 * Return the text from this {@link Element}, using the
+	 * {@link BasicSupport#getElementProcessorFullArticle()} processor logic.
+	 * 
+	 * @param article
+	 *            the element to extract the text from
+	 * 
+	 * @return the text
+	 */
+	protected String getArticleText(Element article) {
+		StringBuilder builder = new StringBuilder();
+		ElementProcessor eProc = getElementProcessorFullArticle();
+		if (eProc != null) {
+			for (String line : toLines(article, eProc)) {
+				builder.append(line + "\n");
+			}
+		} else {
+			builder.append(article.text());
+		}
+
+		// Content is too tight with a single break per line:
+		return builder.toString().replace("\n", "\n\n") //
+				.replace("\n\n\n\n", "\n\n") //
+				.replace("\n\n\n\n", "\n\n") //
+				.trim();
+	}
+
+	/**
+	 * Return the full article if available (this is the article to retrieve
+	 * from the newly downloaded page at {@link Story#getUrlInternal()}).
 	 * 
 	 * @param doc
 	 *            the (full article) document to work on
@@ -342,6 +382,23 @@ public abstract class BasicSupport {
 	 */
 	abstract protected ElementProcessor getElementProcessorFullArticle();
 
+	/**
+	 * Open a network resource.
+	 * <p>
+	 * You need to close the returned {@link InputStream} when done.
+	 * 
+	 * @param url
+	 *            the source to open
+	 * 
+	 * @return the content
+	 * 
+	 * @throws IOException
+	 *             in case of I/O error
+	 */
+	protected InputStream open(URL url) throws IOException {
+		return downloader.open(url, url, cookies, null, null, null);
+	}
+
 	/**
 	 * Convert the comment elements into {@link Comment}s
 	 * 
@@ -480,6 +537,18 @@ public abstract class BasicSupport {
 		this.type = type;
 	}
 
+	/**
+	 * Add a cookie for all site connections.
+	 * 
+	 * @param name
+	 *            the cookie name
+	 * @param value
+	 *            the value
+	 */
+	protected void addCookie(String name, String value) {
+		cookies.put(name, value);
+	}
+
 	/**
 	 * The {@link String} to append to the selector (the selector will be
 	 * constructed as "this string" then "/type/".
@@ -529,6 +598,12 @@ public abstract class BasicSupport {
 			case PHORONIX:
 				support = new Phoronix();
 				break;
+			case SEPT_SUR_SEPT:
+				support = new SeptSurSept();
+				break;
+			case REDDIT:
+				support = new Reddit();
+				break;
 			}
 
 			if (support != null) {