X-Git-Url: http://git.nikiroo.be/?a=blobdiff_plain;f=src%2Fbe%2Fnikiroo%2Fgofetch%2Fsupport%2FBasicSupport.java;h=9555b9db363a2bdffe408d1698db0bb35f73d380;hb=21f1a997d7addd93226243b0d6d80ce8afcde153;hp=a748262857debc001dcb07cd557a7563a007e3d7;hpb=a71d4075a8591b0655277b1a0e606ee48d228869;p=gofetch.git

diff --git a/src/be/nikiroo/gofetch/support/BasicSupport.java b/src/be/nikiroo/gofetch/support/BasicSupport.java
index a748262..9555b9d 100644
--- a/src/be/nikiroo/gofetch/support/BasicSupport.java
+++ b/src/be/nikiroo/gofetch/support/BasicSupport.java
@@ -3,12 +3,13 @@ package be.nikiroo.gofetch.support;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
-import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Map.Entry;
 
 import org.jsoup.helper.DataUtil;
@@ -32,14 +33,32 @@ import be.nikiroo.utils.StringUtils;
  */
 public abstract class BasicSupport {
 	/**
-	 * The downloader to use for all websites via {@link BasicSupport#open(URL)}
+	 * The {@link Downloader} to use for all web sites via
+	 * {@link BasicSupport#open(URL)}
 	 */
 	static private Downloader downloader = new Downloader("gofetcher");
 
 	static private String preselector;
 
+	/**
+	 * The optional cookies to use to get the site data.
+	 */
+	private Map<String, String> cookies = new HashMap<String, String>();
+
 	private Type type;
 
+	/**
+	 * Login on the web site (this method does nothing by default, but can be
+	 * overridden if needed).
+	 * 
+	 * @throws IOException
+	 *             in case of I/O error
+	 * 
+	 */
+	@SuppressWarnings("unused")
+	public void login() throws IOException {
+	}
+
 	/**
 	 * The website textual description, to add in the dispatcher page.
 	 * <p>
@@ -82,6 +101,7 @@ public abstract class BasicSupport {
 	public List<Story> list() throws IOException {
 		List<Story> list = new ArrayList<Story>();
 
+		login();
 		for (Entry<URL, String> entry : getUrls()) {
 			URL url = entry.getKey();
 			String defaultCateg = entry.getValue();
@@ -113,7 +133,8 @@ public abstract class BasicSupport {
 						id = "0" + id;
 					}
 				} else {
-					id = date.replace(":", "_").replace("+", "_");
+					id = date.replace(":", "_").replace("+", "_")
+							.replace("/", "-");
 				}
 
 				date = date(date);
@@ -280,21 +301,7 @@ public abstract class BasicSupport {
 			Document doc = DataUtil.load(in, "UTF-8", url.toString());
 			Element article = getFullArticle(doc);
 			if (article != null) {
-				StringBuilder builder = new StringBuilder();
-				ElementProcessor eProc = getElementProcessorFullArticle();
-				if (eProc != null) {
-					for (String line : toLines(article, eProc)) {
-						builder.append(line + "\n");
-					}
-				} else {
-					builder.append(article.text());
-				}
-
-				// Content is too tight with a single break per line:
-				fullContent = builder.toString().replace("\n", "\n\n") //
-						.replace("\n\n\n\n", "\n\n") //
-						.replace("\n\n\n\n", "\n\n") //
-						.trim();
+				fullContent = getArticleText(article);
 			}
 
 			if (fullContent.isEmpty()) {
@@ -312,7 +319,35 @@ public abstract class BasicSupport {
 	}
 
 	/**
-	 * Return the full article if available.
+	 * Return the text from this {@link Element}, using the
+	 * {@link BasicSupport#getElementProcessorFullArticle()} processor logic.
+	 * 
+	 * @param article
+	 *            the element to extract the text from
+	 * 
+	 * @return the text
+	 */
+	protected String getArticleText(Element article) {
+		StringBuilder builder = new StringBuilder();
+		ElementProcessor eProc = getElementProcessorFullArticle();
+		if (eProc != null) {
+			for (String line : toLines(article, eProc)) {
+				builder.append(line + "\n");
+			}
+		} else {
+			builder.append(article.text());
+		}
+
+		// Content is too tight with a single break per line:
+		return builder.toString().replace("\n", "\n\n") //
+				.replace("\n\n\n\n", "\n\n") //
+				.replace("\n\n\n\n", "\n\n") //
+				.trim();
+	}
+
+	/**
+	 * Return the full article if available (this is the article to retrieve
+	 * from the newly downloaded page at {@link Story#getUrlInternal()}).
 	 * 
 	 * @param doc
 	 *            the (full article) document to work on
@@ -362,7 +397,7 @@ public abstract class BasicSupport {
 	 *             in case of I/O error
 	 */
 	protected InputStream open(URL url) throws IOException {
-		return downloader.open(url);
+		return downloader.open(url, url, cookies, null, null, null);
 	}
 
 	/**
@@ -503,6 +538,18 @@ public abstract class BasicSupport {
 		this.type = type;
 	}
 
+	/**
+	 * Add a cookie for all site connections.
+	 * 
+	 * @param name
+	 *            the cookie name
+	 * @param value
+	 *            the value
+	 */
+	protected void addCookie(String name, String value) {
+		cookies.put(name, value);
+	}
+
 	/**
 	 * The {@link String} to append to the selector (the selector will be
 	 * constructed as "this string" then "/type/".
@@ -531,9 +578,6 @@ public abstract class BasicSupport {
 			case SLASHDOT:
 				support = new Slashdot();
 				break;
-			case PIPEDOT:
-				support = new Pipedot();
-				break;
 			case LWN:
 				support = new LWN();
 				break;
@@ -552,6 +596,12 @@ public abstract class BasicSupport {
 			case PHORONIX:
 				support = new Phoronix();
 				break;
+			case SEPT_SUR_SEPT:
+				support = new SeptSurSept();
+				break;
+			case REDDIT:
+				support = new Reddit();
+				break;
 			}
 
 			if (support != null) {
@@ -770,7 +820,7 @@ public abstract class BasicSupport {
 			Date dat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")
 					.parse(date.trim());
 			return out.format(dat);
-		} catch (ParseException e) {
+		} catch (Exception e) {
 			return date;
 		}
 	}