echo "MAIN = be/nikiroo/gofetch/Main" > Makefile
echo "MORE = " >> Makefile
-echo "TEST = " >> Makefile
+echo "TEST = be/nikiroo/gofetch/test/Test" >> Makefile
echo "TEST_PARAMS = $cols $ok $ko" >> Makefile
echo "NAME = gofetch" >> Makefile
echo "PREFIX = $PREFIX" >> Makefile
* @author niki
*/
public abstract class BasicSupport {
- /** The downloader to use for all websites. */
- static protected Downloader downloader = new Downloader("gofetcher");
+ /**
+ * The downloader to use for all websites via {@link BasicSupport#open(URL)}
+ */
+ static private Downloader downloader = new Downloader("gofetcher");
static private String preselector;
* @return the selector
*/
public String getSelector() {
- return getSelector(type);
+ return getSelector(getType());
}
/**
defaultCateg = "";
}
- InputStream in = downloader.open(url);
+ InputStream in = open(url);
Document doc = DataUtil.load(in, "UTF-8", url.toString());
List<Element> articles = getArticles(doc);
for (Element article : articles) {
String fullContent = "";
URL url = new URL(story.getUrlInternal());
- InputStream in = downloader.open(url);
+ InputStream in = open(url);
try {
Document doc = DataUtil.load(in, "UTF-8", url.toString());
Element article = getFullArticle(doc);
*/
abstract protected ElementProcessor getElementProcessorFullArticle();
+ /**
+ * Open a network resource.
+ * <p>
+ * You need to close the returned {@link InputStream} when done.
+ *
+ * @param url
+ * the source to open
+ *
+ * @return the content
+ *
+ * @throws IOException
+ * in case of I/O error
+ */
+ protected InputStream open(URL url) throws IOException {
+ return downloader.open(url);
+ }
+
/**
* Convert the comment elements into {@link Comment}s
*
Element a = linkToComments.getElementsByTag("a").first();
if (a != null) {
String url = a.absUrl("href");
- InputStream in = downloader.open(new URL(url));
+ InputStream in = open(new URL(url));
try {
doc = DataUtil.load(in, "UTF-8", url.toString());
return doc.getElementsByClass("b-post");
try {
URL url = new URL("https://forums.theregister.co.uk/forum/1"
+ intUrl.getPath());
- InputStream in = downloader.open(url);
+ InputStream in = open(url);
try {
doc = DataUtil.load(in, "UTF-8", url.toString());
Element posts = doc.getElementById("forum_posts");
--- /dev/null
+package be.nikiroo.gofetch.test;
+
+import be.nikiroo.utils.test.TestLauncher;
+
+/**
+ * Tests for GoFetch.
+ *
+ * @author niki
+ */
+public class Test extends TestLauncher {
+ public Test(String[] args) {
+ super("GoFetch", args);
+ addSeries(new TestLWN(args));
+ }
+
+ public static void main(String[] args) {
+ System.exit(new Test(args).launch());
+ }
+}
--- /dev/null
+package be.nikiroo.gofetch.test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Map;
+
+import be.nikiroo.gofetch.data.Story;
+import be.nikiroo.gofetch.output.Gopher;
+import be.nikiroo.gofetch.output.Html;
+import be.nikiroo.gofetch.output.Output;
+import be.nikiroo.gofetch.support.BasicSupport;
+import be.nikiroo.utils.IOUtils;
+import be.nikiroo.utils.test.TestCase;
+import be.nikiroo.utils.test.TestLauncher;
+
+/**
+ * Base class for {@link BasicSupport}s testing.
+ *
+ * @author niki
+ */
+abstract class TestBase extends TestLauncher {
+ public TestBase(BasicSupport support, String[] args) {
+ super(support.getType().toString(), args);
+ addTest(support);
+ }
+
+ static protected InputStream doOpen(Map<URL, File> map, URL url)
+ throws IOException {
+ File file = map.get(url);
+ if (file == null) {
+ throw new FileNotFoundException("Test file not found for URL: "
+ + url);
+ }
+
+ return new FileInputStream(file);
+
+ }
+
+ private void addTest(final BasicSupport support) {
+ addTest(new TestCase("Processing example data") {
+ @Override
+ public void test() throws Exception {
+ File expected = new File("test/expected/" + support.getType());
+ File actual = new File("test/result/" + support.getType());
+
+ Output gopher = new Gopher(support.getType(), "", "", 70);
+ Output html = new Html(support.getType(), "", "", 80);
+
+ for (Story story : support.list()) {
+ IOUtils.writeSmallFile(new File(actual, story.getId()
+ + ".header"), gopher.exportHeader(story));
+ IOUtils.writeSmallFile(
+ new File(actual, story.getId() + ""),
+ gopher.export(story));
+ IOUtils.writeSmallFile(new File(actual, story.getId()
+ + ".header.html"), html.exportHeader(story));
+ IOUtils.writeSmallFile(new File(actual, story.getId()
+ + ".html"), html.export(story));
+ }
+
+ assertEquals(expected, actual);
+ }
+ });
+ }
+}
--- /dev/null
+package be.nikiroo.gofetch.test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+
+import be.nikiroo.gofetch.support.LWN;
+import be.nikiroo.gofetch.support.Type;
+
+public class TestLWN extends TestBase {
+
+ static private Map<URL, File> getMap() throws MalformedURLException {
+ Map<URL, File> map = new HashMap<URL, File>();
+ map.put(new URL("http://fanfan.be/"), new File("/tmp/none"));
+ return map;
+ }
+
+ public TestLWN(String[] args) {
+ super(new LWN() {
+ @Override
+ protected InputStream open(URL url) throws IOException {
+ return doOpen(getMap(), url);
+ }
+
+ @Override
+ public Type getType() {
+ return Type.LWN;
+ }
+ }, args);
+ }
+}
git://git.nikiroo.be / gofetch.git / commitdiff