* @author niki
*/
public abstract class BasicSupport {
- /** The downloader to use for all websites. */
- protected static Downloader downloader = new Downloader("gofetcher");
+ /**
+ * The downloader to use for all websites via {@link BasicSupport#open(URL)}
+ */
+ static private Downloader downloader = new Downloader("gofetcher");
static private String preselector;
* @return the selector
*/
public String getSelector() {
- return getSelector(type);
+ return getSelector(getType());
}
/**
defaultCateg = "";
}
- InputStream in = downloader.open(url);
+ InputStream in = open(url);
Document doc = DataUtil.load(in, "UTF-8", url.toString());
List<Element> articles = getArticles(doc);
for (Element article : articles) {
continue;
}
- if (id.isEmpty()) {
+ if (!id.isEmpty()) {
+ while (id.length() < 10) {
+ id = "0" + id;
+ }
+ } else {
id = date.replace(":", "_").replace("+", "_");
}
String fullContent = "";
URL url = new URL(story.getUrlInternal());
- InputStream in = downloader.open(url);
+ InputStream in = open(url);
try {
Document doc = DataUtil.load(in, "UTF-8", url.toString());
Element article = getFullArticle(doc);
*/
abstract protected ElementProcessor getElementProcessorFullArticle();
+ /**
+ * Open a network resource.
+ * <p>
+ * You need to close the returned {@link InputStream} when done.
+ *
+ * @param url
+ * the source to open
+ *
+ * @return the content
+ *
+ * @throws IOException
+ * in case of I/O error
+ */
+ protected InputStream open(URL url) throws IOException {
+ return downloader.open(url);
+ }
+
/**
* Convert the comment elements into {@link Comment}s
*