List<Story> list = new ArrayList<Story>();
URL url = new URL("https://lwn.net/");
- InputStream in = open(url);
+ InputStream in = downloader.open(url);
Document doc = DataUtil.load(in, "UTF-8", url.toString());
- Elements stories = doc.getElementsByClass("pure-u-1");
- for (Element story : stories) {
- Elements titles = story.getElementsByClass("Headline");
- Elements listings = story.getElementsByClass("BlurbListing");
+ Elements articles = doc.getElementsByClass("pure-u-1");
+ for (Element article : articles) {
+ Elements titles = article.getElementsByClass("Headline");
+ Elements listings = article.getElementsByClass("BlurbListing");
if (titles.size() == 0) {
continue;
}
String id = "";
String intUrl = "";
String extUrl = "";
- for (Element idElem : story.getElementsByTag("a")) {
+ for (Element idElem : article.getElementsByTag("a")) {
// Last link is the story link
intUrl = idElem.absUrl("href");
pos = intUrl.indexOf("#Comments");
// Do not try the paid-for stories...
if (!story.getTitle().startsWith("[$]")) {
URL url = new URL(story.getUrlInternal());
- InputStream in = open(url);
+ InputStream in = downloader.open(url);
Document doc = DataUtil.load(in, "UTF-8", url.toString());
Elements fullContentElements = doc
.getElementsByClass("ArticleText");
}
private List<String> toLines(Element element) {
- return toLines(element, new QuoteProcessor() {
+ return toLines(element, new BasicElementProcessor() {
@Override
public String processText(String text) {
while (text.startsWith(">")) { // comments