1 package be
.nikiroo
.gofetch
.support
;
3 import java
.io
.IOException
;
4 import java
.io
.InputStream
;
6 import java
.net
.URLConnection
;
7 import java
.util
.ArrayList
;
9 import java
.util
.zip
.GZIPInputStream
;
11 import org
.jsoup
.helper
.StringUtil
;
12 import org
.jsoup
.nodes
.Element
;
13 import org
.jsoup
.nodes
.Node
;
14 import org
.jsoup
.nodes
.TextNode
;
15 import org
.jsoup
.select
.Elements
;
16 import org
.jsoup
.select
.NodeTraversor
;
17 import org
.jsoup
.select
.NodeVisitor
;
19 import be
.nikiroo
.gofetch
.data
.Story
;
21 public abstract class BasicSupport
{
23 SLASHDOT
, PIPEDOT
, LWN
,
26 public interface QuoteProcessor
{
27 public boolean detectQuote(Node node
);
29 public String
processText(String text
);
31 public boolean ignoreNode(Node node
);
34 static private String preselector
;
38 abstract public List
<Story
> list() throws IOException
;
41 * Fetch the full article content as well as all the comments associated to
42 * this {@link Story}, if any (can be empty, but not NULL).
45 * the story to fetch the comments of
48 * in case of I/O error
50 abstract public void fetch(Story story
) throws IOException
;
52 abstract public String
getDescription();
54 public String
getSelector() {
55 return getSelector(type
);
58 public Type
getType() {
62 protected void setType(Type type
) {
68 * the preselector to set
70 static public void setPreselector(String preselector
) {
71 BasicSupport
.preselector
= preselector
;
74 static public BasicSupport
getSupport(Type type
) {
75 BasicSupport support
= null;
80 support
= new Slashdot();
83 support
= new Pipedot();
90 if (support
!= null) {
91 support
.setType(type
);
98 static public String
getSelector(Type type
) {
99 return preselector
+ "/" + type
+ "/";
102 // TODO: check Downloader.java?
103 static protected InputStream
open(URL url
) throws IOException
{
104 URLConnection conn
= url
.openConnection();
106 InputStream in
= conn
.getInputStream();
107 if ("gzip".equals(conn
.getContentEncoding())) {
108 in
= new GZIPInputStream(in
);
115 * Get the first {@link Element} of the given class, or an empty span
116 * {@link Element} if none found.
119 * the element to look in
121 * the class to look for
123 * @return the value or an empty span {@link Element}
125 static protected Element
firstOrEmpty(Element element
, String className
) {
126 Elements subElements
= element
.getElementsByClass(className
);
127 if (subElements
.size() > 0) {
128 return subElements
.get(0);
131 return new Element("span");
135 * Get the first {@link Element} of the given tag, or an empty span
136 * {@link Element} if none found.
139 * the element to look in
141 * the tag to look for
143 * @return the value or an empty span {@link Element}
145 static protected Element
firstOrEmptyTag(Element element
, String tagName
) {
146 Elements subElements
= element
.getElementsByTag(tagName
);
147 if (subElements
.size() > 0) {
148 return subElements
.get(0);
151 return new Element("span");
154 static protected List
<String
> toLines(Element element
,
155 final QuoteProcessor quoteProcessor
) {
156 final List
<String
> lines
= new ArrayList
<String
>();
157 final StringBuilder currentLine
= new StringBuilder();
158 final List
<Integer
> quoted
= new ArrayList
<Integer
>();
159 final List
<Node
> ignoredNodes
= new ArrayList
<Node
>();
161 if (element
!= null) {
162 new NodeTraversor(new NodeVisitor() {
164 public void head(Node node
, int depth
) {
165 if (quoteProcessor
.ignoreNode(node
)
166 || ignoredNodes
.contains(node
.parentNode())) {
167 ignoredNodes
.add(node
);
172 for (int i
= 0; i
< quoted
.size(); i
++) {
177 boolean enterQuote
= quoteProcessor
.detectQuote(node
);
178 boolean leaveQuote
= quoted
.contains(depth
);
185 quoted
.remove(Integer
.valueOf(depth
));
188 if (enterQuote
|| leaveQuote
) {
189 if (currentLine
.length() > 0) {
190 if (currentLine
.charAt(currentLine
.length() - 1) == '\n') {
191 currentLine
.setLength(currentLine
.length() - 1);
193 for (String l
: currentLine
.toString().split("\n")) {
197 currentLine
.setLength(0);
200 if (node
instanceof Element
) {
201 Element element
= (Element
) node
;
202 boolean block
= element
.isBlock()
203 || element
.tagName().equalsIgnoreCase("br");
204 if (block
&& currentLine
.length() > 0) {
205 currentLine
.append("\n");
207 } else if (node
instanceof TextNode
) {
208 TextNode textNode
= (TextNode
) node
;
209 String line
= StringUtil
.normaliseWhitespace(textNode
212 currentLine
.append(quoteProcessor
.processText(line
));
213 currentLine
.append(" ");
218 public void tail(Node node
, int depth
) {
220 }).traverse(element
);
223 if (currentLine
.length() > 0) {
225 for (int i
= 0; i
< quoted
.size(); i
++) {
229 if (currentLine
.length() > 0) {
230 if (currentLine
.charAt(currentLine
.length() - 1) == '\n') {
231 currentLine
.setLength(currentLine
.length() - 1);
233 for (String l
: currentLine
.toString().split("\n")) {
239 for (int i
= 0; i
< lines
.size(); i
++) {
240 lines
.set(i
, lines
.get(i
).replace(" ", " ").trim());