1 package be
.nikiroo
.gofetch
;
4 import java
.io
.FileWriter
;
5 import java
.io
.FilenameFilter
;
6 import java
.io
.IOException
;
7 import java
.util
.Arrays
;
10 import be
.nikiroo
.gofetch
.data
.Comment
;
11 import be
.nikiroo
.gofetch
.data
.Story
;
12 import be
.nikiroo
.gofetch
.output
.Gopher
;
13 import be
.nikiroo
.gofetch
.output
.Html
;
14 import be
.nikiroo
.gofetch
.output
.Output
;
15 import be
.nikiroo
.gofetch
.support
.BasicSupport
;
16 import be
.nikiroo
.gofetch
.support
.BasicSupport
.Type
;
17 import be
.nikiroo
.utils
.IOUtils
;
20 * The class that will manage the fetch operations.
22 * It will scrap the required websites and process them to disk.
26 public class Fetcher
{
28 private String preselector
;
29 private int maxStories
;
30 private String hostname
;
35 * Prepare a new {@link Fetcher}.
38 * the target directory where to save the files (won't have
39 * impact on the files' content)
41 * the sub directory and (pre-)selector to use for the resources
42 * (<b>will</b> have an impact on the files' content)
44 * the type of news to get (or the special keyword ALL to get all
45 * of the supported sources)
47 * the maximum number of stories to show on the resume page
49 * the gopher host to use (<b>will</b> have an impact on the
52 * the gopher port to use (<b>will</b> have an impact on the
55 public Fetcher(File dir
, String preselector
, Type type
, int maxStories
,
56 String hostname
, int port
) {
58 this.preselector
= preselector
;
60 this.maxStories
= maxStories
;
61 this.hostname
= hostname
;
66 * Start the fetching operation.
68 * This method will handle the main pages itself, and will call
69 * {@link Fetcher#list(BasicSupport)} for the stories.
72 * in case of I/O error
74 public void start() throws IOException
{
75 File cache
= new File(dir
, preselector
);
77 File cacheHtml
= new File(cache
, "index.html");
78 cache
= new File(cache
, ".cache");
80 Output gopher
= new Gopher(null, hostname
, port
);
81 Output html
= new Html(null);
83 FileWriter writer
= new FileWriter(cache
);
85 FileWriter writerHtml
= new FileWriter(cacheHtml
);
87 writer
.append(gopher
.getIndexHeader());
88 writerHtml
.append(html
.getIndexHeader());
92 types
= Type
.values();
94 types
= new Type
[] { type
};
97 BasicSupport
.setPreselector(preselector
);
98 for (Type type
: types
) {
99 BasicSupport support
= BasicSupport
.getSupport(type
);
102 writer
.append("1" + support
.getDescription()).append("\t")
103 .append("1" + support
.getSelector()) //
104 .append("\t").append(hostname
) //
105 .append("\t").append(Integer
.toString(port
)) //
107 String ref
= support
.getSelector();
108 while (ref
.startsWith("/")) {
109 ref
= ref
.substring(1);
111 writerHtml
.append("<div class='site'><a href='../" + ref
112 + "'>" + support
.getDescription() + "</a></div>");
115 writer
.append(gopher
.getIndexFooter());
116 writerHtml
.append(html
.getIndexFooter());
126 * Process the stories for the given {@link BasicSupport} to disk.
129 * the {@link BasicSupport} to download from
131 * @throws IOException
132 * in case of I/O error
134 private void list(BasicSupport support
) throws IOException
{
135 Output gopher
= new Gopher(support
.getType(), hostname
, port
);
136 Output html
= new Html(support
.getType());
138 new File(dir
, support
.getSelector()).mkdirs();
141 .print("Listing recent news for " + support
.getType() + "...");
142 List
<Story
> stories
= support
.list();
143 System
.err
.println(" " + stories
.size() + " stories found!");
145 for (Story story
: stories
) {
146 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".header",
147 gopher
.export(story
));
148 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".header.html",
151 System
.err
.println(String
.format("%02d/%02d", i
, stories
.size())
152 + " Fetching comments for story " + story
.getId() + "...");
153 List
<Comment
> comments
= support
.getComments(story
);
155 IOUtils
.writeSmallFile(dir
, story
.getSelector(),
156 gopher
.export(story
, comments
));
157 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".html",
158 html
.export(story
, comments
));
163 File varDir
= new File(dir
, support
.getSelector());
164 String
[] headers
= varDir
.list(new FilenameFilter() {
166 public boolean accept(File dir
, String name
) {
167 return name
.endsWith(".header");
171 File cache
= new File(varDir
, ".cache");
172 File cacheHtml
= new File(varDir
, "index.html");
173 FileWriter writer
= new FileWriter(cache
);
175 FileWriter writerHtml
= new FileWriter(cacheHtml
);
177 if (headers
.length
> 0) {
178 Arrays
.sort(headers
);
179 int from
= headers
.length
- 1;
180 int to
= headers
.length
- maxStories
;
184 for (i
= from
; i
>= to
; i
--) {
185 writer
.append(IOUtils
.readSmallFile(new File(varDir
,
188 writerHtml
.append(IOUtils
.readSmallFile(new File(
189 varDir
, headers
[i
] + ".html")));