bf3af6e9664d07edcafff0df57de4766025a5975
1 package be
.nikiroo
.gofetch
;
4 import java
.io
.FileWriter
;
5 import java
.io
.FilenameFilter
;
6 import java
.io
.IOException
;
7 import java
.util
.Arrays
;
10 import be
.nikiroo
.gofetch
.data
.Comment
;
11 import be
.nikiroo
.gofetch
.data
.Story
;
12 import be
.nikiroo
.gofetch
.output
.Gopher
;
13 import be
.nikiroo
.gofetch
.output
.Html
;
14 import be
.nikiroo
.gofetch
.output
.Output
;
15 import be
.nikiroo
.gofetch
.support
.BasicSupport
;
16 import be
.nikiroo
.gofetch
.support
.BasicSupport
.Type
;
17 import be
.nikiroo
.utils
.IOUtils
;
20 * The class that will manage the fetch operations.
22 * It will scrap the required websites and process them to disk.
26 public class Fetcher
{
28 private String preselector
;
29 private int maxStories
;
30 private String hostname
;
35 * Prepare a new {@link Fetcher}.
38 * the target directory where to save the files (won't have
39 * impact on the files' content)
41 * the sub directory and (pre-)selector to use for the resources
42 * (<b>will</b> have an impact on the files' content)
44 * the type of news to get (or the special keyword ALL to get all
45 * of the supported sources)
47 * the maximum number of stories to show on the resume page
49 * the gopher host to use (<b>will</b> have an impact on the
52 * the gopher port to use (<b>will</b> have an impact on the
55 public Fetcher(File dir
, String preselector
, Type type
, int maxStories
,
56 String hostname
, int port
) {
58 this.preselector
= preselector
;
60 this.maxStories
= maxStories
;
61 this.hostname
= hostname
;
66 * Start the fetching operation.
68 * This method will handle the main pages itself, and will call
69 * {@link Fetcher#list(BasicSupport)} for the stories.
72 * in case of I/O error
74 public void start() throws IOException
{
75 File cache
= new File(dir
, preselector
);
77 File cacheHtml
= new File(cache
, "index.html");
78 cache
= new File(cache
, ".cache");
80 Output gopher
= new Gopher(null, hostname
, preselector
, port
);
81 Output html
= new Html(null, hostname
, preselector
, port
);
83 FileWriter writer
= new FileWriter(cache
);
85 FileWriter writerHtml
= new FileWriter(cacheHtml
);
87 writer
.append(gopher
.getIndexHeader());
88 writerHtml
.append(html
.getIndexHeader());
90 BasicSupport
.setPreselector(preselector
);
91 for (Type type
: Type
.values()) {
92 BasicSupport support
= BasicSupport
.getSupport(type
);
94 if (type
== this.type
|| this.type
== null) {
98 writer
.append("1" + support
.getDescription()).append("\t")
99 .append("1" + support
.getSelector()) //
100 .append("\t").append(hostname
) //
101 .append("\t").append(Integer
.toString(port
)) //
103 String ref
= support
.getSelector();
104 while (ref
.startsWith("/")) {
105 ref
= ref
.substring(1);
107 writerHtml
.append("<div class='site'><a href='../" + ref
108 + "'>" + support
.getDescription() + "</a></div>\n");
111 writer
.append(gopher
.getIndexFooter());
112 writerHtml
.append(html
.getIndexFooter());
122 * Process the stories for the given {@link BasicSupport} to disk.
125 * the {@link BasicSupport} to download from
127 * @throws IOException
128 * in case of I/O error
130 private void list(BasicSupport support
) throws IOException
{
131 Output gopher
= new Gopher(support
.getType(), hostname
, preselector
,
133 Output html
= new Html(support
.getType(), hostname
, preselector
, port
);
135 new File(dir
, support
.getSelector()).mkdirs();
138 .print("Listing recent news for " + support
.getType() + "...");
139 List
<Story
> stories
= support
.list();
140 System
.err
.println(" " + stories
.size() + " stories found!");
142 for (Story story
: stories
) {
143 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".header",
144 gopher
.export(story
));
145 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".header.html",
148 System
.err
.println(String
.format("%02d/%02d", i
, stories
.size())
149 + " Fetching comments for story " + story
.getId() + "...");
150 List
<Comment
> comments
= support
.getComments(story
);
152 IOUtils
.writeSmallFile(dir
, story
.getSelector(),
153 gopher
.export(story
, comments
));
154 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".html",
155 html
.export(story
, comments
));
160 File varDir
= new File(dir
, support
.getSelector());
161 String
[] headers
= varDir
.list(new FilenameFilter() {
163 public boolean accept(File dir
, String name
) {
164 return name
.endsWith(".header");
168 File cache
= new File(varDir
, ".cache");
169 File cacheHtml
= new File(varDir
, "index.html");
170 FileWriter writer
= new FileWriter(cache
);
172 FileWriter writerHtml
= new FileWriter(cacheHtml
);
174 if (headers
.length
> 0) {
175 Arrays
.sort(headers
);
176 int from
= headers
.length
- 1;
177 int to
= headers
.length
- maxStories
;
181 for (i
= from
; i
>= to
; i
--) {
182 writer
.append(IOUtils
.readSmallFile(new File(varDir
,
185 writerHtml
.append(IOUtils
.readSmallFile(new File(
186 varDir
, headers
[i
] + ".html")));