1 package be
.nikiroo
.gofetch
;
4 import java
.io
.FileWriter
;
5 import java
.io
.FilenameFilter
;
6 import java
.io
.IOException
;
7 import java
.util
.Arrays
;
10 import be
.nikiroo
.gofetch
.data
.Story
;
11 import be
.nikiroo
.gofetch
.output
.Gopher
;
12 import be
.nikiroo
.gofetch
.output
.Html
;
13 import be
.nikiroo
.gofetch
.output
.Output
;
14 import be
.nikiroo
.gofetch
.support
.BasicSupport
;
15 import be
.nikiroo
.gofetch
.support
.BasicSupport
.Type
;
16 import be
.nikiroo
.utils
.IOUtils
;
19 * The class that will manage the fetch operations.
21 * It will scrap the required websites and process them to disk.
25 public class Fetcher
{
27 private String preselector
;
28 private int maxStories
;
29 private String hostname
;
34 * Prepare a new {@link Fetcher}.
37 * the target directory where to save the files (won't have
38 * impact on the files' content)
40 * the sub directory and (pre-)selector to use for the resources
41 * (<b>will</b> have an impact on the files' content)
43 * the type of news to get (or the special keyword ALL to get all
44 * of the supported sources)
46 * the maximum number of stories to show on the resume page
48 * the gopher host to use (<b>will</b> have an impact on the
51 * the gopher port to use (<b>will</b> have an impact on the
54 public Fetcher(File dir
, String preselector
, Type type
, int maxStories
,
55 String hostname
, int port
) {
57 this.preselector
= preselector
;
59 this.maxStories
= maxStories
;
60 this.hostname
= hostname
;
65 * Start the fetching operation.
67 * This method will handle the main pages itself, and will call
68 * {@link Fetcher#list(BasicSupport)} for the stories.
71 * in case of I/O error
73 public void start() throws IOException
{
74 StringBuilder gopherBuilder
= new StringBuilder();
75 StringBuilder htmlBuilder
= new StringBuilder();
77 BasicSupport
.setPreselector(preselector
);
78 for (Type type
: Type
.values()) {
79 BasicSupport support
= BasicSupport
.getSupport(type
);
81 if (type
== this.type
|| this.type
== null) {
85 gopherBuilder
.append("1" + support
.getDescription()).append("\t")
86 .append("1" + support
.getSelector()) //
87 .append("\t").append(hostname
) //
88 .append("\t").append(Integer
.toString(port
)) //
91 String ref
= support
.getSelector();
92 while (ref
.startsWith("/")) {
93 ref
= ref
.substring(1);
95 htmlBuilder
.append("<div class='site'><a href='../" + ref
+ "'>"
96 + support
.getDescription() + "</a></div>\n");
99 File gopherCache
= new File(dir
, preselector
);
100 gopherCache
.mkdirs();
101 File htmlIndex
= new File(gopherCache
, "index.html");
102 gopherCache
= new File(gopherCache
, ".cache");
104 Output gopher
= new Gopher(null, hostname
, preselector
, port
);
105 Output html
= new Html(null, hostname
, preselector
, port
);
107 FileWriter writer
= new FileWriter(gopherCache
);
109 writer
.append(gopher
.getIndexHeader());
110 writer
.append(gopherBuilder
.toString());
111 writer
.append(gopher
.getIndexFooter());
117 writer
= new FileWriter(htmlIndex
);
118 writer
.append(html
.getIndexHeader());
119 writer
.append(htmlBuilder
.toString());
120 writer
.append(html
.getIndexFooter());
127 * Process the stories for the given {@link BasicSupport} to disk.
130 * the {@link BasicSupport} to download from
132 * @throws IOException
133 * in case of I/O error
135 private void list(BasicSupport support
) throws IOException
{
138 .print("Listing recent news for " + support
.getType() + "...");
139 List
<Story
> stories
= support
.list();
140 System
.err
.println(" " + stories
.size() + " stories found!");
142 // Get comments (and update stories if needed):
144 for (Story story
: stories
) {
145 System
.err
.println(String
.format("%02d/%02d", i
, stories
.size())
146 + " Fetching full story " + story
.getId() + "...");
147 support
.fetch(story
);
151 Output gopher
= new Gopher(support
.getType(), hostname
, preselector
,
153 Output html
= new Html(support
.getType(), hostname
, preselector
, port
);
155 new File(dir
, support
.getSelector()).mkdirs();
157 for (Story story
: stories
) {
158 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".header",
159 gopher
.exportHeader(story
));
160 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".header.html",
161 html
.exportHeader(story
));
163 IOUtils
.writeSmallFile(dir
, story
.getSelector(),
164 gopher
.export(story
));
165 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".html",
169 // Finding headers of all stories in cache:
170 File varDir
= new File(dir
, support
.getSelector());
171 String
[] headers
= varDir
.list(new FilenameFilter() {
173 public boolean accept(File dir
, String name
) {
174 return name
.endsWith(".header");
178 // Finding which ones to show:
181 if (headers
.length
> 0) {
182 Arrays
.sort(headers
);
183 from
= headers
.length
- 1;
184 to
= headers
.length
- maxStories
;
190 // Writing the cache/index files with the stories:
191 File gopherCache
= new File(varDir
, ".cache");
192 FileWriter writer
= new FileWriter(gopherCache
);
194 for (i
= from
; i
>= to
; i
--) {
195 writer
.append(IOUtils
196 .readSmallFile(new File(varDir
, headers
[i
])));
202 File htmlIndex
= new File(varDir
, "index.html");
203 writer
= new FileWriter(htmlIndex
);
205 for (i
= from
; i
>= to
; i
--) {
206 writer
.append(IOUtils
.readSmallFile(new File(varDir
, headers
[i
]