1 package be
.nikiroo
.gofetch
;
4 import java
.io
.FileWriter
;
5 import java
.io
.FilenameFilter
;
6 import java
.io
.IOException
;
7 import java
.util
.ArrayList
;
8 import java
.util
.Arrays
;
9 import java
.util
.Collections
;
10 import java
.util
.List
;
12 import be
.nikiroo
.gofetch
.data
.Story
;
13 import be
.nikiroo
.gofetch
.output
.Gopher
;
14 import be
.nikiroo
.gofetch
.output
.Html
;
15 import be
.nikiroo
.gofetch
.output
.Output
;
16 import be
.nikiroo
.gofetch
.support
.BasicSupport
;
17 import be
.nikiroo
.gofetch
.support
.BasicSupport
.Type
;
18 import be
.nikiroo
.utils
.IOUtils
;
21 * The class that will manage the fetch operations.
23 * It will scrap the required websites and process them to disk.
27 public class Fetcher
{
29 private String preselector
;
30 private int maxStories
;
31 private String hostname
;
36 * Prepare a new {@link Fetcher}.
39 * the target directory where to save the files (won't have
40 * impact on the files' content)
42 * the sub directory and (pre-)selector to use for the resources
43 * (<b>will</b> have an impact on the files' content)
45 * the type of news to get (or the special keyword ALL to get all
46 * of the supported sources)
48 * the maximum number of stories to show on the resume page
50 * the gopher host to use (<b>will</b> have an impact on the
53 * the gopher port to use (<b>will</b> have an impact on the
56 public Fetcher(File dir
, String preselector
, Type type
, int maxStories
,
57 String hostname
, int port
) {
59 this.preselector
= preselector
;
61 this.maxStories
= maxStories
;
62 this.hostname
= hostname
;
67 * Start the fetching operation.
69 * This method will handle the main pages itself, and will call
70 * {@link Fetcher#list(BasicSupport)} for the stories.
73 * in case of I/O error
75 public void start() throws IOException
{
76 StringBuilder gopherBuilder
= new StringBuilder();
77 StringBuilder htmlBuilder
= new StringBuilder();
79 BasicSupport
.setPreselector(preselector
);
80 for (Type type
: Type
.values()) {
81 BasicSupport support
= BasicSupport
.getSupport(type
);
83 if (type
== this.type
|| this.type
== null) {
87 gopherBuilder
.append(getLink(support
.getDescription(),
88 support
.getSelector(), false));
90 String ref
= support
.getSelector();
91 while (ref
.startsWith("/")) {
92 ref
= ref
.substring(1);
94 ref
= "../" + ref
+ "/index.html";
96 htmlBuilder
.append(getLink(support
.getDescription(), ref
, true));
99 File gopherCache
= new File(dir
, preselector
);
100 gopherCache
.mkdirs();
101 File htmlIndex
= new File(gopherCache
, "index.html");
102 gopherCache
= new File(gopherCache
, ".cache");
104 Output gopher
= new Gopher(null, hostname
, preselector
, port
);
105 Output html
= new Html(null, hostname
, preselector
, port
);
107 FileWriter writer
= new FileWriter(gopherCache
);
109 writer
.append(gopher
.getIndexHeader());
110 writer
.append(gopherBuilder
.toString());
111 writer
.append(gopher
.getIndexFooter());
117 writer
= new FileWriter(htmlIndex
);
118 writer
.append(html
.getIndexHeader());
119 writer
.append(htmlBuilder
.toString());
120 writer
.append(html
.getIndexFooter());
127 * Process the stories for the given {@link BasicSupport} to disk.
130 * the {@link BasicSupport} to download from
132 * @throws IOException
133 * in case of I/O error
135 private void list(BasicSupport support
) throws IOException
{
138 .print("Listing recent news for " + support
.getType() + "...");
139 List
<Story
> stories
= support
.list();
140 System
.err
.println(" " + stories
.size() + " stories found!");
142 // Get comments (and update stories if needed):
144 for (Story story
: stories
) {
145 System
.err
.println(String
.format("%02d/%02d", i
, stories
.size())
146 + " Fetching full story " + story
.getId() + "...");
147 support
.fetch(story
);
151 Output gopher
= new Gopher(support
.getType(), hostname
, preselector
,
153 Output html
= new Html(support
.getType(), hostname
, preselector
, port
);
155 new File(dir
, support
.getSelector()).mkdirs();
157 for (Story story
: stories
) {
158 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".header",
159 gopher
.exportHeader(story
));
160 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".header.html",
161 html
.exportHeader(story
));
163 IOUtils
.writeSmallFile(dir
, story
.getSelector(),
164 gopher
.export(story
));
165 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".html",
169 // Finding headers of all stories in cache:
170 File varDir
= new File(dir
, support
.getSelector());
171 String
[] headers
= varDir
.list(new FilenameFilter() {
173 public boolean accept(File dir
, String name
) {
174 return name
.endsWith(".header");
179 Arrays
.sort(headers
);
180 List
<String
> tmp
= Arrays
.asList(headers
);
181 Collections
.reverse(tmp
);
182 headers
= tmp
.toArray(new String
[] {});
185 // Write the index (with "MORE" links if needed)
187 List
<String
> gopherLines
= new ArrayList
<String
>();
188 List
<String
> htmlLines
= new ArrayList
<String
>();
189 for (i
= 0; i
< headers
.length
; i
++) {
191 .add(IOUtils
.readSmallFile(new File(varDir
, headers
[i
])));
192 htmlLines
.add(IOUtils
.readSmallFile(new File(varDir
, headers
[i
]
195 boolean enoughStories
= (i
> 0 && i
% maxStories
== 0);
196 boolean last
= i
== headers
.length
- 1;
197 if (enoughStories
|| last
) {
199 gopherLines
.add(getLink("More", preselector
+ ".cache_"
200 + (page
+ 1), false));
201 htmlLines
.add(getLink("More", "index_" + (page
+ 1)
205 write(gopherLines
, varDir
, ".cache", "", page
);
206 write(htmlLines
, varDir
, "index", ".html", page
);
207 gopherLines
= new ArrayList
<String
>();
208 htmlLines
= new ArrayList
<String
>();
214 private void write(List
<String
> lines
, File varDir
, String basename
,
215 String ext
, int page
) throws IOException
{
216 File file
= new File(varDir
, basename
+ (page
> 0 ?
"_" + page
: "")
219 FileWriter writer
= new FileWriter(file
);
221 for (String line
: lines
) {
222 writer
.append(line
).append("\r\n");
229 private String
getLink(String name
, String ref
, boolean html
) {
231 return new StringBuilder().append("1" + name
).append("\t")
232 .append("1" + ref
) //
233 .append("\t").append(hostname
) //
234 .append("\t").append(Integer
.toString(port
)) //
235 .append("\r\n").toString();
238 return new StringBuilder().append(
239 "<div class='site'><a href='" + ref
+ "'>" + name
240 + "</a></div>\n").toString();