1 package be
.nikiroo
.gofetch
;
4 import java
.io
.FileWriter
;
5 import java
.io
.FilenameFilter
;
6 import java
.io
.IOException
;
7 import java
.util
.ArrayList
;
8 import java
.util
.Arrays
;
9 import java
.util
.Collections
;
10 import java
.util
.List
;
12 import be
.nikiroo
.gofetch
.data
.Story
;
13 import be
.nikiroo
.gofetch
.output
.Gopher
;
14 import be
.nikiroo
.gofetch
.output
.Html
;
15 import be
.nikiroo
.gofetch
.output
.Output
;
16 import be
.nikiroo
.gofetch
.support
.BasicSupport
;
17 import be
.nikiroo
.gofetch
.support
.BasicSupport
.Type
;
18 import be
.nikiroo
.utils
.IOUtils
;
21 * The class that will manage the fetch operations.
23 * It will scrap the required websites and process them to disk.
27 public class Fetcher
{
29 private String preselector
;
30 private int maxStories
;
31 private String hostname
;
36 * Prepare a new {@link Fetcher}.
39 * the target directory where to save the files (won't have
40 * impact on the files' content)
42 * the sub directory and (pre-)selector to use for the resources
43 * (<b>will</b> have an impact on the files' content)
45 * the type of news to get (or NULL to get all of the supported
48 * the maximum number of stories to show on the resume page
50 * the gopher host to use (<b>will</b> have an impact on the
53 * the gopher port to use (<b>will</b> have an impact on the
56 public Fetcher(File dir
, String preselector
, Type type
, int maxStories
,
57 String hostname
, int port
) {
59 this.preselector
= preselector
;
61 this.maxStories
= maxStories
;
62 this.hostname
= hostname
;
67 * Start the fetching operation.
69 * This method will handle the main pages itself, and will call
70 * {@link Fetcher#list(BasicSupport)} for the stories.
73 * in case of I/O error
75 public void start() throws IOException
{
76 StringBuilder gopherBuilder
= new StringBuilder();
77 StringBuilder htmlBuilder
= new StringBuilder();
79 BasicSupport
.setPreselector(preselector
);
80 for (Type type
: Type
.values()) {
81 BasicSupport support
= BasicSupport
.getSupport(type
);
83 if (type
== this.type
|| this.type
== null) {
86 } catch (Exception e
) {
87 new Exception("Failed to process support: " + type
, e
)
92 gopherBuilder
.append(getLink(support
.getDescription(),
93 support
.getSelector(), true, false));
95 String ref
= support
.getSelector();
96 while (ref
.startsWith("/")) {
97 ref
= ref
.substring(1);
99 ref
= "../" + ref
+ "/index.html";
101 htmlBuilder
.append(getLink(support
.getDescription(), ref
, true,
105 File gopherCache
= new File(dir
, preselector
);
106 gopherCache
.mkdirs();
107 File htmlIndex
= new File(gopherCache
, "index.html");
108 gopherCache
= new File(gopherCache
, "gophermap");
110 Output gopher
= new Gopher(null, hostname
, preselector
, port
);
111 Output html
= new Html(null, hostname
, preselector
, port
);
113 FileWriter writer
= new FileWriter(gopherCache
);
115 writer
.append(gopher
.getIndexHeader());
116 writer
.append(gopherBuilder
.toString());
117 writer
.append(gopher
.getIndexFooter());
123 writer
= new FileWriter(htmlIndex
);
124 writer
.append(html
.getIndexHeader());
125 writer
.append(htmlBuilder
.toString());
126 writer
.append(html
.getIndexFooter());
133 * Process the stories for the given {@link BasicSupport} to disk.
136 * the {@link BasicSupport} to download from
138 * @throws IOException
139 * in case of I/O error
141 private void list(BasicSupport support
) throws IOException
{
144 .print("Listing recent news for " + support
.getType() + "...");
145 List
<Story
> stories
= support
.list();
146 System
.err
.println(" " + stories
.size() + " stories found!");
148 // Get comments (and update stories if needed):
150 for (Story story
: stories
) {
151 System
.err
.println(String
.format("%02d/%02d", i
, stories
.size())
152 + " Fetching full story " + story
.getId() + "...");
153 support
.fetch(story
);
157 Output gopher
= new Gopher(support
.getType(), hostname
, preselector
,
159 Output html
= new Html(support
.getType(), hostname
, preselector
, port
);
161 new File(dir
, support
.getSelector()).mkdirs();
163 for (Story story
: stories
) {
164 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".header",
165 gopher
.exportHeader(story
));
166 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".header.html",
167 html
.exportHeader(story
));
169 IOUtils
.writeSmallFile(dir
, story
.getSelector(),
170 gopher
.export(story
));
171 IOUtils
.writeSmallFile(dir
, story
.getSelector() + ".html",
175 // Finding headers of all stories in cache:
176 File varDir
= new File(dir
, support
.getSelector());
177 String
[] headers
= varDir
.list(new FilenameFilter() {
179 public boolean accept(File dir
, String name
) {
180 return name
.endsWith(".header");
185 Arrays
.sort(headers
);
186 List
<String
> tmp
= Arrays
.asList(headers
);
187 Collections
.reverse(tmp
);
188 headers
= tmp
.toArray(new String
[] {});
191 // Write the index (with "MORE" links if needed)
193 List
<String
> gopherLines
= new ArrayList
<String
>();
194 List
<String
> htmlLines
= new ArrayList
<String
>();
195 for (i
= 0; i
< headers
.length
; i
++) {
196 File gopherFile
= new File(varDir
, headers
[i
]);
197 File htmlFile
= new File(varDir
, headers
[i
] + ".html");
199 if (gopherFile
.exists())
200 gopherLines
.add(IOUtils
.readSmallFile(gopherFile
));
201 if (htmlFile
.exists())
202 htmlLines
.add(IOUtils
.readSmallFile(htmlFile
));
204 boolean enoughStories
= (i
> 0 && i
% maxStories
== 0);
205 boolean last
= i
== headers
.length
- 1;
206 if (enoughStories
|| last
) {
208 gopherLines
.add(getLink("More", support
.getSelector()
209 + "gophermap_" + (page
+ 1), true, false));
211 htmlLines
.add(getLink("More", "index_" + (page
+ 1)
212 + ".html", true, true));
215 write(gopherLines
, varDir
, "gophermap", "", page
);
216 write(htmlLines
, varDir
, "index", ".html", page
);
217 gopherLines
= new ArrayList
<String
>();
218 htmlLines
= new ArrayList
<String
>();
224 private void write(List
<String
> lines
, File varDir
, String basename
,
225 String ext
, int page
) throws IOException
{
226 File file
= new File(varDir
, basename
+ (page
> 0 ?
"_" + page
: "")
229 FileWriter writer
= new FileWriter(file
);
231 for (String line
: lines
) {
232 writer
.append(line
).append("\r\n");
243 * the link name (what the user will see)
245 * the actual link reference (the target)
247 * menu (gophermap, i) mode -- not used in html mode
249 * TRUE for html mode, FALSE for gopher mode
251 * @return the ready-to-use link in a {@link String}
253 private String
getLink(String name
, String ref
, boolean menu
, boolean html
) {
255 return new StringBuilder().append((menu ?
"1" : "0") + name
)
256 .append("\t").append(ref
) //
257 .append("\t").append(hostname
) //
258 .append("\t").append(Integer
.toString(port
)) //
259 .append("\r\n").toString();
262 return new StringBuilder().append(
263 "<div class='site'><a href='" + ref
+ "'>" + name
264 + "</a></div>\n").toString();