Commit | Line | Data |
---|---|---|
73785268 NR |
1 | package be.nikiroo.gofetch; |
2 | ||
3 | import java.io.File; | |
4 | import java.io.FileWriter; | |
5 | import java.io.FilenameFilter; | |
6 | import java.io.IOException; | |
e402343e | 7 | import java.util.ArrayList; |
73785268 | 8 | import java.util.Arrays; |
e402343e | 9 | import java.util.Collections; |
73785268 NR |
10 | import java.util.List; |
11 | ||
73785268 NR |
12 | import be.nikiroo.gofetch.data.Story; |
13 | import be.nikiroo.gofetch.output.Gopher; | |
14 | import be.nikiroo.gofetch.output.Html; | |
15 | import be.nikiroo.gofetch.output.Output; | |
16 | import be.nikiroo.gofetch.support.BasicSupport; | |
3e62b034 | 17 | import be.nikiroo.gofetch.support.Type; |
73785268 NR |
18 | import be.nikiroo.utils.IOUtils; |
19 | ||
20 | /** | |
21 | * The class that will manage the fetch operations. | |
22 | * <p> | |
23 | * It will scrap the required websites and process them to disk. | |
24 | * | |
25 | * @author niki | |
26 | */ | |
27 | public class Fetcher { | |
28 | private File dir; | |
29 | private String preselector; | |
30 | private int maxStories; | |
31 | private String hostname; | |
32 | private int port; | |
33 | private Type type; | |
34 | ||
35 | /** | |
36 | * Prepare a new {@link Fetcher}. | |
37 | * | |
38 | * @param dir | |
39 | * the target directory where to save the files (won't have | |
40 | * impact on the files' content) | |
41 | * @param preselector | |
42 | * the sub directory and (pre-)selector to use for the resources | |
43 | * (<b>will</b> have an impact on the files' content) | |
44 | * @param type | |
e570bb42 NR |
45 | * the type of news to get (or NULL to get all of the supported |
46 | * sources) | |
73785268 NR |
47 | * @param maxStories |
48 | * the maximum number of stories to show on the resume page | |
49 | * @param hostname | |
50 | * the gopher host to use (<b>will</b> have an impact on the | |
51 | * files' content) | |
52 | * @param port | |
53 | * the gopher port to use (<b>will</b> have an impact on the | |
54 | * files' content) | |
55 | */ | |
56 | public Fetcher(File dir, String preselector, Type type, int maxStories, | |
57 | String hostname, int port) { | |
58 | this.dir = dir; | |
59 | this.preselector = preselector; | |
60 | this.type = type; | |
61 | this.maxStories = maxStories; | |
62 | this.hostname = hostname; | |
63 | this.port = port; | |
64 | } | |
65 | ||
66 | /** | |
67 | * Start the fetching operation. | |
68 | * <p> | |
69 | * This method will handle the main pages itself, and will call | |
70 | * {@link Fetcher#list(BasicSupport)} for the stories. | |
71 | * | |
72 | * @throws IOException | |
73 | * in case of I/O error | |
74 | */ | |
75 | public void start() throws IOException { | |
5c056aad NR |
76 | StringBuilder gopherBuilder = new StringBuilder(); |
77 | StringBuilder htmlBuilder = new StringBuilder(); | |
78 | ||
79 | BasicSupport.setPreselector(preselector); | |
80 | for (Type type : Type.values()) { | |
81 | BasicSupport support = BasicSupport.getSupport(type); | |
82 | ||
83 | if (type == this.type || this.type == null) { | |
e570bb42 NR |
84 | try { |
85 | list(support); | |
86 | } catch (Exception e) { | |
87 | new Exception("Failed to process support: " + type, e) | |
88 | .printStackTrace(); | |
89 | } | |
5c056aad NR |
90 | } |
91 | ||
e402343e | 92 | gopherBuilder.append(getLink(support.getDescription(), |
588b54b8 | 93 | support.getSelector(), true, false)); |
5c056aad NR |
94 | |
95 | String ref = support.getSelector(); | |
96 | while (ref.startsWith("/")) { | |
97 | ref = ref.substring(1); | |
98 | } | |
e402343e NR |
99 | ref = "../" + ref + "/index.html"; |
100 | ||
136ab801 NR |
101 | htmlBuilder.append(getLink(support.getDescription(), ref, true, |
102 | true)); | |
5c056aad NR |
103 | } |
104 | ||
105 | File gopherCache = new File(dir, preselector); | |
106 | gopherCache.mkdirs(); | |
107 | File htmlIndex = new File(gopherCache, "index.html"); | |
93e09a08 | 108 | gopherCache = new File(gopherCache, "gophermap"); |
73785268 | 109 | |
70b18499 NR |
110 | Output gopher = new Gopher(null, hostname, preselector, port); |
111 | Output html = new Html(null, hostname, preselector, port); | |
73785268 | 112 | |
5c056aad | 113 | FileWriter writer = new FileWriter(gopherCache); |
73785268 | 114 | try { |
c715ea02 | 115 | writer.append(gopher.getMainIndexHeader()); |
5c056aad | 116 | writer.append(gopherBuilder.toString()); |
c715ea02 | 117 | writer.append(gopher.getMainIndexFooter()); |
5c056aad NR |
118 | } finally { |
119 | writer.close(); | |
120 | } | |
121 | ||
122 | try { | |
123 | writer = new FileWriter(htmlIndex); | |
c715ea02 | 124 | writer.append(html.getMainIndexHeader()); |
5c056aad | 125 | writer.append(htmlBuilder.toString()); |
c715ea02 | 126 | writer.append(html.getMainIndexFooter()); |
73785268 NR |
127 | } finally { |
128 | writer.close(); | |
129 | } | |
130 | } | |
131 | ||
132 | /** | |
133 | * Process the stories for the given {@link BasicSupport} to disk. | |
134 | * | |
135 | * @param support | |
136 | * the {@link BasicSupport} to download from | |
137 | * | |
138 | * @throws IOException | |
139 | * in case of I/O error | |
140 | **/ | |
141 | private void list(BasicSupport support) throws IOException { | |
5c056aad NR |
142 | // Get stories: |
143 | System.err | |
144 | .print("Listing recent news for " + support.getType() + "..."); | |
145 | List<Story> stories = support.list(); | |
146 | System.err.println(" " + stories.size() + " stories found!"); | |
147 | ||
148 | // Get comments (and update stories if needed): | |
149 | int i = 1; | |
a2d8693b | 150 | List<Story> fetchedStories = new ArrayList<Story>(stories.size()); |
5c056aad | 151 | for (Story story : stories) { |
a2d8693b | 152 | System.err.print(String.format("%02d/%02d", i, stories.size()) |
5c056aad | 153 | + " Fetching full story " + story.getId() + "..."); |
a2d8693b NR |
154 | try { |
155 | support.fetch(story); | |
156 | fetchedStories.add(story); | |
157 | System.err.println(); | |
158 | } catch (IOException e) { | |
159 | System.err.println(" Failed to get story!"); | |
160 | } | |
5c056aad NR |
161 | i++; |
162 | } | |
a2d8693b | 163 | stories = fetchedStories; |
5c056aad | 164 | |
70b18499 NR |
165 | Output gopher = new Gopher(support.getType(), hostname, preselector, |
166 | port); | |
167 | Output html = new Html(support.getType(), hostname, preselector, port); | |
73785268 NR |
168 | |
169 | new File(dir, support.getSelector()).mkdirs(); | |
170 | ||
73785268 NR |
171 | for (Story story : stories) { |
172 | IOUtils.writeSmallFile(dir, story.getSelector() + ".header", | |
5c056aad | 173 | gopher.exportHeader(story)); |
73785268 | 174 | IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html", |
5c056aad | 175 | html.exportHeader(story)); |
73785268 NR |
176 | |
177 | IOUtils.writeSmallFile(dir, story.getSelector(), | |
5c056aad | 178 | gopher.export(story)); |
73785268 | 179 | IOUtils.writeSmallFile(dir, story.getSelector() + ".html", |
5c056aad | 180 | html.export(story)); |
73785268 NR |
181 | } |
182 | ||
5c056aad | 183 | // Finding headers of all stories in cache: |
73785268 NR |
184 | File varDir = new File(dir, support.getSelector()); |
185 | String[] headers = varDir.list(new FilenameFilter() { | |
186 | @Override | |
187 | public boolean accept(File dir, String name) { | |
188 | return name.endsWith(".header"); | |
189 | } | |
190 | }); | |
191 | ||
e402343e NR |
192 | // Reverse sort: |
193 | Arrays.sort(headers); | |
194 | List<String> tmp = Arrays.asList(headers); | |
195 | Collections.reverse(tmp); | |
196 | headers = tmp.toArray(new String[] {}); | |
197 | // | |
198 | ||
c715ea02 | 199 | // Write the main index (with "MORE" links if needed) |
e402343e NR |
200 | int page = 0; |
201 | List<String> gopherLines = new ArrayList<String>(); | |
202 | List<String> htmlLines = new ArrayList<String>(); | |
c715ea02 NR |
203 | gopherLines.add(gopher.getIndexHeader(support)); |
204 | htmlLines.add(html.getIndexHeader(support)); | |
e402343e | 205 | for (i = 0; i < headers.length; i++) { |
136ab801 NR |
206 | File gopherFile = new File(varDir, headers[i]); |
207 | File htmlFile = new File(varDir, headers[i] + ".html"); | |
208 | ||
209 | if (gopherFile.exists()) | |
210 | gopherLines.add(IOUtils.readSmallFile(gopherFile)); | |
211 | if (htmlFile.exists()) | |
212 | htmlLines.add(IOUtils.readSmallFile(htmlFile)); | |
e402343e NR |
213 | |
214 | boolean enoughStories = (i > 0 && i % maxStories == 0); | |
215 | boolean last = i == headers.length - 1; | |
216 | if (enoughStories || last) { | |
217 | if (!last) { | |
136ab801 NR |
218 | gopherLines.add(getLink("More", support.getSelector() |
219 | + "gophermap_" + (page + 1), true, false)); | |
220 | ||
221 | htmlLines.add(getLink("More", "index_" + (page + 1) | |
222 | + ".html", true, true)); | |
e402343e NR |
223 | } |
224 | ||
c715ea02 NR |
225 | gopherLines.add(gopher.getIndexFooter(support)); |
226 | htmlLines.add(html.getIndexFooter(support)); | |
93e09a08 | 227 | write(gopherLines, varDir, "gophermap", "", page); |
e402343e NR |
228 | write(htmlLines, varDir, "index", ".html", page); |
229 | gopherLines = new ArrayList<String>(); | |
230 | htmlLines = new ArrayList<String>(); | |
231 | page++; | |
5c056aad NR |
232 | } |
233 | } | |
e402343e | 234 | } |
5c056aad | 235 | |
c715ea02 NR |
236 | /** |
237 | * Write an index/gophermap file with the given link content for the | |
238 | * selected supported web site. | |
239 | * | |
240 | * @param lines | |
241 | * the link content (the stories and a short description) | |
242 | * @param varDir | |
243 | * the base directory to write into | |
244 | * @param basename | |
245 | * the base file name | |
246 | * @param ext | |
247 | * the file extension (for instance, ".html") | |
248 | * @param page | |
249 | * the page number (0 = main index) | |
250 | * | |
251 | * @throws IOException | |
252 | * in case of I/O errors | |
253 | */ | |
e402343e NR |
254 | private void write(List<String> lines, File varDir, String basename, |
255 | String ext, int page) throws IOException { | |
256 | File file = new File(varDir, basename + (page > 0 ? "_" + page : "") | |
257 | + ext); | |
258 | ||
259 | FileWriter writer = new FileWriter(file); | |
5c056aad | 260 | try { |
e402343e NR |
261 | for (String line : lines) { |
262 | writer.append(line).append("\r\n"); | |
5c056aad NR |
263 | } |
264 | } finally { | |
265 | writer.close(); | |
266 | } | |
e402343e | 267 | } |
5c056aad | 268 | |
029feaed | 269 | /** |
136ab801 | 270 | * Create a link. |
029feaed NR |
271 | * |
272 | * @param name | |
136ab801 | 273 | * the link name (what the user will see) |
029feaed | 274 | * @param ref |
136ab801 | 275 | * the actual link reference (the target) |
6c13eccd | 276 | * @param menu |
136ab801 | 277 | * menu (gophermap, i) mode -- not used in html mode |
029feaed | 278 | * @param html |
136ab801 NR |
279 | * TRUE for html mode, FALSE for gopher mode |
280 | * | |
281 | * @return the ready-to-use link in a {@link String} | |
029feaed | 282 | */ |
6c13eccd | 283 | private String getLink(String name, String ref, boolean menu, boolean html) { |
e402343e | 284 | if (!html) { |
6c13eccd | 285 | return new StringBuilder().append((menu ? "1" : "0") + name) |
029feaed | 286 | .append("\t").append(ref) // |
e402343e NR |
287 | .append("\t").append(hostname) // |
288 | .append("\t").append(Integer.toString(port)) // | |
289 | .append("\r\n").toString(); | |
73785268 | 290 | } |
e402343e NR |
291 | |
292 | return new StringBuilder().append( | |
293 | "<div class='site'><a href='" + ref + "'>" + name | |
294 | + "</a></div>\n").toString(); | |
73785268 NR |
295 | } |
296 | } |