Commit | Line | Data |
---|---|---|
73785268 NR |
1 | package be.nikiroo.gofetch; |
2 | ||
3 | import java.io.File; | |
4 | import java.io.FileWriter; | |
5 | import java.io.FilenameFilter; | |
6 | import java.io.IOException; | |
e402343e | 7 | import java.util.ArrayList; |
73785268 | 8 | import java.util.Arrays; |
e402343e | 9 | import java.util.Collections; |
73785268 NR |
10 | import java.util.List; |
11 | ||
73785268 NR |
12 | import be.nikiroo.gofetch.data.Story; |
13 | import be.nikiroo.gofetch.output.Gopher; | |
14 | import be.nikiroo.gofetch.output.Html; | |
15 | import be.nikiroo.gofetch.output.Output; | |
16 | import be.nikiroo.gofetch.support.BasicSupport; | |
17 | import be.nikiroo.gofetch.support.BasicSupport.Type; | |
18 | import be.nikiroo.utils.IOUtils; | |
19 | ||
20 | /** | |
21 | * The class that will manage the fetch operations. | |
22 | * <p> | |
23 | * It will scrap the required websites and process them to disk. | |
24 | * | |
25 | * @author niki | |
26 | */ | |
27 | public class Fetcher { | |
28 | private File dir; | |
29 | private String preselector; | |
30 | private int maxStories; | |
31 | private String hostname; | |
32 | private int port; | |
33 | private Type type; | |
34 | ||
35 | /** | |
36 | * Prepare a new {@link Fetcher}. | |
37 | * | |
38 | * @param dir | |
39 | * the target directory where to save the files (won't have | |
40 | * impact on the files' content) | |
41 | * @param preselector | |
42 | * the sub directory and (pre-)selector to use for the resources | |
43 | * (<b>will</b> have an impact on the files' content) | |
44 | * @param type | |
e570bb42 NR |
45 | * the type of news to get (or NULL to get all of the supported |
46 | * sources) | |
73785268 NR |
47 | * @param maxStories |
48 | * the maximum number of stories to show on the resume page | |
49 | * @param hostname | |
50 | * the gopher host to use (<b>will</b> have an impact on the | |
51 | * files' content) | |
52 | * @param port | |
53 | * the gopher port to use (<b>will</b> have an impact on the | |
54 | * files' content) | |
55 | */ | |
56 | public Fetcher(File dir, String preselector, Type type, int maxStories, | |
57 | String hostname, int port) { | |
58 | this.dir = dir; | |
59 | this.preselector = preselector; | |
60 | this.type = type; | |
61 | this.maxStories = maxStories; | |
62 | this.hostname = hostname; | |
63 | this.port = port; | |
64 | } | |
65 | ||
66 | /** | |
67 | * Start the fetching operation. | |
68 | * <p> | |
69 | * This method will handle the main pages itself, and will call | |
70 | * {@link Fetcher#list(BasicSupport)} for the stories. | |
71 | * | |
72 | * @throws IOException | |
73 | * in case of I/O error | |
74 | */ | |
75 | public void start() throws IOException { | |
5c056aad NR |
76 | StringBuilder gopherBuilder = new StringBuilder(); |
77 | StringBuilder htmlBuilder = new StringBuilder(); | |
78 | ||
79 | BasicSupport.setPreselector(preselector); | |
80 | for (Type type : Type.values()) { | |
81 | BasicSupport support = BasicSupport.getSupport(type); | |
82 | ||
83 | if (type == this.type || this.type == null) { | |
e570bb42 NR |
84 | try { |
85 | list(support); | |
86 | } catch (Exception e) { | |
87 | new Exception("Failed to process support: " + type, e) | |
88 | .printStackTrace(); | |
89 | } | |
5c056aad NR |
90 | } |
91 | ||
e402343e | 92 | gopherBuilder.append(getLink(support.getDescription(), |
588b54b8 | 93 | support.getSelector(), true, false)); |
5c056aad NR |
94 | |
95 | String ref = support.getSelector(); | |
96 | while (ref.startsWith("/")) { | |
97 | ref = ref.substring(1); | |
98 | } | |
e402343e NR |
99 | ref = "../" + ref + "/index.html"; |
100 | ||
136ab801 NR |
101 | htmlBuilder.append(getLink(support.getDescription(), ref, true, |
102 | true)); | |
5c056aad NR |
103 | } |
104 | ||
105 | File gopherCache = new File(dir, preselector); | |
106 | gopherCache.mkdirs(); | |
107 | File htmlIndex = new File(gopherCache, "index.html"); | |
93e09a08 | 108 | gopherCache = new File(gopherCache, "gophermap"); |
73785268 | 109 | |
70b18499 NR |
110 | Output gopher = new Gopher(null, hostname, preselector, port); |
111 | Output html = new Html(null, hostname, preselector, port); | |
73785268 | 112 | |
5c056aad | 113 | FileWriter writer = new FileWriter(gopherCache); |
73785268 | 114 | try { |
5c056aad NR |
115 | writer.append(gopher.getIndexHeader()); |
116 | writer.append(gopherBuilder.toString()); | |
117 | writer.append(gopher.getIndexFooter()); | |
118 | } finally { | |
119 | writer.close(); | |
120 | } | |
121 | ||
122 | try { | |
123 | writer = new FileWriter(htmlIndex); | |
124 | writer.append(html.getIndexHeader()); | |
125 | writer.append(htmlBuilder.toString()); | |
126 | writer.append(html.getIndexFooter()); | |
73785268 NR |
127 | } finally { |
128 | writer.close(); | |
129 | } | |
130 | } | |
131 | ||
132 | /** | |
133 | * Process the stories for the given {@link BasicSupport} to disk. | |
134 | * | |
135 | * @param support | |
136 | * the {@link BasicSupport} to download from | |
137 | * | |
138 | * @throws IOException | |
139 | * in case of I/O error | |
140 | **/ | |
141 | private void list(BasicSupport support) throws IOException { | |
5c056aad NR |
142 | // Get stories: |
143 | System.err | |
144 | .print("Listing recent news for " + support.getType() + "..."); | |
145 | List<Story> stories = support.list(); | |
146 | System.err.println(" " + stories.size() + " stories found!"); | |
147 | ||
148 | // Get comments (and update stories if needed): | |
149 | int i = 1; | |
150 | for (Story story : stories) { | |
151 | System.err.println(String.format("%02d/%02d", i, stories.size()) | |
152 | + " Fetching full story " + story.getId() + "..."); | |
153 | support.fetch(story); | |
154 | i++; | |
155 | } | |
156 | ||
70b18499 NR |
157 | Output gopher = new Gopher(support.getType(), hostname, preselector, |
158 | port); | |
159 | Output html = new Html(support.getType(), hostname, preselector, port); | |
73785268 NR |
160 | |
161 | new File(dir, support.getSelector()).mkdirs(); | |
162 | ||
73785268 NR |
163 | for (Story story : stories) { |
164 | IOUtils.writeSmallFile(dir, story.getSelector() + ".header", | |
5c056aad | 165 | gopher.exportHeader(story)); |
73785268 | 166 | IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html", |
5c056aad | 167 | html.exportHeader(story)); |
73785268 NR |
168 | |
169 | IOUtils.writeSmallFile(dir, story.getSelector(), | |
5c056aad | 170 | gopher.export(story)); |
73785268 | 171 | IOUtils.writeSmallFile(dir, story.getSelector() + ".html", |
5c056aad | 172 | html.export(story)); |
73785268 NR |
173 | } |
174 | ||
5c056aad | 175 | // Finding headers of all stories in cache: |
73785268 NR |
176 | File varDir = new File(dir, support.getSelector()); |
177 | String[] headers = varDir.list(new FilenameFilter() { | |
178 | @Override | |
179 | public boolean accept(File dir, String name) { | |
180 | return name.endsWith(".header"); | |
181 | } | |
182 | }); | |
183 | ||
e402343e NR |
184 | // Reverse sort: |
185 | Arrays.sort(headers); | |
186 | List<String> tmp = Arrays.asList(headers); | |
187 | Collections.reverse(tmp); | |
188 | headers = tmp.toArray(new String[] {}); | |
189 | // | |
190 | ||
191 | // Write the index (with "MORE" links if needed) | |
192 | int page = 0; | |
193 | List<String> gopherLines = new ArrayList<String>(); | |
194 | List<String> htmlLines = new ArrayList<String>(); | |
195 | for (i = 0; i < headers.length; i++) { | |
136ab801 NR |
196 | File gopherFile = new File(varDir, headers[i]); |
197 | File htmlFile = new File(varDir, headers[i] + ".html"); | |
198 | ||
199 | if (gopherFile.exists()) | |
200 | gopherLines.add(IOUtils.readSmallFile(gopherFile)); | |
201 | if (htmlFile.exists()) | |
202 | htmlLines.add(IOUtils.readSmallFile(htmlFile)); | |
e402343e NR |
203 | |
204 | boolean enoughStories = (i > 0 && i % maxStories == 0); | |
205 | boolean last = i == headers.length - 1; | |
206 | if (enoughStories || last) { | |
207 | if (!last) { | |
136ab801 NR |
208 | gopherLines.add(getLink("More", support.getSelector() |
209 | + "gophermap_" + (page + 1), true, false)); | |
210 | ||
211 | htmlLines.add(getLink("More", "index_" + (page + 1) | |
212 | + ".html", true, true)); | |
e402343e NR |
213 | } |
214 | ||
93e09a08 | 215 | write(gopherLines, varDir, "gophermap", "", page); |
e402343e NR |
216 | write(htmlLines, varDir, "index", ".html", page); |
217 | gopherLines = new ArrayList<String>(); | |
218 | htmlLines = new ArrayList<String>(); | |
219 | page++; | |
5c056aad NR |
220 | } |
221 | } | |
e402343e | 222 | } |
5c056aad | 223 | |
e402343e NR |
224 | private void write(List<String> lines, File varDir, String basename, |
225 | String ext, int page) throws IOException { | |
226 | File file = new File(varDir, basename + (page > 0 ? "_" + page : "") | |
227 | + ext); | |
228 | ||
229 | FileWriter writer = new FileWriter(file); | |
5c056aad | 230 | try { |
e402343e NR |
231 | for (String line : lines) { |
232 | writer.append(line).append("\r\n"); | |
5c056aad NR |
233 | } |
234 | } finally { | |
235 | writer.close(); | |
236 | } | |
e402343e | 237 | } |
5c056aad | 238 | |
029feaed | 239 | /** |
136ab801 | 240 | * Create a link. |
029feaed NR |
241 | * |
242 | * @param name | |
136ab801 | 243 | * the link name (what the user will see) |
029feaed | 244 | * @param ref |
136ab801 | 245 | * the actual link reference (the target) |
6c13eccd | 246 | * @param menu |
136ab801 | 247 | * menu (gophermap, i) mode -- not used in html mode |
029feaed | 248 | * @param html |
136ab801 NR |
249 | * TRUE for html mode, FALSE for gopher mode |
250 | * | |
251 | * @return the ready-to-use link in a {@link String} | |
029feaed | 252 | */ |
6c13eccd | 253 | private String getLink(String name, String ref, boolean menu, boolean html) { |
e402343e | 254 | if (!html) { |
6c13eccd | 255 | return new StringBuilder().append((menu ? "1" : "0") + name) |
029feaed | 256 | .append("\t").append(ref) // |
e402343e NR |
257 | .append("\t").append(hostname) // |
258 | .append("\t").append(Integer.toString(port)) // | |
259 | .append("\r\n").toString(); | |
73785268 | 260 | } |
e402343e NR |
261 | |
262 | return new StringBuilder().append( | |
263 | "<div class='site'><a href='" + ref + "'>" + name | |
264 | + "</a></div>\n").toString(); | |
73785268 NR |
265 | } |
266 | } |