Commit | Line | Data |
---|---|---|
73785268 NR |
1 | package be.nikiroo.gofetch; |
2 | ||
3 | import java.io.File; | |
4 | import java.io.FileWriter; | |
5 | import java.io.FilenameFilter; | |
6 | import java.io.IOException; | |
e402343e | 7 | import java.util.ArrayList; |
73785268 | 8 | import java.util.Arrays; |
e402343e | 9 | import java.util.Collections; |
73785268 NR |
10 | import java.util.List; |
11 | ||
73785268 NR |
12 | import be.nikiroo.gofetch.data.Story; |
13 | import be.nikiroo.gofetch.output.Gopher; | |
14 | import be.nikiroo.gofetch.output.Html; | |
15 | import be.nikiroo.gofetch.output.Output; | |
16 | import be.nikiroo.gofetch.support.BasicSupport; | |
17 | import be.nikiroo.gofetch.support.BasicSupport.Type; | |
18 | import be.nikiroo.utils.IOUtils; | |
19 | ||
20 | /** | |
21 | * The class that will manage the fetch operations. | |
22 | * <p> | |
23 | * It will scrap the required websites and process them to disk. | |
24 | * | |
25 | * @author niki | |
26 | */ | |
27 | public class Fetcher { | |
28 | private File dir; | |
29 | private String preselector; | |
30 | private int maxStories; | |
31 | private String hostname; | |
32 | private int port; | |
33 | private Type type; | |
34 | ||
35 | /** | |
36 | * Prepare a new {@link Fetcher}. | |
37 | * | |
38 | * @param dir | |
39 | * the target directory where to save the files (won't have | |
40 | * impact on the files' content) | |
41 | * @param preselector | |
42 | * the sub directory and (pre-)selector to use for the resources | |
43 | * (<b>will</b> have an impact on the files' content) | |
44 | * @param type | |
45 | * the type of news to get (or the special keyword ALL to get all | |
46 | * of the supported sources) | |
47 | * @param maxStories | |
48 | * the maximum number of stories to show on the resume page | |
49 | * @param hostname | |
50 | * the gopher host to use (<b>will</b> have an impact on the | |
51 | * files' content) | |
52 | * @param port | |
53 | * the gopher port to use (<b>will</b> have an impact on the | |
54 | * files' content) | |
55 | */ | |
56 | public Fetcher(File dir, String preselector, Type type, int maxStories, | |
57 | String hostname, int port) { | |
58 | this.dir = dir; | |
59 | this.preselector = preselector; | |
60 | this.type = type; | |
61 | this.maxStories = maxStories; | |
62 | this.hostname = hostname; | |
63 | this.port = port; | |
64 | } | |
65 | ||
66 | /** | |
67 | * Start the fetching operation. | |
68 | * <p> | |
69 | * This method will handle the main pages itself, and will call | |
70 | * {@link Fetcher#list(BasicSupport)} for the stories. | |
71 | * | |
72 | * @throws IOException | |
73 | * in case of I/O error | |
74 | */ | |
75 | public void start() throws IOException { | |
5c056aad NR |
76 | StringBuilder gopherBuilder = new StringBuilder(); |
77 | StringBuilder htmlBuilder = new StringBuilder(); | |
78 | ||
79 | BasicSupport.setPreselector(preselector); | |
80 | for (Type type : Type.values()) { | |
81 | BasicSupport support = BasicSupport.getSupport(type); | |
82 | ||
83 | if (type == this.type || this.type == null) { | |
84 | list(support); | |
85 | } | |
86 | ||
e402343e NR |
87 | gopherBuilder.append(getLink(support.getDescription(), |
88 | support.getSelector(), false)); | |
5c056aad NR |
89 | |
90 | String ref = support.getSelector(); | |
91 | while (ref.startsWith("/")) { | |
92 | ref = ref.substring(1); | |
93 | } | |
e402343e NR |
94 | ref = "../" + ref + "/index.html"; |
95 | ||
96 | htmlBuilder.append(getLink(support.getDescription(), ref, true)); | |
5c056aad NR |
97 | } |
98 | ||
99 | File gopherCache = new File(dir, preselector); | |
100 | gopherCache.mkdirs(); | |
101 | File htmlIndex = new File(gopherCache, "index.html"); | |
102 | gopherCache = new File(gopherCache, ".cache"); | |
73785268 | 103 | |
70b18499 NR |
104 | Output gopher = new Gopher(null, hostname, preselector, port); |
105 | Output html = new Html(null, hostname, preselector, port); | |
73785268 | 106 | |
5c056aad | 107 | FileWriter writer = new FileWriter(gopherCache); |
73785268 | 108 | try { |
5c056aad NR |
109 | writer.append(gopher.getIndexHeader()); |
110 | writer.append(gopherBuilder.toString()); | |
111 | writer.append(gopher.getIndexFooter()); | |
112 | } finally { | |
113 | writer.close(); | |
114 | } | |
115 | ||
116 | try { | |
117 | writer = new FileWriter(htmlIndex); | |
118 | writer.append(html.getIndexHeader()); | |
119 | writer.append(htmlBuilder.toString()); | |
120 | writer.append(html.getIndexFooter()); | |
73785268 NR |
121 | } finally { |
122 | writer.close(); | |
123 | } | |
124 | } | |
125 | ||
126 | /** | |
127 | * Process the stories for the given {@link BasicSupport} to disk. | |
128 | * | |
129 | * @param support | |
130 | * the {@link BasicSupport} to download from | |
131 | * | |
132 | * @throws IOException | |
133 | * in case of I/O error | |
134 | **/ | |
135 | private void list(BasicSupport support) throws IOException { | |
5c056aad NR |
136 | // Get stories: |
137 | System.err | |
138 | .print("Listing recent news for " + support.getType() + "..."); | |
139 | List<Story> stories = support.list(); | |
140 | System.err.println(" " + stories.size() + " stories found!"); | |
141 | ||
142 | // Get comments (and update stories if needed): | |
143 | int i = 1; | |
144 | for (Story story : stories) { | |
145 | System.err.println(String.format("%02d/%02d", i, stories.size()) | |
146 | + " Fetching full story " + story.getId() + "..."); | |
147 | support.fetch(story); | |
148 | i++; | |
149 | } | |
150 | ||
70b18499 NR |
151 | Output gopher = new Gopher(support.getType(), hostname, preselector, |
152 | port); | |
153 | Output html = new Html(support.getType(), hostname, preselector, port); | |
73785268 NR |
154 | |
155 | new File(dir, support.getSelector()).mkdirs(); | |
156 | ||
73785268 NR |
157 | for (Story story : stories) { |
158 | IOUtils.writeSmallFile(dir, story.getSelector() + ".header", | |
5c056aad | 159 | gopher.exportHeader(story)); |
73785268 | 160 | IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html", |
5c056aad | 161 | html.exportHeader(story)); |
73785268 NR |
162 | |
163 | IOUtils.writeSmallFile(dir, story.getSelector(), | |
5c056aad | 164 | gopher.export(story)); |
73785268 | 165 | IOUtils.writeSmallFile(dir, story.getSelector() + ".html", |
5c056aad | 166 | html.export(story)); |
73785268 NR |
167 | } |
168 | ||
5c056aad | 169 | // Finding headers of all stories in cache: |
73785268 NR |
170 | File varDir = new File(dir, support.getSelector()); |
171 | String[] headers = varDir.list(new FilenameFilter() { | |
172 | @Override | |
173 | public boolean accept(File dir, String name) { | |
174 | return name.endsWith(".header"); | |
175 | } | |
176 | }); | |
177 | ||
e402343e NR |
178 | // Reverse sort: |
179 | Arrays.sort(headers); | |
180 | List<String> tmp = Arrays.asList(headers); | |
181 | Collections.reverse(tmp); | |
182 | headers = tmp.toArray(new String[] {}); | |
183 | // | |
184 | ||
185 | // Write the index (with "MORE" links if needed) | |
186 | int page = 0; | |
187 | List<String> gopherLines = new ArrayList<String>(); | |
188 | List<String> htmlLines = new ArrayList<String>(); | |
189 | for (i = 0; i < headers.length; i++) { | |
190 | gopherLines | |
191 | .add(IOUtils.readSmallFile(new File(varDir, headers[i]))); | |
192 | htmlLines.add(IOUtils.readSmallFile(new File(varDir, headers[i] | |
193 | + ".html"))); | |
194 | ||
195 | boolean enoughStories = (i > 0 && i % maxStories == 0); | |
196 | boolean last = i == headers.length - 1; | |
197 | if (enoughStories || last) { | |
198 | if (!last) { | |
8a62bdfd NR |
199 | gopherLines.add(getLink("More", preselector + ".cache_" |
200 | + (page + 1), false)); | |
e402343e NR |
201 | htmlLines.add(getLink("More", "index_" + (page + 1) |
202 | + ".html", true)); | |
203 | } | |
204 | ||
205 | write(gopherLines, varDir, ".cache", "", page); | |
206 | write(htmlLines, varDir, "index", ".html", page); | |
207 | gopherLines = new ArrayList<String>(); | |
208 | htmlLines = new ArrayList<String>(); | |
209 | page++; | |
5c056aad NR |
210 | } |
211 | } | |
e402343e | 212 | } |
5c056aad | 213 | |
e402343e NR |
214 | private void write(List<String> lines, File varDir, String basename, |
215 | String ext, int page) throws IOException { | |
216 | File file = new File(varDir, basename + (page > 0 ? "_" + page : "") | |
217 | + ext); | |
218 | ||
219 | FileWriter writer = new FileWriter(file); | |
5c056aad | 220 | try { |
e402343e NR |
221 | for (String line : lines) { |
222 | writer.append(line).append("\r\n"); | |
5c056aad NR |
223 | } |
224 | } finally { | |
225 | writer.close(); | |
226 | } | |
e402343e | 227 | } |
5c056aad | 228 | |
e402343e NR |
229 | private String getLink(String name, String ref, boolean html) { |
230 | if (!html) { | |
231 | return new StringBuilder().append("1" + name).append("\t") | |
232 | .append("1" + ref) // | |
233 | .append("\t").append(hostname) // | |
234 | .append("\t").append(Integer.toString(port)) // | |
235 | .append("\r\n").toString(); | |
73785268 | 236 | } |
e402343e NR |
237 | |
238 | return new StringBuilder().append( | |
239 | "<div class='site'><a href='" + ref + "'>" + name | |
240 | + "</a></div>\n").toString(); | |
73785268 NR |
241 | } |
242 | } |