Add title in index pages, add reference in story
[gofetch.git] / src / be / nikiroo / gofetch / Fetcher.java
CommitLineData
73785268
NR
1package be.nikiroo.gofetch;
2
3import java.io.File;
4import java.io.FileWriter;
5import java.io.FilenameFilter;
6import java.io.IOException;
e402343e 7import java.util.ArrayList;
73785268 8import java.util.Arrays;
e402343e 9import java.util.Collections;
73785268
NR
10import java.util.List;
11
73785268
NR
12import be.nikiroo.gofetch.data.Story;
13import be.nikiroo.gofetch.output.Gopher;
14import be.nikiroo.gofetch.output.Html;
15import be.nikiroo.gofetch.output.Output;
16import be.nikiroo.gofetch.support.BasicSupport;
3e62b034 17import be.nikiroo.gofetch.support.Type;
73785268
NR
18import be.nikiroo.utils.IOUtils;
19
20/**
21 * The class that will manage the fetch operations.
22 * <p>
23 * It will scrap the required websites and process them to disk.
24 *
25 * @author niki
26 */
27public class Fetcher {
28 private File dir;
29 private String preselector;
30 private int maxStories;
31 private String hostname;
32 private int port;
33 private Type type;
34
35 /**
36 * Prepare a new {@link Fetcher}.
37 *
38 * @param dir
39 * the target directory where to save the files (won't have
40 * impact on the files' content)
41 * @param preselector
42 * the sub directory and (pre-)selector to use for the resources
43 * (<b>will</b> have an impact on the files' content)
44 * @param type
e570bb42
NR
45 * the type of news to get (or NULL to get all of the supported
46 * sources)
73785268
NR
47 * @param maxStories
48 * the maximum number of stories to show on the resume page
49 * @param hostname
50 * the gopher host to use (<b>will</b> have an impact on the
51 * files' content)
52 * @param port
53 * the gopher port to use (<b>will</b> have an impact on the
54 * files' content)
55 */
56 public Fetcher(File dir, String preselector, Type type, int maxStories,
57 String hostname, int port) {
58 this.dir = dir;
59 this.preselector = preselector;
60 this.type = type;
61 this.maxStories = maxStories;
62 this.hostname = hostname;
63 this.port = port;
64 }
65
66 /**
67 * Start the fetching operation.
68 * <p>
69 * This method will handle the main pages itself, and will call
70 * {@link Fetcher#list(BasicSupport)} for the stories.
71 *
72 * @throws IOException
73 * in case of I/O error
74 */
75 public void start() throws IOException {
5c056aad
NR
76 StringBuilder gopherBuilder = new StringBuilder();
77 StringBuilder htmlBuilder = new StringBuilder();
78
79 BasicSupport.setPreselector(preselector);
80 for (Type type : Type.values()) {
81 BasicSupport support = BasicSupport.getSupport(type);
82
83 if (type == this.type || this.type == null) {
e570bb42
NR
84 try {
85 list(support);
86 } catch (Exception e) {
87 new Exception("Failed to process support: " + type, e)
88 .printStackTrace();
89 }
5c056aad
NR
90 }
91
e402343e 92 gopherBuilder.append(getLink(support.getDescription(),
588b54b8 93 support.getSelector(), true, false));
5c056aad
NR
94
95 String ref = support.getSelector();
96 while (ref.startsWith("/")) {
97 ref = ref.substring(1);
98 }
e402343e
NR
99 ref = "../" + ref + "/index.html";
100
136ab801
NR
101 htmlBuilder.append(getLink(support.getDescription(), ref, true,
102 true));
5c056aad
NR
103 }
104
105 File gopherCache = new File(dir, preselector);
106 gopherCache.mkdirs();
107 File htmlIndex = new File(gopherCache, "index.html");
93e09a08 108 gopherCache = new File(gopherCache, "gophermap");
73785268 109
70b18499
NR
110 Output gopher = new Gopher(null, hostname, preselector, port);
111 Output html = new Html(null, hostname, preselector, port);
73785268 112
5c056aad 113 FileWriter writer = new FileWriter(gopherCache);
73785268 114 try {
c715ea02 115 writer.append(gopher.getMainIndexHeader());
5c056aad 116 writer.append(gopherBuilder.toString());
c715ea02 117 writer.append(gopher.getMainIndexFooter());
5c056aad
NR
118 } finally {
119 writer.close();
120 }
121
122 try {
123 writer = new FileWriter(htmlIndex);
c715ea02 124 writer.append(html.getMainIndexHeader());
5c056aad 125 writer.append(htmlBuilder.toString());
c715ea02 126 writer.append(html.getMainIndexFooter());
73785268
NR
127 } finally {
128 writer.close();
129 }
130 }
131
132 /**
133 * Process the stories for the given {@link BasicSupport} to disk.
134 *
135 * @param support
136 * the {@link BasicSupport} to download from
137 *
138 * @throws IOException
139 * in case of I/O error
140 **/
141 private void list(BasicSupport support) throws IOException {
5c056aad
NR
142 // Get stories:
143 System.err
144 .print("Listing recent news for " + support.getType() + "...");
145 List<Story> stories = support.list();
146 System.err.println(" " + stories.size() + " stories found!");
147
148 // Get comments (and update stories if needed):
149 int i = 1;
a2d8693b 150 List<Story> fetchedStories = new ArrayList<Story>(stories.size());
5c056aad 151 for (Story story : stories) {
a2d8693b 152 System.err.print(String.format("%02d/%02d", i, stories.size())
5c056aad 153 + " Fetching full story " + story.getId() + "...");
a2d8693b
NR
154 try {
155 support.fetch(story);
156 fetchedStories.add(story);
157 System.err.println();
158 } catch (IOException e) {
159 System.err.println(" Failed to get story!");
160 }
5c056aad
NR
161 i++;
162 }
a2d8693b 163 stories = fetchedStories;
5c056aad 164
70b18499
NR
165 Output gopher = new Gopher(support.getType(), hostname, preselector,
166 port);
167 Output html = new Html(support.getType(), hostname, preselector, port);
73785268
NR
168
169 new File(dir, support.getSelector()).mkdirs();
170
73785268
NR
171 for (Story story : stories) {
172 IOUtils.writeSmallFile(dir, story.getSelector() + ".header",
5c056aad 173 gopher.exportHeader(story));
73785268 174 IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html",
5c056aad 175 html.exportHeader(story));
73785268
NR
176
177 IOUtils.writeSmallFile(dir, story.getSelector(),
5c056aad 178 gopher.export(story));
73785268 179 IOUtils.writeSmallFile(dir, story.getSelector() + ".html",
5c056aad 180 html.export(story));
73785268
NR
181 }
182
5c056aad 183 // Finding headers of all stories in cache:
73785268
NR
184 File varDir = new File(dir, support.getSelector());
185 String[] headers = varDir.list(new FilenameFilter() {
186 @Override
187 public boolean accept(File dir, String name) {
188 return name.endsWith(".header");
189 }
190 });
191
e402343e
NR
192 // Reverse sort:
193 Arrays.sort(headers);
194 List<String> tmp = Arrays.asList(headers);
195 Collections.reverse(tmp);
196 headers = tmp.toArray(new String[] {});
197 //
198
c715ea02 199 // Write the main index (with "MORE" links if needed)
e402343e
NR
200 int page = 0;
201 List<String> gopherLines = new ArrayList<String>();
202 List<String> htmlLines = new ArrayList<String>();
c715ea02
NR
203 gopherLines.add(gopher.getIndexHeader(support));
204 htmlLines.add(html.getIndexHeader(support));
e402343e 205 for (i = 0; i < headers.length; i++) {
136ab801
NR
206 File gopherFile = new File(varDir, headers[i]);
207 File htmlFile = new File(varDir, headers[i] + ".html");
208
209 if (gopherFile.exists())
210 gopherLines.add(IOUtils.readSmallFile(gopherFile));
211 if (htmlFile.exists())
212 htmlLines.add(IOUtils.readSmallFile(htmlFile));
e402343e
NR
213
214 boolean enoughStories = (i > 0 && i % maxStories == 0);
215 boolean last = i == headers.length - 1;
216 if (enoughStories || last) {
217 if (!last) {
136ab801
NR
218 gopherLines.add(getLink("More", support.getSelector()
219 + "gophermap_" + (page + 1), true, false));
220
221 htmlLines.add(getLink("More", "index_" + (page + 1)
222 + ".html", true, true));
e402343e
NR
223 }
224
c715ea02
NR
225 gopherLines.add(gopher.getIndexFooter(support));
226 htmlLines.add(html.getIndexFooter(support));
93e09a08 227 write(gopherLines, varDir, "gophermap", "", page);
e402343e
NR
228 write(htmlLines, varDir, "index", ".html", page);
229 gopherLines = new ArrayList<String>();
230 htmlLines = new ArrayList<String>();
231 page++;
5c056aad
NR
232 }
233 }
e402343e 234 }
5c056aad 235
c715ea02
NR
236 /**
237 * Write an index/gophermap file with the given link content for the
238 * selected supported web site.
239 *
240 * @param lines
241 * the link content (the stories and a short description)
242 * @param varDir
243 * the base directory to write into
244 * @param basename
245 * the base file name
246 * @param ext
247 * the file extension (for instance, ".html")
248 * @param page
249 * the page number (0 = main index)
250 *
251 * @throws IOException
252 * in case of I/O errors
253 */
e402343e
NR
254 private void write(List<String> lines, File varDir, String basename,
255 String ext, int page) throws IOException {
256 File file = new File(varDir, basename + (page > 0 ? "_" + page : "")
257 + ext);
258
259 FileWriter writer = new FileWriter(file);
5c056aad 260 try {
e402343e
NR
261 for (String line : lines) {
262 writer.append(line).append("\r\n");
5c056aad
NR
263 }
264 } finally {
265 writer.close();
266 }
e402343e 267 }
5c056aad 268
029feaed 269 /**
136ab801 270 * Create a link.
029feaed
NR
271 *
272 * @param name
136ab801 273 * the link name (what the user will see)
029feaed 274 * @param ref
136ab801 275 * the actual link reference (the target)
6c13eccd 276 * @param menu
136ab801 277 * menu (gophermap, i) mode -- not used in html mode
029feaed 278 * @param html
136ab801
NR
279 * TRUE for html mode, FALSE for gopher mode
280 *
281 * @return the ready-to-use link in a {@link String}
029feaed 282 */
6c13eccd 283 private String getLink(String name, String ref, boolean menu, boolean html) {
e402343e 284 if (!html) {
6c13eccd 285 return new StringBuilder().append((menu ? "1" : "0") + name)
029feaed 286 .append("\t").append(ref) //
e402343e
NR
287 .append("\t").append(hostname) //
288 .append("\t").append(Integer.toString(port)) //
289 .append("\r\n").toString();
73785268 290 }
e402343e
NR
291
292 return new StringBuilder().append(
293 "<div class='site'><a href='" + ref + "'>" + name
294 + "</a></div>\n").toString();
73785268
NR
295 }
296}