Commit | Line | Data |
---|---|---|
73785268 NR |
1 | package be.nikiroo.gofetch; |
2 | ||
3 | import java.io.File; | |
4 | import java.io.FileWriter; | |
5 | import java.io.FilenameFilter; | |
6 | import java.io.IOException; | |
7 | import java.util.Arrays; | |
8 | import java.util.List; | |
9 | ||
10 | import be.nikiroo.gofetch.data.Comment; | |
11 | import be.nikiroo.gofetch.data.Story; | |
12 | import be.nikiroo.gofetch.output.Gopher; | |
13 | import be.nikiroo.gofetch.output.Html; | |
14 | import be.nikiroo.gofetch.output.Output; | |
15 | import be.nikiroo.gofetch.support.BasicSupport; | |
16 | import be.nikiroo.gofetch.support.BasicSupport.Type; | |
17 | import be.nikiroo.utils.IOUtils; | |
18 | ||
19 | /** | |
20 | * The class that will manage the fetch operations. | |
21 | * <p> | |
22 | * It will scrap the required websites and process them to disk. | |
23 | * | |
24 | * @author niki | |
25 | */ | |
26 | public class Fetcher { | |
27 | private File dir; | |
28 | private String preselector; | |
29 | private int maxStories; | |
30 | private String hostname; | |
31 | private int port; | |
32 | private Type type; | |
33 | ||
34 | /** | |
35 | * Prepare a new {@link Fetcher}. | |
36 | * | |
37 | * @param dir | |
38 | * the target directory where to save the files (won't have | |
39 | * impact on the files' content) | |
40 | * @param preselector | |
41 | * the sub directory and (pre-)selector to use for the resources | |
42 | * (<b>will</b> have an impact on the files' content) | |
43 | * @param type | |
44 | * the type of news to get (or the special keyword ALL to get all | |
45 | * of the supported sources) | |
46 | * @param maxStories | |
47 | * the maximum number of stories to show on the resume page | |
48 | * @param hostname | |
49 | * the gopher host to use (<b>will</b> have an impact on the | |
50 | * files' content) | |
51 | * @param port | |
52 | * the gopher port to use (<b>will</b> have an impact on the | |
53 | * files' content) | |
54 | */ | |
55 | public Fetcher(File dir, String preselector, Type type, int maxStories, | |
56 | String hostname, int port) { | |
57 | this.dir = dir; | |
58 | this.preselector = preselector; | |
59 | this.type = type; | |
60 | this.maxStories = maxStories; | |
61 | this.hostname = hostname; | |
62 | this.port = port; | |
63 | } | |
64 | ||
65 | /** | |
66 | * Start the fetching operation. | |
67 | * <p> | |
68 | * This method will handle the main pages itself, and will call | |
69 | * {@link Fetcher#list(BasicSupport)} for the stories. | |
70 | * | |
71 | * @throws IOException | |
72 | * in case of I/O error | |
73 | */ | |
74 | public void start() throws IOException { | |
75 | File cache = new File(dir, preselector); | |
76 | cache.mkdirs(); | |
77 | File cacheHtml = new File(cache, "index.html"); | |
78 | cache = new File(cache, ".cache"); | |
79 | ||
70b18499 NR |
80 | Output gopher = new Gopher(null, hostname, preselector, port); |
81 | Output html = new Html(null, hostname, preselector, port); | |
73785268 NR |
82 | |
83 | FileWriter writer = new FileWriter(cache); | |
84 | try { | |
85 | FileWriter writerHtml = new FileWriter(cacheHtml); | |
86 | try { | |
87 | writer.append(gopher.getIndexHeader()); | |
88 | writerHtml.append(html.getIndexHeader()); | |
89 | ||
73785268 | 90 | BasicSupport.setPreselector(preselector); |
70b18499 | 91 | for (Type type : Type.values()) { |
73785268 | 92 | BasicSupport support = BasicSupport.getSupport(type); |
70b18499 NR |
93 | |
94 | if (type == this.type || this.type == null) { | |
95 | list(support); | |
96 | } | |
73785268 NR |
97 | |
98 | writer.append("1" + support.getDescription()).append("\t") | |
99 | .append("1" + support.getSelector()) // | |
100 | .append("\t").append(hostname) // | |
101 | .append("\t").append(Integer.toString(port)) // | |
102 | .append("\r\n"); | |
103 | String ref = support.getSelector(); | |
104 | while (ref.startsWith("/")) { | |
105 | ref = ref.substring(1); | |
106 | } | |
107 | writerHtml.append("<div class='site'><a href='../" + ref | |
108 | + "'>" + support.getDescription() + "</a></div>"); | |
109 | } | |
110 | ||
111 | writer.append(gopher.getIndexFooter()); | |
112 | writerHtml.append(html.getIndexFooter()); | |
113 | } finally { | |
114 | writerHtml.close(); | |
115 | } | |
116 | } finally { | |
117 | writer.close(); | |
118 | } | |
119 | } | |
120 | ||
121 | /** | |
122 | * Process the stories for the given {@link BasicSupport} to disk. | |
123 | * | |
124 | * @param support | |
125 | * the {@link BasicSupport} to download from | |
126 | * | |
127 | * @throws IOException | |
128 | * in case of I/O error | |
129 | **/ | |
130 | private void list(BasicSupport support) throws IOException { | |
70b18499 NR |
131 | Output gopher = new Gopher(support.getType(), hostname, preselector, |
132 | port); | |
133 | Output html = new Html(support.getType(), hostname, preselector, port); | |
73785268 NR |
134 | |
135 | new File(dir, support.getSelector()).mkdirs(); | |
136 | ||
137 | System.err | |
138 | .print("Listing recent news for " + support.getType() + "..."); | |
139 | List<Story> stories = support.list(); | |
140 | System.err.println(" " + stories.size() + " stories found!"); | |
141 | int i = 1; | |
142 | for (Story story : stories) { | |
143 | IOUtils.writeSmallFile(dir, story.getSelector() + ".header", | |
144 | gopher.export(story)); | |
145 | IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html", | |
146 | html.export(story)); | |
147 | ||
148 | System.err.println(String.format("%02d/%02d", i, stories.size()) | |
149 | + " Fetching comments for story " + story.getId() + "..."); | |
150 | List<Comment> comments = support.getComments(story); | |
151 | ||
152 | IOUtils.writeSmallFile(dir, story.getSelector(), | |
153 | gopher.export(story, comments)); | |
154 | IOUtils.writeSmallFile(dir, story.getSelector() + ".html", | |
155 | html.export(story, comments)); | |
156 | ||
157 | i++; | |
158 | } | |
159 | ||
160 | File varDir = new File(dir, support.getSelector()); | |
161 | String[] headers = varDir.list(new FilenameFilter() { | |
162 | @Override | |
163 | public boolean accept(File dir, String name) { | |
164 | return name.endsWith(".header"); | |
165 | } | |
166 | }); | |
167 | ||
168 | File cache = new File(varDir, ".cache"); | |
169 | File cacheHtml = new File(varDir, "index.html"); | |
170 | FileWriter writer = new FileWriter(cache); | |
171 | try { | |
172 | FileWriter writerHtml = new FileWriter(cacheHtml); | |
173 | try { | |
174 | if (headers.length > 0) { | |
175 | Arrays.sort(headers); | |
176 | int from = headers.length - 1; | |
177 | int to = headers.length - maxStories; | |
178 | if (to < 0) { | |
179 | to = 0; | |
180 | } | |
181 | for (i = from; i >= to; i--) { | |
182 | writer.append(IOUtils.readSmallFile(new File(varDir, | |
183 | headers[i]))); | |
184 | ||
185 | writerHtml.append(IOUtils.readSmallFile(new File( | |
186 | varDir, headers[i] + ".html"))); | |
187 | } | |
188 | } | |
189 | } finally { | |
190 | writerHtml.close(); | |
191 | } | |
192 | } finally { | |
193 | writer.close(); | |
194 | } | |
195 | } | |
196 | } |