Fetcher.java

   1 package be.nikiroo.gofetch;
   2
   3 import java.io.File;
   4 import java.io.FileWriter;
   5 import java.io.FilenameFilter;
   6 import java.io.IOException;
   7 import java.util.ArrayList;
   8 import java.util.Arrays;
   9 import java.util.Collections;
  10 import java.util.List;
  11
  12 import be.nikiroo.gofetch.data.Story;
  13 import be.nikiroo.gofetch.output.Gopher;
  14 import be.nikiroo.gofetch.output.Html;
  15 import be.nikiroo.gofetch.output.Output;
  16 import be.nikiroo.gofetch.support.BasicSupport;
  17 import be.nikiroo.gofetch.support.Type;
  18 import be.nikiroo.utils.IOUtils;
  19
  20 /**
  21  * The class that will manage the fetch operations.
  22  * <p>
  23  * It will scrap the required websites and process them to disk.
  24  *
  25  * @author niki
  26  */
  27 public class Fetcher {
  28         private File dir;
  29         private String preselector;
  30         private int maxStories;
  31         private String hostname;
  32         private int port;
  33         private Type type;
  34
  35         /**
  36          * Prepare a new {@link Fetcher}.
  37          *
  38          * @param dir
  39          *            the target directory where to save the files (won't have
  40          *            impact on the files' content)
  41          * @param preselector
  42          *            the sub directory and (pre-)selector to use for the resources
  43          *            (<b>will</b> have an impact on the files' content)
  44          * @param type
  45          *            the type of news to get (or NULL to get all of the supported
  46          *            sources)
  47          * @param maxStories
  48          *            the maximum number of stories to show on the resume page
  49          * @param hostname
  50          *            the gopher host to use (<b>will</b> have an impact on the
  51          *            files' content)
  52          * @param port
  53          *            the gopher port to use (<b>will</b> have an impact on the
  54          *            files' content)
  55          */
  56         public Fetcher(File dir, String preselector, Type type, int maxStories,
  57                         String hostname, int port) {
  58                 this.dir = dir;
  59                 this.preselector = preselector;
  60                 this.type = type;
  61                 this.maxStories = maxStories;
  62                 this.hostname = hostname;
  63                 this.port = port;
  64         }
  65
  66         /**
  67          * Start the fetching operation.
  68          * <p>
  69          * This method will handle the main pages itself, and will call
  70          * {@link Fetcher#list(BasicSupport)} for the stories.
  71          *
  72          * @throws IOException
  73          *             in case of I/O error
  74          */
  75         public void start() throws IOException {
  76                 StringBuilder gopherBuilder = new StringBuilder();
  77                 StringBuilder htmlBuilder = new StringBuilder();
  78
  79                 BasicSupport.setPreselector(preselector);
  80                 for (Type type : Type.values()) {
  81                         BasicSupport support = BasicSupport.getSupport(type);
  82
  83                         if (type == this.type || this.type == null) {
  84                                 try {
  85                                         list(support);
  86                                 } catch (Exception e) {
  87                                         new Exception("Failed to process support: " + type, e)
  88                                                         .printStackTrace();
  89                                 }
  90                         }
  91
  92                         gopherBuilder.append(getLink(support.getDescription(),
  93                                         support.getSelector(), true, false));
  94
  95                         String ref = support.getSelector();
  96                         while (ref.startsWith("/")) {
  97                                 ref = ref.substring(1);
  98                         }
  99                         ref = "../" + ref + "/index.html";
 100
 101                         htmlBuilder.append(getLink(support.getDescription(), ref, true,
 102                                         true));
 103                 }
 104
 105                 File gopherCache = new File(dir, preselector);
 106                 gopherCache.mkdirs();
 107                 File htmlIndex = new File(gopherCache, "index.html");
 108                 gopherCache = new File(gopherCache, "gophermap");
 109
 110                 Output gopher = new Gopher(null, hostname, preselector, port);
 111                 Output html = new Html(null, hostname, preselector, port);
 112
 113                 FileWriter writer = new FileWriter(gopherCache);
 114                 try {
 115                         writer.append(gopher.getIndexHeader());
 116                         writer.append(gopherBuilder.toString());
 117                         writer.append(gopher.getIndexFooter());
 118                 } finally {
 119                         writer.close();
 120                 }
 121
 122                 try {
 123                         writer = new FileWriter(htmlIndex);
 124                         writer.append(html.getIndexHeader());
 125                         writer.append(htmlBuilder.toString());
 126                         writer.append(html.getIndexFooter());
 127                 } finally {
 128                         writer.close();
 129                 }
 130         }
 131
 132         /**
 133          * Process the stories for the given {@link BasicSupport} to disk.
 134          *
 135          * @param support
 136          *            the {@link BasicSupport} to download from
 137          *
 138          * @throws IOException
 139          *             in case of I/O error
 140          **/
 141         private void list(BasicSupport support) throws IOException {
 142                 // Get stories:
 143                 System.err
 144                                 .print("Listing recent news for " + support.getType() + "...");
 145                 List<Story> stories = support.list();
 146                 System.err.println(" " + stories.size() + " stories found!");
 147
 148                 // Get comments (and update stories if needed):
 149                 int i = 1;
 150                 List<Story> fetchedStories = new ArrayList<Story>(stories.size());
 151                 for (Story story : stories) {
 152                         System.err.print(String.format("%02d/%02d", i, stories.size())
 153                                         + " Fetching full story " + story.getId() + "...");
 154                         try {
 155                                 support.fetch(story);
 156                                 fetchedStories.add(story);
 157                                 System.err.println();
 158                         } catch (IOException e) {
 159                                 System.err.println(" Failed to get story!");
 160                         }
 161                         i++;
 162                 }
 163                 stories = fetchedStories;
 164
 165                 Output gopher = new Gopher(support.getType(), hostname, preselector,
 166                                 port);
 167                 Output html = new Html(support.getType(), hostname, preselector, port);
 168
 169                 new File(dir, support.getSelector()).mkdirs();
 170
 171                 for (Story story : stories) {
 172                         IOUtils.writeSmallFile(dir, story.getSelector() + ".header",
 173                                         gopher.exportHeader(story));
 174                         IOUtils.writeSmallFile(dir, story.getSelector() + ".header.html",
 175                                         html.exportHeader(story));
 176
 177                         IOUtils.writeSmallFile(dir, story.getSelector(),
 178                                         gopher.export(story));
 179                         IOUtils.writeSmallFile(dir, story.getSelector() + ".html",
 180                                         html.export(story));
 181                 }
 182
 183                 // Finding headers of all stories in cache:
 184                 File varDir = new File(dir, support.getSelector());
 185                 String[] headers = varDir.list(new FilenameFilter() {
 186                         @Override
 187                         public boolean accept(File dir, String name) {
 188                                 return name.endsWith(".header");
 189                         }
 190                 });
 191
 192                 // Reverse sort:
 193                 Arrays.sort(headers);
 194                 List<String> tmp = Arrays.asList(headers);
 195                 Collections.reverse(tmp);
 196                 headers = tmp.toArray(new String[] {});
 197                 //
 198
 199                 // Write the index (with "MORE" links if needed)
 200                 int page = 0;
 201                 List<String> gopherLines = new ArrayList<String>();
 202                 List<String> htmlLines = new ArrayList<String>();
 203                 for (i = 0; i < headers.length; i++) {
 204                         File gopherFile = new File(varDir, headers[i]);
 205                         File htmlFile = new File(varDir, headers[i] + ".html");
 206
 207                         if (gopherFile.exists())
 208                                 gopherLines.add(IOUtils.readSmallFile(gopherFile));
 209                         if (htmlFile.exists())
 210                                 htmlLines.add(IOUtils.readSmallFile(htmlFile));
 211
 212                         boolean enoughStories = (i > 0 && i % maxStories == 0);
 213                         boolean last = i == headers.length - 1;
 214                         if (enoughStories || last) {
 215                                 if (!last) {
 216                                         gopherLines.add(getLink("More", support.getSelector()
 217                                                         + "gophermap_" + (page + 1), true, false));
 218
 219                                         htmlLines.add(getLink("More", "index_" + (page + 1)
 220                                                         + ".html", true, true));
 221                                 }
 222
 223                                 write(gopherLines, varDir, "gophermap", "", page);
 224                                 write(htmlLines, varDir, "index", ".html", page);
 225                                 gopherLines = new ArrayList<String>();
 226                                 htmlLines = new ArrayList<String>();
 227                                 page++;
 228                         }
 229                 }
 230         }
 231
 232         private void write(List<String> lines, File varDir, String basename,
 233                         String ext, int page) throws IOException {
 234                 File file = new File(varDir, basename + (page > 0 ? "_" + page : "")
 235                                 + ext);
 236
 237                 FileWriter writer = new FileWriter(file);
 238                 try {
 239                         for (String line : lines) {
 240                                 writer.append(line).append("\r\n");
 241                         }
 242                 } finally {
 243                         writer.close();
 244                 }
 245         }
 246
 247         /**
 248          * Create a link.
 249          *
 250          * @param name
 251          *            the link name (what the user will see)
 252          * @param ref
 253          *            the actual link reference (the target)
 254          * @param menu
 255          *            menu (gophermap, i) mode -- not used in html mode
 256          * @param html
 257          *            TRUE for html mode, FALSE for gopher mode
 258          *
 259          * @return the ready-to-use link in a {@link String}
 260          */
 261         private String getLink(String name, String ref, boolean menu, boolean html) {
 262                 if (!html) {
 263                         return new StringBuilder().append((menu ? "1" : "0") + name)
 264                                         .append("\t").append(ref) //
 265                                         .append("\t").append(hostname) //
 266                                         .append("\t").append(Integer.toString(port)) //
 267                                         .append("\r\n").toString();
 268                 }
 269
 270                 return new StringBuilder().append(
 271                                 "<div class='site'><a href='" + ref + "'>" + name
 272                                                 + "</a></div>\n").toString();
 273         }
 274 }