src/be/nikiroo/gofetch/support/Reddit.java

   1 package be.nikiroo.gofetch.support;
   2
   3 import java.io.IOException;
   4 import java.net.URL;
   5 import java.text.SimpleDateFormat;
   6 import java.util.AbstractMap;
   7 import java.util.ArrayList;
   8 import java.util.Date;
   9 import java.util.HashMap;
  10 import java.util.LinkedList;
  11 import java.util.List;
  12 import java.util.Map;
  13 import java.util.Map.Entry;
  14
  15 import org.jsoup.nodes.Document;
  16 import org.jsoup.nodes.Element;
  17 import org.jsoup.select.Elements;
  18
  19 import be.nikiroo.gofetch.data.Comment;
  20 import be.nikiroo.gofetch.data.Story;
  21
  22 /**
  23  * Support <a href="https://www.reddit.com/">https://www.reddit.com/</a>.
  24  *
  25  * @author niki
  26  */
  27 public class Reddit extends BasicSupport {
  28         @Override
  29         public String getDescription() {
  30                 return "Reddit: The front page of the internet";
  31         }
  32
  33         @Override
  34         protected List<Entry<URL, String>> getUrls() throws IOException {
  35                 List<Entry<URL, String>> urls = new ArrayList<Entry<URL, String>>();
  36                 String base = "https://www.reddit.com/r/";
  37                 urls.add(new AbstractMap.SimpleEntry<URL, String>(new URL(base
  38                                 + "linux_gaming" + "/new/"), "linux_gaming"));
  39
  40                 return urls;
  41         }
  42
  43         @Override
  44         protected List<Element> getArticles(Document doc) {
  45                 List<Element> list = doc.getElementsByClass("thing");
  46                 if (list.isEmpty()) {
  47                         list = doc.getElementsByClass("Post");
  48                 }
  49                 if (list.isEmpty()) {
  50                         list = doc.getElementsByClass("scrollerItem");
  51                 }
  52
  53                 return list;
  54         }
  55
  56         @Override
  57         protected String getArticleId(Document doc, Element article) {
  58                 String date = getArticleDate(doc, article);
  59                 String title = getArticleTitle(doc, article);
  60
  61                 String id = (date + "_" + title).replaceAll("[^a-zA-Z0-9_-]", "_");
  62                 if (id.length() > 40) {
  63                         id = id.substring(0, 40);
  64                 }
  65
  66                 return id;
  67         }
  68
  69         @Override
  70         protected String getArticleTitle(Document doc, Element article) {
  71                 Elements els = article.getElementsByAttributeValue("data-event-action",
  72                                 "title");
  73                 if (els == null || els.isEmpty()) {
  74                         els = article.getElementsByTag("h2");
  75                 }
  76
  77                 return els.first().text().trim();
  78         }
  79
  80         @Override
  81         protected String getArticleAuthor(Document doc, Element article) {
  82                 return article.getElementsByAttributeValueStarting("href", "/user/")
  83                                 .text().trim();
  84         }
  85
  86         @Override
  87         protected String getArticleDate(Document doc, Element article) {
  88                 Element el = article.getElementsByClass("live-timestamp").first();
  89                 if (el == null) {
  90                         el = article.getElementsByAttributeValue("data-click-id",
  91                                         "timestamp").first();
  92                 }
  93
  94                 String dateAgo = el.text().trim();
  95                 return new SimpleDateFormat("yyyy-MM-dd_HH-mm")
  96                                 .format(getDate(dateAgo));
  97         }
  98
  99         @Override
 100         protected String getArticleCategory(Document doc, Element article,
 101                         String currentCategory) {
 102                 Elements categEls = article.getElementsByAttributeValueStarting("href",
 103                                 "/r/" + currentCategory + "/search=?q=flair_name");
 104
 105                 if (categEls.size() > 0) {
 106                         return currentCategory + ", " + categEls.first().text().trim();
 107                 }
 108
 109                 return currentCategory;
 110         }
 111
 112         @Override
 113         protected String getArticleDetails(Document doc, Element article) {
 114                 return "";
 115         }
 116
 117         @Override
 118         protected String getArticleIntUrl(Document doc, Element article) {
 119                 String url = article.absUrl("data-permalink");
 120                 if (url == null || url.isEmpty()) {
 121                         url = article
 122                                         .getElementsByAttributeValue("data-click-id", "timestamp")
 123                                         .first().absUrl("href");
 124                 }
 125
 126                 return url;
 127         }
 128
 129         @Override
 130         protected String getArticleExtUrl(Document doc, Element article) {
 131                 Elements els = article.getElementsByAttributeValue("data-event-action",
 132                                 "title");
 133                 if (els == null || els.isEmpty()) {
 134                         els = article.getElementsByAttributeValue("data-click-id", "body");
 135                 }
 136
 137                 Element url = els.first();
 138                 if (!url.attr("href").trim().startsWith("/")) {
 139                         return url.absUrl("href");
 140                 }
 141
 142                 return "";
 143         }
 144
 145         @Override
 146         protected String getArticleContent(Document doc, Element article) {
 147                 Elements els = article.getElementsByClass("h2");
 148                 if (els != null && !els.isEmpty()) {
 149                         return els.first().text().trim();
 150                 }
 151
 152                 return "";
 153         }
 154
 155         @Override
 156         protected Element getFullArticle(Document doc) {
 157                 Element element = doc.getElementsByAttributeValue("data-click-id",
 158                                 "body").first();
 159                 if (element == null) {
 160                         element = doc.getElementsByClass("ckueCN").first();
 161                 }
 162
 163                 return element;
 164         }
 165
 166         @Override
 167         protected ElementProcessor getElementProcessorFullArticle() {
 168                 return new BasicElementProcessor();
 169         }
 170
 171         @Override
 172         protected List<Element> getFullArticleCommentPosts(Document doc, URL intUrl) {
 173                 Elements posts = doc.getElementsByClass("jHfOJm");
 174                 if (posts.isEmpty()) {
 175                         posts = doc.getElementsByClass("eCeBkc");
 176                 }
 177                 if (posts.isEmpty()) {
 178                         posts = doc.getElementsByClass("gxtxxZ");
 179                 }
 180
 181                 return posts;
 182         }
 183
 184         @Override
 185         protected List<Element> getCommentCommentPosts(Document doc,
 186                         Element container) {
 187                 List<Element> elements = new LinkedList<Element>();
 188                 for (Element el : container.children()) {
 189                         elements.addAll(el.getElementsByClass("jHfOJm"));
 190
 191                 }
 192
 193                 if (elements.isEmpty()) {
 194                         for (Element el : container.children()) {
 195                                 elements.addAll(el.getElementsByClass("Comment"));
 196                         }
 197                 }
 198
 199                 return elements;
 200         }
 201
 202         @Override
 203         protected String getCommentId(Element post) {
 204                 int level = 1;
 205                 Elements els = post.getElementsByClass("imyGpC");
 206                 if (els.isEmpty())
 207                         els.addAll(post.getElementsByClass("emJXdb"));
 208
 209                 if (!els.isEmpty()) {
 210                         String l = els.first().text().trim().replace("level ", "");
 211                         try {
 212                                 level = Integer.parseInt(l);
 213                         } catch (NumberFormatException e) {
 214                         }
 215                 }
 216
 217                 return Integer.toString(level);
 218         }
 219
 220         @Override
 221         protected String getCommentAuthor(Element post) {
 222                 // Since we have no title, we switch with author
 223                 return "";
 224         }
 225
 226         @Override
 227         protected String getCommentTitle(Element post) {
 228                 // Since we have no title, we switch with author
 229
 230                 Element authorEl = post.getElementsByClass("RVnoX").first();
 231                 if (authorEl == null)
 232                         authorEl = post.getElementsByClass("kzePTH").first();
 233                 if (authorEl == null)
 234                         authorEl = post.getElementsByClass("jczTlv").first();
 235
 236                 if (authorEl != null)
 237                         return authorEl.text().trim();
 238
 239                 return "";
 240         }
 241
 242         @Override
 243         protected String getCommentDate(Element post) {
 244                 Element elAgo = post.getElementsByClass("hJDlLH").first();
 245                 if (elAgo == null)
 246                         elAgo = post.getElementsByClass("hDplaG").first();
 247
 248                 if (elAgo != null) {
 249                         String dateAgo = elAgo.text().trim();
 250                         return new SimpleDateFormat("yyyy-MM-dd_HH-mm")
 251                                         .format(getDate(dateAgo));
 252                 }
 253
 254                 return "";
 255         }
 256
 257         @Override
 258         protected Element getCommentContentElement(Element post) {
 259                 return post.getElementsByClass("ckueCN").first();
 260         }
 261
 262         @Override
 263         protected ElementProcessor getElementProcessorComment() {
 264                 return new BasicElementProcessor();
 265         }
 266
 267         @Override
 268         public void fetch(Story story) throws IOException {
 269                 super.fetch(story);
 270
 271                 List<Comment> comments = new LinkedList<Comment>();
 272                 Map<Integer, Comment> lastOfLevel = new HashMap<Integer, Comment>();
 273
 274                 for (Comment c : story.getComments()) {
 275                         int level = Integer.parseInt(c.getId());
 276                         lastOfLevel.put(level, c);
 277                         if (level <= 1) {
 278                                 comments.add(c);
 279                         } else {
 280                                 Comment parent = lastOfLevel.get(level - 1);
 281                                 if (parent != null) {
 282                                         parent.add(c);
 283                                 } else {
 284                                         // bad data
 285                                         comments.add(c);
 286                                 }
 287                         }
 288                 }
 289
 290                 story.setComments(comments);
 291         }
 292
 293         // 2 hours ago -> 18/10/2018 21:00
 294         private Date getDate(String dateAgo) {
 295                 int h = 0;
 296                 if (dateAgo.endsWith("hour ago")) {
 297                         h = 1;
 298                 } else if (dateAgo.endsWith("hours ago")) {
 299                         dateAgo = dateAgo.replace("hours ago", "").trim();
 300                         h = Integer.parseInt(dateAgo);
 301                 } else if (dateAgo.endsWith("day ago")) {
 302                         h = 24;
 303                 } else if (dateAgo.endsWith("days ago")) {
 304                         dateAgo = dateAgo.replace("days ago", "").trim();
 305                         h = Integer.parseInt(dateAgo) * 24;
 306                 }
 307
 308                 long now = new Date().getTime(); // in ms since 1970
 309                 now = now / (1000l * 60l * 60l); // in hours since 1970
 310                 long then = now - h; // in hours since 1970
 311                 then = then * (1000l * 60l * 60l); // in ms since 1970
 312
 313                 return new Date(then);
 314         }
 315 }