From 413bcc29038d9c46c785142332839d41fd3c10e6 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Fri, 3 Apr 2020 17:43:12 +0200 Subject: [PATCH] add mangahub, remove mangfox --- README-fr.md | 2 +- README.md | 2 +- changelog-fr.md | 5 + changelog.md | 5 + .../fanfix/bundles/resources_core.properties | 4 +- .../bundles/resources_core_fr.properties | 2 +- .../fanfix/searchable/BasicSearchable.java | 2 +- .../fanfix/supported/BasicSupport.java | 4 +- src/be/nikiroo/fanfix/supported/MangaFox.java | 304 ------------------ src/be/nikiroo/fanfix/supported/MangaHub.java | 207 ++++++++++++ .../nikiroo/fanfix/supported/SupportType.java | 6 +- 11 files changed, 227 insertions(+), 316 deletions(-) delete mode 100644 src/be/nikiroo/fanfix/supported/MangaFox.java create mode 100644 src/be/nikiroo/fanfix/supported/MangaHub.java diff --git a/README-fr.md b/README-fr.md index 777840a5..85be7450 100644 --- a/README-fr.md +++ b/README-fr.md @@ -46,7 +46,7 @@ Pour le moment, les sites suivants sont supportés : - http://FimFiction.net/ : fanfictions dévouées à la série My Little Pony - http://Fanfiction.net/ : fanfictions venant d'une multitude d'univers différents, depuis les shows télévisés aux livres en passant par les jeux-vidéos -- http://mangafox.me/ : un site répertoriant une quantité non négligeable de mangas +- http://mangahub.io/ : un site répertoriant une quantité non négligeable de mangas (English) - https://e621.net/ : un site Furry proposant des comics, y compris de MLP - https://sofurry.com/ : même chose, mais orienté sur les histoires plutôt que les images - https://e-hentai.org/ : support ajouté sur demande : n'hésitez pas à demander un site ! diff --git a/README.md b/README.md index 43a0f406..7bc50900 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Currently, the following websites are supported: - http://FimFiction.net/: fan fictions devoted to the My Little Pony show - http://Fanfiction.net/: fan fictions of many, many different universes, from TV shows to novels to games -- http://mangafox.me/: a well filled repository of mangas, or, as their website states: most popular manga scanlations read online for free at mangafox, as well as a close-knit community to chat and make friends +- http://mangahub.io/: a well filled repository of mangas (English) - https://e621.net/: a Furry website supporting comics, including MLP - https://sofurry.com/: same thing, but story-oriented - https://e-hentai.org/: done upon request (so, feel free to ask for more websites!) diff --git a/changelog-fr.md b/changelog-fr.md index e9a5a059..b0bb4e92 100644 --- a/changelog-fr.md +++ b/changelog-fr.md @@ -1,5 +1,10 @@ # Fanfix +# Version WIP + +- new: MangaHub: un site de manga (English) +- new: MangaFox: retrait du support (site désagréable) + # Version 3.0.1 - fix: update pour e621 (et ce n'est plus un BasicSupport_Deprecated) diff --git a/changelog.md b/changelog.md index 825bc16f..e43c4ec4 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,10 @@ # Fanfix +# Version WIP + +- new: MangaHub: a manga website (English) +- new: MangaFox removed (too many unfriendly changes, bye) + # Version 3.0.1 - fix: update for e621 (and it is no more a BasicSupport_Deprecated) diff --git a/src/be/nikiroo/fanfix/bundles/resources_core.properties b/src/be/nikiroo/fanfix/bundles/resources_core.properties index d656ed66..dc7881ad 100644 --- a/src/be/nikiroo/fanfix/bundles/resources_core.properties +++ b/src/be/nikiroo/fanfix/bundles/resources_core.properties @@ -124,9 +124,7 @@ INPUT_DESC_FANFICTION = Fanfictions of many, many different universes, from TV s INPUT_DESC_FIMFICTION = Fanfictions devoted to the My Little Pony show # Description of this input type # (FORMAT: STRING) -INPUT_DESC_MANGAFOX = A well filled repository of mangas, or, as their website states: \n\ -\tMost popular manga scanlations read online for free at mangafox, \n\ -\tas well as a close-knit community to chat and make friends. +INPUT_DESC_MANGAHUB = A well filled repository of mangas, in English # Description of this input type # (FORMAT: STRING) INPUT_DESC_E621 = Furry website supporting comics, including MLP diff --git a/src/be/nikiroo/fanfix/bundles/resources_core_fr.properties b/src/be/nikiroo/fanfix/bundles/resources_core_fr.properties index 9bf36263..a64a5a09 100644 --- a/src/be/nikiroo/fanfix/bundles/resources_core_fr.properties +++ b/src/be/nikiroo/fanfix/bundles/resources_core_fr.properties @@ -109,7 +109,7 @@ INPUT_DESC_FANFICTION = Fanfictions venant d'une multitude d'univers différents INPUT_DESC_FIMFICTION = Fanfictions dévouées à la série My Little Pony # Description of this input type # (FORMAT: STRING) -INPUT_DESC_MANGAFOX = Un site répertoriant une quantité non négligeable de mangas +INPUT_DESC_MANGAHUB = Un site répertoriant une quantité non négligeable de mangas, en anglais # Description of this input type # (FORMAT: STRING) INPUT_DESC_E621 = Un site Furry proposant des comics, y compris de MLP diff --git a/src/be/nikiroo/fanfix/searchable/BasicSearchable.java b/src/be/nikiroo/fanfix/searchable/BasicSearchable.java index d38505e7..2b94725b 100644 --- a/src/be/nikiroo/fanfix/searchable/BasicSearchable.java +++ b/src/be/nikiroo/fanfix/searchable/BasicSearchable.java @@ -247,7 +247,7 @@ public abstract class BasicSearchable { case FANFICTION: support = new Fanfiction(type); break; - case MANGAFOX: + case MANGAHUB: // TODO break; case E621: diff --git a/src/be/nikiroo/fanfix/supported/BasicSupport.java b/src/be/nikiroo/fanfix/supported/BasicSupport.java index d3c0ebb9..d07fbcd9 100644 --- a/src/be/nikiroo/fanfix/supported/BasicSupport.java +++ b/src/be/nikiroo/fanfix/supported/BasicSupport.java @@ -493,8 +493,8 @@ public abstract class BasicSupport { case TEXT: support = new Text(); break; - case MANGAFOX: - support = new MangaFox(); + case MANGAHUB: + support = new MangaHub(); break; case E621: support = new E621(); diff --git a/src/be/nikiroo/fanfix/supported/MangaFox.java b/src/be/nikiroo/fanfix/supported/MangaFox.java deleted file mode 100644 index a9db419e..00000000 --- a/src/be/nikiroo/fanfix/supported/MangaFox.java +++ /dev/null @@ -1,304 +0,0 @@ -package be.nikiroo.fanfix.supported; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map.Entry; - -import org.jsoup.helper.DataUtil; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import be.nikiroo.fanfix.Instance; -import be.nikiroo.fanfix.data.MetaData; -import be.nikiroo.utils.Image; -import be.nikiroo.utils.Progress; -import be.nikiroo.utils.StringUtils; - -class MangaFox extends BasicSupport { - @Override - protected boolean isHtml() { - return true; - } - - @Override - protected MetaData getMeta() throws IOException { - MetaData meta = new MetaData(); - - meta.setTitle(getTitle()); - // No date anymore on mangafox - // meta.setDate(); - meta.setAuthor(getAuthor()); - meta.setTags(getTags()); - meta.setSource(getType().getSourceName()); - meta.setUrl(getSource().toString()); - meta.setPublisher(getType().getSourceName()); - meta.setUuid(getSource().toString()); - meta.setLuid(""); - meta.setLang("en"); - meta.setSubject("manga"); - meta.setType(getType().toString()); - meta.setImageDocument(true); - meta.setCover(getCover()); - - return meta; - } - - private String getTitle() { - Element doc = getSourceNode(); - - Element el = doc.getElementsByClass("detail-info-right-title-font").first(); - if (el != null) { - return StringUtils.unhtml(el.text()).trim(); - } - - return null; - } - - private String getAuthor() { - StringBuilder builder = new StringBuilder(); - for (String author : getListA("detail-info-right-say")) { - if (builder.length() > 0) - builder.append(", "); - builder.append(author); - } - - return builder.toString(); - } - - private List getTags() { - return getListA("detail-info-right-tag-list"); - } - - private List getListA(String uniqueClass) { - List list = new ArrayList(); - - Element doc = getSourceNode(); - Element el = doc.getElementsByClass(uniqueClass).first(); - if (el != null) { - for (Element valueA : el.getElementsByTag("a")) { - list.add(StringUtils.unhtml(valueA.text()).trim()); - } - } - - return list; - } - - @Override - protected String getDesc() { - Element doc = getSourceNode(); - Element title = doc.getElementsByClass("fullcontent").first(); - if (title != null) { - return StringUtils.unhtml(title.text()).trim(); - } - - return null; - } - - private Image getCover() { - Element doc = getSourceNode(); - Element cover = doc.getElementsByClass("detail-info-cover-img").first(); - if (cover != null) { - String coverUrl = cover.absUrl("src"); - - InputStream coverIn; - try { - coverIn = openEx(coverUrl); - try { - return new Image(coverIn); - } finally { - coverIn.close(); - } - } catch (IOException e) { - Instance.getTraceHandler().error(e); - } - } - - return null; - } - - @Override - protected List> getChapters(Progress pg) { - List> urls = new ArrayList>(); - - String prefix = getTitle(); // each chapter starts with this prefix, then a - // chapter number (including "x.5"), then name - - // normally, only one list... - Element doc = getSourceNode(); - for (Element list : doc.getElementsByClass("detail-main-list")) { - for (Element el : list.getElementsByTag("a")) { - String title = el.attr("title"); - if (title.startsWith(prefix)) { - title = title.substring(prefix.length()).trim(); - } - - String url = el.absUrl("href"); - - try { - urls.add(new AbstractMap.SimpleEntry(title, new URL(url))); - } catch (Exception e) { - Instance.getTraceHandler().error(e); - } - } - } - - // by default, the chapters are in reversed order - Collections.reverse(urls); - - return urls; - } - - @Override - protected String getChapterContent(URL chapUrl, int number, Progress pg) throws IOException { - if (pg == null) { - pg = new Progress(); - } - - StringBuilder builder = new StringBuilder(); - - Document chapDoc = DataUtil.load(Instance.getCache().open(chapUrl, this, false), "UTF-8", chapUrl.toString()); - - // Example of what we want: - // URL: http://fanfox.net/manga/solo_leveling/c110.5/1.html#ipg1 - // IMAGE, not working: - // http://s.fanfox.net/store/manga/29037/110.5/compressed/s034.jpg?token=f630767b0c96f6cc793fc8f1fc177c0ae9342eb1&ttl=1585929600 - // IMAGE, working: - // http://s.fanfox.net/store/manga/29037/000.0/compressed/m2018110o_143554_925.jpg?token=7d74569986335d49651ef1040f7dcb9dbd559b1b&ttl=1585929600 - // NOTE: (c110.5 -> 110.5, c000 -> 000.0) - // NOTE: image key: m2018110o_143554_925 can be found in the script, but not - // sorted - - // 0. Get the javascript content - StringBuilder javascript = new StringBuilder(); - for (Element script : chapDoc.getElementsByTag("script")) { - javascript.append(script.html()); - javascript.append("\n"); - } - - // 1. Get the chapter url part - String chap = chapUrl.getPath(); - chap = chap.split("#")[0]; - if (chap.endsWith("/1.html")) { - chap = chap.substring(0, chap.length() - "/1.html".length()); - } - int pos = chap.lastIndexOf("/"); - chap = chap.substring(pos + 1); - if (!chap.contains(".")) { - chap = chap + ".0"; - } - if (chap.startsWith("c")) { - chap = chap.substring(1); - } - - // 2. Token: - // chapKeys = getImageKeys(javascript); - // http://s.fanfox.net/store/manga/29037/000.0/compressed/m2018110o_143554_925.jpg?token=7d74569986335d49651ef1040f7dcb9dbd559b1b&ttl=1585929600 - String base = "http://s.fanfox.net/store/manga/%s/%s/compressed/%s.jpg?%s"; - for (String key : chapKeys) { - String img = String.format(base, comicId, chap, key, token); - builder.append("["); - builder.append(img); - builder.append("]
"); - } - - return builder.toString(); - } - - private int getIntVar(StringBuilder builder, String var) { - var = "var " + var; - - int pos = builder.indexOf(var) + var.length(); - String value = builder.subSequence(pos, pos + 20).toString(); - value = value.split("=")[1].trim(); - value = value.split(";")[0].trim(); - - return Integer.parseInt(value); - } - - private List getImageKeys(StringBuilder builder) { - List chapKeys = new ArrayList(); - - String start = "|compressed|"; - String stop = ">"; - int pos = builder.indexOf(start) + start.length(); - int pos2 = builder.indexOf(stop, pos) - stop.length(); - - String data = builder.substring(pos, pos2); - data = data.replace("|", "'"); - for (String key : data.split("'")) { - if (key.startsWith("m") && !key.equals("manga")) { - chapKeys.add(key); - } - } - - Collections.sort(chapKeys); - return chapKeys; - } - - /** - * Open the URL through the cache, but: retry a second time after 100ms if it - * fails, remove the query part of the {@link URL} before saving it to the cache - * (so it can be recalled later). - * - * @param url the {@link URL} - * - * @return the resource - * - * @throws IOException in case of I/O error - */ - private InputStream openEx(String url) throws IOException { - try { - return Instance.getCache().open(new URL(url), withoutQuery(url), this, true); - } catch (Exception e) { - // second chance - try { - Thread.sleep(100); - } catch (InterruptedException ee) { - } - - return Instance.getCache().open(new URL(url), withoutQuery(url), this, true); - } - } - - /** - * Return the same input {@link URL} but without the query part. - * - * @param url the inpiut {@link URL} as a {@link String} - * - * @return the input {@link URL} without query - */ - private URL withoutQuery(String url) { - URL o = null; - try { - // Remove the query from o (originalUrl), so it can be cached - // correctly - o = new URL(url); - o = new URL(o.getProtocol() + "://" + o.getHost() + o.getPath()); - - return o; - } catch (MalformedURLException e) { - return null; - } - } - - @Override - protected boolean supports(URL url) { - return "mangafox.me".equals(url.getHost()) || "www.mangafox.me".equals(url.getHost()) - || "fanfox.net".equals(url.getHost()) || "www.fanfox.net".equals(url.getHost()); - } -} diff --git a/src/be/nikiroo/fanfix/supported/MangaHub.java b/src/be/nikiroo/fanfix/supported/MangaHub.java new file mode 100644 index 00000000..437914ac --- /dev/null +++ b/src/be/nikiroo/fanfix/supported/MangaHub.java @@ -0,0 +1,207 @@ +package be.nikiroo.fanfix.supported; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map.Entry; + +import org.jsoup.nodes.Element; + +import be.nikiroo.fanfix.Instance; +import be.nikiroo.fanfix.data.MetaData; +import be.nikiroo.utils.Image; +import be.nikiroo.utils.Progress; +import be.nikiroo.utils.StringUtils; + +/** + * Support class for MangaHub, a website + * dedicated to Manga. + * + * @author niki + */ +class MangaHub extends BasicSupport { + @Override + protected boolean isHtml() { + return true; + } + + @Override + protected MetaData getMeta() throws IOException { + MetaData meta = new MetaData(); + + meta.setTitle(getTitle()); + meta.setDate(""); + meta.setAuthor(getAuthor()); + meta.setTags(getTags()); + meta.setSource(getType().getSourceName()); + meta.setUrl(getSource().toString()); + meta.setPublisher(getType().getSourceName()); + meta.setUuid(getSource().toString()); + meta.setLuid(""); + meta.setLang("en"); + meta.setSubject("manga"); + meta.setType(getType().toString()); + meta.setImageDocument(true); + meta.setCover(getCover()); + + return meta; + } + + private String getTitle() { + Element doc = getSourceNode(); + + Element el = doc.getElementsByTag("h1").first(); + if (el != null) { + return StringUtils.unhtml(el.text()).trim(); + } + + return null; + } + + private String getAuthor() { + String author = ""; + + Element el = getSourceNode().select("h1+div span:not([class])").first(); + if (el != null) + author = StringUtils.unhtml(el.text()).trim(); + return author; + } + + private List getTags() { + return getListA("genre-label"); + } + + private List getListA(String uniqueClass) { + List list = new ArrayList(); + + Element doc = getSourceNode(); + Element el = doc.getElementsByClass(uniqueClass).first(); + if (el != null) { + for (Element valueA : el.getElementsByTag("a")) { + list.add(StringUtils.unhtml(valueA.text()).trim()); + } + } + + return list; + } + + @Override + protected String getDesc() { + Element doc = getSourceNode(); + Element title = doc.getElementsByClass("fullcontent").first(); + if (title != null) { + return StringUtils.unhtml(title.text()).trim(); + } + + return null; + } + + private Image getCover() { + Element doc = getSourceNode(); + Element cover = doc.getElementsByClass("manga-thumb").first(); + if (cover != null) { + try { + return bsImages.getImage(this, new URL(cover.absUrl("src"))); + } catch (MalformedURLException e) { + Instance.getTraceHandler().error(e); + } + } + + return null; + } + + @Override + protected List> getChapters(Progress pg) { + List> urls = new ArrayList>(); + + Element doc = getSourceNode(); + for (Element el : doc.getElementsByClass("list-group-item")) { + Element urlEl = el.getElementsByTag("a").first(); + if (urlEl == null) + continue; + + String url = urlEl.absUrl("href"); + + String title = ""; + el = el.getElementsByClass("text-secondary").first(); + if (el != null) { + title = StringUtils.unhtml(el.text()).trim(); + } + + try { + urls.add(new AbstractMap.SimpleEntry(title, new URL(url))); + } catch (Exception e) { + Instance.getTraceHandler().error(e); + } + } + + // by default, the chapters are in reversed order + Collections.reverse(urls); + + return urls; + } + + @Override + protected String getChapterContent(URL chapUrl, int number, Progress pg) throws IOException { + if (pg == null) { + pg = new Progress(); + } + + // 1. Get the title and chapter url part + String path = chapUrl.getPath(); + if (path.endsWith("/")) { + path = path.substring(0, path.length() - "/".length()); + } + String tab[] = path.split("/"); + String chap = tab[tab.length - 1]; + String title = tab[tab.length - 2]; + + if (chap.startsWith("chapter-")) { + chap = chap.substring("chapter-".length()); + } + + // 2. generate an image base + String base = "https://img.mghubcdn.com/file/imghub/" + title + "/" + chap + "/"; + + // 3. add each chapter + StringBuilder builder = new StringBuilder(); + + int i = 1; + String url = base + i + ".jpg"; + while (getHttpStatus(new URL(url)) != 404) { + builder.append("["); + builder.append(url); + builder.append("]
"); + + i++; + url = base + i + ".jpg"; + } + + return builder.toString(); + } + + // HTTP response code, or -1 if other error + private int getHttpStatus(URL url) { + try { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + try { + connection.setRequestMethod("HEAD"); + return connection.getResponseCode(); + } finally { + connection.disconnect(); + } + } catch (Exception e) { + return -1; + } + } + + @Override + protected boolean supports(URL url) { + return "mangahub.io".equals(url.getHost()) || "www.mangahub.io".equals(url.getHost()); + } +} diff --git a/src/be/nikiroo/fanfix/supported/SupportType.java b/src/be/nikiroo/fanfix/supported/SupportType.java index ba18949e..0df30f26 100644 --- a/src/be/nikiroo/fanfix/supported/SupportType.java +++ b/src/be/nikiroo/fanfix/supported/SupportType.java @@ -20,7 +20,7 @@ public enum SupportType { /** Fanfictions from a lot of different universes */ FANFICTION, /** Website with lots of Mangas */ - MANGAFOX, + MANGAHUB, /** Furry website with comics support */ E621, /** Furry website with stories */ @@ -59,8 +59,8 @@ public enum SupportType { return "info-text"; case MANGA_LEL: return "MangaLEL"; - case MANGAFOX: - return "MangaFox.me"; + case MANGAHUB: + return "MangaHub.io"; case TEXT: return "text"; case YIFFSTAR: -- 2.27.0