From 12443642377be74159578c99af7a9883571e38bf Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Sun, 14 Apr 2019 13:43:36 +0200 Subject: [PATCH] fix cache, MangaLEL +search --- changelog-fr.md | 1 + changelog.md | 1 + src/be/nikiroo/fanfix/DataLoader.java | 125 +++++++++--------- src/be/nikiroo/fanfix/Instance.java | 4 +- src/be/nikiroo/fanfix/VersionCheck.java | 3 +- .../nikiroo/fanfix/searchable/Fanfiction.java | 9 +- .../nikiroo/fanfix/searchable/MangaLel.java | 67 +++++++--- src/be/nikiroo/fanfix/supported/MangaFox.java | 8 +- src/be/nikiroo/fanfix/supported/MangaLel.java | 58 ++++---- .../nikiroo/fanfix/test/BasicSupportTest.java | 5 - 10 files changed, 162 insertions(+), 119 deletions(-) diff --git a/changelog-fr.md b/changelog-fr.md index 5a635f72..6fe48c60 100644 --- a/changelog-fr.md +++ b/changelog-fr.md @@ -8,6 +8,7 @@ - fix: correction de DEBUG=0 - gui: correction pour le focus - MangaLEL: site web changé +- search: supporte MangaLEL # Version 2.0.2 diff --git a/changelog.md b/changelog.md index dc932337..24fbceee 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,7 @@ - fix: fix DEBUG=0 - gui: focus fix - MangaLEL: website has changed +- search: MangaLEL support # Version 2.0.2 diff --git a/src/be/nikiroo/fanfix/DataLoader.java b/src/be/nikiroo/fanfix/DataLoader.java index e2af0709..51855c41 100644 --- a/src/be/nikiroo/fanfix/DataLoader.java +++ b/src/be/nikiroo/fanfix/DataLoader.java @@ -11,7 +11,6 @@ import be.nikiroo.fanfix.supported.BasicSupport; import be.nikiroo.utils.Cache; import be.nikiroo.utils.CacheMemory; import be.nikiroo.utils.Downloader; -import be.nikiroo.utils.IOUtils; import be.nikiroo.utils.Image; import be.nikiroo.utils.ImageUtils; import be.nikiroo.utils.TraceHandler; @@ -27,7 +26,7 @@ import be.nikiroo.utils.TraceHandler; */ public class DataLoader { private Downloader downloader; - private Cache downloadCache; + private Downloader downloaderNoCache; private Cache cache; /** @@ -51,9 +50,11 @@ public class DataLoader { */ public DataLoader(File dir, String UA, int hoursChanging, int hoursStable) throws IOException { - downloader = new Downloader(UA); - downloadCache = new Cache(dir, hoursChanging, hoursStable); - cache = downloadCache; + downloader = new Downloader(UA, new Cache(dir, hoursChanging, + hoursStable)); + downloaderNoCache = new Downloader(UA); + + cache = downloader.getCache(); } /** @@ -65,7 +66,7 @@ public class DataLoader { */ public DataLoader(String UA) { downloader = new Downloader(UA); - downloadCache = null; + downloaderNoCache = downloader; cache = new CacheMemory(); } @@ -77,9 +78,10 @@ public class DataLoader { */ public void setTraceHandler(TraceHandler tracer) { downloader.setTraceHandler(tracer); + downloaderNoCache.setTraceHandler(tracer); cache.setTraceHandler(tracer); - if (downloadCache != null) { - downloadCache.setTraceHandler(tracer); + if (downloader.getCache() != null) { + downloader.getCache().setTraceHandler(tracer); } } @@ -87,6 +89,8 @@ public class DataLoader { /** * Open a resource (will load it from the cache if possible, or save it into * the cache after downloading if not). + *

+ * The cached resource will be assimilated to the given original {@link URL} * * @param url * the resource to open @@ -102,8 +106,7 @@ public class DataLoader { */ public InputStream open(URL url, BasicSupport support, boolean stable) throws IOException { - // MUST NOT return null - return open(url, support, stable, url); + return open(url, url, support, stable, null, null, null); } /** @@ -114,72 +117,71 @@ public class DataLoader { * * @param url * the resource to open + * @param originalUrl + * the original {@link URL} before any redirection occurs, which + * is also used for the cache ID if needed (so we can retrieve + * the content with this URL if needed) * @param support * the support to use to download the resource * @param stable * TRUE for more stable resources, FALSE when they often change - * @param originalUrl - * the original {@link URL} used to locate the cached resource * * @return the opened resource, NOT NULL * * @throws IOException * in case of I/O error */ - public InputStream open(URL url, BasicSupport support, boolean stable, - URL originalUrl) throws IOException { - // MUST NOT return null - try { - InputStream in = null; - - if (downloadCache != null) { - in = downloadCache.load(originalUrl, false, stable); - Instance.getTraceHandler().trace( - "Cache " + (in != null ? "hit" : "miss") + ": " + url); - } - - if (in == null) { - try { - in = openNoCache(url, support, null, null, null); - if (downloadCache != null) { - downloadCache.save(in, originalUrl); - // ..But we want a resetable stream - in.close(); - in = downloadCache.load(originalUrl, true, stable); - } else { - InputStream resetIn = IOUtils.forceResetableStream(in); - if (resetIn != in) { - in.close(); - in = resetIn; - } - } - } catch (IOException e) { - throw new IOException("Cannot save the url: " - + (url == null ? "null" : url.toString()), e); - } - } - - return in; - } catch (IOException e) { - throw new IOException("Cannot open the url: " - + (url == null ? "null" : url.toString()), e); - } + public InputStream open(URL url, URL originalUrl, BasicSupport support, + boolean stable) throws IOException { + return open(url, originalUrl, support, stable, null, null, null); } /** - * Open the given {@link URL} without using the cache, but still update the - * cookies. + * Open a resource (will load it from the cache if possible, or save it into + * the cache after downloading if not). + *

+ * The cached resource will be assimilated to the given original {@link URL} * * @param url - * the {@link URL} to open + * the resource to open + * @param originalUrl + * the original {@link URL} before any redirection occurs, which + * is also used for the cache ID if needed (so we can retrieve + * the content with this URL if needed) + * @param support + * the support to use to download the resource + * @param stable + * TRUE for more stable resources, FALSE when they often change + * @param postParams + * the POST parameters + * @param getParams + * the GET parameters (priority over POST) + * @param oauth + * OAuth authorization (aka, "bearer XXXXXXX") * - * @return the {@link InputStream} of the opened page + * @return the opened resource, NOT NULL * * @throws IOException * in case of I/O error */ - public InputStream openNoCache(URL url) throws IOException { - return downloader.open(url); + public InputStream open(URL url, URL originalUrl, BasicSupport support, + boolean stable, Map postParams, + Map getParams, String oauth) throws IOException { + + Map cookiesValues = null; + URL currentReferer = url; + + if (support != null) { + cookiesValues = support.getCookies(); + currentReferer = support.getCurrentReferer(); + // priority: arguments + if (oauth == null) { + oauth = support.getOAuth(); + } + } + + return downloader.open(url, originalUrl, currentReferer, cookiesValues, + postParams, getParams, oauth, stable); } /** @@ -217,8 +219,8 @@ public class DataLoader { } } - return downloader.open(url, currentReferer, cookiesValues, postParams, - getParams, oauth); + return downloaderNoCache.open(url, currentReferer, cookiesValues, + postParams, getParams, oauth); } /** @@ -236,8 +238,8 @@ public class DataLoader { */ public void refresh(URL url, BasicSupport support, boolean stable) throws IOException { - if (downloadCache != null && !downloadCache.check(url, false, stable)) { - open(url, support, stable).close(); + if (check(url, stable)) { + open(url, url, support, stable, null, null, null).close(); } } @@ -254,7 +256,8 @@ public class DataLoader { * */ public boolean check(URL url, boolean stable) { - return downloadCache != null && downloadCache.check(url, false, stable); + return downloader.getCache() != null + && downloader.getCache().check(url, false, stable); } /** diff --git a/src/be/nikiroo/fanfix/Instance.java b/src/be/nikiroo/fanfix/Instance.java index c547a2b1..24d67afb 100644 --- a/src/be/nikiroo/fanfix/Instance.java +++ b/src/be/nikiroo/fanfix/Instance.java @@ -17,6 +17,7 @@ import be.nikiroo.fanfix.library.LocalLibrary; import be.nikiroo.fanfix.library.RemoteLibrary; import be.nikiroo.utils.Cache; import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.Image; import be.nikiroo.utils.Proxy; import be.nikiroo.utils.TempFiles; import be.nikiroo.utils.TraceHandler; @@ -71,7 +72,8 @@ public class Instance { remoteDir = new File(configDir, "remote"); lib = createDefaultLibrary(remoteDir); - // create cache + // create cache and TMP + Image.setTemporaryFilesRoot(new File(configDir, "tmp.images")); File tmp = getFile(Config.CACHE_DIR); if (tmp == null) { // Could have used: System.getProperty("java.io.tmpdir") diff --git a/src/be/nikiroo/fanfix/VersionCheck.java b/src/be/nikiroo/fanfix/VersionCheck.java index 76758af8..2c9a0328 100644 --- a/src/be/nikiroo/fanfix/VersionCheck.java +++ b/src/be/nikiroo/fanfix/VersionCheck.java @@ -120,7 +120,8 @@ public class VersionCheck { InputStream in = null; for (String url : new String[] { urlFrBE, urlFr, urlDefault }) { try { - in = Instance.getCache().openNoCache(new URL(url)); + in = Instance.getCache() + .open(new URL(url), null, false); break; } catch (IOException e) { } diff --git a/src/be/nikiroo/fanfix/searchable/Fanfiction.java b/src/be/nikiroo/fanfix/searchable/Fanfiction.java index d0b6f99b..d25153eb 100644 --- a/src/be/nikiroo/fanfix/searchable/Fanfiction.java +++ b/src/be/nikiroo/fanfix/searchable/Fanfiction.java @@ -280,10 +280,11 @@ class Fanfiction extends BasicSearchable { in.close(); } } catch (Exception e) { - Instance.getTraceHandler() - .error(new Exception( - "Cannot download cover for Fanfiction story in search mode", - e)); + // Should not happen on Fanfiction.net + Instance.getTraceHandler().error( + new Exception( + "Cannot download cover for Fanfiction story in search mode: " + + meta.getTitle(), e)); } } } diff --git a/src/be/nikiroo/fanfix/searchable/MangaLel.java b/src/be/nikiroo/fanfix/searchable/MangaLel.java index c8a522c8..879b1638 100644 --- a/src/be/nikiroo/fanfix/searchable/MangaLel.java +++ b/src/be/nikiroo/fanfix/searchable/MangaLel.java @@ -86,27 +86,18 @@ class MangaLel extends BasicSearchable { for (Element result : doc.getElementsByClass("rechercheAffichage")) { Element a = result.getElementsByTag("a").first(); if (a != null) { + int projectId = -1; + MetaData meta = new MetaData(); - meta.setUrl(a.absUrl("href")); - Element img = result.getElementsByTag("img").first(); - if (img != null) { - String coverUrl = img.absUrl("src"); - try { - InputStream in = Instance.getCache().open( - new URL(coverUrl), getSupport(), true); - try { - meta.setCover(new Image(in)); - } finally { - in.close(); - } - } catch (Exception e) { - Instance.getTraceHandler() - .error(new Exception( - "Cannot download cover for MangaLEL story in search mode", - e)); - } - } + // Target: + // http://mangas-lecture-en-ligne.fr/index_lel.php?page=presentationProjet&idProjet=218 + + // a.absUrl("href"): + // http://mangas-lecture-en-ligne.fr/index_lel?onCommence=oui&idChapitre=2805 + + // ...but we need the PROJECT id, not the CHAPTER id -> use + // Elements infos = result.getElementsByClass("texte"); if (infos != null) { @@ -125,7 +116,43 @@ class MangaLel extends BasicSearchable { getVal(tab, 5))); } - metas.add(meta); + Element img = result.getElementsByTag("img").first(); + if (img != null) { + try { + String[] tab = img.attr("src").split("/"); + String str = tab[tab.length - 1]; + tab = str.split("\\."); + str = tab[0]; + projectId = Integer.parseInt(str); + + String coverUrl = img.absUrl("src"); + try { + InputStream in = Instance.getCache().open( + new URL(coverUrl), getSupport(), true); + try { + meta.setCover(new Image(in)); + } finally { + in.close(); + } + } catch (Exception e) { + // Happen often on MangaLEL... + Instance.getTraceHandler().trace( + "Cannot download cover for MangaLEL story in search mode: " + + meta.getTitle()); + } + } catch (Exception e) { + // no project id... cannot use the story :( + Instance.getTraceHandler().error( + "Cannot find ProjectId for MangaLEL story in search mode: " + + meta.getTitle()); + } + } + + if (projectId >= 0) { + meta.setUrl("http://mangas-lecture-en-ligne.fr/index_lel.php?page=presentationProjet&idProjet=" + + projectId); + metas.add(meta); + } } } diff --git a/src/be/nikiroo/fanfix/supported/MangaFox.java b/src/be/nikiroo/fanfix/supported/MangaFox.java index 5dee0d6a..dae2d314 100644 --- a/src/be/nikiroo/fanfix/supported/MangaFox.java +++ b/src/be/nikiroo/fanfix/supported/MangaFox.java @@ -336,8 +336,8 @@ class MangaFox extends BasicSupport { */ private InputStream openEx(String url) throws IOException { try { - return Instance.getCache().open(new URL(url), this, true, - withoutQuery(url)); + return Instance.getCache().open(new URL(url), withoutQuery(url), + this, true); } catch (Exception e) { // second chance try { @@ -345,8 +345,8 @@ class MangaFox extends BasicSupport { } catch (InterruptedException ee) { } - return Instance.getCache().open(new URL(url), this, true, - withoutQuery(url)); + return Instance.getCache().open(new URL(url), withoutQuery(url), + this, true); } } diff --git a/src/be/nikiroo/fanfix/supported/MangaLel.java b/src/be/nikiroo/fanfix/supported/MangaLel.java index 020ee86f..1ba51bc0 100644 --- a/src/be/nikiroo/fanfix/supported/MangaLel.java +++ b/src/be/nikiroo/fanfix/supported/MangaLel.java @@ -60,30 +60,29 @@ class MangaLel extends BasicSupport { private String getAuthor() { Element doc = getSourceNode(); - Elements tabEls = doc.getElementsByClass("projet-titre"); - - String value = ""; - if (tabEls.size() >= 2) { - value = StringUtils.unhtml(tabEls.get(1).text()).trim(); + Element tabEls = doc.getElementsByClass("presentation-projet").first(); + if (tabEls != null) { + String[] tab = tabEls.outerHtml().split("
"); + return getVal(tab, 1); } - return value; + return ""; } private List getTags() { - List tags = new ArrayList(); - Element doc = getSourceNode(); - Elements tabEls = doc.getElementsByClass("projet-titre"); - - if (tabEls.size() >= 4) { - String values = StringUtils.unhtml(tabEls.get(3).text()).trim(); - for (String value : values.split(",")) { - tags.add(value); + Element tabEls = doc.getElementsByClass("presentation-projet").first(); + if (tabEls != null) { + String[] tab = tabEls.outerHtml().split("
"); + List tags = new ArrayList(); + for (String tag : getVal(tab, 3).split(" ")) { + tags.add(tag); } + return tags; } - return tags; + return new ArrayList(); + } private String getDate() { @@ -118,14 +117,13 @@ class MangaLel extends BasicSupport { @Override protected String getDesc() { Element doc = getSourceNode(); - Elements tabEls = doc.getElementsByClass("projet-titre"); - - String value = ""; - if (tabEls.size() >= 5) { - value = StringUtils.unhtml(tabEls.get(4).text()).trim(); + Element tabEls = doc.getElementsByClass("presentation-projet").first(); + if (tabEls != null) { + String[] tab = tabEls.outerHtml().split("
"); + return getVal(tab, 4); } - return value; + return ""; } private Image getCover() { @@ -167,6 +165,20 @@ class MangaLel extends BasicSupport { return null; } + private String getVal(String[] tab, int i) { + String val = ""; + + if (i < tab.length) { + val = StringUtils.unhtml(tab[i]); + int pos = val.indexOf(":"); + if (pos >= 0) { + val = val.substring(pos + 1).trim(); + } + } + + return val; + } + @Override protected List> getChapters(Progress pg) throws IOException { @@ -203,12 +215,12 @@ class MangaLel extends BasicSupport { Element content = pageDoc.getElementById("content"); Elements linkEls = content.getElementsByTag("img"); for (Element linkEl : linkEls) { - if (linkEl.attr("src").trim().isEmpty()) { + if (linkEl.absUrl("src").isEmpty()) { continue; } builder.append("["); - builder.append(linkEl.absUrl("src").trim()); + builder.append(linkEl.absUrl("src")); builder.append("]
"); } diff --git a/src/be/nikiroo/fanfix/test/BasicSupportTest.java b/src/be/nikiroo/fanfix/test/BasicSupportTest.java index a3f5221d..b731c441 100644 --- a/src/be/nikiroo/fanfix/test/BasicSupportTest.java +++ b/src/be/nikiroo/fanfix/test/BasicSupportTest.java @@ -394,11 +394,6 @@ class BasicSupportTest extends TestLauncher { } private class BasicSupportEmpty extends BasicSupport_Deprecated { - @Override - protected String getSourceName() { - return null; - } - @Override protected boolean supports(URL url) { return false; -- 2.27.0