From 6e06d2cc9bf068a8ec4ad105aaef955a6eb509a1 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Mon, 27 Feb 2017 05:28:22 +0100 Subject: [PATCH] Partially fix YiffStar support - works more or less correctly except login mode - should not use login mode in final form (yet it should be fixed) --- src/be/nikiroo/fanfix/Cache.java | 188 ++++++++++++++---- src/be/nikiroo/fanfix/bundles/Config.java | 4 + .../nikiroo/fanfix/bundles/config.properties | 6 + .../fanfix/supported/BasicSupport.java | 17 +- src/be/nikiroo/fanfix/supported/YiffStar.java | 33 ++- 5 files changed, 203 insertions(+), 45 deletions(-) diff --git a/src/be/nikiroo/fanfix/Cache.java b/src/be/nikiroo/fanfix/Cache.java index b290756..8e8392a 100644 --- a/src/be/nikiroo/fanfix/Cache.java +++ b/src/be/nikiroo/fanfix/Cache.java @@ -6,6 +6,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStreamWriter; import java.net.CookieHandler; import java.net.CookieManager; import java.net.CookiePolicy; @@ -15,6 +16,7 @@ import java.net.HttpURLConnection; import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; +import java.net.URLEncoder; import java.util.Date; import java.util.Map; import java.util.zip.GZIPInputStream; @@ -84,6 +86,13 @@ public class Cache { CookieHandler.setDefault(cookies); } + /** + * Clear all the cookies currently in the jar. + */ + public void clearCookies() { + cookies.getCookieStore().removeAll(); + } + /** * Open a resource (will load it from the cache if possible, or save it into * the cache after downloading if not). @@ -150,6 +159,108 @@ public class Cache { } } + /** + * Open the given {@link URL} without using the cache, but still using and + * updating the cookies. + * + * @param url + * the {@link URL} to open + * @param support + * the {@link BasicSupport} used for the cookies + * + * @return the {@link InputStream} of the opened page + * + * @throws IOException + * in case of I/O error + */ + public InputStream openNoCache(URL url, BasicSupport support) + throws IOException { + return openNoCache(url, support, url, null); + } + + /** + * Open the given {@link URL} without using the cache, but still using and + * updating the cookies. + * + * @param url + * the {@link URL} to open + * @param support + * the {@link BasicSupport} used for the cookies + * @param postParams + * the POST parameters + * + * @return the {@link InputStream} of the opened page + * + * @throws IOException + * in case of I/O error + */ + public InputStream openNoCache(URL url, BasicSupport support, + Map postParams) throws IOException { + return openNoCache(url, support, url, postParams); + } + + /** + * Open the given {@link URL} without using the cache, but still using and + * updating the cookies. + * + * @param url + * the {@link URL} to open + * @param support + * the {@link BasicSupport} used for the cookies + * @param originalUrl + * the original {@link URL} before any redirection occurs + * @param postParams + * the POST parameters + * + * @return the {@link InputStream} of the opened page + * + * @throws IOException + * in case of I/O error + */ + private InputStream openNoCache(URL url, BasicSupport support, + final URL originalUrl, Map postParams) + throws IOException { + + URLConnection conn = openConnectionWithCookies(url, support); + if (postParams != null) { + StringBuilder postData = new StringBuilder(); + for (Map.Entry param : postParams.entrySet()) { + if (postData.length() != 0) + postData.append('&'); + postData.append(URLEncoder.encode(param.getKey(), "UTF-8")); + postData.append('='); + postData.append(URLEncoder.encode( + String.valueOf(param.getValue()), "UTF-8")); + } + + conn.setDoOutput(true); + + OutputStreamWriter writer = new OutputStreamWriter( + conn.getOutputStream()); + + writer.write(postData.toString()); + writer.flush(); + writer.close(); + } + + conn.connect(); + + // Check if redirect + if (conn instanceof HttpURLConnection + && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) { + String newUrl = conn.getHeaderField("Location"); + return openNoCache(new URL(newUrl), support, originalUrl, + postParams); + } + + InputStream in = conn.getInputStream(); + if ("gzip".equals(conn.getContentEncoding())) { + in = new GZIPInputStream(in); + } + + return in; + } + /** * Refresh the resource into cache if needed. * @@ -295,33 +406,7 @@ public class Cache { */ private void save(URL url, BasicSupport support, URL originalUrl) throws IOException { - URLConnection conn = url.openConnection(); - - conn.setRequestProperty("User-Agent", UA); - conn.setRequestProperty("Cookie", generateCookies(support)); - conn.setRequestProperty("Accept-Encoding", "gzip"); - if (support != null && support.getCurrentReferer() != null) { - conn.setRequestProperty("Referer", support.getCurrentReferer() - .toString()); - conn.setRequestProperty("Host", support.getCurrentReferer() - .getHost()); - } - - conn.connect(); - - // Check if redirect - if (conn instanceof HttpURLConnection - && ((HttpURLConnection) conn).getResponseCode() / 100 == 3) { - String newUrl = conn.getHeaderField("Location"); - save(new URL(newUrl), support, originalUrl); - return; - } - - InputStream in = conn.getInputStream(); - if ("gzip".equals(conn.getContentEncoding())) { - in = new GZIPInputStream(in); - } - + InputStream in = openNoCache(url, support, originalUrl, null); try { File cached = getCached(originalUrl); BufferedOutputStream out = new BufferedOutputStream( @@ -340,6 +425,37 @@ public class Cache { } } + /** + * Open a connection on the given {@link URL}, and manage the cookies that + * come with it. + * + * @param url + * the {@link URL} to open + * @param support + * the {@link BasicSupport} to use for cookie generation + * + * @return the connection + * + * @throws IOException + * in case of I/O error + */ + private URLConnection openConnectionWithCookies(URL url, + BasicSupport support) throws IOException { + URLConnection conn = url.openConnection(); + + conn.setRequestProperty("User-Agent", UA); + conn.setRequestProperty("Cookie", generateCookies(support)); + conn.setRequestProperty("Accept-Encoding", "gzip"); + if (support != null && support.getCurrentReferer() != null) { + conn.setRequestProperty("Referer", support.getCurrentReferer() + .toString()); + conn.setRequestProperty("Host", support.getCurrentReferer() + .getHost()); + } + + return conn; + } + /** * Check if the {@link File} is too old according to * {@link Cache#tooOldChanging}. @@ -409,14 +525,18 @@ public class Cache { } if (support != null) { - for (Map.Entry set : support.getCookies() - .entrySet()) { - if (builder.length() > 0) { - builder.append(';'); + try { + for (Map.Entry set : support.getCookies() + .entrySet()) { + if (builder.length() > 0) { + builder.append(';'); + } + builder.append(set.getKey()); + builder.append('='); + builder.append(set.getValue()); } - builder.append(set.getKey()); - builder.append('='); - builder.append(set.getValue()); + } catch (IOException e) { + Instance.syserr(e); } } diff --git a/src/be/nikiroo/fanfix/bundles/Config.java b/src/be/nikiroo/fanfix/bundles/Config.java index ca35733..7cc7bee 100644 --- a/src/be/nikiroo/fanfix/bundles/Config.java +++ b/src/be/nikiroo/fanfix/bundles/Config.java @@ -44,4 +44,8 @@ public enum Config { CHAPTER_EN, // @Meta(what = "Chapter identification string", where = "", format = "", info = "used to identify a starting chapter in text mode") CHAPTER_FR, // + @Meta(what = "Login information", where = "", format = "", info = "used to login on YiffStar to have access to all the stories") + LOGIN_YIFFSTAR_USER, // + @Meta(what = "Login information", where = "", format = "", info = "used to login on YiffStar to have access to all the stories") + LOGIN_YIFFSTAR_PASS, // } diff --git a/src/be/nikiroo/fanfix/bundles/config.properties b/src/be/nikiroo/fanfix/bundles/config.properties index 0b1d0ac..4148f66 100644 --- a/src/be/nikiroo/fanfix/bundles/config.properties +++ b/src/be/nikiroo/fanfix/bundles/config.properties @@ -56,3 +56,9 @@ CHAPTER_EN = Chapter # (WHAT: Chapter identification string) # used to identify a starting chapter in text mode CHAPTER_FR = Chapitre +# (WHAT: Login information) +# used to login on YiffStar to have access to all the stories +LOGIN_YIFFSTAR_USER = +# (WHAT: Login information) +# used to login on YiffStar to have access to all the stories +LOGIN_YIFFSTAR_PASS = diff --git a/src/be/nikiroo/fanfix/supported/BasicSupport.java b/src/be/nikiroo/fanfix/supported/BasicSupport.java index 1291822..b528cac 100644 --- a/src/be/nikiroo/fanfix/supported/BasicSupport.java +++ b/src/be/nikiroo/fanfix/supported/BasicSupport.java @@ -243,6 +243,16 @@ public abstract class BasicSupport { protected abstract String getChapterContent(URL source, InputStream in, int number) throws IOException; + /** + * Log into the support (can be a no-op depending upon the support). + * + * @throws IOException + * in case of I/O error + */ + public void login() throws IOException { + + } + /** * Return the list of cookies (values included) that must be used to * correctly fetch the resources. @@ -251,8 +261,11 @@ public abstract class BasicSupport { * it. * * @return the cookies + * + * @throws IOException + * in case of I/O error */ - public Map getCookies() { + public Map getCookies() throws IOException { return new HashMap(); } @@ -304,6 +317,8 @@ public abstract class BasicSupport { */ protected Story processMeta(URL url, boolean close, boolean getDesc) throws IOException { + login(); + url = getCanonicalUrl(url); setCurrentReferer(url); diff --git a/src/be/nikiroo/fanfix/supported/YiffStar.java b/src/be/nikiroo/fanfix/supported/YiffStar.java index e9c10c9..4f4e8d6 100644 --- a/src/be/nikiroo/fanfix/supported/YiffStar.java +++ b/src/be/nikiroo/fanfix/supported/YiffStar.java @@ -5,12 +5,14 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Scanner; import be.nikiroo.fanfix.Instance; +import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.data.MetaData; import be.nikiroo.utils.StringUtils; @@ -65,12 +67,21 @@ class YiffStar extends BasicSupport { } @Override - public Map getCookies() { - // TODO - // Cookies will actually be retained by the cache manager once logged in - // But we need to connect here and notify the cache manager + public void login() throws IOException { + Map post = new HashMap(); + post.put("LoginForm[sfLoginUsername]", + Instance.getConfig().getString(Config.LOGIN_YIFFSTAR_USER)); + post.put("LoginForm[sfLoginPassword]", + Instance.getConfig().getString(Config.LOGIN_YIFFSTAR_PASS)); + post.put("YII_CSRF_TOKEN", ""); - return super.getCookies(); + // Cookies will actually be retained by the cache manager once logged in + // TODO: not working yet, once fixed can be removed (adding "/guest" to + // URLs fix the access problem!): + /* + * Instance.getCache() .openNoCache(new + * URL("https://www.sofurry.com/user/login"), this, post).close(); + */ } @Override @@ -78,11 +89,13 @@ class YiffStar extends BasicSupport { if (source.getPath().startsWith("/view")) { InputStream in = Instance.getCache().open(source, this, false); String line = getLine(in, "/browse/folder/", 0); - String[] tab = line.split("\""); - if (tab.length > 1) { - String groupUrl = source.getProtocol() + "://" - + source.getHost() + tab[1]; - return new URL(groupUrl); + if (line != null) { + String[] tab = line.split("\""); + if (tab.length > 1) { + String groupUrl = source.getProtocol() + "://" + + source.getHost() + tab[1]; + return new URL(groupUrl); + } } } -- 2.27.0