From a4143cd74a90e17a811a4581cbeb213fed1f6304 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Sun, 26 Feb 2017 17:35:36 +0100 Subject: [PATCH] New support: YiffStar (still no logged-in content) The next step is to login, keep the cookies, and only then download the stories --- src/be/nikiroo/fanfix/Library.java | 6 +- .../fanfix/reader/LocalReaderFrame.java | 15 +- .../fanfix/supported/BasicSupport.java | 63 ++++- src/be/nikiroo/fanfix/supported/E621.java | 2 +- .../nikiroo/fanfix/supported/Fanfiction.java | 1 + src/be/nikiroo/fanfix/supported/Html.java | 58 ++--- src/be/nikiroo/fanfix/supported/YiffStar.java | 235 ++++++++++++++++++ 7 files changed, 319 insertions(+), 61 deletions(-) create mode 100644 src/be/nikiroo/fanfix/supported/YiffStar.java diff --git a/src/be/nikiroo/fanfix/Library.java b/src/be/nikiroo/fanfix/Library.java index 4db3868..6e0262b 100644 --- a/src/be/nikiroo/fanfix/Library.java +++ b/src/be/nikiroo/fanfix/Library.java @@ -390,7 +390,11 @@ public class Library { * @return the target */ private File getFile(MetaData key) { - String title = key.getTitle().replaceAll("[^a-zA-Z0-9._+-]", "_"); + String title = key.getTitle(); + if (title == null) { + title = ""; + } + title = title.replaceAll("[^a-zA-Z0-9._+-]", "_"); return new File(getDir(key), key.getLuid() + "_" + title); } diff --git a/src/be/nikiroo/fanfix/reader/LocalReaderFrame.java b/src/be/nikiroo/fanfix/reader/LocalReaderFrame.java index 7ce16cc..0da3475 100644 --- a/src/be/nikiroo/fanfix/reader/LocalReaderFrame.java +++ b/src/be/nikiroo/fanfix/reader/LocalReaderFrame.java @@ -450,15 +450,11 @@ class LocalReaderFrame extends JFrame { private void outOfUi(final Progress pg, final Runnable run) { pgBar.setProgress(pg); - SwingUtilities.invokeLater(new Runnable() { - public void run() { - setEnabled(false); - pgBar.addActioListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - pgBar.setProgress(null); - setEnabled(true); - } - }); + setEnabled(false); + pgBar.addActioListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + pgBar.setProgress(null); + setEnabled(true); } }); @@ -532,6 +528,7 @@ class LocalReaderFrame extends JFrame { SwingUtilities.invokeLater(new Runnable() { public void run() { if (!ok) { + Instance.syserr(e); JOptionPane.showMessageDialog( LocalReaderFrame.this, "Cannot import: " + url, e.getMessage(), diff --git a/src/be/nikiroo/fanfix/supported/BasicSupport.java b/src/be/nikiroo/fanfix/supported/BasicSupport.java index 2b4715a..1291822 100644 --- a/src/be/nikiroo/fanfix/supported/BasicSupport.java +++ b/src/be/nikiroo/fanfix/supported/BasicSupport.java @@ -59,6 +59,8 @@ public abstract class BasicSupport { MANGAFOX, /** Furry website with comics support */ E621, + /** Furry website with stories */ + YIFFSTAR, /** CBZ files */ CBZ, /** HTML files */ @@ -254,6 +256,21 @@ public abstract class BasicSupport { return new HashMap(); } + /** + * Return the canonical form of the main {@link URL}. + * + * @param source + * the source {@link URL} + * + * @return the canonical form of this {@link URL} + * + * @throws IOException + * in case of I/O error + */ + public URL getCanonicalUrl(URL source) throws IOException { + return source; + } + /** * Process the given story resource into a partially filled {@link Story} * object containing the name and metadata, except for the description. @@ -287,6 +304,10 @@ public abstract class BasicSupport { */ protected Story processMeta(URL url, boolean close, boolean getDesc) throws IOException { + url = getCanonicalUrl(url); + + setCurrentReferer(url); + in = openInput(url); if (in == null) { return null; @@ -324,6 +345,8 @@ public abstract class BasicSupport { in.close(); } } + + setCurrentReferer(null); } } @@ -348,8 +371,7 @@ public abstract class BasicSupport { pg.setMinMax(0, 100); } - setCurrentReferer(url); - + url = getCanonicalUrl(url); pg.setProgress(1); try { Story story = processMeta(url, false, true); @@ -359,6 +381,8 @@ public abstract class BasicSupport { return null; } + setCurrentReferer(url); + story.setChapters(new ArrayList()); List> chapters = getChapters(url, getInput()); @@ -400,12 +424,12 @@ public abstract class BasicSupport { in.close(); } - currentReferer = null; + setCurrentReferer(null); } } /** - * The support type.$ + * The support type. * * @return the type */ @@ -661,6 +685,11 @@ public abstract class BasicSupport { /** * Return the list of supported image extensions. * + * @param emptyAllowed + * TRUE to allow an empty extension on first place, which can be + * used when you may already have an extension in your input but + * are not sure about it + * * @return the extensions */ static String[] getImageExt(boolean emptyAllowed) { @@ -671,6 +700,18 @@ public abstract class BasicSupport { } } + /** + * Check if the given resource can be a local image or a remote image, then + * refresh the cache with it if it is. + * + * @param source + * the story source + * @param line + * the resource to check + * + * @return the image if found, or NULL + * + */ static BufferedImage getImage(BasicSupport support, URL source, String line) { URL url = getImageUrl(support, source, line); if (url != null) { @@ -782,6 +823,14 @@ public abstract class BasicSupport { return Instance.getCache().open(source, this, false); } + /** + * Reset the given {@link InputStream} and return it. + * + * @param in + * the {@link InputStream} to reset + * + * @return the same {@link InputStream} after reset + */ protected InputStream reset(InputStream in) { try { in.reset(); @@ -834,7 +883,7 @@ public abstract class BasicSupport { * paragraphs (quotes or not)). * * @param para - * the paragraph to requotify (not necessaraly a quote) + * the paragraph to requotify (not necessarily a quote) * * @return the correctly (or so we hope) quotified paragraphs */ @@ -1111,7 +1160,7 @@ public abstract class BasicSupport { } /** - * Remove the HTML from the inpit if {@link BasicSupport#isHtml()} is + * Remove the HTML from the input if {@link BasicSupport#isHtml()} is * true. * * @param input @@ -1186,6 +1235,8 @@ public abstract class BasicSupport { return new MangaFox().setType(type); case E621: return new E621().setType(type); + case YIFFSTAR: + return new YiffStar().setType(type); case CBZ: return new Cbz().setType(type); case HTML: diff --git a/src/be/nikiroo/fanfix/supported/E621.java b/src/be/nikiroo/fanfix/supported/E621.java index cde7f0a..4014765 100644 --- a/src/be/nikiroo/fanfix/supported/E621.java +++ b/src/be/nikiroo/fanfix/supported/E621.java @@ -46,7 +46,7 @@ class E621 extends BasicSupport { meta.setUuid(source.toString()); meta.setLuid(""); meta.setLang("EN"); - meta.setSubject(""); + meta.setSubject("Furry"); meta.setType(getType().toString()); meta.setImageDocument(true); meta.setCover(getCover(source)); diff --git a/src/be/nikiroo/fanfix/supported/Fanfiction.java b/src/be/nikiroo/fanfix/supported/Fanfiction.java index a73f7de..e84acaa 100644 --- a/src/be/nikiroo/fanfix/supported/Fanfiction.java +++ b/src/be/nikiroo/fanfix/supported/Fanfiction.java @@ -329,6 +329,7 @@ class Fanfiction extends BasicSupport { } builder.append(line); + builder.append('\n'); } } diff --git a/src/be/nikiroo/fanfix/supported/Html.java b/src/be/nikiroo/fanfix/supported/Html.java index 036479d..fffbcd7 100644 --- a/src/be/nikiroo/fanfix/supported/Html.java +++ b/src/be/nikiroo/fanfix/supported/Html.java @@ -1,17 +1,10 @@ package be.nikiroo.fanfix.supported; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; -import java.util.List; -import java.util.Map.Entry; - -import be.nikiroo.fanfix.data.MetaData; -import be.nikiroo.utils.MarkableFileInputStream; /** * Support class for HTML files created with this program (as we need some @@ -20,8 +13,6 @@ import be.nikiroo.utils.MarkableFileInputStream; * @author niki */ class Html extends InfoText { - private URL fakeSource; - @Override public String getSourceName() { return "html"; @@ -43,41 +34,20 @@ class Html extends InfoText { } @Override - protected MetaData getMeta(URL source, InputStream in) throws IOException { - return super.getMeta(fakeSource, in); - } - - @Override - protected String getDesc(URL source, InputStream in) throws IOException { - return super.getDesc(fakeSource, in); - } - - @Override - protected List> getChapters(URL source, InputStream in) - throws IOException { - return super.getChapters(fakeSource, in); - } - - @Override - protected String getChapterContent(URL source, InputStream in, int number) - throws IOException { - return super.getChapterContent(fakeSource, in, number); - } - - @Override - protected InputStream openInput(URL source) throws IOException { - try { - File fakeFile = new File(source.toURI()); // "story/index.html" - fakeFile = new File(fakeFile.getParent()); // "story" - fakeFile = new File(fakeFile, fakeFile.getName()); // "story/story" - fakeSource = fakeFile.toURI().toURL(); - return new MarkableFileInputStream(new FileInputStream(fakeFile)); - } catch (URISyntaxException e) { - throw new IOException( - "file not supported (maybe not created with this program or corrupt)", - e); - } catch (MalformedURLException e) { - throw new IOException("file not supported (bad URL)", e); + public URL getCanonicalUrl(URL source) throws IOException { + if (source.toString().endsWith(File.separator + "index.html")) { + try { + File fakeFile = new File(source.toURI()); // "story/index.html" + fakeFile = new File(fakeFile.getParent()); // "story" + fakeFile = new File(fakeFile, fakeFile.getName()); // "story/story" + return fakeFile.toURI().toURL(); + } catch (URISyntaxException e) { + throw new IOException( + "file not supported (maybe not created with this program or corrupt)", + e); + } } + + return source; } } diff --git a/src/be/nikiroo/fanfix/supported/YiffStar.java b/src/be/nikiroo/fanfix/supported/YiffStar.java new file mode 100644 index 0000000..e9c10c9 --- /dev/null +++ b/src/be/nikiroo/fanfix/supported/YiffStar.java @@ -0,0 +1,235 @@ +package be.nikiroo.fanfix.supported; + +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Scanner; + +import be.nikiroo.fanfix.Instance; +import be.nikiroo.fanfix.data.MetaData; +import be.nikiroo.utils.StringUtils; + +/** + * Support class for SoFurry.com, a Furry + * website supporting images and stories (we only retrieve the stories). + * + * @author niki + */ +class YiffStar extends BasicSupport { + + @Override + public String getSourceName() { + return "YiffStar"; + } + + @Override + protected MetaData getMeta(URL source, InputStream in) throws IOException { + MetaData meta = new MetaData(); + + meta.setTitle(getTitle(reset(in))); + meta.setAuthor(getAuthor(source, reset(in))); + meta.setDate(""); + meta.setTags(getTags(reset(in))); + meta.setSource(getSourceName()); + meta.setUrl(source.toString()); + meta.setPublisher(getSourceName()); + meta.setUuid(source.toString()); + meta.setLuid(""); + meta.setLang("EN"); + meta.setSubject("Furry"); + meta.setType(getType().toString()); + meta.setImageDocument(false); + meta.setCover(getCover(source, reset(in))); + + return meta; + } + + @Override + protected boolean supports(URL url) { + String host = url.getHost(); + if (host.startsWith("www.")) { + host = host.substring("www.".length()); + } + + return "sofurry.com".equals(host); + } + + @Override + protected boolean isHtml() { + return true; + } + + @Override + public Map getCookies() { + // TODO + // Cookies will actually be retained by the cache manager once logged in + // But we need to connect here and notify the cache manager + + return super.getCookies(); + } + + @Override + public URL getCanonicalUrl(URL source) throws IOException { + if (source.getPath().startsWith("/view")) { + InputStream in = Instance.getCache().open(source, this, false); + String line = getLine(in, "/browse/folder/", 0); + String[] tab = line.split("\""); + if (tab.length > 1) { + String groupUrl = source.getProtocol() + "://" + + source.getHost() + tab[1]; + return new URL(groupUrl); + } + } + + return super.getCanonicalUrl(source); + } + + private List getTags(InputStream in) { + List tags = new ArrayList(); + + String line = getLine(in, "class=\"sf-story-big-tags", 0); + if (line != null) { + String[] tab = StringUtils.unhtml(line).split(","); + for (String possibleTag : tab) { + String tag = possibleTag.trim(); + if (!tag.isEmpty() && !tag.equals("...") && !tags.contains(tag)) { + tags.add(tag); + } + } + } + + return tags; + } + + private BufferedImage getCover(URL source, InputStream in) + throws IOException { + + List> chaps = getChapters(source, in); + if (!chaps.isEmpty()) { + in = Instance.getCache().open(chaps.get(0).getValue(), this, true); + String line = getLine(in, " name=\"og:image\"", 0); + if (line != null) { + int pos = -1; + for (int i = 0; i < 3; i++) { + pos = line.indexOf('"', pos + 1); + } + + if (pos >= 0) { + line = line.substring(pos + 1); + pos = line.indexOf('"'); + if (pos >= 0) { + line = line.substring(0, pos); + if (line.contains("/thumb?")) { + line = line.replace("/thumb?", + "/auxiliaryContent?type=25&"); + return getImage(this, null, line); + } + } + } + } + } + + return null; + } + + private String getAuthor(URL source, InputStream in) throws IOException { + String author = getLine(in, "class=\"onlinestatus", 0); + if (author != null) { + return StringUtils.unhtml(author).trim(); + } + + return null; + } + + private String getTitle(InputStream in) throws IOException { + String title = getLine(in, "class=\"sflabel pagetitle", 0); + if (title != null) { + if (title.contains("(series)")) { + title = title.replace("(series)", ""); + } + return StringUtils.unhtml(title).trim(); + } + + return null; + } + + @Override + protected String getDesc(URL source, InputStream in) throws IOException { + return null; // TODO: no description at all? Cannot find one... + } + + @Override + protected List> getChapters(URL source, InputStream in) + throws IOException { + List> urls = new ArrayList>(); + + @SuppressWarnings("resource") + Scanner scan = new Scanner(in, "UTF-8"); + scan.useDelimiter("\\n"); + while (scan.hasNext()) { + String line = scan.next(); + if (line.contains("\"/view/") && line.contains("title=")) { + String[] tab = line.split("\""); + if (tab.length > 5) { + String link = tab[5]; + if (link.startsWith("/")) { + link = source.getProtocol() + "://" + source.getHost() + + link; + } + final URL value = new URL(link); + final String key = StringUtils.unhtml(line).trim(); + urls.add(new Entry() { + public URL setValue(URL value) { + return null; + } + + public URL getValue() { + return value; + } + + public String getKey() { + return key; + } + }); + } + } + } + + return urls; + } + + @Override + protected String getChapterContent(URL source, InputStream in, int number) + throws IOException { + StringBuilder builder = new StringBuilder(); + + String startAt = "id=\"sfContentBody"; + String endAt = "id=\"recommendationArea"; + boolean ok = false; + + @SuppressWarnings("resource") + Scanner scan = new Scanner(in, "UTF-8"); + scan.useDelimiter("\\n"); + while (scan.hasNext()) { + String line = scan.next(); + if (!ok && line.contains(startAt)) { + ok = true; + } else if (ok && line.contains(endAt)) { + ok = false; + break; + } + + if (ok) { + builder.append(line); + builder.append('\n'); + } + } + + return builder.toString(); + } +} -- 2.27.0