package be.nikiroo.fanfix.supported;
import java.awt.image.BufferedImage;
+import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
-import javax.imageio.ImageIO;
-
import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.bundles.Config;
import be.nikiroo.fanfix.bundles.StringId;
import be.nikiroo.fanfix.data.Paragraph;
import be.nikiroo.fanfix.data.Paragraph.ParagraphType;
import be.nikiroo.fanfix.data.Story;
+import be.nikiroo.utils.IOUtils;
+import be.nikiroo.utils.Progress;
import be.nikiroo.utils.StringUtils;
/**
MANGAFOX,
/** Furry website with comics support */
E621,
+ /** Furry website with stories */
+ YIFFSTAR,
/** CBZ files */
- CBZ;
+ CBZ,
+ /** HTML files */
+ HTML;
/**
* A description of this support type (more information than the
private InputStream in;
private SupportType type;
- private URL currentReferer; // with on 'r', as in 'HTTP'...
+ private URL currentReferer; // with only one 'r', as in 'HTTP'...
// quote chars
private char openQuote = Instance.getTrans().getChar(
protected abstract String getChapterContent(URL source, InputStream in,
int number) throws IOException;
+ /**
+ * Log into the support (can be a no-op depending upon the support).
+ *
+ * @throws IOException
+ * in case of I/O error
+ */
+ public void login() throws IOException {
+
+ }
+
/**
* Return the list of cookies (values included) that must be used to
* correctly fetch the resources.
* it.
*
* @return the cookies
+ *
+ * @throws IOException
+ * in case of I/O error
*/
- public Map<String, String> getCookies() {
+ public Map<String, String> getCookies() throws IOException {
return new HashMap<String, String>();
}
+ /**
+ * Return the canonical form of the main {@link URL}.
+ *
+ * @param source
+ * the source {@link URL}
+ *
+ * @return the canonical form of this {@link URL}
+ *
+ * @throws IOException
+ * in case of I/O error
+ */
+ public URL getCanonicalUrl(URL source) throws IOException {
+ return source;
+ }
+
/**
* Process the given story resource into a partially filled {@link Story}
* object containing the name and metadata, except for the description.
*/
protected Story processMeta(URL url, boolean close, boolean getDesc)
throws IOException {
- in = Instance.getCache().open(url, this, false);
+ login();
+
+ url = getCanonicalUrl(url);
+
+ setCurrentReferer(url);
+
+ in = openInput(url);
if (in == null) {
return null;
}
Story story = new Story();
MetaData meta = getMeta(url, getInput());
+ if (meta.getCreationDate() == null
+ || meta.getCreationDate().isEmpty()) {
+ meta.setCreationDate(StringUtils.fromTime(new Date().getTime()));
+ }
story.setMeta(meta);
if (meta != null && meta.getCover() == null) {
in.close();
}
}
+
+ setCurrentReferer(null);
}
}
*
* @param url
* the story resource
+ * @param pg
+ * the optional progress reporter
*
* @return the {@link Story}
*
* @throws IOException
* in case of I/O error
*/
- public Story process(URL url) throws IOException {
- setCurrentReferer(url);
+ public Story process(URL url, Progress pg) throws IOException {
+ if (pg == null) {
+ pg = new Progress();
+ } else {
+ pg.setMinMax(0, 100);
+ }
+ url = getCanonicalUrl(url);
+ pg.setProgress(1);
try {
Story story = processMeta(url, false, true);
+ pg.setProgress(10);
if (story == null) {
+ pg.setProgress(100);
return null;
}
+ pg.setName("Retrieving " + story.getMeta().getTitle());
+
+ setCurrentReferer(url);
+
story.setChapters(new ArrayList<Chapter>());
List<Entry<String, URL>> chapters = getChapters(url, getInput());
+ pg.setProgress(20);
+
int i = 1;
if (chapters != null) {
+ Progress pgChaps = new Progress(0, chapters.size());
+ pg.addProgress(pgChaps, 80);
+
+ long words = 0;
for (Entry<String, URL> chap : chapters) {
setCurrentReferer(chap.getValue());
InputStream chapIn = Instance.getCache().open(
chap.getValue(), this, true);
try {
- story.getChapters().add(
- makeChapter(url, i, chap.getKey(),
- getChapterContent(url, chapIn, i)));
+ Chapter cc = makeChapter(url, i, chap.getKey(),
+ getChapterContent(url, chapIn, i));
+ words += cc.getWords();
+ story.getChapters().add(cc);
+ if (story.getMeta() != null) {
+ story.getMeta().setWords(words);
+ }
} finally {
chapIn.close();
}
- i++;
+
+ pgChaps.setProgress(i++);
}
+ } else {
+ pg.setProgress(100);
}
return story;
in.close();
}
- currentReferer = null;
+ setCurrentReferer(null);
}
}
/**
- * The support type.$
+ * The support type.
*
* @return the type
*/
*/
protected Chapter makeChapter(URL source, int number, String name,
String content) throws IOException {
-
// Chapter name: process it correctly, then remove the possible
// redundant "Chapter x: " in front of it
String chapterName = processPara(name).getContent().trim();
Chapter chap = new Chapter(number, chapterName);
- if (content == null) {
- return chap;
+ if (content != null) {
+ List<Paragraph> paras = makeParagraphs(source, content);
+ long words = 0;
+ for (Paragraph para : paras) {
+ words += para.getWords();
+ }
+ chap.setParagraphs(paras);
+ chap.setWords(words);
}
+ return chap;
+
+ }
+
+ /**
+ * Convert the given content into {@link Paragraph}s.
+ *
+ * @param source
+ * the source URL of the story
+ * @param content
+ * the textual content
+ *
+ * @return the {@link Paragraph}s
+ *
+ * @throws IOException
+ * in case of I/O error
+ */
+ protected List<Paragraph> makeParagraphs(URL source, String content)
+ throws IOException {
if (isHtml()) {
// Special <HR> processing:
content = content.replaceAll("(<hr [^>]*>)|(<hr/>)|(<hr>)",
"\n* * *\n");
}
+ List<Paragraph> paras = new ArrayList<Paragraph>();
InputStream in = new ByteArrayInputStream(content.getBytes("UTF-8"));
try {
- @SuppressWarnings("resource")
- Scanner scan = new Scanner(in, "UTF-8");
- scan.useDelimiter("(\\n|</p>)"); // \n for test, </p> for html
-
- List<Paragraph> paras = new ArrayList<Paragraph>();
- while (scan.hasNext()) {
- String line = scan.next().trim();
- boolean image = false;
- if (line.startsWith("[") && line.endsWith("]")) {
- URL url = getImageUrl(source,
- line.substring(1, line.length() - 1).trim());
- if (url != null) {
- paras.add(new Paragraph(url));
- image = true;
- }
+ BufferedReader buff = new BufferedReader(new InputStreamReader(in,
+ "UTF-8"));
+
+ for (String encodedLine = buff.readLine(); encodedLine != null; encodedLine = buff
+ .readLine()) {
+ String lines[];
+ if (isHtml()) {
+ lines = encodedLine.split("(<p>|</p>|<br>|<br/>|\\n)");
+ } else {
+ lines = new String[] { encodedLine };
}
- if (!image) {
- paras.add(processPara(line));
+ for (String aline : lines) {
+ String line = aline.trim();
+
+ URL image = null;
+ if (line.startsWith("[") && line.endsWith("]")) {
+ image = getImageUrl(this, source,
+ line.substring(1, line.length() - 1).trim());
+ }
+
+ if (image != null) {
+ paras.add(new Paragraph(image));
+ } else {
+ paras.add(processPara(line));
+ }
}
}
+ } finally {
+ in.close();
+ }
- // Check quotes for "bad" format
- List<Paragraph> newParas = new ArrayList<Paragraph>();
- for (Paragraph para : paras) {
- newParas.addAll(requotify(para));
- }
- paras = newParas;
-
- // Remove double blanks/brks
- boolean space = false;
- boolean brk = true;
- for (int i = 0; i < paras.size(); i++) {
- Paragraph para = paras.get(i);
- boolean thisSpace = para.getType() == ParagraphType.BLANK;
- boolean thisBrk = para.getType() == ParagraphType.BREAK;
-
- if (space && thisBrk) {
- paras.remove(i - 1);
- i--;
- } else if ((space || brk) && (thisSpace || thisBrk)) {
- paras.remove(i);
- i--;
- }
+ // Check quotes for "bad" format
+ List<Paragraph> newParas = new ArrayList<Paragraph>();
+ for (Paragraph para : paras) {
+ newParas.addAll(requotify(para));
+ }
+ paras = newParas;
- space = thisSpace;
- brk = thisBrk;
- }
+ // Remove double blanks/brks
+ fixBlanksBreaks(paras);
- // Remove blank/brk at start
- if (paras.size() > 0
- && (paras.get(0).getType() == ParagraphType.BLANK || paras
- .get(0).getType() == ParagraphType.BREAK)) {
- paras.remove(0);
- }
+ return paras;
+ }
- // Remove blank/brk at end
- int last = paras.size() - 1;
- if (paras.size() > 0
- && (paras.get(last).getType() == ParagraphType.BLANK || paras
- .get(last).getType() == ParagraphType.BREAK)) {
- paras.remove(last);
+ /**
+ * Fix the {@link ParagraphType#BLANK}s and {@link ParagraphType#BREAK}s of
+ * those {@link Paragraph}s.
+ * <p>
+ * The resulting list will not contain a starting or trailing blank/break
+ * nor 2 blanks or breaks following each other.
+ *
+ * @param paras
+ * the list of {@link Paragraph}s to fix
+ */
+ protected void fixBlanksBreaks(List<Paragraph> paras) {
+ boolean space = false;
+ boolean brk = true;
+ for (int i = 0; i < paras.size(); i++) {
+ Paragraph para = paras.get(i);
+ boolean thisSpace = para.getType() == ParagraphType.BLANK;
+ boolean thisBrk = para.getType() == ParagraphType.BREAK;
+
+ if (i > 0 && space && thisBrk) {
+ paras.remove(i - 1);
+ i--;
+ } else if ((space || brk) && (thisSpace || thisBrk)) {
+ paras.remove(i);
+ i--;
}
- chap.setParagraphs(paras);
+ space = thisSpace;
+ brk = thisBrk;
+ }
- return chap;
- } finally {
- in.close();
+ // Remove blank/brk at start
+ if (paras.size() > 0
+ && (paras.get(0).getType() == ParagraphType.BLANK || paras.get(
+ 0).getType() == ParagraphType.BREAK)) {
+ paras.remove(0);
+ }
+
+ // Remove blank/brk at end
+ int last = paras.size() - 1;
+ if (paras.size() > 0
+ && (paras.get(last).getType() == ParagraphType.BLANK || paras
+ .get(last).getType() == ParagraphType.BREAK)) {
+ paras.remove(last);
}
}
+ /**
+ * Get the default cover related to this subject (see <tt>.info</tt> files).
+ *
+ * @param subject
+ * the subject
+ *
+ * @return the cover if any, or NULL
+ */
static BufferedImage getDefaultCover(String subject) {
if (subject != null && !subject.isEmpty()
&& Instance.getCoverDir() != null) {
try {
File fileCover = new File(Instance.getCoverDir(), subject);
- return getImage(fileCover.toURI().toURL(), subject);
+ return getImage(null, fileCover.toURI().toURL(), subject);
} catch (MalformedURLException e) {
}
}
/**
* Return the list of supported image extensions.
*
+ * @param emptyAllowed
+ * TRUE to allow an empty extension on first place, which can be
+ * used when you may already have an extension in your input but
+ * are not sure about it
+ *
* @return the extensions
*/
static String[] getImageExt(boolean emptyAllowed) {
}
}
- static BufferedImage getImage(URL source, String line) {
- URL url = getImageUrl(source, line);
+ /**
+ * Check if the given resource can be a local image or a remote image, then
+ * refresh the cache with it if it is.
+ *
+ * @param source
+ * the story source
+ * @param line
+ * the resource to check
+ *
+ * @return the image if found, or NULL
+ *
+ */
+ static BufferedImage getImage(BasicSupport support, URL source, String line) {
+ URL url = getImageUrl(support, source, line);
if (url != null) {
InputStream in = null;
try {
in = Instance.getCache().open(url, getSupport(url), true);
- return ImageIO.read(in);
+ return IOUtils.toImage(in);
} catch (IOException e) {
} finally {
if (in != null) {
* @return the image URL if found, or NULL
*
*/
- static URL getImageUrl(URL source, String line) {
+ static URL getImageUrl(BasicSupport support, URL source, String line) {
URL url = null;
if (line != null) {
if (source != null) {
path = new File(source.getFile()).getParent();
try {
- String urlBase = new File(new File(path), line.trim())
- .toURI().toURL().toString();
+ String basePath = new File(new File(path), line.trim())
+ .getAbsolutePath();
for (String ext : getImageExt(true)) {
- if (new File(urlBase + ext).exists()) {
- url = new File(urlBase + ext).toURI().toURL();
+ if (new File(basePath + ext).exists()) {
+ url = new File(basePath + ext).toURI().toURL();
}
}
} catch (Exception e) {
for (String ext : getImageExt(true)) {
if (Instance.getCache().check(new URL(line + ext))) {
url = new URL(line + ext);
+ break;
}
}
for (String ext : getImageExt(true)) {
try {
url = new URL(line + ext);
- Instance.getCache().refresh(url,
- getSupport(url), true);
+ Instance.getCache().refresh(url, support, true);
break;
} catch (IOException e) {
// no image with this ext
// refresh the cached file
if (url != null) {
try {
- Instance.getCache().refresh(url, getSupport(url), true);
+ Instance.getCache().refresh(url, support, true);
} catch (IOException e) {
// woops, broken image
url = null;
return url;
}
+ /**
+ * Open the input file that will be used through the support.
+ *
+ * @param source
+ * the source {@link URL}
+ *
+ * @return the {@link InputStream}
+ *
+ * @throws IOException
+ * in case of I/O error
+ */
+ protected InputStream openInput(URL source) throws IOException {
+ return Instance.getCache().open(source, this, false);
+ }
+
+ /**
+ * Reset the given {@link InputStream} and return it.
+ *
+ * @param in
+ * the {@link InputStream} to reset
+ *
+ * @return the same {@link InputStream} after reset
+ */
protected InputStream reset(InputStream in) {
try {
in.reset();
* paragraphs (quotes or not)).
*
* @param para
- * the paragraph to requotify (not necessaraly a quote)
+ * the paragraph to requotify (not necessarily a quote)
*
* @return the correctly (or so we hope) quotified paragraphs
*/
- pr
ivate List<Paragraph> requotify(Paragraph para) {
+ pr
otected List<Paragraph> requotify(Paragraph para) {
List<Paragraph> newParas = new ArrayList<Paragraph>();
if (para.getType() == ParagraphType.QUOTE
boolean singleQ = line.startsWith("" + openQuote);
boolean doubleQ = line.startsWith("" + openDoubleQuote);
+ // Do not try when more than one quote at a time
+ // (some stories are not easily readable if we do)
+ if (singleQ
+ && line.indexOf(closeQuote, 1) < line
+ .lastIndexOf(closeQuote)) {
+ newParas.add(para);
+ return newParas;
+ }
+ if (doubleQ
+ && line.indexOf(closeDoubleQuote, 1) < line
+ .lastIndexOf(closeDoubleQuote)) {
+ newParas.add(para);
+ return newParas;
+ }
+ //
+
if (!singleQ && !doubleQ) {
line = openDoubleQuote + line + closeDoubleQuote;
- newParas.add(new Paragraph(ParagraphType.QUOTE, line));
+ newParas.add(new Paragraph(ParagraphType.QUOTE, line, para
+ .getWords()));
} else {
+ char open = singleQ ? openQuote : openDoubleQuote;
char close = singleQ ? closeQuote : closeDoubleQuote;
- int posClose = line.indexOf(close, 1);
- int posDot = line.indexOf(".");
- while (posDot >= 0 && posDot < posClose) {
- posDot = line.indexOf(".", posDot + 1);
+
+ int posDot = -1;
+ boolean inQuote = false;
+ int i = 0;
+ for (char car : line.toCharArray()) {
+ if (car == open) {
+ inQuote = true;
+ } else if (car == close) {
+ inQuote = false;
+ } else if (car == '.' && !inQuote) {
+ posDot = i;
+ break;
+ }
+ i++;
}
if (posDot >= 0) {
String rest = line.substring(posDot + 1).trim();
line = line.substring(0, posDot + 1).trim();
- newParas.add(new Paragraph(ParagraphType.QUOTE, line));
+ long words = 1;
+ for (char car : line.toCharArray()) {
+ if (car == ' ') {
+ words++;
+ }
+ }
+ newParas.add(new Paragraph(ParagraphType.QUOTE, line, words));
if (!rest.isEmpty()) {
newParas.addAll(requotify(processPara(rest)));
}
*
* @return the processed {@link Paragraph}
*/
- private Paragraph processPara(String line) {
+ protected Paragraph processPara(String line) {
line = ifUnhtml(line).trim();
boolean space = true;
boolean tentativeCloseQuote = false;
char prev = '\0';
int dashCount = 0;
+ long words = 1;
StringBuilder builder = new StringBuilder();
for (char car : line.toCharArray()) {
if (tentativeCloseQuote) {
tentativeCloseQuote = false;
- if ((car >= 'a' && car <= 'z') || (car >= 'A' && car <= 'Z')
- || (car >= '0' && car <= '9')) {
+ if (Character.isLetterOrDigit(car)) {
builder.append("'");
} else {
- builder.append(closeQuote);
+ // handle double-single quotes as double quotes
+ if (prev == car) {
+ builder.append(closeDoubleQuote);
+ continue;
+ } else {
+ builder.append(closeQuote);
+ }
}
}
case '\t':
case '\n': // just in case
case '\r': // just in case
+ if (builder.length() > 0
+ && builder.charAt(builder.length() - 1) != ' ') {
+ words++;
+ }
builder.append(' ');
break;
case '\'':
if (space || (brk && quote)) {
quote = true;
- builder.append(openQuote);
- } else if (prev == ' ') {
- builder.append(openQuote);
+ // handle double-single quotes as double quotes
+ if (prev == car) {
+ builder.deleteCharAt(builder.length() - 1);
+ builder.append(openDoubleQuote);
+ } else {
+ builder.append(openQuote);
+ }
+ } else if (prev == ' ' || prev == car) {
+ // handle double-single quotes as double quotes
+ if (prev == car) {
+ builder.deleteCharAt(builder.length() - 1);
+ builder.append(openDoubleQuote);
+ } else {
+ builder.append(openQuote);
+ }
} else {
// it is a quote ("I'm off") or a 'quote' ("This
// 'good' restaurant"...)
quote = true;
builder.append(openQuote);
} else {
- builder.append(openQuote);
+ // handle double-single quotes as double quotes
+ if (prev == car) {
+ builder.deleteCharAt(builder.length() - 1);
+ builder.append(openDoubleQuote);
+ } else {
+ builder.append(openQuote);
+ }
}
space = false;
brk = false;
case '」':
space = false;
brk = false;
- builder.append(closeQuote);
+ // handle double-single quotes as double quotes
+ if (prev == car) {
+ builder.deleteCharAt(builder.length() - 1);
+ builder.append(closeDoubleQuote);
+ } else {
+ builder.append(closeQuote);
+ }
break;
case '«':
type = ParagraphType.QUOTE;
}
- return new Paragraph(type, line);
+ return new Paragraph(type, line, words);
}
/**
- * Remove the HTML from the inpit <b>if</b> {@link BasicSupport#isHtml()} is
+ * Remove the HTML from the input <b>if</b> {@link BasicSupport#isHtml()} is
* true.
*
* @param input
}
}
- for (SupportType type : new SupportType[] { SupportType.TEXT,
- SupportType.INFO_TEXT }) {
+ for (SupportType type : new SupportType[] { SupportType.INFO_TEXT,
+ SupportType.TEXT }) {
BasicSupport support = getSupport(type);
if (support != null && support.supports(url)) {
return support;
return new MangaFox().setType(type);
case E621:
return new E621().setType(type);
+ case YIFFSTAR:
+ return new YiffStar().setType(type);
case CBZ:
return new Cbz().setType(type);
+ case HTML:
+ return new Html().setType(type);
}
return null;
git://git.nikiroo.be / fanfix.git / blobdiff