* the source of the story
* @param in
* the input (the main resource)
+ * @param pg
+ * the optional progress reporter
*
* @return the chapters
*
* in case of I/O error
*/
protected abstract List<Entry<String, URL>> getChapters(URL source,
- InputStream in) throws IOException;
+ InputStream in, Progress pg) throws IOException;
/**
* Return the content of the chapter (possibly HTML encoded, if
* the input (the main resource)
* @param number
* the chapter number
+ * @param pg
+ * the optional progress reporter
*
* @return the content
*
* in case of I/O error
*/
protected abstract String getChapterContent(URL source, InputStream in,
- int number) throws IOException;
+ int number, Progress pg) throws IOException;
/**
* Log into the support (can be a no-op depending upon the support).
* in case of I/O error
*/
public Story processMeta(URL url) throws IOException {
- return processMeta(url, true, false);
+ return processMeta(url, true, false, null);
}
/**
*
* @param close
* close "this" and "in" when done
+ * @param pg
+ * the optional progress reporter
*
* @return the {@link Story}
*
* @throws IOException
* in case of I/O error
*/
- protected Story processMeta(URL url, boolean close, boolean getDesc)
- throws IOException {
+ protected Story processMeta(URL url, boolean close, boolean getDesc,
+ Progress pg) throws IOException {
+ if (pg == null) {
+ pg = new Progress();
+ } else {
+ pg.setMinMax(0, 100);
+ }
+
login();
+ pg.setProgress(10);
url = getCanonicalUrl(url);
try {
preprocess(url, getInput());
+ pg.setProgress(30);
Story story = new Story();
MetaData meta = getMeta(url, getInput());
}
story.setMeta(meta);
+ pg.setProgress(50);
+
if (meta != null && meta.getCover() == null) {
meta.setCover(getDefaultCover(meta.getSubject()));
}
+ pg.setProgress(60);
+
if (getDesc) {
String descChapterName = Instance.getTrans().getString(
StringId.DESCRIPTION);
story.getMeta().setResume(
makeChapter(url, 0, descChapterName,
- getDesc(url, getInput())));
+ getDesc(url, getInput()), null));
}
+ pg.setProgress(100);
return story;
} finally {
if (close) {
url = getCanonicalUrl(url);
pg.setProgress(1);
try {
- Story story = processMeta(url, false, true);
- pg.setProgress(10);
+ Progress pgMeta = new Progress();
+ pg.addProgress(pgMeta, 10);
+ Story story = processMeta(url, false, true, pgMeta);
+ if (!pgMeta.isDone()) {
+ pgMeta.setProgress(pgMeta.getMax()); // 10%
+ }
+
if (story == null) {
- pg.setProgress(100);
+ pg.setProgress(90);
return null;
}
setCurrentReferer(url);
+ Progress pgGetChapters = new Progress();
+ pg.addProgress(pgGetChapters, 10);
story.setChapters(new ArrayList<Chapter>());
+ List<Entry<String, URL>> chapters = getChapters(url, getInput(),
+ pgGetChapters);
+ if (!pgGetChapters.isDone()) {
+ pgGetChapters.setProgress(pgGetChapters.getMax()); // 20%
+ }
- List<Entry<String, URL>> chapters = getChapters(url, getInput());
- pg.setProgress(20);
-
- int i = 1;
if (chapters != null) {
- Progress pgChaps = new Progress(0, chapters.size());
+ Progress pgChaps = new Progress("Extracting chapters", 0,
+ chapters.size() * 300);
pg.addProgress(pgChaps, 80);
long words = 0;
+ int i = 1;
for (Entry<String, URL> chap : chapters) {
+ pgChaps.setName("Extracting chapter " + i);
setCurrentReferer(chap.getValue());
InputStream chapIn = Instance.getCache().open(
chap.getValue(), this, true);
+ pgChaps.setProgress(i * 100);
try {
+ Progress pgGetChapterContent = new Progress();
+ Progress pgMakeChapter = new Progress();
+ pgChaps.addProgress(pgGetChapterContent, 100);
+ pgChaps.addProgress(pgMakeChapter, 100);
+
+ String content = getChapterContent(url, chapIn, i,
+ pgGetChapterContent);
+ if (!pgGetChapterContent.isDone()) {
+ pgGetChapterContent.setProgress(pgGetChapterContent
+ .getMax());
+ }
+
Chapter cc = makeChapter(url, i, chap.getKey(),
- getChapterContent(url, chapIn, i));
+ content, pgMakeChapter);
+ if (!pgMakeChapter.isDone()) {
+ pgMakeChapter.setProgress(pgMakeChapter.getMax());
+ }
+
words += cc.getWords();
story.getChapters().add(cc);
if (story.getMeta() != null) {
chapIn.close();
}
- pgChaps.setProgress(i++);
+ i++;
}
+
+ pgChaps.setName("Extracting chapters");
} else {
- pg.setProgress(100);
+ pg.setProgress(80);
}
return story;
* the chapter name
* @param content
* the chapter content
+ * @param pg
+ * the optional progress reporter
*
* @return the {@link Chapter}
*
* in case of I/O error
*/
protected Chapter makeChapter(URL source, int number, String name,
- String content) throws IOException {
+ String content, Progress pg) throws IOException {
// Chapter name: process it correctly, then remove the possible
// redundant "Chapter x: " in front of it
String chapterName = processPara(name).getContent().trim();
Chapter chap = new Chapter(number, chapterName);
if (content != null) {
- List<Paragraph> paras = makeParagraphs(source, content);
+ List<Paragraph> paras = makeParagraphs(source, content, pg);
long words = 0;
for (Paragraph para : paras) {
words += para.getWords();
* the source URL of the story
* @param content
* the textual content
+ * @param pg
+ * the optional progress reporter
*
* @return the {@link Paragraph}s
*
* @throws IOException
* in case of I/O error
*/
- protected List<Paragraph> makeParagraphs(URL source, String content)
- throws IOException {
+ protected List<Paragraph> makeParagraphs(URL source, String content,
+ Progress pg) throws IOException {
+ if (pg == null) {
+ pg = new Progress();
+ }
+
if (isHtml()) {
// Special <HR> processing:
content = content.replaceAll("(<hr [^>]*>)|(<hr/>)|(<hr>)",
if (content != null && !content.trim().isEmpty()) {
if (isHtml()) {
- for (String line : content.split("(<p>|</p>|<br>|<br/>)")) {
+ String[] tab = content.split("(<p>|</p>|<br>|<br/>)");
+ pg.setMinMax(0, tab.length);
+ int i = 1;
+ for (String line : tab) {
+ if (line.startsWith("[") && line.endsWith("]")) {
+ pg.setName("Extracting image " + i);
+ }
paras.add(makeParagraph(source, line.trim()));
+ pg.setProgress(i++);
}
+ pg.setName(null);
} else {
+ List<String> lines = new ArrayList<String>();
BufferedReader buff = null;
try {
buff = new BufferedReader(
content.getBytes("UTF-8")), "UTF-8"));
for (String line = buff.readLine(); line != null; line = buff
.readLine()) {
- paras.add(makeParagraph(source, line.trim()));
+ lines.add(line.trim());
}
} finally {
if (buff != null) {
buff.close();
}
}
+
+ pg.setMinMax(0, lines.size());
+ int i = 0;
+ for (String line : lines) {
+ if (line.startsWith("[") && line.endsWith("]")) {
+ pg.setName("Extracting image " + i);
+ }
+ paras.add(makeParagraph(source, line));
+ pg.setProgress(i++);
+ }
+ pg.setName(null);
}
// Check quotes for "bad" format
pg.setMinMax(0, 100);
}
- Story story = processMeta(url, false, true);
+ Progress pgMeta = new Progress();
+ pg.addProgress(pgMeta, 10);
+ Story story = processMeta(url, false, true, pgMeta);
+ if (!pgMeta.isDone()) {
+ pgMeta.setProgress(pgMeta.getMax()); // 10%
+ }
+
story.setChapters(new ArrayList<Chapter>());
Chapter chap = new Chapter(1, null);
story.getChapters().add(chap);
ZipInputStream zipIn = new ZipInputStream(getInput());
- pg.setProgress(10);
List<String> images = new ArrayList<String>();
for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn
.getNextEntry()) {
private BufferedImage getCover(URL source) throws IOException {
InputStream in = Instance.getCache().open(source, this, true);
String images = getChapterContent(new URL(source.toString() + "?page="
- + 1), in, 1);
+ + 1), in, 1, null);
if (!images.isEmpty()) {
int pos = images.indexOf("<br/>");
if (pos >= 0) {
}
@Override
- protected List<Entry<String, URL>> getChapters(URL source, InputStream in)
- throws IOException {
+ protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
+ Progress pg) throws IOException {
List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
int last = 1; // no pool/show when only one page
}
@Override
- protected String getChapterContent(URL source, InputStream in, int number)
- throws IOException {
+ protected String getChapterContent(URL source, InputStream in, int number,
+ Progress pg) throws IOException {
StringBuilder builder = new StringBuilder();
String staticSite = "https://static1.e621.net";
if (source.getHost().contains("e926")) {
import be.nikiroo.fanfix.data.MetaData;
import be.nikiroo.utils.IOUtils;
import be.nikiroo.utils.MarkableFileInputStream;
+import be.nikiroo.utils.Progress;
/**
* Support class for EPUB files created with this program (as we need some
}
@Override
- protected List<Entry<String, URL>> getChapters(URL source, InputStream in)
- throws IOException {
+ protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
+ Progress pg) throws IOException {
if (fakeIn != null) {
fakeIn.reset();
- return super.getChapters(fakeSource, fakeIn);
+ return super.getChapters(fakeSource, fakeIn, pg);
}
return null;
}
@Override
- protected String getChapterContent(URL source, InputStream in, int number)
- throws IOException {
+ protected String getChapterContent(URL source, InputStream in, int number,
+ Progress pg) throws IOException {
if (fakeIn != null) {
fakeIn.reset();
- return super.getChapterContent(fakeSource, fakeIn, number);
+ return super.getChapterContent(fakeSource, fakeIn, number, pg);
}
return null;
import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.bundles.Config;
import be.nikiroo.fanfix.data.MetaData;
+import be.nikiroo.utils.Progress;
import be.nikiroo.utils.StringUtils;
/**
}
@Override
- protected List<Entry<String, URL>> getChapters(URL source, InputStream in) {
+ protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
+ Progress pg) {
List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
String base = source.toString();
}
@Override
- protected String getChapterContent(URL source, InputStream in, int number) {
+ protected String getChapterContent(URL source, InputStream in, int number,
+ Progress pg) {
StringBuilder builder = new StringBuilder();
String startAt = "class='storytext ";
String endAt1 = "function review_init";
import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.data.MetaData;
+import be.nikiroo.utils.Progress;
import be.nikiroo.utils.StringUtils;
/**
}
@Override
- protected List<Entry<String, URL>> getChapters(URL source, InputStream in) {
+ protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
+ Progress pg) {
List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
@SuppressWarnings("resource")
Scanner scan = new Scanner(in, "UTF-8");
}
@Override
- protected String getChapterContent(URL source, InputStream in, int number) {
+ protected String getChapterContent(URL source, InputStream in, int number,
+ Progress pg) {
return getLine(in, "<div id=\"chapter_container\">", 1);
}
import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.data.MetaData;
import be.nikiroo.utils.IOUtils;
+import be.nikiroo.utils.Progress;
import be.nikiroo.utils.StringUtils;
class MangaFox extends BasicSupport {
}
@Override
- protected List<Entry<String, URL>> getChapters(URL source, InputStream in) {
+ protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
+ Progress pg) {
List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
String volumeAt = "<h3 class=\"volume\">";
}
}
- // to help with the retry and the originalUrl
- refresh(url);
-
try {
final String key = name;
final URL value = new URL(url);
}
}
+ if (pg == null) {
+ pg = new Progress(0, urls.size());
+ } else {
+ pg.setMinMax(0, urls.size());
+ }
+
+ int i = 1;
+ for (Entry<String, URL> entry : urls) {
+ // to help with the retry and the originalUrl
+ refresh(entry.getValue().toString());
+ pg.setProgress(i++);
+ }
+
// the chapters are in reversed order
Collections.reverse(urls);
}
@Override
- protected String getChapterContent(URL source, InputStream in, int number) {
+ protected String getChapterContent(URL source, InputStream in, int number,
+ Progress pg) {
+ if (pg == null) {
+ pg = new Progress();
+ } else {
+ // Since we have no idea how many images we have, we cycle from 0
+ // to max, then again, then again...
+ pg.setMinMax(0, 20);
+ }
+
StringBuilder builder = new StringBuilder();
String base = getCurrentReferer().toString();
int pos = base.lastIndexOf('/');
base = base.substring(0, pos + 1); // including the '/' at the end
+ int i = 1;
boolean close = false;
while (in != null) {
String linkNextLine = getLine(in, "return enlarge()", 0);
// to help with the retry and the originalUrl, part 2
refresh(linkImage);
+ pg.setProgress((i++) % pg.getMax());
if (close) {
try {
url = new URL(base + linkNext);
in = openEx(base + linkNext);
setCurrentReferer(url);
+ pg.setProgress((i++) % pg.getMax());
} catch (IOException e) {
Instance.syserr(new IOException(
"Cannot get the next manga page which is: "
import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.bundles.Config;
import be.nikiroo.fanfix.data.MetaData;
+import be.nikiroo.utils.Progress;
/**
* Support class for local stories encoded in textual format, with a few rules:
@Override
protected String getDesc(URL source, InputStream in) throws IOException {
- return getChapterContent(source, in, 0);
+ return getChapterContent(source, in, 0, null);
}
private BufferedImage getCover(URL source) throws IOException {
}
@Override
- protected List<Entry<String, URL>> getChapters(URL source, InputStream in)
- throws IOException {
+ protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
+ Progress pg) throws IOException {
List<Entry<String, URL>> chaps = new ArrayList<Entry<String, URL>>();
@SuppressWarnings("resource")
Scanner scan = new Scanner(in, "UTF-8");
}
@Override
- protected String getChapterContent(URL source, InputStream in, int number)
- throws IOException {
+ protected String getChapterContent(URL source, InputStream in, int number,
+ Progress pg) throws IOException {
StringBuilder builder = new StringBuilder();
@SuppressWarnings("resource")
Scanner scan = new Scanner(in, "UTF-8");
import be.nikiroo.fanfix.Instance;
import be.nikiroo.fanfix.bundles.Config;
import be.nikiroo.fanfix.data.MetaData;
+import be.nikiroo.utils.Progress;
import be.nikiroo.utils.StringUtils;
/**
private BufferedImage getCover(URL source, InputStream in)
throws IOException {
- List<Entry<String, URL>> chaps = getChapters(source, in);
+ List<Entry<String, URL>> chaps = getChapters(source, in, null);
if (!chaps.isEmpty()) {
in = Instance.getCache().open(chaps.get(0).getValue(), this, true);
String line = getLine(in, " name=\"og:image\"", 0);
}
@Override
- protected List<Entry<String, URL>> getChapters(URL source, InputStream in)
- throws IOException {
+ protected List<Entry<String, URL>> getChapters(URL source, InputStream in,
+ Progress pg) throws IOException {
List<Entry<String, URL>> urls = new ArrayList<Entry<String, URL>>();
@SuppressWarnings("resource")
}
@Override
- protected String getChapterContent(URL source, InputStream in, int number)
- throws IOException {
+ protected String getChapterContent(URL source, InputStream in, int number,
+ Progress pg) throws IOException {
StringBuilder builder = new StringBuilder();
String startAt = "id=\"sfContentBody";
import be.nikiroo.fanfix.supported.BasicSupport;
import be.nikiroo.fanfix.supported.BasicSupport.SupportType;
import be.nikiroo.utils.IOUtils;
+import be.nikiroo.utils.Progress;
import be.nikiroo.utils.test.TestCase;
import be.nikiroo.utils.test.TestLauncher;
List<Paragraph> paras = null;
- paras = support.makeParagraphs(null, "");
+ paras = support.makeParagraphs(null, "", null);
assertEquals(
"An empty content should not generate paragraphs",
0, paras.size());
paras = support.makeParagraphs(null,
- "Line 1</p><p>Line 2</p><p>Line 3</p>");
+ "Line 1</p><p>Line 2</p><p>Line 3</p>", null);
assertEquals(5, paras.size());
assertEquals("Line 1", paras.get(0).getContent());
assertEquals(ParagraphType.BLANK, paras.get(1)
assertEquals("Line 3", paras.get(4).getContent());
paras = support.makeParagraphs(null,
- "<p>Line1</p><p>Line2</p><p>Line3</p>");
+ "<p>Line1</p><p>Line2</p><p>Line3</p>", null);
assertEquals(6, paras.size());
}
});
List<Paragraph> paras = null;
paras = support
- .makeParagraphs(null,
- "<p>Line1</p><p>Line2</p><p>Line3<br/><br><p></p>");
+ .makeParagraphs(
+ null,
+ "<p>Line1</p><p>Line2</p><p>Line3<br/><br><p></p>",
+ null);
assertEquals(5, paras.size());
paras = support
- .makeParagraphs(null,
- "<p>Line1</p><p>Line2</p><p>Line3<br/><br><p></p>* * *");
+ .makeParagraphs(
+ null,
+ "<p>Line1</p><p>Line2</p><p>Line3<br/><br><p></p>* * *",
+ null);
assertEquals(5, paras.size());
- paras = support.makeParagraphs(null, "1<p>* * *<p>2");
+ paras = support.makeParagraphs(null, "1<p>* * *<p>2",
+ null);
assertEquals(3, paras.size());
assertEquals(ParagraphType.BREAK, paras.get(1)
.getType());
paras = support.makeParagraphs(null,
- "1<p><br/><p>* * *<p>2");
+ "1<p><br/><p>* * *<p>2", null);
assertEquals(3, paras.size());
assertEquals(ParagraphType.BREAK, paras.get(1)
.getType());
paras = support.makeParagraphs(null,
- "1<p>* * *<br/><p><br><p>2");
+ "1<p>* * *<br/><p><br><p>2", null);
assertEquals(3, paras.size());
assertEquals(ParagraphType.BREAK, paras.get(1)
.getType());
paras = support.makeParagraphs(null,
- "1<p><br/><br>* * *<br/><p><br><p>2");
+ "1<p><br/><br>* * *<br/><p><br><p>2", null);
assertEquals(3, paras.size());
assertEquals(ParagraphType.BREAK, paras.get(1)
.getType());
@Override
protected List<Entry<String, URL>> getChapters(URL source,
- InputStream in) throws IOException {
+ InputStream in, Progress pg) throws IOException {
return null;
}
@Override
protected String getChapterContent(URL source, InputStream in,
- int number) throws IOException {
+ int number, Progress pg) throws IOException {
return null;
}
@Override
// and make it public!
- public List<Paragraph> makeParagraphs(URL source, String content)
- throws IOException {
- return super.makeParagraphs(source, content);
+ public List<Paragraph> makeParagraphs(URL source, String content,
+ Progress pg) throws IOException {
+ return super.makeParagraphs(source, content, pg);
}
@Override
git://git.nikiroo.be / fanfix.git / commitdiff