From: Niki Roo Date: Sun, 5 Mar 2017 22:18:51 +0000 (+0100) Subject: Improve importing progress reporting X-Git-Url: https://git.nikiroo.be/?a=commitdiff_plain;h=ed08c17162aa8cbdb0cbe6a6045815b987236b9f;p=nikiroo-utils.git Improve importing progress reporting - We know report more information, and more often --- diff --git a/src/be/nikiroo/fanfix/supported/BasicSupport.java b/src/be/nikiroo/fanfix/supported/BasicSupport.java index 1811318..97338ba 100644 --- a/src/be/nikiroo/fanfix/supported/BasicSupport.java +++ b/src/be/nikiroo/fanfix/supported/BasicSupport.java @@ -216,6 +216,8 @@ public abstract class BasicSupport { * the source of the story * @param in * the input (the main resource) + * @param pg + * the optional progress reporter * * @return the chapters * @@ -223,7 +225,7 @@ public abstract class BasicSupport { * in case of I/O error */ protected abstract List> getChapters(URL source, - InputStream in) throws IOException; + InputStream in, Progress pg) throws IOException; /** * Return the content of the chapter (possibly HTML encoded, if @@ -235,6 +237,8 @@ public abstract class BasicSupport { * the input (the main resource) * @param number * the chapter number + * @param pg + * the optional progress reporter * * @return the content * @@ -242,7 +246,7 @@ public abstract class BasicSupport { * in case of I/O error */ protected abstract String getChapterContent(URL source, InputStream in, - int number) throws IOException; + int number, Progress pg) throws IOException; /** * Log into the support (can be a no-op depending upon the support). @@ -298,7 +302,7 @@ public abstract class BasicSupport { * in case of I/O error */ public Story processMeta(URL url) throws IOException { - return processMeta(url, true, false); + return processMeta(url, true, false, null); } /** @@ -310,15 +314,24 @@ public abstract class BasicSupport { * * @param close * close "this" and "in" when done + * @param pg + * the optional progress reporter * * @return the {@link Story} * * @throws IOException * in case of I/O error */ - protected Story processMeta(URL url, boolean close, boolean getDesc) - throws IOException { + protected Story processMeta(URL url, boolean close, boolean getDesc, + Progress pg) throws IOException { + if (pg == null) { + pg = new Progress(); + } else { + pg.setMinMax(0, 100); + } + login(); + pg.setProgress(10); url = getCanonicalUrl(url); @@ -331,6 +344,7 @@ public abstract class BasicSupport { try { preprocess(url, getInput()); + pg.setProgress(30); Story story = new Story(); MetaData meta = getMeta(url, getInput()); @@ -340,18 +354,23 @@ public abstract class BasicSupport { } story.setMeta(meta); + pg.setProgress(50); + if (meta != null && meta.getCover() == null) { meta.setCover(getDefaultCover(meta.getSubject())); } + pg.setProgress(60); + if (getDesc) { String descChapterName = Instance.getTrans().getString( StringId.DESCRIPTION); story.getMeta().setResume( makeChapter(url, 0, descChapterName, - getDesc(url, getInput()))); + getDesc(url, getInput()), null)); } + pg.setProgress(100); return story; } finally { if (close) { @@ -394,10 +413,15 @@ public abstract class BasicSupport { url = getCanonicalUrl(url); pg.setProgress(1); try { - Story story = processMeta(url, false, true); - pg.setProgress(10); + Progress pgMeta = new Progress(); + pg.addProgress(pgMeta, 10); + Story story = processMeta(url, false, true, pgMeta); + if (!pgMeta.isDone()) { + pgMeta.setProgress(pgMeta.getMax()); // 10% + } + if (story == null) { - pg.setProgress(100); + pg.setProgress(90); return null; } @@ -405,24 +429,47 @@ public abstract class BasicSupport { setCurrentReferer(url); + Progress pgGetChapters = new Progress(); + pg.addProgress(pgGetChapters, 10); story.setChapters(new ArrayList()); + List> chapters = getChapters(url, getInput(), + pgGetChapters); + if (!pgGetChapters.isDone()) { + pgGetChapters.setProgress(pgGetChapters.getMax()); // 20% + } - List> chapters = getChapters(url, getInput()); - pg.setProgress(20); - - int i = 1; if (chapters != null) { - Progress pgChaps = new Progress(0, chapters.size()); + Progress pgChaps = new Progress("Extracting chapters", 0, + chapters.size() * 300); pg.addProgress(pgChaps, 80); long words = 0; + int i = 1; for (Entry chap : chapters) { + pgChaps.setName("Extracting chapter " + i); setCurrentReferer(chap.getValue()); InputStream chapIn = Instance.getCache().open( chap.getValue(), this, true); + pgChaps.setProgress(i * 100); try { + Progress pgGetChapterContent = new Progress(); + Progress pgMakeChapter = new Progress(); + pgChaps.addProgress(pgGetChapterContent, 100); + pgChaps.addProgress(pgMakeChapter, 100); + + String content = getChapterContent(url, chapIn, i, + pgGetChapterContent); + if (!pgGetChapterContent.isDone()) { + pgGetChapterContent.setProgress(pgGetChapterContent + .getMax()); + } + Chapter cc = makeChapter(url, i, chap.getKey(), - getChapterContent(url, chapIn, i)); + content, pgMakeChapter); + if (!pgMakeChapter.isDone()) { + pgMakeChapter.setProgress(pgMakeChapter.getMax()); + } + words += cc.getWords(); story.getChapters().add(cc); if (story.getMeta() != null) { @@ -432,10 +479,12 @@ public abstract class BasicSupport { chapIn.close(); } - pgChaps.setProgress(i++); + i++; } + + pgChaps.setName("Extracting chapters"); } else { - pg.setProgress(100); + pg.setProgress(80); } return story; @@ -531,6 +580,8 @@ public abstract class BasicSupport { * the chapter name * @param content * the chapter content + * @param pg + * the optional progress reporter * * @return the {@link Chapter} * @@ -538,7 +589,7 @@ public abstract class BasicSupport { * in case of I/O error */ protected Chapter makeChapter(URL source, int number, String name, - String content) throws IOException { + String content, Progress pg) throws IOException { // Chapter name: process it correctly, then remove the possible // redundant "Chapter x: " in front of it String chapterName = processPara(name).getContent().trim(); @@ -566,7 +617,7 @@ public abstract class BasicSupport { Chapter chap = new Chapter(number, chapterName); if (content != null) { - List paras = makeParagraphs(source, content); + List paras = makeParagraphs(source, content, pg); long words = 0; for (Paragraph para : paras) { words += para.getWords(); @@ -586,14 +637,20 @@ public abstract class BasicSupport { * the source URL of the story * @param content * the textual content + * @param pg + * the optional progress reporter * * @return the {@link Paragraph}s * * @throws IOException * in case of I/O error */ - protected List makeParagraphs(URL source, String content) - throws IOException { + protected List makeParagraphs(URL source, String content, + Progress pg) throws IOException { + if (pg == null) { + pg = new Progress(); + } + if (isHtml()) { // Special
processing: content = content.replaceAll("(
]*>)|(
)|(
)", @@ -604,10 +661,19 @@ public abstract class BasicSupport { if (content != null && !content.trim().isEmpty()) { if (isHtml()) { - for (String line : content.split("(

|

|
|
)")) { + String[] tab = content.split("(

|

|
|
)"); + pg.setMinMax(0, tab.length); + int i = 1; + for (String line : tab) { + if (line.startsWith("[") && line.endsWith("]")) { + pg.setName("Extracting image " + i); + } paras.add(makeParagraph(source, line.trim())); + pg.setProgress(i++); } + pg.setName(null); } else { + List lines = new ArrayList(); BufferedReader buff = null; try { buff = new BufferedReader( @@ -615,13 +681,24 @@ public abstract class BasicSupport { content.getBytes("UTF-8")), "UTF-8")); for (String line = buff.readLine(); line != null; line = buff .readLine()) { - paras.add(makeParagraph(source, line.trim())); + lines.add(line.trim()); } } finally { if (buff != null) { buff.close(); } } + + pg.setMinMax(0, lines.size()); + int i = 0; + for (String line : lines) { + if (line.startsWith("[") && line.endsWith("]")) { + pg.setName("Extracting image " + i); + } + paras.add(makeParagraph(source, line)); + pg.setProgress(i++); + } + pg.setName(null); } // Check quotes for "bad" format diff --git a/src/be/nikiroo/fanfix/supported/Cbz.java b/src/be/nikiroo/fanfix/supported/Cbz.java index 7113248..b041b6d 100644 --- a/src/be/nikiroo/fanfix/supported/Cbz.java +++ b/src/be/nikiroo/fanfix/supported/Cbz.java @@ -65,14 +65,19 @@ class Cbz extends Epub { pg.setMinMax(0, 100); } - Story story = processMeta(url, false, true); + Progress pgMeta = new Progress(); + pg.addProgress(pgMeta, 10); + Story story = processMeta(url, false, true, pgMeta); + if (!pgMeta.isDone()) { + pgMeta.setProgress(pgMeta.getMax()); // 10% + } + story.setChapters(new ArrayList()); Chapter chap = new Chapter(1, null); story.getChapters().add(chap); ZipInputStream zipIn = new ZipInputStream(getInput()); - pg.setProgress(10); List images = new ArrayList(); for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn .getNextEntry()) { diff --git a/src/be/nikiroo/fanfix/supported/E621.java b/src/be/nikiroo/fanfix/supported/E621.java index 05d122e..527e092 100644 --- a/src/be/nikiroo/fanfix/supported/E621.java +++ b/src/be/nikiroo/fanfix/supported/E621.java @@ -90,7 +90,7 @@ class E621 extends BasicSupport { private BufferedImage getCover(URL source) throws IOException { InputStream in = Instance.getCache().open(source, this, true); String images = getChapterContent(new URL(source.toString() + "?page=" - + 1), in, 1); + + 1), in, 1, null); if (!images.isEmpty()) { int pos = images.indexOf("
"); if (pos >= 0) { @@ -191,8 +191,8 @@ class E621 extends BasicSupport { } @Override - protected List> getChapters(URL source, InputStream in) - throws IOException { + protected List> getChapters(URL source, InputStream in, + Progress pg) throws IOException { List> urls = new ArrayList>(); int last = 1; // no pool/show when only one page @@ -240,8 +240,8 @@ class E621 extends BasicSupport { } @Override - protected String getChapterContent(URL source, InputStream in, int number) - throws IOException { + protected String getChapterContent(URL source, InputStream in, int number, + Progress pg) throws IOException { StringBuilder builder = new StringBuilder(); String staticSite = "https://static1.e621.net"; if (source.getHost().contains("e926")) { diff --git a/src/be/nikiroo/fanfix/supported/Epub.java b/src/be/nikiroo/fanfix/supported/Epub.java index 437197a..47da1ac 100644 --- a/src/be/nikiroo/fanfix/supported/Epub.java +++ b/src/be/nikiroo/fanfix/supported/Epub.java @@ -16,6 +16,7 @@ import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.data.MetaData; import be.nikiroo.utils.IOUtils; import be.nikiroo.utils.MarkableFileInputStream; +import be.nikiroo.utils.Progress; /** * Support class for EPUB files created with this program (as we need some @@ -60,22 +61,22 @@ class Epub extends InfoText { } @Override - protected List> getChapters(URL source, InputStream in) - throws IOException { + protected List> getChapters(URL source, InputStream in, + Progress pg) throws IOException { if (fakeIn != null) { fakeIn.reset(); - return super.getChapters(fakeSource, fakeIn); + return super.getChapters(fakeSource, fakeIn, pg); } return null; } @Override - protected String getChapterContent(URL source, InputStream in, int number) - throws IOException { + protected String getChapterContent(URL source, InputStream in, int number, + Progress pg) throws IOException { if (fakeIn != null) { fakeIn.reset(); - return super.getChapterContent(fakeSource, fakeIn, number); + return super.getChapterContent(fakeSource, fakeIn, number, pg); } return null; diff --git a/src/be/nikiroo/fanfix/supported/Fanfiction.java b/src/be/nikiroo/fanfix/supported/Fanfiction.java index 9073656..1d1f3f4 100644 --- a/src/be/nikiroo/fanfix/supported/Fanfiction.java +++ b/src/be/nikiroo/fanfix/supported/Fanfiction.java @@ -15,6 +15,7 @@ import java.util.Scanner; import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.data.MetaData; +import be.nikiroo.utils.Progress; import be.nikiroo.utils.StringUtils; /** @@ -210,7 +211,8 @@ class Fanfiction extends BasicSupport { } @Override - protected List> getChapters(URL source, InputStream in) { + protected List> getChapters(URL source, InputStream in, + Progress pg) { List> urls = new ArrayList>(); String base = source.toString(); @@ -286,7 +288,8 @@ class Fanfiction extends BasicSupport { } @Override - protected String getChapterContent(URL source, InputStream in, int number) { + protected String getChapterContent(URL source, InputStream in, int number, + Progress pg) { StringBuilder builder = new StringBuilder(); String startAt = "class='storytext "; String endAt1 = "function review_init"; diff --git a/src/be/nikiroo/fanfix/supported/Fimfiction.java b/src/be/nikiroo/fanfix/supported/Fimfiction.java index 377e369..6293c11 100644 --- a/src/be/nikiroo/fanfix/supported/Fimfiction.java +++ b/src/be/nikiroo/fanfix/supported/Fimfiction.java @@ -14,6 +14,7 @@ import java.util.Scanner; import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.data.MetaData; +import be.nikiroo.utils.Progress; import be.nikiroo.utils.StringUtils; /** @@ -176,7 +177,8 @@ class Fimfiction extends BasicSupport { } @Override - protected List> getChapters(URL source, InputStream in) { + protected List> getChapters(URL source, InputStream in, + Progress pg) { List> urls = new ArrayList>(); @SuppressWarnings("resource") Scanner scan = new Scanner(in, "UTF-8"); @@ -232,7 +234,8 @@ class Fimfiction extends BasicSupport { } @Override - protected String getChapterContent(URL source, InputStream in, int number) { + protected String getChapterContent(URL source, InputStream in, int number, + Progress pg) { return getLine(in, "
", 1); } diff --git a/src/be/nikiroo/fanfix/supported/MangaFox.java b/src/be/nikiroo/fanfix/supported/MangaFox.java index d86cc73..a379e87 100644 --- a/src/be/nikiroo/fanfix/supported/MangaFox.java +++ b/src/be/nikiroo/fanfix/supported/MangaFox.java @@ -14,6 +14,7 @@ import java.util.Scanner; import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.data.MetaData; import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.Progress; import be.nikiroo.utils.StringUtils; class MangaFox extends BasicSupport { @@ -197,7 +198,8 @@ class MangaFox extends BasicSupport { } @Override - protected List> getChapters(URL source, InputStream in) { + protected List> getChapters(URL source, InputStream in, + Progress pg) { List> urls = new ArrayList>(); String volumeAt = "

"; @@ -240,9 +242,6 @@ class MangaFox extends BasicSupport { } } - // to help with the retry and the originalUrl - refresh(url); - try { final String key = name; final URL value = new URL(url); @@ -265,6 +264,19 @@ class MangaFox extends BasicSupport { } } + if (pg == null) { + pg = new Progress(0, urls.size()); + } else { + pg.setMinMax(0, urls.size()); + } + + int i = 1; + for (Entry entry : urls) { + // to help with the retry and the originalUrl + refresh(entry.getValue().toString()); + pg.setProgress(i++); + } + // the chapters are in reversed order Collections.reverse(urls); @@ -272,12 +284,22 @@ class MangaFox extends BasicSupport { } @Override - protected String getChapterContent(URL source, InputStream in, int number) { + protected String getChapterContent(URL source, InputStream in, int number, + Progress pg) { + if (pg == null) { + pg = new Progress(); + } else { + // Since we have no idea how many images we have, we cycle from 0 + // to max, then again, then again... + pg.setMinMax(0, 20); + } + StringBuilder builder = new StringBuilder(); String base = getCurrentReferer().toString(); int pos = base.lastIndexOf('/'); base = base.substring(0, pos + 1); // including the '/' at the end + int i = 1; boolean close = false; while (in != null) { String linkNextLine = getLine(in, "return enlarge()", 0); @@ -317,6 +339,7 @@ class MangaFox extends BasicSupport { // to help with the retry and the originalUrl, part 2 refresh(linkImage); + pg.setProgress((i++) % pg.getMax()); if (close) { try { @@ -333,6 +356,7 @@ class MangaFox extends BasicSupport { url = new URL(base + linkNext); in = openEx(base + linkNext); setCurrentReferer(url); + pg.setProgress((i++) % pg.getMax()); } catch (IOException e) { Instance.syserr(new IOException( "Cannot get the next manga page which is: " diff --git a/src/be/nikiroo/fanfix/supported/Text.java b/src/be/nikiroo/fanfix/supported/Text.java index 3b486ce..6715695 100644 --- a/src/be/nikiroo/fanfix/supported/Text.java +++ b/src/be/nikiroo/fanfix/supported/Text.java @@ -14,6 +14,7 @@ import java.util.Scanner; import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.data.MetaData; +import be.nikiroo.utils.Progress; /** * Support class for local stories encoded in textual format, with a few rules: @@ -147,7 +148,7 @@ class Text extends BasicSupport { @Override protected String getDesc(URL source, InputStream in) throws IOException { - return getChapterContent(source, in, 0); + return getChapterContent(source, in, 0, null); } private BufferedImage getCover(URL source) throws IOException { @@ -169,8 +170,8 @@ class Text extends BasicSupport { } @Override - protected List> getChapters(URL source, InputStream in) - throws IOException { + protected List> getChapters(URL source, InputStream in, + Progress pg) throws IOException { List> chaps = new ArrayList>(); @SuppressWarnings("resource") Scanner scan = new Scanner(in, "UTF-8"); @@ -208,8 +209,8 @@ class Text extends BasicSupport { } @Override - protected String getChapterContent(URL source, InputStream in, int number) - throws IOException { + protected String getChapterContent(URL source, InputStream in, int number, + Progress pg) throws IOException { StringBuilder builder = new StringBuilder(); @SuppressWarnings("resource") Scanner scan = new Scanner(in, "UTF-8"); diff --git a/src/be/nikiroo/fanfix/supported/YiffStar.java b/src/be/nikiroo/fanfix/supported/YiffStar.java index ed75b10..a2126db 100644 --- a/src/be/nikiroo/fanfix/supported/YiffStar.java +++ b/src/be/nikiroo/fanfix/supported/YiffStar.java @@ -15,6 +15,7 @@ import java.util.Scanner; import be.nikiroo.fanfix.Instance; import be.nikiroo.fanfix.bundles.Config; import be.nikiroo.fanfix.data.MetaData; +import be.nikiroo.utils.Progress; import be.nikiroo.utils.StringUtils; /** @@ -130,7 +131,7 @@ class YiffStar extends BasicSupport { private BufferedImage getCover(URL source, InputStream in) throws IOException { - List> chaps = getChapters(source, in); + List> chaps = getChapters(source, in, null); if (!chaps.isEmpty()) { in = Instance.getCache().open(chaps.get(0).getValue(), this, true); String line = getLine(in, " name=\"og:image\"", 0); @@ -185,8 +186,8 @@ class YiffStar extends BasicSupport { } @Override - protected List> getChapters(URL source, InputStream in) - throws IOException { + protected List> getChapters(URL source, InputStream in, + Progress pg) throws IOException { List> urls = new ArrayList>(); @SuppressWarnings("resource") @@ -225,8 +226,8 @@ class YiffStar extends BasicSupport { } @Override - protected String getChapterContent(URL source, InputStream in, int number) - throws IOException { + protected String getChapterContent(URL source, InputStream in, int number, + Progress pg) throws IOException { StringBuilder builder = new StringBuilder(); String startAt = "id=\"sfContentBody"; diff --git a/src/be/nikiroo/fanfix/test/BasicSupportTest.java b/src/be/nikiroo/fanfix/test/BasicSupportTest.java index f989663..645d042 100644 --- a/src/be/nikiroo/fanfix/test/BasicSupportTest.java +++ b/src/be/nikiroo/fanfix/test/BasicSupportTest.java @@ -17,6 +17,7 @@ import be.nikiroo.fanfix.data.Story; import be.nikiroo.fanfix.supported.BasicSupport; import be.nikiroo.fanfix.supported.BasicSupport.SupportType; import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.Progress; import be.nikiroo.utils.test.TestCase; import be.nikiroo.utils.test.TestLauncher; @@ -60,13 +61,13 @@ public class BasicSupportTest extends TestLauncher { List paras = null; - paras = support.makeParagraphs(null, ""); + paras = support.makeParagraphs(null, "", null); assertEquals( "An empty content should not generate paragraphs", 0, paras.size()); paras = support.makeParagraphs(null, - "Line 1

Line 2

Line 3

"); + "Line 1

Line 2

Line 3

", null); assertEquals(5, paras.size()); assertEquals("Line 1", paras.get(0).getContent()); assertEquals(ParagraphType.BLANK, paras.get(1) @@ -77,7 +78,7 @@ public class BasicSupportTest extends TestLauncher { assertEquals("Line 3", paras.get(4).getContent()); paras = support.makeParagraphs(null, - "

Line1

Line2

Line3

"); + "

Line1

Line2

Line3

", null); assertEquals(6, paras.size()); } }); @@ -95,34 +96,39 @@ public class BasicSupportTest extends TestLauncher { List paras = null; paras = support - .makeParagraphs(null, - "

Line1

Line2

Line3

"); + .makeParagraphs( + null, + "

Line1

Line2

Line3

", + null); assertEquals(5, paras.size()); paras = support - .makeParagraphs(null, - "

Line1

Line2

Line3

* * *"); + .makeParagraphs( + null, + "

Line1

Line2

Line3

* * *", + null); assertEquals(5, paras.size()); - paras = support.makeParagraphs(null, "1

* * *

2"); + paras = support.makeParagraphs(null, "1

* * *

2", + null); assertEquals(3, paras.size()); assertEquals(ParagraphType.BREAK, paras.get(1) .getType()); paras = support.makeParagraphs(null, - "1


* * *

2"); + "1


* * *

2", null); assertEquals(3, paras.size()); assertEquals(ParagraphType.BREAK, paras.get(1) .getType()); paras = support.makeParagraphs(null, - "1

* * *


2"); + "1

* * *


2", null); assertEquals(3, paras.size()); assertEquals(ParagraphType.BREAK, paras.get(1) .getType()); paras = support.makeParagraphs(null, - "1



* * *


2"); + "1



* * *


2", null); assertEquals(3, paras.size()); assertEquals(ParagraphType.BREAK, paras.get(1) .getType()); @@ -417,21 +423,21 @@ public class BasicSupportTest extends TestLauncher { @Override protected List> getChapters(URL source, - InputStream in) throws IOException { + InputStream in, Progress pg) throws IOException { return null; } @Override protected String getChapterContent(URL source, InputStream in, - int number) throws IOException { + int number, Progress pg) throws IOException { return null; } @Override // and make it public! - public List makeParagraphs(URL source, String content) - throws IOException { - return super.makeParagraphs(source, content); + public List makeParagraphs(URL source, String content, + Progress pg) throws IOException { + return super.makeParagraphs(source, content, pg); } @Override