From 2e7584daaf5d2f06d326c21297a71d02dd275a35 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Wed, 24 Apr 2019 09:28:48 +0200 Subject: [PATCH 01/16] new: NextableInputStream, step 1 --- src/be/nikiroo/utils/NextableInputStream.java | 66 +++++++++++++++++++ .../test_code/NextableInputStreamTest.java | 33 ++++++++++ 2 files changed, 99 insertions(+) create mode 100644 src/be/nikiroo/utils/NextableInputStream.java create mode 100644 src/be/nikiroo/utils/test_code/NextableInputStreamTest.java diff --git a/src/be/nikiroo/utils/NextableInputStream.java b/src/be/nikiroo/utils/NextableInputStream.java new file mode 100644 index 0000000..0bcbe14 --- /dev/null +++ b/src/be/nikiroo/utils/NextableInputStream.java @@ -0,0 +1,66 @@ +package be.nikiroo.utils; + +import java.io.IOException; +import java.io.InputStream; + +public class NextableInputStream extends InputStream { + private InputStream in; + private boolean eof; + private int pos = 0; + private int len = 0; + private byte[] buffer = new byte[4096]; + + public NextableInputStream(InputStream in) { + this.in = in; + } + + @Override + public int read() throws IOException { + preRead(); + if (eof) { + return -1; + } + + return buffer[pos++]; + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int boff, int blen) throws IOException { + if (b == null) { + throw new NullPointerException(); + } else if (boff < 0 || blen < 0 || blen > b.length - boff) { + throw new IndexOutOfBoundsException(); + } else if (blen == 0) { + return 0; + } + + int done = 0; + while (!eof && done < blen) { + preRead(); + for (int i = pos; i < blen && i < len; i++) { + b[boff + done] = buffer[i]; + pos++; + done++; + } + } + + return done > 0 ? done : -1; + } + + private void preRead() throws IOException { + if (in != null && !eof && pos >= len) { + pos = 0; + len = in.read(buffer); + // checkNexts(); + } + + if (pos >= len) { + eof = true; + } + } +} diff --git a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java new file mode 100644 index 0000000..1b0064d --- /dev/null +++ b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java @@ -0,0 +1,33 @@ +package be.nikiroo.utils.test_code; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; + +import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.NextableInputStream; +import be.nikiroo.utils.test.TestCase; +import be.nikiroo.utils.test.TestLauncher; + +public class NextableInputStreamTest extends TestLauncher { + public NextableInputStreamTest(String[] args) { + super("NextableInputStream test", args); + + addTest(new TestCase("Simple byte array reading") { + @Override + public void test() throws Exception { + byte[] expected = new byte[] { 42, 12, 0, 127 }; + InputStream bin = new ByteArrayInputStream(expected); + NextableInputStream in = new NextableInputStream(bin); + byte[] actual = IOUtils.toByteArray(in); + + assertEquals( + "The resulting array has not the same number of items", + expected.length, actual.length); + for (int i = 0; i < expected.length; i++) { + assertEquals("Item " + i + " (0-based) is not the same", + expected[i], actual[i]); + } + } + }); + } +} -- 2.27.0 From 4098af704dfa22ce4a60003940753c28030374fa Mon Sep 17 00:00:00 2001 From: Niki Date: Wed, 24 Apr 2019 16:29:50 +0200 Subject: [PATCH 02/16] NextableInputStream, step 2 --- src/be/nikiroo/utils/NextableInputStream.java | 67 +++++++++++++++++-- .../utils/NextableInputStreamStep.java | 52 ++++++++++++++ .../test_code/NextableInputStreamTest.java | 62 +++++++++++++++++ src/be/nikiroo/utils/test_code/Test.java | 1 + 4 files changed, 175 insertions(+), 7 deletions(-) create mode 100755 src/be/nikiroo/utils/NextableInputStreamStep.java diff --git a/src/be/nikiroo/utils/NextableInputStream.java b/src/be/nikiroo/utils/NextableInputStream.java index 0bcbe14..6f3afc2 100644 --- a/src/be/nikiroo/utils/NextableInputStream.java +++ b/src/be/nikiroo/utils/NextableInputStream.java @@ -2,8 +2,13 @@ package be.nikiroo.utils; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; public class NextableInputStream extends InputStream { + private List steps = new ArrayList(); + private NextableInputStreamStep step = null; + private InputStream in; private boolean eof; private int pos = 0; @@ -14,6 +19,25 @@ public class NextableInputStream extends InputStream { this.in = in; } + public void addStep(NextableInputStreamStep step) { + steps.add(step); + } + + public boolean next() { + if (!hasMoreData() && step != null) { + len = step.getResumeLen(); + pos += step.getSkip(); + eof = false; + step = null; + + checkNexts(false); + + return true; + } + + return false; + } + @Override public int read() throws IOException { preRead(); @@ -40,27 +64,56 @@ public class NextableInputStream extends InputStream { } int done = 0; - while (!eof && done < blen) { + while (hasMoreData() && done < blen) { preRead(); - for (int i = pos; i < blen && i < len; i++) { - b[boff + done] = buffer[i]; - pos++; - done++; + if (hasMoreData()) { + for (int i = pos; i < blen && i < len; i++) { + b[boff + done] = buffer[i]; + pos++; + done++; + } } } return done > 0 ? done : -1; } + @Override + public int available() throws IOException { + return Math.max(0, len - pos); + } + private void preRead() throws IOException { - if (in != null && !eof && pos >= len) { + if (!eof && in != null && pos >= len && step == null) { pos = 0; len = in.read(buffer); - // checkNexts(); + checkNexts(true); } if (pos >= len) { eof = true; } } + + private boolean hasMoreData() { + return !(eof && pos >= len); + } + + private void checkNexts(boolean newBuffer) { + if (!eof) { + for (NextableInputStreamStep step : steps) { + if (newBuffer) { + step.clearBuffer(); + } + + int stopAt = step.stop(buffer, pos, len); + if (stopAt >= 0) { + this.step = step; + len = stopAt; + eof = true; + break; + } + } + } + } } diff --git a/src/be/nikiroo/utils/NextableInputStreamStep.java b/src/be/nikiroo/utils/NextableInputStreamStep.java new file mode 100755 index 0000000..ab22562 --- /dev/null +++ b/src/be/nikiroo/utils/NextableInputStreamStep.java @@ -0,0 +1,52 @@ +package be.nikiroo.utils; + +public class NextableInputStreamStep { + private int stopAt; + private boolean disabled; + private int pos; + private int resumeLen; + private int last = -1; + private int skip; + + public NextableInputStreamStep(int byt) { + stopAt = byt; + } + + // do NOT stop twice on the same item + public int stop(byte[] buffer, int pos, int len) { + for (int i = pos; i < len; i++) { + if (buffer[i] == stopAt) { + if (i > this.last) { + // we skip the sep + this.skip = 1; + + this.pos = pos; + this.resumeLen = len; + this.last = i; + return i; + } + } + } + + return -1; + } + + public int getResumeLen() { + return resumeLen; + } + + public int getSkip() { + return skip; + } + + public void clearBuffer() { + this.last = -1; + this.pos = 0; + this.skip = 0; + this.resumeLen = 0; + } + + public boolean isEnabled() { + return !disabled; + } +} diff --git a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java index 1b0064d..51ff128 100644 --- a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java +++ b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java @@ -5,6 +5,7 @@ import java.io.InputStream; import be.nikiroo.utils.IOUtils; import be.nikiroo.utils.NextableInputStream; +import be.nikiroo.utils.NextableInputStreamStep; import be.nikiroo.utils.test.TestCase; import be.nikiroo.utils.test.TestLauncher; @@ -29,5 +30,66 @@ public class NextableInputStreamTest extends TestLauncher { } } }); + + addTest(new TestCase("Stop at 12") { + @Override + public void test() throws Exception { + byte[] expected = new byte[] { 42, 12, 0, 127 }; + InputStream bin = new ByteArrayInputStream(expected); + NextableInputStream in = new NextableInputStream(bin); + in.addStep(new NextableInputStreamStep(12)); + byte[] actual = IOUtils.toByteArray(in); + + assertEquals( + "The resulting array has not the correct number of items", + 1, actual.length); + for (int i = 0; i < actual.length; i++) { + assertEquals("Item " + i + " (0-based) is not the same", + expected[i], actual[i]); + } + } + }); + + addTest(new TestCase("Stop at 12, resume, stop again, resume") { + @Override + public void test() throws Exception { + byte[] expected = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 }; + NextableInputStream in = new NextableInputStream( + new ByteArrayInputStream(expected)); + in.addStep(new NextableInputStreamStep(12)); + + byte[] actual1 = IOUtils.toByteArray(in); + byte[] expected1 = new byte[] { 42 }; + assertEquals( + "The FIRST resulting array has not the correct number of items", + expected1.length, actual1.length); + for (int i = 0; i < actual1.length; i++) { + assertEquals("Item " + i + " (0-based) is not the same", + expected1[i], actual1[i]); + } + + assertEquals("Cannot get SECOND entry", true, in.next()); + byte[] actual2 = IOUtils.toByteArray(in); + byte[] expected2 = new byte[] { 0, 127 }; + assertEquals( + "The SECOND resulting array has not the correct number of items", + expected2.length, actual2.length); + for (int i = 0; i < actual2.length; i++) { + assertEquals("Item " + i + " (0-based) is not the same", + expected2[i], actual2[i]); + } + + assertEquals("Cannot get next THIRD entry", true, in.next()); + byte[] actual3 = IOUtils.toByteArray(in); + byte[] expected3 = new byte[] { 51, 11 }; + assertEquals( + "The THIRD resulting array has not the correct number of items", + expected3.length, actual3.length); + for (int i = 0; i < actual3.length; i++) { + assertEquals("Item " + i + " (0-based) is not the same", + expected3[i], actual3[i]); + } + } + }); } } diff --git a/src/be/nikiroo/utils/test_code/Test.java b/src/be/nikiroo/utils/test_code/Test.java index f25c02d..01766ac 100644 --- a/src/be/nikiroo/utils/test_code/Test.java +++ b/src/be/nikiroo/utils/test_code/Test.java @@ -33,6 +33,7 @@ public class Test extends TestLauncher { addSeries(new StringUtilsTest(args)); addSeries(new TempFilesTest(args)); addSeries(new CryptUtilsTest(args)); + addSeries(new NextableInputStreamTest(args)); // TODO: test cache and downloader Cache cache = null; -- 2.27.0 From 63b46ca9f1703134ef2979b72d474e9c9b8f5737 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Wed, 24 Apr 2019 21:34:56 +0200 Subject: [PATCH 03/16] Simple NextableInputStream --- src/be/nikiroo/utils/NextableInputStream.java | 125 +++++++++++++----- .../utils/NextableInputStreamStep.java | 68 ++++++++-- .../test_code/NextableInputStreamTest.java | 92 ++++++------- 3 files changed, 195 insertions(+), 90 deletions(-) diff --git a/src/be/nikiroo/utils/NextableInputStream.java b/src/be/nikiroo/utils/NextableInputStream.java index 6f3afc2..0def936 100644 --- a/src/be/nikiroo/utils/NextableInputStream.java +++ b/src/be/nikiroo/utils/NextableInputStream.java @@ -2,12 +2,19 @@ package be.nikiroo.utils; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; +/** + * This {@link InputStream} can be separated into sub-streams (you can process + * it as a normal {@link InputStream} but, when it is spent, you can call + * {@link NextableInputStream#next()} on it to unlock new data). + *

+ * The separation in sub-streams is done via {@link NextableInputStreamStep}. + * + * @author niki + */ public class NextableInputStream extends InputStream { - private List steps = new ArrayList(); - private NextableInputStreamStep step = null; + private NextableInputStreamStep step; + private boolean stopped; private InputStream in; private boolean eof; @@ -15,24 +22,51 @@ public class NextableInputStream extends InputStream { private int len = 0; private byte[] buffer = new byte[4096]; - public NextableInputStream(InputStream in) { + /** + * Create a new {@link NextableInputStream} that wraps the given + * {@link InputStream}. + * + * @param in + * the {@link InputStream} to wrap + * @param step + * how to separate it into sub-streams (can be NULL, but in that + * case it will behave as a normal {@link InputStream}) + */ + public NextableInputStream(InputStream in, NextableInputStreamStep step) { this.in = in; + this.step = step; } - public void addStep(NextableInputStreamStep step) { - steps.add(step); - } - - public boolean next() { - if (!hasMoreData() && step != null) { + /** + * Unblock the processing of the next sub-stream. + *

+ * It can only be called when the "current" stream is spent (i.e., you must + * first process the stream until it is spent). + *

+ * We consider that when the under-laying {@link InputStream} is also spent, + * we cannot have a next sub-stream (it will thus return FALSE). + *

+ * {@link IOException}s can happen when we have no data available in the + * buffer; in that case, we fetch more data to know if we can have a next + * sub-stream or not. + * + * @return TRUE if we unblocked the next sub-stream, FALSE if not + * + * @throws IOException + * in case of I/O error + */ + public boolean next() throws IOException { + if (!hasMoreData() && stopped) { len = step.getResumeLen(); - pos += step.getSkip(); + pos += step.getResumeSkip(); eof = false; - step = null; - - checkNexts(false); - return true; + if (!preRead()) { + checkBuffer(false); + } + + // consider that if EOF, there is no next + return hasMoreData(); } return false; @@ -83,36 +117,63 @@ public class NextableInputStream extends InputStream { return Math.max(0, len - pos); } - private void preRead() throws IOException { - if (!eof && in != null && pos >= len && step == null) { + /** + * Check if we still have some data in the buffer and, if not, fetch some. + * + * @return TRUE if we fetched some data, FALSE if there are still some in + * the buffer + * + * @throws IOException + * in case of I/O error + */ + private boolean preRead() throws IOException { + boolean hasRead = false; + if (!eof && in != null && pos >= len && !stopped) { pos = 0; len = in.read(buffer); - checkNexts(true); + checkBuffer(true); + hasRead = true; } if (pos >= len) { eof = true; } + + return hasRead; } + /** + * We have more data available in the buffer or we can fetch more. + * + * @return TRUE if it is the case, FALSE if not + */ private boolean hasMoreData() { return !(eof && pos >= len); } - private void checkNexts(boolean newBuffer) { - if (!eof) { - for (NextableInputStreamStep step : steps) { - if (newBuffer) { - step.clearBuffer(); - } + /** + * Check that the buffer didn't overshot to the next item, and fix + * {@link NextableInputStream#len} if needed. + *

+ * If {@link NextableInputStream#len} is fixed, + * {@link NextableInputStream#eof} and {@link NextableInputStream#stopped} + * are set to TRUE. + * + * @param newBuffer + * we changed the buffer, we need to clear some information in + * the {@link NextableInputStreamStep} + */ + private void checkBuffer(boolean newBuffer) { + if (step != null) { + if (newBuffer) { + step.clearBuffer(); + } - int stopAt = step.stop(buffer, pos, len); - if (stopAt >= 0) { - this.step = step; - len = stopAt; - eof = true; - break; - } + int stopAt = step.stop(buffer, pos, len); + if (stopAt >= 0) { + len = stopAt; + eof = true; + stopped = true; } } } diff --git a/src/be/nikiroo/utils/NextableInputStreamStep.java b/src/be/nikiroo/utils/NextableInputStreamStep.java index ab22562..a2ee039 100755 --- a/src/be/nikiroo/utils/NextableInputStreamStep.java +++ b/src/be/nikiroo/utils/NextableInputStreamStep.java @@ -1,26 +1,58 @@ package be.nikiroo.utils; +import java.io.InputStream; + +/** + * Divide an {@link InputStream} into sub-streams. + * + * @author niki + */ public class NextableInputStreamStep { private int stopAt; - private boolean disabled; - private int pos; private int resumeLen; private int last = -1; private int skip; + /** + * Create a new divider that will separate the sub-streams each time it sees + * this byte. + *

+ * Note that the byte will be bypassed by the {@link InputStream} as far as + * the consumers will be aware. + * + * @param byt + * the byte at which to separate two sub-streams + */ public NextableInputStreamStep(int byt) { stopAt = byt; } - // do NOT stop twice on the same item + /** + * Check if we need to stop the {@link InputStream} reading at some point in + * the current buffer. + *

+ * If we do, return the index at which to stop; if not, return -1. + *

+ * This method will not return the same index a second time (unless + * we cleared the buffer). + * + * @param buffer + * the buffer to check + * @param pos + * the current position of what was read in the buffer + * @param len + * the maximum index to use in the buffer (anything above that is + * not to be used) + * + * @return the index at which to stop, or -1 + */ public int stop(byte[] buffer, int pos, int len) { for (int i = pos; i < len; i++) { if (buffer[i] == stopAt) { if (i > this.last) { // we skip the sep this.skip = 1; - - this.pos = pos; + this.resumeLen = len; this.last = i; return i; @@ -31,22 +63,34 @@ public class NextableInputStreamStep { return -1; } + /** + * Get the maximum index to use in the buffer used in + * {@link NextableInputStreamStep#stop(byte[], int, int)} at resume time. + * + * @return the index + */ public int getResumeLen() { return resumeLen; } - - public int getSkip() { + + /** + * Get the number of bytes to skip at resume time. + * + * @return the number of bytes to skip + */ + public int getResumeSkip() { return skip; } + /** + * Clear the information we may have kept about the current buffer + *

+ * You should call this method each time you change the content of the + * buffer used in {@link NextableInputStreamStep#stop(byte[], int, int)}. + */ public void clearBuffer() { this.last = -1; - this.pos = 0; this.skip = 0; this.resumeLen = 0; } - - public boolean isEnabled() { - return !disabled; - } } diff --git a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java index 51ff128..b5f6780 100644 --- a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java +++ b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java @@ -1,7 +1,6 @@ package be.nikiroo.utils.test_code; import java.io.ByteArrayInputStream; -import java.io.InputStream; import be.nikiroo.utils.IOUtils; import be.nikiroo.utils.NextableInputStream; @@ -17,8 +16,8 @@ public class NextableInputStreamTest extends TestLauncher { @Override public void test() throws Exception { byte[] expected = new byte[] { 42, 12, 0, 127 }; - InputStream bin = new ByteArrayInputStream(expected); - NextableInputStream in = new NextableInputStream(bin); + NextableInputStream in = new NextableInputStream( + new ByteArrayInputStream(expected), null); byte[] actual = IOUtils.toByteArray(in); assertEquals( @@ -35,61 +34,62 @@ public class NextableInputStreamTest extends TestLauncher { @Override public void test() throws Exception { byte[] expected = new byte[] { 42, 12, 0, 127 }; - InputStream bin = new ByteArrayInputStream(expected); - NextableInputStream in = new NextableInputStream(bin); - in.addStep(new NextableInputStreamStep(12)); - byte[] actual = IOUtils.toByteArray(in); + NextableInputStream in = new NextableInputStream( + new ByteArrayInputStream(expected), + new NextableInputStreamStep(12)); - assertEquals( - "The resulting array has not the correct number of items", - 1, actual.length); - for (int i = 0; i < actual.length; i++) { - assertEquals("Item " + i + " (0-based) is not the same", - expected[i], actual[i]); - } + checkNext(this, false, "FIRST", in, new byte[] { 42 }); } }); addTest(new TestCase("Stop at 12, resume, stop again, resume") { @Override public void test() throws Exception { - byte[] expected = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 }; + byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 }; NextableInputStream in = new NextableInputStream( - new ByteArrayInputStream(expected)); - in.addStep(new NextableInputStreamStep(12)); + new ByteArrayInputStream(data), + new NextableInputStreamStep(12)); - byte[] actual1 = IOUtils.toByteArray(in); - byte[] expected1 = new byte[] { 42 }; - assertEquals( - "The FIRST resulting array has not the correct number of items", - expected1.length, actual1.length); - for (int i = 0; i < actual1.length; i++) { - assertEquals("Item " + i + " (0-based) is not the same", - expected1[i], actual1[i]); - } + checkNext(this, false, "FIRST", in, new byte[] { 42 }); + checkNext(this, true, "SECOND", in, new byte[] { 0, 127 }); + checkNext(this, true, "THIRD", in, new byte[] { 51, 11 }); + } + }); - assertEquals("Cannot get SECOND entry", true, in.next()); - byte[] actual2 = IOUtils.toByteArray(in); - byte[] expected2 = new byte[] { 0, 127 }; - assertEquals( - "The SECOND resulting array has not the correct number of items", - expected2.length, actual2.length); - for (int i = 0; i < actual2.length; i++) { - assertEquals("Item " + i + " (0-based) is not the same", - expected2[i], actual2[i]); - } + addTest(new TestCase("Encapsulation") { + @Override + public void test() throws Exception { + byte[] data = new byte[] { 42, 12, 0, 4, 127, 12, 5 }; + NextableInputStream in4 = new NextableInputStream( + new ByteArrayInputStream(data), + new NextableInputStreamStep(4)); + NextableInputStream subIn12 = new NextableInputStream(in4, + new NextableInputStreamStep(12)); - assertEquals("Cannot get next THIRD entry", true, in.next()); - byte[] actual3 = IOUtils.toByteArray(in); - byte[] expected3 = new byte[] { 51, 11 }; - assertEquals( - "The THIRD resulting array has not the correct number of items", - expected3.length, actual3.length); - for (int i = 0; i < actual3.length; i++) { - assertEquals("Item " + i + " (0-based) is not the same", - expected3[i], actual3[i]); - } + checkNext(this, false, "SUB FIRST", subIn12, new byte[] { 42 }); + checkNext(this, true, "SUB SECOND", subIn12, new byte[] { 0 }); + + assertEquals("The subIn still has some data", false, + subIn12.next()); + + checkNext(this, true, "MAIN LAST", in4, new byte[] { 127, 12, 5 }); } }); } + + static void checkNext(TestCase test, boolean callNext, String prefix, + NextableInputStream in, byte[] expected) throws Exception { + if (callNext) { + test.assertEquals("Cannot get " + prefix + " entry", true, + in.next()); + } + byte[] actual = IOUtils.toByteArray(in); + test.assertEquals("The " + prefix + + " resulting array has not the correct number of items", + expected.length, actual.length); + for (int i = 0; i < actual.length; i++) { + test.assertEquals("Item " + i + " (0-based) is not the same", + expected[i], actual[i]); + } + } } -- 2.27.0 From dd3034f78a3492ee114b6c78d0fbc468544f8343 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Wed, 24 Apr 2019 21:41:36 +0200 Subject: [PATCH 04/16] more tests --- .../test_code/NextableInputStreamTest.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java index b5f6780..87d64ac 100644 --- a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java +++ b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java @@ -71,8 +71,24 @@ public class NextableInputStreamTest extends TestLauncher { assertEquals("The subIn still has some data", false, subIn12.next()); - - checkNext(this, true, "MAIN LAST", in4, new byte[] { 127, 12, 5 }); + + checkNext(this, true, "MAIN LAST", in4, + new byte[] { 127, 12, 5 }); + } + }); + + addTest(new TestCase("UTF-8 text lines test") { + @Override + public void test() throws Exception { + String ln1 = "Ligne première"; + String ln2 = "Ligne la deuxième du nom"; + byte[] data = (ln1 + "\n" + ln2).getBytes("UTF-8"); + NextableInputStream in = new NextableInputStream( + new ByteArrayInputStream(data), + new NextableInputStreamStep('\n')); + + checkNext(this, false, "FIRST", in, ln1.getBytes("UTF-8")); + checkNext(this, true, "SECOND", in, ln2.getBytes("UTF-8")); } }); } -- 2.27.0 From d6f9bd9f5687106d55c42a91a20e7c95c30ed3e4 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Thu, 25 Apr 2019 09:27:57 +0200 Subject: [PATCH 05/16] NextableInputStream: more options --- src/be/nikiroo/utils/NextableInputStream.java | 281 ++++++++++++++++-- .../test_code/NextableInputStreamTest.java | 86 +++++- 2 files changed, 335 insertions(+), 32 deletions(-) diff --git a/src/be/nikiroo/utils/NextableInputStream.java b/src/be/nikiroo/utils/NextableInputStream.java index 0def936..69fd33b 100644 --- a/src/be/nikiroo/utils/NextableInputStream.java +++ b/src/be/nikiroo/utils/NextableInputStream.java @@ -14,13 +14,18 @@ import java.io.InputStream; */ public class NextableInputStream extends InputStream { private NextableInputStreamStep step; + private boolean started; private boolean stopped; + private boolean closed; private InputStream in; + private int openCounter; private boolean eof; - private int pos = 0; - private int len = 0; - private byte[] buffer = new byte[4096]; + private int pos; + private int len; + private byte[] buffer; + + private long bytesRead; /** * Create a new {@link NextableInputStream} that wraps the given @@ -35,6 +40,84 @@ public class NextableInputStream extends InputStream { public NextableInputStream(InputStream in, NextableInputStreamStep step) { this.in = in; this.step = step; + + this.buffer = new byte[4096]; + this.pos = 0; + this.len = 0; + } + + /** + * Create a new {@link NextableInputStream} that wraps the given bytes array + * as a data source. + * + * @param in + * the array to wrap, cannot be NULL + * @param step + * how to separate it into sub-streams (can be NULL, but in that + * case it will behave as a normal {@link InputStream}) + */ + public NextableInputStream(byte[] in, NextableInputStreamStep step) { + this(in, step, 0, in.length); + } + + /** + * Create a new {@link NextableInputStream} that wraps the given bytes array + * as a data source. + * + * @param in + * the array to wrap, cannot be NULL + * @param step + * how to separate it into sub-streams (can be NULL, but in that + * case it will behave as a normal {@link InputStream}) + * @param offset + * the offset to start the reading at + * @param length + * the number of bytes to take into account in the array, + * starting from the offset + * + * @throws NullPointerException + * if the array is NULL + * @throws IndexOutOfBoundsException + * if the offset and length do not correspond to the given array + */ + public NextableInputStream(byte[] in, NextableInputStreamStep step, + int offset, int length) { + if (in == null) { + throw new NullPointerException(); + } else if (offset < 0 || length < 0 || length > in.length - offset) { + throw new IndexOutOfBoundsException(); + } + + this.in = null; + this.step = step; + + this.buffer = in; + this.pos = offset; + this.len = length; + + checkBuffer(true); + } + + /** + * Return this very same {@link NextableInputStream}, but keep a counter of + * how many streams were open this way. When calling + * {@link NextableInputStream#close()}, decrease this counter if it is not + * already zero instead of actually closing the stream. + *

+ * You are now responsible for it — you must close it. + *

+ * This method allows you to use a wrapping stream around this one and still + * close the wrapping stream. + * + * @return the same stream, but you are now responsible for closing it + * + * @throws IOException + * in case of I/O error or if the stream is closed + */ + public synchronized InputStream open() throws IOException { + checkClose(); + openCounter++; + return this; } /** @@ -53,27 +136,61 @@ public class NextableInputStream extends InputStream { * @return TRUE if we unblocked the next sub-stream, FALSE if not * * @throws IOException - * in case of I/O error + * in case of I/O error or if the stream is closed */ public boolean next() throws IOException { - if (!hasMoreData() && stopped) { - len = step.getResumeLen(); - pos += step.getResumeSkip(); - eof = false; + return next(false); + } - if (!preRead()) { - checkBuffer(false); - } + /** + * Unblock the next sub-stream as would have done + * {@link NextableInputStream#next()}, but disable the sub-stream systems. + *

+ * That is, the next stream, if any, will be the last one and will not be + * subject to the {@link NextableInputStreamStep}. + * + * @return TRUE if we unblocked the next sub-stream, FALSE if not + * + * @throws IOException + * in case of I/O error or if the stream is closed + */ + public boolean nextAll() throws IOException { + return next(true); + } - // consider that if EOF, there is no next - return hasMoreData(); - } + public boolean startWith() { + // TODO + return false; + } + public boolean startWiths() { + // TODO return false; } + /** + * The number of bytes read from the under-laying {@link InputStream}. + * + * @return the number of bytes + */ + public long getBytesRead() { + return bytesRead; + } + + /** + * Check if this stream is totally spent (no more data to read or to + * process). + * + * @return TRUE if it is + */ + public boolean eof() { + return closed || (len < 0 && !hasMoreData()); + } + @Override public int read() throws IOException { + checkClose(); + preRead(); if (eof) { return -1; @@ -89,6 +206,8 @@ public class NextableInputStream extends InputStream { @Override public int read(byte[] b, int boff, int blen) throws IOException { + checkClose(); + if (b == null) { throw new NullPointerException(); } else if (boff < 0 || blen < 0 || blen > b.length - boff) { @@ -113,10 +232,73 @@ public class NextableInputStream extends InputStream { } @Override - public int available() throws IOException { + public long skip(long n) throws IOException { + // TODO Auto-generated method stub + return super.skip(n); + } + + @Override + public int available() { + if (closed) { + return 0; + } + return Math.max(0, len - pos); } + /** + * Closes this stream and releases any system resources associated with the + * stream. + *

+ * Including the under-laying {@link InputStream}. + *

+ * Note: if you called the {@link NextableInputStream#open()} method + * prior to this one, it will just decrease the internal count of how many + * open streams it held and do nothing else. The stream will actually be + * closed when you have called {@link NextableInputStream#close()} once more + * than {@link NextableInputStream#open()}. + * + * @exception IOException + * in case of I/O error + */ + @Override + public synchronized void close() throws IOException { + close(true); + } + + /** + * Closes this stream and releases any system resources associated with the + * stream. + *

+ * Including the under-laying {@link InputStream} if + * incudingSubStream is true. + *

+ * You can call this method multiple times, it will not cause an + * {@link IOException} for subsequent calls. + *

+ * Note: if you called the {@link NextableInputStream#open()} method + * prior to this one, it will just decrease the internal count of how many + * open streams it held and do nothing else. The stream will actually be + * closed when you have called {@link NextableInputStream#close()} once more + * than {@link NextableInputStream#open()}. + * + * @exception IOException + * in case of I/O error + */ + public synchronized void close(boolean includingSubStream) + throws IOException { + if (!closed) { + if (openCounter > 0) { + openCounter--; + } else { + closed = true; + if (includingSubStream && in != null) { + in.close(); + } + } + } + } + /** * Check if we still have some data in the buffer and, if not, fetch some. * @@ -131,6 +313,10 @@ public class NextableInputStream extends InputStream { if (!eof && in != null && pos >= len && !stopped) { pos = 0; len = in.read(buffer); + if (len > 0) { + bytesRead += len; + } + checkBuffer(true); hasRead = true; } @@ -148,7 +334,7 @@ public class NextableInputStream extends InputStream { * @return TRUE if it is the case, FALSE if not */ private boolean hasMoreData() { - return !(eof && pos >= len); + return !closed && started && !(eof && pos >= len); } /** @@ -164,7 +350,7 @@ public class NextableInputStream extends InputStream { * the {@link NextableInputStreamStep} */ private void checkBuffer(boolean newBuffer) { - if (step != null) { + if (step != null && len > 0) { if (newBuffer) { step.clearBuffer(); } @@ -177,4 +363,65 @@ public class NextableInputStream extends InputStream { } } } + + /** + * The implementation of {@link NextableInputStream#next()} and + * {@link NextableInputStream#nextAll()}. + * + * @param all + * TRUE for {@link NextableInputStream#nextAll()}, FALSE for + * {@link NextableInputStream#next()} + * + * @return TRUE if we unblocked the next sub-stream, FALSE if not + * + * @throws IOException + * in case of I/O error or if the stream is closed + */ + private boolean next(boolean all) throws IOException { + checkClose(); + + if (!started) { + // First call before being allowed to read + started = true; + + if (all) { + step = null; + } + + return true; + } + + if (step != null && !hasMoreData() && stopped) { + len = step.getResumeLen(); + pos += step.getResumeSkip(); + eof = false; + + if (all) { + step = null; + } + + if (!preRead()) { + checkBuffer(false); + } + + // consider that if EOF, there is no next + return hasMoreData(); + } + + return false; + } + + /** + * Check that the stream was not closed, and throw an {@link IOException} if + * it was. + * + * @throws IOException + * if it was closed + */ + private void checkClose() throws IOException { + if (closed) { + throw new IOException( + "This NextableInputStream was closed, you cannot use it anymore."); + } + } } diff --git a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java index 87d64ac..e57da84 100644 --- a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java +++ b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java @@ -18,6 +18,7 @@ public class NextableInputStreamTest extends TestLauncher { byte[] expected = new byte[] { 42, 12, 0, 127 }; NextableInputStream in = new NextableInputStream( new ByteArrayInputStream(expected), null); + in.next(); byte[] actual = IOUtils.toByteArray(in); assertEquals( @@ -38,7 +39,7 @@ public class NextableInputStreamTest extends TestLauncher { new ByteArrayInputStream(expected), new NextableInputStreamStep(12)); - checkNext(this, false, "FIRST", in, new byte[] { 42 }); + checkNext(this, "FIRST", in, new byte[] { 42 }); } }); @@ -50,9 +51,9 @@ public class NextableInputStreamTest extends TestLauncher { new ByteArrayInputStream(data), new NextableInputStreamStep(12)); - checkNext(this, false, "FIRST", in, new byte[] { 42 }); - checkNext(this, true, "SECOND", in, new byte[] { 0, 127 }); - checkNext(this, true, "THIRD", in, new byte[] { 51, 11 }); + checkNext(this, "FIRST", in, new byte[] { 42 }); + checkNext(this, "SECOND", in, new byte[] { 0, 127 }); + checkNext(this, "THIRD", in, new byte[] { 51, 11 }); } }); @@ -66,14 +67,14 @@ public class NextableInputStreamTest extends TestLauncher { NextableInputStream subIn12 = new NextableInputStream(in4, new NextableInputStreamStep(12)); - checkNext(this, false, "SUB FIRST", subIn12, new byte[] { 42 }); - checkNext(this, true, "SUB SECOND", subIn12, new byte[] { 0 }); + in4.next(); + checkNext(this, "SUB FIRST", subIn12, new byte[] { 42 }); + checkNext(this, "SUB SECOND", subIn12, new byte[] { 0 }); assertEquals("The subIn still has some data", false, subIn12.next()); - checkNext(this, true, "MAIN LAST", in4, - new byte[] { 127, 12, 5 }); + checkNext(this, "MAIN LAST", in4, new byte[] { 127, 12, 5 }); } }); @@ -87,18 +88,73 @@ public class NextableInputStreamTest extends TestLauncher { new ByteArrayInputStream(data), new NextableInputStreamStep('\n')); - checkNext(this, false, "FIRST", in, ln1.getBytes("UTF-8")); - checkNext(this, true, "SECOND", in, ln2.getBytes("UTF-8")); + checkNext(this, "FIRST", in, ln1.getBytes("UTF-8")); + checkNext(this, "SECOND", in, ln2.getBytes("UTF-8")); + } + }); + + addTest(new TestCase("nextAll()") { + @Override + public void test() throws Exception { + byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 }; + NextableInputStream in = new NextableInputStream( + new ByteArrayInputStream(data), + new NextableInputStreamStep(12)); + + checkNext(this, "FIRST", in, new byte[] { 42 }); + checkNextAll(this, "REST", in, new byte[] { 0, 127, 12, 51, 11, + 12 }); + assertEquals("The stream still has some data", false, in.next()); + } + }); + + addTest(new TestCase("getBytesRead()") { + @Override + public void test() throws Exception { + byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 }; + NextableInputStream in = new NextableInputStream( + new ByteArrayInputStream(data), + new NextableInputStreamStep(12)); + + in.nextAll(); + IOUtils.toByteArray(in); + + assertEquals("The number of bytes read is not correct", + data.length, in.getBytesRead()); + } + }); + + addTest(new TestCase("bytes array input") { + @Override + public void test() throws Exception { + byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 }; + NextableInputStream in = new NextableInputStream(data, + new NextableInputStreamStep(12)); + + checkNext(this, "FIRST", in, new byte[] { 42 }); + checkNext(this, "SECOND", in, new byte[] { 0, 127 }); + checkNext(this, "THIRD", in, new byte[] { 51, 11 }); } }); } - static void checkNext(TestCase test, boolean callNext, String prefix, - NextableInputStream in, byte[] expected) throws Exception { - if (callNext) { - test.assertEquals("Cannot get " + prefix + " entry", true, - in.next()); + static void checkNext(TestCase test, String prefix, NextableInputStream in, + byte[] expected) throws Exception { + test.assertEquals("Cannot get " + prefix + " entry", true, in.next()); + byte[] actual = IOUtils.toByteArray(in); + test.assertEquals("The " + prefix + + " resulting array has not the correct number of items", + expected.length, actual.length); + for (int i = 0; i < actual.length; i++) { + test.assertEquals("Item " + i + " (0-based) is not the same", + expected[i], actual[i]); } + } + + static void checkNextAll(TestCase test, String prefix, + NextableInputStream in, byte[] expected) throws Exception { + test.assertEquals("Cannot get " + prefix + " entries", true, + in.nextAll()); byte[] actual = IOUtils.toByteArray(in); test.assertEquals("The " + prefix + " resulting array has not the correct number of items", -- 2.27.0 From 32d89af14cbf18589a1daca5f999f8cedd13dd40 Mon Sep 17 00:00:00 2001 From: Niki Date: Thu, 25 Apr 2019 15:56:13 +0200 Subject: [PATCH 06/16] NextableInputStream: startsWith() --- src/be/nikiroo/utils/NextableInputStream.java | 85 +++++++++++++++++-- 1 file changed, 77 insertions(+), 8 deletions(-) diff --git a/src/be/nikiroo/utils/NextableInputStream.java b/src/be/nikiroo/utils/NextableInputStream.java index 69fd33b..7ced598 100644 --- a/src/be/nikiroo/utils/NextableInputStream.java +++ b/src/be/nikiroo/utils/NextableInputStream.java @@ -2,6 +2,7 @@ package be.nikiroo.utils; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; /** * This {@link InputStream} can be separated into sub-streams (you can process @@ -25,6 +26,12 @@ public class NextableInputStream extends InputStream { private int len; private byte[] buffer; + // special use, prefetched next buffer + private byte[] buffer2; + private int pos2; + private int len2; + private byte[] originalBuffer; + private long bytesRead; /** @@ -42,6 +49,7 @@ public class NextableInputStream extends InputStream { this.step = step; this.buffer = new byte[4096]; + this.originalBuffer = this.buffer; this.pos = 0; this.len = 0; } @@ -92,6 +100,7 @@ public class NextableInputStream extends InputStream { this.step = step; this.buffer = in; + this.originalBuffer = this.buffer; this.pos = offset; this.len = length; @@ -158,13 +167,48 @@ public class NextableInputStream extends InputStream { return next(true); } - public boolean startWith() { - // TODO - return false; + // max is buffer.size ! + public boolean startsWiths(String search) throws IOException { + return startsWith(search.getBytes("UTF-8")); } - public boolean startWiths() { - // TODO + // max is buffer.size ! + public boolean startsWith(byte[] search) throws IOException { + if (search.length > originalBuffer.length) { + throw new IOException( + "This stream does not support searching for more than " + + buffer.length + " bytes"); + } + + checkClose(); + + if (available() < search.length) { + preRead(); + } + + if (available() >= search.length) { + // Easy path + return startsWith(search, buffer, pos); + } else if (!eof) { + // Harder path + if (buffer2 == null && buffer.length == originalBuffer.length) { + buffer2 = Arrays.copyOf(buffer, buffer.length * 2); + + pos2 = buffer.length; + len2 = in.read(buffer2, pos2, buffer.length); + if (len2 > 0) { + bytesRead += len2; + } + + // Note: here, len/len2 = INDEX of last good byte + len2 += pos2; + } + + if (available() + (len2 - pos2) >= search.length) { + return startsWith(search, buffer2, pos2); + } + } + return false; } @@ -312,9 +356,20 @@ public class NextableInputStream extends InputStream { boolean hasRead = false; if (!eof && in != null && pos >= len && !stopped) { pos = 0; - len = in.read(buffer); - if (len > 0) { - bytesRead += len; + if (buffer2 != null) { + buffer = buffer2; + pos = pos2; + len = len2; + + buffer2 = null; + pos2 = 0; + len2 = 0; + } else { + buffer = originalBuffer; + len = in.read(buffer); + if (len > 0) { + bytesRead += len; + } } checkBuffer(true); @@ -424,4 +479,18 @@ public class NextableInputStream extends InputStream { "This NextableInputStream was closed, you cannot use it anymore."); } } + + // buffer must be > search + static private boolean startsWith(byte[] search, byte[] buffer, + int offset) { + boolean same = true; + for (int i = 0; i < search.length; i++) { + if (search[i] != buffer[offset + i]) { + same = false; + break; + } + } + + return same; + } } -- 2.27.0 From 473e5f319a4cacc584b5daaf8f0c5d1f18bbf5d0 Mon Sep 17 00:00:00 2001 From: Niki Date: Thu, 25 Apr 2019 17:02:21 +0200 Subject: [PATCH 07/16] NextableInputStream: better skip() + more tests --- src/be/nikiroo/utils/NextableInputStream.java | 17 +++- .../test_code/NextableInputStreamTest.java | 97 ++++++++++++++++--- 2 files changed, 101 insertions(+), 13 deletions(-) diff --git a/src/be/nikiroo/utils/NextableInputStream.java b/src/be/nikiroo/utils/NextableInputStream.java index 7ced598..b5374a1 100644 --- a/src/be/nikiroo/utils/NextableInputStream.java +++ b/src/be/nikiroo/utils/NextableInputStream.java @@ -277,8 +277,21 @@ public class NextableInputStream extends InputStream { @Override public long skip(long n) throws IOException { - // TODO Auto-generated method stub - return super.skip(n); + if (n <= 0) { + return 0; + } + + long skipped = 0; + while (hasMoreData() && n > 0) { + preRead(); + + long inBuffer = Math.min(n, available()); + pos += inBuffer; + n -= inBuffer; + skipped += inBuffer; + } + + return skipped; } @Override diff --git a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java index e57da84..4664cbf 100644 --- a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java +++ b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java @@ -1,6 +1,7 @@ package be.nikiroo.utils.test_code; import java.io.ByteArrayInputStream; +import java.io.IOException; import be.nikiroo.utils.IOUtils; import be.nikiroo.utils.NextableInputStream; @@ -102,9 +103,10 @@ public class NextableInputStreamTest extends TestLauncher { new NextableInputStreamStep(12)); checkNext(this, "FIRST", in, new byte[] { 42 }); - checkNextAll(this, "REST", in, new byte[] { 0, 127, 12, 51, 11, - 12 }); - assertEquals("The stream still has some data", false, in.next()); + checkNextAll(this, "REST", in, + new byte[] { 0, 127, 12, 51, 11, 12 }); + assertEquals("The stream still has some data", false, + in.next()); } }); @@ -136,25 +138,98 @@ public class NextableInputStreamTest extends TestLauncher { checkNext(this, "THIRD", in, new byte[] { 51, 11 }); } }); + + addTest(new TestCase("Skip data") { + @Override + public void test() throws Exception { + byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 }; + NextableInputStream in = new NextableInputStream(data, null); + in.next(); + + in.skip(4); + checkArrays(this, "ONLY", in, new byte[] { 12, 51, 11, 12 }); + } + }); + + addTest(new TestCase("Starts with") { + @Override + public void test() throws Exception { + byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 }; + NextableInputStream in = new NextableInputStream(data, null); + in.next(); + + // yes + assertEquals("It actually starts with that", true, + in.startsWith(new byte[] { 42 })); + assertEquals("It actually starts with that", true, + in.startsWith(new byte[] { 42, 12 })); + assertEquals("It actually is the same array", true, + in.startsWith(data)); + + // no + assertEquals("It actually does not start with that", false, + in.startsWith(new byte[] { 12 })); + assertEquals("It actually does not start with that", false, + in.startsWith( + new byte[] { 42, 12, 0, 127, 12, 51, 11, 11 })); + + // too big + try { + in.startsWith( + new byte[] { 42, 12, 0, 127, 12, 51, 11, 12, 0 }); + fail("Searching a prefix bigger than the array should throw an IOException"); + } catch (IOException e) { + } + } + }); + + addTest(new TestCase("Starts with strings") { + @Override + public void test() throws Exception { + String text = "Fanfan et Toto vont à la mer"; + byte[] data = text.getBytes("UTF-8"); + NextableInputStream in = new NextableInputStream(data, null); + in.next(); + + // yes + assertEquals("It actually starts with that", true, + in.startsWiths("F")); + assertEquals("It actually starts with that", true, + in.startsWiths("Fanfan et")); + assertEquals("It actually is the same text", true, + in.startsWiths(text)); + + // no + assertEquals("It actually does not start with that", false, + in.startsWiths("Toto")); + assertEquals("It actually does not start with that", false, + in.startsWiths("Fanfan et Toto vont à la mee")); + + // too big + try { + in.startsWiths("Fanfan et Toto vont à la mer."); + fail("Searching a prefix bigger than the array should throw an IOException"); + } catch (IOException e) { + } + } + }); } static void checkNext(TestCase test, String prefix, NextableInputStream in, byte[] expected) throws Exception { test.assertEquals("Cannot get " + prefix + " entry", true, in.next()); - byte[] actual = IOUtils.toByteArray(in); - test.assertEquals("The " + prefix - + " resulting array has not the correct number of items", - expected.length, actual.length); - for (int i = 0; i < actual.length; i++) { - test.assertEquals("Item " + i + " (0-based) is not the same", - expected[i], actual[i]); - } + checkArrays(test, prefix, in, expected); } static void checkNextAll(TestCase test, String prefix, NextableInputStream in, byte[] expected) throws Exception { test.assertEquals("Cannot get " + prefix + " entries", true, in.nextAll()); + checkArrays(test, prefix, in, expected); + } + + static void checkArrays(TestCase test, String prefix, + NextableInputStream in, byte[] expected) throws Exception { byte[] actual = IOUtils.toByteArray(in); test.assertEquals("The " + prefix + " resulting array has not the correct number of items", -- 2.27.0 From e378894c40d45740da6498fe9020e7628d485e35 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Thu, 25 Apr 2019 18:40:29 +0200 Subject: [PATCH 08/16] IOUtils: fix readSmallStream --- src/be/nikiroo/utils/IOUtils.java | 44 +++++++++++++------------------ 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/src/be/nikiroo/utils/IOUtils.java b/src/be/nikiroo/utils/IOUtils.java index a263eb1..9cdaba8 100644 --- a/src/be/nikiroo/utils/IOUtils.java +++ b/src/be/nikiroo/utils/IOUtils.java @@ -1,14 +1,11 @@ package be.nikiroo.utils; -import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; @@ -56,8 +53,10 @@ public class IOUtils { public static void write(InputStream in, OutputStream out) throws IOException { byte buffer[] = new byte[4096]; - for (int len = 0; (len = in.read(buffer)) > 0;) { + int len = in.read(buffer); + while (len > 0) { out.write(buffer, 0, len); + len = in.read(buffer); } } @@ -229,11 +228,11 @@ public class IOUtils { */ public static void writeSmallFile(File file, String content) throws IOException { - FileWriter writerVersion = new FileWriter(file); + FileOutputStream out = new FileOutputStream(file); try { - writerVersion.write(content); + out.write(content.getBytes("UTF-8")); } finally { - writerVersion.close(); + out.close(); } } @@ -269,19 +268,14 @@ public class IOUtils { * in case of I/O error */ public static String readSmallStream(InputStream stream) throws IOException { - // do NOT close the reader, or the related stream will be closed, too - // reader.close(); - BufferedReader reader = new BufferedReader( - new InputStreamReader(stream)); - - StringBuilder builder = new StringBuilder(); - for (String line = reader.readLine(); line != null; line = reader - .readLine()) { - builder.append(line); - builder.append("\n"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + write(stream, out); + return out.toString("UTF-8"); + } finally { + // do NOT close, or the related stream will be closed, too + // out.close(); } - - return builder.toString(); } /** @@ -451,12 +445,12 @@ public class IOUtils { */ public static byte[] toByteArray(InputStream in) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - write(in, out); - - byte[] array = out.toByteArray(); - out.close(); - - return array; + try { + write(in, out); + return out.toByteArray(); + } finally { + out.close(); + } } /** -- 2.27.0 From a64a1754f23ed89422072b2b9d60be5426c3aa10 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Thu, 25 Apr 2019 18:41:11 +0200 Subject: [PATCH 09/16] changelog --- changelog.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 79e97c2..5bdc946 100644 --- a/changelog.md +++ b/changelog.md @@ -4,7 +4,8 @@ - new: server: count the bytes we rec/send - new: CryptUtils -- serial: SSL -> CryptUtils +- fix: IOUtils.readSmallStream and \n at the end +- change: serial: SSL -> CryptUtils ## Version 4.7.2 -- 2.27.0 From 39d16a807a9327ec2106dbf26ab054dcd5f10ada Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Fri, 26 Apr 2019 17:54:00 +0200 Subject: [PATCH 10/16] formatNumber change --- src/be/nikiroo/utils/StringUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/be/nikiroo/utils/StringUtils.java b/src/be/nikiroo/utils/StringUtils.java index 1ee9ac4..954d341 100644 --- a/src/be/nikiroo/utils/StringUtils.java +++ b/src/be/nikiroo/utils/StringUtils.java @@ -844,15 +844,15 @@ public class StringUtils { if (value >= 1000000000l) { mult = 1000000000l; userValue = value / 1000000000l; - suffix = "G"; + suffix = " G"; } else if (value >= 1000000l) { mult = 1000000l; userValue = value / 1000000l; - suffix = "M"; + suffix = " M"; } else if (value >= 1000l) { mult = 1000l; userValue = value / 1000l; - suffix = "k"; + suffix = " k"; } String deci = ""; -- 2.27.0 From 59509e75f7a88cefad9c5478324bbeb64a871d31 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Fri, 26 Apr 2019 18:16:21 +0200 Subject: [PATCH 11/16] NextableInputStream: better perfs --- src/be/nikiroo/utils/NextableInputStream.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/be/nikiroo/utils/NextableInputStream.java b/src/be/nikiroo/utils/NextableInputStream.java index b5374a1..d9f27ce 100644 --- a/src/be/nikiroo/utils/NextableInputStream.java +++ b/src/be/nikiroo/utils/NextableInputStream.java @@ -224,6 +224,9 @@ public class NextableInputStream extends InputStream { /** * Check if this stream is totally spent (no more data to read or to * process). + *

+ * Note: an empty stream that is still not started will return FALSE, as we + * don't know yet if it is empty. * * @return TRUE if it is */ @@ -264,10 +267,11 @@ public class NextableInputStream extends InputStream { while (hasMoreData() && done < blen) { preRead(); if (hasMoreData()) { - for (int i = pos; i < blen && i < len; i++) { - b[boff + done] = buffer[i]; - pos++; - done++; + int now = Math.min(blen, len) - pos; + if (now > 0) { + System.arraycopy(buffer, pos, b, boff, now); + pos += now; + done += now; } } } @@ -494,8 +498,7 @@ public class NextableInputStream extends InputStream { } // buffer must be > search - static private boolean startsWith(byte[] search, byte[] buffer, - int offset) { + static private boolean startsWith(byte[] search, byte[] buffer, int offset) { boolean same = true; for (int i = 0; i < search.length; i++) { if (search[i] != buffer[offset + i]) { -- 2.27.0 From 33895a7b06d1a8a7c3555da06b215ab1085ac4df Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Fri, 26 Apr 2019 23:09:29 +0200 Subject: [PATCH 12/16] NextableInputStream: separate base behaviour from nextable --- src/be/nikiroo/utils/BufferedInputStream.java | 400 ++++++++++++++++++ src/be/nikiroo/utils/NextableInputStream.java | 315 +------------- 2 files changed, 412 insertions(+), 303 deletions(-) create mode 100644 src/be/nikiroo/utils/BufferedInputStream.java diff --git a/src/be/nikiroo/utils/BufferedInputStream.java b/src/be/nikiroo/utils/BufferedInputStream.java new file mode 100644 index 0000000..4a8a8c3 --- /dev/null +++ b/src/be/nikiroo/utils/BufferedInputStream.java @@ -0,0 +1,400 @@ +package be.nikiroo.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +/** + * A simple {@link InputStream} that is buffered with a bytes array. + *

+ * It is mostly intended to be used as a base class to create new + * {@link InputStream}s with special operation modes, and to give some default + * methods. + * + * @author niki + */ +public class BufferedInputStream extends InputStream { + protected int pos; + protected int len; + protected byte[] buffer; + protected boolean eof; + + private boolean closed; + private InputStream in; + private int openCounter; + + // special use, prefetched next buffer + private byte[] buffer2; + private int pos2; + private int len2; + private byte[] originalBuffer; + + private long bytesRead; + + /** + * Create a new {@link BufferedInputStream} that wraps the given + * {@link InputStream}. + * + * @param in + * the {@link InputStream} to wrap + */ + public BufferedInputStream(InputStream in) { + this.in = in; + + this.buffer = new byte[4096]; + this.originalBuffer = this.buffer; + this.pos = 0; + this.len = 0; + } + + /** + * Create a new {@link BufferedInputStream} that wraps the given bytes array + * as a data source. + * + * @param in + * the array to wrap, cannot be NULL + */ + public BufferedInputStream(byte[] in) { + this(in, 0, in.length); + } + + /** + * Create a new {@link BufferedInputStream} that wraps the given bytes array + * as a data source. + * + * @param in + * the array to wrap, cannot be NULL + * @param offset + * the offset to start the reading at + * @param length + * the number of bytes to take into account in the array, + * starting from the offset + * + * @throws NullPointerException + * if the array is NULL + * @throws IndexOutOfBoundsException + * if the offset and length do not correspond to the given array + */ + public BufferedInputStream(byte[] in, int offset, int length) { + if (in == null) { + throw new NullPointerException(); + } else if (offset < 0 || length < 0 || length > in.length - offset) { + throw new IndexOutOfBoundsException(); + } + + this.in = null; + + this.buffer = in; + this.originalBuffer = this.buffer; + this.pos = offset; + this.len = length; + } + + /** + * Return this very same {@link BufferedInputStream}, but keep a counter of + * how many streams were open this way. When calling + * {@link BufferedInputStream#close()}, decrease this counter if it is not + * already zero instead of actually closing the stream. + *

+ * You are now responsible for it — you must close it. + *

+ * This method allows you to use a wrapping stream around this one and still + * close the wrapping stream. + * + * @return the same stream, but you are now responsible for closing it + * + * @throws IOException + * in case of I/O error or if the stream is closed + */ + public synchronized InputStream open() throws IOException { + checkClose(); + openCounter++; + return this; + } + + // max is buffer.size ! + public boolean startsWiths(String search) throws IOException { + return startsWith(search.getBytes("UTF-8")); + } + + // max is buffer.size ! + public boolean startsWith(byte[] search) throws IOException { + if (search.length > originalBuffer.length) { + throw new IOException( + "This stream does not support searching for more than " + + buffer.length + " bytes"); + } + + checkClose(); + + if (available() < search.length) { + preRead(); + } + + if (available() >= search.length) { + // Easy path + return startsWith(search, buffer, pos); + } else if (!eof) { + // Harder path + if (buffer2 == null && buffer.length == originalBuffer.length) { + buffer2 = Arrays.copyOf(buffer, buffer.length * 2); + + pos2 = buffer.length; + len2 = in.read(buffer2, pos2, buffer.length); + if (len2 > 0) { + bytesRead += len2; + } + + // Note: here, len/len2 = INDEX of last good byte + len2 += pos2; + } + + if (available() + (len2 - pos2) >= search.length) { + return startsWith(search, buffer2, pos2); + } + } + + return false; + } + + /** + * The number of bytes read from the under-laying {@link InputStream}. + * + * @return the number of bytes + */ + public long getBytesRead() { + return bytesRead; + } + + /** + * Check if this stream is totally spent (no more data to read or to + * process). + *

+ * Note: an empty stream that is still not started will return FALSE, as we + * don't know yet if it is empty. + * + * @return TRUE if it is + */ + public boolean eof() { + return closed || (len < 0 && !hasMoreData()); + } + + @Override + public int read() throws IOException { + checkClose(); + + preRead(); + if (eof) { + return -1; + } + + return buffer[pos++]; + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int boff, int blen) throws IOException { + checkClose(); + + if (b == null) { + throw new NullPointerException(); + } else if (boff < 0 || blen < 0 || blen > b.length - boff) { + throw new IndexOutOfBoundsException(); + } else if (blen == 0) { + return 0; + } + + int done = 0; + while (hasMoreData() && done < blen) { + preRead(); + if (hasMoreData()) { + int now = Math.min(blen, len) - pos; + if (now > 0) { + System.arraycopy(buffer, pos, b, boff, now); + pos += now; + done += now; + } + } + } + + return done > 0 ? done : -1; + } + + @Override + public long skip(long n) throws IOException { + if (n <= 0) { + return 0; + } + + long skipped = 0; + while (hasMoreData() && n > 0) { + preRead(); + + long inBuffer = Math.min(n, available()); + pos += inBuffer; + n -= inBuffer; + skipped += inBuffer; + } + + return skipped; + } + + @Override + public int available() { + if (closed) { + return 0; + } + + return Math.max(0, len - pos); + } + + /** + * Closes this stream and releases any system resources associated with the + * stream. + *

+ * Including the under-laying {@link InputStream}. + *

+ * Note: if you called the {@link BufferedInputStream#open()} method + * prior to this one, it will just decrease the internal count of how many + * open streams it held and do nothing else. The stream will actually be + * closed when you have called {@link BufferedInputStream#close()} once more + * than {@link BufferedInputStream#open()}. + * + * @exception IOException + * in case of I/O error + */ + @Override + public synchronized void close() throws IOException { + close(true); + } + + /** + * Closes this stream and releases any system resources associated with the + * stream. + *

+ * Including the under-laying {@link InputStream} if + * incudingSubStream is true. + *

+ * You can call this method multiple times, it will not cause an + * {@link IOException} for subsequent calls. + *

+ * Note: if you called the {@link BufferedInputStream#open()} method + * prior to this one, it will just decrease the internal count of how many + * open streams it held and do nothing else. The stream will actually be + * closed when you have called {@link BufferedInputStream#close()} once more + * than {@link BufferedInputStream#open()}. + * + * @exception IOException + * in case of I/O error + */ + public synchronized void close(boolean includingSubStream) + throws IOException { + if (!closed) { + if (openCounter > 0) { + openCounter--; + } else { + closed = true; + if (includingSubStream && in != null) { + in.close(); + } + } + } + } + + /** + * Check if we still have some data in the buffer and, if not, fetch some. + * + * @return TRUE if we fetched some data, FALSE if there are still some in + * the buffer + * + * @throws IOException + * in case of I/O error + */ + protected boolean preRead() throws IOException { + boolean hasRead = false; + if (!eof && in != null && pos >= len) { + pos = 0; + if (buffer2 != null) { + buffer = buffer2; + pos = pos2; + len = len2; + + buffer2 = null; + pos2 = 0; + len2 = 0; + } else { + buffer = originalBuffer; + + len = read(in, buffer); + if (len > 0) { + bytesRead += len; + } + } + + hasRead = true; + } + + if (pos >= len) { + eof = true; + } + + return hasRead; + } + + /** + * Read the under-laying stream into the local buffer. + * + * @param in + * the under-laying {@link InputStream} + * @param buffer + * the buffer we use in this {@link BufferedInputStream} + * + * @return the number of bytes read + * + * @throws IOException + * in case of I/O error + */ + protected int read(InputStream in, byte[] buffer) throws IOException { + return in.read(buffer); + } + + /** + * We have more data available in the buffer or we can fetch more. + * + * @return TRUE if it is the case, FALSE if not + */ + protected boolean hasMoreData() { + return !closed && !(eof && pos >= len); + } + + /** + * Check that the stream was not closed, and throw an {@link IOException} if + * it was. + * + * @throws IOException + * if it was closed + */ + protected void checkClose() throws IOException { + if (closed) { + throw new IOException( + "This NextableInputStream was closed, you cannot use it anymore."); + } + } + + // buffer must be > search + static private boolean startsWith(byte[] search, byte[] buffer, int offset) { + boolean same = true; + for (int i = 0; i < search.length; i++) { + if (search[i] != buffer[offset + i]) { + same = false; + break; + } + } + + return same; + } +} diff --git a/src/be/nikiroo/utils/NextableInputStream.java b/src/be/nikiroo/utils/NextableInputStream.java index d9f27ce..37322c9 100644 --- a/src/be/nikiroo/utils/NextableInputStream.java +++ b/src/be/nikiroo/utils/NextableInputStream.java @@ -2,7 +2,6 @@ package be.nikiroo.utils; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; /** * This {@link InputStream} can be separated into sub-streams (you can process @@ -13,26 +12,10 @@ import java.util.Arrays; * * @author niki */ -public class NextableInputStream extends InputStream { +public class NextableInputStream extends BufferedInputStream { private NextableInputStreamStep step; private boolean started; private boolean stopped; - private boolean closed; - - private InputStream in; - private int openCounter; - private boolean eof; - private int pos; - private int len; - private byte[] buffer; - - // special use, prefetched next buffer - private byte[] buffer2; - private int pos2; - private int len2; - private byte[] originalBuffer; - - private long bytesRead; /** * Create a new {@link NextableInputStream} that wraps the given @@ -45,13 +28,8 @@ public class NextableInputStream extends InputStream { * case it will behave as a normal {@link InputStream}) */ public NextableInputStream(InputStream in, NextableInputStreamStep step) { - this.in = in; + super(in); this.step = step; - - this.buffer = new byte[4096]; - this.originalBuffer = this.buffer; - this.pos = 0; - this.len = 0; } /** @@ -90,45 +68,11 @@ public class NextableInputStream extends InputStream { */ public NextableInputStream(byte[] in, NextableInputStreamStep step, int offset, int length) { - if (in == null) { - throw new NullPointerException(); - } else if (offset < 0 || length < 0 || length > in.length - offset) { - throw new IndexOutOfBoundsException(); - } - - this.in = null; + super(in, offset, length); this.step = step; - - this.buffer = in; - this.originalBuffer = this.buffer; - this.pos = offset; - this.len = length; - checkBuffer(true); } - /** - * Return this very same {@link NextableInputStream}, but keep a counter of - * how many streams were open this way. When calling - * {@link NextableInputStream#close()}, decrease this counter if it is not - * already zero instead of actually closing the stream. - *

- * You are now responsible for it — you must close it. - *

- * This method allows you to use a wrapping stream around this one and still - * close the wrapping stream. - * - * @return the same stream, but you are now responsible for closing it - * - * @throws IOException - * in case of I/O error or if the stream is closed - */ - public synchronized InputStream open() throws IOException { - checkClose(); - openCounter++; - return this; - } - /** * Unblock the processing of the next sub-stream. *

@@ -167,199 +111,6 @@ public class NextableInputStream extends InputStream { return next(true); } - // max is buffer.size ! - public boolean startsWiths(String search) throws IOException { - return startsWith(search.getBytes("UTF-8")); - } - - // max is buffer.size ! - public boolean startsWith(byte[] search) throws IOException { - if (search.length > originalBuffer.length) { - throw new IOException( - "This stream does not support searching for more than " - + buffer.length + " bytes"); - } - - checkClose(); - - if (available() < search.length) { - preRead(); - } - - if (available() >= search.length) { - // Easy path - return startsWith(search, buffer, pos); - } else if (!eof) { - // Harder path - if (buffer2 == null && buffer.length == originalBuffer.length) { - buffer2 = Arrays.copyOf(buffer, buffer.length * 2); - - pos2 = buffer.length; - len2 = in.read(buffer2, pos2, buffer.length); - if (len2 > 0) { - bytesRead += len2; - } - - // Note: here, len/len2 = INDEX of last good byte - len2 += pos2; - } - - if (available() + (len2 - pos2) >= search.length) { - return startsWith(search, buffer2, pos2); - } - } - - return false; - } - - /** - * The number of bytes read from the under-laying {@link InputStream}. - * - * @return the number of bytes - */ - public long getBytesRead() { - return bytesRead; - } - - /** - * Check if this stream is totally spent (no more data to read or to - * process). - *

- * Note: an empty stream that is still not started will return FALSE, as we - * don't know yet if it is empty. - * - * @return TRUE if it is - */ - public boolean eof() { - return closed || (len < 0 && !hasMoreData()); - } - - @Override - public int read() throws IOException { - checkClose(); - - preRead(); - if (eof) { - return -1; - } - - return buffer[pos++]; - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int boff, int blen) throws IOException { - checkClose(); - - if (b == null) { - throw new NullPointerException(); - } else if (boff < 0 || blen < 0 || blen > b.length - boff) { - throw new IndexOutOfBoundsException(); - } else if (blen == 0) { - return 0; - } - - int done = 0; - while (hasMoreData() && done < blen) { - preRead(); - if (hasMoreData()) { - int now = Math.min(blen, len) - pos; - if (now > 0) { - System.arraycopy(buffer, pos, b, boff, now); - pos += now; - done += now; - } - } - } - - return done > 0 ? done : -1; - } - - @Override - public long skip(long n) throws IOException { - if (n <= 0) { - return 0; - } - - long skipped = 0; - while (hasMoreData() && n > 0) { - preRead(); - - long inBuffer = Math.min(n, available()); - pos += inBuffer; - n -= inBuffer; - skipped += inBuffer; - } - - return skipped; - } - - @Override - public int available() { - if (closed) { - return 0; - } - - return Math.max(0, len - pos); - } - - /** - * Closes this stream and releases any system resources associated with the - * stream. - *

- * Including the under-laying {@link InputStream}. - *

- * Note: if you called the {@link NextableInputStream#open()} method - * prior to this one, it will just decrease the internal count of how many - * open streams it held and do nothing else. The stream will actually be - * closed when you have called {@link NextableInputStream#close()} once more - * than {@link NextableInputStream#open()}. - * - * @exception IOException - * in case of I/O error - */ - @Override - public synchronized void close() throws IOException { - close(true); - } - - /** - * Closes this stream and releases any system resources associated with the - * stream. - *

- * Including the under-laying {@link InputStream} if - * incudingSubStream is true. - *

- * You can call this method multiple times, it will not cause an - * {@link IOException} for subsequent calls. - *

- * Note: if you called the {@link NextableInputStream#open()} method - * prior to this one, it will just decrease the internal count of how many - * open streams it held and do nothing else. The stream will actually be - * closed when you have called {@link NextableInputStream#close()} once more - * than {@link NextableInputStream#open()}. - * - * @exception IOException - * in case of I/O error - */ - public synchronized void close(boolean includingSubStream) - throws IOException { - if (!closed) { - if (openCounter > 0) { - openCounter--; - } else { - closed = true; - if (includingSubStream && in != null) { - in.close(); - } - } - } - } - /** * Check if we still have some data in the buffer and, if not, fetch some. * @@ -369,35 +120,19 @@ public class NextableInputStream extends InputStream { * @throws IOException * in case of I/O error */ - private boolean preRead() throws IOException { - boolean hasRead = false; - if (!eof && in != null && pos >= len && !stopped) { - pos = 0; - if (buffer2 != null) { - buffer = buffer2; - pos = pos2; - len = len2; - - buffer2 = null; - pos2 = 0; - len2 = 0; - } else { - buffer = originalBuffer; - len = in.read(buffer); - if (len > 0) { - bytesRead += len; - } - } - + @Override + protected boolean preRead() throws IOException { + if (!stopped) { + boolean bufferChanged = super.preRead(); checkBuffer(true); - hasRead = true; + return bufferChanged; } if (pos >= len) { eof = true; } - return hasRead; + return false; } /** @@ -405,8 +140,9 @@ public class NextableInputStream extends InputStream { * * @return TRUE if it is the case, FALSE if not */ - private boolean hasMoreData() { - return !closed && started && !(eof && pos >= len); + @Override + protected boolean hasMoreData() { + return started && super.hasMoreData(); } /** @@ -482,31 +218,4 @@ public class NextableInputStream extends InputStream { return false; } - - /** - * Check that the stream was not closed, and throw an {@link IOException} if - * it was. - * - * @throws IOException - * if it was closed - */ - private void checkClose() throws IOException { - if (closed) { - throw new IOException( - "This NextableInputStream was closed, you cannot use it anymore."); - } - } - - // buffer must be > search - static private boolean startsWith(byte[] search, byte[] buffer, int offset) { - boolean same = true; - for (int i = 0; i < search.length; i++) { - if (search[i] != buffer[offset + i]) { - same = false; - break; - } - } - - return same; - } } -- 2.27.0 From 6ef54b36380b9d123bbdb6b4056cf64b7489225e Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Sat, 27 Apr 2019 00:33:32 +0200 Subject: [PATCH 13/16] ReplaceInputStream (wip) --- src/be/nikiroo/utils/BufferedInputStream.java | 2 +- src/be/nikiroo/utils/ReplaceInputStream.java | 68 +++++++++++++ .../test_code/BufferedInputStreamTest.java | 46 +++++++++ .../test_code/ReplaceInputStreamTest.java | 98 +++++++++++++++++++ src/be/nikiroo/utils/test_code/Test.java | 2 + 5 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 src/be/nikiroo/utils/ReplaceInputStream.java create mode 100644 src/be/nikiroo/utils/test_code/BufferedInputStreamTest.java create mode 100644 src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java diff --git a/src/be/nikiroo/utils/BufferedInputStream.java b/src/be/nikiroo/utils/BufferedInputStream.java index 4a8a8c3..9b95574 100644 --- a/src/be/nikiroo/utils/BufferedInputStream.java +++ b/src/be/nikiroo/utils/BufferedInputStream.java @@ -386,7 +386,7 @@ public class BufferedInputStream extends InputStream { } // buffer must be > search - static private boolean startsWith(byte[] search, byte[] buffer, int offset) { + static protected boolean startsWith(byte[] search, byte[] buffer, int offset) { boolean same = true; for (int i = 0; i < search.length; i++) { if (search[i] != buffer[offset + i]) { diff --git a/src/be/nikiroo/utils/ReplaceInputStream.java b/src/be/nikiroo/utils/ReplaceInputStream.java new file mode 100644 index 0000000..5ce9ef3 --- /dev/null +++ b/src/be/nikiroo/utils/ReplaceInputStream.java @@ -0,0 +1,68 @@ +package be.nikiroo.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; + +public class ReplaceInputStream extends BufferedInputStream { + private byte[] from; + private byte[] to; + + private byte[] source; + private int spos; + private int slen; + + public ReplaceInputStream(InputStream in, String from, String to) { + this(in, bytes(from), bytes(to)); + } + + public ReplaceInputStream(InputStream in, byte[] from, byte[] to) { + super(in); + this.from = from; + this.to = to; + + source = new byte[4096]; + spos = 0; + slen = 0; + } + + @Override + protected int read(InputStream in, byte[] buffer) throws IOException { + if (buffer.length < to.length) { + throw new IOException( + "Underlaying buffer is too small to contain the replace String"); + } + + if (spos >= slen) { + spos = 0; + slen = in.read(source); + } + + // Note: very simple, not efficient implementation, sorry. + int count = 0; + int i = spos; + while (i < slen && count < buffer.length - to.length) { + if (from.length > 0 && startsWith(from, source, spos)) { + System.arraycopy(to, 0, buffer, spos, to.length); + count += to.length; + i += to.length; + spos += to.length; + } else { + buffer[count++] = source[i++]; + spos++; + } + } + + return count; + } + + static private byte[] bytes(String str) { + try { + return str.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + // All conforming JVM must support UTF-8 + e.printStackTrace(); + return null; + } + } +} diff --git a/src/be/nikiroo/utils/test_code/BufferedInputStreamTest.java b/src/be/nikiroo/utils/test_code/BufferedInputStreamTest.java new file mode 100644 index 0000000..e2fc80b --- /dev/null +++ b/src/be/nikiroo/utils/test_code/BufferedInputStreamTest.java @@ -0,0 +1,46 @@ +package be.nikiroo.utils.test_code; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; + +import be.nikiroo.utils.BufferedInputStream; +import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.test.TestCase; +import be.nikiroo.utils.test.TestLauncher; + +public class BufferedInputStreamTest extends TestLauncher { + public BufferedInputStreamTest(String[] args) { + super("BufferedInputStream test", args); + + addTest(new TestCase("Simple InputStream reading") { + @Override + public void test() throws Exception { + byte[] expected = new byte[] { 42, 12, 0, 127 }; + BufferedInputStream in = new BufferedInputStream( + new ByteArrayInputStream(expected)); + checkArrays(this, "FIRST", in, expected); + } + }); + + addTest(new TestCase("Simple byte array reading") { + @Override + public void test() throws Exception { + byte[] expected = new byte[] { 42, 12, 0, 127 }; + BufferedInputStream in = new BufferedInputStream(expected); + checkArrays(this, "FIRST", in, expected); + } + }); + } + + static void checkArrays(TestCase test, String prefix, InputStream in, + byte[] expected) throws Exception { + byte[] actual = IOUtils.toByteArray(in); + test.assertEquals("The " + prefix + + " resulting array has not the correct number of items", + expected.length, actual.length); + for (int i = 0; i < actual.length; i++) { + test.assertEquals("Item " + i + " (0-based) is not the same", + expected[i], actual[i]); + } + } +} diff --git a/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java b/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java new file mode 100644 index 0000000..d6a741f --- /dev/null +++ b/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java @@ -0,0 +1,98 @@ +package be.nikiroo.utils.test_code; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; + +import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.ReplaceInputStream; +import be.nikiroo.utils.test.TestCase; +import be.nikiroo.utils.test.TestLauncher; + +public class ReplaceInputStreamTest extends TestLauncher { + public ReplaceInputStreamTest(String[] args) { + super("ReplaceInputStream test", args); + + addTest(new TestCase("Simple InputStream, empty replace") { + @Override + public void test() throws Exception { + byte[] expected = new byte[] { 42, 12, 0, 127 }; + ReplaceInputStream in = new ReplaceInputStream( + new ByteArrayInputStream(expected), new byte[0], + new byte[0]); + checkArrays(this, "FIRST", in, expected); + } + }); + + addTest(new TestCase("Simple InputStream, simple replace") { + @Override + public void test() throws Exception { + byte[] expected = new byte[] { 42, 12, 0, 127 }; + ReplaceInputStream in = new ReplaceInputStream( + new ByteArrayInputStream(expected), new byte[] { 0 }, + new byte[] { 10 }); + + checkArrays(this, "FIRST", in, new byte[] { 42, 12, 10, 127 }); + } + }); + + addTest(new TestCase("Simple byte array reading, 3/4 replace") { + @Override + public void test() throws Exception { + byte[] expected = new byte[] { 42, 12, 0, 127 }; + ReplaceInputStream in = new ReplaceInputStream( + new ByteArrayInputStream(expected), new byte[] { 12, 0, + 127 }, new byte[] { 10, 10, 10 }); + + checkArrays(this, "FIRST", in, new byte[] { 42, 10, 10, 10 }); + } + }); + + addTest(new TestCase("Simple byte array reading, longer replace") { + @Override + public void test() throws Exception { + byte[] expected = new byte[] { 42, 12, 0, 127 }; + ReplaceInputStream in = new ReplaceInputStream( + new ByteArrayInputStream(expected), new byte[] { 0 }, + new byte[] { 10, 10, 10 }); + + // TODO NOT OK!! + System.out.println(); + for (int i = 0; i < expected.length; i++) { + System.out.println("expected[" + i + "] = " + expected[i]); + } + byte[] actual = IOUtils.readSmallStream(in).getBytes("UTF-8"); + for (int i = 0; i < actual.length; i++) { + System.out.println("actual[" + i + "] = " + actual[i]); + } + System.exit(1); + // + + checkArrays(this, "FIRST", in, new byte[] { 42, 12, 10, 10, 10, + 127 }); + } + }); + + addTest(new TestCase("Simple byte array reading, shorter replace") { + @Override + public void test() throws Exception { + byte[] expected = new byte[] { 42, 12, 0, 127 }; + ReplaceInputStream in = new ReplaceInputStream( + new ByteArrayInputStream(expected), new byte[] { 42, + 12, 0 }, new byte[] { 10 }); + checkArrays(this, "FIRST", in, new byte[] { 10, 127 }); + } + }); + } + + static void checkArrays(TestCase test, String prefix, InputStream in, + byte[] expected) throws Exception { + byte[] actual = IOUtils.toByteArray(in); + test.assertEquals("The " + prefix + + " resulting array has not the correct number of items", + expected.length, actual.length); + for (int i = 0; i < actual.length; i++) { + test.assertEquals("Item " + i + " (0-based) is not the same", + expected[i], actual[i]); + } + } +} diff --git a/src/be/nikiroo/utils/test_code/Test.java b/src/be/nikiroo/utils/test_code/Test.java index 01766ac..24d6bba 100644 --- a/src/be/nikiroo/utils/test_code/Test.java +++ b/src/be/nikiroo/utils/test_code/Test.java @@ -33,7 +33,9 @@ public class Test extends TestLauncher { addSeries(new StringUtilsTest(args)); addSeries(new TempFilesTest(args)); addSeries(new CryptUtilsTest(args)); + addSeries(new BufferedInputStreamTest(args)); addSeries(new NextableInputStreamTest(args)); + addSeries(new ReplaceInputStreamTest(args)); // TODO: test cache and downloader Cache cache = null; -- 2.27.0 From 876dbf8b4e0f2498fb41f626f98e5fae46600b7b Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Sat, 27 Apr 2019 13:23:06 +0200 Subject: [PATCH 14/16] ReplaceInputStream ready + tests --- src/be/nikiroo/utils/ReplaceInputStream.java | 49 ++++++++++--- .../test_code/ReplaceInputStreamTest.java | 70 +++++++++++-------- 2 files changed, 80 insertions(+), 39 deletions(-) diff --git a/src/be/nikiroo/utils/ReplaceInputStream.java b/src/be/nikiroo/utils/ReplaceInputStream.java index 5ce9ef3..ff2ec88 100644 --- a/src/be/nikiroo/utils/ReplaceInputStream.java +++ b/src/be/nikiroo/utils/ReplaceInputStream.java @@ -4,6 +4,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; +/** + * This {@link InputStream} will replace some of its content by replacing it + * with something else. + * + * @author niki + */ public class ReplaceInputStream extends BufferedInputStream { private byte[] from; private byte[] to; @@ -12,10 +18,32 @@ public class ReplaceInputStream extends BufferedInputStream { private int spos; private int slen; + /** + * Create a {@link ReplaceInputStream} that will replace from with + * to. + * + * @param in + * the under-laying {@link InputStream} + * @param from + * the {@link String} to replace + * @param to + * the {@link String} to replace with + */ public ReplaceInputStream(InputStream in, String from, String to) { this(in, bytes(from), bytes(to)); } + /** + * Create a {@link ReplaceInputStream} that will replace from with + * to. + * + * @param in + * the under-laying {@link InputStream} + * @param from + * the value to replace + * @param to + * the value to replace with + */ public ReplaceInputStream(InputStream in, byte[] from, byte[] to) { super(in); this.from = from; @@ -28,9 +56,9 @@ public class ReplaceInputStream extends BufferedInputStream { @Override protected int read(InputStream in, byte[] buffer) throws IOException { - if (buffer.length < to.length) { + if (buffer.length < to.length || source.length < to.length * 2) { throw new IOException( - "Underlaying buffer is too small to contain the replace String"); + "An underlaying buffer is too small for this replace value"); } if (spos >= slen) { @@ -40,22 +68,27 @@ public class ReplaceInputStream extends BufferedInputStream { // Note: very simple, not efficient implementation, sorry. int count = 0; - int i = spos; - while (i < slen && count < buffer.length - to.length) { + while (spos < slen && count < buffer.length - to.length) { if (from.length > 0 && startsWith(from, source, spos)) { System.arraycopy(to, 0, buffer, spos, to.length); count += to.length; - i += to.length; - spos += to.length; + spos += from.length; } else { - buffer[count++] = source[i++]; - spos++; + buffer[count++] = source[spos++]; } } return count; } + /** + * Return the bytes array representation of the given {@link String} in + * UTF-8. + * + * @param str + * the string to transform into bytes + * @return the content in bytes + */ static private byte[] bytes(String str) { try { return str.getBytes("UTF-8"); diff --git a/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java b/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java index d6a741f..c0ca923 100644 --- a/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java +++ b/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java @@ -8,80 +8,88 @@ import be.nikiroo.utils.ReplaceInputStream; import be.nikiroo.utils.test.TestCase; import be.nikiroo.utils.test.TestLauncher; -public class ReplaceInputStreamTest extends TestLauncher { +class ReplaceInputStreamTest extends TestLauncher { public ReplaceInputStreamTest(String[] args) { super("ReplaceInputStream test", args); - addTest(new TestCase("Simple InputStream, empty replace") { + addTest(new TestCase("Empty replace") { @Override public void test() throws Exception { - byte[] expected = new byte[] { 42, 12, 0, 127 }; + byte[] data = new byte[] { 42, 12, 0, 127 }; ReplaceInputStream in = new ReplaceInputStream( - new ByteArrayInputStream(expected), new byte[0], + new ByteArrayInputStream(data), new byte[0], new byte[0]); - checkArrays(this, "FIRST", in, expected); + + checkArrays(this, "FIRST", in, data); } }); - addTest(new TestCase("Simple InputStream, simple replace") { + addTest(new TestCase("Simple replace") { @Override public void test() throws Exception { - byte[] expected = new byte[] { 42, 12, 0, 127 }; + byte[] data = new byte[] { 42, 12, 0, 127 }; ReplaceInputStream in = new ReplaceInputStream( - new ByteArrayInputStream(expected), new byte[] { 0 }, + new ByteArrayInputStream(data), new byte[] { 0 }, new byte[] { 10 }); checkArrays(this, "FIRST", in, new byte[] { 42, 12, 10, 127 }); } }); - addTest(new TestCase("Simple byte array reading, 3/4 replace") { + addTest(new TestCase("3/4 replace") { @Override public void test() throws Exception { - byte[] expected = new byte[] { 42, 12, 0, 127 }; + byte[] data = new byte[] { 42, 12, 0, 127 }; ReplaceInputStream in = new ReplaceInputStream( - new ByteArrayInputStream(expected), new byte[] { 12, 0, - 127 }, new byte[] { 10, 10, 10 }); + new ByteArrayInputStream(data), + new byte[] { 12, 0, 127 }, new byte[] { 10, 10, 10 }); checkArrays(this, "FIRST", in, new byte[] { 42, 10, 10, 10 }); } }); - addTest(new TestCase("Simple byte array reading, longer replace") { + addTest(new TestCase("Lnger replace") { @Override public void test() throws Exception { - byte[] expected = new byte[] { 42, 12, 0, 127 }; + byte[] data = new byte[] { 42, 12, 0, 127 }; ReplaceInputStream in = new ReplaceInputStream( - new ByteArrayInputStream(expected), new byte[] { 0 }, + new ByteArrayInputStream(data), new byte[] { 0 }, new byte[] { 10, 10, 10 }); - // TODO NOT OK!! - System.out.println(); - for (int i = 0; i < expected.length; i++) { - System.out.println("expected[" + i + "] = " + expected[i]); - } - byte[] actual = IOUtils.readSmallStream(in).getBytes("UTF-8"); - for (int i = 0; i < actual.length; i++) { - System.out.println("actual[" + i + "] = " + actual[i]); - } - System.exit(1); - // - checkArrays(this, "FIRST", in, new byte[] { 42, 12, 10, 10, 10, 127 }); } }); - addTest(new TestCase("Simple byte array reading, shorter replace") { + addTest(new TestCase("Shorter replace") { @Override public void test() throws Exception { - byte[] expected = new byte[] { 42, 12, 0, 127 }; + byte[] data = new byte[] { 42, 12, 0, 127 }; ReplaceInputStream in = new ReplaceInputStream( - new ByteArrayInputStream(expected), new byte[] { 42, - 12, 0 }, new byte[] { 10 }); + new ByteArrayInputStream(data), + new byte[] { 42, 12, 0 }, new byte[] { 10 }); + checkArrays(this, "FIRST", in, new byte[] { 10, 127 }); } }); + + addTest(new TestCase("String replace") { + @Override + public void test() throws Exception { + byte[] data = "I like red".getBytes("UTF-8"); + ReplaceInputStream in = new ReplaceInputStream( + new ByteArrayInputStream(data), + "red".getBytes("UTF-8"), "blue".getBytes("UTF-8")); + + checkArrays(this, "FIRST", in, "I like blue".getBytes("UTF-8")); + + data = "I like blue".getBytes("UTF-8"); + in = new ReplaceInputStream(new ByteArrayInputStream(data), + "blue".getBytes("UTF-8"), "red".getBytes("UTF-8")); + + checkArrays(this, "FIRST", in, "I like red".getBytes("UTF-8")); + } + }); } static void checkArrays(TestCase test, String prefix, InputStream in, -- 2.27.0 From eeaa5ebc0ebe6ffd860273ed6727024737085a25 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Sat, 27 Apr 2019 15:41:53 +0200 Subject: [PATCH 15/16] jDoc + code cleanup --- src/be/nikiroo/utils/BufferedInputStream.java | 82 +++++++++++++++++-- src/be/nikiroo/utils/ReplaceInputStream.java | 2 +- 2 files changed, 74 insertions(+), 10 deletions(-) diff --git a/src/be/nikiroo/utils/BufferedInputStream.java b/src/be/nikiroo/utils/BufferedInputStream.java index 9b95574..b236f8d 100644 --- a/src/be/nikiroo/utils/BufferedInputStream.java +++ b/src/be/nikiroo/utils/BufferedInputStream.java @@ -14,9 +14,13 @@ import java.util.Arrays; * @author niki */ public class BufferedInputStream extends InputStream { + /** The current position in the buffer. */ protected int pos; + /** The index of the last usable position of the buffer. */ protected int len; + /** The buffer itself. */ protected byte[] buffer; + /** An End-Of-File (or buffer, here) marker. */ protected boolean eof; private boolean closed; @@ -112,12 +116,42 @@ public class BufferedInputStream extends InputStream { return this; } - // max is buffer.size ! + /** + * Check if the current content (what will be read next) starts with the + * given search term. + *

+ * Note: the search term size must be smaller or equal the internal + * buffer size. + * + * @param search + * the term to search for + * + * @return TRUE if the content that will be read starts with it + * + * @throws IOException + * in case of I/O error or if the size of the search term is + * greater than the internal buffer + */ public boolean startsWiths(String search) throws IOException { return startsWith(search.getBytes("UTF-8")); } - // max is buffer.size ! + /** + * Check if the current content (what will be read next) starts with the + * given search term. + *

+ * Note: the search term size must be smaller or equal the internal + * buffer size. + * + * @param search + * the term to search for + * + * @return TRUE if the content that will be read starts with it + * + * @throws IOException + * in case of I/O error or if the size of the search term is + * greater than the internal buffer + */ public boolean startsWith(byte[] search) throws IOException { if (search.length > originalBuffer.length) { throw new IOException( @@ -133,7 +167,7 @@ public class BufferedInputStream extends InputStream { if (available() >= search.length) { // Easy path - return startsWith(search, buffer, pos); + return startsWith(search, buffer, pos, len); } else if (!eof) { // Harder path if (buffer2 == null && buffer.length == originalBuffer.length) { @@ -149,9 +183,7 @@ public class BufferedInputStream extends InputStream { len2 += pos2; } - if (available() + (len2 - pos2) >= search.length) { - return startsWith(search, buffer2, pos2); - } + return startsWith(search, buffer2, pos2, len2); } return false; @@ -288,6 +320,9 @@ public class BufferedInputStream extends InputStream { * closed when you have called {@link BufferedInputStream#close()} once more * than {@link BufferedInputStream#open()}. * + * @param includingSubStream + * also close the under-laying stream + * * @exception IOException * in case of I/O error */ @@ -381,12 +416,41 @@ public class BufferedInputStream extends InputStream { protected void checkClose() throws IOException { if (closed) { throw new IOException( - "This NextableInputStream was closed, you cannot use it anymore."); + "This BufferedInputStream was closed, you cannot use it anymore."); } } - // buffer must be > search - static protected boolean startsWith(byte[] search, byte[] buffer, int offset) { + /** + * Check if the buffer starts with the given search term (given as an array, + * a start position and a end position). + *

+ * Note: the parameter len is the index of the last + * position, not the length. + *

+ * Note: the search term size must be smaller or equal the internal + * buffer size. + * + * @param search + * the term to search for + * @param buffer + * the buffer to look into + * @param offset + * the offset at which to start the search + * @param len + * the maximum index of the data to check (this is not a + * length, but an index) + * + * @return TRUE if the search content is present at the given location and + * does not exceed the len index + */ + static protected boolean startsWith(byte[] search, byte[] buffer, + int offset, int len) { + + // Check if there even is enough space for it + if (search.length > (len - offset)) { + return false; + } + boolean same = true; for (int i = 0; i < search.length; i++) { if (search[i] != buffer[offset + i]) { diff --git a/src/be/nikiroo/utils/ReplaceInputStream.java b/src/be/nikiroo/utils/ReplaceInputStream.java index ff2ec88..a456463 100644 --- a/src/be/nikiroo/utils/ReplaceInputStream.java +++ b/src/be/nikiroo/utils/ReplaceInputStream.java @@ -69,7 +69,7 @@ public class ReplaceInputStream extends BufferedInputStream { // Note: very simple, not efficient implementation, sorry. int count = 0; while (spos < slen && count < buffer.length - to.length) { - if (from.length > 0 && startsWith(from, source, spos)) { + if (from.length > 0 && startsWith(from, source, spos, slen)) { System.arraycopy(to, 0, buffer, spos, to.length); count += to.length; spos += from.length; -- 2.27.0 From eeb2cc17c4dbbbf9eb7f4c6ef695226a66fbf106 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Sat, 27 Apr 2019 17:08:12 +0200 Subject: [PATCH 16/16] fix BufferedInputStream offset + jdoc --- src/be/nikiroo/utils/BufferedInputStream.java | 5 +---- src/be/nikiroo/utils/NextableInputStream.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/be/nikiroo/utils/BufferedInputStream.java b/src/be/nikiroo/utils/BufferedInputStream.java index b236f8d..9db12f1 100644 --- a/src/be/nikiroo/utils/BufferedInputStream.java +++ b/src/be/nikiroo/utils/BufferedInputStream.java @@ -201,9 +201,6 @@ public class BufferedInputStream extends InputStream { /** * Check if this stream is totally spent (no more data to read or to * process). - *

- * Note: an empty stream that is still not started will return FALSE, as we - * don't know yet if it is empty. * * @return TRUE if it is */ @@ -246,7 +243,7 @@ public class BufferedInputStream extends InputStream { if (hasMoreData()) { int now = Math.min(blen, len) - pos; if (now > 0) { - System.arraycopy(buffer, pos, b, boff, now); + System.arraycopy(buffer, pos, b, boff + done, now); pos += now; done += now; } diff --git a/src/be/nikiroo/utils/NextableInputStream.java b/src/be/nikiroo/utils/NextableInputStream.java index 37322c9..7e2766a 100644 --- a/src/be/nikiroo/utils/NextableInputStream.java +++ b/src/be/nikiroo/utils/NextableInputStream.java @@ -111,6 +111,20 @@ public class NextableInputStream extends BufferedInputStream { return next(true); } + /** + * Check if this stream is totally spent (no more data to read or to + * process). + *

+ * Note: an empty stream that is still not started will return FALSE, as we + * don't know yet if it is empty. + * + * @return TRUE if it is + */ + @Override + public boolean eof() { + return super.eof(); + } + /** * Check if we still have some data in the buffer and, if not, fetch some. * -- 2.27.0