From: Niki Roo <niki@nikiroo.be>
Date: Thu, 25 Apr 2019 15:41:06 +0000 (+0200)
Subject: Merge branch 'master' into streamify
X-Git-Url: http://git.nikiroo.be/?p=nikiroo-utils.git;a=commitdiff_plain;h=1d20e650389b5aaaaa5de4213cefefade582da5d;hp=dc41a952e6375fe6aa1579614fe717d260fc78a5

Merge branch 'master' into streamify
---

diff --git a/src/be/nikiroo/utils/NextableInputStream.java b/src/be/nikiroo/utils/NextableInputStream.java
index 0def936..b5374a1 100644
--- a/src/be/nikiroo/utils/NextableInputStream.java
+++ b/src/be/nikiroo/utils/NextableInputStream.java
@@ -2,6 +2,7 @@ package be.nikiroo.utils;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Arrays;
 
 /**
  * This {@link InputStream} can be separated into sub-streams (you can process
@@ -14,13 +15,24 @@ import java.io.InputStream;
  */
 public class NextableInputStream extends InputStream {
 	private NextableInputStreamStep step;
+	private boolean started;
 	private boolean stopped;
+	private boolean closed;
 
 	private InputStream in;
+	private int openCounter;
 	private boolean eof;
-	private int pos = 0;
-	private int len = 0;
-	private byte[] buffer = new byte[4096];
+	private int pos;
+	private int len;
+	private byte[] buffer;
+
+	// special use, prefetched next buffer
+	private byte[] buffer2;
+	private int pos2;
+	private int len2;
+	private byte[] originalBuffer;
+
+	private long bytesRead;
 
 	/**
 	 * Create a new {@link NextableInputStream} that wraps the given
@@ -35,6 +47,86 @@ public class NextableInputStream extends InputStream {
 	public NextableInputStream(InputStream in, NextableInputStreamStep step) {
 		this.in = in;
 		this.step = step;
+
+		this.buffer = new byte[4096];
+		this.originalBuffer = this.buffer;
+		this.pos = 0;
+		this.len = 0;
+	}
+
+	/**
+	 * Create a new {@link NextableInputStream} that wraps the given bytes array
+	 * as a data source.
+	 * 
+	 * @param in
+	 *            the array to wrap, cannot be NULL
+	 * @param step
+	 *            how to separate it into sub-streams (can be NULL, but in that
+	 *            case it will behave as a normal {@link InputStream})
+	 */
+	public NextableInputStream(byte[] in, NextableInputStreamStep step) {
+		this(in, step, 0, in.length);
+	}
+
+	/**
+	 * Create a new {@link NextableInputStream} that wraps the given bytes array
+	 * as a data source.
+	 * 
+	 * @param in
+	 *            the array to wrap, cannot be NULL
+	 * @param step
+	 *            how to separate it into sub-streams (can be NULL, but in that
+	 *            case it will behave as a normal {@link InputStream})
+	 * @param offset
+	 *            the offset to start the reading at
+	 * @param length
+	 *            the number of bytes to take into account in the array,
+	 *            starting from the offset
+	 * 
+	 * @throws NullPointerException
+	 *             if the array is NULL
+	 * @throws IndexOutOfBoundsException
+	 *             if the offset and length do not correspond to the given array
+	 */
+	public NextableInputStream(byte[] in, NextableInputStreamStep step,
+			int offset, int length) {
+		if (in == null) {
+			throw new NullPointerException();
+		} else if (offset < 0 || length < 0 || length > in.length - offset) {
+			throw new IndexOutOfBoundsException();
+		}
+
+		this.in = null;
+		this.step = step;
+
+		this.buffer = in;
+		this.originalBuffer = this.buffer;
+		this.pos = offset;
+		this.len = length;
+
+		checkBuffer(true);
+	}
+
+	/**
+	 * Return this very same {@link NextableInputStream}, but keep a counter of
+	 * how many streams were open this way. When calling
+	 * {@link NextableInputStream#close()}, decrease this counter if it is not
+	 * already zero instead of actually closing the stream.
+	 * <p>
+	 * You are now responsible for it &mdash; you <b>must</b> close it.
+	 * <p>
+	 * This method allows you to use a wrapping stream around this one and still
+	 * close the wrapping stream.
+	 * 
+	 * @return the same stream, but you are now responsible for closing it
+	 * 
+	 * @throws IOException
+	 *             in case of I/O error or if the stream is closed
+	 */
+	public synchronized InputStream open() throws IOException {
+		checkClose();
+		openCounter++;
+		return this;
 	}
 
 	/**
@@ -53,27 +145,96 @@ public class NextableInputStream extends InputStream {
 	 * @return TRUE if we unblocked the next sub-stream, FALSE if not
 	 * 
 	 * @throws IOException
-	 *             in case of I/O error
+	 *             in case of I/O error or if the stream is closed
 	 */
 	public boolean next() throws IOException {
-		if (!hasMoreData() && stopped) {
-			len = step.getResumeLen();
-			pos += step.getResumeSkip();
-			eof = false;
+		return next(false);
+	}
 
-			if (!preRead()) {
-				checkBuffer(false);
+	/**
+	 * Unblock the next sub-stream as would have done
+	 * {@link NextableInputStream#next()}, but disable the sub-stream systems.
+	 * <p>
+	 * That is, the next stream, if any, will be the last one and will not be
+	 * subject to the {@link NextableInputStreamStep}.
+	 * 
+	 * @return TRUE if we unblocked the next sub-stream, FALSE if not
+	 * 
+	 * @throws IOException
+	 *             in case of I/O error or if the stream is closed
+	 */
+	public boolean nextAll() throws IOException {
+		return next(true);
+	}
+
+	// max is buffer.size !
+	public boolean startsWiths(String search) throws IOException {
+		return startsWith(search.getBytes("UTF-8"));
+	}
+
+	// max is buffer.size !
+	public boolean startsWith(byte[] search) throws IOException {
+		if (search.length > originalBuffer.length) {
+			throw new IOException(
+					"This stream does not support searching for more than "
+							+ buffer.length + " bytes");
+		}
+
+		checkClose();
+
+		if (available() < search.length) {
+			preRead();
+		}
+
+		if (available() >= search.length) {
+			// Easy path
+			return startsWith(search, buffer, pos);
+		} else if (!eof) {
+			// Harder path
+			if (buffer2 == null && buffer.length == originalBuffer.length) {
+				buffer2 = Arrays.copyOf(buffer, buffer.length * 2);
+
+				pos2 = buffer.length;
+				len2 = in.read(buffer2, pos2, buffer.length);
+				if (len2 > 0) {
+					bytesRead += len2;
+				}
+
+				// Note: here, len/len2 = INDEX of last good byte
+				len2 += pos2;
 			}
 
-			// consider that if EOF, there is no next
-			return hasMoreData();
+			if (available() + (len2 - pos2) >= search.length) {
+				return startsWith(search, buffer2, pos2);
+			}
 		}
 
 		return false;
 	}
 
+	/**
+	 * The number of bytes read from the under-laying {@link InputStream}.
+	 * 
+	 * @return the number of bytes
+	 */
+	public long getBytesRead() {
+		return bytesRead;
+	}
+
+	/**
+	 * Check if this stream is totally spent (no more data to read or to
+	 * process).
+	 * 
+	 * @return TRUE if it is
+	 */
+	public boolean eof() {
+		return closed || (len < 0 && !hasMoreData());
+	}
+
 	@Override
 	public int read() throws IOException {
+		checkClose();
+
 		preRead();
 		if (eof) {
 			return -1;
@@ -89,6 +250,8 @@ public class NextableInputStream extends InputStream {
 
 	@Override
 	public int read(byte[] b, int boff, int blen) throws IOException {
+		checkClose();
+
 		if (b == null) {
 			throw new NullPointerException();
 		} else if (boff < 0 || blen < 0 || blen > b.length - boff) {
@@ -113,10 +276,86 @@ public class NextableInputStream extends InputStream {
 	}
 
 	@Override
-	public int available() throws IOException {
+	public long skip(long n) throws IOException {
+		if (n <= 0) {
+			return 0;
+		}
+
+		long skipped = 0;
+		while (hasMoreData() && n > 0) {
+			preRead();
+
+			long inBuffer = Math.min(n, available());
+			pos += inBuffer;
+			n -= inBuffer;
+			skipped += inBuffer;
+		}
+
+		return skipped;
+	}
+
+	@Override
+	public int available() {
+		if (closed) {
+			return 0;
+		}
+
 		return Math.max(0, len - pos);
 	}
 
+	/**
+	 * Closes this stream and releases any system resources associated with the
+	 * stream.
+	 * <p>
+	 * Including the under-laying {@link InputStream}.
+	 * <p>
+	 * <b>Note:</b> if you called the {@link NextableInputStream#open()} method
+	 * prior to this one, it will just decrease the internal count of how many
+	 * open streams it held and do nothing else. The stream will actually be
+	 * closed when you have called {@link NextableInputStream#close()} once more
+	 * than {@link NextableInputStream#open()}.
+	 * 
+	 * @exception IOException
+	 *                in case of I/O error
+	 */
+	@Override
+	public synchronized void close() throws IOException {
+		close(true);
+	}
+
+	/**
+	 * Closes this stream and releases any system resources associated with the
+	 * stream.
+	 * <p>
+	 * Including the under-laying {@link InputStream} if
+	 * <tt>incudingSubStream</tt> is true.
+	 * <p>
+	 * You can call this method multiple times, it will not cause an
+	 * {@link IOException} for subsequent calls.
+	 * <p>
+	 * <b>Note:</b> if you called the {@link NextableInputStream#open()} method
+	 * prior to this one, it will just decrease the internal count of how many
+	 * open streams it held and do nothing else. The stream will actually be
+	 * closed when you have called {@link NextableInputStream#close()} once more
+	 * than {@link NextableInputStream#open()}.
+	 * 
+	 * @exception IOException
+	 *                in case of I/O error
+	 */
+	public synchronized void close(boolean includingSubStream)
+			throws IOException {
+		if (!closed) {
+			if (openCounter > 0) {
+				openCounter--;
+			} else {
+				closed = true;
+				if (includingSubStream && in != null) {
+					in.close();
+				}
+			}
+		}
+	}
+
 	/**
 	 * Check if we still have some data in the buffer and, if not, fetch some.
 	 * 
@@ -130,7 +369,22 @@ public class NextableInputStream extends InputStream {
 		boolean hasRead = false;
 		if (!eof && in != null && pos >= len && !stopped) {
 			pos = 0;
-			len = in.read(buffer);
+			if (buffer2 != null) {
+				buffer = buffer2;
+				pos = pos2;
+				len = len2;
+
+				buffer2 = null;
+				pos2 = 0;
+				len2 = 0;
+			} else {
+				buffer = originalBuffer;
+				len = in.read(buffer);
+				if (len > 0) {
+					bytesRead += len;
+				}
+			}
+
 			checkBuffer(true);
 			hasRead = true;
 		}
@@ -148,7 +402,7 @@ public class NextableInputStream extends InputStream {
 	 * @return TRUE if it is the case, FALSE if not
 	 */
 	private boolean hasMoreData() {
-		return !(eof && pos >= len);
+		return !closed && started && !(eof && pos >= len);
 	}
 
 	/**
@@ -164,7 +418,7 @@ public class NextableInputStream extends InputStream {
 	 *            the {@link NextableInputStreamStep}
 	 */
 	private void checkBuffer(boolean newBuffer) {
-		if (step != null) {
+		if (step != null && len > 0) {
 			if (newBuffer) {
 				step.clearBuffer();
 			}
@@ -177,4 +431,79 @@ public class NextableInputStream extends InputStream {
 			}
 		}
 	}
+
+	/**
+	 * The implementation of {@link NextableInputStream#next()} and
+	 * {@link NextableInputStream#nextAll()}.
+	 * 
+	 * @param all
+	 *            TRUE for {@link NextableInputStream#nextAll()}, FALSE for
+	 *            {@link NextableInputStream#next()}
+	 * 
+	 * @return TRUE if we unblocked the next sub-stream, FALSE if not
+	 * 
+	 * @throws IOException
+	 *             in case of I/O error or if the stream is closed
+	 */
+	private boolean next(boolean all) throws IOException {
+		checkClose();
+
+		if (!started) {
+			// First call before being allowed to read
+			started = true;
+
+			if (all) {
+				step = null;
+			}
+
+			return true;
+		}
+
+		if (step != null && !hasMoreData() && stopped) {
+			len = step.getResumeLen();
+			pos += step.getResumeSkip();
+			eof = false;
+
+			if (all) {
+				step = null;
+			}
+
+			if (!preRead()) {
+				checkBuffer(false);
+			}
+
+			// consider that if EOF, there is no next
+			return hasMoreData();
+		}
+
+		return false;
+	}
+
+	/**
+	 * Check that the stream was not closed, and throw an {@link IOException} if
+	 * it was.
+	 * 
+	 * @throws IOException
+	 *             if it was closed
+	 */
+	private void checkClose() throws IOException {
+		if (closed) {
+			throw new IOException(
+					"This NextableInputStream was closed, you cannot use it anymore.");
+		}
+	}
+
+	// buffer must be > search
+	static private boolean startsWith(byte[] search, byte[] buffer,
+			int offset) {
+		boolean same = true;
+		for (int i = 0; i < search.length; i++) {
+			if (search[i] != buffer[offset + i]) {
+				same = false;
+				break;
+			}
+		}
+
+		return same;
+	}
 }
diff --git a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java
index 87d64ac..4664cbf 100644
--- a/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java
+++ b/src/be/nikiroo/utils/test_code/NextableInputStreamTest.java
@@ -1,6 +1,7 @@
 package be.nikiroo.utils.test_code;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
 
 import be.nikiroo.utils.IOUtils;
 import be.nikiroo.utils.NextableInputStream;
@@ -18,6 +19,7 @@ public class NextableInputStreamTest extends TestLauncher {
 				byte[] expected = new byte[] { 42, 12, 0, 127 };
 				NextableInputStream in = new NextableInputStream(
 						new ByteArrayInputStream(expected), null);
+				in.next();
 				byte[] actual = IOUtils.toByteArray(in);
 
 				assertEquals(
@@ -38,7 +40,7 @@ public class NextableInputStreamTest extends TestLauncher {
 						new ByteArrayInputStream(expected),
 						new NextableInputStreamStep(12));
 
-				checkNext(this, false, "FIRST", in, new byte[] { 42 });
+				checkNext(this, "FIRST", in, new byte[] { 42 });
 			}
 		});
 
@@ -50,9 +52,9 @@ public class NextableInputStreamTest extends TestLauncher {
 						new ByteArrayInputStream(data),
 						new NextableInputStreamStep(12));
 
-				checkNext(this, false, "FIRST", in, new byte[] { 42 });
-				checkNext(this, true, "SECOND", in, new byte[] { 0, 127 });
-				checkNext(this, true, "THIRD", in, new byte[] { 51, 11 });
+				checkNext(this, "FIRST", in, new byte[] { 42 });
+				checkNext(this, "SECOND", in, new byte[] { 0, 127 });
+				checkNext(this, "THIRD", in, new byte[] { 51, 11 });
 			}
 		});
 
@@ -66,14 +68,14 @@ public class NextableInputStreamTest extends TestLauncher {
 				NextableInputStream subIn12 = new NextableInputStream(in4,
 						new NextableInputStreamStep(12));
 
-				checkNext(this, false, "SUB FIRST", subIn12, new byte[] { 42 });
-				checkNext(this, true, "SUB SECOND", subIn12, new byte[] { 0 });
+				in4.next();
+				checkNext(this, "SUB FIRST", subIn12, new byte[] { 42 });
+				checkNext(this, "SUB SECOND", subIn12, new byte[] { 0 });
 
 				assertEquals("The subIn still has some data", false,
 						subIn12.next());
 
-				checkNext(this, true, "MAIN LAST", in4,
-						new byte[] { 127, 12, 5 });
+				checkNext(this, "MAIN LAST", in4, new byte[] { 127, 12, 5 });
 			}
 		});
 
@@ -87,18 +89,147 @@ public class NextableInputStreamTest extends TestLauncher {
 						new ByteArrayInputStream(data),
 						new NextableInputStreamStep('\n'));
 
-				checkNext(this, false, "FIRST", in, ln1.getBytes("UTF-8"));
-				checkNext(this, true, "SECOND", in, ln2.getBytes("UTF-8"));
+				checkNext(this, "FIRST", in, ln1.getBytes("UTF-8"));
+				checkNext(this, "SECOND", in, ln2.getBytes("UTF-8"));
+			}
+		});
+
+		addTest(new TestCase("nextAll()") {
+			@Override
+			public void test() throws Exception {
+				byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 };
+				NextableInputStream in = new NextableInputStream(
+						new ByteArrayInputStream(data),
+						new NextableInputStreamStep(12));
+
+				checkNext(this, "FIRST", in, new byte[] { 42 });
+				checkNextAll(this, "REST", in,
+						new byte[] { 0, 127, 12, 51, 11, 12 });
+				assertEquals("The stream still has some data", false,
+						in.next());
+			}
+		});
+
+		addTest(new TestCase("getBytesRead()") {
+			@Override
+			public void test() throws Exception {
+				byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 };
+				NextableInputStream in = new NextableInputStream(
+						new ByteArrayInputStream(data),
+						new NextableInputStreamStep(12));
+
+				in.nextAll();
+				IOUtils.toByteArray(in);
+
+				assertEquals("The number of bytes read is not correct",
+						data.length, in.getBytesRead());
+			}
+		});
+
+		addTest(new TestCase("bytes array input") {
+			@Override
+			public void test() throws Exception {
+				byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 };
+				NextableInputStream in = new NextableInputStream(data,
+						new NextableInputStreamStep(12));
+
+				checkNext(this, "FIRST", in, new byte[] { 42 });
+				checkNext(this, "SECOND", in, new byte[] { 0, 127 });
+				checkNext(this, "THIRD", in, new byte[] { 51, 11 });
+			}
+		});
+
+		addTest(new TestCase("Skip data") {
+			@Override
+			public void test() throws Exception {
+				byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 };
+				NextableInputStream in = new NextableInputStream(data, null);
+				in.next();
+
+				in.skip(4);
+				checkArrays(this, "ONLY", in, new byte[] { 12, 51, 11, 12 });
+			}
+		});
+
+		addTest(new TestCase("Starts with") {
+			@Override
+			public void test() throws Exception {
+				byte[] data = new byte[] { 42, 12, 0, 127, 12, 51, 11, 12 };
+				NextableInputStream in = new NextableInputStream(data, null);
+				in.next();
+
+				// yes
+				assertEquals("It actually starts with that", true,
+						in.startsWith(new byte[] { 42 }));
+				assertEquals("It actually starts with that", true,
+						in.startsWith(new byte[] { 42, 12 }));
+				assertEquals("It actually is the same array", true,
+						in.startsWith(data));
+
+				// no
+				assertEquals("It actually does not start with that", false,
+						in.startsWith(new byte[] { 12 }));
+				assertEquals("It actually does not start with that", false,
+						in.startsWith(
+								new byte[] { 42, 12, 0, 127, 12, 51, 11, 11 }));
+
+				// too big
+				try {
+					in.startsWith(
+							new byte[] { 42, 12, 0, 127, 12, 51, 11, 12, 0 });
+					fail("Searching a prefix bigger than the array should throw an IOException");
+				} catch (IOException e) {
+				}
+			}
+		});
+
+		addTest(new TestCase("Starts with strings") {
+			@Override
+			public void test() throws Exception {
+				String text = "Fanfan et Toto vont à la mer";
+				byte[] data = text.getBytes("UTF-8");
+				NextableInputStream in = new NextableInputStream(data, null);
+				in.next();
+
+				// yes
+				assertEquals("It actually starts with that", true,
+						in.startsWiths("F"));
+				assertEquals("It actually starts with that", true,
+						in.startsWiths("Fanfan et"));
+				assertEquals("It actually is the same text", true,
+						in.startsWiths(text));
+
+				// no
+				assertEquals("It actually does not start with that", false,
+						in.startsWiths("Toto"));
+				assertEquals("It actually does not start with that", false,
+						in.startsWiths("Fanfan et Toto vont à la mee"));
+
+				// too big
+				try {
+					in.startsWiths("Fanfan et Toto vont à la mer.");
+					fail("Searching a prefix bigger than the array should throw an IOException");
+				} catch (IOException e) {
+				}
 			}
 		});
 	}
 
-	static void checkNext(TestCase test, boolean callNext, String prefix,
+	static void checkNext(TestCase test, String prefix, NextableInputStream in,
+			byte[] expected) throws Exception {
+		test.assertEquals("Cannot get " + prefix + " entry", true, in.next());
+		checkArrays(test, prefix, in, expected);
+	}
+
+	static void checkNextAll(TestCase test, String prefix,
+			NextableInputStream in, byte[] expected) throws Exception {
+		test.assertEquals("Cannot get " + prefix + " entries", true,
+				in.nextAll());
+		checkArrays(test, prefix, in, expected);
+	}
+
+	static void checkArrays(TestCase test, String prefix,
 			NextableInputStream in, byte[] expected) throws Exception {
-		if (callNext) {
-			test.assertEquals("Cannot get " + prefix + " entry", true,
-					in.next());
-		}
 		byte[] actual = IOUtils.toByteArray(in);
 		test.assertEquals("The " + prefix
 				+ " resulting array has not the correct number of items",