From efe7b5989ac8a72375a1abfaf9aa3f50a18ea58d Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Wed, 20 May 2020 13:28:17 +0200 Subject: [PATCH] fix remplace/buffered input stream --- .../utils/streams/BufferedInputStream.java | 75 ++++++---- .../utils/streams/ReplaceInputStream.java | 130 +++++++++++++----- .../test_code/ReplaceInputStreamTest.java | 101 +++++++++++++- 3 files changed, 234 insertions(+), 72 deletions(-) diff --git a/src/be/nikiroo/utils/streams/BufferedInputStream.java b/src/be/nikiroo/utils/streams/BufferedInputStream.java index babd2ce8..3cad70d2 100644 --- a/src/be/nikiroo/utils/streams/BufferedInputStream.java +++ b/src/be/nikiroo/utils/streams/BufferedInputStream.java @@ -41,6 +41,7 @@ public class BufferedInputStream extends InputStream { private boolean closed; private InputStream in; private int openCounter; + private byte[] singleByteReader = new byte[1]; /** array + offset of pushed-back buffers */ private List> backBuffers; @@ -48,7 +49,7 @@ public class BufferedInputStream extends InputStream { private long bytesRead; /** - * Create a new {@link BufferedInputStream2} that wraps the given + * Create a new {@link BufferedInputStream} that wraps the given * {@link InputStream}. * * @param in @@ -64,8 +65,8 @@ public class BufferedInputStream extends InputStream { } /** - * Create a new {@link BufferedInputStream2} that wraps the given bytes - * array as a data source. + * Create a new {@link BufferedInputStream} that wraps the given bytes array + * as a data source. * * @param in * the array to wrap, cannot be NULL @@ -75,8 +76,8 @@ public class BufferedInputStream extends InputStream { } /** - * Create a new {@link BufferedInputStream2} that wraps the given bytes - * array as a data source. + * Create a new {@link BufferedInputStream} that wraps the given bytes array + * as a data source. * * @param in * the array to wrap, cannot be NULL @@ -107,9 +108,9 @@ public class BufferedInputStream extends InputStream { } /** - * Return this very same {@link BufferedInputStream2}, but keep a counter of + * Return this very same {@link BufferedInputStream}, but keep a counter of * how many streams were open this way. When calling - * {@link BufferedInputStream2#close()}, decrease this counter if it is not + * {@link BufferedInputStream#close()}, decrease this counter if it is not * already zero instead of actually closing the stream. *

* You are now responsible for it — you must close it. @@ -281,14 +282,11 @@ public class BufferedInputStream extends InputStream { @Override public int read() throws IOException { - checkClose(); - - preRead(); - if (eof) { + if (read(singleByteReader) < 0) { return -1; } - return buffer[start++]; + return singleByteReader[0]; } @Override @@ -309,6 +307,10 @@ public class BufferedInputStream extends InputStream { } // Read from the pushed-back buffers if any + if (backBuffers.isEmpty()) { + preRead(); // an implementation could pushback in preRead() + } + if (!backBuffers.isEmpty()) { int read = 0; @@ -317,7 +319,7 @@ public class BufferedInputStream extends InputStream { byte[] bbBuffer = bb.getKey(); int bbOffset = bb.getValue(); int bbSize = bbBuffer.length - bbOffset; - + if (bbSize > blen) { read = blen; System.arraycopy(bbBuffer, bbOffset, b, boff, read); @@ -402,11 +404,11 @@ public class BufferedInputStream extends InputStream { *

* Including the under-laying {@link InputStream}. *

- * Note: if you called the {@link BufferedInputStream2#open()} method + * Note: if you called the {@link BufferedInputStream#open()} method * prior to this one, it will just decrease the internal count of how many * open streams it held and do nothing else. The stream will actually be - * closed when you have called {@link BufferedInputStream2#close()} once - * more than {@link BufferedInputStream2#open()}. + * closed when you have called {@link BufferedInputStream#close()} once more + * than {@link BufferedInputStream#open()}. * * @exception IOException * in case of I/O error @@ -426,11 +428,11 @@ public class BufferedInputStream extends InputStream { * You can call this method multiple times, it will not cause an * {@link IOException} for subsequent calls. *

- * Note: if you called the {@link BufferedInputStream2#open()} method + * Note: if you called the {@link BufferedInputStream#open()} method * prior to this one, it will just decrease the internal count of how many * open streams it held and do nothing else. The stream will actually be - * closed when you have called {@link BufferedInputStream2#close()} once - * more than {@link BufferedInputStream2#open()}. + * closed when you have called {@link BufferedInputStream#close()} once more + * than {@link BufferedInputStream#open()}. * * @param includingSubStream * also close the under-laying stream @@ -511,7 +513,7 @@ public class BufferedInputStream extends InputStream { boolean hasRead = false; if (in != null && !eof && start >= stop) { start = 0; - stop = read(in, buffer, 0, buffer.length); + stop = read(in, buffer); if (stop > 0) { bytesRead += stop; } @@ -539,26 +541,43 @@ public class BufferedInputStream extends InputStream { new AbstractMap.SimpleEntry(buffer, offset)); } + /** + * Push back some data that will be read again at the next read call. + * + * @param buffer + * the buffer to push back + * @param offset + * the offset at which to start reading in the buffer + * @param len + * the length to copy + */ + protected void pushback(byte[] buffer, int offset, int len) { + // TODO: not efficient! + if (buffer.length != len) { + byte[] lenNotSupportedYet = new byte[len]; + System.arraycopy(buffer, offset, lenNotSupportedYet, 0, len); + buffer = lenNotSupportedYet; + offset = 0; + } + + pushback(buffer, offset); + } + /** * Read the under-laying stream into the given local buffer. * * @param in * the under-laying {@link InputStream} * @param buffer - * the buffer we use in this {@link BufferedInputStream2} - * @param off - * the offset - * @param len - * the length in bytes + * the buffer we use in this {@link BufferedInputStream} * * @return the number of bytes read * * @throws IOException * in case of I/O error */ - protected int read(InputStream in, byte[] buffer, int off, int len) - throws IOException { - return in.read(buffer, off, len); + protected int read(InputStream in, byte[] buffer) throws IOException { + return in.read(buffer, 0, buffer.length); } /** diff --git a/src/be/nikiroo/utils/streams/ReplaceInputStream.java b/src/be/nikiroo/utils/streams/ReplaceInputStream.java index ae576e25..0860f78c 100644 --- a/src/be/nikiroo/utils/streams/ReplaceInputStream.java +++ b/src/be/nikiroo/utils/streams/ReplaceInputStream.java @@ -2,6 +2,8 @@ package be.nikiroo.utils.streams; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; import be.nikiroo.utils.StringUtils; @@ -23,12 +25,8 @@ public class ReplaceInputStream extends BufferedInputStream { private byte[][] froms; private byte[][] tos; + private int bufferSize; private int maxFromSize; - private int maxToSize; - - private byte[] source; - private int spos; - private int slen; /** * Create a {@link ReplaceInputStream} that will replace from with @@ -108,56 +106,112 @@ public class ReplaceInputStream extends BufferedInputStream { maxFromSize = Math.max(maxFromSize, froms[i].length); } - maxToSize = 0; + int maxToSize = 0; for (int i = 0; i < tos.length; i++) { maxToSize = Math.max(maxToSize, tos[i].length); } // We need at least maxFromSize so we can iterate and replace - source = new byte[Math.max(2 * Math.max(maxToSize, maxFromSize), - MIN_BUFFER_SIZE)]; - spos = 0; - slen = 0; + bufferSize = Math.max(4 * Math.max(maxToSize, maxFromSize), + MIN_BUFFER_SIZE); } @Override - protected int read(InputStream in, byte[] buffer, int off, int len) - throws IOException { - if (len < maxToSize || source.length < maxToSize * 2) { - throw new IOException( - "An underlaying buffer is too small for these replace values"); - } + protected boolean preRead() throws IOException { + boolean rep = super.preRead(); + start = stop; + return rep; + } - // We need at least one byte of data to process - if (available() < Math.max(maxFromSize, 1) && !eof) { - spos = 0; - slen = in.read(source); + @Override + protected int read(InputStream in, byte[] buffer) throws IOException { + buffer = null; // do not use the buffer. + + byte[] newBuffer = new byte[bufferSize]; + int read = 0; + while (read < bufferSize / 2) { + int thisTime = in.read(newBuffer, read, bufferSize / 2 - read); + if (thisTime <= 0) { + break; + } + read += thisTime; } - // Note: very simple, not efficient implementation; sorry. - int count = 0; - while (spos < slen && count < len - maxToSize) { - boolean replaced = false; - for (int i = 0; i < froms.length; i++) { - if (froms[i] != null && froms[i].length > 0 - && StreamUtils.startsWith(froms[i], source, spos, slen)) { - if (tos[i] != null && tos[i].length > 0) { - System.arraycopy(tos[i], 0, buffer, off + count, - tos[i].length); - count += tos[i].length; + List bbBuffers = new ArrayList(); + List bbOffsets = new ArrayList(); + List bbLengths = new ArrayList(); + + int offset = 0; + for (int i = 0; i < read; i++) { + for (int fromIndex = 0; fromIndex < froms.length; fromIndex++) { + byte[] from = froms[fromIndex]; + byte[] to = tos[fromIndex]; + + if (from.length > 0 + && StreamUtils.startsWith(from, newBuffer, i, read)) { + if (i - offset > 0) { + bbBuffers.add(newBuffer); + bbOffsets.add(offset); + bbLengths.add(i - offset); } - - spos += froms[i].length; - replaced = true; - break; + + if (to.length > 0) { + bbBuffers.add(to); + bbOffsets.add(0); + bbLengths.add(to.length); + } + + i += from.length; + offset = i; } } + } + + if (offset < read) { + bbBuffers.add(newBuffer); + bbOffsets.add(offset); + bbLengths.add(read - offset); + } - if (!replaced) { - buffer[off + count++] = source[spos++]; + for (int i = bbBuffers.size() - 1; i >= 0; i--) { + // DEBUG("pushback", bbBuffers.get(i), bbOffsets.get(i), + // bbLengths.get(i)); + pushback(bbBuffers.get(i), bbOffsets.get(i), bbLengths.get(i)); + } + + return read; + } + + // static public void DEBUG(String title, byte[] b, int off, int len) { + // String str = new String(b,off,len); + // if(str.length()>20) { + // str=str.substring(0,10)+" ... + // "+str.substring(str.length()-10,str.length()); + // } + // } + + @Override + public String toString() { + StringBuilder rep = new StringBuilder(); + rep.append(getClass().getSimpleName()).append("\n"); + + for (int i = 0; i < froms.length; i++) { + byte[] from = froms[i]; + byte[] to = tos[i]; + + rep.append("\t"); + rep.append("bytes[").append(from.length).append("]"); + if (from.length <= 20) { + rep.append(" (").append(new String(from)).append(")"); + } + rep.append(" -> "); + rep.append("bytes[").append(to.length).append("]"); + if (to.length <= 20) { + rep.append(" (").append(new String(to)).append(")"); } + rep.append("\n"); } - return count; + return "[" + rep + "]"; } } diff --git a/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java b/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java index d08a91ed..efab8c73 100644 --- a/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java +++ b/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java @@ -83,33 +83,122 @@ class ReplaceInputStreamTest extends TestLauncher { checkArrays(this, "FIRST", in, "I like blue".getBytes("UTF-8")); - data = "I like blue".getBytes("UTF-8"); + data = "I like blue hammers".getBytes("UTF-8"); in = new ReplaceInputStream(new ByteArrayInputStream(data), "blue", "red"); - checkArrays(this, "FIRST", in, "I like red".getBytes("UTF-8")); + checkArrays(this, "SECOND", in, "I like red hammers".getBytes("UTF-8")); } }); addTest(new TestCase("Multiple replaces") { @Override public void test() throws Exception { - byte[] data = "I like red".getBytes("UTF-8"); + byte[] data = "I like red cabage".getBytes("UTF-8"); ReplaceInputStream in = new ReplaceInputStream( new ByteArrayInputStream(data), // - new String[] {"like", "red"}, // - new String[] {"dislike", "green"} // + new String[] { "red", "like" }, // + new String[] { "green", "very very much like" } // + ); + + String result = new String(IOUtils.toByteArray(in), "UTF-8"); + assertEquals("I very very much like green cabage", result); + } + }); + + addTest(new TestCase("Multiple replaces") { + @Override + public void test() throws Exception { + String str= ("" // + + "\n" // + + "\n" // + + "\n" // + + "\n" // + + "\t\n" // + + "\t\n" // + + "\t${title}\n" // + + "\t\n" // + + "\t\n" // + + "\n" // + + "\n" // + + "\t

\n" // + + "${banner}${content}\t
\n" // + + "\n" // + + "" // + ); + byte[] data = str.getBytes("UTF-8"); + + String title = "Fanfix"; + String banner = ""; + String content = ""; + + InputStream in = new ReplaceInputStream( + new ByteArrayInputStream(data), // + new String[] { "${title}", "${banner}", "${content}" }, // + new String[] { title, banner, content } // ); String result = new String(IOUtils.toByteArray(in), "UTF-8"); - assertEquals("I dislike green", result); + assertEquals(str // + .replace("${title}", title) // + .replace("${banner}", banner) // + .replace("${content}", content) // + , result); } }); + + } static void checkArrays(TestCase test, String prefix, InputStream in, byte[] expected) throws Exception { byte[] actual = IOUtils.toByteArray(in); + +// System.out.println("\nActual:"); +// for(byte byt : actual) { +// System.out.print(byt+" "); +// } +// System.out.println("\nExpected:"); +// for(byte byt : expected) { +// System.out.print(byt+" "); +// } + test.assertEquals("The " + prefix + " resulting array has not the correct number of items", expected.length, actual.length); -- 2.27.0