From: Niki Roo <niki@nikiroo.be>
Date: Wed, 20 May 2020 11:28:17 +0000 (+0200)
Subject: fix remplace/buffered input stream
X-Git-Url: https://git.nikiroo.be/?a=commitdiff_plain;h=efe7b5989ac8a72375a1abfaf9aa3f50a18ea58d;p=nikiroo-utils.git

fix remplace/buffered input stream
---

diff --git a/src/be/nikiroo/utils/streams/BufferedInputStream.java b/src/be/nikiroo/utils/streams/BufferedInputStream.java
index babd2ce..3cad70d 100644
--- a/src/be/nikiroo/utils/streams/BufferedInputStream.java
+++ b/src/be/nikiroo/utils/streams/BufferedInputStream.java
@@ -41,6 +41,7 @@ public class BufferedInputStream extends InputStream {
 	private boolean closed;
 	private InputStream in;
 	private int openCounter;
+	private byte[] singleByteReader = new byte[1];
 
 	/** array + offset of pushed-back buffers */
 	private List<Entry<byte[], Integer>> backBuffers;
@@ -48,7 +49,7 @@ public class BufferedInputStream extends InputStream {
 	private long bytesRead;
 
 	/**
-	 * Create a new {@link BufferedInputStream2} that wraps the given
+	 * Create a new {@link BufferedInputStream} that wraps the given
 	 * {@link InputStream}.
 	 * 
 	 * @param in
@@ -64,8 +65,8 @@ public class BufferedInputStream extends InputStream {
 	}
 
 	/**
-	 * Create a new {@link BufferedInputStream2} that wraps the given bytes
-	 * array as a data source.
+	 * Create a new {@link BufferedInputStream} that wraps the given bytes array
+	 * as a data source.
 	 * 
 	 * @param in
 	 *            the array to wrap, cannot be NULL
@@ -75,8 +76,8 @@ public class BufferedInputStream extends InputStream {
 	}
 
 	/**
-	 * Create a new {@link BufferedInputStream2} that wraps the given bytes
-	 * array as a data source.
+	 * Create a new {@link BufferedInputStream} that wraps the given bytes array
+	 * as a data source.
 	 * 
 	 * @param in
 	 *            the array to wrap, cannot be NULL
@@ -107,9 +108,9 @@ public class BufferedInputStream extends InputStream {
 	}
 
 	/**
-	 * Return this very same {@link BufferedInputStream2}, but keep a counter of
+	 * Return this very same {@link BufferedInputStream}, but keep a counter of
 	 * how many streams were open this way. When calling
-	 * {@link BufferedInputStream2#close()}, decrease this counter if it is not
+	 * {@link BufferedInputStream#close()}, decrease this counter if it is not
 	 * already zero instead of actually closing the stream.
 	 * <p>
 	 * You are now responsible for it &mdash; you <b>must</b> close it.
@@ -281,14 +282,11 @@ public class BufferedInputStream extends InputStream {
 
 	@Override
 	public int read() throws IOException {
-		checkClose();
-
-		preRead();
-		if (eof) {
+		if (read(singleByteReader) < 0) {
 			return -1;
 		}
 
-		return buffer[start++];
+		return singleByteReader[0];
 	}
 
 	@Override
@@ -309,6 +307,10 @@ public class BufferedInputStream extends InputStream {
 		}
 
 		// Read from the pushed-back buffers if any
+		if (backBuffers.isEmpty()) {
+			preRead(); // an implementation could pushback in preRead()
+		}
+		
 		if (!backBuffers.isEmpty()) {
 			int read = 0;
 
@@ -317,7 +319,7 @@ public class BufferedInputStream extends InputStream {
 			byte[] bbBuffer = bb.getKey();
 			int bbOffset = bb.getValue();
 			int bbSize = bbBuffer.length - bbOffset;
-
+			
 			if (bbSize > blen) {
 				read = blen;
 				System.arraycopy(bbBuffer, bbOffset, b, boff, read);
@@ -402,11 +404,11 @@ public class BufferedInputStream extends InputStream {
 	 * <p>
 	 * Including the under-laying {@link InputStream}.
 	 * <p>
-	 * <b>Note:</b> if you called the {@link BufferedInputStream2#open()} method
+	 * <b>Note:</b> if you called the {@link BufferedInputStream#open()} method
 	 * prior to this one, it will just decrease the internal count of how many
 	 * open streams it held and do nothing else. The stream will actually be
-	 * closed when you have called {@link BufferedInputStream2#close()} once
-	 * more than {@link BufferedInputStream2#open()}.
+	 * closed when you have called {@link BufferedInputStream#close()} once more
+	 * than {@link BufferedInputStream#open()}.
 	 * 
 	 * @exception IOException
 	 *                in case of I/O error
@@ -426,11 +428,11 @@ public class BufferedInputStream extends InputStream {
 	 * You can call this method multiple times, it will not cause an
 	 * {@link IOException} for subsequent calls.
 	 * <p>
-	 * <b>Note:</b> if you called the {@link BufferedInputStream2#open()} method
+	 * <b>Note:</b> if you called the {@link BufferedInputStream#open()} method
 	 * prior to this one, it will just decrease the internal count of how many
 	 * open streams it held and do nothing else. The stream will actually be
-	 * closed when you have called {@link BufferedInputStream2#close()} once
-	 * more than {@link BufferedInputStream2#open()}.
+	 * closed when you have called {@link BufferedInputStream#close()} once more
+	 * than {@link BufferedInputStream#open()}.
 	 * 
 	 * @param includingSubStream
 	 *            also close the under-laying stream
@@ -511,7 +513,7 @@ public class BufferedInputStream extends InputStream {
 		boolean hasRead = false;
 		if (in != null && !eof && start >= stop) {
 			start = 0;
-			stop = read(in, buffer, 0, buffer.length);
+			stop = read(in, buffer);
 			if (stop > 0) {
 				bytesRead += stop;
 			}
@@ -539,26 +541,43 @@ public class BufferedInputStream extends InputStream {
 				new AbstractMap.SimpleEntry<byte[], Integer>(buffer, offset));
 	}
 
+	/**
+	 * Push back some data that will be read again at the next read call.
+	 * 
+	 * @param buffer
+	 *            the buffer to push back
+	 * @param offset
+	 *            the offset at which to start reading in the buffer
+	 * @param len
+	 *            the length to copy
+	 */
+	protected void pushback(byte[] buffer, int offset, int len) {
+		// TODO: not efficient!
+		if (buffer.length != len) {
+			byte[] lenNotSupportedYet = new byte[len];
+			System.arraycopy(buffer, offset, lenNotSupportedYet, 0, len);
+			buffer = lenNotSupportedYet;
+			offset = 0;
+		}
+
+		pushback(buffer, offset);
+	}
+
 	/**
 	 * Read the under-laying stream into the given local buffer.
 	 * 
 	 * @param in
 	 *            the under-laying {@link InputStream}
 	 * @param buffer
-	 *            the buffer we use in this {@link BufferedInputStream2}
-	 * @param off
-	 *            the offset
-	 * @param len
-	 *            the length in bytes
+	 *            the buffer we use in this {@link BufferedInputStream}
 	 * 
 	 * @return the number of bytes read
 	 * 
 	 * @throws IOException
 	 *             in case of I/O error
 	 */
-	protected int read(InputStream in, byte[] buffer, int off, int len)
-			throws IOException {
-		return in.read(buffer, off, len);
+	protected int read(InputStream in, byte[] buffer) throws IOException {
+		return in.read(buffer, 0, buffer.length);
 	}
 
 	/**
diff --git a/src/be/nikiroo/utils/streams/ReplaceInputStream.java b/src/be/nikiroo/utils/streams/ReplaceInputStream.java
index ae576e2..0860f78 100644
--- a/src/be/nikiroo/utils/streams/ReplaceInputStream.java
+++ b/src/be/nikiroo/utils/streams/ReplaceInputStream.java
@@ -2,6 +2,8 @@ package be.nikiroo.utils.streams;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
 
 import be.nikiroo.utils.StringUtils;
 
@@ -23,12 +25,8 @@ public class ReplaceInputStream extends BufferedInputStream {
 
 	private byte[][] froms;
 	private byte[][] tos;
+	private int bufferSize;
 	private int maxFromSize;
-	private int maxToSize;
-
-	private byte[] source;
-	private int spos;
-	private int slen;
 
 	/**
 	 * Create a {@link ReplaceInputStream} that will replace <tt>from</tt> with
@@ -108,56 +106,112 @@ public class ReplaceInputStream extends BufferedInputStream {
 			maxFromSize = Math.max(maxFromSize, froms[i].length);
 		}
 
-		maxToSize = 0;
+		int maxToSize = 0;
 		for (int i = 0; i < tos.length; i++) {
 			maxToSize = Math.max(maxToSize, tos[i].length);
 		}
 
 		// We need at least maxFromSize so we can iterate and replace
-		source = new byte[Math.max(2 * Math.max(maxToSize, maxFromSize),
-				MIN_BUFFER_SIZE)];
-		spos = 0;
-		slen = 0;
+		bufferSize = Math.max(4 * Math.max(maxToSize, maxFromSize),
+				MIN_BUFFER_SIZE);
 	}
 
 	@Override
-	protected int read(InputStream in, byte[] buffer, int off, int len)
-			throws IOException {
-		if (len < maxToSize || source.length < maxToSize * 2) {
-			throw new IOException(
-					"An underlaying buffer is too small for these replace values");
-		}
+	protected boolean preRead() throws IOException {
+		boolean rep = super.preRead();
+		start = stop;
+		return rep;
+	}
 
-		// We need at least one byte of data to process
-		if (available() < Math.max(maxFromSize, 1) && !eof) {
-			spos = 0;
-			slen = in.read(source);
+	@Override
+	protected int read(InputStream in, byte[] buffer) throws IOException {
+		buffer = null; // do not use the buffer.
+
+		byte[] newBuffer = new byte[bufferSize];
+		int read = 0;
+		while (read < bufferSize / 2) {
+			int thisTime = in.read(newBuffer, read, bufferSize / 2 - read);
+			if (thisTime <= 0) {
+				break;
+			}
+			read += thisTime;
 		}
 
-		// Note: very simple, not efficient implementation; sorry.
-		int count = 0;
-		while (spos < slen && count < len - maxToSize) {
-			boolean replaced = false;
-			for (int i = 0; i < froms.length; i++) {
-				if (froms[i] != null && froms[i].length > 0
-						&& StreamUtils.startsWith(froms[i], source, spos, slen)) {
-					if (tos[i] != null && tos[i].length > 0) {
-						System.arraycopy(tos[i], 0, buffer, off + count,
-								tos[i].length);
-						count += tos[i].length;
+		List<byte[]> bbBuffers = new ArrayList<byte[]>();
+		List<Integer> bbOffsets = new ArrayList<Integer>();
+		List<Integer> bbLengths = new ArrayList<Integer>();
+
+		int offset = 0;
+		for (int i = 0; i < read; i++) {
+			for (int fromIndex = 0; fromIndex < froms.length; fromIndex++) {
+				byte[] from = froms[fromIndex];
+				byte[] to = tos[fromIndex];
+
+				if (from.length > 0
+						&& StreamUtils.startsWith(from, newBuffer, i, read)) {
+					if (i - offset > 0) {
+						bbBuffers.add(newBuffer);
+						bbOffsets.add(offset);
+						bbLengths.add(i - offset);
 					}
-					
-					spos += froms[i].length;
-					replaced = true;
-					break;
+
+					if (to.length > 0) {
+						bbBuffers.add(to);
+						bbOffsets.add(0);
+						bbLengths.add(to.length);
+					}
+
+					i += from.length;
+					offset = i;
 				}
 			}
+		}
+
+		if (offset < read) {
+			bbBuffers.add(newBuffer);
+			bbOffsets.add(offset);
+			bbLengths.add(read - offset);
+		}
 
-			if (!replaced) {
-				buffer[off + count++] = source[spos++];
+		for (int i = bbBuffers.size() - 1; i >= 0; i--) {
+			// DEBUG("pushback", bbBuffers.get(i), bbOffsets.get(i),
+			// bbLengths.get(i));
+			pushback(bbBuffers.get(i), bbOffsets.get(i), bbLengths.get(i));
+		}
+
+		return read;
+	}
+
+	// static public void DEBUG(String title, byte[] b, int off, int len) {
+	// String str = new String(b,off,len);
+	// if(str.length()>20) {
+	// str=str.substring(0,10)+" ...
+	// "+str.substring(str.length()-10,str.length());
+	// }
+	// }
+
+	@Override
+	public String toString() {
+		StringBuilder rep = new StringBuilder();
+		rep.append(getClass().getSimpleName()).append("\n");
+
+		for (int i = 0; i < froms.length; i++) {
+			byte[] from = froms[i];
+			byte[] to = tos[i];
+
+			rep.append("\t");
+			rep.append("bytes[").append(from.length).append("]");
+			if (from.length <= 20) {
+				rep.append(" (").append(new String(from)).append(")");
+			}
+			rep.append(" -> ");
+			rep.append("bytes[").append(to.length).append("]");
+			if (to.length <= 20) {
+				rep.append(" (").append(new String(to)).append(")");
 			}
+			rep.append("\n");
 		}
 
-		return count;
+		return "[" + rep + "]";
 	}
 }
diff --git a/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java b/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java
index d08a91e..efab8c7 100644
--- a/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java
+++ b/src/be/nikiroo/utils/test_code/ReplaceInputStreamTest.java
@@ -83,33 +83,122 @@ class ReplaceInputStreamTest extends TestLauncher {
 
 				checkArrays(this, "FIRST", in, "I like blue".getBytes("UTF-8"));
 
-				data = "I like blue".getBytes("UTF-8");
+				data = "I like blue hammers".getBytes("UTF-8");
 				in = new ReplaceInputStream(new ByteArrayInputStream(data),
 						"blue", "red");
 
-				checkArrays(this, "FIRST", in, "I like red".getBytes("UTF-8"));
+				checkArrays(this, "SECOND", in, "I like red hammers".getBytes("UTF-8"));
 			}
 		});
 		
 		addTest(new TestCase("Multiple replaces") {
 			@Override
 			public void test() throws Exception {
-				byte[] data = "I like red".getBytes("UTF-8");
+				byte[] data = "I like red cabage".getBytes("UTF-8");
 				ReplaceInputStream in = new ReplaceInputStream(
 						new ByteArrayInputStream(data), //
-						new String[] {"like", "red"}, //
-						new String[] {"dislike", "green"} //
+						new String[] { "red", "like" }, //
+						new String[] { "green", "very very much like" } //
+				);
+				
+				String result = new String(IOUtils.toByteArray(in), "UTF-8");
+				assertEquals("I very very much like green cabage", result);
+			}
+		});
+		
+		addTest(new TestCase("Multiple replaces") {
+			@Override
+			public void test() throws Exception {
+				String str= ("" //
+						+ "<!DOCTYPE html>\n" //
+						+ "<html>\n" //
+						+ "<head>\n" //
+						+ "<!--\n" //
+						+ "\tCopyright 2020 David ROULET\n" //
+						+ "\t\n" //
+						+ "\tThis file is part of fanfix.\n" //
+						+ "\t\n" //
+						+ "\tfanfix is free software: you can redistribute it and/or modify\n" //
+						+ "\tit under the terms of the GNU Affero General Public License as published by\n" //
+						+ "\tthe Free Software Foundation, either version 3 of the License, or\n" //
+						+ "\t(at your option) any later version.\n" //
+						+ "\t\n" //
+						+ "\tfanfix is distributed in the hope that it will be useful,\n" //
+						+ "\tbut WITHOUT ANY WARRANTY; without even the implied warranty of\n" //
+						+ "\tMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n" //
+						+ "\tGNU Affero General Public License for more details.\n" //
+						+ "\t\n" //
+						+ "\tYou should have received a copy of the GNU Affero General Public License\n" //
+						+ "\talong with fanfix.  If not, see <https://www.gnu.org/licenses/>.\n" //
+						+ "\t___________________________________________________________________________\n" //
+						+ "\n" //
+						+ "       This website was coded by:\n" //
+						+ "       \t\tA kangaroo.\n" //
+						+ "                                                  _  _\n" //
+						+ "                                                 (\\\\( \\\n" //
+						+ "                                                  `.\\-.)\n" //
+						+ "                              _...._            _,-'   `-.\n" //
+						+ "\\                           ,'      `-._.- -.,-'       .  \\\n" //
+						+ " \\`.                      ,'                               `.\n" //
+						+ "  \\ `-...__              /                           .   .:  y\n" //
+						+ "   `._     ``-...__     /                           ,'```-._/\n" //
+						+ "      `-._         ```-'                      |    /_          //\n" //
+						+ "          `.._                   _            ;   <_ \\        //\n" //
+						+ "              ``-.___             `.           `-._ \\ \\      //\n" //
+						+ "                     `- <           `.     (\\ _/)/ `.\\/     //\n" //
+						+ "                         \\            \\     `       ^^^^^^^^^\n" //
+						+ "\t___________________________________________________________________________\n" //
+						+ "\t\n" //
+						+ "-->\n" //
+						+ "\t<meta http-equiv='content-type' content='text/html; charset=UTF-8'>\n" //
+						+ "\t<meta name='viewport' content='width=device-width, initial-scale=1.0'>\n" //
+						+ "\t<title>${title}</title>\n" //
+						+ "\t<link rel='stylesheet' type='text/css' href='/style.css' />\n" //
+						+ "\t<link rel='icon' type='image/x-icon' href='/${favicon}' />\n" //
+						+ "</head>\n" //
+						+ "<body>\n" //
+						+ "\t<div class='main'>\n" //
+						+ "${banner}${content}\t</div>\n" //
+						+ "</body>\n" //
+						+ "" //
+				);
+				byte[] data = str.getBytes("UTF-8");
+
+				String title = "Fanfix";
+				String banner = "<div class='banner'>Super banner v3</div>";
+				String content = "";
+
+				InputStream in = new ReplaceInputStream(
+						new ByteArrayInputStream(data), //
+						new String[] { "${title}", "${banner}", "${content}" }, //
+						new String[] { title, banner, content } //
 				);
 
 				String result = new String(IOUtils.toByteArray(in), "UTF-8");
-				assertEquals("I dislike green", result);
+				assertEquals(str //
+						.replace("${title}", title) //
+						.replace("${banner}", banner) //
+						.replace("${content}", content) //
+				, result);
 			}
 		});
+		
+		
 	}
 
 	static void checkArrays(TestCase test, String prefix, InputStream in,
 			byte[] expected) throws Exception {
 		byte[] actual = IOUtils.toByteArray(in);
+		
+//		System.out.println("\nActual:");
+//		for(byte byt : actual) {
+//			System.out.print(byt+" ");
+//		}
+//		System.out.println("\nExpected:");
+//		for(byte byt : expected) {
+//			System.out.print(byt+" ");
+//		}
+		
 		test.assertEquals("The " + prefix
 				+ " resulting array has not the correct number of items",
 				expected.length, actual.length);