From: Niki Roo Date: Wed, 20 May 2020 14:41:05 +0000 (+0200) Subject: Merge commit '53c2b6a134b08402e1daf3e4c84b9b888de9cc9c' X-Git-Url: http://git.nikiroo.be/?p=fanfix.git;a=commitdiff_plain;h=7665ac5903eff6055e601dc0ad7540032d2f8a14;hp=90eaf7c6209fc6ad21a9f9a37bc301e6078c930e Merge commit '53c2b6a134b08402e1daf3e4c84b9b888de9cc9c' --- diff --git a/streams/BufferedInputStream.java b/streams/BufferedInputStream.java index 683fa55..3cad70d 100644 --- a/streams/BufferedInputStream.java +++ b/streams/BufferedInputStream.java @@ -2,7 +2,10 @@ package be.nikiroo.utils.streams; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; import be.nikiroo.utils.StringUtils; @@ -18,10 +21,11 @@ import be.nikiroo.utils.StringUtils; public class BufferedInputStream extends InputStream { /** * The size of the internal buffer (can be different if you pass your own - * buffer, of course). + * buffer, of course, and can also expand to search for longer "startsWith" + * data). *

- * A second buffer of twice the size can sometimes be created as needed for - * the {@link BufferedInputStream#startsWith(byte[])} search operation. + * Note that special "push-back" buffers can also be created during the life + * of this stream. */ static private final int BUFFER_SIZE = 4096; @@ -37,12 +41,10 @@ public class BufferedInputStream extends InputStream { private boolean closed; private InputStream in; private int openCounter; + private byte[] singleByteReader = new byte[1]; - // special use, prefetched next buffer - private byte[] buffer2; - private int pos2; - private int len2; - private byte[] originalBuffer; + /** array + offset of pushed-back buffers */ + private List> backBuffers; private long bytesRead; @@ -57,9 +59,9 @@ public class BufferedInputStream extends InputStream { this.in = in; this.buffer = new byte[BUFFER_SIZE]; - this.originalBuffer = this.buffer; this.start = 0; this.stop = 0; + this.backBuffers = new ArrayList>(); } /** @@ -100,18 +102,9 @@ public class BufferedInputStream extends InputStream { this.in = null; this.buffer = in; - this.originalBuffer = this.buffer; this.start = offset; this.stop = length; - } - - /** - * The internal buffer size (can be useful to know for search methods). - * - * @return the size of the internal buffer, in bytes. - */ - public int getInternalBufferSize() { - return originalBuffer.length; + this.backBuffers = new ArrayList>(); } /** @@ -174,7 +167,7 @@ public class BufferedInputStream extends InputStream { */ public boolean is(byte[] search) throws IOException { if (startsWith(search)) { - return (stop - start) == search.length; + return available() == search.length; } return false; @@ -220,40 +213,27 @@ public class BufferedInputStream extends InputStream { * greater than the internal buffer */ public boolean startsWith(byte[] search) throws IOException { - if (search.length > originalBuffer.length) { - throw new IOException( - "This stream does not support searching for more than " - + buffer.length + " bytes"); - } - checkClose(); - if (available() < search.length) { + while (consolidatePushBack(search.length) < search.length) { preRead(); - } - - if (available() >= search.length) { - // Easy path - return StreamUtils.startsWith(search, buffer, start, stop); - } else if (in != null && !eof) { - // Harder path - if (buffer2 == null && buffer.length == originalBuffer.length) { - buffer2 = Arrays.copyOf(buffer, buffer.length * 2); - - pos2 = buffer.length; - len2 = read(in, buffer2, pos2, buffer.length); - if (len2 > 0) { - bytesRead += len2; - } - - // Note: here, len/len2 = INDEX of last good byte - len2 += pos2; + if (start >= stop) { + // Not enough data left to start with that + return false; } - return StreamUtils.startsWith(search, buffer2, pos2, len2); + byte[] newBuffer = new byte[stop - start]; + System.arraycopy(buffer, start, newBuffer, 0, stop - start); + pushback(newBuffer, 0); + start = stop; } - return false; + Entry bb = backBuffers.get(backBuffers.size() - 1); + byte[] bbBuffer = bb.getKey(); + int bbOffset = bb.getValue(); + + return StreamUtils.startsWith(search, bbBuffer, bbOffset, + bbBuffer.length); } /** @@ -266,8 +246,7 @@ public class BufferedInputStream extends InputStream { } /** - * Check if this stream is spent (no more data to read or to - * process). + * Check if this stream is spent (no more data to read or to process). * * @return TRUE if it is * @@ -303,14 +282,11 @@ public class BufferedInputStream extends InputStream { @Override public int read() throws IOException { - checkClose(); - - preRead(); - if (eof) { + if (read(singleByteReader) < 0) { return -1; } - return buffer[start++]; + return singleByteReader[0]; } @Override @@ -330,6 +306,32 @@ public class BufferedInputStream extends InputStream { return 0; } + // Read from the pushed-back buffers if any + if (backBuffers.isEmpty()) { + preRead(); // an implementation could pushback in preRead() + } + + if (!backBuffers.isEmpty()) { + int read = 0; + + Entry bb = backBuffers + .remove(backBuffers.size() - 1); + byte[] bbBuffer = bb.getKey(); + int bbOffset = bb.getValue(); + int bbSize = bbBuffer.length - bbOffset; + + if (bbSize > blen) { + read = blen; + System.arraycopy(bbBuffer, bbOffset, b, boff, read); + pushback(bbBuffer, bbOffset + read); + } else { + read = bbSize; + System.arraycopy(bbBuffer, bbOffset, b, boff, read); + } + + return read; + } + int done = 0; while (hasMoreData() && done < blen) { preRead(); @@ -353,6 +355,23 @@ public class BufferedInputStream extends InputStream { } long skipped = 0; + while (!backBuffers.isEmpty() && n > 0) { + Entry bb = backBuffers + .remove(backBuffers.size() - 1); + byte[] bbBuffer = bb.getKey(); + int bbOffset = bb.getValue(); + int bbSize = bbBuffer.length - bbOffset; + + int localSkip = 0; + localSkip = (int) Math.min(n, bbSize); + + n -= localSkip; + bbSize -= localSkip; + + if (bbSize > 0) { + pushback(bbBuffer, bbOffset + localSkip); + } + } while (hasMoreData() && n > 0) { preRead(); @@ -371,7 +390,12 @@ public class BufferedInputStream extends InputStream { return 0; } - return Math.max(0, stop - start); + int avail = 0; + for (Entry entry : backBuffers) { + avail += entry.getKey().length - entry.getValue(); + } + + return avail + Math.max(0, stop - start); } /** @@ -430,6 +454,52 @@ public class BufferedInputStream extends InputStream { } } + /** + * Consolidate the push-back buffers so the last one is at least the given + * size, if possible. + *

+ * If there is not enough data in the push-back buffers, they will all be + * consolidated. + * + * @param size + * the minimum size of the consolidated buffer, or -1 to force + * the consolidation of all push-back buffers + * + * @return the size of the last, consolidated buffer; can be less than the + * requested size if not enough data + */ + protected int consolidatePushBack(int size) { + int bbIndex = -1; + int bbUpToSize = 0; + for (Entry entry : backBuffers) { + bbIndex++; + bbUpToSize += entry.getKey().length - entry.getValue(); + + if (size >= 0 && bbUpToSize >= size) { + break; + } + } + + // Index 0 means "the last buffer is already big enough" + if (bbIndex > 0) { + byte[] consolidatedBuffer = new byte[bbUpToSize]; + int consolidatedPos = 0; + for (int i = 0; i <= bbIndex; i++) { + Entry bb = backBuffers + .remove(backBuffers.size() - 1); + byte[] bbBuffer = bb.getKey(); + int bbOffset = bb.getValue(); + int bbSize = bbBuffer.length - bbOffset; + System.arraycopy(bbBuffer, bbOffset, consolidatedBuffer, + consolidatedPos, bbSize); + } + + pushback(consolidatedBuffer, 0); + } + + return bbUpToSize; + } + /** * Check if we still have some data in the buffer and, if not, fetch some. * @@ -443,21 +513,9 @@ public class BufferedInputStream extends InputStream { boolean hasRead = false; if (in != null && !eof && start >= stop) { start = 0; - if (buffer2 != null) { - buffer = buffer2; - start = pos2; - stop = len2; - - buffer2 = null; - pos2 = 0; - len2 = 0; - } else { - buffer = originalBuffer; - - stop = read(in, buffer, 0, buffer.length); - if (stop > 0) { - bytesRead += stop; - } + stop = read(in, buffer); + if (stop > 0) { + bytesRead += stop; } hasRead = true; @@ -471,25 +529,55 @@ public class BufferedInputStream extends InputStream { } /** - * Read the under-laying stream into the local buffer. + * Push back some data that will be read again at the next read call. + * + * @param buffer + * the buffer to push back + * @param offset + * the offset at which to start reading in the buffer + */ + protected void pushback(byte[] buffer, int offset) { + backBuffers.add( + new AbstractMap.SimpleEntry(buffer, offset)); + } + + /** + * Push back some data that will be read again at the next read call. + * + * @param buffer + * the buffer to push back + * @param offset + * the offset at which to start reading in the buffer + * @param len + * the length to copy + */ + protected void pushback(byte[] buffer, int offset, int len) { + // TODO: not efficient! + if (buffer.length != len) { + byte[] lenNotSupportedYet = new byte[len]; + System.arraycopy(buffer, offset, lenNotSupportedYet, 0, len); + buffer = lenNotSupportedYet; + offset = 0; + } + + pushback(buffer, offset); + } + + /** + * Read the under-laying stream into the given local buffer. * * @param in * the under-laying {@link InputStream} * @param buffer * the buffer we use in this {@link BufferedInputStream} - * @param off - * the offset - * @param len - * the length in bytes * * @return the number of bytes read * * @throws IOException * in case of I/O error */ - protected int read(InputStream in, byte[] buffer, int off, int len) - throws IOException { - return in.read(buffer, off, len); + protected int read(InputStream in, byte[] buffer) throws IOException { + return in.read(buffer, 0, buffer.length); } /** @@ -503,7 +591,7 @@ public class BufferedInputStream extends InputStream { return false; } - return (start < stop) || !eof; + return !backBuffers.isEmpty() || (start < stop) || !eof; } /** diff --git a/streams/ReplaceInputStream.java b/streams/ReplaceInputStream.java index 1cc5139..0860f78 100644 --- a/streams/ReplaceInputStream.java +++ b/streams/ReplaceInputStream.java @@ -2,6 +2,8 @@ package be.nikiroo.utils.streams; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; import be.nikiroo.utils.StringUtils; @@ -23,12 +25,8 @@ public class ReplaceInputStream extends BufferedInputStream { private byte[][] froms; private byte[][] tos; + private int bufferSize; private int maxFromSize; - private int maxToSize; - - private byte[] source; - private int spos; - private int slen; /** * Create a {@link ReplaceInputStream} that will replace from with @@ -108,55 +106,112 @@ public class ReplaceInputStream extends BufferedInputStream { maxFromSize = Math.max(maxFromSize, froms[i].length); } - maxToSize = 0; + int maxToSize = 0; for (int i = 0; i < tos.length; i++) { maxToSize = Math.max(maxToSize, tos[i].length); } // We need at least maxFromSize so we can iterate and replace - source = new byte[Math.max(2 * maxFromSize, MIN_BUFFER_SIZE)]; - spos = 0; - slen = 0; + bufferSize = Math.max(4 * Math.max(maxToSize, maxFromSize), + MIN_BUFFER_SIZE); } @Override - protected int read(InputStream in, byte[] buffer, int off, int len) - throws IOException { - if (len < maxToSize || source.length < maxToSize * 2) { - throw new IOException( - "An underlaying buffer is too small for these replace values"); - } + protected boolean preRead() throws IOException { + boolean rep = super.preRead(); + start = stop; + return rep; + } - // We need at least one byte of data to process - if (available() < Math.max(maxFromSize, 1) && !eof) { - spos = 0; - slen = in.read(source); + @Override + protected int read(InputStream in, byte[] buffer) throws IOException { + buffer = null; // do not use the buffer. + + byte[] newBuffer = new byte[bufferSize]; + int read = 0; + while (read < bufferSize / 2) { + int thisTime = in.read(newBuffer, read, bufferSize / 2 - read); + if (thisTime <= 0) { + break; + } + read += thisTime; } - // Note: very simple, not efficient implementation; sorry. - int count = 0; - while (spos < slen && count < len - maxToSize) { - boolean replaced = false; - for (int i = 0; i < froms.length; i++) { - if (froms[i] != null && froms[i].length > 0 - && StreamUtils.startsWith(froms[i], source, spos, slen)) { - if (tos[i] != null && tos[i].length > 0) { - System.arraycopy(tos[i], 0, buffer, off + spos, - tos[i].length); - count += tos[i].length; + List bbBuffers = new ArrayList(); + List bbOffsets = new ArrayList(); + List bbLengths = new ArrayList(); + + int offset = 0; + for (int i = 0; i < read; i++) { + for (int fromIndex = 0; fromIndex < froms.length; fromIndex++) { + byte[] from = froms[fromIndex]; + byte[] to = tos[fromIndex]; + + if (from.length > 0 + && StreamUtils.startsWith(from, newBuffer, i, read)) { + if (i - offset > 0) { + bbBuffers.add(newBuffer); + bbOffsets.add(offset); + bbLengths.add(i - offset); } - spos += froms[i].length; - replaced = true; - break; + if (to.length > 0) { + bbBuffers.add(to); + bbOffsets.add(0); + bbLengths.add(to.length); + } + + i += from.length; + offset = i; } } + } + + if (offset < read) { + bbBuffers.add(newBuffer); + bbOffsets.add(offset); + bbLengths.add(read - offset); + } - if (!replaced) { - buffer[off + count++] = source[spos++]; + for (int i = bbBuffers.size() - 1; i >= 0; i--) { + // DEBUG("pushback", bbBuffers.get(i), bbOffsets.get(i), + // bbLengths.get(i)); + pushback(bbBuffers.get(i), bbOffsets.get(i), bbLengths.get(i)); + } + + return read; + } + + // static public void DEBUG(String title, byte[] b, int off, int len) { + // String str = new String(b,off,len); + // if(str.length()>20) { + // str=str.substring(0,10)+" ... + // "+str.substring(str.length()-10,str.length()); + // } + // } + + @Override + public String toString() { + StringBuilder rep = new StringBuilder(); + rep.append(getClass().getSimpleName()).append("\n"); + + for (int i = 0; i < froms.length; i++) { + byte[] from = froms[i]; + byte[] to = tos[i]; + + rep.append("\t"); + rep.append("bytes[").append(from.length).append("]"); + if (from.length <= 20) { + rep.append(" (").append(new String(from)).append(")"); + } + rep.append(" -> "); + rep.append("bytes[").append(to.length).append("]"); + if (to.length <= 20) { + rep.append(" (").append(new String(to)).append(")"); } + rep.append("\n"); } - return count; + return "[" + rep + "]"; } } diff --git a/test_code/NextableInputStreamTest.java b/test_code/NextableInputStreamTest.java index 463a123..4e59823 100644 --- a/test_code/NextableInputStreamTest.java +++ b/test_code/NextableInputStreamTest.java @@ -1,7 +1,6 @@ package be.nikiroo.utils.test_code; import java.io.ByteArrayInputStream; -import java.io.IOException; import be.nikiroo.utils.IOUtils; import be.nikiroo.utils.streams.NextableInputStream; @@ -177,12 +176,10 @@ public class NextableInputStreamTest extends TestLauncher { 11 })); // too big - try { - in.startsWith(new byte[] { 42, 12, 0, 127, 12, 51, 11, 12, - 0 }); - fail("Searching a prefix bigger than the array should throw an IOException"); - } catch (IOException e) { - } + assertEquals( + "A search term bigger than the whole data cannot be found in the data", + false, in.startsWith(new byte[] { 42, 12, 0, 127, 12, + 51, 11, 12, 0 })); in.close(); } @@ -209,13 +206,11 @@ public class NextableInputStreamTest extends TestLauncher { in.startsWith("Toto")); assertEquals("It actually does not start with that", false, in.startsWith("Fanfan et Toto vont à la mee")); - + // too big - try { - in.startsWith("Fanfan et Toto vont à la mer."); - fail("Searching a prefix bigger than the array should throw an IOException"); - } catch (IOException e) { - } + assertEquals( + "A search term bigger than the whole data cannot be found in the data", + false, in.startsWith("Fanfan et Toto vont à la mer.")); in.close(); } diff --git a/test_code/ReplaceInputStreamTest.java b/test_code/ReplaceInputStreamTest.java index e6e2112..efab8c7 100644 --- a/test_code/ReplaceInputStreamTest.java +++ b/test_code/ReplaceInputStreamTest.java @@ -48,7 +48,7 @@ class ReplaceInputStreamTest extends TestLauncher { } }); - addTest(new TestCase("Lnger replace") { + addTest(new TestCase("Longer replace") { @Override public void test() throws Exception { byte[] data = new byte[] { 42, 12, 0, 127 }; @@ -79,22 +79,126 @@ class ReplaceInputStreamTest extends TestLauncher { byte[] data = "I like red".getBytes("UTF-8"); ReplaceInputStream in = new ReplaceInputStream( new ByteArrayInputStream(data), - "red".getBytes("UTF-8"), "blue".getBytes("UTF-8")); + "red", "blue"); checkArrays(this, "FIRST", in, "I like blue".getBytes("UTF-8")); - data = "I like blue".getBytes("UTF-8"); + data = "I like blue hammers".getBytes("UTF-8"); in = new ReplaceInputStream(new ByteArrayInputStream(data), - "blue".getBytes("UTF-8"), "red".getBytes("UTF-8")); + "blue", "red"); - checkArrays(this, "FIRST", in, "I like red".getBytes("UTF-8")); + checkArrays(this, "SECOND", in, "I like red hammers".getBytes("UTF-8")); } }); + + addTest(new TestCase("Multiple replaces") { + @Override + public void test() throws Exception { + byte[] data = "I like red cabage".getBytes("UTF-8"); + ReplaceInputStream in = new ReplaceInputStream( + new ByteArrayInputStream(data), // + new String[] { "red", "like" }, // + new String[] { "green", "very very much like" } // + ); + + String result = new String(IOUtils.toByteArray(in), "UTF-8"); + assertEquals("I very very much like green cabage", result); + } + }); + + addTest(new TestCase("Multiple replaces") { + @Override + public void test() throws Exception { + String str= ("" // + + "\n" // + + "\n" // + + "\n" // + + "\n" // + + "\t\n" // + + "\t\n" // + + "\t${title}\n" // + + "\t\n" // + + "\t\n" // + + "\n" // + + "\n" // + + "\t

\n" // + + "${banner}${content}\t
\n" // + + "\n" // + + "" // + ); + byte[] data = str.getBytes("UTF-8"); + + String title = "Fanfix"; + String banner = ""; + String content = ""; + + InputStream in = new ReplaceInputStream( + new ByteArrayInputStream(data), // + new String[] { "${title}", "${banner}", "${content}" }, // + new String[] { title, banner, content } // + ); + + String result = new String(IOUtils.toByteArray(in), "UTF-8"); + assertEquals(str // + .replace("${title}", title) // + .replace("${banner}", banner) // + .replace("${content}", content) // + , result); + } + }); + + } static void checkArrays(TestCase test, String prefix, InputStream in, byte[] expected) throws Exception { byte[] actual = IOUtils.toByteArray(in); + +// System.out.println("\nActual:"); +// for(byte byt : actual) { +// System.out.print(byt+" "); +// } +// System.out.println("\nExpected:"); +// for(byte byt : expected) { +// System.out.print(byt+" "); +// } + test.assertEquals("The " + prefix + " resulting array has not the correct number of items", expected.length, actual.length); diff --git a/ui/UIUtils.java b/ui/UIUtils.java index 6c40389..e4eb000 100644 --- a/ui/UIUtils.java +++ b/ui/UIUtils.java @@ -22,7 +22,6 @@ import javax.swing.UnsupportedLookAndFeelException; import javax.swing.event.HyperlinkEvent; import javax.swing.event.HyperlinkListener; -import be.nikiroo.fanfix.Instance; import be.nikiroo.utils.Version; import be.nikiroo.utils.VersionCheck; @@ -306,9 +305,9 @@ public class UIUtils { try { Desktop.getDesktop().browse(e.getURL().toURI()); } catch (IOException ee) { - Instance.getInstance().getTraceHandler().error(ee); + ee.printStackTrace(); } catch (URISyntaxException ee) { - Instance.getInstance().getTraceHandler().error(ee); + ee.printStackTrace(); } } });