From: Niki Roo Date: Sat, 27 Apr 2019 17:32:25 +0000 (+0200) Subject: Merge branch 'master' into streamify X-Git-Url: http://git.nikiroo.be/?p=nikiroo-utils.git;a=commitdiff_plain;h=67b76a57e0789b4c2afb0091b9d5bcd120290ed1;hp=627f866e6cd474bd7494750dc3846c5ad898f2ec Merge branch 'master' into streamify --- diff --git a/src/be/nikiroo/utils/serial/CustomSerializer.java b/src/be/nikiroo/utils/serial/CustomSerializer.java index be89316..36a6c21 100644 --- a/src/be/nikiroo/utils/serial/CustomSerializer.java +++ b/src/be/nikiroo/utils/serial/CustomSerializer.java @@ -1,39 +1,85 @@ package be.nikiroo.utils.serial; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.streams.NextableInputStream; +import be.nikiroo.utils.streams.NextableInputStreamStep; public abstract class CustomSerializer { - protected abstract String toString(Object value); + protected abstract void toStream(OutputStream out, Object value) + throws IOException; - protected abstract Object fromString(String content) throws IOException; + protected abstract Object fromStream(InputStream in) throws IOException; protected abstract String getType(); /** - * Encode the object into the given builder if possible (if supported). + * Encode the object into the given {@link OutputStream} if supported. * - * @param builder + * @param out * the builder to append to * @param value * the object to encode - * @return TRUE if success, FALSE if not (the content of the builder won't - * be changed in case of failure) + * + * @return FALSE if the value is not supported, TRUE if the operation was + * successful (if the value is supported by the operation was not + * successful, you will get an {@link IOException}) + * + * @throws IOException + * in case of I/O error */ - public boolean encode(StringBuilder builder, Object value) { - int prev = builder.length(); - String customString = toString(value); - builder.append("custom^").append(getType()).append("^"); - if (!SerialUtils.encode(builder, customString)) { - builder.delete(prev, builder.length()); - return false; - } + public boolean encode(OutputStream out, Object value) throws IOException { + SerialUtils.write(out, "custom^"); + SerialUtils.write(out, getType()); + SerialUtils.write(out, "^"); + // TODO: manage ENTER + toStream(out, value); return true; } - public Object decode(String encodedValue) throws IOException { - return fromString((String) SerialUtils.decode(contentOf(encodedValue))); + public Object decode(InputStream in) throws IOException { + // TODO: manage ENTER + NextableInputStream stream = new NextableInputStream(in, + new NextableInputStreamStep('^')); + + try { + if (!stream.next()) { + throw new IOException("Cannot find the first custom^ element"); + } + + String custom = IOUtils.readSmallStream(stream); + if (!"custom".equals(custom)) { + throw new IOException( + "Cannot find the first custom^ element, it is: " + + custom + "^"); + } + + if (!stream.next()) { + throw new IOException("Cannot find the second custom^" + + getType() + " element"); + } + + String type = IOUtils.readSmallStream(stream); + if (!getType().equals(type)) { + throw new IOException("Cannot find the second custom^" + + getType() + " element, it is: custom^" + type + "^"); + } + + if (!stream.nextAll()) { + throw new IOException("Cannot find the third custom^" + + getType() + "^value element"); + } + + // TODO: manage ENTER + return fromStream(stream); + } finally { + stream.close(false); + } } public static boolean isCustom(String encodedValue) { diff --git a/src/be/nikiroo/utils/serial/Exporter.java b/src/be/nikiroo/utils/serial/Exporter.java index dc96d97..6325484 100644 --- a/src/be/nikiroo/utils/serial/Exporter.java +++ b/src/be/nikiroo/utils/serial/Exporter.java @@ -2,11 +2,10 @@ package be.nikiroo.utils.serial; import java.io.IOException; import java.io.NotSerializableException; +import java.io.OutputStream; import java.util.HashMap; import java.util.Map; -import be.nikiroo.utils.StringUtils; - /** * A simple class to serialise objects to {@link String}. *

@@ -17,14 +16,19 @@ import be.nikiroo.utils.StringUtils; */ public class Exporter { private Map map; - private StringBuilder builder; + private OutputStream out; /** * Create a new {@link Exporter}. */ - public Exporter() { + public Exporter(OutputStream out) { + if (out == null) { + throw new NullPointerException( + "Cannot create an be.nikiroo.utils.serials.Exporter that will export to NULL"); + } + + this.out = out; map = new HashMap(); - builder = new StringBuilder(); } /** @@ -42,98 +46,12 @@ public class Exporter { * if the object cannot be serialised (in this case, the * {@link Exporter} can contain bad, most probably not * importable data) + * @throws IOException + * in case of I/O error */ - public Exporter append(Object o) throws NotSerializableException { - SerialUtils.append(builder, o, map); + public Exporter append(Object o) throws NotSerializableException, + IOException { + SerialUtils.append(out, o, map); return this; } - - /** - * Clear the current content. - */ - public void clear() { - builder.setLength(0); - map.clear(); - } - - /** - * Append the exported items in a serialised form into the given - * {@link StringBuilder}. - * - * @param toBuilder - * the {@link StringBuilder} - * @param b64 - * TRUE to have BASE64-coded content, FALSE to have raw content, - * NULL to let the system decide - * @param zip - * TRUE to zip the BASE64 output if the output is indeed in - * BASE64 format, FALSE not to - */ - public void appendTo(StringBuilder toBuilder, Boolean b64, boolean zip) { - if (b64 == null && builder.length() < 128) { - b64 = false; - } - - if (b64 == null || b64) { - try { - String zipped = StringUtils.base64(builder.toString(), zip); - if (b64 != null || zipped.length() < builder.length() - 4) { - toBuilder.append(zip ? "ZIP:" : "B64:"); - toBuilder.append(zipped); - return; - } - } catch (IOException e) { - throw new RuntimeException( - "Base64 conversion of data failed, maybe not enough memory?", - e); - } - } - - toBuilder.append(builder); - } - - /** - * The exported items in a serialised form. - * - * @deprecated use {@link Exporter#toString(Boolean, boolean)} instead - * - * @param zip - * TRUE to have zipped (and BASE64-coded) content, FALSE to have - * raw content, NULL to let the system decide - * - * @return the items currently in this {@link Exporter} - */ - @Deprecated - public String toString(Boolean zip) { - return toString(zip, zip == null || zip); - } - - /** - * The exported items in a serialised form. - * - * @param b64 - * TRUE to have BASE64-coded content, FALSE to have raw content, - * NULL to let the system decide - * @param zip - * TRUE to zip the BASE64 output if the output is indeed in - * BASE64 format, FALSE not to - * - * @return the items currently in this {@link Exporter} - */ - public String toString(Boolean b64, boolean zip) { - StringBuilder toBuilder = new StringBuilder(); - appendTo(toBuilder, b64, zip); - return toBuilder.toString(); - } - - /** - * The exported items in a serialised form (possibly BASE64-coded, possibly - * zipped). - * - * @return the items currently in this {@link Exporter} - */ - @Override - public String toString() { - return toString(null, true); - } } \ No newline at end of file diff --git a/src/be/nikiroo/utils/serial/Importer.java b/src/be/nikiroo/utils/serial/Importer.java index bca157c..ad2d284 100644 --- a/src/be/nikiroo/utils/serial/Importer.java +++ b/src/be/nikiroo/utils/serial/Importer.java @@ -1,11 +1,14 @@ package be.nikiroo.utils.serial; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.io.InputStream; import java.lang.reflect.Field; import java.util.HashMap; import java.util.Map; +import be.nikiroo.utils.IOUtils; +import be.nikiroo.utils.streams.NextableInputStream; +import be.nikiroo.utils.streams.NextableInputStreamStep; import be.nikiroo.utils.StringUtils; /** @@ -19,9 +22,6 @@ import be.nikiroo.utils.StringUtils; * @author niki */ public class Importer { - static private Integer SIZE_ID = null; - static private byte[] NEWLINE = null; - private Boolean link; private Object me; private Importer child; @@ -29,15 +29,6 @@ public class Importer { private String currentFieldName; - static { - try { - SIZE_ID = "EXT:".getBytes("UTF-8").length; - NEWLINE = "\n".getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - // UTF-8 is mandated to exist on confirming jre's - } - } - /** * Create a new {@link Importer}. */ @@ -70,77 +61,43 @@ public class Importer { * if a class described in the serialised data cannot be found * @throws IOException * if the content cannot be read (for instance, corrupt data) + * @throws NullPointerException + * if the stream is empty */ - public Importer read(String data) throws NoSuchFieldException, - NoSuchMethodException, ClassNotFoundException, IOException { - return read(data.getBytes("UTF-8"), 0); - } + public Importer read(InputStream in) throws NoSuchFieldException, + NoSuchMethodException, ClassNotFoundException, IOException, + NullPointerException { - /** - * Read some data into this {@link Importer}: it can be the full serialised - * content, or a number of lines of it (any given line MUST be - * complete though) and accumulate it with the already present data. - * - * @param data - * the data to parse - * @param offset - * the offset at which to start reading the data (we ignore - * anything that goes before that offset) - * - * @return itself so it can be chained - * - * @throws NoSuchFieldException - * if the serialised data contains information about a field - * which does actually not exist in the class we know of - * @throws NoSuchMethodException - * if a class described in the serialised data cannot be created - * because it is not compatible with this code - * @throws ClassNotFoundException - * if a class described in the serialised data cannot be found - * @throws IOException - * if the content cannot be read (for instance, corrupt data) - */ - private Importer read(byte[] data, int offset) throws NoSuchFieldException, - NoSuchMethodException, ClassNotFoundException, IOException { + NextableInputStream stream = new NextableInputStream(in, + new NextableInputStreamStep('\n')); - int dataStart = offset; - while (dataStart < data.length) { - String id = ""; - if (data.length - dataStart >= SIZE_ID) { - id = new String(data, dataStart, SIZE_ID); + try { + if (in == null) { + throw new NullPointerException("InputStream is null"); } - - boolean zip = id.equals("ZIP:"); - boolean b64 = id.equals("B64:"); - if (zip || b64) { - dataStart += SIZE_ID; + if (stream.eof()) { + throw new NullPointerException("InputStream is empty"); } - int count = find(data, dataStart, NEWLINE); - count -= dataStart; - if (count < 0) { - count = data.length - dataStart; - } + while (stream.next()) { + boolean zip = stream.startsWiths("ZIP:"); + boolean b64 = stream.startsWiths("B64:"); - if (zip || b64) { - boolean unpacked = false; - try { - byte[] line = StringUtils.unbase64(data, dataStart, count, + if (zip || b64) { + stream.skip("XXX:".length()); + InputStream decoded = StringUtils.unbase64(stream.open(), zip); - unpacked = true; - read(line, 0); - } catch (IOException e) { - throw new IOException("Internal error when decoding " - + (unpacked ? "unpacked " : "") - + (zip ? "ZIP" : "B64") - + " content: input may be corrupt"); + try { + read(decoded); + } finally { + decoded.close(); + } + } else { + processLine(stream); } - } else { - String line = new String(data, dataStart, count, "UTF-8"); - processLine(line); } - - dataStart += count + NEWLINE.length; + } finally { + stream.close(false); } return this; @@ -166,11 +123,12 @@ public class Importer { * @throws IOException * if the content cannot be read (for instance, corrupt data) */ - private boolean processLine(String line) throws NoSuchFieldException, + private boolean processLine(InputStream in) throws NoSuchFieldException, NoSuchMethodException, ClassNotFoundException, IOException { + // Defer to latest child if any if (child != null) { - if (child.processLine(line)) { + if (child.processLine(in)) { if (currentFieldName != null) { setField(currentFieldName, child.getValue()); currentFieldName = null; @@ -181,6 +139,9 @@ public class Importer { return false; } + // TODO use the stream, Luke + String line = IOUtils.readSmallStream(in); + if (line.equals("{")) { // START: new child if needed if (link != null) { child = new Importer(map); @@ -255,37 +216,6 @@ public class Importer { } } - /** - * Find the given needle in the data and return its position (or -1 if not - * found). - * - * @param data - * the data to look through - * @param offset - * the offset at wich to start searching - * @param needle - * the needle to find - * - * @return the position of the needle if found, -1 if not found - */ - private int find(byte[] data, int offset, byte[] needle) { - for (int i = offset; i + needle.length - 1 < data.length; i++) { - boolean same = true; - for (int j = 0; j < needle.length; j++) { - if (data[i + j] != needle[j]) { - same = false; - break; - } - } - - if (same) { - return i; - } - } - - return -1; - } - /** * Return the current deserialised value. * diff --git a/src/be/nikiroo/utils/serial/SerialUtils.java b/src/be/nikiroo/utils/serial/SerialUtils.java index 706d579..204d1d9 100644 --- a/src/be/nikiroo/utils/serial/SerialUtils.java +++ b/src/be/nikiroo/utils/serial/SerialUtils.java @@ -1,7 +1,11 @@ package be.nikiroo.utils.serial; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.NotSerializableException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; import java.lang.reflect.Array; import java.lang.reflect.Constructor; import java.lang.reflect.Field; @@ -13,7 +17,11 @@ import java.util.List; import java.util.Map; import java.util.UnknownFormatConversionException; +import be.nikiroo.utils.IOUtils; import be.nikiroo.utils.Image; +import be.nikiroo.utils.streams.NextableInputStream; +import be.nikiroo.utils.streams.NextableInputStreamStep; +import be.nikiroo.utils.StringUtils; /** * Small class to help with serialisation. @@ -53,50 +61,57 @@ public class SerialUtils { // Array types: customTypes.put("[]", new CustomSerializer() { @Override - protected String toString(Object value) { + protected void toStream(OutputStream out, Object value) + throws IOException { + // TODO: we use \n to separate, and b64 to un-\n -- but we could + // use \\n ? String type = value.getClass().getCanonicalName(); type = type.substring(0, type.length() - 2); // remove the [] - StringBuilder builder = new StringBuilder(); - builder.append(type).append("\n"); + write(out, type); + write(out, "\r"); try { for (int i = 0; true; i++) { Object item = Array.get(value, i); // encode it normally if direct value - if (!SerialUtils.encode(builder, item)) { + if (!SerialUtils.encode(out, item)) { try { - // use ZIP: if not - new Exporter().append(item).appendTo(builder, - true, false); + // TODO: bad escaping? + write(out, "B64:"); + OutputStream bout = StringUtils.base64(out, + false, false); + new Exporter(bout).append(item); } catch (NotSerializableException e) { throw new UnknownFormatConversionException(e .getMessage()); } } - builder.append("\n"); + write(out, "\r"); } } catch (ArrayIndexOutOfBoundsException e) { // Done. } - - return builder.toString(); - } - - @Override - protected String getType() { - return "[]"; } @Override - protected Object fromString(String content) throws IOException { - String[] tab = content.split("\n"); + protected Object fromStream(InputStream in) throws IOException { + NextableInputStream stream = new NextableInputStream(in, + new NextableInputStreamStep('\r')); try { + List list = new ArrayList(); + stream.next(); + String type = IOUtils.readSmallStream(stream); + + while (stream.next()) { + Object value = new Importer().read(stream).getValue(); + list.add(value); + } + Object array = Array.newInstance( - SerialUtils.getClass(tab[0]), tab.length - 1); - for (int i = 1; i < tab.length; i++) { - Object value = new Importer().read(tab[i]).getValue(); - Array.set(array, i - 1, value); + SerialUtils.getClass(type), list.size()); + for (int i = 0; i < list.size(); i++) { + Array.set(array, i, list.get(i)); } return array; @@ -107,23 +122,33 @@ public class SerialUtils { throw new IOException(e.getMessage()); } } + + @Override + protected String getType() { + return "[]"; + } }); // URL: customTypes.put("java.net.URL", new CustomSerializer() { @Override - protected String toString(Object value) { + protected void toStream(OutputStream out, Object value) + throws IOException { + String val = ""; if (value != null) { - return ((URL) value).toString(); + val = ((URL) value).toString(); } - return null; + + out.write(val.getBytes("UTF-8")); } @Override - protected Object fromString(String content) throws IOException { - if (content != null) { - return new URL(content); + protected Object fromStream(InputStream in) throws IOException { + String val = IOUtils.readSmallStream(in); + if (!val.isEmpty()) { + return new URL(val); } + return null; } @@ -136,8 +161,20 @@ public class SerialUtils { // Images (this is currently the only supported image type by default) customTypes.put("be.nikiroo.utils.Image", new CustomSerializer() { @Override - protected String toString(Object value) { - return ((Image) value).toBase64(); + protected void toStream(OutputStream out, Object value) + throws IOException { + Image img = (Image) value; + OutputStream encoded = StringUtils.base64(out, false, false); + try { + InputStream in = img.newInputStream(); + try { + IOUtils.write(in, encoded); + } finally { + in.close(); + } + } finally { + encoded.close(); + } } @Override @@ -146,9 +183,9 @@ public class SerialUtils { } @Override - protected Object fromString(String content) { + protected Object fromStream(InputStream in) throws IOException { try { - return new Image(content); + return new Image(in); } catch (IOException e) { throw new UnknownFormatConversionException(e.getMessage()); } @@ -212,7 +249,7 @@ public class SerialUtils { ctor = clazz.getDeclaredConstructor(classes .toArray(new Class[] {})); } catch (NoSuchMethodException nsme) { - // TODO: it seems e do not always need a parameter for each + // TODO: it seems we do not always need a parameter for each // level, so we currently try "ALL" levels or "FIRST" level // only -> we should check the actual rule and use it ctor = clazz.getDeclaredConstructor(classes.get(0)); @@ -252,14 +289,14 @@ public class SerialUtils { } /** - * Serialise the given object into this {@link StringBuilder}. + * Serialise the given object into this {@link OutputStream}. *

* Important: If the operation fails (with a * {@link NotSerializableException}), the {@link StringBuilder} will be * corrupted (will contain bad, most probably not importable data). * - * @param builder - * the output {@link StringBuilder} to serialise to + * @param out + * the output {@link OutputStream} to serialise to * @param o * the object to serialise * @param map @@ -271,9 +308,11 @@ public class SerialUtils { * if the object cannot be serialised (in this case, the * {@link StringBuilder} can contain bad, most probably not * importable data) + * @throws IOException + * in case of I/O errors */ - static void append(StringBuilder builder, Object o, Map map) - throws NotSerializableException { + static void append(OutputStream out, Object o, Map map) + throws NotSerializableException, IOException { Field[] fields = new Field[] {}; String type = ""; @@ -295,9 +334,13 @@ public class SerialUtils { } } - builder.append("{\nREF ").append(type).append("@").append(id) - .append(":"); - if (!encode(builder, o)) { // check if direct value + write(out, "{\nREF "); + write(out, type); + write(out, "@"); + write(out, id); + write(out, ":"); + + if (!encode(out, o)) { // check if direct value try { for (Field field : fields) { field.setAccessible(true); @@ -311,16 +354,15 @@ public class SerialUtils { continue; } - builder.append("\n"); - builder.append(field.getName()); - builder.append(":"); - Object value; + write(out, "\n"); + write(out, field.getName()); + write(out, ":"); - value = field.get(o); + Object value = field.get(o); - if (!encode(builder, value)) { - builder.append("\n"); - append(builder, value, map); + if (!encode(out, value)) { + write(out, "\n"); + append(out, value, map); } } } catch (IllegalArgumentException e) { @@ -331,11 +373,12 @@ public class SerialUtils { // setAccessible) } } - builder.append("\n}"); + write(out, "\n}"); } /** - * Encode the object into the given builder if possible and if supported. + * Encode the object into the given {@link OutputStream} if possible and if + * supported. *

* A supported object in this context means an object we can directly * encode, like an Integer or a String. Custom objects and arrays are also @@ -343,47 +386,57 @@ public class SerialUtils { *

* For compound objects, you should use {@link Exporter}. * - * @param builder - * the builder to append to + * @param out + * the {@link OutputStream} to append to * @param value * the object to encode (can be NULL, which will be encoded) * - * @return TRUE if success, FALSE if not (the content of the builder won't - * be changed in case of failure) + * @return TRUE if success, FALSE if not (the content of the + * {@link OutputStream} won't be changed in case of failure) + * + * @throws IOException + * in case of I/O error */ - static boolean encode(StringBuilder builder, Object value) { + static boolean encode(OutputStream out, Object value) throws IOException { if (value == null) { - builder.append("NULL"); + write(out, "NULL"); } else if (value.getClass().getSimpleName().endsWith("[]")) { // Simple name does support [] suffix and do not return NULL for // inner anonymous classes - return customTypes.get("[]").encode(builder, value); + return customTypes.get("[]").encode(out, value); } else if (customTypes.containsKey(value.getClass().getCanonicalName())) { return customTypes.get(value.getClass().getCanonicalName())// - .encode(builder, value); + .encode(out, value); } else if (value instanceof String) { - encodeString(builder, (String) value); + encodeString(out, (String) value); } else if (value instanceof Boolean) { - builder.append(value); + write(out, value); } else if (value instanceof Byte) { - builder.append(value).append('b'); + write(out, value); + write(out, "b"); } else if (value instanceof Character) { - encodeString(builder, "" + value); - builder.append('c'); + encodeString(out, "" + value); + write(out, "c"); } else if (value instanceof Short) { - builder.append(value).append('s'); + write(out, value); + write(out, "s"); } else if (value instanceof Integer) { - builder.append(value); + write(out, value); } else if (value instanceof Long) { - builder.append(value).append('L'); + write(out, value); + write(out, "L"); } else if (value instanceof Float) { - builder.append(value).append('F'); + write(out, value); + write(out, "F"); } else if (value instanceof Double) { - builder.append(value).append('d'); + write(out, value); + write(out, "d"); } else if (value instanceof Enum) { String type = value.getClass().getCanonicalName(); - builder.append(type).append(".").append(((Enum) value).name()) - .append(";"); + write(out, type); + write(out, "."); + write(out, ((Enum) value).name()); + write(out, ";"); } else { return false; } @@ -419,7 +472,14 @@ public class SerialUtils { // custom:TYPE_NAME:"content is String-encoded" String type = CustomSerializer.typeOf(encodedValue); if (customTypes.containsKey(type)) { - return customTypes.get(type).decode(encodedValue); + // TODO: we should start with a stream + InputStream streamEncodedValue = new ByteArrayInputStream( + encodedValue.getBytes("UTF-8")); + try { + return customTypes.get(type).decode(streamEncodedValue); + } finally { + streamEncodedValue.close(); + } } throw new IOException("Unknown custom type: " + type); } else if (encodedValue.equals("NULL") @@ -456,6 +516,27 @@ public class SerialUtils { } } + /** + * Write the given {@link String} into the given {@link OutputStream} in + * UTF-8. + * + * @param out + * the {@link OutputStream} + * @param data + * the data to write, cannot be NULL + * + * @throws IOException + * in case of I/O error + */ + static void write(OutputStream out, Object data) throws IOException { + try { + out.write(data.toString().getBytes("UTF-8")); + } catch (UnsupportedEncodingException e) { + // A conforming JVM is required to support UTF-8 + e.printStackTrace(); + } + } + /** * Return the corresponding class or throw an {@link Exception} if it * cannot. @@ -507,7 +588,7 @@ public class SerialUtils { } @SuppressWarnings({ "unchecked", "rawtypes" }) - private static Enum decodeEnum(String escaped) { + static private Enum decodeEnum(String escaped) { // escaped: be.xxx.EnumType.VALUE; int pos = escaped.lastIndexOf("."); String type = escaped.substring(0, pos); @@ -522,32 +603,56 @@ public class SerialUtils { } // aa bb -> "aa\tbb" - private static void encodeString(StringBuilder builder, String raw) { - builder.append('\"'); + static void encodeString(OutputStream out, String raw) throws IOException { + out.write('\"'); + // TODO !! utf-8 required for (char car : raw.toCharArray()) { - switch (car) { - case '\\': - builder.append("\\\\"); - break; - case '\r': - builder.append("\\r"); - break; - case '\n': - builder.append("\\n"); - break; - case '"': - builder.append("\\\""); - break; - default: - builder.append(car); - break; + encodeString(out, car); + } + out.write('\"'); + } + + // aa bb -> "aa\tbb" + static void encodeString(OutputStream out, InputStream raw) + throws IOException { + out.write('\"'); + byte buffer[] = new byte[4096]; + for (int len = 0; (len = raw.read(buffer)) > 0;) { + for (int i = 0; i < len; i++) { + // TODO: not 100% correct, look up howto for UTF-8 + encodeString(out, (char) buffer[i]); } } - builder.append('\"'); + out.write('\"'); + } + + // for encode string, NOT to encode a char by itself! + static void encodeString(OutputStream out, char raw) throws IOException { + switch (raw) { + case '\\': + out.write('\\'); + out.write('\\'); + break; + case '\r': + out.write('\\'); + out.write('r'); + break; + case '\n': + out.write('\\'); + out.write('n'); + break; + case '"': + out.write('\\'); + out.write('\"'); + break; + default: + out.write(raw); + break; + } } // "aa\tbb" -> aa bb - private static String decodeString(String escaped) { + static String decodeString(String escaped) { StringBuilder builder = new StringBuilder(); boolean escaping = false; diff --git a/src/be/nikiroo/utils/serial/server/ConnectAction.java b/src/be/nikiroo/utils/serial/server/ConnectAction.java index a377ced..8dbc7aa 100644 --- a/src/be/nikiroo/utils/serial/server/ConnectAction.java +++ b/src/be/nikiroo/utils/serial/server/ConnectAction.java @@ -199,8 +199,9 @@ abstract class ConnectAction { * @param data * the data to send * - * @return the answer (which can be NULL) if this action is a client, always - * NULL if it is a server + * @return the answer (which can be NULL if no answer, or NULL for an answer + * which is NULL) if this action is a client, always NULL if it is a + * server * * @throws IOException * in case of I/O error @@ -216,10 +217,18 @@ abstract class ConnectAction { protected Object sendObject(Object data) throws IOException, NoSuchFieldException, NoSuchMethodException, ClassNotFoundException { synchronized (lock) { - String rep = sendString(new Exporter().append(data).toString(true, - true)); - if (rep != null) { - return new Importer().read(rep).getValue(); + new Exporter(out).append(data); + + if (server) { + out.flush(); + return null; + } + + contentToSend = true; + try { + return recObject(); + } catch (NullPointerException e) { + // We accept no data here } return null; @@ -253,12 +262,18 @@ abstract class ConnectAction { protected Object recObject() throws IOException, NoSuchFieldException, NoSuchMethodException, ClassNotFoundException, java.lang.NullPointerException { - String str = recString(); - if (str == null) { - throw new NullPointerException("No more data available"); - } + synchronized (lock) { + if (server || contentToSend) { + if (contentToSend) { + out.flush(); + contentToSend = false; + } - return new Importer().read(str).getValue(); + return new Importer().read(in).getValue(); + } + + return null; + } } /** @@ -365,15 +380,15 @@ abstract class ConnectAction { */ private void writeLine(OutputStream out, String line) throws IOException { if (crypt == null) { - out.write(line.getBytes()); + out.write(line.getBytes("UTF-8")); bytesSent += line.length(); } else { // TODO: how NOT to create so many big Strings? String b64 = crypt.encrypt64(line, false); - out.write(b64.getBytes()); + out.write(b64.getBytes("UTF-8")); bytesSent += b64.length(); } - out.write("\n".getBytes()); + out.write("\n".getBytes("UTF-8")); bytesSent++; } } \ No newline at end of file diff --git a/src/be/nikiroo/utils/serial/server/ServerBridge.java b/src/be/nikiroo/utils/serial/server/ServerBridge.java index 6c2ed01..ff4a931 100644 --- a/src/be/nikiroo/utils/serial/server/ServerBridge.java +++ b/src/be/nikiroo/utils/serial/server/ServerBridge.java @@ -1,6 +1,8 @@ package be.nikiroo.utils.serial.server; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.lang.reflect.Array; import java.net.Socket; import java.net.UnknownHostException; @@ -123,6 +125,7 @@ public class ServerBridge extends Server { @Override protected ConnectActionServer createConnectActionServer(Socket s) { + // Bad impl, not up to date (should work, but not efficient) return new ConnectActionServerString(s, key) { @Override public void action(final Version clientVersion) throws Exception { @@ -226,6 +229,7 @@ public class ServerBridge extends Server { * the data to trace */ private void trace(String prefix, String data) { + // TODO: we convert to string and back int size = data == null ? 0 : data.length(); String ssize = StringUtils.formatNumber(size) + "bytes"; @@ -241,22 +245,29 @@ public class ServerBridge extends Server { } } - Object obj = new Importer().read(data).getValue(); - if (obj == null) { - getTraceHandler().trace("NULL", 2); - getTraceHandler().trace("NULL", 3); - getTraceHandler().trace("NULL", 4); - } else { - if (obj.getClass().isArray()) { - getTraceHandler().trace( - "(" + obj.getClass() + ") with " - + Array.getLength(obj) + "element(s)", - 3); + InputStream stream = new ByteArrayInputStream( + data.getBytes("UTF-8")); + try { + Object obj = new Importer().read(stream).getValue(); + if (obj == null) { + getTraceHandler().trace("NULL", 2); + getTraceHandler().trace("NULL", 3); + getTraceHandler().trace("NULL", 4); } else { - getTraceHandler().trace("(" + obj.getClass() + ")", 2); + if (obj.getClass().isArray()) { + getTraceHandler().trace( + "(" + obj.getClass() + ") with " + + Array.getLength(obj) + + "element(s)", 3); + } else { + getTraceHandler().trace("(" + obj.getClass() + ")", + 2); + } + getTraceHandler().trace("" + obj.toString(), 3); + getTraceHandler().trace(data, 4); } - getTraceHandler().trace("" + obj.toString(), 3); - getTraceHandler().trace(data, 4); + } finally { + stream.close(); } } catch (NoSuchMethodException e) { getTraceHandler().trace("(not an object)", 2); diff --git a/src/be/nikiroo/utils/test_code/SerialTest.java b/src/be/nikiroo/utils/test_code/SerialTest.java index f3ed346..1581965 100644 --- a/src/be/nikiroo/utils/test_code/SerialTest.java +++ b/src/be/nikiroo/utils/test_code/SerialTest.java @@ -1,6 +1,12 @@ package be.nikiroo.utils.test_code; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.NotSerializableException; import java.net.URL; +import java.util.Arrays; import be.nikiroo.utils.serial.Exporter; import be.nikiroo.utils.serial.Importer; @@ -15,6 +21,50 @@ class SerialTest extends TestLauncher { this(null); } + private void encodeRecodeTest(TestCase test, Object data) throws Exception { + byte[] encoded = toBytes(data, true); + Object redata = fromBytes(toBytes(data, false)); + byte[] reencoded = toBytes(redata, true); + + // We suppose text mode + if (encoded.length < 256 && reencoded.length < 256) { + test.assertEquals("Different data after encode/decode/encode", + new String(encoded, "UTF-8"), + new String(reencoded, "UTF-8")); + } else { + test.assertEquals("Different data after encode/decode/encode", + true, Arrays.equals(encoded, reencoded)); + } + } + + // try to remove pointer addresses + private byte[] toBytes(Object data, boolean clearRefs) + throws NotSerializableException, IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Exporter(out).append(data); + out.flush(); + + if (clearRefs) { + String tmp = new String(out.toByteArray(), "UTF-8"); + tmp = tmp.replaceAll("@[0-9]*", "@REF"); + return tmp.getBytes("UTF-8"); + } + + return out.toByteArray(); + } + + private Object fromBytes(byte[] data) throws NoSuchFieldException, + NoSuchMethodException, ClassNotFoundException, + NullPointerException, IOException { + + InputStream in = new ByteArrayInputStream(data); + try { + return new Importer().read(in).getValue(); + } finally { + in.close(); + } + } + public SerialTest(String[] args) { super("Serial test", args); @@ -22,14 +72,7 @@ class SerialTest extends TestLauncher { @Override public void test() throws Exception { Data data = new Data(42); - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); + encodeRecodeTest(this, data); } }); @@ -43,18 +86,9 @@ class SerialTest extends TestLauncher { @SuppressWarnings("unused") int value = 42; }; - - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); + encodeRecodeTest(this, data); } }); - addTest(new TestCase() { @SuppressWarnings("unused") private TestCase me = setName("Array of anonymous inner classes"); @@ -66,211 +100,91 @@ class SerialTest extends TestLauncher { int value = 42; } }; - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - // Comparing the 2 strings won't be useful, because the @REFs - // will be ZIP-encoded; so we parse and re-encode the object - encoded = new Exporter().append(data[0]).toString(false, false); - try { - reencoded = new Exporter().append(((Data[]) redata)[0]) - .toString(false, false); - } catch (Exception e) { - fail("Cannot cast the returned data into its original object", - e); - } - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); + byte[] encoded = toBytes(data, false); + Object redata = fromBytes(encoded); + + // Comparing the 2 arrays won't be useful, because the @REFs + // will be ZIP-encoded; so we parse and re-encode each object + + byte[] encoded1 = toBytes(data[0], true); + byte[] reencoded1 = toBytes(((Object[]) redata)[0], true); + + assertEquals("Different data after encode/decode/encode", true, + Arrays.equals(encoded1, reencoded1)); } }); - addTest(new TestCase("URL Import/Export") { @Override public void test() throws Exception { URL data = new URL("https://fanfan.be/"); - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); + encodeRecodeTest(this, data); } }); - addTest(new TestCase("URL-String Import/Export") { @Override public void test() throws Exception { String data = new URL("https://fanfan.be/").toString(); - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); - assertEquals(data, redata); + encodeRecodeTest(this, data); } }); - addTest(new TestCase("URL/URL-String arrays Import/Export") { @Override public void test() throws Exception { final String url = "https://fanfan.be/"; - Object[] data = new Object[] { new URL(url), url }; - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); - assertEquals(data[0], ((Object[]) redata)[0]); - assertEquals(data[1], ((Object[]) redata)[1]); + + byte[] encoded = toBytes(data, false); + Object redata = fromBytes(encoded); + + // Comparing the 2 arrays won't be useful, because the @REFs + // will be ZIP-encoded; so we parse and re-encode each object + + byte[] encoded1 = toBytes(data[0], true); + byte[] reencoded1 = toBytes(((Object[]) redata)[0], true); + byte[] encoded2 = toBytes(data[1], true); + byte[] reencoded2 = toBytes(((Object[]) redata)[1], true); + + assertEquals("Different data 1 after encode/decode/encode", + true, Arrays.equals(encoded1, reencoded1)); + assertEquals("Different data 2 after encode/decode/encode", + true, Arrays.equals(encoded2, reencoded2)); } }); - addTest(new TestCase("Import/Export with nested objects") { @Override public void test() throws Exception { Data data = new DataObject(new Data(21)); - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); + encodeRecodeTest(this, data); } }); - addTest(new TestCase("Import/Export with nested objects forming a loop") { @Override public void test() throws Exception { DataLoop data = new DataLoop("looping"); data.next = new DataLoop("level 2"); data.next.next = data; - - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); + encodeRecodeTest(this, data); } }); - addTest(new TestCase("Array in Object Import/Export") { @Override public void test() throws Exception { Object data = new DataArray();// new String[] { "un", "deux" }; - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); + encodeRecodeTest(this, data); } }); - addTest(new TestCase("Array Import/Export") { @Override public void test() throws Exception { Object data = new String[] { "un", "deux" }; - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); + encodeRecodeTest(this, data); } }); - addTest(new TestCase("Enum Import/Export") { @Override public void test() throws Exception { Object data = EnumToSend.FANFAN; - String encoded = new Exporter().append(data).toString(false, - false); - Object redata = new Importer().read(encoded).getValue(); - String reencoded = new Exporter().append(redata).toString( - false, false); - - assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), - reencoded.replaceAll("@[0-9]*", "@REF")); - } - }); - - addTest(new TestCase("B64 and ZIP String test") { - @Override - public void test() throws Exception { - Object data = "Fanfan la tulipe"; - String encoded = new Exporter().append(data).toString(true, - false); - String redata = (String) new Importer().read(encoded) - .getValue(); - - assertEquals("Items not identical after B64", data, redata); - - encoded = new Exporter().append(data).toString(true, true); - redata = (String) new Importer().read(encoded).getValue(); - - assertEquals("Items not identical after ZIP", data, redata); - } - }); - - addTest(new TestCase("B64 and ZIP Data test") { - @Override - public void test() throws Exception { - Object data = new Data(55); - String encoded = new Exporter().append(data).toString(true, - false); - Data redata = (Data) new Importer().read(encoded).getValue(); - - assertEquals("Items not identical after B64", data, redata); - - encoded = new Exporter().append(data).toString(true, true); - redata = (Data) new Importer().read(encoded).getValue(); - - assertEquals("Items not identical after ZIP", data, redata); - } - }); - - addTest(new TestCase("B64 and ZIP 70000 chars test") { - @Override - public void test() throws Exception { - StringBuilder builder = new StringBuilder(); - for (int i = 0; i < 7000; i++) { - builder.append("0123456789"); - } - - Object data = builder.toString(); - String encoded = new Exporter().append(data).toString(true, - false); - String redata = (String) new Importer().read(encoded) - .getValue(); - - assertEquals("Items not identical after B64", data, redata); - - encoded = new Exporter().append(data).toString(true, true); - redata = (String) new Importer().read(encoded).getValue(); - - assertEquals("Items not identical after ZIP", data, redata); + encodeRecodeTest(this, data); } }); }