From ce0974c4b695f842fa7ec81f3c53d016d1959854 Mon Sep 17 00:00:00 2001 From: Niki Roo Date: Sat, 1 Jul 2017 18:40:16 +0200 Subject: [PATCH] New Server class to send/rec objects via network --- VERSION | 2 +- changelog | 7 + configure.sh | 2 +- .../nikiroo/utils/serial/ConnectAction.java | 147 ++++++++++++ .../utils/serial/ConnectActionClient.java | 31 +++ .../utils/serial/ConnectActionServer.java | 19 ++ src/be/nikiroo/utils/serial/Importer.java | 26 ++- src/be/nikiroo/utils/serial/SerialUtils.java | 110 +++++++-- src/be/nikiroo/utils/serial/Server.java | 219 ++++++++++++++++++ src/be/nikiroo/utils/test/SerialTest.java | 49 ++++ 10 files changed, 583 insertions(+), 29 deletions(-) create mode 100644 src/be/nikiroo/utils/serial/ConnectAction.java create mode 100644 src/be/nikiroo/utils/serial/ConnectActionClient.java create mode 100644 src/be/nikiroo/utils/serial/ConnectActionServer.java create mode 100644 src/be/nikiroo/utils/serial/Server.java diff --git a/VERSION b/VERSION index 26ca5946..4cda8f19 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.1 +1.5.2 diff --git a/changelog b/changelog index 4dc73412..40962052 100644 --- a/changelog +++ b/changelog @@ -1,3 +1,10 @@ +Version 1.6.0 +------------- + +Serialisation utilities + Server class to send/receive objects via network easily + Serialiser now supports Arrays + fixes + Version 1.5.1 ------------- diff --git a/configure.sh b/configure.sh index 713c9f46..b88e39aa 100755 --- a/configure.sh +++ b/configure.sh @@ -45,7 +45,7 @@ fi; echo "MAIN = be/nikiroo/utils/test/Test" > Makefile -echo "MORE = be/nikiroo/utils/StringUtils be/nikiroo/utils/IOUtils be/nikiroo/utils/MarkableFileInputStream be/nikiroo/utils/ui/UIUtils be/nikiroo/utils/ui/WrapLayout be/nikiroo/utils/ui/ProgressBar be/nikiroo/utils/resources/TransBundle" >> Makefile +echo "MORE = be/nikiroo/utils/MarkableFileInputStream be/nikiroo/utils/ui/UIUtils be/nikiroo/utils/ui/WrapLayout be/nikiroo/utils/ui/ProgressBar" >> Makefile echo "TEST = be/nikiroo/utils/test/Test" >> Makefile echo "TEST_PARAMS = $cols $ok $ko" >> Makefile echo "NAME = nikiroo-utils" >> Makefile diff --git a/src/be/nikiroo/utils/serial/ConnectAction.java b/src/be/nikiroo/utils/serial/ConnectAction.java new file mode 100644 index 00000000..ec49a1dd --- /dev/null +++ b/src/be/nikiroo/utils/serial/ConnectAction.java @@ -0,0 +1,147 @@ +package be.nikiroo.utils.serial; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.Socket; + +import be.nikiroo.utils.Version; + +abstract class ConnectAction { + private Socket s; + private boolean server; + private Version version; + + private Object lock = new Object(); + private BufferedReader in; + private OutputStreamWriter out; + private boolean contentToSend; + + // serverVersion = null on server (or bad clients) + abstract public void action(Version serverVersion) throws Exception; + + // server = version NULL + protected ConnectAction(Socket s, boolean server, Version version) { + this.s = s; + this.server = server; + + if (version == null) { + this.version = new Version(); + } else { + this.version = version; + } + } + + public void connectAsync() { + new Thread(new Runnable() { + public void run() { + connect(); + } + }).start(); + } + + public void connect() { + try { + in = new BufferedReader(new InputStreamReader(s.getInputStream(), + "UTF-8")); + try { + out = new OutputStreamWriter(s.getOutputStream(), "UTF-8"); + try { + if (server) { + action(version); + } else { + String v = sendString("VERSION " + version.toString()); + if (v != null && v.startsWith("VERSION ")) { + v = v.substring("VERSION ".length()); + } + + action(new Version(v)); + } + } finally { + out.close(); + } + } finally { + in.close(); + } + } catch (Exception e) { + onError(e); + } finally { + try { + s.close(); + } catch (Exception e) { + onError(e); + } + } + } + + // (also, server never get anything) + public Object send(Object data) throws IOException, NoSuchFieldException, + NoSuchMethodException, ClassNotFoundException { + synchronized (lock) { + String rep = sendString(new Exporter().append(data).toString(true)); + return new Importer().read(rep).getValue(); + } + } + + public Object flush() throws NoSuchFieldException, NoSuchMethodException, + ClassNotFoundException, IOException, java.lang.NullPointerException { + String str = flushString(); + if (str == null) { + throw new NullPointerException("No more data from client"); + } + + return new Importer().read(str).getValue(); + } + + protected void onClientVersionReceived(Version clientVersion) { + + } + + protected void onError(Exception e) { + + } + + // \n included in line, but not in rep (also, server never get anything) + private String sendString(String line) throws IOException { + synchronized (lock) { + out.write(line); + out.write("\n"); + + if (server) { + out.flush(); + return null; + } else { + contentToSend = true; + return flushString(); + } + } + } + + // server can receive something even without pending content + private String flushString() throws IOException { + synchronized (lock) { + if (server || contentToSend) { + if (contentToSend) { + out.flush(); + contentToSend = false; + } + + String line = in.readLine(); + if (server && line != null && line.startsWith("VERSION ")) { + // "VERSION client-version" (VERSION 1.0.0) + Version clientVersion = new Version( + line.substring("VERSION ".length())); + onClientVersionReceived(clientVersion); + sendString("VERSION " + version.toString()); + + line = in.readLine(); + } + + return line; + } else { + return null; + } + } + } +} \ No newline at end of file diff --git a/src/be/nikiroo/utils/serial/ConnectActionClient.java b/src/be/nikiroo/utils/serial/ConnectActionClient.java new file mode 100644 index 00000000..5b982021 --- /dev/null +++ b/src/be/nikiroo/utils/serial/ConnectActionClient.java @@ -0,0 +1,31 @@ +package be.nikiroo.utils.serial; + +import java.io.IOException; +import java.net.Socket; + +import be.nikiroo.utils.Version; + +public class ConnectActionClient extends ConnectAction { + protected ConnectActionClient(Socket s) { + super(s, false, Version.getCurrentVersion()); + } + + protected ConnectActionClient(Socket s, Version version) { + super(s, false, version); + } + + protected ConnectActionClient(String host, int port, boolean ssl) + throws IOException { + super(Server.createSocket(host, port, ssl), false, Version + .getCurrentVersion()); + } + + protected ConnectActionClient(String host, int port, boolean ssl, + Version version) throws IOException { + super(Server.createSocket(host, port, ssl), false, version); + } + + @Override + public void action(Version serverVersion) throws Exception { + } +} \ No newline at end of file diff --git a/src/be/nikiroo/utils/serial/ConnectActionServer.java b/src/be/nikiroo/utils/serial/ConnectActionServer.java new file mode 100644 index 00000000..7c040652 --- /dev/null +++ b/src/be/nikiroo/utils/serial/ConnectActionServer.java @@ -0,0 +1,19 @@ +package be.nikiroo.utils.serial; + +import java.net.Socket; + +import be.nikiroo.utils.Version; + +public class ConnectActionServer extends ConnectAction { + protected ConnectActionServer(Socket s) { + super(s, true, Version.getCurrentVersion()); + } + + protected ConnectActionServer(Socket s, Version version) { + super(s, true, version); + } + + @Override + public void action(Version serverVersion) throws Exception { + } +} \ No newline at end of file diff --git a/src/be/nikiroo/utils/serial/Importer.java b/src/be/nikiroo/utils/serial/Importer.java index 61093f75..d21f1dd3 100644 --- a/src/be/nikiroo/utils/serial/Importer.java +++ b/src/be/nikiroo/utils/serial/Importer.java @@ -65,9 +65,10 @@ public class Importer { if (line.startsWith("ZIP:")) { line = StringUtils.unzip64(line.substring("ZIP:".length())); + read(line); + } else { + processLine(line); } - processLine(line); - } scan.close(); } catch (IOException e) { @@ -118,18 +119,33 @@ public class Importer { } else if (line.equals("}")) { // STOP: report self to parent return true; } else if (line.startsWith("REF ")) { // REF: create/link self - String ref = line.substring(4).split("@")[1]; + String[] tab = line.substring("REF ".length()).split("@"); + String type = tab[0]; + tab = tab[1].split(":"); + String ref = tab[0]; + link = map.containsKey(ref); if (link) { me = map.get(ref); } else { - me = SerialUtils.createObject(line.substring(4).split("@")[0]); + if (line.endsWith(":")) { + // construct + me = SerialUtils.createObject(type); + } else { + // direct value + int pos = line.indexOf(":"); + String encodedValue = line.substring(pos + 1); + me = SerialUtils.decode(encodedValue); + } map.put(ref, me); } - } else { // FIELD: new field + } else { // FIELD: new field *or* direct simple value if (line.endsWith(":")) { // field value is compound currentFieldName = line.substring(0, line.length() - 1); + } else if (line.startsWith(":") || !line.contains(":")) { + // not a field value but a direct value + me = SerialUtils.decode(line); } else { // field value is direct int pos = line.indexOf(":"); diff --git a/src/be/nikiroo/utils/serial/SerialUtils.java b/src/be/nikiroo/utils/serial/SerialUtils.java index 49817b25..7bf1b17a 100644 --- a/src/be/nikiroo/utils/serial/SerialUtils.java +++ b/src/be/nikiroo/utils/serial/SerialUtils.java @@ -1,10 +1,13 @@ package be.nikiroo.utils.serial; import java.io.NotSerializableException; +import java.lang.reflect.Array; import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.UnknownFormatConversionException; /** * Small class to help with serialisation. @@ -20,6 +23,62 @@ public class SerialUtils { static { customTypes = new HashMap(); // TODO: add "default" custom serialisers if any (Bitmap?) + + // Array types: + customTypes.put("[]", new CustomSerializer() { + @Override + protected String toString(Object value) { + String type = value.getClass().getCanonicalName(); + type = type.substring(0, type.length() - 2); // remove the [] + + StringBuilder builder = new StringBuilder(); + builder.append(type).append("\n"); + try { + for (int i = 0; true; i++) { + Object item = Array.get(value, i); + // encode it normally if direct value + if (!SerialUtils.encode(builder, item)) { + try { + // use ZIP: if not + builder.append(new Exporter().append(item) + .toString(true)); + } catch (NotSerializableException e) { + throw new UnknownFormatConversionException(e + .getMessage()); + } + } + builder.append("\n"); + } + } catch (ArrayIndexOutOfBoundsException e) { + // Done. + } + + return builder.toString(); + } + + @Override + protected String getType() { + return "[]"; + } + + @Override + protected Object fromString(String content) { + String[] tab = content.split("\n"); + + try { + Object array = Array.newInstance( + SerialUtils.getClass(tab[0]), tab.length - 1); + for (int i = 1; i < tab.length; i++) { + Object value = new Importer().read(tab[i]).getValue(); + Array.set(array, i - 1, value); + } + + return array; + } catch (Exception e) { + throw new UnknownFormatConversionException(e.getMessage()); + } + } + }); } /** @@ -122,32 +181,37 @@ public class SerialUtils { } } - builder.append("{\nREF ").append(type).append("@").append(id); - try { - for (Field field : fields) { - field.setAccessible(true); + builder.append("{\nREF ").append(type).append("@").append(id) + .append(":"); + if (!encode(builder, o)) { // check if direct value + try { + for (Field field : fields) { + field.setAccessible(true); - if (field.getName().startsWith("this$")) { - // Do not keep this links of nested classes - continue; - } + if (field.getName().startsWith("this$")) { + // Do not keep this links of nested classes + continue; + } - builder.append("\n"); - builder.append(field.getName()); - builder.append(":"); - Object value; + builder.append("\n"); + builder.append(field.getName()); + builder.append(":"); + Object value; - value = field.get(o); + value = field.get(o); - if (!encode(builder, value)) { - builder.append("\n"); - append(builder, value, map); + if (!encode(builder, value)) { + builder.append("\n"); + append(builder, value, map); + } } + } catch (IllegalArgumentException e) { + e.printStackTrace(); // should not happen (see + // setAccessible) + } catch (IllegalAccessException e) { + e.printStackTrace(); // should not happen (see + // setAccessible) } - } catch (IllegalArgumentException e) { - e.printStackTrace(); // should not happen (see setAccessible) - } catch (IllegalAccessException e) { - e.printStackTrace(); // should not happen (see setAccessible) } builder.append("\n}"); } @@ -156,6 +220,8 @@ public class SerialUtils { static boolean encode(StringBuilder builder, Object value) { if (value == null) { builder.append("NULL"); + } else if (value.getClass().getCanonicalName().endsWith("[]")) { + customTypes.get("[]").encode(builder, value); } else if (customTypes.containsKey(value.getClass().getCanonicalName())) { customTypes.get(value.getClass().getCanonicalName())// .encode(builder, value); @@ -166,7 +232,7 @@ public class SerialUtils { } else if (value instanceof Byte) { builder.append(value).append('b'); } else if (value instanceof Character) { - encodeString(builder, (String) value); + encodeString(builder, "" + value); builder.append('c'); } else if (value instanceof Short) { builder.append(value).append('s'); @@ -197,7 +263,7 @@ public class SerialUtils { if (customTypes.containsKey(type)) { return customTypes.get(type).decode(encodedValue); } else { - throw new java.util.UnknownFormatConversionException( + throw new UnknownFormatConversionException( "Unknown custom type: " + type); } } else if (encodedValue.equals("NULL") || encodedValue.equals("null")) { diff --git a/src/be/nikiroo/utils/serial/Server.java b/src/be/nikiroo/utils/serial/Server.java new file mode 100644 index 00000000..50b8f461 --- /dev/null +++ b/src/be/nikiroo/utils/serial/Server.java @@ -0,0 +1,219 @@ +package be.nikiroo.utils.serial; + +import java.io.IOException; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.ArrayList; +import java.util.List; + +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLServerSocketFactory; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; + +import be.nikiroo.utils.Version; + +abstract public class Server implements Runnable { + static private final String[] ANON_CIPHERS = getAnonCiphers(); + + private Version serverVersion = new Version(); + private int port; + private boolean ssl; + private ServerSocket ss; + private boolean started; + private boolean exiting = false; + private int counter; + private Object lock = new Object(); + private Object counterLock = new Object(); + + public Server(Version version, int port, boolean ssl) throws IOException { + this.serverVersion = version; + this.port = port; + this.ssl = ssl; + this.ss = createSocketServer(port, ssl); + } + + public void start() { + synchronized (lock) { + if (!started) { + started = true; + new Thread(this).start(); + } + } + } + + public void stop() { + stop(0, true); + } + + // wait = wait before returning (sync VS async) timeout in ms, 0 or -1 for + // never + public void stop(final long timeout, final boolean wait) { + if (wait) { + stop(timeout); + } else { + new Thread(new Runnable() { + public void run() { + stop(timeout); + } + }).start(); + } + } + + // timeout in ms, 0 or -1 or never + private void stop(long timeout) { + synchronized (lock) { + if (started && !exiting) { + exiting = true; + + try { + new ConnectActionClient(createSocket(null, port, ssl)) { + @Override + public void action(Version serverVersion) + throws Exception { + } + }.connect(); + + long time = 0; + while (ss != null && timeout > 0 && timeout > time) { + Thread.sleep(10); + time += 10; + } + } catch (Exception e) { + if (ss != null) { + counter = 0; // will stop the main thread + onError(e); + } + } + } + + // only return when stopped + while (started || exiting) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + } + } + } + } + + public void run() { + try { + while (started && !exiting) { + count(1); + Socket s = ss.accept(); + new ConnectActionServer(s) { + private Version clientVersion = new Version(); + + @Override + public void action(Version dummy) throws Exception { + try { + for (Object data = flush(); true; data = flush()) { + Object rep = null; + try { + rep = onRequest(this, clientVersion, data); + } catch (Exception e) { + onError(e); + } + send(rep); + } + } catch (NullPointerException e) { + // Client has no data any more, we quit + } + } + + @Override + public void connect() { + try { + super.connect(); + } finally { + count(-1); + } + }; + + @Override + protected void onClientVersionReceived(Version clientVersion) { + this.clientVersion = clientVersion; + }; + }.connectAsync(); + } + + // Will be covered by @link{Server#stop(long)} for timeouts + while (counter > 0) { + Thread.sleep(10); + } + } catch (Exception e) { + if (counter > 0) { + onError(e); + } + } finally { + try { + ss.close(); + } catch (Exception e) { + onError(e); + } + + ss = null; + + started = false; + exiting = false; + counter = 0; + } + } + + abstract protected Object onRequest(ConnectActionServer action, + Version clientVersion, Object data) throws Exception; + + protected void onError(Exception e) { + if (e == null) { + e = new Exception("Unknown error"); + } + + e.printStackTrace(); + } + + private int count(int change) { + synchronized (counterLock) { + counter += change; + return counter; + } + } + + static Socket createSocket(String host, int port, boolean ssl) + throws IOException { + Socket s; + if (ssl) { + s = SSLSocketFactory.getDefault().createSocket(host, port); + ((SSLSocket) s).setEnabledCipherSuites(ANON_CIPHERS); + } else { + s = new Socket(host, port); + } + + return s; + } + + static ServerSocket createSocketServer(int port, boolean ssl) + throws IOException { + ServerSocket ss; + if (ssl) { + ss = SSLServerSocketFactory.getDefault().createServerSocket(port); + ((SSLServerSocket) ss).setEnabledCipherSuites(ANON_CIPHERS); + } else { + ss = new ServerSocket(port); + } + + return ss; + } + + private static String[] getAnonCiphers() { + List anonCiphers = new ArrayList(); + for (String cipher : ((SSLSocketFactory) SSLSocketFactory.getDefault()) + .getSupportedCipherSuites()) { + if (cipher.contains("_anon_")) { + anonCiphers.add(cipher); + } + } + + return anonCiphers.toArray(new String[] {}); + } +} diff --git a/src/be/nikiroo/utils/test/SerialTest.java b/src/be/nikiroo/utils/test/SerialTest.java index f73c39ec..22d1206c 100644 --- a/src/be/nikiroo/utils/test/SerialTest.java +++ b/src/be/nikiroo/utils/test/SerialTest.java @@ -1,9 +1,26 @@ package be.nikiroo.utils.test; +import be.nikiroo.utils.Version; +import be.nikiroo.utils.serial.ConnectActionServer; import be.nikiroo.utils.serial.Exporter; import be.nikiroo.utils.serial.Importer; +import be.nikiroo.utils.serial.Server; class SerialTest extends TestLauncher { + private void not_used() { + // TODO: test Server ; but this will at least help dependency checking + try { + Server server = new Server(null, 0, false) { + @Override + protected Object onRequest(ConnectActionServer action, + Version clientVersion, Object data) throws Exception { + return null; + } + }; + } catch (Exception e) { + } + } + private SerialTest() { super("Serial test", null); } @@ -55,6 +72,38 @@ class SerialTest extends TestLauncher { reencoded.replaceAll("@[0-9]*", "@REF")); } }); + + addTest(new TestCase("Array in Object Import/Export") { + @Override + public void test() throws Exception { + Object data = new DataArray();// new String[] { "un", "deux" }; + String encoded = new Exporter().append(data).toString(false); + Object redata = new Importer().read(encoded).getValue(); + String reencoded = new Exporter().append(redata) + .toString(false); + + assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), + reencoded.replaceAll("@[0-9]*", "@REF")); + } + }); + + addTest(new TestCase("Array Import/Export") { + @Override + public void test() throws Exception { + Object data = new String[] { "un", "deux" }; + String encoded = new Exporter().append(data).toString(false); + Object redata = new Importer().read(encoded).getValue(); + String reencoded = new Exporter().append(redata) + .toString(false); + + assertEquals(encoded.replaceAll("@[0-9]*", "@REF"), + reencoded.replaceAll("@[0-9]*", "@REF")); + } + }); + } + + class DataArray { + public String[] data = new String[] { "un", "deux" }; } @SuppressWarnings("unused") -- 2.27.0