src/be/nikiroo/utils/serial/SerialUtils.java

   1 package be.nikiroo.utils.serial;
   2
   3 import java.io.ByteArrayInputStream;
   4 import java.io.IOException;
   5 import java.io.InputStream;
   6 import java.io.NotSerializableException;
   7 import java.io.OutputStream;
   8 import java.io.UnsupportedEncodingException;
   9 import java.lang.reflect.Array;
  10 import java.lang.reflect.Constructor;
  11 import java.lang.reflect.Field;
  12 import java.lang.reflect.Modifier;
  13 import java.net.URL;
  14 import java.util.ArrayList;
  15 import java.util.HashMap;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.UnknownFormatConversionException;
  19
  20 import be.nikiroo.utils.IOUtils;
  21 import be.nikiroo.utils.Image;
  22 import be.nikiroo.utils.StringUtils;
  23 import be.nikiroo.utils.streams.Base64InputStream;
  24 import be.nikiroo.utils.streams.Base64OutputStream;
  25 import be.nikiroo.utils.streams.NextableInputStream;
  26 import be.nikiroo.utils.streams.NextableInputStreamStep;
  27
  28 /**
  29  * Small class to help with serialisation.
  30  * <p>
  31  * Note that we do not support inner classes (but we do support nested classes)
  32  * and all objects require an empty constructor to be deserialised.
  33  * <p>
  34  * It is possible to add support to custom types (both the encoder and the
  35  * decoder will require the custom classes) -- see {@link CustomSerializer}.
  36  * <p>
  37  * Default supported types are:
  38  * <ul>
  39  * <li>NULL (as a null value)</li>
  40  * <li>String</li>
  41  * <li>Boolean</li>
  42  * <li>Byte</li>
  43  * <li>Character</li>
  44  * <li>Short</li>
  45  * <li>Long</li>
  46  * <li>Float</li>
  47  * <li>Double</li>
  48  * <li>Integer</li>
  49  * <li>Enum (any enum whose name and value is known by the caller)</li>
  50  * <li>java.awt.image.BufferedImage (as a {@link CustomSerializer})</li>
  51  * <li>An array of the above (as a {@link CustomSerializer})</li>
  52  * <li>URL</li>
  53  * </ul>
  54  *
  55  * @author niki
  56  */
  57 public class SerialUtils {
  58         private static Map<String, CustomSerializer> customTypes;
  59
  60         static {
  61                 customTypes = new HashMap<String, CustomSerializer>();
  62
  63                 // Array types:
  64                 customTypes.put("[]", new CustomSerializer() {
  65                         @Override
  66                         protected void toStream(OutputStream out, Object value)
  67                                         throws IOException {
  68
  69                                 // TODO: we use \n to separate, and b64 to un-\n
  70                                 // -- but we could use \\n ?
  71                                 String type = value.getClass().getCanonicalName();
  72                                 type = type.substring(0, type.length() - 2); // remove the []
  73
  74                                 write(out, type);
  75                                 try {
  76                                         for (int i = 0; true; i++) {
  77                                                 Object item = Array.get(value, i);
  78
  79                                                 // encode it normally if direct value
  80                                                 write(out, "\r");
  81                                                 if (!SerialUtils.encode(out, item)) {
  82                                                         try {
  83                                                                 // TODO: bad escaping?
  84                                                                 write(out, "B64:");
  85                                                                 OutputStream out64 = new Base64OutputStream(
  86                                                                                 out, true);
  87                                                                 new Exporter(out64).append(item);
  88                                                         } catch (NotSerializableException e) {
  89                                                                 throw new UnknownFormatConversionException(e
  90                                                                                 .getMessage());
  91                                                         }
  92                                                 }
  93                                         }
  94                                 } catch (ArrayIndexOutOfBoundsException e) {
  95                                         // Done.
  96                                 }
  97                         }
  98
  99                         @Override
 100                         protected Object fromStream(InputStream in) throws IOException {
 101                                 NextableInputStream stream = new NextableInputStream(in,
 102                                                 new NextableInputStreamStep('\r'));
 103
 104                                 try {
 105                                         List<Object> list = new ArrayList<Object>();
 106                                         stream.next();
 107                                         String type = IOUtils.readSmallStream(stream);
 108
 109                                         while (stream.next()) {
 110                                                 Object value = new Importer().read(stream).getValue();
 111                                                 list.add(value);
 112                                         }
 113
 114                                         Object array = Array.newInstance(
 115                                                         SerialUtils.getClass(type), list.size());
 116                                         for (int i = 0; i < list.size(); i++) {
 117                                                 Array.set(array, i, list.get(i));
 118                                         }
 119
 120                                         return array;
 121                                 } catch (Exception e) {
 122                                         if (e instanceof IOException) {
 123                                                 throw (IOException) e;
 124                                         }
 125                                         throw new IOException(e.getMessage());
 126                                 }
 127                         }
 128
 129                         @Override
 130                         protected String getType() {
 131                                 return "[]";
 132                         }
 133                 });
 134
 135                 // URL:
 136                 customTypes.put("java.net.URL", new CustomSerializer() {
 137                         @Override
 138                         protected void toStream(OutputStream out, Object value)
 139                                         throws IOException {
 140                                 String val = "";
 141                                 if (value != null) {
 142                                         val = ((URL) value).toString();
 143                                 }
 144
 145                                 out.write(val.getBytes("UTF-8"));
 146                         }
 147
 148                         @Override
 149                         protected Object fromStream(InputStream in) throws IOException {
 150                                 String val = IOUtils.readSmallStream(in);
 151                                 if (!val.isEmpty()) {
 152                                         return new URL(val);
 153                                 }
 154
 155                                 return null;
 156                         }
 157
 158                         @Override
 159                         protected String getType() {
 160                                 return "java.net.URL";
 161                         }
 162                 });
 163
 164                 // Images (this is currently the only supported image type by default)
 165                 customTypes.put("be.nikiroo.utils.Image", new CustomSerializer() {
 166                         @Override
 167                         protected void toStream(OutputStream out, Object value)
 168                                         throws IOException {
 169                                 Image img = (Image) value;
 170                                 OutputStream encoded = new Base64OutputStream(out, true);
 171                                 try {
 172                                         InputStream in = img.newInputStream();
 173                                         try {
 174                                                 IOUtils.write(in, encoded);
 175                                         } finally {
 176                                                 in.close();
 177                                         }
 178                                 } finally {
 179                                         encoded.flush();
 180                                         // Cannot close!
 181                                 }
 182                         }
 183
 184                         @Override
 185                         protected String getType() {
 186                                 return "be.nikiroo.utils.Image";
 187                         }
 188
 189                         @Override
 190                         protected Object fromStream(InputStream in) throws IOException {
 191                                 try {
 192                                         // Cannot close it!
 193                                         InputStream decoded = new Base64InputStream(in, false);
 194                                         return new Image(decoded);
 195                                 } catch (IOException e) {
 196                                         throw new UnknownFormatConversionException(e.getMessage());
 197                                 }
 198                         }
 199                 });
 200         }
 201
 202         /**
 203          * Create an empty object of the given type.
 204          *
 205          * @param type
 206          *            the object type (its class name)
 207          *
 208          * @return the new object
 209          *
 210          * @throws ClassNotFoundException
 211          *             if the class cannot be found
 212          * @throws NoSuchMethodException
 213          *             if the given class is not compatible with this code
 214          */
 215         public static Object createObject(String type)
 216                         throws ClassNotFoundException, NoSuchMethodException {
 217
 218                 String desc = null;
 219                 try {
 220                         Class<?> clazz = getClass(type);
 221                         String className = clazz.getName();
 222                         List<Object> args = new ArrayList<Object>();
 223                         List<Class<?>> classes = new ArrayList<Class<?>>();
 224                         Constructor<?> ctor = null;
 225                         if (className.contains("$")) {
 226                                 for (String parentName = className.substring(0,
 227                                                 className.lastIndexOf('$'));; parentName = parentName
 228                                                 .substring(0, parentName.lastIndexOf('$'))) {
 229                                         Object parent = createObject(parentName);
 230                                         args.add(parent);
 231                                         classes.add(parent.getClass());
 232
 233                                         if (!parentName.contains("$")) {
 234                                                 break;
 235                                         }
 236                                 }
 237
 238                                 // Better error description in case there is no empty
 239                                 // constructor:
 240                                 desc = "";
 241                                 String end = "";
 242                                 for (Class<?> parent = clazz; parent != null
 243                                                 && !parent.equals(Object.class); parent = parent
 244                                                 .getSuperclass()) {
 245                                         if (!desc.isEmpty()) {
 246                                                 desc += " [:";
 247                                                 end += "]";
 248                                         }
 249                                         desc += parent;
 250                                 }
 251                                 desc += end;
 252                                 //
 253
 254                                 try {
 255                                         ctor = clazz.getDeclaredConstructor(classes
 256                                                         .toArray(new Class[] {}));
 257                                 } catch (NoSuchMethodException nsme) {
 258                                         // TODO: it seems we do not always need a parameter for each
 259                                         // level, so we currently try "ALL" levels or "FIRST" level
 260                                         // only -> we should check the actual rule and use it
 261                                         ctor = clazz.getDeclaredConstructor(classes.get(0));
 262                                         Object firstParent = args.get(0);
 263                                         args.clear();
 264                                         args.add(firstParent);
 265                                 }
 266                                 desc = null;
 267                         } else {
 268                                 ctor = clazz.getDeclaredConstructor();
 269                         }
 270
 271                         ctor.setAccessible(true);
 272                         return ctor.newInstance(args.toArray());
 273                 } catch (ClassNotFoundException e) {
 274                         throw e;
 275                 } catch (NoSuchMethodException e) {
 276                         if (desc != null) {
 277                                 throw new NoSuchMethodException("Empty constructor not found: "
 278                                                 + desc);
 279                         }
 280                         throw e;
 281                 } catch (Exception e) {
 282                         throw new NoSuchMethodException("Cannot instantiate: " + type);
 283                 }
 284         }
 285
 286         /**
 287          * Insert a custom serialiser that will take precedence over the default one
 288          * or the target class.
 289          *
 290          * @param serializer
 291          *            the custom serialiser
 292          */
 293         static public void addCustomSerializer(CustomSerializer serializer) {
 294                 customTypes.put(serializer.getType(), serializer);
 295         }
 296
 297         /**
 298          * Serialise the given object into this {@link OutputStream}.
 299          * <p>
 300          * <b>Important: </b>If the operation fails (with a
 301          * {@link NotSerializableException}), the {@link StringBuilder} will be
 302          * corrupted (will contain bad, most probably not importable data).
 303          *
 304          * @param out
 305          *            the output {@link OutputStream} to serialise to
 306          * @param o
 307          *            the object to serialise
 308          * @param map
 309          *            the map of already serialised objects (if the given object or
 310          *            one of its descendant is already present in it, only an ID
 311          *            will be serialised)
 312          *
 313          * @throws NotSerializableException
 314          *             if the object cannot be serialised (in this case, the
 315          *             {@link StringBuilder} can contain bad, most probably not
 316          *             importable data)
 317          * @throws IOException
 318          *             in case of I/O errors
 319          */
 320         static void append(OutputStream out, Object o, Map<Integer, Object> map)
 321                         throws NotSerializableException, IOException {
 322
 323                 Field[] fields = new Field[] {};
 324                 String type = "";
 325                 String id = "NULL";
 326
 327                 if (o != null) {
 328                         int hash = System.identityHashCode(o);
 329                         fields = o.getClass().getDeclaredFields();
 330                         type = o.getClass().getCanonicalName();
 331                         if (type == null) {
 332                                 // Anonymous inner classes support
 333                                 type = o.getClass().getName();
 334                         }
 335                         id = Integer.toString(hash);
 336                         if (map.containsKey(hash)) {
 337                                 fields = new Field[] {};
 338                         } else {
 339                                 map.put(hash, o);
 340                         }
 341                 }
 342
 343                 write(out, "{\nREF ");
 344                 write(out, type);
 345                 write(out, "@");
 346                 write(out, id);
 347                 write(out, ":");
 348
 349                 if (!encode(out, o)) { // check if direct value
 350                         try {
 351                                 for (Field field : fields) {
 352                                         field.setAccessible(true);
 353
 354                                         if (field.getName().startsWith("this$")
 355                                                         || field.isSynthetic()
 356                                                         || (field.getModifiers() & Modifier.STATIC) == Modifier.STATIC) {
 357                                                 // Do not keep this links of nested classes
 358                                                 // Do not keep synthetic fields
 359                                                 // Do not keep final fields
 360                                                 continue;
 361                                         }
 362
 363                                         write(out, "\n");
 364                                         write(out, field.getName());
 365                                         write(out, ":");
 366
 367                                         Object value = field.get(o);
 368
 369                                         if (!encode(out, value)) {
 370                                                 write(out, "\n");
 371                                                 append(out, value, map);
 372                                         }
 373                                 }
 374                         } catch (IllegalArgumentException e) {
 375                                 e.printStackTrace(); // should not happen (see
 376                                                                                 // setAccessible)
 377                         } catch (IllegalAccessException e) {
 378                                 e.printStackTrace(); // should not happen (see
 379                                                                                 // setAccessible)
 380                         }
 381
 382                         write(out, "\n}");
 383                 }
 384         }
 385
 386         /**
 387          * Encode the object into the given {@link OutputStream} if possible and if
 388          * supported.
 389          * <p>
 390          * A supported object in this context means an object we can directly
 391          * encode, like an Integer or a String. Custom objects and arrays are also
 392          * considered supported, but <b>compound objects are not supported here</b>.
 393          * <p>
 394          * For compound objects, you should use {@link Exporter}.
 395          *
 396          * @param out
 397          *            the {@link OutputStream} to append to
 398          * @param value
 399          *            the object to encode (can be NULL, which will be encoded)
 400          *
 401          * @return TRUE if success, FALSE if not (the content of the
 402          *         {@link OutputStream} won't be changed in case of failure)
 403          *
 404          * @throws IOException
 405          *             in case of I/O error
 406          */
 407         static boolean encode(OutputStream out, Object value) throws IOException {
 408                 if (value == null) {
 409                         write(out, "NULL");
 410                 } else if (value.getClass().getSimpleName().endsWith("[]")) {
 411                         // Simple name does support [] suffix and do not return NULL for
 412                         // inner anonymous classes
 413                         customTypes.get("[]").encode(out, value);
 414                 } else if (customTypes.containsKey(value.getClass().getCanonicalName())) {
 415                         customTypes.get(value.getClass().getCanonicalName())//
 416                                         .encode(out, value);
 417                 } else if (value instanceof String) {
 418                         encodeString(out, (String) value);
 419                 } else if (value instanceof Boolean) {
 420                         write(out, value);
 421                 } else if (value instanceof Byte) {
 422                         write(out, value);
 423                         write(out, "b");
 424                 } else if (value instanceof Character) {
 425                         encodeString(out, "" + value);
 426                         write(out, "c");
 427                 } else if (value instanceof Short) {
 428                         write(out, value);
 429                         write(out, "s");
 430                 } else if (value instanceof Integer) {
 431                         write(out, value);
 432                 } else if (value instanceof Long) {
 433                         write(out, value);
 434                         write(out, "L");
 435                 } else if (value instanceof Float) {
 436                         write(out, value);
 437                         write(out, "F");
 438                 } else if (value instanceof Double) {
 439                         write(out, value);
 440                         write(out, "d");
 441                 } else if (value instanceof Enum) {
 442                         String type = value.getClass().getCanonicalName();
 443                         write(out, type);
 444                         write(out, ".");
 445                         write(out, ((Enum<?>) value).name());
 446                         write(out, ";");
 447                 } else {
 448                         return false;
 449                 }
 450
 451                 return true;
 452         }
 453
 454         /**
 455          * Decode the data into an equivalent supported source object.
 456          * <p>
 457          * A supported object in this context means an object we can directly
 458          * encode, like an Integer or a String. Custom objects and arrays are also
 459          * considered supported, but <b>compound objects are not supported here</b>.
 460          * <p>
 461          * For compound objects, you should use {@link Importer}.
 462          *
 463          * @param encodedValue
 464          *            the encoded data, cannot be NULL
 465          *
 466          * @return the object (can be NULL for NULL encoded values)
 467          *
 468          * @throws IOException
 469          *             if the content cannot be converted
 470          */
 471         static Object decode(String encodedValue) throws IOException {
 472                 try {
 473                         String cut = "";
 474                         if (encodedValue.length() > 1) {
 475                                 cut = encodedValue.substring(0, encodedValue.length() - 1);
 476                         }
 477
 478                         if (CustomSerializer.isCustom(encodedValue)) {
 479                                 // custom:TYPE_NAME:"content is String-encoded"
 480                                 String type = CustomSerializer.typeOf(encodedValue);
 481                                 if (customTypes.containsKey(type)) {
 482                                         // TODO: we should start with a stream
 483                                         InputStream streamEncodedValue = new ByteArrayInputStream(
 484                                                         encodedValue.getBytes("UTF-8"));
 485                                         try {
 486                                                 return customTypes.get(type).decode(streamEncodedValue);
 487                                         } finally {
 488                                                 streamEncodedValue.close();
 489                                         }
 490                                 }
 491                                 throw new IOException("Unknown custom type: " + type);
 492                         } else if (encodedValue.equals("NULL")
 493                                         || encodedValue.equals("null")) {
 494                                 return null;
 495                         } else if (encodedValue.endsWith("\"")) {
 496                                 return decodeString(encodedValue);
 497                         } else if (encodedValue.equals("true")) {
 498                                 return true;
 499                         } else if (encodedValue.equals("false")) {
 500                                 return false;
 501                         } else if (encodedValue.endsWith("b")) {
 502                                 return Byte.parseByte(cut);
 503                         } else if (encodedValue.endsWith("c")) {
 504                                 return decodeString(cut).charAt(0);
 505                         } else if (encodedValue.endsWith("s")) {
 506                                 return Short.parseShort(cut);
 507                         } else if (encodedValue.endsWith("L")) {
 508                                 return Long.parseLong(cut);
 509                         } else if (encodedValue.endsWith("F")) {
 510                                 return Float.parseFloat(cut);
 511                         } else if (encodedValue.endsWith("d")) {
 512                                 return Double.parseDouble(cut);
 513                         } else if (encodedValue.endsWith(";")) {
 514                                 return decodeEnum(encodedValue);
 515                         } else {
 516                                 return Integer.parseInt(encodedValue);
 517                         }
 518                 } catch (Exception e) {
 519                         if (e instanceof IOException) {
 520                                 throw (IOException) e;
 521                         }
 522                         throw new IOException(e.getMessage(), e);
 523                 }
 524         }
 525
 526         /**
 527          * Write the given {@link String} into the given {@link OutputStream} in
 528          * UTF-8.
 529          *
 530          * @param out
 531          *            the {@link OutputStream}
 532          * @param data
 533          *            the data to write, cannot be NULL
 534          *
 535          * @throws IOException
 536          *             in case of I/O error
 537          */
 538         static void write(OutputStream out, Object data) throws IOException {
 539                 try {
 540                         out.write(data.toString().getBytes("UTF-8"));
 541                 } catch (UnsupportedEncodingException e) {
 542                         // A conforming JVM is required to support UTF-8
 543                         e.printStackTrace();
 544                 }
 545         }
 546
 547         /**
 548          * Return the corresponding class or throw an {@link Exception} if it
 549          * cannot.
 550          *
 551          * @param type
 552          *            the class name to look for
 553          *
 554          * @return the class (will never be NULL)
 555          *
 556          * @throws ClassNotFoundException
 557          *             if the class cannot be found
 558          * @throws NoSuchMethodException
 559          *             if the class cannot be created (usually because it or its
 560          *             enclosing class doesn't have an empty constructor)
 561          */
 562         static private Class<?> getClass(String type)
 563                         throws ClassNotFoundException, NoSuchMethodException {
 564                 Class<?> clazz = null;
 565                 try {
 566                         clazz = Class.forName(type);
 567                 } catch (ClassNotFoundException e) {
 568                         int pos = type.length();
 569                         pos = type.lastIndexOf(".", pos);
 570                         if (pos >= 0) {
 571                                 String parentType = type.substring(0, pos);
 572                                 String nestedType = type.substring(pos + 1);
 573                                 Class<?> javaParent = null;
 574                                 try {
 575                                         javaParent = getClass(parentType);
 576                                         parentType = javaParent.getName();
 577                                         clazz = Class.forName(parentType + "$" + nestedType);
 578                                 } catch (Exception ee) {
 579                                 }
 580
 581                                 if (javaParent == null) {
 582                                         throw new NoSuchMethodException(
 583                                                         "Class not found: "
 584                                                                         + type
 585                                                                         + " (the enclosing class cannot be created: maybe it doesn't have an empty constructor?)");
 586                                 }
 587                         }
 588                 }
 589
 590                 if (clazz == null) {
 591                         throw new ClassNotFoundException("Class not found: " + type);
 592                 }
 593
 594                 return clazz;
 595         }
 596
 597         @SuppressWarnings({ "unchecked", "rawtypes" })
 598         static private Enum<?> decodeEnum(String escaped) {
 599                 // escaped: be.xxx.EnumType.VALUE;
 600                 int pos = escaped.lastIndexOf(".");
 601                 String type = escaped.substring(0, pos);
 602                 String name = escaped.substring(pos + 1, escaped.length() - 1);
 603
 604                 try {
 605                         return Enum.valueOf((Class<Enum>) getClass(type), name);
 606                 } catch (Exception e) {
 607                         throw new UnknownFormatConversionException("Unknown enum: <" + type
 608                                         + "> " + name);
 609                 }
 610         }
 611
 612         // aa bb -> "aa\tbb"
 613         static void encodeString(OutputStream out, String raw) throws IOException {
 614                 // TODO: not. efficient.
 615                 out.write('\"');
 616                 // TODO !! utf-8 required
 617                 for (char car : raw.toCharArray()) {
 618                         encodeString(out, car);
 619                 }
 620                 out.write('\"');
 621         }
 622
 623         // aa bb -> "aa\tbb"
 624         static void encodeString(OutputStream out, InputStream raw)
 625                         throws IOException {
 626                 out.write('\"');
 627                 byte buffer[] = new byte[4096];
 628                 for (int len = 0; (len = raw.read(buffer)) > 0;) {
 629                         for (int i = 0; i < len; i++) {
 630                                 // TODO: not 100% correct, look up howto for UTF-8
 631                                 encodeString(out, (char) buffer[i]);
 632                         }
 633                 }
 634                 out.write('\"');
 635         }
 636
 637         // for encode string, NOT to encode a char by itself!
 638         static void encodeString(OutputStream out, char raw) throws IOException {
 639                 switch (raw) {
 640                 case '\\':
 641                         out.write('\\');
 642                         out.write('\\');
 643                         break;
 644                 case '\r':
 645                         out.write('\\');
 646                         out.write('r');
 647                         break;
 648                 case '\n':
 649                         out.write('\\');
 650                         out.write('n');
 651                         break;
 652                 case '"':
 653                         out.write('\\');
 654                         out.write('\"');
 655                         break;
 656                 default:
 657                         out.write(raw);
 658                         break;
 659                 }
 660         }
 661
 662         // "aa\tbb" -> aa bb
 663         static String decodeString(String escaped) {
 664                 StringBuilder builder = new StringBuilder();
 665
 666                 boolean escaping = false;
 667                 for (char car : escaped.toCharArray()) {
 668                         if (!escaping) {
 669                                 if (car == '\\') {
 670                                         escaping = true;
 671                                 } else {
 672                                         builder.append(car);
 673                                 }
 674                         } else {
 675                                 switch (car) {
 676                                 case '\\':
 677                                         builder.append('\\');
 678                                         break;
 679                                 case 'r':
 680                                         builder.append('\r');
 681                                         break;
 682                                 case 'n':
 683                                         builder.append('\n');
 684                                         break;
 685                                 case '"':
 686                                         builder.append('"');
 687                                         break;
 688                                 }
 689                                 escaping = false;
 690                         }
 691                 }
 692
 693                 return builder.substring(1, builder.length() - 1);
 694         }
 695 }