src/be/nikiroo/utils/serial/SerialUtils.java

   1 package be.nikiroo.utils.serial;
   2
   3 import java.io.ByteArrayInputStream;
   4 import java.io.IOException;
   5 import java.io.InputStream;
   6 import java.io.NotSerializableException;
   7 import java.io.OutputStream;
   8 import java.lang.reflect.Array;
   9 import java.lang.reflect.Constructor;
  10 import java.lang.reflect.Field;
  11 import java.lang.reflect.Modifier;
  12 import java.net.URL;
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.List;
  16 import java.util.Map;
  17 import java.util.UnknownFormatConversionException;
  18
  19 import be.nikiroo.utils.IOUtils;
  20 import be.nikiroo.utils.Image;
  21 import be.nikiroo.utils.StringUtils;
  22 import be.nikiroo.utils.streams.Base64InputStream;
  23 import be.nikiroo.utils.streams.Base64OutputStream;
  24 import be.nikiroo.utils.streams.NextableInputStream;
  25 import be.nikiroo.utils.streams.NextableInputStreamStep;
  26
  27 /**
  28  * Small class to help with serialisation.
  29  * <p>
  30  * Note that we do not support inner classes (but we do support nested classes)
  31  * and all objects require an empty constructor to be deserialised.
  32  * <p>
  33  * It is possible to add support to custom types (both the encoder and the
  34  * decoder will require the custom classes) -- see {@link CustomSerializer}.
  35  * <p>
  36  * Default supported types are:
  37  * <ul>
  38  * <li>NULL (as a null value)</li>
  39  * <li>String</li>
  40  * <li>Boolean</li>
  41  * <li>Byte</li>
  42  * <li>Character</li>
  43  * <li>Short</li>
  44  * <li>Long</li>
  45  * <li>Float</li>
  46  * <li>Double</li>
  47  * <li>Integer</li>
  48  * <li>Enum (any enum whose name and value is known by the caller)</li>
  49  * <li>java.awt.image.BufferedImage (as a {@link CustomSerializer})</li>
  50  * <li>An array of the above (as a {@link CustomSerializer})</li>
  51  * <li>URL</li>
  52  * </ul>
  53  *
  54  * @author niki
  55  */
  56 public class SerialUtils {
  57         private static Map<String, CustomSerializer> customTypes;
  58
  59         static {
  60                 customTypes = new HashMap<String, CustomSerializer>();
  61
  62                 // Array types:
  63                 customTypes.put("[]", new CustomSerializer() {
  64                         @Override
  65                         protected void toStream(OutputStream out, Object value)
  66                                         throws IOException {
  67
  68                                 // TODO: we use \n to separate, and b64 to un-\n
  69                                 // -- but we could use \\n ?
  70                                 String type = value.getClass().getCanonicalName();
  71                                 type = type.substring(0, type.length() - 2); // remove the []
  72
  73                                 write(out, type);
  74                                 try {
  75                                         for (int i = 0; true; i++) {
  76                                                 Object item = Array.get(value, i);
  77
  78                                                 // encode it normally if direct value
  79                                                 write(out, "\r");
  80                                                 if (!SerialUtils.encode(out, item)) {
  81                                                         try {
  82                                                                 // TODO: bad escaping?
  83                                                                 write(out, "B64:");
  84                                                                 OutputStream out64 = new Base64OutputStream(
  85                                                                                 out, true);
  86                                                                 new Exporter(out64).append(item);
  87                                                         } catch (NotSerializableException e) {
  88                                                                 throw new UnknownFormatConversionException(e
  89                                                                                 .getMessage());
  90                                                         }
  91                                                 }
  92                                         }
  93                                 } catch (ArrayIndexOutOfBoundsException e) {
  94                                         // Done.
  95                                 }
  96                         }
  97
  98                         @Override
  99                         protected Object fromStream(InputStream in) throws IOException {
 100                                 NextableInputStream stream = new NextableInputStream(in,
 101                                                 new NextableInputStreamStep('\r'));
 102
 103                                 try {
 104                                         List<Object> list = new ArrayList<Object>();
 105                                         stream.next();
 106                                         String type = IOUtils.readSmallStream(stream);
 107
 108                                         while (stream.next()) {
 109                                                 Object value = new Importer().read(stream).getValue();
 110                                                 list.add(value);
 111                                         }
 112
 113                                         Object array = Array.newInstance(
 114                                                         SerialUtils.getClass(type), list.size());
 115                                         for (int i = 0; i < list.size(); i++) {
 116                                                 Array.set(array, i, list.get(i));
 117                                         }
 118
 119                                         return array;
 120                                 } catch (Exception e) {
 121                                         if (e instanceof IOException) {
 122                                                 throw (IOException) e;
 123                                         }
 124                                         throw new IOException(e.getMessage());
 125                                 }
 126                         }
 127
 128                         @Override
 129                         protected String getType() {
 130                                 return "[]";
 131                         }
 132                 });
 133
 134                 // URL:
 135                 customTypes.put("java.net.URL", new CustomSerializer() {
 136                         @Override
 137                         protected void toStream(OutputStream out, Object value)
 138                                         throws IOException {
 139                                 String val = "";
 140                                 if (value != null) {
 141                                         val = ((URL) value).toString();
 142                                 }
 143
 144                                 out.write(StringUtils.getBytes(val));
 145                         }
 146
 147                         @Override
 148                         protected Object fromStream(InputStream in) throws IOException {
 149                                 String val = IOUtils.readSmallStream(in);
 150                                 if (!val.isEmpty()) {
 151                                         return new URL(val);
 152                                 }
 153
 154                                 return null;
 155                         }
 156
 157                         @Override
 158                         protected String getType() {
 159                                 return "java.net.URL";
 160                         }
 161                 });
 162
 163                 // Images (this is currently the only supported image type by default)
 164                 customTypes.put("be.nikiroo.utils.Image", new CustomSerializer() {
 165                         @Override
 166                         protected void toStream(OutputStream out, Object value)
 167                                         throws IOException {
 168                                 Image img = (Image) value;
 169                                 OutputStream encoded = new Base64OutputStream(out, true);
 170                                 try {
 171                                         InputStream in = img.newInputStream();
 172                                         try {
 173                                                 IOUtils.write(in, encoded);
 174                                         } finally {
 175                                                 in.close();
 176                                         }
 177                                 } finally {
 178                                         encoded.flush();
 179                                         // Cannot close!
 180                                 }
 181                         }
 182
 183                         @Override
 184                         protected String getType() {
 185                                 return "be.nikiroo.utils.Image";
 186                         }
 187
 188                         @Override
 189                         protected Object fromStream(InputStream in) throws IOException {
 190                                 try {
 191                                         // Cannot close it!
 192                                         InputStream decoded = new Base64InputStream(in, false);
 193                                         return new Image(decoded);
 194                                 } catch (IOException e) {
 195                                         throw new UnknownFormatConversionException(e.getMessage());
 196                                 }
 197                         }
 198                 });
 199         }
 200
 201         /**
 202          * Create an empty object of the given type.
 203          *
 204          * @param type
 205          *            the object type (its class name)
 206          *
 207          * @return the new object
 208          *
 209          * @throws ClassNotFoundException
 210          *             if the class cannot be found
 211          * @throws NoSuchMethodException
 212          *             if the given class is not compatible with this code
 213          */
 214         public static Object createObject(String type)
 215                         throws ClassNotFoundException, NoSuchMethodException {
 216
 217                 String desc = null;
 218                 try {
 219                         Class<?> clazz = getClass(type);
 220                         String className = clazz.getName();
 221                         List<Object> args = new ArrayList<Object>();
 222                         List<Class<?>> classes = new ArrayList<Class<?>>();
 223                         Constructor<?> ctor = null;
 224                         if (className.contains("$")) {
 225                                 for (String parentName = className.substring(0,
 226                                                 className.lastIndexOf('$'));; parentName = parentName
 227                                                 .substring(0, parentName.lastIndexOf('$'))) {
 228                                         Object parent = createObject(parentName);
 229                                         args.add(parent);
 230                                         classes.add(parent.getClass());
 231
 232                                         if (!parentName.contains("$")) {
 233                                                 break;
 234                                         }
 235                                 }
 236
 237                                 // Better error description in case there is no empty
 238                                 // constructor:
 239                                 desc = "";
 240                                 String end = "";
 241                                 for (Class<?> parent = clazz; parent != null
 242                                                 && !parent.equals(Object.class); parent = parent
 243                                                 .getSuperclass()) {
 244                                         if (!desc.isEmpty()) {
 245                                                 desc += " [:";
 246                                                 end += "]";
 247                                         }
 248                                         desc += parent;
 249                                 }
 250                                 desc += end;
 251                                 //
 252
 253                                 try {
 254                                         ctor = clazz.getDeclaredConstructor(classes
 255                                                         .toArray(new Class[] {}));
 256                                 } catch (NoSuchMethodException nsme) {
 257                                         // TODO: it seems we do not always need a parameter for each
 258                                         // level, so we currently try "ALL" levels or "FIRST" level
 259                                         // only -> we should check the actual rule and use it
 260                                         ctor = clazz.getDeclaredConstructor(classes.get(0));
 261                                         Object firstParent = args.get(0);
 262                                         args.clear();
 263                                         args.add(firstParent);
 264                                 }
 265                                 desc = null;
 266                         } else {
 267                                 ctor = clazz.getDeclaredConstructor();
 268                         }
 269
 270                         ctor.setAccessible(true);
 271                         return ctor.newInstance(args.toArray());
 272                 } catch (ClassNotFoundException e) {
 273                         throw e;
 274                 } catch (NoSuchMethodException e) {
 275                         if (desc != null) {
 276                                 throw new NoSuchMethodException("Empty constructor not found: "
 277                                                 + desc);
 278                         }
 279                         throw e;
 280                 } catch (Exception e) {
 281                         throw new NoSuchMethodException("Cannot instantiate: " + type);
 282                 }
 283         }
 284
 285         /**
 286          * Insert a custom serialiser that will take precedence over the default one
 287          * or the target class.
 288          *
 289          * @param serializer
 290          *            the custom serialiser
 291          */
 292         static public void addCustomSerializer(CustomSerializer serializer) {
 293                 customTypes.put(serializer.getType(), serializer);
 294         }
 295
 296         /**
 297          * Serialise the given object into this {@link OutputStream}.
 298          * <p>
 299          * <b>Important: </b>If the operation fails (with a
 300          * {@link NotSerializableException}), the {@link StringBuilder} will be
 301          * corrupted (will contain bad, most probably not importable data).
 302          *
 303          * @param out
 304          *            the output {@link OutputStream} to serialise to
 305          * @param o
 306          *            the object to serialise
 307          * @param map
 308          *            the map of already serialised objects (if the given object or
 309          *            one of its descendant is already present in it, only an ID
 310          *            will be serialised)
 311          *
 312          * @throws NotSerializableException
 313          *             if the object cannot be serialised (in this case, the
 314          *             {@link StringBuilder} can contain bad, most probably not
 315          *             importable data)
 316          * @throws IOException
 317          *             in case of I/O errors
 318          */
 319         static void append(OutputStream out, Object o, Map<Integer, Object> map)
 320                         throws NotSerializableException, IOException {
 321
 322                 Field[] fields = new Field[] {};
 323                 String type = "";
 324                 String id = "NULL";
 325
 326                 if (o != null) {
 327                         int hash = System.identityHashCode(o);
 328                         fields = o.getClass().getDeclaredFields();
 329                         type = o.getClass().getCanonicalName();
 330                         if (type == null) {
 331                                 // Anonymous inner classes support
 332                                 type = o.getClass().getName();
 333                         }
 334                         id = Integer.toString(hash);
 335                         if (map.containsKey(hash)) {
 336                                 fields = new Field[] {};
 337                         } else {
 338                                 map.put(hash, o);
 339                         }
 340                 }
 341
 342                 write(out, "{\nREF ");
 343                 write(out, type);
 344                 write(out, "@");
 345                 write(out, id);
 346                 write(out, ":");
 347
 348                 if (!encode(out, o)) { // check if direct value
 349                         try {
 350                                 for (Field field : fields) {
 351                                         field.setAccessible(true);
 352
 353                                         if (field.getName().startsWith("this$")
 354                                                         || field.isSynthetic()
 355                                                         || (field.getModifiers() & Modifier.STATIC) == Modifier.STATIC) {
 356                                                 // Do not keep this links of nested classes
 357                                                 // Do not keep synthetic fields
 358                                                 // Do not keep final fields
 359                                                 continue;
 360                                         }
 361
 362                                         write(out, "\n");
 363                                         write(out, field.getName());
 364                                         write(out, ":");
 365
 366                                         Object value = field.get(o);
 367
 368                                         if (!encode(out, value)) {
 369                                                 write(out, "\n");
 370                                                 append(out, value, map);
 371                                         }
 372                                 }
 373                         } catch (IllegalArgumentException e) {
 374                                 e.printStackTrace(); // should not happen (see
 375                                                                                 // setAccessible)
 376                         } catch (IllegalAccessException e) {
 377                                 e.printStackTrace(); // should not happen (see
 378                                                                                 // setAccessible)
 379                         }
 380
 381                         write(out, "\n}");
 382                 }
 383         }
 384
 385         /**
 386          * Encode the object into the given {@link OutputStream} if possible and if
 387          * supported.
 388          * <p>
 389          * A supported object in this context means an object we can directly
 390          * encode, like an Integer or a String. Custom objects and arrays are also
 391          * considered supported, but <b>compound objects are not supported here</b>.
 392          * <p>
 393          * For compound objects, you should use {@link Exporter}.
 394          *
 395          * @param out
 396          *            the {@link OutputStream} to append to
 397          * @param value
 398          *            the object to encode (can be NULL, which will be encoded)
 399          *
 400          * @return TRUE if success, FALSE if not (the content of the
 401          *         {@link OutputStream} won't be changed in case of failure)
 402          *
 403          * @throws IOException
 404          *             in case of I/O error
 405          */
 406         static boolean encode(OutputStream out, Object value) throws IOException {
 407                 if (value == null) {
 408                         write(out, "NULL");
 409                 } else if (value.getClass().getSimpleName().endsWith("[]")) {
 410                         // Simple name does support [] suffix and do not return NULL for
 411                         // inner anonymous classes
 412                         customTypes.get("[]").encode(out, value);
 413                 } else if (customTypes.containsKey(value.getClass().getCanonicalName())) {
 414                         customTypes.get(value.getClass().getCanonicalName())//
 415                                         .encode(out, value);
 416                 } else if (value instanceof String) {
 417                         encodeString(out, (String) value);
 418                 } else if (value instanceof Boolean) {
 419                         write(out, value);
 420                 } else if (value instanceof Byte) {
 421                         write(out, value);
 422                         write(out, "b");
 423                 } else if (value instanceof Character) {
 424                         encodeString(out, "" + value);
 425                         write(out, "c");
 426                 } else if (value instanceof Short) {
 427                         write(out, value);
 428                         write(out, "s");
 429                 } else if (value instanceof Integer) {
 430                         write(out, value);
 431                 } else if (value instanceof Long) {
 432                         write(out, value);
 433                         write(out, "L");
 434                 } else if (value instanceof Float) {
 435                         write(out, value);
 436                         write(out, "F");
 437                 } else if (value instanceof Double) {
 438                         write(out, value);
 439                         write(out, "d");
 440                 } else if (value instanceof Enum) {
 441                         String type = value.getClass().getCanonicalName();
 442                         write(out, type);
 443                         write(out, ".");
 444                         write(out, ((Enum<?>) value).name());
 445                         write(out, ";");
 446                 } else {
 447                         return false;
 448                 }
 449
 450                 return true;
 451         }
 452
 453         /**
 454          * Decode the data into an equivalent supported source object.
 455          * <p>
 456          * A supported object in this context means an object we can directly
 457          * encode, like an Integer or a String. Custom objects and arrays are also
 458          * considered supported, but <b>compound objects are not supported here</b>.
 459          * <p>
 460          * For compound objects, you should use {@link Importer}.
 461          *
 462          * @param encodedValue
 463          *            the encoded data, cannot be NULL
 464          *
 465          * @return the object (can be NULL for NULL encoded values)
 466          *
 467          * @throws IOException
 468          *             if the content cannot be converted
 469          */
 470         static Object decode(String encodedValue) throws IOException {
 471                 try {
 472                         String cut = "";
 473                         if (encodedValue.length() > 1) {
 474                                 cut = encodedValue.substring(0, encodedValue.length() - 1);
 475                         }
 476
 477                         if (CustomSerializer.isCustom(encodedValue)) {
 478                                 // custom:TYPE_NAME:"content is String-encoded"
 479                                 String type = CustomSerializer.typeOf(encodedValue);
 480                                 if (customTypes.containsKey(type)) {
 481                                         // TODO: we should start with a stream
 482                                         InputStream streamEncodedValue = new ByteArrayInputStream(
 483                                                         StringUtils.getBytes(encodedValue));
 484                                         try {
 485                                                 return customTypes.get(type).decode(streamEncodedValue);
 486                                         } finally {
 487                                                 streamEncodedValue.close();
 488                                         }
 489                                 }
 490                                 throw new IOException("Unknown custom type: " + type);
 491                         } else if (encodedValue.equals("NULL")
 492                                         || encodedValue.equals("null")) {
 493                                 return null;
 494                         } else if (encodedValue.endsWith("\"")) {
 495                                 return decodeString(encodedValue);
 496                         } else if (encodedValue.equals("true")) {
 497                                 return true;
 498                         } else if (encodedValue.equals("false")) {
 499                                 return false;
 500                         } else if (encodedValue.endsWith("b")) {
 501                                 return Byte.parseByte(cut);
 502                         } else if (encodedValue.endsWith("c")) {
 503                                 return decodeString(cut).charAt(0);
 504                         } else if (encodedValue.endsWith("s")) {
 505                                 return Short.parseShort(cut);
 506                         } else if (encodedValue.endsWith("L")) {
 507                                 return Long.parseLong(cut);
 508                         } else if (encodedValue.endsWith("F")) {
 509                                 return Float.parseFloat(cut);
 510                         } else if (encodedValue.endsWith("d")) {
 511                                 return Double.parseDouble(cut);
 512                         } else if (encodedValue.endsWith(";")) {
 513                                 return decodeEnum(encodedValue);
 514                         } else {
 515                                 return Integer.parseInt(encodedValue);
 516                         }
 517                 } catch (Exception e) {
 518                         if (e instanceof IOException) {
 519                                 throw (IOException) e;
 520                         }
 521                         throw new IOException(e.getMessage(), e);
 522                 }
 523         }
 524
 525         /**
 526          * Write the given {@link String} into the given {@link OutputStream} in
 527          * UTF-8.
 528          *
 529          * @param out
 530          *            the {@link OutputStream}
 531          * @param data
 532          *            the data to write, cannot be NULL
 533          *
 534          * @throws IOException
 535          *             in case of I/O error
 536          */
 537         static void write(OutputStream out, Object data) throws IOException {
 538                 out.write(StringUtils.getBytes(data.toString()));
 539         }
 540
 541         /**
 542          * Return the corresponding class or throw an {@link Exception} if it
 543          * cannot.
 544          *
 545          * @param type
 546          *            the class name to look for
 547          *
 548          * @return the class (will never be NULL)
 549          *
 550          * @throws ClassNotFoundException
 551          *             if the class cannot be found
 552          * @throws NoSuchMethodException
 553          *             if the class cannot be created (usually because it or its
 554          *             enclosing class doesn't have an empty constructor)
 555          */
 556         static private Class<?> getClass(String type)
 557                         throws ClassNotFoundException, NoSuchMethodException {
 558                 Class<?> clazz = null;
 559                 try {
 560                         clazz = Class.forName(type);
 561                 } catch (ClassNotFoundException e) {
 562                         int pos = type.length();
 563                         pos = type.lastIndexOf(".", pos);
 564                         if (pos >= 0) {
 565                                 String parentType = type.substring(0, pos);
 566                                 String nestedType = type.substring(pos + 1);
 567                                 Class<?> javaParent = null;
 568                                 try {
 569                                         javaParent = getClass(parentType);
 570                                         parentType = javaParent.getName();
 571                                         clazz = Class.forName(parentType + "$" + nestedType);
 572                                 } catch (Exception ee) {
 573                                 }
 574
 575                                 if (javaParent == null) {
 576                                         throw new NoSuchMethodException(
 577                                                         "Class not found: "
 578                                                                         + type
 579                                                                         + " (the enclosing class cannot be created: maybe it doesn't have an empty constructor?)");
 580                                 }
 581                         }
 582                 }
 583
 584                 if (clazz == null) {
 585                         throw new ClassNotFoundException("Class not found: " + type);
 586                 }
 587
 588                 return clazz;
 589         }
 590
 591         @SuppressWarnings({ "unchecked", "rawtypes" })
 592         static private Enum<?> decodeEnum(String escaped) {
 593                 // escaped: be.xxx.EnumType.VALUE;
 594                 int pos = escaped.lastIndexOf(".");
 595                 String type = escaped.substring(0, pos);
 596                 String name = escaped.substring(pos + 1, escaped.length() - 1);
 597
 598                 try {
 599                         return Enum.valueOf((Class<Enum>) getClass(type), name);
 600                 } catch (Exception e) {
 601                         throw new UnknownFormatConversionException("Unknown enum: <" + type
 602                                         + "> " + name);
 603                 }
 604         }
 605
 606         // aa bb -> "aa\tbb"
 607         static void encodeString(OutputStream out, String raw) throws IOException {
 608                 // TODO: not. efficient.
 609                 out.write('\"');
 610                 // TODO !! utf-8 required
 611                 for (char car : raw.toCharArray()) {
 612                         encodeString(out, car);
 613                 }
 614                 out.write('\"');
 615         }
 616
 617         // aa bb -> "aa\tbb"
 618         static void encodeString(OutputStream out, InputStream raw)
 619                         throws IOException {
 620                 out.write('\"');
 621                 byte buffer[] = new byte[4096];
 622                 for (int len = 0; (len = raw.read(buffer)) > 0;) {
 623                         for (int i = 0; i < len; i++) {
 624                                 // TODO: not 100% correct, look up howto for UTF-8
 625                                 encodeString(out, (char) buffer[i]);
 626                         }
 627                 }
 628                 out.write('\"');
 629         }
 630
 631         // for encode string, NOT to encode a char by itself!
 632         static void encodeString(OutputStream out, char raw) throws IOException {
 633                 switch (raw) {
 634                 case '\\':
 635                         out.write('\\');
 636                         out.write('\\');
 637                         break;
 638                 case '\r':
 639                         out.write('\\');
 640                         out.write('r');
 641                         break;
 642                 case '\n':
 643                         out.write('\\');
 644                         out.write('n');
 645                         break;
 646                 case '"':
 647                         out.write('\\');
 648                         out.write('\"');
 649                         break;
 650                 default:
 651                         out.write(raw);
 652                         break;
 653                 }
 654         }
 655
 656         // "aa\tbb" -> aa bb
 657         static String decodeString(String escaped) {
 658                 StringBuilder builder = new StringBuilder();
 659
 660                 boolean escaping = false;
 661                 for (char car : escaped.toCharArray()) {
 662                         if (!escaping) {
 663                                 if (car == '\\') {
 664                                         escaping = true;
 665                                 } else {
 666                                         builder.append(car);
 667                                 }
 668                         } else {
 669                                 switch (car) {
 670                                 case '\\':
 671                                         builder.append('\\');
 672                                         break;
 673                                 case 'r':
 674                                         builder.append('\r');
 675                                         break;
 676                                 case 'n':
 677                                         builder.append('\n');
 678                                         break;
 679                                 case '"':
 680                                         builder.append('"');
 681                                         break;
 682                                 }
 683                                 escaping = false;
 684                         }
 685                 }
 686
 687                 return builder.substring(1, builder.length() - 1);
 688         }
 689 }