src/be/nikiroo/utils/serial/SerialUtils.java

   1 package be.nikiroo.utils.serial;
   2
   3 import java.io.IOException;
   4 import java.io.InputStream;
   5 import java.io.NotSerializableException;
   6 import java.io.OutputStream;
   7 import java.io.UnsupportedEncodingException;
   8 import java.lang.reflect.Array;
   9 import java.lang.reflect.Constructor;
  10 import java.lang.reflect.Field;
  11 import java.lang.reflect.Modifier;
  12 import java.net.URL;
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.List;
  16 import java.util.Map;
  17 import java.util.UnknownFormatConversionException;
  18
  19 import be.nikiroo.utils.IOUtils;
  20 import be.nikiroo.utils.Image;
  21 import be.nikiroo.utils.NextableInputStream;
  22 import be.nikiroo.utils.NextableInputStreamStep;
  23 import be.nikiroo.utils.StringUtils;
  24
  25 /**
  26  * Small class to help with serialisation.
  27  * <p>
  28  * Note that we do not support inner classes (but we do support nested classes)
  29  * and all objects require an empty constructor to be deserialised.
  30  * <p>
  31  * It is possible to add support to custom types (both the encoder and the
  32  * decoder will require the custom classes) -- see {@link CustomSerializer}.
  33  * <p>
  34  * Default supported types are:
  35  * <ul>
  36  * <li>NULL (as a null value)</li>
  37  * <li>String</li>
  38  * <li>Boolean</li>
  39  * <li>Byte</li>
  40  * <li>Character</li>
  41  * <li>Short</li>
  42  * <li>Long</li>
  43  * <li>Float</li>
  44  * <li>Double</li>
  45  * <li>Integer</li>
  46  * <li>Enum (any enum whose name and value is known by the caller)</li>
  47  * <li>java.awt.image.BufferedImage (as a {@link CustomSerializer})</li>
  48  * <li>An array of the above (as a {@link CustomSerializer})</li>
  49  * <li>URL</li>
  50  * </ul>
  51  *
  52  * @author niki
  53  */
  54 public class SerialUtils {
  55         private static Map<String, CustomSerializer> customTypes;
  56
  57         static {
  58                 customTypes = new HashMap<String, CustomSerializer>();
  59
  60                 // Array types:
  61                 customTypes.put("[]", new CustomSerializer() {
  62                         @Override
  63                         protected void toStream(OutputStream out, Object value)
  64                                         throws IOException {
  65                                 // TODO: we use \n to separate, and b64 to un-\n -- but we could
  66                                 // use \\n ?
  67                                 String type = value.getClass().getCanonicalName();
  68                                 type = type.substring(0, type.length() - 2); // remove the []
  69
  70                                 write(out, type);
  71                                 write(out, "\n");
  72                                 try {
  73                                         for (int i = 0; true; i++) {
  74                                                 Object item = Array.get(value, i);
  75                                                 // encode it normally if direct value
  76                                                 if (!SerialUtils.encode(out, item)) {
  77                                                         try {
  78                                                                 // TODO: use ZIP: if not?
  79                                                                 new Exporter(out).append(item);
  80                                                         } catch (NotSerializableException e) {
  81                                                                 throw new UnknownFormatConversionException(e
  82                                                                                 .getMessage());
  83                                                         }
  84                                                 }
  85                                                 write(out, "\n");
  86                                         }
  87                                 } catch (ArrayIndexOutOfBoundsException e) {
  88                                         // Done.
  89                                 }
  90                         }
  91
  92                         @Override
  93                         protected Object fromStream(InputStream in) throws IOException {
  94                                 NextableInputStream stream = new NextableInputStream(in,
  95                                                 new NextableInputStreamStep('\n'));
  96
  97                                 try {
  98                                         List<Object> list = new ArrayList<Object>();
  99                                         stream.next();
 100                                         String type = IOUtils.readSmallStream(stream);
 101
 102                                         while (stream.next()) {
 103                                                 Object value = new Importer().read(stream).getValue();
 104                                                 list.add(value);
 105                                         }
 106
 107                                         Object array = Array.newInstance(
 108                                                         SerialUtils.getClass(type), list.size());
 109                                         for (int i = 0; i < list.size(); i++) {
 110                                                 Array.set(array, i, list.get(i));
 111                                         }
 112
 113                                         return array;
 114                                 } catch (Exception e) {
 115                                         if (e instanceof IOException) {
 116                                                 throw (IOException) e;
 117                                         }
 118                                         throw new IOException(e.getMessage());
 119                                 }
 120                         }
 121
 122                         @Override
 123                         protected String getType() {
 124                                 return "[]";
 125                         }
 126                 });
 127
 128                 // URL:
 129                 customTypes.put("java.net.URL", new CustomSerializer() {
 130
 131                         @Override
 132                         protected void toStream(OutputStream out, Object value)
 133                                         throws IOException {
 134                                 String val = "";
 135                                 if (value != null) {
 136                                         val = ((URL) value).toString();
 137                                 }
 138
 139                                 out.write(val.getBytes("UTF-8"));
 140                         }
 141
 142                         @Override
 143                         protected Object fromStream(InputStream in) throws IOException {
 144                                 String val = IOUtils.readSmallStream(in);
 145                                 if (!val.isEmpty()) {
 146                                         return new URL(val);
 147                                 }
 148
 149                                 return null;
 150                         }
 151
 152                         @Override
 153                         protected String getType() {
 154                                 return "java.net.URL";
 155                         }
 156                 });
 157
 158                 // Images (this is currently the only supported image type by default)
 159                 customTypes.put("be.nikiroo.utils.Image", new CustomSerializer() {
 160                         @Override
 161                         protected void toStream(OutputStream out, Object value)
 162                                         throws IOException {
 163                                 Image img = (Image) value;
 164                                 OutputStream encoded = StringUtils.base64(out, false, false);
 165                                 try {
 166                                         InputStream in = img.newInputStream();
 167                                         try {
 168                                                 IOUtils.write(in, encoded);
 169                                         } finally {
 170                                                 in.close();
 171                                         }
 172                                 } finally {
 173                                         encoded.close();
 174                                 }
 175                         }
 176
 177                         @Override
 178                         protected String getType() {
 179                                 return "be.nikiroo.utils.Image";
 180                         }
 181
 182                         @Override
 183                         protected Object fromStream(InputStream in) throws IOException {
 184                                 try {
 185                                         return new Image(in);
 186                                 } catch (IOException e) {
 187                                         throw new UnknownFormatConversionException(e.getMessage());
 188                                 }
 189                         }
 190                 });
 191         }
 192
 193         /**
 194          * Create an empty object of the given type.
 195          *
 196          * @param type
 197          *            the object type (its class name)
 198          *
 199          * @return the new object
 200          *
 201          * @throws ClassNotFoundException
 202          *             if the class cannot be found
 203          * @throws NoSuchMethodException
 204          *             if the given class is not compatible with this code
 205          */
 206         public static Object createObject(String type)
 207                         throws ClassNotFoundException, NoSuchMethodException {
 208
 209                 String desc = null;
 210                 try {
 211                         Class<?> clazz = getClass(type);
 212                         String className = clazz.getName();
 213                         List<Object> args = new ArrayList<Object>();
 214                         List<Class<?>> classes = new ArrayList<Class<?>>();
 215                         Constructor<?> ctor = null;
 216                         if (className.contains("$")) {
 217                                 for (String parentName = className.substring(0,
 218                                                 className.lastIndexOf('$'));; parentName = parentName
 219                                                 .substring(0, parentName.lastIndexOf('$'))) {
 220                                         Object parent = createObject(parentName);
 221                                         args.add(parent);
 222                                         classes.add(parent.getClass());
 223
 224                                         if (!parentName.contains("$")) {
 225                                                 break;
 226                                         }
 227                                 }
 228
 229                                 // Better error description in case there is no empty
 230                                 // constructor:
 231                                 desc = "";
 232                                 String end = "";
 233                                 for (Class<?> parent = clazz; parent != null
 234                                                 && !parent.equals(Object.class); parent = parent
 235                                                 .getSuperclass()) {
 236                                         if (!desc.isEmpty()) {
 237                                                 desc += " [:";
 238                                                 end += "]";
 239                                         }
 240                                         desc += parent;
 241                                 }
 242                                 desc += end;
 243                                 //
 244
 245                                 try {
 246                                         ctor = clazz.getDeclaredConstructor(classes
 247                                                         .toArray(new Class[] {}));
 248                                 } catch (NoSuchMethodException nsme) {
 249                                         // TODO: it seems e do not always need a parameter for each
 250                                         // level, so we currently try "ALL" levels or "FIRST" level
 251                                         // only -> we should check the actual rule and use it
 252                                         ctor = clazz.getDeclaredConstructor(classes.get(0));
 253                                         Object firstParent = args.get(0);
 254                                         args.clear();
 255                                         args.add(firstParent);
 256                                 }
 257                                 desc = null;
 258                         } else {
 259                                 ctor = clazz.getDeclaredConstructor();
 260                         }
 261
 262                         ctor.setAccessible(true);
 263                         return ctor.newInstance(args.toArray());
 264                 } catch (ClassNotFoundException e) {
 265                         throw e;
 266                 } catch (NoSuchMethodException e) {
 267                         if (desc != null) {
 268                                 throw new NoSuchMethodException("Empty constructor not found: "
 269                                                 + desc);
 270                         }
 271                         throw e;
 272                 } catch (Exception e) {
 273                         throw new NoSuchMethodException("Cannot instantiate: " + type);
 274                 }
 275         }
 276
 277         /**
 278          * Insert a custom serialiser that will take precedence over the default one
 279          * or the target class.
 280          *
 281          * @param serializer
 282          *            the custom serialiser
 283          */
 284         static public void addCustomSerializer(CustomSerializer serializer) {
 285                 customTypes.put(serializer.getType(), serializer);
 286         }
 287
 288         /**
 289          * Serialise the given object into this {@link OutputStream}.
 290          * <p>
 291          * <b>Important: </b>If the operation fails (with a
 292          * {@link NotSerializableException}), the {@link StringBuilder} will be
 293          * corrupted (will contain bad, most probably not importable data).
 294          *
 295          * @param out
 296          *            the output {@link OutputStream} to serialise to
 297          * @param o
 298          *            the object to serialise
 299          * @param map
 300          *            the map of already serialised objects (if the given object or
 301          *            one of its descendant is already present in it, only an ID
 302          *            will be serialised)
 303          *
 304          * @throws NotSerializableException
 305          *             if the object cannot be serialised (in this case, the
 306          *             {@link StringBuilder} can contain bad, most probably not
 307          *             importable data)
 308          * @throws IOException
 309          *             in case of I/O errors
 310          */
 311         static void append(OutputStream out, Object o, Map<Integer, Object> map)
 312                         throws NotSerializableException, IOException {
 313
 314                 Field[] fields = new Field[] {};
 315                 String type = "";
 316                 String id = "NULL";
 317
 318                 if (o != null) {
 319                         int hash = System.identityHashCode(o);
 320                         fields = o.getClass().getDeclaredFields();
 321                         type = o.getClass().getCanonicalName();
 322                         if (type == null) {
 323                                 // Anonymous inner classes support
 324                                 type = o.getClass().getName();
 325                         }
 326                         id = Integer.toString(hash);
 327                         if (map.containsKey(hash)) {
 328                                 fields = new Field[] {};
 329                         } else {
 330                                 map.put(hash, o);
 331                         }
 332                 }
 333
 334                 write(out, "{\nREF ");
 335                 write(out, type);
 336                 write(out, "@");
 337                 write(out, id);
 338                 write(out, ":");
 339
 340                 if (!encode(out, o)) { // check if direct value
 341                         try {
 342                                 for (Field field : fields) {
 343                                         field.setAccessible(true);
 344
 345                                         if (field.getName().startsWith("this$")
 346                                                         || field.isSynthetic()
 347                                                         || (field.getModifiers() & Modifier.STATIC) == Modifier.STATIC) {
 348                                                 // Do not keep this links of nested classes
 349                                                 // Do not keep synthetic fields
 350                                                 // Do not keep final fields
 351                                                 continue;
 352                                         }
 353
 354                                         write(out, "\n");
 355                                         write(out, field.getName());
 356                                         write(out, ":");
 357
 358                                         Object value = field.get(o);
 359
 360                                         if (!encode(out, value)) {
 361                                                 write(out, "\n");
 362                                                 append(out, value, map);
 363                                         }
 364                                 }
 365                         } catch (IllegalArgumentException e) {
 366                                 e.printStackTrace(); // should not happen (see
 367                                                                                 // setAccessible)
 368                         } catch (IllegalAccessException e) {
 369                                 e.printStackTrace(); // should not happen (see
 370                                                                                 // setAccessible)
 371                         }
 372                 }
 373                 write(out, "\n}");
 374         }
 375
 376         /**
 377          * Encode the object into the given {@link OutputStream} if possible and if
 378          * supported.
 379          * <p>
 380          * A supported object in this context means an object we can directly
 381          * encode, like an Integer or a String. Custom objects and arrays are also
 382          * considered supported, but <b>compound objects are not supported here</b>.
 383          * <p>
 384          * For compound objects, you should use {@link Exporter}.
 385          *
 386          * @param out
 387          *            the {@link OutputStream} to append to
 388          * @param value
 389          *            the object to encode (can be NULL, which will be encoded)
 390          *
 391          * @return TRUE if success, FALSE if not (the content of the
 392          *         {@link OutputStream} won't be changed in case of failure)
 393          *
 394          * @throws IOException
 395          *             in case of I/O error
 396          */
 397         static boolean encode(OutputStream out, Object value) throws IOException {
 398                 if (value == null) {
 399                         write(out, "NULL");
 400                 } else if (value.getClass().getSimpleName().endsWith("[]")) {
 401                         // Simple name does support [] suffix and do not return NULL for
 402                         // inner anonymous classes
 403                         return customTypes.get("[]").encode(out, value);
 404                 } else if (customTypes.containsKey(value.getClass().getCanonicalName())) {
 405                         return customTypes.get(value.getClass().getCanonicalName())//
 406                                         .encode(out, value);
 407                 } else if (value instanceof String) {
 408                         encodeString(out, (String) value);
 409                 } else if (value instanceof Boolean) {
 410                         write(out, value);
 411                 } else if (value instanceof Byte) {
 412                         write(out, value);
 413                         write(out, "b");
 414                 } else if (value instanceof Character) {
 415                         encodeString(out, "" + value);
 416                         write(out, "c");
 417                 } else if (value instanceof Short) {
 418                         write(out, value);
 419                         write(out, "s");
 420                 } else if (value instanceof Integer) {
 421                         write(out, value);
 422                 } else if (value instanceof Long) {
 423                         write(out, value);
 424                         write(out, "L");
 425                 } else if (value instanceof Float) {
 426                         write(out, value);
 427                         write(out, "F");
 428                 } else if (value instanceof Double) {
 429                         write(out, value);
 430                         write(out, "d");
 431                 } else if (value instanceof Enum) {
 432                         String type = value.getClass().getCanonicalName();
 433                         write(out, type);
 434                         write(out, ".");
 435                         write(out, ((Enum<?>) value).name());
 436                         write(out, ";");
 437                 } else {
 438                         return false;
 439                 }
 440
 441                 return true;
 442         }
 443
 444         /**
 445          * Decode the data into an equivalent supported source object.
 446          * <p>
 447          * A supported object in this context means an object we can directly
 448          * encode, like an Integer or a String. Custom objects and arrays are also
 449          * considered supported, but <b>compound objects are not supported here</b>.
 450          * <p>
 451          * For compound objects, you should use {@link Importer}.
 452          *
 453          * @param encodedValue
 454          *            the encoded data, cannot be NULL
 455          *
 456          * @return the object (can be NULL for NULL encoded values)
 457          *
 458          * @throws IOException
 459          *             if the content cannot be converted
 460          */
 461         static Object decode(String encodedValue) throws IOException {
 462                 try {
 463                         String cut = "";
 464                         if (encodedValue.length() > 1) {
 465                                 cut = encodedValue.substring(0, encodedValue.length() - 1);
 466                         }
 467
 468                         if (CustomSerializer.isCustom(encodedValue)) {
 469                                 // custom:TYPE_NAME:"content is String-encoded"
 470                                 String type = CustomSerializer.typeOf(encodedValue);
 471                                 if (customTypes.containsKey(type)) {
 472                                         return customTypes.get(type).decode(encodedValue);
 473                                 }
 474                                 throw new IOException("Unknown custom type: " + type);
 475                         } else if (encodedValue.equals("NULL")
 476                                         || encodedValue.equals("null")) {
 477                                 return null;
 478                         } else if (encodedValue.endsWith("\"")) {
 479                                 return decodeString(encodedValue);
 480                         } else if (encodedValue.equals("true")) {
 481                                 return true;
 482                         } else if (encodedValue.equals("false")) {
 483                                 return false;
 484                         } else if (encodedValue.endsWith("b")) {
 485                                 return Byte.parseByte(cut);
 486                         } else if (encodedValue.endsWith("c")) {
 487                                 return decodeString(cut).charAt(0);
 488                         } else if (encodedValue.endsWith("s")) {
 489                                 return Short.parseShort(cut);
 490                         } else if (encodedValue.endsWith("L")) {
 491                                 return Long.parseLong(cut);
 492                         } else if (encodedValue.endsWith("F")) {
 493                                 return Float.parseFloat(cut);
 494                         } else if (encodedValue.endsWith("d")) {
 495                                 return Double.parseDouble(cut);
 496                         } else if (encodedValue.endsWith(";")) {
 497                                 return decodeEnum(encodedValue);
 498                         } else {
 499                                 return Integer.parseInt(encodedValue);
 500                         }
 501                 } catch (Exception e) {
 502                         if (e instanceof IOException) {
 503                                 throw (IOException) e;
 504                         }
 505                         throw new IOException(e.getMessage());
 506                 }
 507         }
 508
 509         /**
 510          * Write the given {@link String} into the given {@link OutputStream} in
 511          * UTF-8.
 512          *
 513          * @param out
 514          *            the {@link OutputStream}
 515          * @param data
 516          *            the data to write, cannot be NULL
 517          *
 518          * @throws IOException
 519          *             in case of I/O error
 520          */
 521         static void write(OutputStream out, Object data) throws IOException {
 522                 try {
 523                         out.write(data.toString().getBytes("UTF-8"));
 524                 } catch (UnsupportedEncodingException e) {
 525                         // A conforming JVM is required to support UTF-8
 526                         e.printStackTrace();
 527                 }
 528         }
 529
 530         /**
 531          * Return the corresponding class or throw an {@link Exception} if it
 532          * cannot.
 533          *
 534          * @param type
 535          *            the class name to look for
 536          *
 537          * @return the class (will never be NULL)
 538          *
 539          * @throws ClassNotFoundException
 540          *             if the class cannot be found
 541          * @throws NoSuchMethodException
 542          *             if the class cannot be created (usually because it or its
 543          *             enclosing class doesn't have an empty constructor)
 544          */
 545         static private Class<?> getClass(String type)
 546                         throws ClassNotFoundException, NoSuchMethodException {
 547                 Class<?> clazz = null;
 548                 try {
 549                         clazz = Class.forName(type);
 550                 } catch (ClassNotFoundException e) {
 551                         int pos = type.length();
 552                         pos = type.lastIndexOf(".", pos);
 553                         if (pos >= 0) {
 554                                 String parentType = type.substring(0, pos);
 555                                 String nestedType = type.substring(pos + 1);
 556                                 Class<?> javaParent = null;
 557                                 try {
 558                                         javaParent = getClass(parentType);
 559                                         parentType = javaParent.getName();
 560                                         clazz = Class.forName(parentType + "$" + nestedType);
 561                                 } catch (Exception ee) {
 562                                 }
 563
 564                                 if (javaParent == null) {
 565                                         throw new NoSuchMethodException(
 566                                                         "Class not found: "
 567                                                                         + type
 568                                                                         + " (the enclosing class cannot be created: maybe it doesn't have an empty constructor?)");
 569                                 }
 570                         }
 571                 }
 572
 573                 if (clazz == null) {
 574                         throw new ClassNotFoundException("Class not found: " + type);
 575                 }
 576
 577                 return clazz;
 578         }
 579
 580         @SuppressWarnings({ "unchecked", "rawtypes" })
 581         static private Enum<?> decodeEnum(String escaped) {
 582                 // escaped: be.xxx.EnumType.VALUE;
 583                 int pos = escaped.lastIndexOf(".");
 584                 String type = escaped.substring(0, pos);
 585                 String name = escaped.substring(pos + 1, escaped.length() - 1);
 586
 587                 try {
 588                         return Enum.valueOf((Class<Enum>) getClass(type), name);
 589                 } catch (Exception e) {
 590                         throw new UnknownFormatConversionException("Unknown enum: <" + type
 591                                         + "> " + name);
 592                 }
 593         }
 594
 595         // aa bb -> "aa\tbb"
 596         static void encodeString(OutputStream out, String raw) throws IOException {
 597                 out.write('\"');
 598                 // TODO !! utf-8 required
 599                 for (char car : raw.toCharArray()) {
 600                         encodeString(out, car);
 601                 }
 602                 out.write('\"');
 603         }
 604
 605         // aa bb -> "aa\tbb"
 606         static void encodeString(OutputStream out, InputStream raw)
 607                         throws IOException {
 608                 out.write('\"');
 609                 byte buffer[] = new byte[4069];
 610                 for (int len = 0; (len = raw.read(buffer)) > 0;) {
 611                         for (int i = 0; i < len; i++) {
 612                                 // TODO: not 100% correct, look up howto for UTF-8
 613                                 encodeString(out, (char) buffer[i]);
 614                         }
 615                 }
 616                 out.write('\"');
 617         }
 618
 619         // for encode string, NOT to encode a char by itself!
 620         static void encodeString(OutputStream out, char raw) throws IOException {
 621                 switch (raw) {
 622                 case '\\':
 623                         out.write('\\');
 624                         out.write('\\');
 625                         break;
 626                 case '\r':
 627                         out.write('\\');
 628                         out.write('r');
 629                         break;
 630                 case '\n':
 631                         out.write('\\');
 632                         out.write('n');
 633                         break;
 634                 case '"':
 635                         out.write('\\');
 636                         out.write('\"');
 637                         break;
 638                 default:
 639                         out.write(raw);
 640                         break;
 641                 }
 642         }
 643
 644         // "aa\tbb" -> aa bb
 645         static String decodeString(String escaped) {
 646                 StringBuilder builder = new StringBuilder();
 647
 648                 boolean escaping = false;
 649                 for (char car : escaped.toCharArray()) {
 650                         if (!escaping) {
 651                                 if (car == '\\') {
 652                                         escaping = true;
 653                                 } else {
 654                                         builder.append(car);
 655                                 }
 656                         } else {
 657                                 switch (car) {
 658                                 case '\\':
 659                                         builder.append('\\');
 660                                         break;
 661                                 case 'r':
 662                                         builder.append('\r');
 663                                         break;
 664                                 case 'n':
 665                                         builder.append('\n');
 666                                         break;
 667                                 case '"':
 668                                         builder.append('"');
 669                                         break;
 670                                 }
 671                                 escaping = false;
 672                         }
 673                 }
 674
 675                 return builder.substring(1, builder.length() - 1);
 676         }
 677 }