now mostly streamified!
[nikiroo-utils.git] / src / be / nikiroo / utils / serial / SerialUtils.java
1 package be.nikiroo.utils.serial;
2
3 import java.io.IOException;
4 import java.io.InputStream;
5 import java.io.NotSerializableException;
6 import java.io.OutputStream;
7 import java.lang.reflect.Array;
8 import java.lang.reflect.Constructor;
9 import java.lang.reflect.Field;
10 import java.lang.reflect.Modifier;
11 import java.net.URL;
12 import java.util.ArrayList;
13 import java.util.HashMap;
14 import java.util.List;
15 import java.util.Map;
16 import java.util.UnknownFormatConversionException;
17
18 import be.nikiroo.utils.IOUtils;
19 import be.nikiroo.utils.Image;
20 import be.nikiroo.utils.StringUtils;
21 import be.nikiroo.utils.streams.Base64InputStream;
22 import be.nikiroo.utils.streams.Base64OutputStream;
23 import be.nikiroo.utils.streams.BufferedInputStream;
24 import be.nikiroo.utils.streams.NextableInputStream;
25 import be.nikiroo.utils.streams.NextableInputStreamStep;
26
27 /**
28 * Small class to help with serialisation.
29 * <p>
30 * Note that we do not support inner classes (but we do support nested classes)
31 * and all objects require an empty constructor to be deserialised.
32 * <p>
33 * It is possible to add support to custom types (both the encoder and the
34 * decoder will require the custom classes) -- see {@link CustomSerializer}.
35 * <p>
36 * Default supported types are:
37 * <ul>
38 * <li>NULL (as a null value)</li>
39 * <li>String</li>
40 * <li>Boolean</li>
41 * <li>Byte</li>
42 * <li>Character</li>
43 * <li>Short</li>
44 * <li>Long</li>
45 * <li>Float</li>
46 * <li>Double</li>
47 * <li>Integer</li>
48 * <li>Enum (any enum whose name and value is known by the caller)</li>
49 * <li>java.awt.image.BufferedImage (as a {@link CustomSerializer})</li>
50 * <li>An array of the above (as a {@link CustomSerializer})</li>
51 * <li>URL</li>
52 * </ul>
53 *
54 * @author niki
55 */
56 public class SerialUtils {
57 private static Map<String, CustomSerializer> customTypes;
58
59 static {
60 customTypes = new HashMap<String, CustomSerializer>();
61
62 // Array types:
63 customTypes.put("[]", new CustomSerializer() {
64 @Override
65 protected void toStream(OutputStream out, Object value)
66 throws IOException {
67
68 // TODO: we use \n to separate, and b64 to un-\n
69 // -- but we could use \\n ?
70 String type = value.getClass().getCanonicalName();
71 type = type.substring(0, type.length() - 2); // remove the []
72
73 write(out, type);
74 try {
75 for (int i = 0; true; i++) {
76 Object item = Array.get(value, i);
77
78 // encode it normally if direct value
79 write(out, "\r");
80 if (!SerialUtils.encode(out, item)) {
81 try {
82 // TODO: bad escaping?
83 write(out, "B64:");
84 OutputStream out64 = new Base64OutputStream(
85 out, true);
86 new Exporter(out64).append(item);
87 } catch (NotSerializableException e) {
88 throw new UnknownFormatConversionException(e
89 .getMessage());
90 }
91 }
92 }
93 } catch (ArrayIndexOutOfBoundsException e) {
94 // Done.
95 }
96 }
97
98 @Override
99 protected Object fromStream(InputStream in) throws IOException {
100 NextableInputStream stream = new NextableInputStream(in,
101 new NextableInputStreamStep('\r'));
102
103 try {
104 List<Object> list = new ArrayList<Object>();
105 stream.next();
106 String type = IOUtils.readSmallStream(stream);
107
108 while (stream.next()) {
109 Object value = new Importer().read(stream).getValue();
110 list.add(value);
111 }
112
113 Object array = Array.newInstance(
114 SerialUtils.getClass(type), list.size());
115 for (int i = 0; i < list.size(); i++) {
116 Array.set(array, i, list.get(i));
117 }
118
119 return array;
120 } catch (Exception e) {
121 if (e instanceof IOException) {
122 throw (IOException) e;
123 }
124 throw new IOException(e.getMessage());
125 }
126 }
127
128 @Override
129 protected String getType() {
130 return "[]";
131 }
132 });
133
134 // URL:
135 customTypes.put("java.net.URL", new CustomSerializer() {
136 @Override
137 protected void toStream(OutputStream out, Object value)
138 throws IOException {
139 String val = "";
140 if (value != null) {
141 val = ((URL) value).toString();
142 }
143
144 out.write(StringUtils.getBytes(val));
145 }
146
147 @Override
148 protected Object fromStream(InputStream in) throws IOException {
149 String val = IOUtils.readSmallStream(in);
150 if (!val.isEmpty()) {
151 return new URL(val);
152 }
153
154 return null;
155 }
156
157 @Override
158 protected String getType() {
159 return "java.net.URL";
160 }
161 });
162
163 // Images (this is currently the only supported image type by default)
164 customTypes.put("be.nikiroo.utils.Image", new CustomSerializer() {
165 @Override
166 protected void toStream(OutputStream out, Object value)
167 throws IOException {
168 Image img = (Image) value;
169 OutputStream encoded = new Base64OutputStream(out, true);
170 try {
171 InputStream in = img.newInputStream();
172 try {
173 IOUtils.write(in, encoded);
174 } finally {
175 in.close();
176 }
177 } finally {
178 encoded.flush();
179 // Cannot close!
180 }
181 }
182
183 @Override
184 protected String getType() {
185 return "be.nikiroo.utils.Image";
186 }
187
188 @Override
189 protected Object fromStream(InputStream in) throws IOException {
190 try {
191 // Cannot close it!
192 InputStream decoded = new Base64InputStream(in, false);
193 return new Image(decoded);
194 } catch (IOException e) {
195 throw new UnknownFormatConversionException(e.getMessage());
196 }
197 }
198 });
199 }
200
201 /**
202 * Create an empty object of the given type.
203 *
204 * @param type
205 * the object type (its class name)
206 *
207 * @return the new object
208 *
209 * @throws ClassNotFoundException
210 * if the class cannot be found
211 * @throws NoSuchMethodException
212 * if the given class is not compatible with this code
213 */
214 public static Object createObject(String type)
215 throws ClassNotFoundException, NoSuchMethodException {
216
217 String desc = null;
218 try {
219 Class<?> clazz = getClass(type);
220 String className = clazz.getName();
221 List<Object> args = new ArrayList<Object>();
222 List<Class<?>> classes = new ArrayList<Class<?>>();
223 Constructor<?> ctor = null;
224 if (className.contains("$")) {
225 for (String parentName = className.substring(0,
226 className.lastIndexOf('$'));; parentName = parentName
227 .substring(0, parentName.lastIndexOf('$'))) {
228 Object parent = createObject(parentName);
229 args.add(parent);
230 classes.add(parent.getClass());
231
232 if (!parentName.contains("$")) {
233 break;
234 }
235 }
236
237 // Better error description in case there is no empty
238 // constructor:
239 desc = "";
240 String end = "";
241 for (Class<?> parent = clazz; parent != null
242 && !parent.equals(Object.class); parent = parent
243 .getSuperclass()) {
244 if (!desc.isEmpty()) {
245 desc += " [:";
246 end += "]";
247 }
248 desc += parent;
249 }
250 desc += end;
251 //
252
253 try {
254 ctor = clazz.getDeclaredConstructor(classes
255 .toArray(new Class[] {}));
256 } catch (NoSuchMethodException nsme) {
257 // TODO: it seems we do not always need a parameter for each
258 // level, so we currently try "ALL" levels or "FIRST" level
259 // only -> we should check the actual rule and use it
260 ctor = clazz.getDeclaredConstructor(classes.get(0));
261 Object firstParent = args.get(0);
262 args.clear();
263 args.add(firstParent);
264 }
265 desc = null;
266 } else {
267 ctor = clazz.getDeclaredConstructor();
268 }
269
270 ctor.setAccessible(true);
271 return ctor.newInstance(args.toArray());
272 } catch (ClassNotFoundException e) {
273 throw e;
274 } catch (NoSuchMethodException e) {
275 if (desc != null) {
276 throw new NoSuchMethodException("Empty constructor not found: "
277 + desc);
278 }
279 throw e;
280 } catch (Exception e) {
281 throw new NoSuchMethodException("Cannot instantiate: " + type);
282 }
283 }
284
285 /**
286 * Insert a custom serialiser that will take precedence over the default one
287 * or the target class.
288 *
289 * @param serializer
290 * the custom serialiser
291 */
292 static public void addCustomSerializer(CustomSerializer serializer) {
293 customTypes.put(serializer.getType(), serializer);
294 }
295
296 /**
297 * Serialise the given object into this {@link OutputStream}.
298 * <p>
299 * <b>Important: </b>If the operation fails (with a
300 * {@link NotSerializableException}), the {@link StringBuilder} will be
301 * corrupted (will contain bad, most probably not importable data).
302 *
303 * @param out
304 * the output {@link OutputStream} to serialise to
305 * @param o
306 * the object to serialise
307 * @param map
308 * the map of already serialised objects (if the given object or
309 * one of its descendant is already present in it, only an ID
310 * will be serialised)
311 *
312 * @throws NotSerializableException
313 * if the object cannot be serialised (in this case, the
314 * {@link StringBuilder} can contain bad, most probably not
315 * importable data)
316 * @throws IOException
317 * in case of I/O errors
318 */
319 static void append(OutputStream out, Object o, Map<Integer, Object> map)
320 throws NotSerializableException, IOException {
321
322 Field[] fields = new Field[] {};
323 String type = "";
324 String id = "NULL";
325
326 if (o != null) {
327 int hash = System.identityHashCode(o);
328 fields = o.getClass().getDeclaredFields();
329 type = o.getClass().getCanonicalName();
330 if (type == null) {
331 // Anonymous inner classes support
332 type = o.getClass().getName();
333 }
334 id = Integer.toString(hash);
335 if (map.containsKey(hash)) {
336 fields = new Field[] {};
337 } else {
338 map.put(hash, o);
339 }
340 }
341
342 write(out, "{\nREF ");
343 write(out, type);
344 write(out, "@");
345 write(out, id);
346 write(out, ":");
347
348 if (!encode(out, o)) { // check if direct value
349 try {
350 for (Field field : fields) {
351 field.setAccessible(true);
352
353 if (field.getName().startsWith("this$")
354 || field.isSynthetic()
355 || (field.getModifiers() & Modifier.STATIC) == Modifier.STATIC) {
356 // Do not keep this links of nested classes
357 // Do not keep synthetic fields
358 // Do not keep final fields
359 continue;
360 }
361
362 write(out, "\n^");
363 write(out, field.getName());
364 write(out, ":");
365
366 Object value = field.get(o);
367
368 if (!encode(out, value)) {
369 write(out, "\n");
370 append(out, value, map);
371 }
372 }
373 } catch (IllegalArgumentException e) {
374 e.printStackTrace(); // should not happen (see
375 // setAccessible)
376 } catch (IllegalAccessException e) {
377 e.printStackTrace(); // should not happen (see
378 // setAccessible)
379 }
380
381 write(out, "\n}");
382 }
383 }
384
385 /**
386 * Encode the object into the given {@link OutputStream} if possible and if
387 * supported.
388 * <p>
389 * A supported object in this context means an object we can directly
390 * encode, like an Integer or a String. Custom objects and arrays are also
391 * considered supported, but <b>compound objects are not supported here</b>.
392 * <p>
393 * For compound objects, you should use {@link Exporter}.
394 *
395 * @param out
396 * the {@link OutputStream} to append to
397 * @param value
398 * the object to encode (can be NULL, which will be encoded)
399 *
400 * @return TRUE if success, FALSE if not (the content of the
401 * {@link OutputStream} won't be changed in case of failure)
402 *
403 * @throws IOException
404 * in case of I/O error
405 */
406 static boolean encode(OutputStream out, Object value) throws IOException {
407 if (value == null) {
408 write(out, "NULL");
409 } else if (value.getClass().getSimpleName().endsWith("[]")) {
410 // Simple name does support [] suffix and do not return NULL for
411 // inner anonymous classes
412 customTypes.get("[]").encode(out, value);
413 } else if (customTypes.containsKey(value.getClass().getCanonicalName())) {
414 customTypes.get(value.getClass().getCanonicalName())//
415 .encode(out, value);
416 } else if (value instanceof String) {
417 encodeString(out, (String) value);
418 } else if (value instanceof Boolean) {
419 write(out, value);
420 } else if (value instanceof Byte) {
421 write(out, "b");
422 write(out, value);
423 } else if (value instanceof Character) {
424 write(out, "c");
425 encodeString(out, "" + value);
426 } else if (value instanceof Short) {
427 write(out, "s");
428 write(out, value);
429 } else if (value instanceof Integer) {
430 write(out, "i");
431 write(out, value);
432 } else if (value instanceof Long) {
433 write(out, "l");
434 write(out, value);
435 } else if (value instanceof Float) {
436 write(out, "f");
437 write(out, value);
438 } else if (value instanceof Double) {
439 write(out, "d");
440 write(out, value);
441 } else if (value instanceof Enum) {
442 write(out, "E:");
443 String type = value.getClass().getCanonicalName();
444 write(out, type);
445 write(out, ".");
446 write(out, ((Enum<?>) value).name());
447 write(out, ";");
448 } else {
449 return false;
450 }
451
452 return true;
453 }
454
455 static boolean isDirectValue(BufferedInputStream encodedValue)
456 throws IOException {
457 if (CustomSerializer.isCustom(encodedValue)) {
458 return false;
459 }
460
461 for (String fullValue : new String[] { "NULL", "null", "true", "false" }) {
462 if (encodedValue.is(fullValue)) {
463 return true;
464 }
465 }
466
467 // TODO: Not efficient
468 for (String prefix : new String[] { "c\"", "\"", "b", "s", "i", "l",
469 "f", "d", "E:" }) {
470 if (encodedValue.startsWith(prefix)) {
471 return true;
472 }
473 }
474
475 return false;
476 }
477
478 /**
479 * Decode the data into an equivalent supported source object.
480 * <p>
481 * A supported object in this context means an object we can directly
482 * encode, like an Integer or a String (see
483 * {@link SerialUtils#decode(String)}.
484 * <p>
485 * Custom objects and arrays are also considered supported here, but
486 * <b>compound objects are not</b>.
487 * <p>
488 * For compound objects, you should use {@link Importer}.
489 *
490 * @param encodedValue
491 * the encoded data, cannot be NULL
492 *
493 * @return the object (can be NULL for NULL encoded values)
494 *
495 * @throws IOException
496 * if the content cannot be converted
497 */
498 static Object decode(BufferedInputStream encodedValue) throws IOException {
499 if (CustomSerializer.isCustom(encodedValue)) {
500 // custom^TYPE^ENCODED_VALUE
501 NextableInputStream content = new NextableInputStream(encodedValue,
502 new NextableInputStreamStep('^'));
503 try {
504 content.next();
505 @SuppressWarnings("unused")
506 String custom = IOUtils.readSmallStream(content);
507 content.next();
508 String type = IOUtils.readSmallStream(content);
509 content.nextAll();
510 if (customTypes.containsKey(type)) {
511 return customTypes.get(type).decode(content);
512 }
513 content.end();
514 throw new IOException("Unknown custom type: " + type);
515 } finally {
516 content.close(false);
517 // TODO: check what happens with thrown Exception in finally
518 encodedValue.end();
519 }
520 }
521
522 String encodedString = IOUtils.readSmallStream(encodedValue);
523 return decode(encodedString);
524 }
525
526 /**
527 * Decode the data into an equivalent supported source object.
528 * <p>
529 * A supported object in this context means an object we can directly
530 * encode, like an Integer or a String.
531 * <p>
532 * For custom objects and arrays, you should use
533 * {@link SerialUtils#decode(InputStream)} or directly {@link Importer}.
534 * <p>
535 * For compound objects, you should use {@link Importer}.
536 *
537 * @param encodedValue
538 * the encoded data, cannot be NULL
539 *
540 * @return the object (can be NULL for NULL encoded values)
541 *
542 * @throws IOException
543 * if the content cannot be converted
544 */
545 static Object decode(String encodedValue) throws IOException {
546 try {
547 String cut = "";
548 if (encodedValue.length() > 1) {
549 cut = encodedValue.substring(1);
550 }
551
552 if (encodedValue.equals("NULL") || encodedValue.equals("null")) {
553 return null;
554 } else if (encodedValue.startsWith("\"")) {
555 return decodeString(encodedValue);
556 } else if (encodedValue.equals("true")) {
557 return true;
558 } else if (encodedValue.equals("false")) {
559 return false;
560 } else if (encodedValue.startsWith("b")) {
561 return Byte.parseByte(cut);
562 } else if (encodedValue.startsWith("c")) {
563 return decodeString(cut).charAt(0);
564 } else if (encodedValue.startsWith("s")) {
565 return Short.parseShort(cut);
566 } else if (encodedValue.startsWith("l")) {
567 return Long.parseLong(cut);
568 } else if (encodedValue.startsWith("f")) {
569 return Float.parseFloat(cut);
570 } else if (encodedValue.startsWith("d")) {
571 return Double.parseDouble(cut);
572 } else if (encodedValue.startsWith("i")) {
573 return Integer.parseInt(cut);
574 } else if (encodedValue.startsWith("E:")) {
575 cut = cut.substring(1);
576 return decodeEnum(cut);
577 } else {
578 throw new IOException("Unrecognized value: " + encodedValue);
579 }
580 } catch (Exception e) {
581 if (e instanceof IOException) {
582 throw (IOException) e;
583 }
584 throw new IOException(e.getMessage(), e);
585 }
586 }
587
588 /**
589 * Write the given {@link String} into the given {@link OutputStream} in
590 * UTF-8.
591 *
592 * @param out
593 * the {@link OutputStream}
594 * @param data
595 * the data to write, cannot be NULL
596 *
597 * @throws IOException
598 * in case of I/O error
599 */
600 static void write(OutputStream out, Object data) throws IOException {
601 out.write(StringUtils.getBytes(data.toString()));
602 }
603
604 /**
605 * Return the corresponding class or throw an {@link Exception} if it
606 * cannot.
607 *
608 * @param type
609 * the class name to look for
610 *
611 * @return the class (will never be NULL)
612 *
613 * @throws ClassNotFoundException
614 * if the class cannot be found
615 * @throws NoSuchMethodException
616 * if the class cannot be created (usually because it or its
617 * enclosing class doesn't have an empty constructor)
618 */
619 static private Class<?> getClass(String type)
620 throws ClassNotFoundException, NoSuchMethodException {
621 Class<?> clazz = null;
622 try {
623 clazz = Class.forName(type);
624 } catch (ClassNotFoundException e) {
625 int pos = type.length();
626 pos = type.lastIndexOf(".", pos);
627 if (pos >= 0) {
628 String parentType = type.substring(0, pos);
629 String nestedType = type.substring(pos + 1);
630 Class<?> javaParent = null;
631 try {
632 javaParent = getClass(parentType);
633 parentType = javaParent.getName();
634 clazz = Class.forName(parentType + "$" + nestedType);
635 } catch (Exception ee) {
636 }
637
638 if (javaParent == null) {
639 throw new NoSuchMethodException(
640 "Class not found: "
641 + type
642 + " (the enclosing class cannot be created: maybe it doesn't have an empty constructor?)");
643 }
644 }
645 }
646
647 if (clazz == null) {
648 throw new ClassNotFoundException("Class not found: " + type);
649 }
650
651 return clazz;
652 }
653
654 @SuppressWarnings({ "unchecked", "rawtypes" })
655 static private Enum<?> decodeEnum(String escaped) {
656 // escaped: be.xxx.EnumType.VALUE;
657 int pos = escaped.lastIndexOf(".");
658 String type = escaped.substring(0, pos);
659 String name = escaped.substring(pos + 1, escaped.length() - 1);
660
661 try {
662 return Enum.valueOf((Class<Enum>) getClass(type), name);
663 } catch (Exception e) {
664 throw new UnknownFormatConversionException("Unknown enum: <" + type
665 + "> " + name);
666 }
667 }
668
669 // aa bb -> "aa\tbb"
670 static void encodeString(OutputStream out, String raw) throws IOException {
671 // TODO: not. efficient.
672 out.write('\"');
673 // TODO !! utf-8 required
674 for (char car : raw.toCharArray()) {
675 encodeString(out, car);
676 }
677 out.write('\"');
678 }
679
680 // aa bb -> "aa\tbb"
681 static void encodeString(OutputStream out, InputStream raw)
682 throws IOException {
683 out.write('\"');
684 byte buffer[] = new byte[4096];
685 for (int len = 0; (len = raw.read(buffer)) > 0;) {
686 for (int i = 0; i < len; i++) {
687 // TODO: not 100% correct, look up howto for UTF-8
688 encodeString(out, (char) buffer[i]);
689 }
690 }
691 out.write('\"');
692 }
693
694 // for encode string, NOT to encode a char by itself!
695 static void encodeString(OutputStream out, char raw) throws IOException {
696 switch (raw) {
697 case '\\':
698 out.write('\\');
699 out.write('\\');
700 break;
701 case '\r':
702 out.write('\\');
703 out.write('r');
704 break;
705 case '\n':
706 out.write('\\');
707 out.write('n');
708 break;
709 case '"':
710 out.write('\\');
711 out.write('\"');
712 break;
713 default:
714 out.write(raw);
715 break;
716 }
717 }
718
719 // "aa\tbb" -> aa bb
720 static String decodeString(String escaped) {
721 StringBuilder builder = new StringBuilder();
722
723 boolean escaping = false;
724 for (char car : escaped.toCharArray()) {
725 if (!escaping) {
726 if (car == '\\') {
727 escaping = true;
728 } else {
729 builder.append(car);
730 }
731 } else {
732 switch (car) {
733 case '\\':
734 builder.append('\\');
735 break;
736 case 'r':
737 builder.append('\r');
738 break;
739 case 'n':
740 builder.append('\n');
741 break;
742 case '"':
743 builder.append('"');
744 break;
745 }
746 escaping = false;
747 }
748 }
749
750 return builder.substring(1, builder.length() - 1);
751 }
752 }