From: Niki <david.roulet@solidaris.be>
Date: Mon, 24 Jun 2024 13:58:53 +0000 (+0200)
Subject: raw ebcdic support + debug, csv... params
X-Git-Url: https://git.nikiroo.be/?a=commitdiff_plain;h=8d05a0c8e73f2188991b9fe456e4d8f83d123554;p=cbook.git

raw ebcdic support + debug, csv... params
---

diff --git a/src/cbook/cbook.h b/src/cbook/cbook.h
index 77ec54c..7aebd7b 100755
--- a/src/cbook/cbook.h
+++ b/src/cbook/cbook.h
@@ -56,6 +56,7 @@ typedef int CBOOK_OUT;
 
 typedef struct {
 	char *name;
+	int raw_ebcdic;
 	int started;
 	int finished;
 	int unaligned;
@@ -65,6 +66,7 @@ typedef struct {
 	char *err_mess;
 	char *err_field;
 	CBOOK_OUT out_fmt;
+	size_t bytes;
 	array_t *lines;
 } book_t;
 
@@ -90,6 +92,7 @@ int read_book(FILE *book_file, book_t *book);
 
 int write_header(FILE *outfile, book_t *book);
 int write_csv(FILE *output, book_t *book, cstring_t *data, size_t lino);
+void write_done();
 
 #endif /* CBOOK_H */
 
diff --git a/src/cbook/cbook_csv.c b/src/cbook/cbook_csv.c
index 32e51f2..135922d 100755
--- a/src/cbook/cbook_csv.c
+++ b/src/cbook/cbook_csv.c
@@ -26,16 +26,21 @@
 
 // DECLARATIONS
 
+static cstring_t *ascii_line = NULL;
+
 // Actual write code, but may write either data or header
-int write_line(FILE *outfile, book_t *book, cstring_t *data, 
+static int write_line(FILE *outfile, book_t *book, cstring_t *data, 
 		int header, size_t lino);
 
 // Write one field (including subfields), consume data and report errors
-char *write_field(FILE *outfile, book_t *book, line_t *field, 
+static char *write_field(FILE *outfile, book_t *book, line_t *field, 
 		char *data, size_t *remaining, size_t lino);
 
 // Write this one field (data is ready), report errors in book->err_mess
-void one_field(FILE *outfile, book_t *book, line_t *field, char *data);
+static void one_field(FILE *outfile, book_t *book, line_t *field, char *data);
+
+// Convert EBCDIC to ASCII
+static void ascii(FILE *outfile, char *ebcdic, size_t sz);
 
 // PUBLIC
 
@@ -45,17 +50,21 @@ int write_header(FILE *outfile, book_t *book) {
 		return 1;
 
 	return write_line(outfile, book, NULL, 1, 0);
-	fwrite("\n", 1, 1, outfile);
 }
 
 int write_csv(FILE *outfile, book_t *book, cstring_t *data, size_t lino) {
 	return write_line(outfile, book, data, 0, lino);
 }
 
+void write_done() {
+	free_cstring(ascii_line);
+	ascii_line = NULL;
+}
+
 // PRIVATE
 
-int write_line(FILE *outfile, book_t *book, cstring_t *dataline, int header,
-		size_t lino) {
+static int write_line(FILE *outfile, book_t *book, cstring_t *dataline, 
+		int header, size_t lino) {
 	size_t remaining = 0;
 	char *data = NULL;
 
@@ -102,11 +111,13 @@ int write_line(FILE *outfile, book_t *book, cstring_t *dataline, int header,
 	}
 
 	fwrite("\n", 1, 1, outfile);
+	if (book->out_fmt == CBOOK_OUT_FIELDS)
+		fwrite("\n", 1, 1, outfile);
 
 	return 1;
 }
 
-char *write_field(FILE *outfile, book_t *book, line_t *field, 
+static char *write_field(FILE *outfile, book_t *book, line_t *field, 
 		char *data, size_t *remaining, size_t lino) {
 	if (field->type == CBOOK_FMT_GROUP) {
 		array_loop(field->children, subfield, line_t *) {
@@ -145,7 +156,11 @@ char *write_field(FILE *outfile, book_t *book, line_t *field,
 	return data;
 }
 
-void one_field(FILE *outfile, book_t *book, line_t *field, char *data) {
+static void one_field(FILE *outfile, book_t *book, line_t *field, char *data) {
+	if (!ascii_line)
+		ascii_line = new_cstring();
+	cstring_clear(ascii_line);
+
 	switch(book->out_fmt) {
 	case CBOOK_OUT_CSV  : break;
 	case CBOOK_OUT_FIELDS:
@@ -156,14 +171,32 @@ void one_field(FILE *outfile, book_t *book, line_t *field, char *data) {
 
 	switch(field->type) {
 	case CBOOK_FMT_CHAR:
-		fwrite(data, 1, field->bytes, outfile);
+		fwrite("\"", 1, 1, outfile);
+		if (book->raw_ebcdic) {
+			ascii(outfile, data, field->bytes);
+			fwrite(ascii_line->string, 1, 
+					ascii_line->length, outfile);
+		} else {
+			char *d = data;
+			if (cstring_find(d, "\"", 0) >= 0) {
+				cstring_t *tmp = new_cstring();
+				cstring_addfN(tmp, d, 0, field->bytes);
+				cstring_replace(tmp, "\"", "\"\"");
+				d = cstring_convert(tmp);
+			}
+			fwrite(d, 1, field->bytes, outfile);
+			if (d != data)
+				free(d);
+		}
+		fwrite("\"", 1, 1, outfile);
+
 		break;
 	case CBOOK_FMT_VARCHAR:
 		// first 2 bytes -> BIN FIXED(15)
 		;
 		// TODO:
 		size_t coded = field->bytes - 2;
-
+		
 		data = data + 2;
 		if (coded > (field->bytes - 2)) {
 			char buf[100];
@@ -173,8 +206,14 @@ void one_field(FILE *outfile, book_t *book, line_t *field, char *data) {
 				buf, NULL
 			);
 		}
-
-		fwrite(data, 1, coded, outfile);
+		
+		size_t bkp = field->bytes;
+		field->bytes = coded;
+		field->type = CBOOK_FMT_CHAR;
+		one_field(outfile, book, field, data);
+		field->type = CBOOK_FMT_VARCHAR;
+		field->bytes = bkp;
+	
 		break;
 	case CBOOK_FMT_DECIMAL:	
 	case CBOOK_FMT_UDECIMAL:
@@ -274,8 +313,11 @@ void one_field(FILE *outfile, book_t *book, line_t *field, char *data) {
 
 		}
 
-		if (!positive)
+		if (positive)
+			fwrite(" ", 1, 1, outfile);
+		else
 			fwrite("-", 1, 1, outfile);
+
 		fwrite(str->string, 1, str->length, outfile);
 
 		free_cstring(str);
@@ -323,3 +365,80 @@ void one_field(FILE *outfile, book_t *book, line_t *field, char *data) {
 		break;
 	}
 }
+
+static void ascii(FILE *outfile, char *ebcdic, size_t sz) {
+	/* - Professor: "So the American government went to 
+	 * IBM to come up with an encryption standard, 
+	 * and they came up with..."
+	 * - Student: "EBCDIC!"
+	 */
+
+	if (!ascii_line)
+		ascii_line = new_cstring();
+
+	cstring_grow_to(ascii_line, sz); // should be enough in most cases
+
+	for (size_t i = 0 ; i < sz ; i++) {
+		unsigned char byte = ebcdic[i];
+		if ((byte >= 0xF0) && (byte <= 0xF9)) { /* 0 - 9 */
+			cstring_add_car(ascii_line, byte - 0xC0);
+		} else if ((byte >= 0xC1) && (byte <= 0xC9)) { /* A - I */
+			cstring_add_car(ascii_line, byte - 0x80);
+		} else if ((byte >= 0xD1) && (byte <= 0xD9)) { /* J - R */
+			cstring_add_car(ascii_line, byte - 0x87);
+		} else if ((byte >= 0xE2) && (byte <= 0xE9)) { /* S - Z */
+			cstring_add_car(ascii_line, byte - 0x8F);
+		} else if ((byte >= 0x81) && (byte <= 0x89)) { /* a - i */
+			cstring_add_car(ascii_line, byte - 0x20);
+		} else if ((byte >= 0x91) && (byte <= 0x99)) { /* j - r */
+			cstring_add_car(ascii_line, byte - 0x27);
+		} else if ((byte >= 0xA2) && (byte <= 0xA9)) { /* s - z */
+			cstring_add_car(ascii_line, byte - 0x2F);
+		} else {
+			switch(byte) {
+			case 0x40: cstring_add_car(ascii_line, ' '); break;
+			case 0x4F: cstring_add_car(ascii_line, '!'); break;
+			case 0x7F: // double it 
+				   cstring_add_car(ascii_line, '"'); 
+				   cstring_add_car(ascii_line, '"'); 
+			break;
+			case 0x7B: cstring_add_car(ascii_line, '#'); break;
+			case 0x5B: cstring_add_car(ascii_line, '$'); break;
+			case 0x6C: cstring_add_car(ascii_line, '%'); break;
+			case 0x50: cstring_add_car(ascii_line, '&'); break;
+			case 0x7D: cstring_add_car(ascii_line, '\''); break;
+			case 0x4D: cstring_add_car(ascii_line, '('); break;
+			case 0x5D: cstring_add_car(ascii_line, ')'); break;
+			case 0x5C: cstring_add_car(ascii_line, '*'); break;
+			case 0x4E: cstring_add_car(ascii_line, '+'); break;
+			case 0x6B: cstring_add_car(ascii_line, ','); break;
+			case 0x60: cstring_add_car(ascii_line, '-'); break;
+			case 0x4B: cstring_add_car(ascii_line, '.'); break;
+			case 0x61: cstring_add_car(ascii_line, '/'); break;
+			case 0x7A: cstring_add_car(ascii_line, ':'); break;
+			case 0x5E: cstring_add_car(ascii_line, ';'); break;
+			case 0x4C: cstring_add_car(ascii_line, '<'); break;
+			case 0x7E: cstring_add_car(ascii_line, '='); break;
+			case 0x6E: cstring_add_car(ascii_line, '>'); break;
+			case 0x6F: cstring_add_car(ascii_line, '?'); break;
+			case 0x7C: cstring_add_car(ascii_line, '@'); break;
+			case 0x4A: cstring_add_car(ascii_line, '['); break;
+			case 0xE0: cstring_add_car(ascii_line, '\\'); break;
+			case 0x5A: cstring_add_car(ascii_line, ']'); break;
+			case 0x5F: cstring_add_car(ascii_line, '^'); break;
+			case 0x6D: cstring_add_car(ascii_line, '_'); break;
+			case 0x79: cstring_add_car(ascii_line, '`'); break;
+			case 0xC0: cstring_add_car(ascii_line, '{'); break;
+			case 0x6A: cstring_add_car(ascii_line, '|'); break;
+			case 0xD0: cstring_add_car(ascii_line, '}'); break;
+			case 0xA1: cstring_add_car(ascii_line, '~'); break;
+			// TODO: 0xC2A5 = 0xE0 \ (idem à 0x5C !!!)
+			default:
+				// 0x2117 = (P) or ℗ char
+				cstring_add_car(ascii_line, 0x21);
+				cstring_add_car(ascii_line, 0x17);
+				break;
+			}
+		}
+	}
+}
diff --git a/src/cbook/cbook_main.c b/src/cbook/cbook_main.c
index c21edcb..4d54c25 100755
--- a/src/cbook/cbook_main.c
+++ b/src/cbook/cbook_main.c
@@ -22,10 +22,11 @@
 #include <errno.h>
 
 #include "cbook.h"
-#include "cutils/cstring.h"
+#include "cutils/cutils.h"
 
 /* Declarations */
 
+int readline(book_t *book, cstring_t *line, FILE *infile, size_t lino);
 void help(char *program);
 
 /* Public */
@@ -35,14 +36,22 @@ int main(int argc, char **argv) {
 	char *input = NULL;
 	char *output = NULL;
 	
-	// TODO: configure that with params
 	CBOOK_OUT out_fmt = CBOOK_OUT_CSV;
-	int ignore_errors = 1;
-	int debug = 1;
+	int ignore_errors = 0;
+	int debug = 0;
+	int raw_ebcdic = 0;
 
 	for (int i = 1; i < argc; i++) {
 		char *arg = argv[i];
-		if (!strcmp("--help", arg) || !strcmp("-h", arg)) {
+		if (!strcmp("--debug", arg) || !strcmp("-d", arg)) {
+			debug = 1;
+		} else if (!strcmp("--output-flat", arg) || !strcmp("-o", arg)){
+			out_fmt = CBOOK_OUT_FIELDS;
+		} else if (!strcmp("--raw-ebcdic", arg) || !strcmp("-r", arg)) {
+			raw_ebcdic = 1;
+		} else if (!strcmp("--ignore-errors", arg)||!strcmp("-i", arg)){
+			ignore_errors = 1;
+		} else if (!strcmp("--help", arg) || !strcmp("-h", arg)) {
 			help(argv[0]);
 			return 0;
 		} else if (!strcmp("--book", arg) || !strcmp("-b", arg)) {
@@ -99,6 +108,7 @@ int main(int argc, char **argv) {
 	}
 	
 	book_t *book = new_book();
+	book->raw_ebcdic = raw_ebcdic;
 	book->out_fmt = out_fmt;
 	book->ignore_errors = ignore_errors;
 	book->debug = debug;
@@ -119,29 +129,67 @@ int main(int argc, char **argv) {
 	
 	size_t lino = 0;
 	cstring_t *line = new_cstring();
-	while(cstring_readline(line, infile)) {
-		lino++;
+	int write_ok = 1;
+	while(readline(book, line, infile, ++lino)) {
 		if (!write_csv(outfile, book, line, lino)) {
-			fprintf(stderr, "Failure to write CSV output file\n");
-			fprintf(stderr, "Error on line %zu, field <%s>: %s\n", 
-				book->err_line,book->err_field,book->err_mess);
-			fwrite("\n", 1, 1, outfile);
-			return 4;
+			write_ok = 0;
+			break;
 		}
-
-		fwrite("\n", 1, 1, outfile);
 	}
+	write_done();
 	
 	if (infile != stdin)
 		fclose(infile);
 	if (outfile != stdout)
 		fclose(stdout);
+	
+	if (!write_ok || book->err_mess) {
+		if (!write_ok)
+			fprintf(stderr, "Failure to write CSV output file\n");
+		else
+			fprintf(stderr, "Failure to read data input file\n");
+		fprintf(stderr, "Error on line %zu, field <%s>: %s\n", 
+			book->err_line,book->err_field,book->err_mess);
+		fwrite("\n", 1, 1, outfile);
+		
+		if (!write_ok)
+			return 4;
+		return 5;
+	}
 
 	return 0;
 }
 
 /* Private */
 
+int readline(book_t *book, cstring_t *line, FILE *infile, size_t lino) {
+	if (!book->raw_ebcdic)
+		return cstring_readline(line, infile);
+	
+	cstring_clear(line);
+	cstring_grow_to(line, book->bytes + 1);
+	size_t count = fread(line->string, 1, book->bytes, infile);
+	if (count != book->bytes) {
+		if (!count && feof(infile))
+			return 0;
+
+		char zu[100];
+		cstring_t *tmp = new_cstring();
+		cstring_add(tmp, "Read error: ");
+		sprintf(zu, "%zu", book->bytes); cstring_add(tmp, zu);
+		cstring_add(tmp, " bytes expected, ");
+		sprintf(zu, "%zu", count);       cstring_add(tmp, zu);
+		cstring_add(tmp, " read");
+		book->err_line = lino;
+		book->err_mess = cstring_convert(tmp);
+		return 0;
+	}
+	line->length = count;
+	line->string[count] = '\0';
+
+	return 1;
+}
+
 void help(char *program) {
 	printf("CBook data conversion program\n");
 	printf("\n");
diff --git a/src/cbook/cbook_pl1.c b/src/cbook/cbook_pl1.c
index 27f8bf0..b8309c0 100755
--- a/src/cbook/cbook_pl1.c
+++ b/src/cbook/cbook_pl1.c
@@ -177,6 +177,11 @@ skip:
 		}
 	}
 	
+	book->bytes = 0;
+	array_loop(book->lines, line, line_t *) {
+		book->bytes += (*line)->bytes;
+	}
+
 	reorder_lines(book);
 
 	return 1;
diff --git a/src/cutils b/src/cutils
index 8a3dcb4..6e1a127 160000
--- a/src/cutils
+++ b/src/cutils
@@ -1 +1 @@
-Subproject commit 8a3dcb4e6495ca5a0fd8846a4e4dc6ed52e90136
+Subproject commit 6e1a12739841bc4f7d4e5a2fc8b36d72fc0e8cb2