From 686d4da2d2ecc203d5f8b524225a4327777825be Mon Sep 17 00:00:00 2001 From: Kevin Lamonte Date: Sun, 15 Sep 2019 08:19:48 -0500 Subject: [PATCH] experimental 24-bit image protocol --- src/jexer/backend/ECMA48Terminal.java | 243 ++++++++++++++++++++++- src/jexer/tterminal/ECMA48.java | 270 +++++++++++++++++++++----- src/jexer/tterminal/Sixel.java | 52 ++++- 3 files changed, 512 insertions(+), 53 deletions(-) diff --git a/src/jexer/backend/ECMA48Terminal.java b/src/jexer/backend/ECMA48Terminal.java index 96ca5df..e2997d2 100644 --- a/src/jexer/backend/ECMA48Terminal.java +++ b/src/jexer/backend/ECMA48Terminal.java @@ -217,7 +217,17 @@ public class ECMA48Terminal extends LogicalScreen private ImageCache iterm2Cache = null; /** - * Base64 encoder used by iTerm2 images. + * If true, emit image data via Jexer image protocol. + */ + private boolean jexerImages = false; + + /** + * The Jexer post-rendered string cache. + */ + private ImageCache jexerCache = null; + + /** + * Base64 encoder used by iTerm2 and Jexer images. */ private java.util.Base64.Encoder base64 = null; @@ -226,6 +236,11 @@ public class ECMA48Terminal extends LogicalScreen */ private boolean setRawMode = false; + /** + * If true, '?' was seen in terminal response. + */ + private boolean decPrivateModeFlag = false; + /** * The terminal's input. If an InputStream is not specified in the * constructor, then this InputStreamReader will be bound to System.in @@ -1133,6 +1148,9 @@ public class ECMA48Terminal extends LogicalScreen "UTF-8")); } + // Request Device Attributes + this.output.printf("\033[c"); + // Request xterm report window/cell dimensions in pixels this.output.printf("%s", xtermReportPixelDimensions()); @@ -1222,6 +1240,9 @@ public class ECMA48Terminal extends LogicalScreen this.output = writer; + // Request Device Attributes + this.output.printf("\033[c"); + // Request xterm report window/cell dimensions in pixels this.output.printf("%s", xtermReportPixelDimensions()); @@ -2031,6 +2052,8 @@ public class ECMA48Terminal extends LogicalScreen if (cellsToDraw.size() > 0) { if (iterm2Images) { sb.append(toIterm2Image(x, y, cellsToDraw)); + } else if (jexerImages) { + sb.append(toJexerImage(x, y, cellsToDraw)); } else { sb.append(toSixel(x, y, cellsToDraw)); } @@ -2062,6 +2085,7 @@ public class ECMA48Terminal extends LogicalScreen params = new ArrayList(); params.clear(); params.add(""); + decPrivateModeFlag = false; } /** @@ -2644,6 +2668,10 @@ public class ECMA48Terminal extends LogicalScreen // Mouse position, SGR (1006) coordinates state = ParseState.MOUSE_SGR; return; + case '?': + // DEC private mode flag + decPrivateModeFlag = true; + return; default: break; } @@ -2773,6 +2801,27 @@ public class ECMA48Terminal extends LogicalScreen events.add(new TKeypressEvent(kbEnd, alt, ctrl, shift)); resetParser(); return; + case 'c': + // Device Attributes + if (decPrivateModeFlag == false) { + break; + } + for (String x: params) { + if (x.equals("4")) { + // Terminal reports sixel support + if (debugToStderr) { + System.err.println("Device Attributes: sixel"); + } + } + if (x.equals("444")) { + // Terminal reports Jexer images support + if (debugToStderr) { + System.err.println("Device Attributes: Jexer images"); + } + jexerImages = true; + } + } + return; case 't': // windowOps if ((params.size() > 2) && (params.get(0).equals("4"))) { @@ -3570,6 +3619,198 @@ public class ECMA48Terminal extends LogicalScreen // End iTerm2 image output support ---------------------------------------- // ------------------------------------------------------------------------ + // ------------------------------------------------------------------------ + // Jexer image output support --------------------------------------------- + // ------------------------------------------------------------------------ + + /** + * Create a Jexer images string representing a row of several cells + * containing bitmap data. + * + * @param x column coordinate. 0 is the left-most column. + * @param y row coordinate. 0 is the top-most row. + * @param cells the cells containing the bitmap data + * @return the string to emit to an ANSI / ECMA-style terminal + */ + private String toJexerImage(final int x, final int y, + final ArrayList cells) { + + StringBuilder sb = new StringBuilder(); + + assert (cells != null); + assert (cells.size() > 0); + assert (cells.get(0).getImage() != null); + + if (jexerImages == false) { + sb.append(normal()); + sb.append(gotoXY(x, y)); + for (int i = 0; i < cells.size(); i++) { + sb.append(' '); + } + return sb.toString(); + } + + if (jexerCache == null) { + jexerCache = new ImageCache(height * 10); + base64 = java.util.Base64.getEncoder(); + } + + // Save and get rows to/from the cache that do NOT have inverted + // cells. + boolean saveInCache = true; + for (Cell cell: cells) { + if (cell.isInvertedImage()) { + saveInCache = false; + } + } + if (saveInCache) { + String cachedResult = jexerCache.get(cells); + if (cachedResult != null) { + // System.err.println("CACHE HIT"); + sb.append(gotoXY(x, y)); + sb.append(cachedResult); + return sb.toString(); + } + // System.err.println("CACHE MISS"); + } + + int imageWidth = cells.get(0).getImage().getWidth(); + int imageHeight = cells.get(0).getImage().getHeight(); + + // cells.get(x).getImage() has a dithered bitmap containing indexes + // into the color palette. Piece these together into one larger + // image for final rendering. + int totalWidth = 0; + int fullWidth = cells.size() * getTextWidth(); + int fullHeight = getTextHeight(); + for (int i = 0; i < cells.size(); i++) { + totalWidth += cells.get(i).getImage().getWidth(); + } + + BufferedImage image = new BufferedImage(fullWidth, + fullHeight, BufferedImage.TYPE_INT_ARGB); + + int [] rgbArray; + for (int i = 0; i < cells.size() - 1; i++) { + int tileWidth = Math.min(cells.get(i).getImage().getWidth(), + imageWidth); + int tileHeight = Math.min(cells.get(i).getImage().getHeight(), + imageHeight); + if (false && cells.get(i).isInvertedImage()) { + // I used to put an all-white cell over the cursor, don't do + // that anymore. + rgbArray = new int[imageWidth * imageHeight]; + for (int j = 0; j < rgbArray.length; j++) { + rgbArray[j] = 0xFFFFFF; + } + } else { + try { + rgbArray = cells.get(i).getImage().getRGB(0, 0, + tileWidth, tileHeight, null, 0, tileWidth); + } catch (Exception e) { + throw new RuntimeException("image " + imageWidth + "x" + + imageHeight + + "tile " + tileWidth + "x" + + tileHeight + + " cells.get(i).getImage() " + + cells.get(i).getImage() + + " i " + i + + " fullWidth " + fullWidth + + " fullHeight " + fullHeight, e); + } + } + + /* + System.err.printf("calling image.setRGB(): %d %d %d %d %d\n", + i * imageWidth, 0, imageWidth, imageHeight, + 0, imageWidth); + System.err.printf(" fullWidth %d fullHeight %d cells.size() %d textWidth %d\n", + fullWidth, fullHeight, cells.size(), getTextWidth()); + */ + + image.setRGB(i * imageWidth, 0, tileWidth, tileHeight, + rgbArray, 0, tileWidth); + if (tileHeight < fullHeight) { + int backgroundColor = cells.get(i).getBackground().getRGB(); + for (int imageX = 0; imageX < image.getWidth(); imageX++) { + for (int imageY = imageHeight; imageY < fullHeight; + imageY++) { + + image.setRGB(imageX, imageY, backgroundColor); + } + } + } + } + totalWidth -= ((cells.size() - 1) * imageWidth); + if (false && cells.get(cells.size() - 1).isInvertedImage()) { + // I used to put an all-white cell over the cursor, don't do that + // anymore. + rgbArray = new int[totalWidth * imageHeight]; + for (int j = 0; j < rgbArray.length; j++) { + rgbArray[j] = 0xFFFFFF; + } + } else { + try { + rgbArray = cells.get(cells.size() - 1).getImage().getRGB(0, 0, + totalWidth, imageHeight, null, 0, totalWidth); + } catch (Exception e) { + throw new RuntimeException("image " + imageWidth + "x" + + imageHeight + " cells.get(cells.size() - 1).getImage() " + + cells.get(cells.size() - 1).getImage(), e); + } + } + image.setRGB((cells.size() - 1) * imageWidth, 0, totalWidth, + imageHeight, rgbArray, 0, totalWidth); + + if (totalWidth < getTextWidth()) { + int backgroundColor = cells.get(cells.size() - 1).getBackground().getRGB(); + + for (int imageX = image.getWidth() - totalWidth; + imageX < image.getWidth(); imageX++) { + + for (int imageY = 0; imageY < fullHeight; imageY++) { + image.setRGB(imageX, imageY, backgroundColor); + } + } + } + + sb.append(String.format("\033]444;%d;%d;0;", image.getWidth(), + Math.min(image.getHeight(), fullHeight))); + + byte [] bytes = new byte[image.getWidth() * image.getHeight() * 3]; + int stride = image.getWidth(); + for (int px = 0; px < stride; px++) { + for (int py = 0; py < image.getHeight(); py++) { + int rgb = image.getRGB(px, py); + bytes[(py * stride * 3) + (px * 3)] = (byte) ((rgb >>> 16) & 0xFF); + bytes[(py * stride * 3) + (px * 3) + 1] = (byte) ((rgb >>> 8) & 0xFF); + bytes[(py * stride * 3) + (px * 3) + 2] = (byte) ( rgb & 0xFF); + } + } + sb.append(base64.encodeToString(bytes)); + sb.append("\007"); + + if (saveInCache) { + // This row is OK to save into the cache. + jexerCache.put(cells, sb.toString()); + } + + return (gotoXY(x, y) + sb.toString()); + } + + /** + * Get the Jexer images support flag. + * + * @return true if this terminal is emitting Jexer images + */ + public boolean hasJexerImages() { + return jexerImages; + } + + // ------------------------------------------------------------------------ + // End Jexer image output support ----------------------------------------- + // ------------------------------------------------------------------------ + /** * Setup system colors to match DOS color palette. */ diff --git a/src/jexer/tterminal/ECMA48.java b/src/jexer/tterminal/ECMA48.java index 4bf7efb..1d34811 100644 --- a/src/jexer/tterminal/ECMA48.java +++ b/src/jexer/tterminal/ECMA48.java @@ -471,6 +471,11 @@ public class ECMA48 implements Runnable { */ private StringBuilder sixelParseBuffer; + /** + * Sixel shared palette. + */ + private HashMap sixelPalette; + /** * The width of a character cell in pixels. */ @@ -872,9 +877,11 @@ public class ECMA48 implements Runnable { // "I am a VT220" - 7 bit version if (!s8c1t) { return "\033[?62;1;6;9;4;22c"; + // return "\033[?62;1;6;9;4;22;444c"; } // "I am a VT220" - 8 bit version return "\u009b?62;1;6;9;4;22c"; + // return "\u009b?62;1;6;9;4;22;444c"; default: throw new IllegalArgumentException("Invalid device type: " + type); } @@ -3144,6 +3151,21 @@ public class ECMA48 implements Runnable { break; + case 80: + if (type == DeviceType.XTERM) { + if (decPrivateModeFlag == true) { + if (value == true) { + // Enable sixel scrolling (default). + // TODO + } else { + // Disable sixel scrolling. + // TODO + } + } + } + + break; + case 1000: if ((type == DeviceType.XTERM) && (decPrivateModeFlag == true) @@ -3209,6 +3231,22 @@ public class ECMA48 implements Runnable { } break; + case 1070: + if (type == DeviceType.XTERM) { + if (decPrivateModeFlag == true) { + if (value == true) { + // Use private color registers for each sixel + // graphic (default). + sixelPalette = null; + } else { + // Use shared color registers for each sixel + // graphic. + sixelPalette = new HashMap(); + } + } + } + break; + default: break; @@ -4753,6 +4791,12 @@ public class ECMA48 implements Runnable { color.getBlue() << 8)); } } + + if (p[0].equals("444") && (p.length == 5)) { + // Jexer image + parseJexerImage(p[1], p[2], p[3], p[4]); + } + } // Go to SCAN_GROUND state @@ -6791,11 +6835,13 @@ public class ECMA48 implements Runnable { if (ch == 0x9C) { parseSixel(); toGround(); + return; } // 0x1B 0x5C goes to GROUND if (ch == 0x1B) { collect((char) ch); + return; } if (ch == 0x5C) { if ((collectBuffer.length() > 0) @@ -6803,29 +6849,20 @@ public class ECMA48 implements Runnable { ) { parseSixel(); toGround(); + return; } } // 00-17, 19, 1C-1F, 20-7E --> put - if (ch <= 0x17) { - sixelParseBuffer.append((char) ch); - return; - } - if (ch == 0x19) { - sixelParseBuffer.append((char) ch); - return; - } - if ((ch >= 0x1C) && (ch <= 0x1F)) { - sixelParseBuffer.append((char) ch); - return; - } - if ((ch >= 0x20) && (ch <= 0x7E)) { + if ((ch <= 0x17) + || (ch == 0x19) + || ((ch >= 0x1C) && (ch <= 0x1F)) + || ((ch >= 0x20) && (ch <= 0x7E)) + ) { sixelParseBuffer.append((char) ch); - return; } // 7F --> ignore - return; case SOSPMAPC_STRING: @@ -6923,9 +6960,43 @@ public class ECMA48 implements Runnable { return mouseProtocol; } - // ------------------------------------------------------------------------ - // Sixel support ---------------------------------------------------------- - // ------------------------------------------------------------------------ + /** + * Draw the left and right cells of a two-cell-wide (full-width) glyph. + * + * @param leftX the x position to draw the left half to + * @param leftY the y position to draw the left half to + * @param rightX the x position to draw the right half to + * @param rightY the y position to draw the right half to + * @param ch the character to draw + */ + private void drawHalves(final int leftX, final int leftY, + final int rightX, final int rightY, final int ch) { + + // System.err.println("drawHalves(): " + Integer.toHexString(ch)); + + if (lastTextHeight != textHeight) { + glyphMaker = GlyphMaker.getInstance(textHeight); + lastTextHeight = textHeight; + } + + Cell cell = new Cell(ch, currentState.attr); + BufferedImage image = glyphMaker.getImage(cell, textWidth * 2, + textHeight); + BufferedImage leftImage = image.getSubimage(0, 0, textWidth, + textHeight); + BufferedImage rightImage = image.getSubimage(textWidth, 0, textWidth, + textHeight); + + Cell left = new Cell(cell); + left.setImage(leftImage); + left.setWidth(Cell.Width.LEFT); + display.get(leftY).replace(leftX, left); + + Cell right = new Cell(cell); + right.setImage(rightImage); + right.setWidth(Cell.Width.RIGHT); + display.get(rightY).replace(rightX, right); + } /** * Set the width of a character cell in pixels. @@ -6954,9 +7025,9 @@ public class ECMA48 implements Runnable { /* System.err.println("parseSixel(): '" + sixelParseBuffer.toString() + "'"); - */ + */ - Sixel sixel = new Sixel(sixelParseBuffer.toString()); + Sixel sixel = new Sixel(sixelParseBuffer.toString(), sixelPalette); BufferedImage image = sixel.getImage(); // System.err.println("parseSixel(): image " + image); @@ -7045,41 +7116,146 @@ public class ECMA48 implements Runnable { } /** - * Draw the left and right cells of a two-cell-wide (full-width) glyph. + * Parse a "Jexer" image string into a bitmap image, and overlay that + * image onto the text cells. * - * @param leftX the x position to draw the left half to - * @param leftY the y position to draw the left half to - * @param rightX the x position to draw the right half to - * @param rightY the y position to draw the right half to - * @param ch the character to draw + * @param pw width token + * @param ph height token + * @param ps scroll token + * @param data pixel data */ - private void drawHalves(final int leftX, final int leftY, - final int rightX, final int rightY, final int ch) { + private void parseJexerImage(final String pw, final String ph, + final String ps, final String data) { - // System.err.println("drawHalves(): " + Integer.toHexString(ch)); + int imageWidth = 0; + int imageHeight = 0; + boolean scroll = false; + try { + imageWidth = Integer.parseInt(pw); + imageHeight = Integer.parseInt(ph); + } catch (NumberFormatException e) { + // SQUASH + return; + } + if ((imageWidth < 1) + || (imageWidth > 10000) + || (imageHeight < 1) + || (imageHeight > 10000) + ) { + return; + } + if (ps.equals("1")) { + scroll = true; + } else if (ps.equals("0")) { + scroll = false; + } else { + return; + } - if (lastTextHeight != textHeight) { - glyphMaker = GlyphMaker.getInstance(textHeight); - lastTextHeight = textHeight; + java.util.Base64.Decoder base64 = java.util.Base64.getDecoder(); + byte [] bytes = base64.decode(data); + if (bytes.length != (imageWidth * imageHeight * 3)) { + return; } - Cell cell = new Cell(ch, currentState.attr); - BufferedImage image = glyphMaker.getImage(cell, textWidth * 2, - textHeight); - BufferedImage leftImage = image.getSubimage(0, 0, textWidth, - textHeight); - BufferedImage rightImage = image.getSubimage(textWidth, 0, textWidth, - textHeight); + BufferedImage image = new BufferedImage(imageWidth, imageHeight, + BufferedImage.TYPE_INT_ARGB); - Cell left = new Cell(cell); - left.setImage(leftImage); - left.setWidth(Cell.Width.LEFT); - display.get(leftY).replace(leftX, left); + for (int x = 0; x < imageWidth; x++) { + for (int y = 0; y < imageHeight; y++) { + int red = bytes[(y * imageWidth * 3) + (x * 3) ]; + if (red < 0) { + red += 256; + } + int green = bytes[(y * imageWidth * 3) + (x * 3) + 1]; + if (green < 0) { + green += 256; + } + int blue = bytes[(y * imageWidth * 3) + (x * 3) + 2]; + if (blue < 0) { + blue += 256; + } + int rgb = 0xFF000000 | (red << 16) | (green << 8) | blue; + image.setRGB(x, y, rgb); + } + } + + /* + * Procedure: + * + * Break up the image into text cell sized pieces as a new array of + * Cells. + * + * Note original column position x0. + * + * For each cell: + * + * 1. Advance (printCharacter(' ')) for horizontal increment, or + * index (linefeed() + cursorPosition(y, x0)) for vertical + * increment. + * + * 2. Set (x, y) cell image data. + * + * 3. For the right and bottom edges: + * + * a. Render the text to pixels using Terminus font. + * + * b. Blit the image on top of the text, using alpha channel. + */ + int cellColumns = image.getWidth() / textWidth; + if (cellColumns * textWidth < image.getWidth()) { + cellColumns++; + } + int cellRows = image.getHeight() / textHeight; + if (cellRows * textHeight < image.getHeight()) { + cellRows++; + } + + // Break the image up into an array of cells. + Cell [][] cells = new Cell[cellColumns][cellRows]; + + for (int x = 0; x < cellColumns; x++) { + for (int y = 0; y < cellRows; y++) { + + int width = textWidth; + if ((x + 1) * textWidth > image.getWidth()) { + width = image.getWidth() - (x * textWidth); + } + int height = textHeight; + if ((y + 1) * textHeight > image.getHeight()) { + height = image.getHeight() - (y * textHeight); + } + + Cell cell = new Cell(); + cell.setImage(image.getSubimage(x * textWidth, + y * textHeight, width, height)); + + cells[x][y] = cell; + } + } + + int x0 = currentState.cursorX; + for (int y = 0; y < cellRows; y++) { + for (int x = 0; x < cellColumns; x++) { + assert (currentState.cursorX <= rightMargin); + DisplayLine line = display.get(currentState.cursorY); + line.replace(currentState.cursorX, cells[x][y]); + // If at the end of the visible screen, stop. + if (currentState.cursorX == rightMargin) { + break; + } + // Room for more image on the visible screen. + currentState.cursorX++; + } + if ((scroll == true) + || ((scroll == false) + && (currentState.cursorY < scrollRegionBottom)) + ) { + linefeed(); + } + cursorPosition(currentState.cursorY, x0); + } - Cell right = new Cell(cell); - right.setImage(rightImage); - right.setWidth(Cell.Width.RIGHT); - display.get(rightY).replace(rightX, right); } } diff --git a/src/jexer/tterminal/Sixel.java b/src/jexer/tterminal/Sixel.java index 8e363ef..a4c00fc 100644 --- a/src/jexer/tterminal/Sixel.java +++ b/src/jexer/tterminal/Sixel.java @@ -72,6 +72,16 @@ public class Sixel { */ private static int HEIGHT_INCREASE = 400; + /** + * Maximum width in pixels. + */ + private static int MAX_WIDTH = 1000; + + /** + * Maximum height in pixels. + */ + private static int MAX_HEIGHT = 1000; + /** * Current scanning state. */ @@ -142,6 +152,11 @@ public class Sixel { */ private Color color = Color.BLACK; + /** + * If set, abort processing this image. + */ + private boolean abort = false; + // ------------------------------------------------------------------------ // Constructors ----------------------------------------------------------- // ------------------------------------------------------------------------ @@ -150,12 +165,14 @@ public class Sixel { * Public constructor. * * @param buffer the sixel data to parse + * @param palette palette to use, or null for a private palette */ - public Sixel(final String buffer) { + public Sixel(final String buffer, final HashMap palette) { this.buffer = buffer; - palette = new HashMap(); - for (int i = 0; i < buffer.length(); i++) { - consume(buffer.charAt(i)); + if (palette == null) { + this.palette = new HashMap(); + } else { + this.palette = palette; } } @@ -169,6 +186,16 @@ public class Sixel { * @return the sixel data as an image. */ public BufferedImage getImage() { + if (buffer != null) { + for (int i = 0; (i < buffer.length()) && (abort == false); i++) { + consume(buffer.charAt(i)); + } + buffer = null; + } + if (abort == true) { + return null; + } + if ((width > 0) && (height > 0) && (image != null)) { /* System.err.println(String.format("%d %d %d %d", width, y + 1, @@ -300,6 +327,9 @@ public class Sixel { if (x > width) { width = x; } + if (width > MAX_WIDTH) { + abort = true; + } return; } @@ -337,6 +367,12 @@ public class Sixel { if (x > width) { width = x; } + if (width > MAX_WIDTH) { + abort = true; + } + if (y + 1 > MAX_HEIGHT) { + abort = true; + } } /** @@ -393,7 +429,13 @@ public class Sixel { if ((pan == pad) && (pah > 0) && (pav > 0)) { rasterWidth = pah; rasterHeight = pav; - resizeImage(rasterWidth, rasterHeight); + if ((rasterWidth <= MAX_WIDTH) && (rasterHeight <= MAX_HEIGHT)) { + resizeImage(rasterWidth, rasterHeight); + } else { + abort = true; + } + } else { + abort = true; } } -- 2.27.0