From 83787cfb2eb1501a328f8c5f2416c2532e3fef1c Mon Sep 17 00:00:00 2001 From: Kevin Lamonte Date: Tue, 29 Oct 2019 14:18:08 -0500 Subject: [PATCH] Jexer image protocol --- src/jexer/TTerminalWidget.java | 16 +- src/jexer/backend/ECMA48Terminal.java | 130 +++++++++++--- src/jexer/tterminal/ECMA48.java | 235 +++++++++++++++----------- 3 files changed, 258 insertions(+), 123 deletions(-) diff --git a/src/jexer/TTerminalWidget.java b/src/jexer/TTerminalWidget.java index a269609..6c8b989 100644 --- a/src/jexer/TTerminalWidget.java +++ b/src/jexer/TTerminalWidget.java @@ -541,9 +541,7 @@ public class TTerminalWidget extends TScrollableWidget int width = getDisplayWidth(); boolean syncEmulator = false; - if ((System.currentTimeMillis() - lastUpdateTime >= 20) - && (dirty == true) - ) { + if (System.currentTimeMillis() - lastUpdateTime >= 50) { // Too much time has passed, draw it all. syncEmulator = true; } else if (emulator.isReading() && (dirty == false)) { @@ -1125,7 +1123,17 @@ public class TTerminalWidget extends TScrollableWidget * Called by emulator when fresh data has come in. */ public void displayChanged() { - dirty = true; + if (emulator != null) { + // Force sync here: EMCA48.run() thread might be setting + // dirty=true while TTerminalWdiget.draw() is setting + // dirty=false. If these writes start interleaving, the display + // stops getting updated. + synchronized (emulator) { + dirty = true; + } + } else { + dirty = true; + } getApplication().postEvent(new TMenuEvent(TMenu.MID_REPAINT)); } diff --git a/src/jexer/backend/ECMA48Terminal.java b/src/jexer/backend/ECMA48Terminal.java index e2997d2..613a3ab 100644 --- a/src/jexer/backend/ECMA48Terminal.java +++ b/src/jexer/backend/ECMA48Terminal.java @@ -83,6 +83,16 @@ public class ECMA48Terminal extends LogicalScreen MOUSE_SGR, } + /** + * Available Jexer images support. + */ + private enum JexerImageOption { + DISABLED, + JPG, + PNG, + RGB, + } + // ------------------------------------------------------------------------ // Variables -------------------------------------------------------------- // ------------------------------------------------------------------------ @@ -217,9 +227,10 @@ public class ECMA48Terminal extends LogicalScreen private ImageCache iterm2Cache = null; /** - * If true, emit image data via Jexer image protocol. + * If not DISABLED, emit image data via Jexer image protocol if the + * terminal supports it. */ - private boolean jexerImages = false; + private JexerImageOption jexerImageOption = JexerImageOption.JPG; /** * The Jexer post-rendered string cache. @@ -1479,7 +1490,7 @@ public class ECMA48Terminal extends LogicalScreen // SQUASH } - // Default to using images for full-width characters. + // Default to not supporting iTerm2 images. if (System.getProperty("jexer.ECMA48.iTerm2Images", "false").equals("true")) { iterm2Images = true; @@ -1487,6 +1498,19 @@ public class ECMA48Terminal extends LogicalScreen iterm2Images = false; } + // Default to using JPG Jexer images if terminal supports it. + String jexerImageStr = System.getProperty("jexer.ECMA48.jexerImages", + "jpg").toLowerCase(); + if (jexerImageStr.equals("false")) { + jexerImageOption = JexerImageOption.DISABLED; + } else if (jexerImageStr.equals("jpg")) { + jexerImageOption = JexerImageOption.JPG; + } else if (jexerImageStr.equals("png")) { + jexerImageOption = JexerImageOption.PNG; + } else if (jexerImageStr.equals("rgb")) { + jexerImageOption = JexerImageOption.RGB; + } + // Set custom colors setCustomSystemColors(); } @@ -2052,7 +2076,7 @@ public class ECMA48Terminal extends LogicalScreen if (cellsToDraw.size() > 0) { if (iterm2Images) { sb.append(toIterm2Image(x, y, cellsToDraw)); - } else if (jexerImages) { + } else if (jexerImageOption != JexerImageOption.DISABLED) { sb.append(toJexerImage(x, y, cellsToDraw)); } else { sb.append(toSixel(x, y, cellsToDraw)); @@ -2806,6 +2830,7 @@ public class ECMA48Terminal extends LogicalScreen if (decPrivateModeFlag == false) { break; } + boolean jexerImages = false; for (String x: params) { if (x.equals("4")) { // Terminal reports sixel support @@ -2821,6 +2846,11 @@ public class ECMA48Terminal extends LogicalScreen jexerImages = true; } } + if (jexerImages == false) { + // Terminal does not support Jexer images, disable + // them. + jexerImageOption = JexerImageOption.DISABLED; + } return; case 't': // windowOps @@ -3424,8 +3454,7 @@ public class ECMA48Terminal extends LogicalScreen int imageWidth = cells.get(0).getImage().getWidth(); int imageHeight = cells.get(0).getImage().getHeight(); - // cells.get(x).getImage() has a dithered bitmap containing indexes - // into the color palette. Piece these together into one larger + // Piece cells.get(x).getImage() pieces together into one larger // image for final rendering. int totalWidth = 0; int fullWidth = cells.size() * getTextWidth(); @@ -3641,7 +3670,7 @@ public class ECMA48Terminal extends LogicalScreen assert (cells.size() > 0); assert (cells.get(0).getImage() != null); - if (jexerImages == false) { + if (jexerImageOption == JexerImageOption.DISABLED) { sb.append(normal()); sb.append(gotoXY(x, y)); for (int i = 0; i < cells.size(); i++) { @@ -3677,8 +3706,7 @@ public class ECMA48Terminal extends LogicalScreen int imageWidth = cells.get(0).getImage().getWidth(); int imageHeight = cells.get(0).getImage().getHeight(); - // cells.get(x).getImage() has a dithered bitmap containing indexes - // into the color palette. Piece these together into one larger + // Piece cells.get(x).getImage() pieces together into one larger // image for final rendering. int totalWidth = 0; int fullWidth = cells.size() * getTextWidth(); @@ -3774,21 +3802,77 @@ public class ECMA48Terminal extends LogicalScreen } } - sb.append(String.format("\033]444;%d;%d;0;", image.getWidth(), - Math.min(image.getHeight(), fullHeight))); - - byte [] bytes = new byte[image.getWidth() * image.getHeight() * 3]; - int stride = image.getWidth(); - for (int px = 0; px < stride; px++) { - for (int py = 0; py < image.getHeight(); py++) { - int rgb = image.getRGB(px, py); - bytes[(py * stride * 3) + (px * 3)] = (byte) ((rgb >>> 16) & 0xFF); - bytes[(py * stride * 3) + (px * 3) + 1] = (byte) ((rgb >>> 8) & 0xFF); - bytes[(py * stride * 3) + (px * 3) + 2] = (byte) ( rgb & 0xFF); + if (jexerImageOption == JexerImageOption.PNG) { + // Encode as PNG + ByteArrayOutputStream pngOutputStream = new ByteArrayOutputStream(1024); + try { + if (!ImageIO.write(image.getSubimage(0, 0, image.getWidth(), + Math.min(image.getHeight(), fullHeight)), + "PNG", pngOutputStream) + ) { + // We failed to render image, bail out. + return ""; + } + } catch (IOException e) { + // We failed to render image, bail out. + return ""; } + + sb.append("\033]444;1;0;"); + sb.append(base64.encodeToString(pngOutputStream.toByteArray())); + sb.append("\007"); + + } else if (jexerImageOption == JexerImageOption.JPG) { + + // Encode as JPG + ByteArrayOutputStream jpgOutputStream = new ByteArrayOutputStream(1024); + + // Convert from ARGB to RGB, otherwise the JPG encode will fail. + BufferedImage jpgImage = new BufferedImage(image.getWidth(), + image.getHeight(), BufferedImage.TYPE_INT_RGB); + int [] pixels = new int[image.getWidth() * image.getHeight()]; + image.getRGB(0, 0, image.getWidth(), image.getHeight(), pixels, + 0, image.getWidth()); + jpgImage.setRGB(0, 0, image.getWidth(), image.getHeight(), pixels, + 0, image.getWidth()); + + try { + if (!ImageIO.write(jpgImage.getSubimage(0, 0, + jpgImage.getWidth(), + Math.min(jpgImage.getHeight(), fullHeight)), + "JPG", jpgOutputStream) + ) { + // We failed to render image, bail out. + return ""; + } + } catch (IOException e) { + // We failed to render image, bail out. + return ""; + } + + sb.append("\033]444;2;0;"); + sb.append(base64.encodeToString(jpgOutputStream.toByteArray())); + sb.append("\007"); + + } else if (jexerImageOption == JexerImageOption.RGB) { + + // RGB + sb.append(String.format("\033]444;0;%d;%d;0;", image.getWidth(), + Math.min(image.getHeight(), fullHeight))); + + byte [] bytes = new byte[image.getWidth() * image.getHeight() * 3]; + int stride = image.getWidth(); + for (int px = 0; px < stride; px++) { + for (int py = 0; py < image.getHeight(); py++) { + int rgb = image.getRGB(px, py); + bytes[(py * stride * 3) + (px * 3)] = (byte) ((rgb >>> 16) & 0xFF); + bytes[(py * stride * 3) + (px * 3) + 1] = (byte) ((rgb >>> 8) & 0xFF); + bytes[(py * stride * 3) + (px * 3) + 2] = (byte) ( rgb & 0xFF); + } + } + sb.append(base64.encodeToString(bytes)); + sb.append("\007"); } - sb.append(base64.encodeToString(bytes)); - sb.append("\007"); if (saveInCache) { // This row is OK to save into the cache. @@ -3804,7 +3888,7 @@ public class ECMA48Terminal extends LogicalScreen * @return true if this terminal is emitting Jexer images */ public boolean hasJexerImages() { - return jexerImages; + return (jexerImageOption != JexerImageOption.DISABLED); } // ------------------------------------------------------------------------ diff --git a/src/jexer/tterminal/ECMA48.java b/src/jexer/tterminal/ECMA48.java index d492295..c393ca5 100644 --- a/src/jexer/tterminal/ECMA48.java +++ b/src/jexer/tterminal/ECMA48.java @@ -32,6 +32,7 @@ import java.awt.Graphics2D; import java.awt.image.BufferedImage; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; import java.io.CharArrayWriter; import java.io.InputStream; import java.io.InputStreamReader; @@ -46,6 +47,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import javax.imageio.ImageIO; import jexer.TKeypress; import jexer.backend.GlyphMaker; @@ -395,7 +397,7 @@ public class ECMA48 implements Runnable { /** * Non-csi collect buffer. */ - private StringBuilder collectBuffer; + private StringBuilder collectBuffer = new StringBuilder(128); /** * When true, use the G1 character set. @@ -470,7 +472,7 @@ public class ECMA48 implements Runnable { /** * Sixel collection buffer. */ - private StringBuilder sixelParseBuffer; + private StringBuilder sixelParseBuffer = new StringBuilder(2048); /** * Sixel shared palette. @@ -893,14 +895,14 @@ public class ECMA48 implements Runnable { case VT220: case XTERM: - // "I am a VT220" - 7 bit version + // "I am a VT220" - 7 bit version, with sixel and Jexer image + // support. if (!s8c1t) { - return "\033[?62;1;6;9;4;22c"; - // return "\033[?62;1;6;9;4;22;444c"; + return "\033[?62;1;6;9;4;22;444c"; } - // "I am a VT220" - 8 bit version - return "\u009b?62;1;6;9;4;22c"; - // return "\u009b?62;1;6;9;4;22;444c"; + // "I am a VT220" - 8 bit version, with sixel and Jexer image + // support. + return "\u009b?62;1;6;9;4;22;444c"; default: throw new IllegalArgumentException("Invalid device type: " + type); } @@ -1261,7 +1263,7 @@ public class ECMA48 implements Runnable { */ private void toGround() { csiParams.clear(); - collectBuffer = new StringBuilder(8); + collectBuffer.setLength(0); scanState = ScanState.GROUND; } @@ -4811,11 +4813,18 @@ public class ECMA48 implements Runnable { } } - if (p[0].equals("444") && (p.length == 5)) { - // Jexer image - parseJexerImage(p[1], p[2], p[3], p[4]); + if (p[0].equals("444")) { + if (p[1].equals("0") && (p.length == 6)) { + // Jexer image - RGB + parseJexerImageRGB(p[2], p[3], p[4], p[5]); + } else if (p[1].equals("1") && (p.length == 4)) { + // Jexer image - PNG + parseJexerImageFile(1, p[2], p[3]); + } else if (p[1].equals("2") && (p.length == 4)) { + // Jexer image - JPG + parseJexerImageFile(2, p[2], p[3]); + } } - } // Go to SCAN_GROUND state @@ -6701,7 +6710,7 @@ public class ECMA48 implements Runnable { // 0x71 goes to DCS_SIXEL if (ch == 0x71) { - sixelParseBuffer = new StringBuilder(); + sixelParseBuffer.setLength(0); scanState = ScanState.DCS_SIXEL; } else if ((ch >= 0x40) && (ch <= 0x7E)) { // 0x40-7E goes to DCS_PASSTHROUGH @@ -6786,7 +6795,7 @@ public class ECMA48 implements Runnable { // 0x71 goes to DCS_SIXEL if (ch == 0x71) { - sixelParseBuffer = new StringBuilder(); + sixelParseBuffer.setLength(0); scanState = ScanState.DCS_SIXEL; } else if ((ch >= 0x40) && (ch <= 0x7E)) { // 0x40-7E goes to DCS_PASSTHROUGH @@ -7050,87 +7059,19 @@ public class ECMA48 implements Runnable { // Sixel data was malformed in some way, bail out. return; } - - /* - * Procedure: - * - * Break up the image into text cell sized pieces as a new array of - * Cells. - * - * Note original column position x0. - * - * For each cell: - * - * 1. Advance (printCharacter(' ')) for horizontal increment, or - * index (linefeed() + cursorPosition(y, x0)) for vertical - * increment. - * - * 2. Set (x, y) cell image data. - * - * 3. For the right and bottom edges: - * - * a. Render the text to pixels using Terminus font. - * - * b. Blit the image on top of the text, using alpha channel. - */ - int cellColumns = image.getWidth() / textWidth; - if (cellColumns * textWidth < image.getWidth()) { - cellColumns++; - } - int cellRows = image.getHeight() / textHeight; - if (cellRows * textHeight < image.getHeight()) { - cellRows++; - } - - // Break the image up into an array of cells. - Cell [][] cells = new Cell[cellColumns][cellRows]; - - for (int x = 0; x < cellColumns; x++) { - for (int y = 0; y < cellRows; y++) { - - int width = textWidth; - if ((x + 1) * textWidth > image.getWidth()) { - width = image.getWidth() - (x * textWidth); - } - int height = textHeight; - if ((y + 1) * textHeight > image.getHeight()) { - height = image.getHeight() - (y * textHeight); - } - - Cell cell = new Cell(); - cell.setImage(image.getSubimage(x * textWidth, - y * textHeight, width, height)); - - cells[x][y] = cell; - } - } - - int x0 = currentState.cursorX; - for (int y = 0; y < cellRows; y++) { - for (int x = 0; x < cellColumns; x++) { - assert (currentState.cursorX <= rightMargin); - - // TODO: Render text of current cell first, then image over - // it (accounting for blank pixels). For now, just copy the - // cell. - DisplayLine line = display.get(currentState.cursorY); - line.replace(currentState.cursorX, cells[x][y]); - - // If at the end of the visible screen, stop. - if (currentState.cursorX == rightMargin) { - break; - } - // Room for more image on the visible screen. - currentState.cursorX++; - } - linefeed(); - cursorPosition(currentState.cursorY, x0); + if ((image.getWidth() < 1) + || (image.getWidth() > 10000) + || (image.getHeight() < 1) + || (image.getHeight() > 10000) + ) { + return; } + imageToCells(image, true); } /** - * Parse a "Jexer" image string into a bitmap image, and overlay that + * Parse a "Jexer" RGB image string into a bitmap image, and overlay that * image onto the text cells. * * @param pw width token @@ -7138,7 +7079,7 @@ public class ECMA48 implements Runnable { * @param ps scroll token * @param data pixel data */ - private void parseJexerImage(final String pw, final String ph, + private void parseJexerImageRGB(final String pw, final String ph, final String ps, final String data) { int imageWidth = 0; @@ -7194,6 +7135,94 @@ public class ECMA48 implements Runnable { } } + imageToCells(image, scroll); + } + + /** + * Parse a "Jexer" PNG or JPG image string into a bitmap image, and + * overlay that image onto the text cells. + * + * @param type 1 for PNG, 2 for JPG + * @param ps scroll token + * @param data pixel data + */ + private void parseJexerImageFile(final int type, final String ps, + final String data) { + + int imageWidth = 0; + int imageHeight = 0; + boolean scroll = false; + BufferedImage image = null; + try { + java.util.Base64.Decoder base64 = java.util.Base64.getDecoder(); + byte [] bytes = base64.decode(data); + + switch (type) { + case 1: + if ((bytes[0] != (byte) 0x89) + || (bytes[1] != 'P') + || (bytes[2] != 'N') + || (bytes[3] != 'G') + || (bytes[4] != (byte) 0x0D) + || (bytes[5] != (byte) 0x0A) + || (bytes[6] != (byte) 0x1A) + || (bytes[7] != (byte) 0x0A) + ) { + // File does not have PNG header, bail out. + return; + } + break; + + case 2: + if ((bytes[0] != (byte) 0XFF) + || (bytes[1] != (byte) 0xD8) + || (bytes[2] != (byte) 0xFF) + ) { + // File does not have JPG header, bail out. + return; + } + break; + + default: + // Unsupported type, bail out. + return; + } + + image = ImageIO.read(new ByteArrayInputStream(bytes)); + } catch (IOException e) { + // SQUASH + return; + } + assert (image != null); + imageWidth = image.getWidth(); + imageHeight = image.getHeight(); + if ((imageWidth < 1) + || (imageWidth > 10000) + || (imageHeight < 1) + || (imageHeight > 10000) + ) { + return; + } + if (ps.equals("1")) { + scroll = true; + } else if (ps.equals("0")) { + scroll = false; + } else { + return; + } + + imageToCells(image, scroll); + } + + /** + * Break up an image into the cells at the current cursor. + * + * @param image the image to display + * @param scroll if true, scroll the image and move the cursor + */ + private void imageToCells(final BufferedImage image, final boolean scroll) { + assert (image != null); + /* * Procedure: * @@ -7249,11 +7278,17 @@ public class ECMA48 implements Runnable { } int x0 = currentState.cursorX; + int y0 = currentState.cursorY; for (int y = 0; y < cellRows; y++) { for (int x = 0; x < cellColumns; x++) { assert (currentState.cursorX <= rightMargin); + + // TODO: Render text of current cell first, then image over + // it (accounting for blank pixels). For now, just copy the + // cell. DisplayLine line = display.get(currentState.cursorY); line.replace(currentState.cursorX, cells[x][y]); + // If at the end of the visible screen, stop. if (currentState.cursorX == rightMargin) { break; @@ -7261,15 +7296,23 @@ public class ECMA48 implements Runnable { // Room for more image on the visible screen. currentState.cursorX++; } - if ((scroll == true) - || ((scroll == false) - && (currentState.cursorY < scrollRegionBottom)) - ) { + if (currentState.cursorY < scrollRegionBottom - 1) { + // Not at the bottom, down a line. + linefeed(); + } else if (scroll == true) { + // At the bottom, scroll as needed. linefeed(); + } else { + // At the bottom, no more scrolling, done. + break; } cursorPosition(currentState.cursorY, x0); } + if (scroll == false) { + cursorPosition(y0, x0); + } + } } -- 2.27.0