Jexer image protocol
authorKevin Lamonte <kevin.lamonte@gmail.com>
Tue, 29 Oct 2019 19:18:08 +0000 (14:18 -0500)
committerKevin Lamonte <kevin.lamonte@gmail.com>
Tue, 29 Oct 2019 19:18:08 +0000 (14:18 -0500)
src/jexer/TTerminalWidget.java
src/jexer/backend/ECMA48Terminal.java
src/jexer/tterminal/ECMA48.java

index a2696092ce82ee7ed0550a0019b6b8fc0c9c785d..6c8b9894a016c70797e123c96a35911bb864914e 100644 (file)
@@ -541,9 +541,7 @@ public class TTerminalWidget extends TScrollableWidget
         int width = getDisplayWidth();
 
         boolean syncEmulator = false;
-        if ((System.currentTimeMillis() - lastUpdateTime >= 20)
-            && (dirty == true)
-        ) {
+        if (System.currentTimeMillis() - lastUpdateTime >= 50) {
             // Too much time has passed, draw it all.
             syncEmulator = true;
         } else if (emulator.isReading() && (dirty == false)) {
@@ -1125,7 +1123,17 @@ public class TTerminalWidget extends TScrollableWidget
      * Called by emulator when fresh data has come in.
      */
     public void displayChanged() {
-        dirty = true;
+        if (emulator != null) {
+            // Force sync here: EMCA48.run() thread might be setting
+            // dirty=true while TTerminalWdiget.draw() is setting
+            // dirty=false.  If these writes start interleaving, the display
+            // stops getting updated.
+            synchronized (emulator) {
+                dirty = true;
+            }
+        } else {
+            dirty = true;
+        }
         getApplication().postEvent(new TMenuEvent(TMenu.MID_REPAINT));
     }
 
index e2997d2f6b17486356ddd9d902d114039258bdef..613a3abd8f8f2feee14022645469019b6c597201 100644 (file)
@@ -83,6 +83,16 @@ public class ECMA48Terminal extends LogicalScreen
         MOUSE_SGR,
     }
 
+    /**
+     * Available Jexer images support.
+     */
+    private enum JexerImageOption {
+        DISABLED,
+        JPG,
+        PNG,
+        RGB,
+    }
+
     // ------------------------------------------------------------------------
     // Variables --------------------------------------------------------------
     // ------------------------------------------------------------------------
@@ -217,9 +227,10 @@ public class ECMA48Terminal extends LogicalScreen
     private ImageCache iterm2Cache = null;
 
     /**
-     * If true, emit image data via Jexer image protocol.
+     * If not DISABLED, emit image data via Jexer image protocol if the
+     * terminal supports it.
      */
-    private boolean jexerImages = false;
+    private JexerImageOption jexerImageOption = JexerImageOption.JPG;
 
     /**
      * The Jexer post-rendered string cache.
@@ -1479,7 +1490,7 @@ public class ECMA48Terminal extends LogicalScreen
             // SQUASH
         }
 
-        // Default to using images for full-width characters.
+        // Default to not supporting iTerm2 images.
         if (System.getProperty("jexer.ECMA48.iTerm2Images",
                 "false").equals("true")) {
             iterm2Images = true;
@@ -1487,6 +1498,19 @@ public class ECMA48Terminal extends LogicalScreen
             iterm2Images = false;
         }
 
+        // Default to using JPG Jexer images if terminal supports it.
+        String jexerImageStr = System.getProperty("jexer.ECMA48.jexerImages",
+            "jpg").toLowerCase();
+        if (jexerImageStr.equals("false")) {
+            jexerImageOption = JexerImageOption.DISABLED;
+        } else if (jexerImageStr.equals("jpg")) {
+            jexerImageOption = JexerImageOption.JPG;
+        } else if (jexerImageStr.equals("png")) {
+            jexerImageOption = JexerImageOption.PNG;
+        } else if (jexerImageStr.equals("rgb")) {
+            jexerImageOption = JexerImageOption.RGB;
+        }
+
         // Set custom colors
         setCustomSystemColors();
     }
@@ -2052,7 +2076,7 @@ public class ECMA48Terminal extends LogicalScreen
                 if (cellsToDraw.size() > 0) {
                     if (iterm2Images) {
                         sb.append(toIterm2Image(x, y, cellsToDraw));
-                    } else if (jexerImages) {
+                    } else if (jexerImageOption != JexerImageOption.DISABLED) {
                         sb.append(toJexerImage(x, y, cellsToDraw));
                     } else {
                         sb.append(toSixel(x, y, cellsToDraw));
@@ -2806,6 +2830,7 @@ public class ECMA48Terminal extends LogicalScreen
                     if (decPrivateModeFlag == false) {
                         break;
                     }
+                    boolean jexerImages = false;
                     for (String x: params) {
                         if (x.equals("4")) {
                             // Terminal reports sixel support
@@ -2821,6 +2846,11 @@ public class ECMA48Terminal extends LogicalScreen
                             jexerImages = true;
                         }
                     }
+                    if (jexerImages == false) {
+                        // Terminal does not support Jexer images, disable
+                        // them.
+                        jexerImageOption = JexerImageOption.DISABLED;
+                    }
                     return;
                 case 't':
                     // windowOps
@@ -3424,8 +3454,7 @@ public class ECMA48Terminal extends LogicalScreen
         int imageWidth = cells.get(0).getImage().getWidth();
         int imageHeight = cells.get(0).getImage().getHeight();
 
-        // cells.get(x).getImage() has a dithered bitmap containing indexes
-        // into the color palette.  Piece these together into one larger
+        // Piece cells.get(x).getImage() pieces together into one larger
         // image for final rendering.
         int totalWidth = 0;
         int fullWidth = cells.size() * getTextWidth();
@@ -3641,7 +3670,7 @@ public class ECMA48Terminal extends LogicalScreen
         assert (cells.size() > 0);
         assert (cells.get(0).getImage() != null);
 
-        if (jexerImages == false) {
+        if (jexerImageOption == JexerImageOption.DISABLED) {
             sb.append(normal());
             sb.append(gotoXY(x, y));
             for (int i = 0; i < cells.size(); i++) {
@@ -3677,8 +3706,7 @@ public class ECMA48Terminal extends LogicalScreen
         int imageWidth = cells.get(0).getImage().getWidth();
         int imageHeight = cells.get(0).getImage().getHeight();
 
-        // cells.get(x).getImage() has a dithered bitmap containing indexes
-        // into the color palette.  Piece these together into one larger
+        // Piece cells.get(x).getImage() pieces together into one larger
         // image for final rendering.
         int totalWidth = 0;
         int fullWidth = cells.size() * getTextWidth();
@@ -3774,21 +3802,77 @@ public class ECMA48Terminal extends LogicalScreen
             }
         }
 
-        sb.append(String.format("\033]444;%d;%d;0;", image.getWidth(),
-                Math.min(image.getHeight(), fullHeight)));
-
-        byte [] bytes = new byte[image.getWidth() * image.getHeight() * 3];
-        int stride = image.getWidth();
-        for (int px = 0; px < stride; px++) {
-            for (int py = 0; py < image.getHeight(); py++) {
-                int rgb = image.getRGB(px, py);
-                bytes[(py * stride * 3) + (px * 3)]     = (byte) ((rgb >>> 16) & 0xFF);
-                bytes[(py * stride * 3) + (px * 3) + 1] = (byte) ((rgb >>>  8) & 0xFF);
-                bytes[(py * stride * 3) + (px * 3) + 2] = (byte) ( rgb         & 0xFF);
+        if (jexerImageOption == JexerImageOption.PNG) {
+            // Encode as PNG
+            ByteArrayOutputStream pngOutputStream = new ByteArrayOutputStream(1024);
+            try {
+                if (!ImageIO.write(image.getSubimage(0, 0, image.getWidth(),
+                            Math.min(image.getHeight(), fullHeight)),
+                        "PNG", pngOutputStream)
+                ) {
+                    // We failed to render image, bail out.
+                    return "";
+                }
+            } catch (IOException e) {
+                // We failed to render image, bail out.
+                return "";
             }
+
+            sb.append("\033]444;1;0;");
+            sb.append(base64.encodeToString(pngOutputStream.toByteArray()));
+            sb.append("\007");
+
+        } else if (jexerImageOption == JexerImageOption.JPG) {
+
+            // Encode as JPG
+            ByteArrayOutputStream jpgOutputStream = new ByteArrayOutputStream(1024);
+
+            // Convert from ARGB to RGB, otherwise the JPG encode will fail.
+            BufferedImage jpgImage = new BufferedImage(image.getWidth(),
+                image.getHeight(), BufferedImage.TYPE_INT_RGB);
+            int [] pixels = new int[image.getWidth() * image.getHeight()];
+            image.getRGB(0, 0, image.getWidth(), image.getHeight(), pixels,
+                0, image.getWidth());
+            jpgImage.setRGB(0, 0, image.getWidth(), image.getHeight(), pixels,
+                0, image.getWidth());
+
+            try {
+                if (!ImageIO.write(jpgImage.getSubimage(0, 0,
+                            jpgImage.getWidth(),
+                            Math.min(jpgImage.getHeight(), fullHeight)),
+                        "JPG", jpgOutputStream)
+                ) {
+                    // We failed to render image, bail out.
+                    return "";
+                }
+            } catch (IOException e) {
+                // We failed to render image, bail out.
+                return "";
+            }
+
+            sb.append("\033]444;2;0;");
+            sb.append(base64.encodeToString(jpgOutputStream.toByteArray()));
+            sb.append("\007");
+
+        } else if (jexerImageOption == JexerImageOption.RGB) {
+
+            // RGB
+            sb.append(String.format("\033]444;0;%d;%d;0;", image.getWidth(),
+                    Math.min(image.getHeight(), fullHeight)));
+
+            byte [] bytes = new byte[image.getWidth() * image.getHeight() * 3];
+            int stride = image.getWidth();
+            for (int px = 0; px < stride; px++) {
+                for (int py = 0; py < image.getHeight(); py++) {
+                    int rgb = image.getRGB(px, py);
+                    bytes[(py * stride * 3) + (px * 3)]     = (byte) ((rgb >>> 16) & 0xFF);
+                    bytes[(py * stride * 3) + (px * 3) + 1] = (byte) ((rgb >>>  8) & 0xFF);
+                    bytes[(py * stride * 3) + (px * 3) + 2] = (byte) ( rgb         & 0xFF);
+                }
+            }
+            sb.append(base64.encodeToString(bytes));
+            sb.append("\007");
         }
-        sb.append(base64.encodeToString(bytes));
-        sb.append("\007");
 
         if (saveInCache) {
             // This row is OK to save into the cache.
@@ -3804,7 +3888,7 @@ public class ECMA48Terminal extends LogicalScreen
      * @return true if this terminal is emitting Jexer images
      */
     public boolean hasJexerImages() {
-        return jexerImages;
+        return (jexerImageOption != JexerImageOption.DISABLED);
     }
 
     // ------------------------------------------------------------------------
index d49229550f26bdae055c48297c5b66fa477cf486..c393ca50d46b3f5bd1909ff78e2d312f1f3d8cb7 100644 (file)
@@ -32,6 +32,7 @@ import java.awt.Graphics2D;
 import java.awt.image.BufferedImage;
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
 import java.io.CharArrayWriter;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -46,6 +47,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import javax.imageio.ImageIO;
 
 import jexer.TKeypress;
 import jexer.backend.GlyphMaker;
@@ -395,7 +397,7 @@ public class ECMA48 implements Runnable {
     /**
      * Non-csi collect buffer.
      */
-    private StringBuilder collectBuffer;
+    private StringBuilder collectBuffer = new StringBuilder(128);
 
     /**
      * When true, use the G1 character set.
@@ -470,7 +472,7 @@ public class ECMA48 implements Runnable {
     /**
      * Sixel collection buffer.
      */
-    private StringBuilder sixelParseBuffer;
+    private StringBuilder sixelParseBuffer = new StringBuilder(2048);
 
     /**
      * Sixel shared palette.
@@ -893,14 +895,14 @@ public class ECMA48 implements Runnable {
 
         case VT220:
         case XTERM:
-            // "I am a VT220" - 7 bit version
+            // "I am a VT220" - 7 bit version, with sixel and Jexer image
+            // support.
             if (!s8c1t) {
-                return "\033[?62;1;6;9;4;22c";
-                // return "\033[?62;1;6;9;4;22;444c";
+                return "\033[?62;1;6;9;4;22;444c";
             }
-            // "I am a VT220" - 8 bit version
-            return "\u009b?62;1;6;9;4;22c";
-            // return "\u009b?62;1;6;9;4;22;444c";
+            // "I am a VT220" - 8 bit version, with sixel and Jexer image
+            // support.
+            return "\u009b?62;1;6;9;4;22;444c";
         default:
             throw new IllegalArgumentException("Invalid device type: " + type);
         }
@@ -1261,7 +1263,7 @@ public class ECMA48 implements Runnable {
      */
     private void toGround() {
         csiParams.clear();
-        collectBuffer = new StringBuilder(8);
+        collectBuffer.setLength(0);
         scanState = ScanState.GROUND;
     }
 
@@ -4811,11 +4813,18 @@ public class ECMA48 implements Runnable {
                     }
                 }
 
-                if (p[0].equals("444") && (p.length == 5)) {
-                    // Jexer image
-                    parseJexerImage(p[1], p[2], p[3], p[4]);
+                if (p[0].equals("444")) {
+                    if (p[1].equals("0") && (p.length == 6)) {
+                        // Jexer image - RGB
+                        parseJexerImageRGB(p[2], p[3], p[4], p[5]);
+                    } else if (p[1].equals("1") && (p.length == 4)) {
+                        // Jexer image - PNG
+                        parseJexerImageFile(1, p[2], p[3]);
+                    } else if (p[1].equals("2") && (p.length == 4)) {
+                        // Jexer image - JPG
+                        parseJexerImageFile(2, p[2], p[3]);
+                    }
                 }
-
             }
 
             // Go to SCAN_GROUND state
@@ -6701,7 +6710,7 @@ public class ECMA48 implements Runnable {
 
             // 0x71 goes to DCS_SIXEL
             if (ch == 0x71) {
-                sixelParseBuffer = new StringBuilder();
+                sixelParseBuffer.setLength(0);
                 scanState = ScanState.DCS_SIXEL;
             } else if ((ch >= 0x40) && (ch <= 0x7E)) {
                 // 0x40-7E goes to DCS_PASSTHROUGH
@@ -6786,7 +6795,7 @@ public class ECMA48 implements Runnable {
 
             // 0x71 goes to DCS_SIXEL
             if (ch == 0x71) {
-                sixelParseBuffer = new StringBuilder();
+                sixelParseBuffer.setLength(0);
                 scanState = ScanState.DCS_SIXEL;
             } else if ((ch >= 0x40) && (ch <= 0x7E)) {
                 // 0x40-7E goes to DCS_PASSTHROUGH
@@ -7050,87 +7059,19 @@ public class ECMA48 implements Runnable {
             // Sixel data was malformed in some way, bail out.
             return;
         }
-
-        /*
-         * Procedure:
-         *
-         * Break up the image into text cell sized pieces as a new array of
-         * Cells.
-         *
-         * Note original column position x0.
-         *
-         * For each cell:
-         *
-         * 1. Advance (printCharacter(' ')) for horizontal increment, or
-         *    index (linefeed() + cursorPosition(y, x0)) for vertical
-         *    increment.
-         *
-         * 2. Set (x, y) cell image data.
-         *
-         * 3. For the right and bottom edges:
-         *
-         *   a. Render the text to pixels using Terminus font.
-         *
-         *   b. Blit the image on top of the text, using alpha channel.
-         */
-        int cellColumns = image.getWidth() / textWidth;
-        if (cellColumns * textWidth < image.getWidth()) {
-            cellColumns++;
-        }
-        int cellRows = image.getHeight() / textHeight;
-        if (cellRows * textHeight < image.getHeight()) {
-            cellRows++;
-        }
-
-        // Break the image up into an array of cells.
-        Cell [][] cells = new Cell[cellColumns][cellRows];
-
-        for (int x = 0; x < cellColumns; x++) {
-            for (int y = 0; y < cellRows; y++) {
-
-                int width = textWidth;
-                if ((x + 1) * textWidth > image.getWidth()) {
-                    width = image.getWidth() - (x * textWidth);
-                }
-                int height = textHeight;
-                if ((y + 1) * textHeight > image.getHeight()) {
-                    height = image.getHeight() - (y * textHeight);
-                }
-
-                Cell cell = new Cell();
-                cell.setImage(image.getSubimage(x * textWidth,
-                        y * textHeight, width, height));
-
-                cells[x][y] = cell;
-            }
-        }
-
-        int x0 = currentState.cursorX;
-        for (int y = 0; y < cellRows; y++) {
-            for (int x = 0; x < cellColumns; x++) {
-                assert (currentState.cursorX <= rightMargin);
-
-                // TODO: Render text of current cell first, then image over
-                // it (accounting for blank pixels).  For now, just copy the
-                // cell.
-                DisplayLine line = display.get(currentState.cursorY);
-                line.replace(currentState.cursorX, cells[x][y]);
-
-                // If at the end of the visible screen, stop.
-                if (currentState.cursorX == rightMargin) {
-                    break;
-                }
-                // Room for more image on the visible screen.
-                currentState.cursorX++;
-            }
-            linefeed();
-            cursorPosition(currentState.cursorY, x0);
+        if ((image.getWidth() < 1)
+            || (image.getWidth() > 10000)
+            || (image.getHeight() < 1)
+            || (image.getHeight() > 10000)
+        ) {
+            return;
         }
 
+        imageToCells(image, true);
     }
 
     /**
-     * Parse a "Jexer" image string into a bitmap image, and overlay that
+     * Parse a "Jexer" RGB image string into a bitmap image, and overlay that
      * image onto the text cells.
      *
      * @param pw width token
@@ -7138,7 +7079,7 @@ public class ECMA48 implements Runnable {
      * @param ps scroll token
      * @param data pixel data
      */
-    private void parseJexerImage(final String pw, final String ph,
+    private void parseJexerImageRGB(final String pw, final String ph,
         final String ps, final String data) {
 
         int imageWidth = 0;
@@ -7194,6 +7135,94 @@ public class ECMA48 implements Runnable {
             }
         }
 
+        imageToCells(image, scroll);
+    }
+
+    /**
+     * Parse a "Jexer" PNG or JPG image string into a bitmap image, and
+     * overlay that image onto the text cells.
+     *
+     * @param type 1 for PNG, 2 for JPG
+     * @param ps scroll token
+     * @param data pixel data
+     */
+    private void parseJexerImageFile(final int type, final String ps,
+        final String data) {
+
+        int imageWidth = 0;
+        int imageHeight = 0;
+        boolean scroll = false;
+        BufferedImage image = null;
+        try {
+            java.util.Base64.Decoder base64 = java.util.Base64.getDecoder();
+            byte [] bytes = base64.decode(data);
+
+            switch (type) {
+            case 1:
+                if ((bytes[0] != (byte) 0x89)
+                    || (bytes[1] != 'P')
+                    || (bytes[2] != 'N')
+                    || (bytes[3] != 'G')
+                    || (bytes[4] != (byte) 0x0D)
+                    || (bytes[5] != (byte) 0x0A)
+                    || (bytes[6] != (byte) 0x1A)
+                    || (bytes[7] != (byte) 0x0A)
+                ) {
+                    // File does not have PNG header, bail out.
+                    return;
+                }
+                break;
+
+            case 2:
+                if ((bytes[0] != (byte) 0XFF)
+                    || (bytes[1] != (byte) 0xD8)
+                    || (bytes[2] != (byte) 0xFF)
+                ) {
+                    // File does not have JPG header, bail out.
+                    return;
+                }
+                break;
+
+            default:
+                // Unsupported type, bail out.
+                return;
+            }
+
+            image = ImageIO.read(new ByteArrayInputStream(bytes));
+        } catch (IOException e) {
+            // SQUASH
+            return;
+        }
+        assert (image != null);
+        imageWidth = image.getWidth();
+        imageHeight = image.getHeight();
+        if ((imageWidth < 1)
+            || (imageWidth > 10000)
+            || (imageHeight < 1)
+            || (imageHeight > 10000)
+        ) {
+            return;
+        }
+        if (ps.equals("1")) {
+            scroll = true;
+        } else if (ps.equals("0")) {
+            scroll = false;
+        } else {
+            return;
+        }
+
+        imageToCells(image, scroll);
+    }
+
+    /**
+     * Break up an image into the cells at the current cursor.
+     *
+     * @param image the image to display
+     * @param scroll if true, scroll the image and move the cursor
+     */
+    private void imageToCells(final BufferedImage image, final boolean scroll) {
+        assert (image != null);
+
         /*
          * Procedure:
          *
@@ -7249,11 +7278,17 @@ public class ECMA48 implements Runnable {
         }
 
         int x0 = currentState.cursorX;
+        int y0 = currentState.cursorY;
         for (int y = 0; y < cellRows; y++) {
             for (int x = 0; x < cellColumns; x++) {
                 assert (currentState.cursorX <= rightMargin);
+
+                // TODO: Render text of current cell first, then image over
+                // it (accounting for blank pixels).  For now, just copy the
+                // cell.
                 DisplayLine line = display.get(currentState.cursorY);
                 line.replace(currentState.cursorX, cells[x][y]);
+
                 // If at the end of the visible screen, stop.
                 if (currentState.cursorX == rightMargin) {
                     break;
@@ -7261,15 +7296,23 @@ public class ECMA48 implements Runnable {
                 // Room for more image on the visible screen.
                 currentState.cursorX++;
             }
-            if ((scroll == true)
-                || ((scroll == false)
-                    && (currentState.cursorY < scrollRegionBottom))
-            ) {
+            if (currentState.cursorY < scrollRegionBottom - 1) {
+                // Not at the bottom, down a line.
+                linefeed();
+            } else if (scroll == true) {
+                // At the bottom, scroll as needed.
                 linefeed();
+            } else {
+                // At the bottom, no more scrolling, done.
+                break;
             }
             cursorPosition(currentState.cursorY, x0);
         }
 
+        if (scroll == false) {
+            cursorPosition(y0, x0);
+        }
+
     }
 
 }