hybrig image rendering

src/main/java/com/gregor/jprototerm/Compositor.java

@@ -5,6 +5,7 @@ import dev.jlibghostty.MouseButton;
 import dev.jlibghostty.MouseEncoderSize;
 import dev.jlibghostty.MouseInput;
 import javafx.geometry.VPos;
+import javafx.scene.Node;
 import javafx.scene.canvas.Canvas;
 import javafx.scene.canvas.GraphicsContext;
 import javafx.scene.input.InputEvent;
@@ -38,6 +39,8 @@ public final class Compositor {
     private static final double TAB_BAR_HEIGHT = 22.0;
 
     private final Canvas canvas = new Canvas();
+    // Kitty images are drawn as retained nodes layered over the canvas, not composited onto it.
+    private final KittyImageOverlay imageOverlay = new KittyImageOverlay();
     private final AppConfig config;
     private final TerminalMetrics metrics;
     private final List<Tab> tabs = new ArrayList<>();
@@ -75,6 +78,11 @@ public final class Compositor {
         return canvas;
     }
 
+    /** The kitty-image overlay, to be stacked directly above {@link #canvas()} in the window. */
+    public Node imageOverlay() {
+        return imageOverlay.node();
+    }
+
     public void setFont(String family, double size) {
         metrics.setFont(family, size);
         paneContentVersion.clear();
@@ -256,6 +264,7 @@ public final class Compositor {
         for (TerminalPane pane : panes) {
             paneContentVersion.put(pane, pane.paintFull(gc, isActive(pane)));
         }
+        imageOverlay.sync(panes);
     }
 
     // Repaint just the panes whose content changed, directly on the retained canvas. Each pane
@@ -272,6 +281,7 @@ public final class Compositor {
                 continue;
             }
             paneContentVersion.put(pane, pane.paintIncremental(gc, isActive(pane)));
+            imageOverlay.updatePane(pane);
         }
     }
 

src/main/java/com/gregor/jprototerm/GhosttyTerminalRenderer.java

@@ -27,6 +27,7 @@ import javafx.scene.text.Text;
 
 import java.io.ByteArrayInputStream;
 import java.nio.IntBuffer;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -60,6 +61,9 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
     // Decoded kitty images for this renderer's pane (kitty graphics state is per-terminal).
     private final Map<KittyImageKey, Image> kittyImageCache = new HashMap<>();
     private final SoftwareBackbuffer software = new SoftwareBackbuffer();
+    // Image placements produced by the last paint; the compositor reads these and renders them
+    // as overlay nodes rather than compositing them onto the canvas. Empty for non-kitty panes.
+    private List<KittyImageNode> kittyImageNodes = List.of();
 
     GhosttyTerminalRenderer(TerminalMetrics metrics) {
         this.metrics = metrics;
@@ -73,15 +77,10 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
         double height = target.height();
         gc.save();
         clip(gc, px, py, width, height, target.clip());
-        boolean withKitty = target.kittyEnabled() && hasKittyGraphics(target);
         RenderStateSnapshot snapshot = target.snapshotFull();
-        if (withKitty) {
-            drawContent(gc, target, snapshot, px, py, width, height, active, true);
-            software.invalidate();
-        } else {
         software.paintFull(gc, snapshot, px, py, width, height, active);
-        }
         gc.restore();
+        collectKittyImages(target, snapshot, px, py);
     }
 
     @Override
@@ -93,10 +92,15 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
         gc.save();
         clip(gc, px, py, width, height, target.clip());
         if (target.kittyEnabled() && hasKittyGraphics(target)) {
-            // Kitty placements can move without a per-row dirty flag, so always redraw whole.
+            // Images render as overlay nodes, not on the canvas, but their positions track the
-            drawContent(gc, target, target.snapshotFull(), px, py, width, height, active, true);
+            // grid (scroll/cursor), so we need a full snapshot to locate placeholder cells. The
-            software.invalidate();
+            // software path itself still repaints only the text rows whose hash changed.
-        } else {
+            RenderStateSnapshot snapshot = target.snapshotFull();
+            software.paintFullOrShifted(gc, snapshot, px, py, width, height, active);
+            gc.restore();
+            collectKittyImages(target, snapshot, px, py);
+            return;
+        }
         RenderStateSnapshot snapshot = target.snapshot();
         int dirty = snapshot == null ? DIRTY_FULL : snapshot.dirty();
         if (dirty == DIRTY_FULL) {
@@ -105,59 +109,13 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
             software.paintDirty(gc, target, snapshot, px, py, width, height, active);
         }
         // dirty == FALSE: nothing visible changed.
-        }
         gc.restore();
+        kittyImageNodes = List.of();
     }
 
-    // Full content render: background, border, all rows, cursor, and (when enabled) kitty
+    @Override
-    // graphics. Used by the kitty direct path and by full redraws.
+    List<KittyImageNode> kittyImages() {
-    private void drawContent(
+        return kittyImageNodes;
-            GraphicsContext gc,
-            RenderTarget target,
-            RenderStateSnapshot snapshot,
-            double x,
-            double y,
-            double width,
-            double height,
-            boolean active,
-            boolean withKitty
-    ) {
-        double cellWidth = metrics.cellWidth();
-        double lineHeight = metrics.lineHeight();
-        gc.setFontSmoothingType(FontSmoothingType.LCD);
-        gc.setFill(PANE_BACKGROUND);
-        gc.fillRect(x, y, width, height);
-        gc.setFont(metrics.font());
-
-        double left = x + TerminalMetrics.PADDING;
-        double top = y + TerminalMetrics.PADDING;
-        double baseline = top + metrics.baselineOffset();
-
-        Map<KittyPlaceholderKey, KittyPlaceholderBounds> placeholderBounds = withKitty
-                ? kittyPlaceholderBounds(snapshot)
-                : Map.of();
-
-        if (withKitty) {
-            drawKittyGraphics(gc, target, KittyPlacementLayer.BELOW_TEXT, placeholderBounds, left, top, cellWidth, lineHeight);
-        }
-
-        if (snapshot != null) {
-            double contentBottom = top + snapshot.rows() * lineHeight;
-            fillVerticalPadding(gc, snapshot, x, y, width, height, top, contentBottom);
-            for (RenderRow row : snapshot.renderRows()) {
-                double y0 = Math.floor(top + (row.row() * lineHeight));
-                double y1 = Math.ceil(top + ((row.row() + 1) * lineHeight));
-                paintSidePadding(gc, row, x, width, left, cellWidth, y0, y1 - y0);
-                drawRow(gc, row, left, top, baseline, cellWidth, lineHeight);
-            }
-            drawCursor(gc, snapshot, left, top, cellWidth, lineHeight);
-        }
-
-        if (withKitty) {
-            drawKittyGraphics(gc, target, KittyPlacementLayer.ABOVE_TEXT, placeholderBounds, left, top, cellWidth, lineHeight);
-        }
-
-        drawBorder(gc, x, y, width, height, active);
     }
 
     // Incremental render: repaint only the rows ghostty flagged dirty, then restore the
@@ -465,15 +423,34 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
                 .orElse(false);
     }
 
-    private void drawKittyGraphics(
+    // Build the image-node descriptors for the current frame. Reads the decoded-image cache and
-            GraphicsContext gc,
+    // each placement's geometry, but draws nothing — the compositor turns these into overlay
+    // nodes clipped to the pane.
+    private void collectKittyImages(RenderTarget target, RenderStateSnapshot snapshot, double px, double py) {
+        if (!target.kittyEnabled() || !hasKittyGraphics(target)) {
+            kittyImageNodes = List.of();
+            return;
+        }
+        double cellWidth = metrics.cellWidth();
+        double lineHeight = metrics.lineHeight();
+        double originX = px + TerminalMetrics.PADDING;
+        double originY = py + TerminalMetrics.PADDING;
+        Map<KittyPlaceholderKey, KittyPlaceholderBounds> placeholderBounds = kittyPlaceholderBounds(snapshot);
+        List<KittyImageNode> nodes = new ArrayList<>();
+        collectLayer(target, KittyPlacementLayer.BELOW_TEXT, placeholderBounds, originX, originY, cellWidth, lineHeight, nodes);
+        collectLayer(target, KittyPlacementLayer.ABOVE_TEXT, placeholderBounds, originX, originY, cellWidth, lineHeight, nodes);
+        kittyImageNodes = nodes;
+    }
+
+    private void collectLayer(
             RenderTarget target,
             KittyPlacementLayer layer,
             Map<KittyPlaceholderKey, KittyPlaceholderBounds> placeholderBounds,
             double originX,
             double originY,
             double cellWidth,
-            double lineHeight
+            double lineHeight,
+            List<KittyImageNode> out
     ) {
         target.kittyGraphics().ifPresent(graphics -> {
             for (KittyPlacement placement : graphics.placements(layer)) {
@@ -482,17 +459,17 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
                     continue;
                 }
 
-                if (placement.virtual()) {
+                KittyImageNode node = placement.virtual()
-                    drawVirtualKittyPlacement(gc, placement, image, placeholderBounds, originX, originY, cellWidth, lineHeight);
+                        ? virtualNode(placement, image, placeholderBounds, originX, originY, cellWidth, lineHeight)
-                } else {
+                        : pinnedNode(placement, image, originX, originY, cellWidth, lineHeight);
-                    drawPinnedKittyPlacement(gc, placement, image, originX, originY, cellWidth, lineHeight);
+                if (node != null) {
+                    out.add(node);
                 }
             }
         });
     }
 
-    private static void drawPinnedKittyPlacement(
+    private static KittyImageNode pinnedNode(
-            GraphicsContext gc,
             KittyPlacement placement,
             Image image,
             double originX,
@@ -502,15 +479,13 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
     ) {
         KittyRenderInfo renderInfo = placement.renderInfo().orElse(null);
         if (renderInfo == null || !renderInfo.viewportVisible()) {
-            return;
+            return null;
         }
 
-        double sourceX = renderInfo.sourceX();
-        double sourceY = renderInfo.sourceY();
         double sourceWidth = renderInfo.sourceWidth();
         double sourceHeight = renderInfo.sourceHeight();
         if (sourceWidth <= 0.0 || sourceHeight <= 0.0) {
-            return;
+            return null;
         }
 
         double x = originX + (renderInfo.viewportColumn() * cellWidth) + placement.xOffset();
@@ -518,14 +493,14 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
         double width = renderInfo.pixelWidth() > 0 ? renderInfo.pixelWidth() : renderInfo.gridColumns() * cellWidth;
         double height = renderInfo.pixelHeight() > 0 ? renderInfo.pixelHeight() : renderInfo.gridRows() * lineHeight;
         if (width <= 0.0 || height <= 0.0) {
-            return;
+            return null;
         }
 
-        gc.drawImage(image, sourceX, sourceY, sourceWidth, sourceHeight, x, y, width, height);
+        return new KittyImageNode(placement.imageId(), placement.placementId(), image,
+                renderInfo.sourceX(), renderInfo.sourceY(), sourceWidth, sourceHeight, x, y, width, height);
     }
 
-    private static void drawVirtualKittyPlacement(
+    private static KittyImageNode virtualNode(
-            GraphicsContext gc,
             KittyPlacement placement,
             Image image,
             Map<KittyPlaceholderKey, KittyPlaceholderBounds> placeholderBounds,
@@ -546,12 +521,12 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
                     .orElse(null);
         }
         if (bounds == null || bounds.isEmpty()) {
-            return;
+            return null;
         }
 
         SourceRect source = sourceRect(placement, image);
         if (source.width() <= 0.0 || source.height() <= 0.0) {
-            return;
+            return null;
         }
 
         long gridColumns = gridColumns(placement, bounds);
@@ -569,13 +544,14 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
         double availableHeight = bounds.rows() * lineHeight;
 
         if (sourceWidth <= 0.0 || sourceHeight <= 0.0 || availableWidth <= 0.0 || availableHeight <= 0.0) {
-            return;
+            return null;
         }
 
         double scale = Math.min(availableWidth / sourceWidth, availableHeight / sourceHeight);
         double width = sourceWidth * scale;
         double height = sourceHeight * scale;
-        gc.drawImage(image, sourceX, sourceY, sourceWidth, sourceHeight, x, y, width, height);
+        return new KittyImageNode(placement.imageId(), placement.placementId(), image,
+                sourceX, sourceY, sourceWidth, sourceHeight, x, y, width, height);
     }
 
     private static long gridColumns(KittyPlacement placement, KittyPlaceholderBounds bounds) {

src/main/java/com/gregor/jprototerm/KittyImageNode.java

@@ -0,0 +1,32 @@
+package com.gregor.jprototerm;
+
+import javafx.scene.image.Image;
+
+/**
+ * A single kitty image to display, produced by the renderer and consumed by {@link
+ * KittyImageOverlay}. Images are not painted onto the canvas; each becomes a retained
+ * {@code ImageView} node positioned over the pane. The {@code source*} fields are the region of
+ * {@link #image()} to show (in image pixels); the {@code x/y/width/height} are where to put it,
+ * in scene coordinates (the same space the pane's clip {@code Shape} lives in).
+ *
+ * <p>{@code imageId}+{@code placementId} identify the placement so the overlay can reuse the
+ * same node across frames instead of recreating it.
+ */
+record KittyImageNode(
+        long imageId,
+        long placementId,
+        Image image,
+        double sourceX,
+        double sourceY,
+        double sourceWidth,
+        double sourceHeight,
+        double x,
+        double y,
+        double width,
+        double height
+) {
+    /** Stable per-pane key for node reuse. Packs the two u32 ids without collision. */
+    long key() {
+        return (imageId << 32) | (placementId & 0xffffffffL);
+    }
+}

src/main/java/com/gregor/jprototerm/KittyImageOverlay.java

@@ -0,0 +1,147 @@
+package com.gregor.jprototerm;
+
+import javafx.geometry.Rectangle2D;
+import javafx.scene.Group;
+import javafx.scene.Node;
+import javafx.scene.image.ImageView;
+import javafx.scene.layout.Pane;
+import javafx.scene.shape.Rectangle;
+import javafx.scene.shape.Shape;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Renders kitty graphics images as retained scene-graph nodes layered over the {@link Compositor}
+ * canvas, instead of compositing them onto the canvas. Each pane gets a {@link Group} clipped to
+ * that pane's region (the same clip {@code Shape} the canvas renderer uses), and each visible
+ * image placement is an {@link ImageView} inside it, reused across frames so an unchanged image
+ * costs nothing to redraw.
+ *
+ * <p>The overlay {@link #node()} is mouse-transparent and sits above the canvas in the window's
+ * {@code StackPane}; its children use scene coordinates, which line up with the canvas because
+ * both fill the same root.
+ */
+final class KittyImageOverlay {
+    private final Pane root = new Pane();
+    private final Map<TerminalPane, PaneOverlay> overlays = new HashMap<>();
+
+    KittyImageOverlay() {
+        // Input belongs to the canvas underneath; the overlay only shows pixels.
+        root.setMouseTransparent(true);
+        root.setManaged(false);
+    }
+
+    Node node() {
+        return root;
+    }
+
+    /**
+     * Full reconcile to {@code panes} (bottom-to-top): drop overlays for panes that went away,
+     * refresh each surviving/added pane's images and clip, and order the per-pane groups to match
+     * the pane z-order. Called on layout frames, after the panes have painted.
+     */
+    void sync(List<TerminalPane> panes) {
+        Iterator<Map.Entry<TerminalPane, PaneOverlay>> it = overlays.entrySet().iterator();
+        while (it.hasNext()) {
+            Map.Entry<TerminalPane, PaneOverlay> entry = it.next();
+            if (!panes.contains(entry.getKey())) {
+                root.getChildren().remove(entry.getValue().group);
+                it.remove();
+            }
+        }
+        for (TerminalPane pane : panes) {
+            updatePane(pane);
+        }
+        List<Node> ordered = new ArrayList<>(panes.size());
+        for (TerminalPane pane : panes) {
+            ordered.add(overlays.get(pane).group);
+        }
+        if (!root.getChildren().equals(ordered)) {
+            root.getChildren().setAll(ordered);
+        }
+    }
+
+    /**
+     * Refresh one pane's images and clip (called on content frames for each repainted pane).
+     * Creates the pane's group if this is the first time it has shown an image.
+     */
+    void updatePane(TerminalPane pane) {
+        List<KittyImageNode> images = pane.kittyImages();
+        PaneOverlay overlay = overlays.get(pane);
+        if (overlay == null) {
+            if (images.isEmpty()) {
+                return;
+            }
+            overlay = new PaneOverlay();
+            overlays.put(pane, overlay);
+            root.getChildren().add(overlay.group);
+        }
+        overlay.group.setClip(clipFor(pane));
+        reconcile(overlay, images);
+    }
+
+    private static void reconcile(PaneOverlay overlay, List<KittyImageNode> images) {
+        Set<Long> seen = new HashSet<>();
+        for (KittyImageNode node : images) {
+            long key = node.key();
+            seen.add(key);
+            ImageView view = overlay.views.get(key);
+            if (view == null) {
+                view = new ImageView();
+                view.setManaged(false);
+                view.setSmooth(true);
+                view.setPreserveRatio(false);
+                overlay.views.put(key, view);
+                overlay.group.getChildren().add(view);
+            }
+            apply(view, node);
+        }
+        if (overlay.views.size() == seen.size()) {
+            return;
+        }
+        Iterator<Map.Entry<Long, ImageView>> it = overlay.views.entrySet().iterator();
+        while (it.hasNext()) {
+            Map.Entry<Long, ImageView> entry = it.next();
+            if (!seen.contains(entry.getKey())) {
+                overlay.group.getChildren().remove(entry.getValue());
+                it.remove();
+            }
+        }
+    }
+
+    private static void apply(ImageView view, KittyImageNode node) {
+        if (view.getImage() != node.image()) {
+            view.setImage(node.image());
+        }
+        view.setViewport(new Rectangle2D(node.sourceX(), node.sourceY(), node.sourceWidth(), node.sourceHeight()));
+        view.setFitWidth(node.width());
+        view.setFitHeight(node.height());
+        view.setLayoutX(node.x());
+        view.setLayoutY(node.y());
+    }
+
+    // The pane's occlusion clip when one is set (rect minus covering panes), else the pane's
+    // plain bounds so an image can't spill outside its pane. Matches Tab's pixel snapping.
+    private static Shape clipFor(TerminalPane pane) {
+        Shape clip = pane.clip();
+        if (clip != null) {
+            return clip;
+        }
+        return new Rectangle(Math.round(pane.x()), Math.round(pane.y()), pane.width(), pane.height());
+    }
+
+    private static final class PaneOverlay {
+        private final Group group = new Group();
+        private final Map<Long, ImageView> views = new HashMap<>();
+
+        private PaneOverlay() {
+            group.setManaged(false);
+        }
+    }
+}

src/main/java/com/gregor/jprototerm/Main.java

@@ -32,7 +32,7 @@ public final class Main extends Application {
         metrics = new TerminalMetrics(config.fontFamily(), config.fontSize());
         compositor = new Compositor(config, metrics);
 
-        StackPane root = new StackPane(compositor.canvas());
+        StackPane root = new StackPane(compositor.canvas(), compositor.imageOverlay());
         compositor.canvas().widthProperty().bind(root.widthProperty());
         compositor.canvas().heightProperty().bind(root.heightProperty());
 

src/main/java/com/gregor/jprototerm/TerminalPane.java

@@ -294,6 +294,14 @@ public final class TerminalPane implements AutoCloseable, RenderTarget {
         return snapshotVersion;
     }
 
+    /**
+     * Kitty image placements from the most recent paint, in scene coordinates. The compositor
+     * renders these as overlay nodes clipped to this pane (see {@link KittyImageOverlay}).
+     */
+    public java.util.List<KittyImageNode> kittyImages() {
+        return renderer.kittyImages();
+    }
+
     @Override
     public void close() {
         if (session != null) {

src/main/java/com/gregor/jprototerm/TerminalRenderer.java

@@ -25,6 +25,14 @@ abstract class TerminalRenderer {
     /** Repaint only what changed since the last frame, clipped to the target's clip region. */
     abstract void paintIncremental(GraphicsContext gc, RenderTarget target, boolean active);
 
+    /**
+     * The kitty image placements produced by the most recent paint, for the compositor to render
+     * as overlay nodes above the canvas. Empty unless the last paint found visible images.
+     */
+    java.util.List<KittyImageNode> kittyImages() {
+        return java.util.List.of();
+    }
+
     protected static void clipRect(GraphicsContext gc, double x, double y, double width, double height) {
         gc.beginPath();
         gc.rect(x, y, width, height);