From 85f2d86c097b32d9893fa414f5e3ef17faacb2d1 Mon Sep 17 00:00:00 2001 From: Gregor Lohaus Date: Mon, 1 Jun 2026 02:45:46 +0200 Subject: [PATCH] hybrig image rendering --- .../com/gregor/jprototerm/Compositor.java | 10 ++ .../jprototerm/GhosttyTerminalRenderer.java | 154 ++++++++---------- .../com/gregor/jprototerm/KittyImageNode.java | 32 ++++ .../gregor/jprototerm/KittyImageOverlay.java | 147 +++++++++++++++++ src/main/java/com/gregor/jprototerm/Main.java | 2 +- .../com/gregor/jprototerm/TerminalPane.java | 8 + .../gregor/jprototerm/TerminalRenderer.java | 8 + 7 files changed, 271 insertions(+), 90 deletions(-) create mode 100644 src/main/java/com/gregor/jprototerm/KittyImageNode.java create mode 100644 src/main/java/com/gregor/jprototerm/KittyImageOverlay.java diff --git a/src/main/java/com/gregor/jprototerm/Compositor.java b/src/main/java/com/gregor/jprototerm/Compositor.java index 69e25b6..638175e 100644 --- a/src/main/java/com/gregor/jprototerm/Compositor.java +++ b/src/main/java/com/gregor/jprototerm/Compositor.java @@ -5,6 +5,7 @@ import dev.jlibghostty.MouseButton; import dev.jlibghostty.MouseEncoderSize; import dev.jlibghostty.MouseInput; import javafx.geometry.VPos; +import javafx.scene.Node; import javafx.scene.canvas.Canvas; import javafx.scene.canvas.GraphicsContext; import javafx.scene.input.InputEvent; @@ -38,6 +39,8 @@ public final class Compositor { private static final double TAB_BAR_HEIGHT = 22.0; private final Canvas canvas = new Canvas(); + // Kitty images are drawn as retained nodes layered over the canvas, not composited onto it. + private final KittyImageOverlay imageOverlay = new KittyImageOverlay(); private final AppConfig config; private final TerminalMetrics metrics; private final List tabs = new ArrayList<>(); @@ -75,6 +78,11 @@ public final class Compositor { return canvas; } + /** The kitty-image overlay, to be stacked directly above {@link #canvas()} in the window. */ + public Node imageOverlay() { + return imageOverlay.node(); + } + public void setFont(String family, double size) { metrics.setFont(family, size); paneContentVersion.clear(); @@ -256,6 +264,7 @@ public final class Compositor { for (TerminalPane pane : panes) { paneContentVersion.put(pane, pane.paintFull(gc, isActive(pane))); } + imageOverlay.sync(panes); } // Repaint just the panes whose content changed, directly on the retained canvas. Each pane @@ -272,6 +281,7 @@ public final class Compositor { continue; } paneContentVersion.put(pane, pane.paintIncremental(gc, isActive(pane))); + imageOverlay.updatePane(pane); } } diff --git a/src/main/java/com/gregor/jprototerm/GhosttyTerminalRenderer.java b/src/main/java/com/gregor/jprototerm/GhosttyTerminalRenderer.java index eee9652..e40993d 100644 --- a/src/main/java/com/gregor/jprototerm/GhosttyTerminalRenderer.java +++ b/src/main/java/com/gregor/jprototerm/GhosttyTerminalRenderer.java @@ -27,6 +27,7 @@ import javafx.scene.text.Text; import java.io.ByteArrayInputStream; import java.nio.IntBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -60,6 +61,9 @@ final class GhosttyTerminalRenderer extends TerminalRenderer { // Decoded kitty images for this renderer's pane (kitty graphics state is per-terminal). private final Map kittyImageCache = new HashMap<>(); private final SoftwareBackbuffer software = new SoftwareBackbuffer(); + // Image placements produced by the last paint; the compositor reads these and renders them + // as overlay nodes rather than compositing them onto the canvas. Empty for non-kitty panes. + private List kittyImageNodes = List.of(); GhosttyTerminalRenderer(TerminalMetrics metrics) { this.metrics = metrics; @@ -73,15 +77,10 @@ final class GhosttyTerminalRenderer extends TerminalRenderer { double height = target.height(); gc.save(); clip(gc, px, py, width, height, target.clip()); - boolean withKitty = target.kittyEnabled() && hasKittyGraphics(target); RenderStateSnapshot snapshot = target.snapshotFull(); - if (withKitty) { - drawContent(gc, target, snapshot, px, py, width, height, active, true); - software.invalidate(); - } else { - software.paintFull(gc, snapshot, px, py, width, height, active); - } + software.paintFull(gc, snapshot, px, py, width, height, active); gc.restore(); + collectKittyImages(target, snapshot, px, py); } @Override @@ -93,71 +92,30 @@ final class GhosttyTerminalRenderer extends TerminalRenderer { gc.save(); clip(gc, px, py, width, height, target.clip()); if (target.kittyEnabled() && hasKittyGraphics(target)) { - // Kitty placements can move without a per-row dirty flag, so always redraw whole. - drawContent(gc, target, target.snapshotFull(), px, py, width, height, active, true); - software.invalidate(); - } else { - RenderStateSnapshot snapshot = target.snapshot(); - int dirty = snapshot == null ? DIRTY_FULL : snapshot.dirty(); - if (dirty == DIRTY_FULL) { - software.paintFullOrShifted(gc, target.snapshotFull(), px, py, width, height, active); - } else if (dirty == DIRTY_PARTIAL) { - software.paintDirty(gc, target, snapshot, px, py, width, height, active); - } - // dirty == FALSE: nothing visible changed. + // Images render as overlay nodes, not on the canvas, but their positions track the + // grid (scroll/cursor), so we need a full snapshot to locate placeholder cells. The + // software path itself still repaints only the text rows whose hash changed. + RenderStateSnapshot snapshot = target.snapshotFull(); + software.paintFullOrShifted(gc, snapshot, px, py, width, height, active); + gc.restore(); + collectKittyImages(target, snapshot, px, py); + return; } + RenderStateSnapshot snapshot = target.snapshot(); + int dirty = snapshot == null ? DIRTY_FULL : snapshot.dirty(); + if (dirty == DIRTY_FULL) { + software.paintFullOrShifted(gc, target.snapshotFull(), px, py, width, height, active); + } else if (dirty == DIRTY_PARTIAL) { + software.paintDirty(gc, target, snapshot, px, py, width, height, active); + } + // dirty == FALSE: nothing visible changed. gc.restore(); + kittyImageNodes = List.of(); } - // Full content render: background, border, all rows, cursor, and (when enabled) kitty - // graphics. Used by the kitty direct path and by full redraws. - private void drawContent( - GraphicsContext gc, - RenderTarget target, - RenderStateSnapshot snapshot, - double x, - double y, - double width, - double height, - boolean active, - boolean withKitty - ) { - double cellWidth = metrics.cellWidth(); - double lineHeight = metrics.lineHeight(); - gc.setFontSmoothingType(FontSmoothingType.LCD); - gc.setFill(PANE_BACKGROUND); - gc.fillRect(x, y, width, height); - gc.setFont(metrics.font()); - - double left = x + TerminalMetrics.PADDING; - double top = y + TerminalMetrics.PADDING; - double baseline = top + metrics.baselineOffset(); - - Map placeholderBounds = withKitty - ? kittyPlaceholderBounds(snapshot) - : Map.of(); - - if (withKitty) { - drawKittyGraphics(gc, target, KittyPlacementLayer.BELOW_TEXT, placeholderBounds, left, top, cellWidth, lineHeight); - } - - if (snapshot != null) { - double contentBottom = top + snapshot.rows() * lineHeight; - fillVerticalPadding(gc, snapshot, x, y, width, height, top, contentBottom); - for (RenderRow row : snapshot.renderRows()) { - double y0 = Math.floor(top + (row.row() * lineHeight)); - double y1 = Math.ceil(top + ((row.row() + 1) * lineHeight)); - paintSidePadding(gc, row, x, width, left, cellWidth, y0, y1 - y0); - drawRow(gc, row, left, top, baseline, cellWidth, lineHeight); - } - drawCursor(gc, snapshot, left, top, cellWidth, lineHeight); - } - - if (withKitty) { - drawKittyGraphics(gc, target, KittyPlacementLayer.ABOVE_TEXT, placeholderBounds, left, top, cellWidth, lineHeight); - } - - drawBorder(gc, x, y, width, height, active); + @Override + List kittyImages() { + return kittyImageNodes; } // Incremental render: repaint only the rows ghostty flagged dirty, then restore the @@ -465,15 +423,34 @@ final class GhosttyTerminalRenderer extends TerminalRenderer { .orElse(false); } - private void drawKittyGraphics( - GraphicsContext gc, + // Build the image-node descriptors for the current frame. Reads the decoded-image cache and + // each placement's geometry, but draws nothing — the compositor turns these into overlay + // nodes clipped to the pane. + private void collectKittyImages(RenderTarget target, RenderStateSnapshot snapshot, double px, double py) { + if (!target.kittyEnabled() || !hasKittyGraphics(target)) { + kittyImageNodes = List.of(); + return; + } + double cellWidth = metrics.cellWidth(); + double lineHeight = metrics.lineHeight(); + double originX = px + TerminalMetrics.PADDING; + double originY = py + TerminalMetrics.PADDING; + Map placeholderBounds = kittyPlaceholderBounds(snapshot); + List nodes = new ArrayList<>(); + collectLayer(target, KittyPlacementLayer.BELOW_TEXT, placeholderBounds, originX, originY, cellWidth, lineHeight, nodes); + collectLayer(target, KittyPlacementLayer.ABOVE_TEXT, placeholderBounds, originX, originY, cellWidth, lineHeight, nodes); + kittyImageNodes = nodes; + } + + private void collectLayer( RenderTarget target, KittyPlacementLayer layer, Map placeholderBounds, double originX, double originY, double cellWidth, - double lineHeight + double lineHeight, + List out ) { target.kittyGraphics().ifPresent(graphics -> { for (KittyPlacement placement : graphics.placements(layer)) { @@ -482,17 +459,17 @@ final class GhosttyTerminalRenderer extends TerminalRenderer { continue; } - if (placement.virtual()) { - drawVirtualKittyPlacement(gc, placement, image, placeholderBounds, originX, originY, cellWidth, lineHeight); - } else { - drawPinnedKittyPlacement(gc, placement, image, originX, originY, cellWidth, lineHeight); + KittyImageNode node = placement.virtual() + ? virtualNode(placement, image, placeholderBounds, originX, originY, cellWidth, lineHeight) + : pinnedNode(placement, image, originX, originY, cellWidth, lineHeight); + if (node != null) { + out.add(node); } } }); } - private static void drawPinnedKittyPlacement( - GraphicsContext gc, + private static KittyImageNode pinnedNode( KittyPlacement placement, Image image, double originX, @@ -502,15 +479,13 @@ final class GhosttyTerminalRenderer extends TerminalRenderer { ) { KittyRenderInfo renderInfo = placement.renderInfo().orElse(null); if (renderInfo == null || !renderInfo.viewportVisible()) { - return; + return null; } - double sourceX = renderInfo.sourceX(); - double sourceY = renderInfo.sourceY(); double sourceWidth = renderInfo.sourceWidth(); double sourceHeight = renderInfo.sourceHeight(); if (sourceWidth <= 0.0 || sourceHeight <= 0.0) { - return; + return null; } double x = originX + (renderInfo.viewportColumn() * cellWidth) + placement.xOffset(); @@ -518,14 +493,14 @@ final class GhosttyTerminalRenderer extends TerminalRenderer { double width = renderInfo.pixelWidth() > 0 ? renderInfo.pixelWidth() : renderInfo.gridColumns() * cellWidth; double height = renderInfo.pixelHeight() > 0 ? renderInfo.pixelHeight() : renderInfo.gridRows() * lineHeight; if (width <= 0.0 || height <= 0.0) { - return; + return null; } - gc.drawImage(image, sourceX, sourceY, sourceWidth, sourceHeight, x, y, width, height); + return new KittyImageNode(placement.imageId(), placement.placementId(), image, + renderInfo.sourceX(), renderInfo.sourceY(), sourceWidth, sourceHeight, x, y, width, height); } - private static void drawVirtualKittyPlacement( - GraphicsContext gc, + private static KittyImageNode virtualNode( KittyPlacement placement, Image image, Map placeholderBounds, @@ -546,12 +521,12 @@ final class GhosttyTerminalRenderer extends TerminalRenderer { .orElse(null); } if (bounds == null || bounds.isEmpty()) { - return; + return null; } SourceRect source = sourceRect(placement, image); if (source.width() <= 0.0 || source.height() <= 0.0) { - return; + return null; } long gridColumns = gridColumns(placement, bounds); @@ -569,13 +544,14 @@ final class GhosttyTerminalRenderer extends TerminalRenderer { double availableHeight = bounds.rows() * lineHeight; if (sourceWidth <= 0.0 || sourceHeight <= 0.0 || availableWidth <= 0.0 || availableHeight <= 0.0) { - return; + return null; } double scale = Math.min(availableWidth / sourceWidth, availableHeight / sourceHeight); double width = sourceWidth * scale; double height = sourceHeight * scale; - gc.drawImage(image, sourceX, sourceY, sourceWidth, sourceHeight, x, y, width, height); + return new KittyImageNode(placement.imageId(), placement.placementId(), image, + sourceX, sourceY, sourceWidth, sourceHeight, x, y, width, height); } private static long gridColumns(KittyPlacement placement, KittyPlaceholderBounds bounds) { diff --git a/src/main/java/com/gregor/jprototerm/KittyImageNode.java b/src/main/java/com/gregor/jprototerm/KittyImageNode.java new file mode 100644 index 0000000..fc2942d --- /dev/null +++ b/src/main/java/com/gregor/jprototerm/KittyImageNode.java @@ -0,0 +1,32 @@ +package com.gregor.jprototerm; + +import javafx.scene.image.Image; + +/** + * A single kitty image to display, produced by the renderer and consumed by {@link + * KittyImageOverlay}. Images are not painted onto the canvas; each becomes a retained + * {@code ImageView} node positioned over the pane. The {@code source*} fields are the region of + * {@link #image()} to show (in image pixels); the {@code x/y/width/height} are where to put it, + * in scene coordinates (the same space the pane's clip {@code Shape} lives in). + * + *

{@code imageId}+{@code placementId} identify the placement so the overlay can reuse the + * same node across frames instead of recreating it. + */ +record KittyImageNode( + long imageId, + long placementId, + Image image, + double sourceX, + double sourceY, + double sourceWidth, + double sourceHeight, + double x, + double y, + double width, + double height +) { + /** Stable per-pane key for node reuse. Packs the two u32 ids without collision. */ + long key() { + return (imageId << 32) | (placementId & 0xffffffffL); + } +} diff --git a/src/main/java/com/gregor/jprototerm/KittyImageOverlay.java b/src/main/java/com/gregor/jprototerm/KittyImageOverlay.java new file mode 100644 index 0000000..e1a7200 --- /dev/null +++ b/src/main/java/com/gregor/jprototerm/KittyImageOverlay.java @@ -0,0 +1,147 @@ +package com.gregor.jprototerm; + +import javafx.geometry.Rectangle2D; +import javafx.scene.Group; +import javafx.scene.Node; +import javafx.scene.image.ImageView; +import javafx.scene.layout.Pane; +import javafx.scene.shape.Rectangle; +import javafx.scene.shape.Shape; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Renders kitty graphics images as retained scene-graph nodes layered over the {@link Compositor} + * canvas, instead of compositing them onto the canvas. Each pane gets a {@link Group} clipped to + * that pane's region (the same clip {@code Shape} the canvas renderer uses), and each visible + * image placement is an {@link ImageView} inside it, reused across frames so an unchanged image + * costs nothing to redraw. + * + *

The overlay {@link #node()} is mouse-transparent and sits above the canvas in the window's + * {@code StackPane}; its children use scene coordinates, which line up with the canvas because + * both fill the same root. + */ +final class KittyImageOverlay { + private final Pane root = new Pane(); + private final Map overlays = new HashMap<>(); + + KittyImageOverlay() { + // Input belongs to the canvas underneath; the overlay only shows pixels. + root.setMouseTransparent(true); + root.setManaged(false); + } + + Node node() { + return root; + } + + /** + * Full reconcile to {@code panes} (bottom-to-top): drop overlays for panes that went away, + * refresh each surviving/added pane's images and clip, and order the per-pane groups to match + * the pane z-order. Called on layout frames, after the panes have painted. + */ + void sync(List panes) { + Iterator> it = overlays.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + if (!panes.contains(entry.getKey())) { + root.getChildren().remove(entry.getValue().group); + it.remove(); + } + } + for (TerminalPane pane : panes) { + updatePane(pane); + } + List ordered = new ArrayList<>(panes.size()); + for (TerminalPane pane : panes) { + ordered.add(overlays.get(pane).group); + } + if (!root.getChildren().equals(ordered)) { + root.getChildren().setAll(ordered); + } + } + + /** + * Refresh one pane's images and clip (called on content frames for each repainted pane). + * Creates the pane's group if this is the first time it has shown an image. + */ + void updatePane(TerminalPane pane) { + List images = pane.kittyImages(); + PaneOverlay overlay = overlays.get(pane); + if (overlay == null) { + if (images.isEmpty()) { + return; + } + overlay = new PaneOverlay(); + overlays.put(pane, overlay); + root.getChildren().add(overlay.group); + } + overlay.group.setClip(clipFor(pane)); + reconcile(overlay, images); + } + + private static void reconcile(PaneOverlay overlay, List images) { + Set seen = new HashSet<>(); + for (KittyImageNode node : images) { + long key = node.key(); + seen.add(key); + ImageView view = overlay.views.get(key); + if (view == null) { + view = new ImageView(); + view.setManaged(false); + view.setSmooth(true); + view.setPreserveRatio(false); + overlay.views.put(key, view); + overlay.group.getChildren().add(view); + } + apply(view, node); + } + if (overlay.views.size() == seen.size()) { + return; + } + Iterator> it = overlay.views.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + if (!seen.contains(entry.getKey())) { + overlay.group.getChildren().remove(entry.getValue()); + it.remove(); + } + } + } + + private static void apply(ImageView view, KittyImageNode node) { + if (view.getImage() != node.image()) { + view.setImage(node.image()); + } + view.setViewport(new Rectangle2D(node.sourceX(), node.sourceY(), node.sourceWidth(), node.sourceHeight())); + view.setFitWidth(node.width()); + view.setFitHeight(node.height()); + view.setLayoutX(node.x()); + view.setLayoutY(node.y()); + } + + // The pane's occlusion clip when one is set (rect minus covering panes), else the pane's + // plain bounds so an image can't spill outside its pane. Matches Tab's pixel snapping. + private static Shape clipFor(TerminalPane pane) { + Shape clip = pane.clip(); + if (clip != null) { + return clip; + } + return new Rectangle(Math.round(pane.x()), Math.round(pane.y()), pane.width(), pane.height()); + } + + private static final class PaneOverlay { + private final Group group = new Group(); + private final Map views = new HashMap<>(); + + private PaneOverlay() { + group.setManaged(false); + } + } +} diff --git a/src/main/java/com/gregor/jprototerm/Main.java b/src/main/java/com/gregor/jprototerm/Main.java index 0e7779a..baf1bb9 100644 --- a/src/main/java/com/gregor/jprototerm/Main.java +++ b/src/main/java/com/gregor/jprototerm/Main.java @@ -32,7 +32,7 @@ public final class Main extends Application { metrics = new TerminalMetrics(config.fontFamily(), config.fontSize()); compositor = new Compositor(config, metrics); - StackPane root = new StackPane(compositor.canvas()); + StackPane root = new StackPane(compositor.canvas(), compositor.imageOverlay()); compositor.canvas().widthProperty().bind(root.widthProperty()); compositor.canvas().heightProperty().bind(root.heightProperty()); diff --git a/src/main/java/com/gregor/jprototerm/TerminalPane.java b/src/main/java/com/gregor/jprototerm/TerminalPane.java index a44f962..de70d3c 100644 --- a/src/main/java/com/gregor/jprototerm/TerminalPane.java +++ b/src/main/java/com/gregor/jprototerm/TerminalPane.java @@ -294,6 +294,14 @@ public final class TerminalPane implements AutoCloseable, RenderTarget { return snapshotVersion; } + /** + * Kitty image placements from the most recent paint, in scene coordinates. The compositor + * renders these as overlay nodes clipped to this pane (see {@link KittyImageOverlay}). + */ + public java.util.List kittyImages() { + return renderer.kittyImages(); + } + @Override public void close() { if (session != null) { diff --git a/src/main/java/com/gregor/jprototerm/TerminalRenderer.java b/src/main/java/com/gregor/jprototerm/TerminalRenderer.java index 286de53..cc11734 100644 --- a/src/main/java/com/gregor/jprototerm/TerminalRenderer.java +++ b/src/main/java/com/gregor/jprototerm/TerminalRenderer.java @@ -25,6 +25,14 @@ abstract class TerminalRenderer { /** Repaint only what changed since the last frame, clipped to the target's clip region. */ abstract void paintIncremental(GraphicsContext gc, RenderTarget target, boolean active); + /** + * The kitty image placements produced by the most recent paint, for the compositor to render + * as overlay nodes above the canvas. Empty unless the last paint found visible images. + */ + java.util.List kittyImages() { + return java.util.List.of(); + } + protected static void clipRect(GraphicsContext gc, double x, double y, double width, double height) { gc.beginPath(); gc.rect(x, y, width, height);