memory savings

src/main/java/com/gregor/jprototerm/Compositor.java

@@ -19,9 +19,11 @@ import javafx.scene.text.TextAlignment;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
 /**
  * Owns the window's tabs and drives rendering and input. It composites only the current tab:
@@ -52,6 +54,16 @@ public final class Compositor {
     // Last content version drawn to the canvas per pane, so a content frame repaints only
     // the panes that actually changed.
     private final Map<TerminalPane, Long> paneContentVersion = new HashMap<>();
+    // Off-screen panes (background tabs, hidden floating groups) keep their full-resolution pixel
+    // backbuffer + GPU image until released. We free them after a short grace period rather than the
+    // instant they're hidden, so rapidly flipping through tabs never thrashes the realloc/upload.
+    private static final long RELEASE_DELAY_NANOS = 750_000_000L;
+    // Hidden pane -> nanoTime it became hidden (the release timer); removed once released or shown.
+    private final Map<TerminalPane, Long> hiddenSince = new HashMap<>();
+    // Panes whose backbuffer is currently released, so we don't release again every frame.
+    private final Set<TerminalPane> released = new HashSet<>();
+    // layoutVersion at the last sweep: lets an idle, all-released steady state skip the scan.
+    private long lastSweepLayoutVersion = Long.MIN_VALUE;
     // Cheap per-frame dirty signal: skip the whole render when none of these changed.
     private double lastWidth = -1.0;
     private double lastHeight = -1.0;
@@ -278,6 +290,7 @@ public final class Compositor {
     // ---- Rendering ------------------------------------------------------------------
 
     public void render() {
+        sweepHiddenPanes();
         switch (nextFrameType()) {
             case IDLE -> { }
             case LAYOUT -> renderLayoutFrame();
@@ -285,6 +298,48 @@ public final class Compositor {
         }
     }
 
+    // Free the backbuffer of any pane that has been off-screen past the grace period, and re-arm the
+    // timer for newly hidden panes. The next layout frame rebuilds a released pane (paintFull goes
+    // through ensure()), so showing a tab again is the only cost. Skips entirely once everything that
+    // can be hidden is already released and the layout hasn't changed, so an idle multi-tab window
+    // does no per-frame work here.
+    private void sweepHiddenPanes() {
+        if (layoutVersion == lastSweepLayoutVersion && hiddenSince.isEmpty()) {
+            return;
+        }
+        lastSweepLayoutVersion = layoutVersion;
+
+        // Fast path: a single tab compositing all of its panes has nothing off-screen.
+        if (tabs.size() <= 1 && (tabs.isEmpty() || !currentTab().hasHiddenPanes())) {
+            hiddenSince.clear();
+            released.clear();
+            return;
+        }
+
+        Set<TerminalPane> visible = new HashSet<>(currentPanes());
+        Set<TerminalPane> live = new HashSet<>();
+        long now = System.nanoTime();
+        for (Tab tab : tabs) {
+            for (TerminalPane pane : tab.allPanes()) {
+                live.add(pane);
+                if (visible.contains(pane)) {
+                    hiddenSince.remove(pane);
+                    released.remove(pane);
+                } else if (!released.contains(pane)) {
+                    Long since = hiddenSince.putIfAbsent(pane, now);
+                    if (since != null && now - since >= RELEASE_DELAY_NANOS) {
+                        pane.releaseRenderResources();
+                        released.add(pane);
+                        hiddenSince.remove(pane);
+                    }
+                }
+            }
+        }
+        // Forget panes that have since closed.
+        hiddenSince.keySet().retainAll(live);
+        released.retainAll(live);
+    }
+
     // Classify this frame and commit the change trackers. A layout change (size, font,
     // tab/pane set, z-order, active pane) needs a full recomposite; otherwise a change to the
     // current tab's content version repaints only the panes that changed; otherwise nothing

src/main/java/com/gregor/jprototerm/GhosttyTerminalRenderer.java

@@ -13,17 +13,12 @@ import dev.jlibghostty.RenderCursorStyle;
 import dev.jlibghostty.RenderRow;
 import dev.jlibghostty.RenderStateSnapshot;
 import javafx.geometry.Rectangle2D;
-import javafx.scene.SnapshotParameters;
-import javafx.scene.canvas.Canvas;
 import javafx.scene.canvas.GraphicsContext;
 import javafx.scene.image.Image;
 import javafx.scene.image.PixelFormat;
 import javafx.scene.image.PixelBuffer;
-import javafx.scene.image.PixelReader;
 import javafx.scene.image.WritableImage;
 import javafx.scene.paint.Color;
-import javafx.scene.text.FontSmoothingType;
-import javafx.scene.text.Text;
 
 import java.io.ByteArrayInputStream;
 import java.nio.IntBuffer;
@@ -120,6 +115,12 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
         return kittyImageNodes;
     }
 
+    @Override
+    void release() {
+        software.release();
+        kittyImageNodes = List.of();
+    }
+
     // Effective background colour of a cell as it is drawn (reverse video swaps fg/bg, an
     // unset colour falls back to the defaults).
     private static Color cellBackgroundColor(RenderCell cell) {
@@ -416,7 +417,6 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
         private WritableImage image;
         private long[] rowHashes = new long[0];
         private CursorState lastCursor = CursorState.none();
-        private GlyphCache glyphs;
         // Half-open [min, max) vertical span of buffer rows written since the last present, so
         // present() can upload only that band to the GPU instead of the whole pane texture.
         private int dirtyMinY = Integer.MAX_VALUE;
@@ -583,7 +583,6 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
             int nextWidth = Math.max(1, (int) Math.round(paneWidth));
             int nextHeight = Math.max(1, (int) Math.round(paneHeight));
             if (nextWidth == width && nextHeight == height && image != null) {
-                ensureGlyphs();
                 return;
             }
 
@@ -593,17 +592,19 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
             pixelBuffer = new PixelBuffer<>(width, height, IntBuffer.wrap(pixels), PixelFormat.getIntArgbPreInstance());
             image = new WritableImage(pixelBuffer);
             invalidate();
-            ensureGlyphs();
         }
 
-        private void ensureGlyphs() {
-            int cellWidth = cellWidth();
-            int lineHeight = lineHeight();
-            double baseline = metrics.baselineOffset();
-            if (glyphs == null || glyphs.font != metrics.font()
-                    || glyphs.cellWidth != cellWidth || glyphs.lineHeight != lineHeight || glyphs.baseline != baseline) {
-                glyphs = new GlyphCache(metrics.font(), cellWidth, lineHeight, baseline);
-            }
+        // Drop the full-resolution pixel buffer and its GPU-backed image. The next ensure() rebuilds
+        // them (and a layout frame's paintFull repaints from scratch), so this is safe to call when
+        // the pane goes off-screen; only the shared glyph atlas (in TerminalMetrics) survives.
+        private void release() {
+            pixels = new int[0];
+            pixelBuffer = null;
+            image = null;
+            width = 0;
+            height = 0;
+            invalidate();
+            resetDirty();
         }
 
         private void present(GraphicsContext gc, double px, double py) {
@@ -804,30 +805,32 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
                 if (cell.kittyPlaceholder().isPresent() || cell.codepoints().length == 0) {
                     continue;
                 }
-                Glyph glyph = glyphs.glyph(cell.text());
+                GlyphCache.Glyph glyph = metrics.glyphCache().glyph(cell.text());
                 int color = rgb(cellForegroundColor(cell));
                 blitGlyph(glyph, x0 + (cell.column() * cellWidth), rowTop, color);
             }
         }
 
-        private void blitGlyph(Glyph glyph, int x, int y, int rgb) {
+        private void blitGlyph(GlyphCache.Glyph glyph, int x, int y, int rgb) {
             int red = (rgb >> 16) & 0xff;
             int green = (rgb >> 8) & 0xff;
             int blue = rgb & 0xff;
             // Clamp the glyph rectangle to the buffer once, so the inner loops carry no
             // per-pixel bounds check (this is the hottest pixel loop on a text repaint).
+            int glyphWidth = glyph.width();
+            byte[] glyphAlpha = glyph.alpha();
             int gyStart = Math.max(0, -y);
-            int gyEnd = Math.min(glyph.height, height - y);
+            int gyEnd = Math.min(glyph.height(), height - y);
             int gxStart = Math.max(0, -x);
-            int gxEnd = Math.min(glyph.width, width - x);
+            int gxEnd = Math.min(glyphWidth, width - x);
             if (gyStart >= gyEnd || gxStart >= gxEnd) {
                 return;
             }
             for (int gy = gyStart; gy < gyEnd; gy++) {
                 int rowOffset = (y + gy) * width;
-                int glyphOffset = gy * glyph.width;
+                int glyphOffset = gy * glyphWidth;
                 for (int gx = gxStart; gx < gxEnd; gx++) {
-                    int alpha = glyph.alpha[glyphOffset + gx] & 0xff;
+                    int alpha = glyphAlpha[glyphOffset + gx] & 0xff;
                     if (alpha == 0) {
                         continue;
                     }
@@ -947,53 +950,6 @@ final class GhosttyTerminalRenderer extends TerminalRenderer {
         }
     }
 
-    private final class GlyphCache {
-        private final javafx.scene.text.Font font;
-        private final int cellWidth;
-        private final int lineHeight;
-        private final double baseline;
-        private final Map<String, Glyph> glyphs = new HashMap<>();
-
-        private GlyphCache(javafx.scene.text.Font font, int cellWidth, int lineHeight, double baseline) {
-            this.font = font;
-            this.cellWidth = cellWidth;
-            this.lineHeight = lineHeight;
-            this.baseline = baseline;
-        }
-
-        private Glyph glyph(String text) {
-            return glyphs.computeIfAbsent(text, this::renderGlyph);
-        }
-
-        private Glyph renderGlyph(String value) {
-            Text measured = new Text(value);
-            measured.setFont(font);
-            int glyphWidth = Math.max(cellWidth, (int) Math.ceil(measured.getLayoutBounds().getWidth()) + 2);
-            Canvas canvas = new Canvas(glyphWidth, lineHeight);
-            GraphicsContext gc = canvas.getGraphicsContext2D();
-            gc.setFontSmoothingType(FontSmoothingType.GRAY);
-            gc.setFont(font);
-            gc.setFill(Color.WHITE);
-            gc.fillText(value, 0.0, baseline);
-
-            SnapshotParameters parameters = new SnapshotParameters();
-            parameters.setFill(Color.TRANSPARENT);
-            WritableImage snapshot = canvas.snapshot(parameters, null);
-            PixelReader reader = snapshot.getPixelReader();
-            byte[] alpha = new byte[glyphWidth * lineHeight];
-            for (int y = 0; y < lineHeight; y++) {
-                int offset = y * glyphWidth;
-                for (int x = 0; x < glyphWidth; x++) {
-                    alpha[offset + x] = (byte) ((reader.getArgb(x, y) >>> 24) & 0xff);
-                }
-            }
-            return new Glyph(glyphWidth, lineHeight, alpha);
-        }
-    }
-
-    private record Glyph(int width, int height, byte[] alpha) {
-    }
-
     private record CursorState(boolean visible, boolean hasViewport, int column, int row, RenderCursorStyle style) {
         private static CursorState none() {
             return new CursorState(false, false, -1, -1, null);

src/main/java/com/gregor/jprototerm/GlyphCache.java

@@ -0,0 +1,90 @@
+package com.gregor.jprototerm;
+
+import javafx.scene.SnapshotParameters;
+import javafx.scene.canvas.Canvas;
+import javafx.scene.canvas.GraphicsContext;
+import javafx.scene.image.PixelReader;
+import javafx.scene.image.WritableImage;
+import javafx.scene.paint.Color;
+import javafx.scene.text.Font;
+import javafx.scene.text.FontSmoothingType;
+import javafx.scene.text.Text;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Rasterized glyph alpha masks for one window's font, shared by every pane's renderer. The atlas is
+ * a pure function of the window's {@link TerminalMetrics} (font family/size, snapped cell geometry,
+ * baseline), and all panes in a window observe the same metrics, so a single shared cache lets N
+ * panes reuse one copy of each glyph instead of each rasterizing and retaining its own. It also
+ * means a pane whose backbuffer was released (see {@link GhosttyTerminalRenderer}) does not have to
+ * re-rasterize glyphs when it is shown again.
+ *
+ * <p>Rasterizing goes through JavaFX ({@link Canvas#snapshot}), so {@link #glyph} must be called on
+ * the FX thread — which is where all rendering happens. The cache self-invalidates when the metrics
+ * change (e.g. a font switch): the next lookup notices and clears.
+ */
+final class GlyphCache {
+    record Glyph(int width, int height, byte[] alpha) {
+    }
+
+    private final TerminalMetrics metrics;
+    private final Map<String, Glyph> glyphs = new HashMap<>();
+    // The metrics snapshot the cached glyphs were rasterized for; a mismatch clears the cache.
+    private Font font;
+    private int cellWidth;
+    private int lineHeight;
+    private double baseline;
+
+    GlyphCache(TerminalMetrics metrics) {
+        this.metrics = metrics;
+    }
+
+    Glyph glyph(String text) {
+        ensureCurrent();
+        return glyphs.computeIfAbsent(text, this::renderGlyph);
+    }
+
+    // Drop the rasterized masks if the font/cell geometry changed since they were built. Cheap to
+    // call per lookup: a no-op unless the window's metrics actually changed under us.
+    private void ensureCurrent() {
+        Font currentFont = metrics.font();
+        int currentCellWidth = Math.max(1, (int) Math.round(metrics.cellWidth()));
+        int currentLineHeight = Math.max(1, (int) Math.round(metrics.lineHeight()));
+        double currentBaseline = metrics.baselineOffset();
+        if (currentFont != font || currentCellWidth != cellWidth
+                || currentLineHeight != lineHeight || currentBaseline != baseline) {
+            font = currentFont;
+            cellWidth = currentCellWidth;
+            lineHeight = currentLineHeight;
+            baseline = currentBaseline;
+            glyphs.clear();
+        }
+    }
+
+    private Glyph renderGlyph(String value) {
+        Text measured = new Text(value);
+        measured.setFont(font);
+        int glyphWidth = Math.max(cellWidth, (int) Math.ceil(measured.getLayoutBounds().getWidth()) + 2);
+        Canvas canvas = new Canvas(glyphWidth, lineHeight);
+        GraphicsContext gc = canvas.getGraphicsContext2D();
+        gc.setFontSmoothingType(FontSmoothingType.GRAY);
+        gc.setFont(font);
+        gc.setFill(Color.WHITE);
+        gc.fillText(value, 0.0, baseline);
+
+        SnapshotParameters parameters = new SnapshotParameters();
+        parameters.setFill(Color.TRANSPARENT);
+        WritableImage snapshot = canvas.snapshot(parameters, null);
+        PixelReader reader = snapshot.getPixelReader();
+        byte[] alpha = new byte[glyphWidth * lineHeight];
+        for (int y = 0; y < lineHeight; y++) {
+            int offset = y * glyphWidth;
+            for (int x = 0; x < glyphWidth; x++) {
+                alpha[offset + x] = (byte) ((reader.getArgb(x, y) >>> 24) & 0xff);
+            }
+        }
+        return new Glyph(glyphWidth, lineHeight, alpha);
+    }
+}

src/main/java/com/gregor/jprototerm/Tab.java

@@ -101,6 +101,19 @@ final class Tab implements AutoCloseable {
         return pane != null && pane == active;
     }
 
+    /** Every pane this tab owns, composited or not (tiled then floating). */
+    List<TerminalPane> allPanes() {
+        List<TerminalPane> all = new ArrayList<>(tiled.size() + floating.size());
+        all.addAll(tiled);
+        all.addAll(floating);
+        return all;
+    }
+
+    /** Whether this tab owns panes that {@link #panes()} does not currently composite. */
+    boolean hasHiddenPanes() {
+        return !floatingVisible && !floating.isEmpty();
+    }
+
     boolean focus(TerminalPane pane) {
         if (pane == active || !isFocusable(pane)) {
             return false;

src/main/java/com/gregor/jprototerm/TerminalMetrics.java

@@ -23,6 +23,9 @@ public final class TerminalMetrics {
     private double cellWidth;
     private double lineHeight;
     private double baselineOffset;
+    // One rasterized-glyph atlas per window, shared by every pane's renderer (the masks are a pure
+    // function of the font geometry below). It self-invalidates when these metrics change.
+    private final GlyphCache glyphCache = new GlyphCache(this);
 
     public TerminalMetrics(String fontFamily, double fontSize) {
         setFont(fontFamily, fontSize);
@@ -59,6 +62,11 @@ public final class TerminalMetrics {
         return baselineOffset;
     }
 
+    /** The window's shared glyph atlas (see {@link GlyphCache}). */
+    public GlyphCache glyphCache() {
+        return glyphCache;
+    }
+
     /** Columns that fit in a pane of the given pixel width (after subtracting the padding). */
     public int columnsFor(double widthPx) {
         return Math.max(1, (int) ((widthPx - 2 * PADDING) / cellWidth));

src/main/java/com/gregor/jprototerm/TerminalPane.java

@@ -350,6 +350,16 @@ public final class TerminalPane implements AutoCloseable, RenderTarget {
         onContentChange.run();
     }
 
+    /**
+     * Drop this pane's large, rebuildable render buffers because it is no longer being composited
+     * (e.g. it sits in a background tab or a hidden floating group). The pixel backbuffer and its GPU
+     * image are freed; the shell, ghostty terminal and shared glyph atlas are untouched, and the next
+     * {@link #paintFull} rebuilds the buffer. Safe to call repeatedly. See {@link TerminalRenderer#release}.
+     */
+    public void releaseRenderResources() {
+        renderer.release();
+    }
+
     /** Paint the whole pane; see {@link TerminalRenderer#paintFull}. */
     public long paintFull(GraphicsContext gc, boolean active) {
         renderer.paintFull(gc, this, active);

src/main/java/com/gregor/jprototerm/TerminalRenderer.java

@@ -33,6 +33,14 @@ abstract class TerminalRenderer {
         return java.util.List.of();
     }
 
+    /**
+     * Release any large, rebuildable render buffers (e.g. a pane's pixel backbuffer) because the
+     * target is no longer being composited. A no-op by default; the next paint must rebuild whatever
+     * was dropped. Called off the paint path, so it must not assume a frame is in progress.
+     */
+    void release() {
+    }
+
     protected static void clipRect(GraphicsContext gc, double x, double y, double width, double height) {
         gc.beginPath();
         gc.rect(x, y, width, height);