osc: update font subpixel mode, and window opaque compositor hint, on alpha changes

When background alpha is changed at runtime (using OSC-11), we (may)
have to update the opaque hint we send to the compositor.

We must also update the subpixel mode used when rendering font
glyphs.

Why?

When the window is fully opaque, we use wl_surface_set_opaque_region()
on the entire surface, to hint to the compositor that it doesn’t have
to blend the window content with whatever is behind the
window. Obviously, if alpha is changed from opaque, to transparent (or
semi-transparent), that hint must be removed.

Sub-pixel mode is harder to explain, but in short, we can’t do
subpixel hinting with a (semi-)transparent background. Thus, similar
to the opaque hint, subpixel antialiasing must be enabled/disabled
when background alpha is changed.

CHANGELOG.md

@@ -76,7 +76,9 @@
 * `XTMODKEYS` state not being reset on a terminal reset.
 * In Gnome dock foot always groups under "foot client".  Change
   instances of footclient and foot to appear as "foot client" and
-  "foot" respectively. ([#1355][1355])
+  "foot" respectively. ([#1355][1355]).
+* Glitchy rendering when alpha (transparency) is changed between
+  opaque and non-opaque at runtime (using OSC-11).
 
 [1317]: https://codeberg.org/dnkl/foot/issues/1317
 [1355]: https://codeberg.org/dnkl/foot/issues/1355

osc.c

@@ -729,8 +729,15 @@ osc_dispatch(struct terminal *term)
 
         case 11:
             term->colors.bg = color;
-            if (have_alpha)
+            if (have_alpha) {
+                const bool changed = term->colors.alpha != alpha;
                 term->colors.alpha = alpha;
+
+                if (changed) {
+                    wayl_win_alpha_changed(term->window);
+                    term_font_subpixel_changed(term);
+                }
+            }
             break;
 
         case 17:

pgo/pgo.c

@@ -94,6 +94,7 @@ wayl_win_init(struct terminal *term, const char *token)
 }
 
 void wayl_win_destroy(struct wl_window *win) {}
+void wayl_win_alpha_changed(struct wl_window *win) {}
 bool wayl_win_set_urgent(struct wl_window *win) { return true; }
 
 bool

wayland.c

@@ -1495,16 +1495,7 @@ wayl_win_init(struct terminal *term, const char *token)
         goto out;
     }
 
-    if (term->colors.alpha == 0xffff) {
+    wayl_win_alpha_changed(win);
-        struct wl_region *region = wl_compositor_create_region(
-            term->wl->compositor);
-
-        if (region != NULL) {
-            wl_region_add(region, 0, 0, INT32_MAX, INT32_MAX);
-            wl_surface_set_opaque_region(win->surface, region);
-            wl_region_destroy(region);
-        }
-    }
 
     wl_surface_add_listener(win->surface, &surface_listener, win);
 
@@ -1798,6 +1789,24 @@ wayl_roundtrip(struct wayland *wayl)
     wayl_flush(wayl);
 }
 
+void
+wayl_win_alpha_changed(struct wl_window *win)
+{
+    struct terminal *term = win->term;
+
+    if (term->colors.alpha == 0xffff) {
+        struct wl_region *region = wl_compositor_create_region(
+            term->wl->compositor);
+
+        if (region != NULL) {
+            wl_region_add(region, 0, 0, INT32_MAX, INT32_MAX);
+            wl_surface_set_opaque_region(win->surface, region);
+            wl_region_destroy(region);
+        }
+    } else
+        wl_surface_set_opaque_region(win->surface, NULL);
+}
+
 #if defined(HAVE_XDG_ACTIVATION)
 static void
 activation_token_for_urgency_done(const char *token, void *data)

wayland.h

@@ -434,6 +434,7 @@ void wayl_roundtrip(struct wayland *wayl);
 struct wl_window *wayl_win_init(struct terminal *term, const char *token);
 void wayl_win_destroy(struct wl_window *win);
 
+void wayl_win_alpha_changed(struct wl_window *win);
 bool wayl_win_set_urgent(struct wl_window *win);
 
 bool wayl_win_csd_titlebar_visible(const struct wl_window *win);