From c51050a9bc16b231de964156cd3ffd20ee12569e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= <daniel@ekloef.se>
Date: Thu, 25 May 2023 18:39:32 +0200
Subject: [PATCH] osc: update font subpixel mode, and window opaque compositor
 hint, on alpha changes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When background alpha is changed at runtime (using OSC-11), we (may)
have to update the opaque hint we send to the compositor.

We must also update the subpixel mode used when rendering font
glyphs.

Why?

When the window is fully opaque, we use wl_surface_set_opaque_region()
on the entire surface, to hint to the compositor that it doesn’t have
to blend the window content with whatever is behind the
window. Obviously, if alpha is changed from opaque, to transparent (or
semi-transparent), that hint must be removed.

Sub-pixel mode is harder to explain, but in short, we can’t do
subpixel hinting with a (semi-)transparent background. Thus, similar
to the opaque hint, subpixel antialiasing must be enabled/disabled
when background alpha is changed.
---
 CHANGELOG.md |  4 +++-
 osc.c        |  9 ++++++++-
 pgo/pgo.c    |  1 +
 wayland.c    | 29 +++++++++++++++++++----------
 wayland.h    |  1 +
 5 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4c8245b2..0c44dd13 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -76,7 +76,9 @@
 * `XTMODKEYS` state not being reset on a terminal reset.
 * In Gnome dock foot always groups under "foot client".  Change
   instances of footclient and foot to appear as "foot client" and
-  "foot" respectively. ([#1355][1355])
+  "foot" respectively. ([#1355][1355]).
+* Glitchy rendering when alpha (transparency) is changed between
+  opaque and non-opaque at runtime (using OSC-11).
 
 [1317]: https://codeberg.org/dnkl/foot/issues/1317
 [1355]: https://codeberg.org/dnkl/foot/issues/1355
diff --git a/osc.c b/osc.c
index 55cfcf84..45d114de 100644
--- a/osc.c
+++ b/osc.c
@@ -729,8 +729,15 @@ osc_dispatch(struct terminal *term)
 
         case 11:
             term->colors.bg = color;
-            if (have_alpha)
+            if (have_alpha) {
+                const bool changed = term->colors.alpha != alpha;
                 term->colors.alpha = alpha;
+
+                if (changed) {
+                    wayl_win_alpha_changed(term->window);
+                    term_font_subpixel_changed(term);
+                }
+            }
             break;
 
         case 17:
diff --git a/pgo/pgo.c b/pgo/pgo.c
index b41b5850..d9ee5855 100644
--- a/pgo/pgo.c
+++ b/pgo/pgo.c
@@ -94,6 +94,7 @@ wayl_win_init(struct terminal *term, const char *token)
 }
 
 void wayl_win_destroy(struct wl_window *win) {}
+void wayl_win_alpha_changed(struct wl_window *win) {}
 bool wayl_win_set_urgent(struct wl_window *win) { return true; }
 
 bool
diff --git a/wayland.c b/wayland.c
index 68a7a4f1..51161cf0 100644
--- a/wayland.c
+++ b/wayland.c
@@ -1495,16 +1495,7 @@ wayl_win_init(struct terminal *term, const char *token)
         goto out;
     }
 
-    if (term->colors.alpha == 0xffff) {
-        struct wl_region *region = wl_compositor_create_region(
-            term->wl->compositor);
-
-        if (region != NULL) {
-            wl_region_add(region, 0, 0, INT32_MAX, INT32_MAX);
-            wl_surface_set_opaque_region(win->surface, region);
-            wl_region_destroy(region);
-        }
-    }
+    wayl_win_alpha_changed(win);
 
     wl_surface_add_listener(win->surface, &surface_listener, win);
 
@@ -1798,6 +1789,24 @@ wayl_roundtrip(struct wayland *wayl)
     wayl_flush(wayl);
 }
 
+void
+wayl_win_alpha_changed(struct wl_window *win)
+{
+    struct terminal *term = win->term;
+
+    if (term->colors.alpha == 0xffff) {
+        struct wl_region *region = wl_compositor_create_region(
+            term->wl->compositor);
+
+        if (region != NULL) {
+            wl_region_add(region, 0, 0, INT32_MAX, INT32_MAX);
+            wl_surface_set_opaque_region(win->surface, region);
+            wl_region_destroy(region);
+        }
+    } else
+        wl_surface_set_opaque_region(win->surface, NULL);
+}
+
 #if defined(HAVE_XDG_ACTIVATION)
 static void
 activation_token_for_urgency_done(const char *token, void *data)
diff --git a/wayland.h b/wayland.h
index 4b6939ab..0d627052 100644
--- a/wayland.h
+++ b/wayland.h
@@ -434,6 +434,7 @@ void wayl_roundtrip(struct wayland *wayl);
 struct wl_window *wayl_win_init(struct terminal *term, const char *token);
 void wayl_win_destroy(struct wl_window *win);
 
+void wayl_win_alpha_changed(struct wl_window *win);
 bool wayl_win_set_urgent(struct wl_window *win);
 
 bool wayl_win_csd_titlebar_visible(const struct wl_window *win);