input: xkb: ignore virtual modifiers

Some compositors (mutter/GNOME is one) adds _virtual_ modifiers to the
set of active modifiers when e.g. Alt, Meta, Super or Hyper is
pressed. For example, pressing Alt+b would result in *both* the Alt
*and* the Mod1 modifier being set.

Since foot makes close to zero assumptions on how the modifiers should
be interpreted, this causes various breakages.

For example, a foot shortcut defined as Mod1+b will not match, since
the Alt modifiers is also set. This has forced users to
redefine/override some of the default key bindings to include the
additional modifiers.

It also causes issues with the kitty keyboard protocol, for some key
combinations. Mainly whether or not to use unshifted key or not,
resulting in incorrect escape sequences.

Since all the "real" modifiers are always set as well, we can safely
ignore the virtual modifiers.

Closes #2009

CHANGELOG.md

@@ -68,8 +68,20 @@
   colors has not configured) is now done by linear RGB interpolation,
   rather than converting to HSL and adjusting the luminance
   ([#2006][2006]).
+* XKB: virtual modifiers are now ignored. This works around various
+  issues seen when running foot under mutter (GNOME) ([#2009][2009]):
+  - Some key combinations generating the wrong escape sequence in the
+    kitty keyboard protocol.
+  - some of foot's default shortcuts not working (mainly those using
+    `Mod1`) out of the box.
+  - **Note: if you have custom key bindings in `foot.ini` that
+    includes one or more of the `Alt`, `Meta`, `Super`, `Hyper`,
+    `NumLock`, `ScrollLock`, `LevelThree` or `LevelFive` modifiers,
+    you need to update them; i.e. remove the virtual modifier(s),
+    leaving only the real modifiers (`Mod1`, `Mod2` etc).**
 
 [2006]: https://codeberg.org/dnkl/foot/issues/2006
+[2009]: https://codeberg.org/dnkl/foot/issues/2009
 
 
 ### Deprecated

input.c

@@ -620,6 +620,54 @@ keyboard_keymap(void *data, struct wl_keyboard *wl_keyboard,
         if (seat->kbd.mod_num != XKB_MOD_INVALID)
             seat->kbd.kitty_significant |= 1 << seat->kbd.mod_num;
 
+
+        /*
+         * Create a mask of all "virtual" modifiers. Some compositors
+         * add these *in addition* to the "real" modifiers (Mod1,
+         * Mod2, etc).
+         *
+         * Since our modifier logic (both for internal shortcut
+         * processing, and e.g. the kitty keyboard protocol) makes
+         * very few assumptions on available modifiers, which keys map
+         * to which modifier etc, the presence of virtual modifiers
+         * causes various things to break.
+         *
+         * For example, if a foot shortcut is Mod1+b (i.e. Alt+b), it
+         * won't match if the compositor _also_ sets the Alt modifier
+         * (the corresponding shortcut in foot would be Alt+Mod1+b).
+         *
+         * See https://codeberg.org/dnkl/foot/issues/2009
+         *
+         * Mutter (GNOME) is known to set the virtual modifiers in
+         * addtiion to the real modifiers.
+         *
+         * As far as I know, there's no compositor that _only_ sets
+         * virtual modifiers (don't think that's even legal...?)
+         */
+        {
+            xkb_mod_index_t alt = xkb_keymap_mod_get_index(seat->kbd.xkb_keymap, XKB_VMOD_NAME_ALT);
+            xkb_mod_index_t meta = xkb_keymap_mod_get_index(seat->kbd.xkb_keymap, XKB_VMOD_NAME_META);
+            xkb_mod_index_t super = xkb_keymap_mod_get_index(seat->kbd.xkb_keymap, XKB_VMOD_NAME_SUPER);
+            xkb_mod_index_t hyper = xkb_keymap_mod_get_index(seat->kbd.xkb_keymap, XKB_VMOD_NAME_HYPER);
+            xkb_mod_index_t num_lock = xkb_keymap_mod_get_index(seat->kbd.xkb_keymap, XKB_VMOD_NAME_NUM);
+            xkb_mod_index_t scroll_lock = xkb_keymap_mod_get_index(seat->kbd.xkb_keymap, XKB_VMOD_NAME_SCROLL);
+            xkb_mod_index_t level_three = xkb_keymap_mod_get_index(seat->kbd.xkb_keymap, XKB_VMOD_NAME_LEVEL3);
+            xkb_mod_index_t level_five = xkb_keymap_mod_get_index(seat->kbd.xkb_keymap, XKB_VMOD_NAME_LEVEL5);
+
+            xkb_mod_index_t ignore = 0;
+
+            if (alt != XKB_MOD_INVALID) ignore |= 1 << alt;
+            if (meta != XKB_MOD_INVALID) ignore |= 1 << meta;
+            if (super != XKB_MOD_INVALID) ignore |= 1 << super;
+            if (hyper != XKB_MOD_INVALID) ignore |= 1 << hyper;
+            if (num_lock != XKB_MOD_INVALID) ignore |= 1 << num_lock;
+            if (scroll_lock != XKB_MOD_INVALID) ignore |= 1 << scroll_lock;
+            if (level_three != XKB_MOD_INVALID) ignore |= 1 << level_three;
+            if (level_five != XKB_MOD_INVALID) ignore |= 1 << level_five;
+
+            seat->kbd.virtual_modifiers = ignore;
+        }
+
         seat->kbd.key_arrow_up = xkb_keymap_key_by_name(seat->kbd.xkb_keymap, "UP");
         seat->kbd.key_arrow_down = xkb_keymap_key_by_name(seat->kbd.xkb_keymap, "DOWN");
     }
@@ -1759,6 +1807,10 @@ keyboard_modifiers(void *data, struct wl_keyboard *wl_keyboard, uint32_t serial,
 {
     struct seat *seat = data;
 
+    mods_depressed &= ~seat->kbd.virtual_modifiers;
+    mods_latched &= ~seat->kbd.virtual_modifiers;
+    mods_locked &= ~seat->kbd.virtual_modifiers;
+
 #if defined(_DEBUG)
     char depressed[256];
     char latched[256];

wayland.h

@@ -136,6 +136,8 @@ struct seat {
         xkb_mod_mask_t legacy_significant;  /* Significant modifiers for the legacy keyboard protocol */
         xkb_mod_mask_t kitty_significant;   /* Significant modifiers for the kitty keyboard protocol */
 
+        xkb_mod_mask_t virtual_modifiers;   /* Set of modifiers to completely ignore */
+
         xkb_keycode_t key_arrow_up;
         xkb_keycode_t key_arrow_down;