From c006ac3a079d00e30f9d1cd7986d8112724629b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= <daniel@ekloef.se>
Date: Thu, 12 Oct 2023 16:16:11 +0200
Subject: [PATCH] shm: memfd_create: fallback to not using MFD_NOEXEC_SEAL

MFD_NOEXEC_SEAL was introduced in linux 6.3. Kernels before that
will *reject* memfd_create() calls that set it.

This caused foot to exit (i.e. not start at all), when compiled on
linux >= 6.3, but run on linux < 6.3.

We _do_ want to use MFD_NOEXEC_SEAL, since a) our memory mapped really
shouldn't be executable, and b) to silence a warning on linux >= 6.3.

To handle all cases, first try *with* MFD_NOEXEC_SEAL. If that fails
with EINVAL, retry *without* it.

Closes #1514
---
 CHANGELOG.md |  5 +++++
 shm.c        | 21 ++++++++++++++++-----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4a520e29..39c22d87 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -53,6 +53,11 @@
 ### Deprecated
 ### Removed
 ### Fixed
+* Foot not starting on linux kernels before 6.3 ([#1514][1514]).
+
+[1514]: https://codeberg.org/dnkl/foot/issues/1514
+
+
 ### Security
 ### Contributors
 
diff --git a/shm.c b/shm.c
index 8ca0ead0..171459f6 100644
--- a/shm.c
+++ b/shm.c
@@ -27,10 +27,8 @@
  #define MAP_UNINITIALIZED 0
 #endif
 
-#if defined(MFD_NOEXEC_SEAL)
-    #define FOOT_MFD_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_NOEXEC_SEAL)
-#else
-    #define FOOT_MFD_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING)
+#if !defined(MFD_NOEXEC_SEAL)
+ #define MFD_NOEXEC_SEAL 0
 #endif
 
 #define TIME_SCROLL 0
@@ -339,7 +337,20 @@ get_new_buffers(struct buffer_chain *chain, size_t count,
 
     /* Backing memory for SHM */
 #if defined(MEMFD_CREATE)
-    pool_fd = memfd_create("foot-wayland-shm-buffer-pool", FOOT_MFD_FLAGS);
+    /*
+     * Older kernels reject MFD_NOEXEC_SEAL with EINVAL. Try first
+     * *with* it, and if that fails, try again *without* it.
+     */
+    errno = 0;
+    pool_fd = memfd_create(
+        "foot-wayland-shm-buffer-pool",
+        MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_NOEXEC_SEAL);
+
+    if (pool_fd < 0 && errno == EINVAL) {
+        pool_fd = memfd_create(
+            "foot-wayland-shm-buffer-pool", MFD_CLOEXEC | MFD_ALLOW_SEALING);
+    }
+
 #elif defined(__FreeBSD__)
     // memfd_create on FreeBSD 13 is SHM_ANON without sealing support
     pool_fd = shm_open(SHM_ANON, O_RDWR | O_CLOEXEC, 0600);