From 05b1c9d0c8a1e657085aa2481338c128fd35cbde Mon Sep 17 00:00:00 2001
From: Robert Mader <robert.mader@collabora.com>
Date: Wed, 29 May 2024 18:58:44 +0200
Subject: [PATCH] gst: Add helper to fixate DMA_DRM caps

When support for modifier-aware DMA_DRM formats was added in f1b75fc6,
the translation between Pipewire parameters and Gstreamer caps was kept
compatible with the] non-DMA_DRM/legacy API by reporting format/modifier
combinations with linear or invalid modifier both as `format` and
`drm-format`.

In cases when a linear modifier ends up being negotiated, this, however,
resurts in non-fixated caps, preventing the negotiation to succeed.

Add a helper that allows to fixate such caps.
---
 src/gst/gstpipewireformat.c | 55 +++++++++++++++++++++++++++++++++++++
 src/gst/gstpipewireformat.h |  2 ++
 2 files changed, 57 insertions(+)

diff --git a/src/gst/gstpipewireformat.c b/src/gst/gstpipewireformat.c
index 7ffe8e916..1e164e08b 100644
--- a/src/gst/gstpipewireformat.c
+++ b/src/gst/gstpipewireformat.c
@@ -1229,3 +1229,58 @@ gst_caps_sanitize (GstCaps **caps)
   *caps = gst_caps_make_writable (*caps);
   gst_caps_filter_and_map_in_place (*caps, filter_dmabuf_caps, NULL);
 }
+
+void
+gst_caps_maybe_fixate_dma_format (GstCaps *caps)
+{
+#ifdef HAVE_GSTREAMER_DMA_DRM
+  GstCapsFeatures *features;
+  GstStructure *structure;
+  const GValue *format_value;
+  const GValue *drm_format_value;
+  const char *format_string;
+  const char *drm_format_string;
+  uint32_t fourcc;
+  uint64_t mod;
+  int drm_idx;
+  int i;
+
+  g_return_if_fail (GST_IS_CAPS (caps));
+
+  if (gst_caps_is_fixed (caps) || gst_caps_get_size(caps) != 1)
+    return;
+
+  features = gst_caps_get_features (caps, 0);
+  if (!gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_DMABUF))
+    return;
+
+  structure = gst_caps_get_structure (caps, 0);
+  if (!gst_structure_has_field (structure, "format") ||
+      !gst_structure_has_field (structure, "drm-format"))
+    return;
+
+  format_value = gst_structure_get_value (structure, "format");
+  drm_format_value = gst_structure_get_value (structure, "drm-format");
+  if (G_VALUE_TYPE (format_value) != GST_TYPE_LIST ||
+      ((GArray *) g_value_peek_pointer (format_value))->len != 2 ||
+      G_VALUE_TYPE (drm_format_value) != G_TYPE_STRING)
+    return;
+
+  drm_format_string = g_value_get_string (drm_format_value);
+  fourcc = gst_video_dma_drm_fourcc_from_string (drm_format_string, &mod);
+  drm_idx = gst_video_dma_drm_fourcc_to_format (fourcc);
+  if (drm_idx == GST_VIDEO_FORMAT_UNKNOWN || mod != DRM_FORMAT_MOD_LINEAR)
+    return;
+
+  for (i = 0; (format_string = get_nth_string (format_value, i)); i++) {
+    int idx;
+
+    idx = gst_video_format_from_string (format_string);
+    if (idx != GST_VIDEO_FORMAT_DMA_DRM && idx != drm_idx)
+      return;
+  }
+
+  gst_caps_set_simple (caps, "format", G_TYPE_STRING, "DMA_DRM", NULL);
+  g_warn_if_fail (gst_caps_is_fixed (caps));
+#endif
+}
diff --git a/src/gst/gstpipewireformat.h b/src/gst/gstpipewireformat.h
index 1c3a239ba..2ecfedd09 100644
--- a/src/gst/gstpipewireformat.h
+++ b/src/gst/gstpipewireformat.h
@@ -17,6 +17,8 @@ GstCaps *        gst_caps_from_format    (const struct spa_pod *format);
 
 void             gst_caps_sanitize       (GstCaps **caps);
 
+void             gst_caps_maybe_fixate_dma_format (GstCaps *caps);
+
 G_END_DECLS
 
 #endif