gst: Sanitize caps before translating

DMABuf caps without concrete formats and modifiers don't map well to the
Pipewire negotiation process.
Introduce a new gst_caps_sanitize() helper function, where such cases
can be handled.

src/gst/gstpipewireformat.c

@@ -1176,3 +1176,43 @@ gst_caps_from_format (const struct spa_pod *format)
   }
   return res;
 }
+
+static gboolean
+filter_dmabuf_caps (GstCapsFeatures *features,
+                    GstStructure    *structure,
+                    gpointer         user_data)
+{
+  const GValue *value;
+  const char *v;
+
+  if (!gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_DMABUF))
+    return TRUE;
+
+  if (!(value = gst_structure_get_value (structure, "format")) ||
+      !(v = get_nth_string (value, 0)))
+    return FALSE;
+
+#ifdef HAVE_GSTREAMER_DMA_DRM
+  {
+    int idx;
+
+    idx = gst_video_format_from_string (v);
+    if (idx == GST_VIDEO_FORMAT_UNKNOWN)
+      return FALSE;
+
+    if (idx == GST_VIDEO_FORMAT_DMA_DRM &&
+        !gst_structure_get_value (structure, "drm-format"))
+      return FALSE;
+  }
+#endif
+
+  return TRUE;
+}
+
+GstCaps *
+gst_caps_sanitize (GstCaps *caps)
+{
+  caps = gst_caps_make_writable (caps);
+  gst_caps_filter_and_map_in_place (caps, filter_dmabuf_caps, NULL);
+  return caps;
+}

src/gst/gstpipewireformat.h

@@ -15,6 +15,8 @@ GPtrArray *      gst_caps_to_format_all  (GstCaps *caps);
 
 GstCaps *        gst_caps_from_format    (const struct spa_pod *format);
 
+GstCaps *        gst_caps_sanitize       (GstCaps *caps);
+
 G_END_DECLS
 
 #endif

src/gst/gstpipewiresrc.c

@@ -844,10 +844,14 @@ gst_pipewire_src_negotiate (GstBaseSrc * basesrc)
     /* no peer, work with our own caps then */
     caps = thiscaps;
   }
+
+  GST_DEBUG_OBJECT (basesrc, "have common caps: %" GST_PTR_FORMAT, caps);
+  gst_caps_sanitize (caps);
+
   if (caps == NULL || gst_caps_is_empty (caps))
     goto no_common_caps;
 
-  GST_DEBUG_OBJECT (basesrc, "have common caps: %" GST_PTR_FORMAT, caps);
+  GST_DEBUG_OBJECT (basesrc, "have common caps (sanitized): %" GST_PTR_FORMAT, caps);
 
   /* open a connection with these caps */
   possible = gst_caps_to_format_all (caps);