gst: handle blocks and size allocation for encoded format

In case of encoded video we get n_planes as 0 from the video info so
passing that as n_datas is failing during the buffer negotiation. Make
sure to use an appropriate value based on whether we have raw video or
not.

Co-authored-by: Taruntej Kanakamalla <taruntej@asymptotic.io>

src/gst/gstpipewirepool.c

@@ -66,7 +66,10 @@ void gst_pipewire_pool_wrap_buffer (GstPipeWirePool *pool, struct pw_buffer *b)
   uint32_t i;
   GstPipeWirePoolData *data;
   /* Default to a large enough value */
-  gsize plane_sizes[GST_VIDEO_MAX_PLANES] = { pool->video_info.size, };
+  gsize plane_0_size = pool->has_rawvideo ?
+    pool->video_info.size :
+    (gsize) pool->video_info.width * pool->video_info.height;
+  gsize plane_sizes[GST_VIDEO_MAX_PLANES] = { plane_0_size, };
 
   GST_DEBUG_OBJECT (pool, "wrap buffer, datas:%d", b->buffer->n_datas);
 
@@ -333,11 +336,20 @@ set_config (GstBufferPool * pool, GstStructure * config)
   if (g_str_has_prefix (gst_structure_get_name (structure), "video/") ||
       g_str_has_prefix (gst_structure_get_name (structure), "image/")) {
     p->has_video = TRUE;
+
     gst_video_info_from_caps (&p->video_info, caps);
 
+    if (GST_VIDEO_FORMAT_INFO_IS_VALID_RAW (p->video_info.finfo)
+#ifdef HAVE_GSTREAMER_DMA_DRM
+        && GST_VIDEO_FORMAT_INFO_FORMAT (p->video_info.finfo) != GST_VIDEO_FORMAT_DMA_DRM
+#endif
+        )
+      p->has_rawvideo = TRUE;
+    else
+      p->has_rawvideo = FALSE;
+
 #ifdef HAVE_GSTREAMER_SHM_ALLOCATOR
-    if (GST_VIDEO_FORMAT_INFO_IS_VALID_RAW (p->video_info.finfo) &&
+    if (p->has_rawvideo) {
-        GST_VIDEO_FORMAT_INFO_FORMAT (p->video_info.finfo) != GST_VIDEO_FORMAT_DMA_DRM) {
       gst_video_alignment_reset (&p->video_align);
       gst_video_info_align (&p->video_info, &p->video_align);
     }
@@ -349,11 +361,11 @@ set_config (GstBufferPool * pool, GstStructure * config)
     g_assert_not_reached ();
   }
 
-  p->add_metavideo = p->has_video && gst_buffer_pool_config_has_option (config,
+  p->add_metavideo = p->has_rawvideo && gst_buffer_pool_config_has_option (config,
       GST_BUFFER_POOL_OPTION_VIDEO_META);
 
 #ifdef HAVE_GSTREAMER_SHM_ALLOCATOR
-  has_videoalign = p->has_video && gst_buffer_pool_config_has_option (config,
+  has_videoalign = p->has_rawvideo && gst_buffer_pool_config_has_option (config,
       GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
 
   if (has_videoalign) {

src/gst/gstpipewirepool.h

@@ -22,6 +22,12 @@ G_DECLARE_FINAL_TYPE (GstPipeWirePool, gst_pipewire_pool, GST, PIPEWIRE_POOL, Gs
 #define PIPEWIRE_POOL_MIN_BUFFERS 2u
 #define PIPEWIRE_POOL_MAX_BUFFERS 16u
 
+/* Only available in GStreamer 1.22+ */
+#ifndef GST_VIDEO_FORMAT_INFO_IS_VALID_RAW
+#define GST_VIDEO_FORMAT_INFO_IS_VALID_RAW(info)              \
+  (info != NULL && (info)->format > GST_VIDEO_FORMAT_ENCODED)
+#endif
+
 typedef struct _GstPipeWirePoolData GstPipeWirePoolData;
 struct _GstPipeWirePoolData {
   GstPipeWirePool *pool;
@@ -42,6 +48,7 @@ struct _GstPipeWirePool {
   guint n_buffers;
 
   gboolean has_video;
+  gboolean has_rawvideo;
   gboolean add_metavideo;
   GstAudioInfo audio_info;
   GstVideoInfo video_info;

src/gst/gstpipewiresink.c

@@ -321,11 +321,18 @@ gst_pipewire_sink_update_params (GstPipeWireSink *sink)
   spa_pod_builder_add (&b,
       SPA_PARAM_BUFFERS_size, SPA_POD_CHOICE_RANGE_Int(size, size, INT32_MAX),
       0);
-  if (sink->is_video) {
+  if (sink->is_rawvideo) {
     /* MUST have n_datas == n_planes */
     spa_pod_builder_add (&b,
         SPA_PARAM_BUFFERS_blocks,
-        SPA_POD_Int(GST_VIDEO_INFO_N_PLANES (&pool->video_info)), 0);
+        SPA_POD_Int(GST_VIDEO_INFO_N_PLANES (&pool->video_info)),
+        0);
+  } else {
+    /* Non-planar data, get a single block */
+    spa_pod_builder_add (&b,
+        SPA_PARAM_BUFFERS_blocks,
+        SPA_POD_Int(1),
+        0);
   }
 
   spa_pod_builder_add (&b,
@@ -343,7 +350,7 @@ gst_pipewire_sink_update_params (GstPipeWireSink *sink)
       SPA_PARAM_META_type, SPA_POD_Id(SPA_META_Header),
       SPA_PARAM_META_size, SPA_POD_Int(sizeof (struct spa_meta_header)));
 
-  if (sink->is_video) {
+  if (sink->is_rawvideo) {
     port_params[n_params++] = spa_pod_builder_add_object (&b,
         SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta,
         SPA_PARAM_META_type, SPA_POD_Id(SPA_META_VideoCrop),
@@ -369,7 +376,7 @@ gst_pipewire_sink_init (GstPipeWireSink * sink)
 
   sink->mode = DEFAULT_PROP_MODE;
   sink->use_bufferpool = DEFAULT_PROP_USE_BUFFERPOOL;
-  sink->is_video = false;
+  sink->is_rawvideo = false;
 
   GST_OBJECT_FLAG_SET (sink, GST_ELEMENT_FLAG_PROVIDE_CLOCK);
 
@@ -387,7 +394,7 @@ gst_pipewire_sink_sink_fixate (GstBaseSink * bsink, GstCaps * caps)
   structure = gst_caps_get_structure (caps, 0);
 
   if (gst_structure_has_name (structure, "video/x-raw")) {
-    pwsink->is_video = true;
+    pwsink->is_rawvideo = true;
     gst_structure_fixate_field_nearest_int (structure, "width", 320);
     gst_structure_fixate_field_nearest_int (structure, "height", 240);
     gst_structure_fixate_field_nearest_fraction (structure, "framerate", 30, 1);
@@ -779,9 +786,21 @@ gst_pipewire_sink_setcaps (GstBaseSink * bsink, GstCaps * caps)
     if (pwsink->use_bufferpool != USE_BUFFERPOOL_YES)
       pwsink->use_bufferpool = USE_BUFFERPOOL_NO;
   } else {
+    GstVideoInfo video_info;
+
     pwsink->rate = rate = 0;
     pwsink->rate_match = false;
-    pwsink->is_video = true;
+
+    gst_video_info_from_caps (&video_info, caps);
+
+    if (GST_VIDEO_FORMAT_INFO_IS_VALID_RAW (video_info.finfo)
+#ifdef HAVE_GSTREAMER_DMA_DRM
+        && GST_VIDEO_FORMAT_INFO_FORMAT (video_info.finfo) != GST_VIDEO_FORMAT_DMA_DRM
+#endif
+        )
+      pwsink->is_rawvideo = TRUE;
+    else
+      pwsink->is_rawvideo = FALSE;
   }
 
   spa_dll_set_bw(&pwsink->stream->dll, SPA_DLL_BW_MIN, 4096, rate);
@@ -866,7 +885,7 @@ gst_pipewire_sink_setcaps (GstBaseSink * bsink, GstCaps * caps)
   config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pwsink->stream->pool));
   gst_buffer_pool_config_get_params (config, NULL, &size, &min_buffers, &max_buffers);
   gst_buffer_pool_config_set_params (config, caps, size, min_buffers, max_buffers);
-  if (pwsink->is_video) {
+  if (pwsink->is_rawvideo) {
     gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
 #ifdef HAVE_GSTREAMER_SHM_ALLOCATOR
     gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
@@ -953,7 +972,7 @@ gst_pipewire_sink_render (GstBaseSink * bsink, GstBuffer * buffer)
       if (res != GST_FLOW_OK)
         goto done;
 
-      if (pwsink->is_video) {
+      if (pwsink->is_rawvideo) {
         GstVideoFrame src, dst;
         gboolean copied = FALSE;
         buf_size = 0; // to break from the loop

src/gst/gstpipewiresink.h

@@ -69,7 +69,7 @@ struct _GstPipeWireSink {
   gboolean negotiated;
   gboolean rate_match;
   gint rate;
-  gboolean is_video;
+  gboolean is_rawvideo;
 
   GstPipeWireSinkMode mode;
   GstPipeWireSinkSlaveMethod slave_method;

src/gst/gstpipewiresrc.c

@@ -777,7 +777,7 @@ static GstBuffer *dequeue_buffer(GstPipeWireSrc *pwsrc)
     pwsrc->transform_value = transform_value;
   }
 
-  if (pwsrc->is_video) {
+  if (pwsrc->is_rawvideo) {
     GstVideoInfo *info = &pwsrc->video_info;
     uint32_t n_datas = b->buffer->n_datas;
     uint32_t n_planes = GST_VIDEO_INFO_N_PLANES (info);
@@ -1285,6 +1285,16 @@ handle_format_change (GstPipeWireSrc *pwsrc,
         gst_video_info_dma_drm_init (&pwsrc->drm_info);
 #endif
         gst_video_info_from_caps (&pwsrc->video_info, pwsrc->caps);
+
+        if (GST_VIDEO_FORMAT_INFO_IS_VALID_RAW (pwsrc->video_info.finfo)
+#ifdef HAVE_GSTREAMER_DMA_DRM
+            && GST_VIDEO_FORMAT_INFO_FORMAT (pwsrc->video_info.finfo) != GST_VIDEO_FORMAT_DMA_DRM
+#endif
+            )
+          pwsrc->is_rawvideo = TRUE;
+        else
+          pwsrc->is_rawvideo = FALSE;
+
 #ifdef HAVE_GSTREAMER_DMA_DRM
       }
 #endif
@@ -1297,6 +1307,7 @@ handle_format_change (GstPipeWireSrc *pwsrc,
   } else {
     pwsrc->negotiated = FALSE;
     pwsrc->is_video = FALSE;
+    pwsrc->is_rawvideo = FALSE;
   }
 
   if (pwsrc->caps) {

src/gst/gstpipewiresrc.h

@@ -64,6 +64,7 @@ struct _GstPipeWireSrc {
   GstCaps *possible_caps;
 
   gboolean is_video;
+  gboolean is_rawvideo;
   GstVideoInfo video_info;
 #ifdef HAVE_GSTREAMER_DMA_DRM
   GstVideoInfoDmaDrm drm_info;