diff --git a/src/pipewire/buffers.c b/src/pipewire/buffers.c
index f15f75753..e0b74eb25 100644
--- a/src/pipewire/buffers.c
+++ b/src/pipewire/buffers.c
@@ -318,10 +318,12 @@ int pw_buffers_negotiate(struct pw_context *context, uint32_t flags,
 		minsize = 8192;
 		max_buffers = 2;
 	}
-
-	if (types == SPA_ID_INVALID &&
-	    SPA_FLAG_IS_SET(flags, PW_BUFFERS_FLAG_SHARED_MEM))
-		types = 1<<SPA_DATA_MemFd;
+	if (SPA_FLAG_IS_SET(flags, PW_BUFFERS_FLAG_SHARED_MEM)) {
+		if (types != SPA_ID_INVALID)
+			SPA_FLAG_CLEAR(types, 1<<SPA_DATA_MemPtr);
+		if (types == 0 || types == SPA_ID_INVALID)
+			types = 1<<SPA_DATA_MemFd;
+	}
 
 	if (SPA_FLAG_IS_SET(flags, PW_BUFFERS_FLAG_NO_MEM))
 		minsize = 0;