bluez5: probe adapter msbc capability via hci commands

Using a probe connection to determine adapter msbc capability causes
problems on some adapters (ff8c3d2, 84bc0490a5, 717004334b,
pipewire#2030) and seems to be a bad idea.

Go back to probing for transparent msbc transport capability via HCI
commands. bluetooth/hci.h may be deprecated later, but for now it's
better to go back to using it.  (In practice, adapters not supporting
esco appear to be fairly rare; kernel commit in 2013 refers to "older
devices", so if we can't use HCI, assume the adapter supports the
necessary modes.)

spa/plugins/bluez5/backend-native.c

@@ -550,11 +550,10 @@ fail:
 
 #ifdef HAVE_BLUEZ_5_BACKEND_HFP_NATIVE
 
-static int sco_create_socket(struct impl *backend, struct spa_bt_adapter *adapter, bool msbc);
-
 static bool device_supports_required_mSBC_transport_modes(
-		struct impl *backend, struct spa_bt_device *device) {
-	int sock;
+		struct impl *backend, struct spa_bt_device *device)
+{
+	int res;
 	bool msbc_ok, msbc_alt1_ok;
 	uint32_t bt_features;
 
@@ -575,43 +574,20 @@ static bool device_supports_required_mSBC_transport_modes(
 	if (!msbc_ok && !msbc_alt1_ok)
 		return false;
 
-	/*
-	 * Check if adapter supports BT_VOICE_TRANSPARENT. Do this without
-	 * directly probing HCI properties.
-	 */
-	sock = sco_create_socket(backend, device->adapter, true);
-	if (sock < 0) {
+	res = spa_bt_adapter_has_msbc(device->adapter);
+	if (res < 0) {
+		spa_log_warn(backend->log,
+				"adapter %s: failed to determine msbc/esco capability (%d)",
+				device->adapter->path, res);
+	} else if (res == 0) {
+		spa_log_info(backend->log,
+				"adapter %s: no msbc/esco transport",
+				device->adapter->path);
 		return false;
 	} else {
-		struct sockaddr_sco addr;
-		socklen_t len;
-		int res;
-
-		/* Connect to non-existent address */
-		len = sizeof(addr);
-		memset(&addr, 0, len);
-		addr.sco_family = AF_BLUETOOTH;
-		bacpy(&addr.sco_bdaddr, BDADDR_LOCAL);
-
-		spa_log_debug(backend->log, "connect to determine adapter msbc support...");
-
-		/* Linux kernel code checks for features needed for BT_VOICE_TRANSPARENT
-		 * among the first checks it does, and fails with EOPNOTSUPP if not
-		 * supported. The connection generally timeouts, so set it
-		 * nonblocking since we are just checking.
-		 */
-		fcntl(sock, F_SETFL, O_NONBLOCK);
-		res = connect(sock, (struct sockaddr *) &addr, len);
-		if (res < 0)
-			res = errno;
-		else
-			res = 0;
-		close(sock);
-
-		spa_log_debug(backend->log, "determined adapter-msbc:%d res:%d",
-				(res != EOPNOTSUPP), res);
-		if (res == EOPNOTSUPP)
-			return false;
+		spa_log_debug(backend->log,
+				"adapter %s: has msbc/esco transport",
+				device->adapter->path);
 	}
 
 	/* Check if USB ALT6 is really available on the device */

spa/plugins/bluez5/defs.h

@@ -332,11 +332,13 @@ struct spa_bt_adapter {
 	uint32_t bluetooth_class;
 	uint32_t profiles;
 	int powered;
+	unsigned int has_msbc:1;
+	unsigned int msbc_probed:1;
 	unsigned int endpoints_registered:1;
 	unsigned int application_registered:1;
 	unsigned int player_registered:1;
-	unsigned int has_battery_provider;
-	unsigned int battery_provider_unavailable;
+	unsigned int has_battery_provider:1;
+	unsigned int battery_provider_unavailable:1;
 };
 
 enum spa_bt_form_factor {
@@ -696,6 +698,8 @@ int spa_bt_quirks_get_features(const struct spa_bt_quirks *quirks,
 		uint32_t *features);
 void spa_bt_quirks_destroy(struct spa_bt_quirks *quirks);
 
+int spa_bt_adapter_has_msbc(struct spa_bt_adapter *adapter);
+
 struct spa_bt_backend_implementation {
 #define SPA_VERSION_BT_BACKEND_IMPLEMENTATION	0
 	uint32_t version;

spa/plugins/bluez5/hci.c

@@ -0,0 +1,93 @@
+/* Spa HSP/HFP native backend HCI support
+ *
+ * Copyright © 2022 Pauli Virtanen
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <errno.h>
+#include <unistd.h>
+#include <stdarg.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <sys/uio.h>
+#include <sys/socket.h>
+
+#include "defs.h"
+
+#ifndef HAVE_BLUEZ_5_HCI
+
+int spa_bt_adapter_has_msbc(struct spa_bt_adapter *adapter)
+{
+	if (adapter->msbc_probed)
+		return adapter->has_msbc;
+	return -EOPNOTSUPP;
+}
+
+#else
+
+#include <bluetooth/bluetooth.h>
+#include <bluetooth/hci.h>
+#include <bluetooth/hci_lib.h>
+
+int spa_bt_adapter_has_msbc(struct spa_bt_adapter *adapter)
+{
+	int hci_id, res;
+	int sock = -1;
+	uint8_t features[8], max_page = 0;
+	struct sockaddr_hci a;
+	const char *str;
+
+	if (adapter->msbc_probed)
+		return adapter->has_msbc;
+
+	str = strrchr(adapter->path, '/');  /* hciXX */
+	if (str == NULL || sscanf(str, "/hci%d", &hci_id) != 1 || hci_id < 0)
+		return -ENOENT;
+
+	sock = socket(AF_BLUETOOTH, SOCK_RAW | SOCK_CLOEXEC, BTPROTO_HCI);
+	if (sock < 0)
+		goto error;
+
+	memset(&a, 0, sizeof(a));
+	a.hci_family = AF_BLUETOOTH;
+	a.hci_dev = hci_id;
+	if (bind(sock, (struct sockaddr *) &a, sizeof(a)) < 0)
+		goto error;
+
+	if (hci_read_local_ext_features(sock, 0, &max_page, features, 1000) < 0)
+		goto error;
+
+	close(sock);
+
+	adapter->msbc_probed = true;
+	adapter->has_msbc = ((features[2] & LMP_TRSP_SCO) && (features[3] & LMP_ESCO)) ? 1 : 0;
+	return adapter->has_msbc;
+
+error:
+	res = -errno;
+	if (sock >= 0)
+		close(sock);
+	return res;
+}
+
+#endif

spa/plugins/bluez5/meson.build

@@ -20,6 +20,10 @@ if not get_option('bluez5-backend-hsphfpd').disabled()
   cdata.set('HAVE_BLUEZ_5_BACKEND_HSPHFPD', 1)
 endif
 
+if dependency('bluez', version: '< 6', required: false).found()
+  cdata.set('HAVE_BLUEZ_5_HCI', 1)
+endif
+
 bluez5_sources = [
   'plugin.c',
   'codec-loader.c',
@@ -32,7 +36,8 @@ bluez5_sources = [
   'quirks.c',
   'player.c',
   'bluez5-device.c',
-  'bluez5-dbus.c'
+  'bluez5-dbus.c',
+  'hci.c'
 ]
 
 bluez5_data = ['bluez-hardware.conf']