diff --git a/spa/include/spa/utils/names.h b/spa/include/spa/utils/names.h index 30b3d698f..d5f9b6d4e 100644 --- a/spa/include/spa/utils/names.h +++ b/spa/include/spa/utils/names.h @@ -73,6 +73,9 @@ extern "C" { #define SPA_NAME_VIDEO_CONVERT "video.convert" /**< converts raw video from one format * to another. Must include at least * format and scaling */ +#define SPA_NAME_VIDEO_CONVERT_VULKAN "video.convert.vulkan" /**< converts raw video from one format + * to another using vulkan. Must + * include at least format and scaling */ #define SPA_NAME_VIDEO_ADAPT "video.adapt" /**< combination of a node and a * video.convert. */ /** keys for alsa factory names */ @@ -134,6 +137,8 @@ extern "C" { "api.vulkan.compute.source" /**< a vulkan compute source. */ #define SPA_NAME_API_VULKAN_COMPUTE_FILTER \ "api.vulkan.compute.filter" /**< a vulkan compute filter. */ +#define SPA_NAME_API_VULKAN_BLIT_FILTER \ + "api.vulkan.blit.filter" /**< a vulkan blit filter. */ /** * \} diff --git a/spa/plugins/videoconvert/dmabuf.h b/spa/plugins/videoconvert/dmabuf.h new file mode 100644 index 000000000..2bc68a1b9 --- /dev/null +++ b/spa/plugins/videoconvert/dmabuf.h @@ -0,0 +1,62 @@ +// Copyright (c) 2023 The wlroots contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Obtained from https://gitlab.freedesktop.org/wlroots/wlroots/ + +/* SPDX-FileCopyrightText: Copyright © 2023 The wlroots contributors */ +/* SPDX-License-Identifier: MIT */ + +#ifndef RENDER_DMABUF_H +#define RENDER_DMABUF_H + +#include +#include + +#include "spa/support/log.h" + +// Copied from to avoid #ifdef soup +#define DMA_BUF_SYNC_READ (1 << 0) +#define DMA_BUF_SYNC_WRITE (2 << 0) +#define DMA_BUF_SYNC_RW (DMA_BUF_SYNC_READ | DMA_BUF_SYNC_WRITE) + +/** + * Check whether DMA-BUF import/export from/to sync_file is available. + * + * If this function returns true, dmabuf_import_sync_file() is supported. + */ +bool dmabuf_check_sync_file_import_export(struct spa_log *log); + +/** + * Import a sync_file into a DMA-BUF with DMA_BUF_IOCTL_IMPORT_SYNC_FILE. + * + * This can be used to make explicit sync interoperate with implicit sync. + */ +bool dmabuf_import_sync_file(struct spa_log *log, int dmabuf_fd, uint32_t flags, int sync_file_fd); + +/** + * Export a sync_file from a DMA-BUF with DMA_BUF_IOCTL_EXPORT_SYNC_FILE. + * + * The sync_file FD is returned on success, -1 is returned on error. + * + * This can be used to make explicit sync interoperate with implicit sync. + */ +int dmabuf_export_sync_file(struct spa_log *log, int dmabuf_fd, uint32_t flags); + +#endif diff --git a/spa/plugins/videoconvert/dmabuf_fallback.c b/spa/plugins/videoconvert/dmabuf_fallback.c new file mode 100644 index 000000000..20db30735 --- /dev/null +++ b/spa/plugins/videoconvert/dmabuf_fallback.c @@ -0,0 +1,42 @@ +// Copyright (c) 2023 The wlroots contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Obtained from https://gitlab.freedesktop.org/wlroots/wlroots/ + +/* SPDX-FileCopyrightText: Copyright © 2023 The wlroots contributors */ +/* SPDX-License-Identifier: MIT */ + +#include +#include + + +bool dmabuf_check_sync_file_import_export(struct spa_log *log) { + return false; +} + +bool dmabuf_import_sync_file(struct spa_log *log, int dmabuf_fd, uint32_t flags, int sync_file_fd) { + spa_log_error("DMA-BUF sync_file import IOCTL not available on this system"); + return false; +} + +int dmabuf_export_sync_file(struct spa_log *log, int dmabuf_fd, uint32_t flags) { + spa_log_error("DMA-BUF sync_file export IOCTL not available on this system"); + return false; +} diff --git a/spa/plugins/videoconvert/dmabuf_linux.c b/spa/plugins/videoconvert/dmabuf_linux.c new file mode 100644 index 000000000..f3b9c6f13 --- /dev/null +++ b/spa/plugins/videoconvert/dmabuf_linux.c @@ -0,0 +1,127 @@ +// Copyright (c) 2023 The wlroots contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Obtained from https://gitlab.freedesktop.org/wlroots/wlroots/ + +/* SPDX-FileCopyrightText: Copyright © 2023 The wlroots contributors */ +/* SPDX-License-Identifier: MIT */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "dmabuf.h" + +bool dmabuf_check_sync_file_import_export(struct spa_log *log) { + /* Unfortunately there's no better way to check the availability of the + * IOCTL than to check the kernel version. See the discussion at: + * https://lore.kernel.org/dri-devel/20220601161303.64797-1-contact@emersion.fr/ + */ + + struct utsname utsname = {0}; + if (uname(&utsname) != 0) { + spa_log_warn(log, "uname failed"); + return false; + } + + if (strcmp(utsname.sysname, "Linux") != 0) { + return false; + } + + // Trim release suffix if any, e.g. "-arch1-1" + for (size_t i = 0; utsname.release[i] != '\0'; i++) { + char ch = utsname.release[i]; + if ((ch < '0' || ch > '9') && ch != '.') { + utsname.release[i] = '\0'; + break; + } + } + + char *rel = strtok(utsname.release, "."); + int major = atoi(rel); + + int minor = 0; + rel = strtok(NULL, "."); + if (rel != NULL) { + minor = atoi(rel); + } + + int patch = 0; + rel = strtok(NULL, "."); + if (rel != NULL) { + patch = atoi(rel); + } + + return KERNEL_VERSION(major, minor, patch) >= KERNEL_VERSION(5, 20, 0); +} + +// TODO: drop these definitions once widespread + +#if !defined(DMA_BUF_IOCTL_IMPORT_SYNC_FILE) + +struct dma_buf_import_sync_file { + __u32 flags; + __s32 fd; +}; + +#define DMA_BUF_IOCTL_IMPORT_SYNC_FILE _IOW(DMA_BUF_BASE, 3, struct dma_buf_import_sync_file) + +#endif + +#if !defined(DMA_BUF_IOCTL_EXPORT_SYNC_FILE) + +struct dma_buf_export_sync_file { + __u32 flags; + __s32 fd; +}; + +#define DMA_BUF_IOCTL_EXPORT_SYNC_FILE _IOWR(DMA_BUF_BASE, 2, struct dma_buf_export_sync_file) + +#endif + +bool dmabuf_import_sync_file(struct spa_log *log, int dmabuf_fd, uint32_t flags, int sync_file_fd) { + struct dma_buf_import_sync_file data = { + .flags = flags, + .fd = sync_file_fd, + }; + if (drmIoctl(dmabuf_fd, DMA_BUF_IOCTL_IMPORT_SYNC_FILE, &data) != 0) { + spa_log_error(log, "drmIoctl(IMPORT_SYNC_FILE) failed with %d (%s)", errno, spa_strerror(-errno)); + return false; + } + return true; +} + +int dmabuf_export_sync_file(struct spa_log *log, int dmabuf_fd, uint32_t flags) { + struct dma_buf_export_sync_file data = { + .flags = flags, + .fd = -1, + }; + if (drmIoctl(dmabuf_fd, DMA_BUF_IOCTL_EXPORT_SYNC_FILE, &data) != 0) { + spa_log_error(log, "drmIoctl(EXPORT_SYNC_FILE) failed with %d (%s)", errno, spa_strerror(-errno)); + return -1; + } + return data.fd; +} diff --git a/spa/plugins/videoconvert/meson.build b/spa/plugins/videoconvert/meson.build index 24673a541..2f2850288 100644 --- a/spa/plugins/videoconvert/meson.build +++ b/spa/plugins/videoconvert/meson.build @@ -3,13 +3,35 @@ videoconvert_sources = [ 'plugin.c' ] +videoconvert_dep = [] +if vulkan_headers +videoconvert_sources += files( + 'pixel-formats.c', + 'videoconvert-vulkan.c', + 'vulkan-blit-utils.c', + 'vulkan-utils.c', + 'utils.c' + ) + +drm = dependency('libdrm') + +if cc.has_header('linux/dma-buf.h') and target_machine.system() == 'linux' + videoconvert_sources += files('dmabuf_linux.c') +else + videoconvert_sources += files('dmabuf_fallback.c') +endif + +videoconvert_dep += vulkan_dep +videoconvert_dep += drm +endif + simd_cargs = [] simd_dependencies = [] videoconvertlib = shared_library('spa-videoconvert', videoconvert_sources, c_args : simd_cargs, - dependencies : [ spa_dep, mathlib ], + dependencies : [ spa_dep, mathlib, videoconvert_dep ], link_with : simd_dependencies, install : true, install_dir : spa_plugindir / 'videoconvert') diff --git a/spa/plugins/videoconvert/pixel-formats.c b/spa/plugins/videoconvert/pixel-formats.c new file mode 100644 index 000000000..06144d758 --- /dev/null +++ b/spa/plugins/videoconvert/pixel-formats.c @@ -0,0 +1,33 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include "pixel-formats.h" + +#include +#include + +struct pixel_info { + uint32_t format; + uint32_t bpp; +} pixel_infos[] = { + { SPA_VIDEO_FORMAT_RGBA_F32, 16 }, + { SPA_VIDEO_FORMAT_BGRA, 4 }, + { SPA_VIDEO_FORMAT_RGBA, 4 }, + { SPA_VIDEO_FORMAT_BGRx, 4 }, + { SPA_VIDEO_FORMAT_RGBx, 4 }, + { SPA_VIDEO_FORMAT_BGR, 3 }, + { SPA_VIDEO_FORMAT_RGB, 3 }, +}; + +bool get_pixel_format_info(uint32_t format, struct pixel_format_info *info) +{ + struct pixel_info *p; + SPA_FOR_EACH_ELEMENT(pixel_infos, p) { + if (p->format != format) + continue; + info->bpp = p->bpp; + return true; + } + return false; +} diff --git a/spa/plugins/videoconvert/pixel-formats.h b/spa/plugins/videoconvert/pixel-formats.h new file mode 100644 index 000000000..949733107 --- /dev/null +++ b/spa/plugins/videoconvert/pixel-formats.h @@ -0,0 +1,12 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include +#include + +struct pixel_format_info { + uint32_t bpp; // bytes per pixel +}; + +bool get_pixel_format_info(uint32_t format, struct pixel_format_info *info); diff --git a/spa/plugins/videoconvert/plugin.c b/spa/plugins/videoconvert/plugin.c index 78528141e..5c2b693c3 100644 --- a/spa/plugins/videoconvert/plugin.c +++ b/spa/plugins/videoconvert/plugin.c @@ -8,6 +8,7 @@ #include extern const struct spa_handle_factory spa_videoadapter_factory; +extern const struct spa_handle_factory spa_videoconvert_vulkan_factory; SPA_LOG_TOPIC_ENUM_DEFINE_REGISTERED; @@ -21,6 +22,9 @@ int spa_handle_factory_enum(const struct spa_handle_factory **factory, uint32_t case 0: *factory = &spa_videoadapter_factory; break; + case 1: + *factory = &spa_videoconvert_vulkan_factory; + break; default: return 0; } diff --git a/spa/plugins/videoconvert/utils.c b/spa/plugins/videoconvert/utils.c new file mode 100644 index 000000000..922eb9ca1 --- /dev/null +++ b/spa/plugins/videoconvert/utils.c @@ -0,0 +1,94 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include "utils.h" + +#include +#include +#include + +// This function enumerates the available formats in vulkan_state::formats, announcing all formats capable to support DmaBufs +// first and then falling back to those supported with SHM buffers. +bool find_EnumFormatInfo(struct vulkan_format_infos *fmtInfos, uint32_t index, uint32_t caps, uint32_t *fmt_idx, bool *has_modifier) { + int64_t fmtIterator = 0; + int64_t maxIterator = 0; + if (caps & VULKAN_BUFFER_TYPE_CAP_SHM) + maxIterator += fmtInfos->formatCount; + if (caps & VULKAN_BUFFER_TYPE_CAP_DMABUF) + maxIterator += fmtInfos->formatCount; + // Count available formats until index underflows, while fmtIterator indexes the current format. + // Iterate twice over formats first time with modifiers, second time without if both caps are supported. + while (index < (uint32_t)-1 && fmtIterator < maxIterator) { + const struct vulkan_format_info *f_info = &fmtInfos->infos[fmtIterator%fmtInfos->formatCount]; + if (caps & VULKAN_BUFFER_TYPE_CAP_DMABUF && fmtIterator < fmtInfos->formatCount) { + // First round, check for modifiers + if (f_info->modifierCount > 0) { + index--; + } + } else if (caps & VULKAN_BUFFER_TYPE_CAP_SHM) { + // Second round, every format should be supported. + index--; + } + fmtIterator++; + } + + if (index != (uint32_t)-1) { + // No more formats available + return false; + } + // Undo end of loop increment + fmtIterator--; + *fmt_idx = fmtIterator%fmtInfos->formatCount; + // Loop finished in first round + *has_modifier = caps & VULKAN_BUFFER_TYPE_CAP_DMABUF && fmtIterator < fmtInfos->formatCount; + return true; +} + +struct spa_pod *build_dsp_EnumFormat(const struct vulkan_format_info *fmt, bool with_modifiers, struct spa_pod_builder *builder) { + struct spa_pod_frame f[2]; + uint32_t i, c; + + spa_pod_builder_push_object(builder, &f[0], SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat); + spa_pod_builder_add(builder, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), 0); + spa_pod_builder_add(builder, SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_dsp), 0); + spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_format, SPA_POD_Id(fmt->spa_format), 0); + if (with_modifiers && fmt->modifierCount > 0) { + spa_pod_builder_prop(builder, SPA_FORMAT_VIDEO_modifier, SPA_POD_PROP_FLAG_MANDATORY | SPA_POD_PROP_FLAG_DONT_FIXATE); + spa_pod_builder_push_choice(builder, &f[1], SPA_CHOICE_Enum, 0); + for (i = 0, c = 0; i < fmt->modifierCount; i++) { + spa_pod_builder_long(builder, fmt->infos[i].props.drmFormatModifier); + if (c++ == 0) + spa_pod_builder_long(builder, fmt->infos[i].props.drmFormatModifier); + } + spa_pod_builder_pop(builder, &f[1]); + } + return spa_pod_builder_pop(builder, &f[0]); +} + +struct spa_pod *build_raw_EnumFormat(const struct vulkan_format_info *fmt, bool with_modifiers, struct spa_pod_builder *builder) { + struct spa_pod_frame f[2]; + uint32_t i, c; + + spa_pod_builder_push_object(builder, &f[0], SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat); + spa_pod_builder_add(builder, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), 0); + spa_pod_builder_add(builder, SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), 0); + spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_format, SPA_POD_Id(fmt->spa_format), 0); + if (with_modifiers && fmt->modifierCount > 0) { + spa_pod_builder_prop(builder, SPA_FORMAT_VIDEO_modifier, SPA_POD_PROP_FLAG_MANDATORY | SPA_POD_PROP_FLAG_DONT_FIXATE); + spa_pod_builder_push_choice(builder, &f[1], SPA_CHOICE_Enum, 0); + for (i = 0, c = 0; i < fmt->modifierCount; i++) { + spa_pod_builder_long(builder, fmt->infos[i].props.drmFormatModifier); + if (c++ == 0) + spa_pod_builder_long(builder, fmt->infos[i].props.drmFormatModifier); + } + spa_pod_builder_pop(builder, &f[1]); + } + spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_size, + SPA_POD_CHOICE_RANGE_Rectangle( + &SPA_RECTANGLE(640, 480), // Arbitrary + &SPA_RECTANGLE(1, 1), + &SPA_RECTANGLE(8192, 4320)), + 0); + return spa_pod_builder_pop(builder, &f[0]); +} diff --git a/spa/plugins/videoconvert/utils.h b/spa/plugins/videoconvert/utils.h new file mode 100644 index 000000000..9716804a3 --- /dev/null +++ b/spa/plugins/videoconvert/utils.h @@ -0,0 +1,11 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include "vulkan-types.h" +#include "spa/pod/builder.h" + +bool find_EnumFormatInfo(struct vulkan_format_infos *fmtInfos, uint32_t index, uint32_t caps, uint32_t *fmt_idx, bool *has_modifier); + +struct spa_pod *build_dsp_EnumFormat(const struct vulkan_format_info *fmt, bool with_modifiers, struct spa_pod_builder *builder); +struct spa_pod *build_raw_EnumFormat(const struct vulkan_format_info *fmt, bool with_modifiers, struct spa_pod_builder *builder); diff --git a/spa/plugins/videoconvert/videoadapter.c b/spa/plugins/videoconvert/videoadapter.c index b47ddc83b..6408173ec 100644 --- a/spa/plugins/videoconvert/videoadapter.c +++ b/spa/plugins/videoconvert/videoadapter.c @@ -28,9 +28,14 @@ #define SPA_LOG_TOPIC_DEFAULT &log_topic SPA_LOG_TOPIC_DEFINE_STATIC(log_topic, "spa.videoadapter"); +#if 1 +#define VIDEOCONVERTER 1 +#define VIDEOCONVERTER_FACTORY spa_videoconvert_vulkan_factory +#endif + #define DEFAULT_ALIGN 16 -#define MAX_PORTS 1 +#define MAX_PORTS (1+1) /** \cond */ @@ -48,14 +53,15 @@ struct impl { struct spa_node *follower; struct spa_hook follower_listener; - uint32_t follower_flags; + uint64_t follower_flags; struct spa_video_info follower_current_format; struct spa_video_info default_format; + int in_set_param; struct spa_handle *hnd_convert; struct spa_node *convert; struct spa_hook convert_listener; - uint32_t convert_flags; + uint64_t convert_flags; uint32_t n_buffers; struct spa_buffer **buffers; @@ -78,6 +84,7 @@ struct impl { struct spa_param_info params[N_NODE_PARAMS]; uint32_t convert_params_flags[N_NODE_PARAMS]; uint32_t follower_params_flags[N_NODE_PARAMS]; + uint64_t follower_port_flags; struct spa_hook_list hooks; struct spa_callbacks callbacks; @@ -85,6 +92,8 @@ struct impl { unsigned int add_listener:1; unsigned int have_format:1; unsigned int started:1; + unsigned int warned:1; + unsigned int ready:1; unsigned int driver:1; unsigned int async:1; unsigned int passthrough:1; @@ -120,6 +129,27 @@ static int follower_enum_params(struct impl *this, return 0; } +static int convert_enum_port_config(struct impl *this, + int seq, uint32_t id, uint32_t start, uint32_t num, + const struct spa_pod *filter, struct spa_pod_builder *builder) +{ + struct spa_pod *f1, *f2 = NULL; + int res; + + f1 = spa_pod_builder_add_object(builder, + SPA_TYPE_OBJECT_ParamPortConfig, id, + SPA_PARAM_PORT_CONFIG_direction, SPA_POD_Id(this->direction)); + + if (filter) { + if ((res = spa_pod_filter(builder, &f2, f1, filter)) < 0) + return res; + } + else { + f2 = f1; + } + return spa_node_enum_params(this->convert, seq, id, start, num, f2); +} + static int impl_node_enum_params(void *object, int seq, uint32_t id, uint32_t start, uint32_t num, const struct spa_pod *filter) @@ -145,11 +175,30 @@ next: switch (id) { case SPA_PARAM_EnumPortConfig: - case SPA_PARAM_PortConfig: if (this->convert == NULL) return 0; - res = spa_node_enum_params(this->convert, seq, id, start, num, filter); - return res; + return convert_enum_port_config(this, seq, id, start, num, filter, &b.b); + case SPA_PARAM_PortConfig: + if (this->passthrough) { + switch (result.index) { + case 0: + result.param = spa_pod_builder_add_object(&b.b, + SPA_TYPE_OBJECT_ParamPortConfig, id, + SPA_PARAM_PORT_CONFIG_direction, SPA_POD_Id(this->direction), + SPA_PARAM_PORT_CONFIG_mode, SPA_POD_Id( + SPA_PARAM_PORT_CONFIG_MODE_passthrough)); + result.next++; + res = 1; + break; + default: + return 0; + } + } else { + if (this->convert == NULL) + return 0; + return convert_enum_port_config(this, seq, id, start, num, filter, &b.b); + } + break; case SPA_PARAM_PropInfo: res = follower_enum_params(this, id, IDX_PropInfo, &result, filter, &b.b); @@ -200,21 +249,6 @@ static int link_io(struct impl *this) spa_zero(this->io_rate_match); this->io_rate_match.rate = 1.0; - if ((res = spa_node_port_set_io(this->follower, - this->direction, 0, - SPA_IO_RateMatch, - &this->io_rate_match, sizeof(this->io_rate_match))) < 0) { - spa_log_debug(this->log, "%p: set RateMatch on follower disabled %d %s", this, - res, spa_strerror(res)); - } - else if ((res = spa_node_port_set_io(this->convert, - SPA_DIRECTION_REVERSE(this->direction), 0, - SPA_IO_RateMatch, - &this->io_rate_match, sizeof(this->io_rate_match))) < 0) { - spa_log_warn(this->log, "%p: set RateMatch on convert failed %d %s", this, - res, spa_strerror(res)); - } - this->io_buffers = SPA_IO_BUFFERS_INIT; if ((res = spa_node_port_set_io(this->follower, @@ -280,7 +314,7 @@ static int debug_params(struct impl *this, struct spa_node *node, if (filter) { spa_log_error(this->log, "with this filter:"); - spa_debug_log_pod(this->log, SPA_LOG_LEVEL_DEBUG, 2, NULL, filter); + spa_debug_log_pod(this->log, SPA_LOG_LEVEL_ERROR, 2, NULL, filter); } else { spa_log_error(this->log, "there was no filter"); } @@ -298,7 +332,7 @@ static int debug_params(struct impl *this, struct spa_node *node, break; } spa_log_error(this->log, "unmatched %s %d:", debug, count); - spa_debug_log_pod(this->log, SPA_LOG_LEVEL_DEBUG, 2, NULL, param); + spa_debug_log_pod(this->log, SPA_LOG_LEVEL_ERROR, 2, NULL, param); count++; } if (count == 0) @@ -318,13 +352,15 @@ static int negotiate_buffers(struct impl *this) uint32_t i, size, buffers, blocks, align, flags, stride = 0; uint32_t *aligns; struct spa_data *datas; - uint32_t follower_flags, conv_flags; + uint64_t follower_flags, conv_flags; spa_log_debug(this->log, "%p: %d", this, this->n_buffers); if (this->target == this->follower) return 0; + spa_log_debug(this->log, "%p: n_buffers:%d", this, this->n_buffers); + if (this->n_buffers > 0) return 0; @@ -478,7 +514,7 @@ static int configure_convert(struct impl *this, uint32_t mode) spa_pod_builder_init(&b, buffer, sizeof(buffer)); - spa_log_debug(this->log, "%p: configure convert %p", this, this->target); + spa_log_debug(this->log, "%p: configure convert %p %d", this, this->target, mode); param = spa_pod_builder_add_object(&b, SPA_TYPE_OBJECT_ParamPortConfig, SPA_PARAM_PortConfig, @@ -488,7 +524,7 @@ static int configure_convert(struct impl *this, uint32_t mode) return spa_node_set_param(this->convert, SPA_PARAM_PortConfig, 0, param); } -extern const struct spa_handle_factory spa_videoconvert_factory; +extern const struct spa_handle_factory VIDEOCONVERTER_FACTORY; static const struct spa_node_events follower_node_events; @@ -530,12 +566,13 @@ static int reconfigure_mode(struct impl *this, bool passthrough, } else { /* add converter ports */ configure_convert(this, SPA_PARAM_PORT_CONFIG_MODE_dsp); - link_io(this); } + link_io(this); } - this->info.change_mask |= SPA_NODE_CHANGE_MASK_FLAGS | SPA_NODE_CHANGE_MASK_PARAMS; - this->info.flags &= ~SPA_NODE_FLAG_NEED_CONFIGURE; + SPA_FLAG_CLEAR(this->info.flags, SPA_NODE_FLAG_NEED_CONFIGURE); + SPA_FLAG_UPDATE(this->info.flags, SPA_NODE_FLAG_ASYNC, + this->async && this->follower == this->target); this->params[IDX_Props].user++; emit_node_info(this, false); @@ -562,9 +599,8 @@ static int impl_node_set_param(void *object, uint32_t id, uint32_t flags, if ((res = spa_format_parse(param, &info.media_type, &info.media_subtype)) < 0) return res; if (info.media_type != SPA_MEDIA_TYPE_video || - info.media_subtype != SPA_MEDIA_SUBTYPE_raw) - return -EINVAL; - + info.media_subtype != SPA_MEDIA_SUBTYPE_raw) + return -EINVAL; if (spa_format_video_raw_parse(param, &info.info.raw) < 0) return -EINVAL; @@ -599,8 +635,9 @@ static int impl_node_set_param(void *object, uint32_t id, uint32_t flags, info.media_subtype != SPA_MEDIA_SUBTYPE_raw) return -ENOTSUP; - if (spa_format_video_raw_parse(format, &info.info.raw) >= 0) + if (spa_format_video_raw_parse(format, &info.info.raw) >= 0) { this->default_format = info; + } } switch (mode) { @@ -629,13 +666,16 @@ static int impl_node_set_param(void *object, uint32_t id, uint32_t flags, } case SPA_PARAM_Props: - if (this->target != this->follower) - res = spa_node_set_param(this->target, id, flags, param); - res2 = spa_node_set_param(this->follower, id, flags, param); + { + int in_set_param = ++this->in_set_param; + res = spa_node_set_param(this->follower, id, flags, param); + if (this->target != this->follower && this->in_set_param == in_set_param) + res2 = spa_node_set_param(this->target, id, flags, param); if (res < 0 && res2 < 0) return res; res = 0; break; + } case SPA_PARAM_ProcessLatency: res = spa_node_set_param(this->follower, id, flags, param); break; @@ -712,10 +752,12 @@ static int negotiate_format(struct impl *this) struct spa_pod_builder b = { 0 }; int res; - if (this->have_format) + if (this->target == this->follower) return 0; - if (this->target == this->follower) + spa_log_debug(this->log, "%p: have_format:%d", this, this->have_format); + + if (this->have_format) return 0; spa_pod_builder_init(&b, buffer, sizeof(buffer)); @@ -787,18 +829,30 @@ static int impl_node_send_command(void *object, const struct spa_command *comman switch (SPA_NODE_COMMAND_ID(command)) { case SPA_NODE_COMMAND_Start: + spa_log_debug(this->log, "%p: starting %d", this, this->started); + if (this->started) + return 0; if ((res = negotiate_format(this)) < 0) return res; if ((res = negotiate_buffers(this)) < 0) return res; + this->ready = true; + this->warned = false; break; case SPA_NODE_COMMAND_Suspend: - configure_format(this, 0, NULL); - SPA_FALLTHROUGH + this->started = false; + this->ready = false; + this->warned = false; + spa_log_debug(this->log, "%p: suspending", this); + break; case SPA_NODE_COMMAND_Pause: this->started = false; + this->ready = false; + this->warned = false; + spa_log_debug(this->log, "%p: pausing", this); break; case SPA_NODE_COMMAND_Flush: + spa_log_debug(this->log, "%p: flushing", this); this->io_buffers.status = SPA_STATUS_OK; break; default: @@ -809,20 +863,35 @@ static int impl_node_send_command(void *object, const struct spa_command *comman spa_log_error(this->log, "%p: can't send command %d: %s", this, SPA_NODE_COMMAND_ID(command), spa_strerror(res)); - return res; } - if (this->target != this->follower) { + if (res >= 0 && this->target != this->follower) { if ((res = spa_node_send_command(this->follower, command)) < 0) { spa_log_error(this->log, "%p: can't send command %d: %s", this, SPA_NODE_COMMAND_ID(command), spa_strerror(res)); - return res; } } switch (SPA_NODE_COMMAND_ID(command)) { case SPA_NODE_COMMAND_Start: - this->started = true; + if (res < 0) { + spa_log_debug(this->log, "%p: start failed", this); + this->ready = false; + configure_format(this, 0, NULL); + } else { + this->started = true; + spa_log_debug(this->log, "%p: started", this); + } + break; + case SPA_NODE_COMMAND_Suspend: + configure_format(this, 0, NULL); + spa_log_debug(this->log, "%p: suspended", this); + break; + case SPA_NODE_COMMAND_Pause: + spa_log_debug(this->log, "%p: paused", this); + break; + case SPA_NODE_COMMAND_Flush: + spa_log_debug(this->log, "%p: flushed", this); break; } return res; @@ -866,10 +935,11 @@ static void convert_node_info(void *data, const struct spa_node_info *info) (this->params[idx].flags & SPA_PARAM_INFO_SERIAL) | (info->params[i].flags & SPA_PARAM_INFO_READWRITE); - if (!this->add_listener) { - this->params[idx].user++; - spa_log_debug(this->log, "param %d changed", info->params[i].id); - } + if (this->add_listener) + continue; + + this->params[idx].user++; + spa_log_debug(this->log, "param %d changed", info->params[i].id); } } emit_node_info(this, false); @@ -880,12 +950,19 @@ static void convert_port_info(void *data, const struct spa_port_info *info) { struct impl *this = data; + struct spa_port_info pi; if (direction != this->direction) { + /* skip the converter output port into the follower */ if (port_id == 0) return; else + /* the monitor ports are exposed */ port_id--; + } else if (info) { + pi = *info; + pi.flags = this->follower_port_flags; + info = π } spa_log_debug(this->log, "%p: port info %d:%d", this, @@ -938,6 +1015,8 @@ static void follower_info(void *data, const struct spa_node_info *info) this->info.flags |= SPA_NODE_FLAG_OUT_PORT_CONFIG; this->info.max_output_ports = MAX_PORTS; } + SPA_FLAG_UPDATE(this->info.flags, SPA_NODE_FLAG_ASYNC, + this->async && this->follower == this->target); spa_log_debug(this->log, "%p: follower info %s", this, this->direction == SPA_DIRECTION_INPUT ? @@ -974,10 +1053,11 @@ static void follower_info(void *data, const struct spa_node_info *info) (this->params[idx].flags & SPA_PARAM_INFO_SERIAL) | (info->params[i].flags & SPA_PARAM_INFO_READWRITE); - if (!this->add_listener) { - this->params[idx].user++; - spa_log_debug(this->log, "param %d changed", info->params[i].id); - } + if (this->add_listener) + continue; + + this->params[idx].user++; + spa_log_debug(this->log, "param %d changed", info->params[i].id); } } emit_node_info(this, false); @@ -987,6 +1067,7 @@ static void follower_info(void *data, const struct spa_node_info *info) } +/* static int recalc_latency(struct impl *this, enum spa_direction direction, uint32_t port_id) { struct spa_pod_builder b = { 0 }; @@ -1005,8 +1086,10 @@ static int recalc_latency(struct impl *this, enum spa_direction direction, uint3 spa_pod_builder_init(&b, buffer, sizeof(buffer)); if ((res = spa_node_port_enum_params_sync(this->follower, direction, port_id, SPA_PARAM_Latency, - &index, NULL, ¶m, &b)) != 1) - return res; + &index, NULL, ¶m, &b)) != 1) { + param = NULL; + break; + } if ((res = spa_latency_parse(param, &latency)) < 0) return res; if (latency.direction == direction) @@ -1019,6 +1102,7 @@ static int recalc_latency(struct impl *this, enum spa_direction direction, uint3 return 0; } +*/ static void follower_port_info(void *data, enum spa_direction direction, uint32_t port_id, @@ -1033,6 +1117,11 @@ static void follower_port_info(void *data, return; } + this->follower_port_flags = info->flags & + (SPA_PORT_FLAG_LIVE | + SPA_PORT_FLAG_PHYSICAL | + SPA_PORT_FLAG_TERMINAL); + spa_log_debug(this->log, "%p: follower port info %s %p %08"PRIx64, this, this->direction == SPA_DIRECTION_INPUT ? "Input" : "Output", info, info->change_mask); @@ -1054,26 +1143,33 @@ static void follower_port_info(void *data, default: continue; } + if (!this->add_listener && this->follower_params_flags[idx] == info->params[i].flags) continue; + this->info.change_mask |= SPA_NODE_CHANGE_MASK_PARAMS; this->follower_params_flags[idx] = info->params[i].flags; this->params[idx].flags = (this->params[idx].flags & SPA_PARAM_INFO_SERIAL) | (info->params[i].flags & SPA_PARAM_INFO_READWRITE); + if (this->add_listener) + continue; + if (idx == IDX_Latency) { - res = recalc_latency(this, direction, port_id); + // res = recalc_latency(this, direction, port_id); + res = 0; spa_log_debug(this->log, "latency: %d (%s)", res, spa_strerror(res)); } - - this->info.change_mask |= SPA_NODE_CHANGE_MASK_PARAMS; - if (!this->add_listener) { - this->params[idx].user++; - spa_log_debug(this->log, "param %d changed", info->params[i].id); + if (idx == IDX_EnumFormat) { + spa_log_debug(this->log, "new formats"); + configure_format(this, 0, NULL); } + + this->params[idx].user++; + spa_log_debug(this->log, "param %d changed", info->params[i].id); } } emit_node_info(this, false); @@ -1124,6 +1220,11 @@ static int follower_ready(void *data, int status) spa_log_trace_fp(this->log, "%p: ready %d", this, status); + if (!this->ready) { + spa_log_info(this->log, "%p: ready stopped node", this); + return -EIO; + } + if (this->target != this->follower) { this->driver = true; @@ -1369,6 +1470,13 @@ static int impl_node_process(void *object) struct impl *this = object; int status = 0, fstatus, retry = 8; + if (!this->started) { + if (!this->warned) + spa_log_warn(this->log, "%p: scheduling stopped node", this); + this->warned = true; + return -EIO; + } + spa_log_trace_fp(this->log, "%p: process convert:%p driver:%d", this, this->convert, this->driver); @@ -1524,8 +1632,8 @@ impl_get_size(const struct spa_handle_factory *factory, { size_t size = 0; -#if 0 - size += spa_handle_factory_get_size(&spa_videoconvert_factory, params); +#if VIDEOCONVERTER + size += spa_handle_factory_get_size(&VIDEOCONVERTER_FACTORY, params); #endif size += sizeof(struct impl); @@ -1540,7 +1648,7 @@ impl_init(const struct spa_handle_factory *factory, uint32_t n_support) { struct impl *this; -#if 0 +#if VIDEOCONVERTER void *iface; #endif const char *str; @@ -1576,13 +1684,16 @@ impl_init(const struct spa_handle_factory *factory, SPA_VERSION_NODE, &impl_node, this); -#if 0 +#if VIDEOCONVERTER this->hnd_convert = SPA_PTROFF(this, sizeof(struct impl), struct spa_handle); - spa_handle_factory_init(&spa_videoconvert_factory, + spa_handle_factory_init(&VIDEOCONVERTER_FACTORY, this->hnd_convert, info, support, n_support); spa_handle_get_interface(this->hnd_convert, SPA_TYPE_INTERFACE_Node, &iface); + if (iface == NULL) + return -EINVAL; + this->convert = iface; #endif this->target = this->convert; @@ -1611,7 +1722,14 @@ impl_init(const struct spa_handle_factory *factory, spa_node_add_listener(this->convert, &this->convert_listener, &convert_node_events, this); - reconfigure_mode(this, true, this->direction, NULL); + if (this->direction == SPA_DIRECTION_INPUT) { + // MODE: CONVERT + configure_convert(this, SPA_PARAM_PORT_CONFIG_MODE_convert); + // reconfigure_mode(this, true, this->direction, NULL); + } else { + // MODE: PASSTHROUGH + reconfigure_mode(this, true, this->direction, NULL); + } link_io(this); diff --git a/spa/plugins/videoconvert/videoconvert-vulkan.c b/spa/plugins/videoconvert/videoconvert-vulkan.c new file mode 100644 index 000000000..f4e8bd052 --- /dev/null +++ b/spa/plugins/videoconvert/videoconvert-vulkan.c @@ -0,0 +1,1285 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2019 Wim Taymans */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pixel-formats.h" +#include "vulkan-blit-utils.h" + +#undef SPA_LOG_TOPIC_DEFAULT +#define SPA_LOG_TOPIC_DEFAULT log_topic +static struct spa_log_topic *log_topic = &SPA_LOG_TOPIC(0, "spa.videoconvert.vulkan"); + +#define NAME "videoconvert-vulkan" +#define MAX_PORTS 1 + + +struct buffer { + uint32_t id; +#define BUFFER_FLAG_OUT (1<<0) + uint32_t flags; + struct spa_buffer *outbuf; + struct spa_meta_header *h; + struct spa_list link; +}; + +struct port { + uint32_t direction; + uint32_t id; + + struct spa_io_buffers *io; + + uint64_t info_all; + struct spa_port_info info; + struct spa_param_info params[5]; + bool is_dsp; + bool have_format; + struct spa_video_info current_format; + + struct buffer buffers[MAX_BUFFERS]; + uint32_t n_buffers; + + struct spa_list empty; + struct spa_list ready; + uint32_t stream_id; +}; + +struct dir { + struct port ports[MAX_PORTS]; + uint32_t n_ports; + + enum spa_direction direction; + enum spa_param_port_config_mode mode; + + struct spa_video_info format; + unsigned int have_format:1; + unsigned int have_profile:1; +}; + +struct impl { + struct spa_handle handle; + struct spa_node node; + + struct spa_log *log; + + struct spa_io_position *position; + + uint64_t info_all; + struct spa_node_info info; +#define IDX_EnumPortConfig 0 +#define IDX_PortConfig 1 +#define IDX_PropInfo 2 +#define IDX_Props 3 +#define N_NODE_PARAMS 4 + struct spa_param_info params[N_NODE_PARAMS]; + + struct spa_hook_list hooks; + struct spa_callbacks callbacks; + + bool started; + bool setup; + + struct vulkan_blit_state state; + struct dir dir[2]; +}; + +#define CHECK_PORT(this,d,p) ((p) < this->dir[d].n_ports) +#define GET_PORT(this,d,p) (&this->dir[d].ports[p]) +#define GET_IN_PORT(this,p) GET_PORT(this,SPA_DIRECTION_INPUT,p) +#define GET_OUT_PORT(this,p) GET_PORT(this,SPA_DIRECTION_OUTPUT,p) + +#define PORT_IS_DSP(this,d,p) (GET_PORT(this,d,p)->is_dsp) +#define PORT_IS_CONTROL(this,d,p) (GET_PORT(this,d,p)->is_control) + +static const struct spa_dict_item node_info_items[] = { + { SPA_KEY_MEDIA_CLASS, "Video/Filter" }, +}; + +static void emit_node_info(struct impl *this, bool full) +{ + uint64_t old = full ? this->info.change_mask : 0; + if (full) + this->info.change_mask = this->info_all; + if (this->info.change_mask) { + this->info.props = &SPA_DICT_INIT_ARRAY(node_info_items); + spa_node_emit_info(&this->hooks, &this->info); + this->info.change_mask = old; + } +} + +static void emit_port_info(struct impl *this, struct port *port, bool full) +{ + uint64_t old = full ? port->info.change_mask : 0; + if (full) + port->info.change_mask = port->info_all; + if (port->info.change_mask) { + struct spa_dict_item items[1]; + + items[0] = SPA_DICT_ITEM_INIT(SPA_KEY_FORMAT_DSP, "32 bit float RGBA video"); + port->info.props = &SPA_DICT_INIT(items, 1); + spa_node_emit_port_info(&this->hooks, port->direction, port->id, &port->info); + port->info.change_mask = old; + } +} + +static int init_port(struct impl *this, enum spa_direction direction, uint32_t port_id, bool is_dsp) +{ + spa_assert(port_id < MAX_PORTS); + + struct port *port = GET_PORT(this, direction, port_id); + + port->stream_id = direction; + port->direction = direction; + port->id = port_id; + port->info_all = SPA_PORT_CHANGE_MASK_FLAGS | + SPA_PORT_CHANGE_MASK_PROPS | + SPA_PORT_CHANGE_MASK_PARAMS; + port->info = SPA_PORT_INFO_INIT(); + port->info.flags = SPA_PORT_FLAG_NO_REF; + port->params[0] = SPA_PARAM_INFO(SPA_PARAM_EnumFormat, SPA_PARAM_INFO_READ); + port->params[1] = SPA_PARAM_INFO(SPA_PARAM_Meta, SPA_PARAM_INFO_READ); + port->params[2] = SPA_PARAM_INFO(SPA_PARAM_IO, SPA_PARAM_INFO_READ); + port->params[3] = SPA_PARAM_INFO(SPA_PARAM_Format, SPA_PARAM_INFO_WRITE); + port->params[4] = SPA_PARAM_INFO(SPA_PARAM_Buffers, 0); + port->info.params = port->params; + port->info.n_params = 5; + port->have_format = false; + port->is_dsp = is_dsp; + spa_vulkan_blit_init_stream(&this->state, &this->state.streams[port->stream_id], + direction, NULL); + if (port->is_dsp) { + port->current_format.media_type = SPA_MEDIA_TYPE_audio; + port->current_format.media_subtype = SPA_MEDIA_SUBTYPE_dsp; + port->current_format.info.dsp.format = SPA_VIDEO_FORMAT_DSP_F32; + } + spa_list_init(&port->empty); + spa_list_init(&port->ready); + + spa_log_info(this->log, "%p: add port %d:%d is_dsp:%d", + this, direction, port_id, is_dsp); + emit_port_info(this, port, true); + + return 0; +} + +static int impl_node_enum_params(void *object, int seq, + uint32_t id, uint32_t start, uint32_t num, + const struct spa_pod *filter) +{ + struct impl *this = object; + struct spa_pod *param; + struct spa_pod_builder b = { 0 }; + uint8_t buffer[1024]; + struct spa_result_node_params result; + uint32_t count = 0; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(num != 0, -EINVAL); + + result.id = id; + result.next = start; + next: + result.index = result.next++; + + spa_pod_builder_init(&b, buffer, sizeof(buffer)); + + switch (id) { + case SPA_PARAM_EnumPortConfig: + { + struct dir *dir; + switch (result.index) { + case 0: + dir = &this->dir[SPA_DIRECTION_INPUT];; + break; + case 1: + dir = &this->dir[SPA_DIRECTION_OUTPUT];; + break; + default: + return 0; + } + param = spa_pod_builder_add_object(&b, + SPA_TYPE_OBJECT_ParamPortConfig, id, + SPA_PARAM_PORT_CONFIG_direction, SPA_POD_Id(dir->direction), + SPA_PARAM_PORT_CONFIG_mode, SPA_POD_CHOICE_ENUM_Id(4, + SPA_PARAM_PORT_CONFIG_MODE_none, + SPA_PARAM_PORT_CONFIG_MODE_none, + SPA_PARAM_PORT_CONFIG_MODE_dsp, + SPA_PARAM_PORT_CONFIG_MODE_convert), + SPA_PARAM_PORT_CONFIG_monitor, SPA_POD_CHOICE_Bool(false), + SPA_PARAM_PORT_CONFIG_control, SPA_POD_CHOICE_Bool(false)); + break; + } + case SPA_PARAM_PortConfig: + { + struct dir *dir; + struct spa_pod_frame f[1]; + + switch (result.index) { + case 0: + dir = &this->dir[SPA_DIRECTION_INPUT];; + break; + case 1: + dir = &this->dir[SPA_DIRECTION_OUTPUT];; + break; + default: + return 0; + } + spa_pod_builder_push_object(&b, &f[0], SPA_TYPE_OBJECT_ParamPortConfig, id); + spa_pod_builder_add(&b, + SPA_PARAM_PORT_CONFIG_direction, SPA_POD_Id(dir->direction), + SPA_PARAM_PORT_CONFIG_mode, SPA_POD_Id(dir->mode), + SPA_PARAM_PORT_CONFIG_monitor, SPA_POD_Bool(false), + SPA_PARAM_PORT_CONFIG_control, SPA_POD_Bool(false), + 0); + + if (dir->have_format) { + spa_pod_builder_prop(&b, SPA_PARAM_PORT_CONFIG_format, 0); + // TODO + if (false && dir->mode == SPA_PARAM_PORT_CONFIG_MODE_dsp) { + spa_format_video_dsp_build(&b, SPA_PARAM_PORT_CONFIG_format, + &dir->format.info.dsp); + } else { + spa_format_video_raw_build(&b, SPA_PARAM_PORT_CONFIG_format, + &dir->format.info.raw); + } + } + param = spa_pod_builder_pop(&b, &f[0]); + break; + } + + default: + return 0; + } + + if (spa_pod_filter(&b, &result.param, param, filter) < 0) + goto next; + + spa_node_emit_result(&this->hooks, seq, 0, SPA_RESULT_TYPE_NODE_PARAMS, &result); + + if (++count != num) + goto next; + + return 0; +} + +static int impl_node_set_io(void *object, uint32_t id, void *data, size_t size) +{ + struct impl *this = object; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + spa_log_debug(this->log, "%p: io %d %p/%zd", this, id, data, size); + + switch (id) { + case SPA_IO_Position: + if (size > 0 && size < sizeof(struct spa_io_position)) + return -EINVAL; + this->position = data; + break; + default: + return -ENOENT; + } + return 0; +} + +static int reconfigure_mode(struct impl *this, enum spa_param_port_config_mode mode, + enum spa_direction direction, struct spa_video_info *info) +{ + struct dir *dir; + uint32_t i; + + dir = &this->dir[direction]; + + if (dir->have_profile && dir->mode == mode && + (info == NULL || memcmp(&dir->format, info, sizeof(*info)) == 0)) + return 0; + + spa_log_info(this->log, "%p: port config direction:%d mode:%d %d %p", this, + direction, mode, dir->n_ports, info); + + for (i = 0; i < dir->n_ports; i++) { + spa_node_emit_port_info(&this->hooks, direction, i, NULL); + } + + this->setup = false; + dir->have_profile = true; + dir->mode = mode; + + switch (mode) { + case SPA_PARAM_PORT_CONFIG_MODE_dsp: + { + if (info) { + dir->n_ports = 1; + // WHY?? + dir->format = *info; + dir->format.info.raw.format = SPA_VIDEO_FORMAT_DSP_F32; + dir->have_format = true; + } else { + dir->n_ports = 0; + } + + for (i = 0; i < dir->n_ports; i++) { + init_port(this, direction, i, true); + } + break; + } + case SPA_PARAM_PORT_CONFIG_MODE_convert: + { + dir->n_ports = 1; + dir->have_format = false; + init_port(this, direction, 0, false); + break; + } + case SPA_PARAM_PORT_CONFIG_MODE_none: + break; + default: + return -ENOTSUP; + } + + this->info.change_mask |= SPA_NODE_CHANGE_MASK_FLAGS | SPA_NODE_CHANGE_MASK_PARAMS; + this->info.flags &= ~SPA_NODE_FLAG_NEED_CONFIGURE; + this->params[IDX_Props].user++; + this->params[IDX_PortConfig].user++; + return 0; +} + +static int impl_node_set_param(void *object, uint32_t id, uint32_t flags, + const struct spa_pod *param) +{ + struct impl *this = object; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + if (param == NULL) + return 0; + + switch (id) { + case SPA_PARAM_PortConfig: + { + struct spa_video_info info = { 0, }, *infop = NULL; + struct spa_pod *format = NULL; + enum spa_direction direction; + enum spa_param_port_config_mode mode; + bool monitor = false, control = false; + int res; + + if (spa_pod_parse_object(param, + SPA_TYPE_OBJECT_ParamPortConfig, NULL, + SPA_PARAM_PORT_CONFIG_direction, SPA_POD_Id(&direction), + SPA_PARAM_PORT_CONFIG_mode, SPA_POD_Id(&mode), + SPA_PARAM_PORT_CONFIG_monitor, SPA_POD_OPT_Bool(&monitor), + SPA_PARAM_PORT_CONFIG_control, SPA_POD_OPT_Bool(&control), + SPA_PARAM_PORT_CONFIG_format, SPA_POD_OPT_Pod(&format)) < 0) + return -EINVAL; + + if (format) { + if (!spa_pod_is_object_type(format, SPA_TYPE_OBJECT_Format)) + return -EINVAL; + + if ((res = spa_format_parse(format, &info.media_type, &info.media_subtype)) < 0) + return res; + + if (info.media_type != SPA_MEDIA_TYPE_video || + info.media_subtype != SPA_MEDIA_SUBTYPE_raw) + return -EINVAL; + + if (spa_format_video_raw_parse(format, &info.info.raw) < 0) + return -EINVAL; + + if (info.info.raw.format == 0) + return -EINVAL; + + infop = &info; + } + + if ((res = reconfigure_mode(this, mode, direction, infop)) < 0) + return res; + + emit_node_info(this, false); + break; + } + + default: + return -ENOENT; + } + return 0; +} + +static int setup_convert(struct impl *this) +{ + struct dir *in, *out; + uint32_t src_format, dst_format; + + in = &this->dir[SPA_DIRECTION_INPUT]; + out = &this->dir[SPA_DIRECTION_OUTPUT]; + + spa_log_debug(this->log, "%p: setup:%d in_format:%d out_format:%d", this, + this->setup, in->have_format, out->have_format); + + if (this->setup) + return 0; + + if (!in->have_format || !out->have_format) + return -EINVAL; + + src_format = in->format.media_subtype == SPA_MEDIA_SUBTYPE_dsp + ? in->format.info.dsp.format + : in->format.info.raw.format; + dst_format = out->format.media_subtype == SPA_MEDIA_SUBTYPE_dsp + ? out->format.info.dsp.format + : out->format.info.raw.format; + spa_log_info(this->log, "%p: %s->%s", this, + spa_debug_type_find_name(spa_type_video_format, src_format), + spa_debug_type_find_name(spa_type_video_format, dst_format)); + + this->setup = true; + + emit_node_info(this, false); + return 0; +} +static inline void reuse_buffer(struct impl *this, struct port *port, uint32_t id) +{ + struct buffer *b = &port->buffers[id]; + + if (SPA_FLAG_IS_SET(b->flags, BUFFER_FLAG_OUT)) { + spa_log_debug(this->log, NAME " %p: reuse buffer %d", this, id); + + SPA_FLAG_CLEAR(b->flags, BUFFER_FLAG_OUT); + spa_list_append(&port->empty, &b->link); + } +} + +static int impl_node_send_command(void *object, const struct spa_command *command) +{ + struct impl *this = object; + int res; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(command != NULL, -EINVAL); + + switch (SPA_NODE_COMMAND_ID(command)) { + case SPA_NODE_COMMAND_Start: + if (this->started) + return 0; + if ((res = setup_convert(this)) < 0) + return res; + this->started = true; + spa_vulkan_blit_start(&this->state); + break; + case SPA_NODE_COMMAND_Suspend: + this->setup = false; + SPA_FALLTHROUGH; + case SPA_NODE_COMMAND_Pause: + this->started = false; + spa_vulkan_blit_stop(&this->state); + break; + case SPA_NODE_COMMAND_Flush: + // reset_node(this); + break; + default: + return -ENOTSUP; + } + return 0; +} + +static int +impl_node_add_listener(void *object, + struct spa_hook *listener, + const struct spa_node_events *events, + void *data) +{ + struct impl *this = object; + struct spa_hook_list save; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + spa_log_trace(this->log, "%p: add listener %p", this, listener); + spa_hook_list_isolate(&this->hooks, &save, listener, events, data); + + emit_node_info(this, true); + emit_port_info(this, &this->dir[0].ports[0], true); + emit_port_info(this, &this->dir[1].ports[0], true); + + spa_hook_list_join(&this->hooks, &save); + + return 0; +} + +static int +impl_node_set_callbacks(void *object, + const struct spa_node_callbacks *callbacks, + void *data) +{ + struct impl *this = object; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + this->callbacks = SPA_CALLBACKS_INIT(callbacks, data); + + return 0; +} + +static int impl_node_add_port(void *object, enum spa_direction direction, uint32_t port_id, + const struct spa_dict *props) +{ + return -ENOTSUP; +} + +static int +impl_node_remove_port(void *object, enum spa_direction direction, uint32_t port_id) +{ + return -ENOTSUP; +} + +static bool port_has_fixated_format(struct port *p) +{ + if (!p->have_format) + return false; + switch (p->current_format.media_subtype) { + case SPA_MEDIA_SUBTYPE_dsp: + return p->current_format.info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER + && p->current_format.info.dsp.flags ^ SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED; + case SPA_MEDIA_SUBTYPE_raw: + return p->current_format.info.raw.flags & SPA_VIDEO_FLAG_MODIFIER + && p->current_format.info.raw.flags ^ SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED; + } + return false; +} + +static int port_enum_formats(void *object, + enum spa_direction direction, uint32_t port_id, + uint32_t index, + const struct spa_pod *filter, + struct spa_pod **param, + struct spa_pod_builder *builder) +{ + struct impl *this = object; + + struct dir *dir = &this->dir[direction]; + struct port *port = GET_PORT(this, direction, port_id); + if (port_has_fixated_format(port)) { + if (index == 0) { + if (port->current_format.media_subtype == SPA_MEDIA_SUBTYPE_dsp) { + spa_log_info(this->log, NAME": enum_formats fixated format idx: %d, format %d, has_modifier 1", + index, port->current_format.info.dsp.format); + *param = spa_format_video_dsp_build(builder, SPA_PARAM_EnumFormat, &port->current_format.info.dsp); + } else { + spa_log_info(this->log, NAME": enum_formats fixated format idx: %d, format %d, has_modifier 1", + index, port->current_format.info.raw.format); + *param = spa_format_video_raw_build(builder, SPA_PARAM_EnumFormat, &port->current_format.info.raw); + } + return 1; + } + switch(dir->mode) { + case SPA_PARAM_PORT_CONFIG_MODE_dsp: + return spa_vulkan_blit_enumerate_dsp_formats(&this->state, index-1, spa_vulkan_blit_get_buffer_caps(&this->state, direction), param, builder); + case SPA_PARAM_PORT_CONFIG_MODE_convert: + default: + return spa_vulkan_blit_enumerate_raw_formats(&this->state, index-1, spa_vulkan_blit_get_buffer_caps(&this->state, direction), param, builder); + } + } else { + switch(dir->mode) { + case SPA_PARAM_PORT_CONFIG_MODE_dsp: + return spa_vulkan_blit_enumerate_dsp_formats(&this->state, index, spa_vulkan_blit_get_buffer_caps(&this->state, direction), param, builder); + case SPA_PARAM_PORT_CONFIG_MODE_convert: + default: + return spa_vulkan_blit_enumerate_raw_formats(&this->state, index, spa_vulkan_blit_get_buffer_caps(&this->state, direction), param, builder); + } + } +} + +static int port_get_buffer_props(struct impl *this, struct port *port, + uint32_t *blocks, uint32_t *size, uint32_t *stride, bool *is_dmabuf) +{ + if (port->current_format.media_subtype == SPA_MEDIA_SUBTYPE_dsp) { + if (this->position == NULL) + return -EIO; + + spa_log_debug(this->log, NAME" %p: %dx%d stride %d", this, + this->position->video.size.width, + this->position->video.size.height, + this->position->video.stride); + + if (port->current_format.info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER) { + *is_dmabuf = true; + + struct vulkan_modifier_info *mod_info = spa_vulkan_blit_get_modifier_info(&this->state, + &port->current_format); + *blocks = mod_info->props.drmFormatModifierPlaneCount; + } else { + *is_dmabuf = false; + *blocks = 1; + *size = this->position->video.stride * this->position->video.size.height; + *stride = this->position->video.stride; + } + return 0; + } else if (port->current_format.media_subtype == SPA_MEDIA_SUBTYPE_raw) { + spa_log_debug(this->log, NAME" %p: %dx%d", this, + port->current_format.info.raw.size.width, + port->current_format.info.raw.size.height); + + if (port->current_format.info.raw.flags & SPA_VIDEO_FLAG_MODIFIER) { + *is_dmabuf = true; + + struct vulkan_modifier_info *mod_info = spa_vulkan_blit_get_modifier_info(&this->state, + &port->current_format); + *blocks = mod_info->props.drmFormatModifierPlaneCount; + } else { + struct pixel_format_info pInfo = {0}; + if (!get_pixel_format_info(port->current_format.info.raw.format, &pInfo)) + return -EINVAL; + uint32_t buffer_stride = pInfo.bpp * port->current_format.info.raw.size.width; + *is_dmabuf = false; + *blocks = 1; + *size = buffer_stride * port->current_format.info.raw.size.height; + *stride = buffer_stride; + } + return 0; + } else { + return -EINVAL; + } +} + +static int +impl_node_port_enum_params(void *object, int seq, + enum spa_direction direction, uint32_t port_id, + uint32_t id, uint32_t start, uint32_t num, + const struct spa_pod *filter) +{ + struct impl *this = object; + struct port *port; + struct spa_pod *param; + struct spa_pod_builder b = { 0 }; + uint8_t buffer[4096]; + struct spa_result_node_params result; + uint32_t count = 0; + int res; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(num != 0, -EINVAL); + + spa_log_debug(this->log, "%p: enum params port %d.%d %d %u", + this, direction, port_id, seq, id); + + spa_return_val_if_fail(CHECK_PORT(this, direction, port_id), -EINVAL); + + port = GET_PORT(this, direction, port_id); + + result.id = id; + result.next = start; + next: + result.index = result.next++; + + spa_pod_builder_init(&b, buffer, sizeof(buffer)); + + switch (id) { + case SPA_PARAM_EnumFormat: + if ((res = port_enum_formats(this, direction, port_id, + result.index, filter, ¶m, &b)) <= 0) + return res; + break; + case SPA_PARAM_Format: + if (!port->have_format) + return -EIO; + if (result.index > 0) + return 0; + + if (port->current_format.media_subtype == SPA_MEDIA_SUBTYPE_dsp) { + param = spa_format_video_dsp_build(&b, id, &port->current_format.info.dsp); + } else { + param = spa_format_video_raw_build(&b, id, &port->current_format.info.raw); + } + break; + case SPA_PARAM_Buffers: + { + if (!port->have_format) + return -EIO; + if (result.index > 0) + return 0; + + int ret; + uint32_t blocks, size, stride; + bool is_dmabuf; + + if ((ret = port_get_buffer_props(this, port, &blocks, &size, &stride, &is_dmabuf)) < 0) + return ret; + + if (is_dmabuf) { + param = spa_pod_builder_add_object(&b, + SPA_TYPE_OBJECT_ParamBuffers, id, + SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(2, 1, MAX_BUFFERS), + SPA_PARAM_BUFFERS_blocks, SPA_POD_Int(blocks), + SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int(1<hooks, seq, 0, SPA_RESULT_TYPE_NODE_PARAMS, &result); + + if (++count != num) + goto next; + + return 0; +} + +static int clear_buffers(struct impl *this, struct port *port) +{ + if (port->n_buffers > 0) { + spa_log_debug(this->log, NAME " %p: clear buffers %p", this, port); + spa_vulkan_blit_stop(&this->state); + spa_vulkan_blit_use_buffers(&this->state, &this->state.streams[port->stream_id], 0, &port->current_format, 0, NULL); + port->n_buffers = 0; + spa_list_init(&port->empty); + spa_list_init(&port->ready); + this->started = false; + } + return 0; +} + +static int port_set_dsp_format(struct impl *this, struct port *port, + uint32_t flags, struct spa_video_info *info, + bool *has_modifier, bool *modifier_fixed, + const struct spa_pod *format) +{ + if (spa_format_video_dsp_parse(format, &info->info.dsp) < 0) + return -EINVAL; + + if (info->info.dsp.format != SPA_VIDEO_FORMAT_DSP_F32) + return -EINVAL; + + this->state.streams[port->stream_id].dim.width = this->position->video.size.width; + this->state.streams[port->stream_id].dim.height = this->position->video.size.height; + this->state.streams[port->stream_id].bpp = 16; + *has_modifier = SPA_FLAG_IS_SET(info->info.dsp.flags, SPA_VIDEO_FLAG_MODIFIER); + + // fixate modifier + if (port->direction == SPA_DIRECTION_OUTPUT + && info->info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER + && info->info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED) { + const struct spa_pod_prop *mod_prop; + if ((mod_prop = spa_pod_find_prop(format, NULL, SPA_FORMAT_VIDEO_modifier)) == NULL) + return -EINVAL; + + const struct spa_pod *mod_pod = &mod_prop->value; + uint32_t modifierCount = SPA_POD_CHOICE_N_VALUES(mod_pod); + uint64_t *modifiers = SPA_POD_CHOICE_VALUES(mod_pod); + if (modifierCount <= 1) + return -EINVAL; + // SPA_POD_CHOICE carries the "preferred" value at position 0 + modifierCount -= 1; + modifiers++; + uint64_t fixed_modifier; + if (spa_vulkan_blit_fixate_modifier(&this->state, &this->state.streams[port->stream_id], info, modifierCount, modifiers, &fixed_modifier) != 0) + return -EINVAL; + + spa_log_info(this->log, NAME ": modifier fixated %"PRIu64, fixed_modifier); + + info->info.dsp.modifier = fixed_modifier; + info->info.dsp.flags &= ~SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED; + *modifier_fixed = true; + } + + return 0; +} + +static int port_set_raw_format(struct impl *this, struct port *port, + uint32_t flags, struct spa_video_info *info, + bool *has_modifier, bool *modifier_fixed, + const struct spa_pod *format) +{ + if (spa_format_video_raw_parse(format, &info->info.raw) < 0) + return -EINVAL; + + struct pixel_format_info pInfo; + if (!get_pixel_format_info(info->info.raw.format, &pInfo)) + return -EINVAL; + this->state.streams[port->stream_id].dim = info->info.raw.size; + this->state.streams[port->stream_id].bpp = pInfo.bpp; + *has_modifier = SPA_FLAG_IS_SET(info->info.raw.flags, SPA_VIDEO_FLAG_MODIFIER); + + // fixate modifier + if (port->direction == SPA_DIRECTION_OUTPUT + && info->info.raw.flags & SPA_VIDEO_FLAG_MODIFIER + && info->info.raw.flags & SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED) { + const struct spa_pod_prop *mod_prop; + if ((mod_prop = spa_pod_find_prop(format, NULL, SPA_FORMAT_VIDEO_modifier)) == NULL) + return -EINVAL; + + const struct spa_pod *mod_pod = &mod_prop->value; + uint32_t modifierCount = SPA_POD_CHOICE_N_VALUES(mod_pod); + uint64_t *modifiers = SPA_POD_CHOICE_VALUES(mod_pod); + if (modifierCount <= 1) + return -EINVAL; + // SPA_POD_CHOICE carries the "preferred" value at position 0 + modifierCount -= 1; + modifiers++; + uint64_t fixed_modifier; + if (spa_vulkan_blit_fixate_modifier(&this->state, &this->state.streams[port->stream_id], info, modifierCount, modifiers, &fixed_modifier) != 0) + return -EINVAL; + + spa_log_info(this->log, NAME ": modifier fixated %"PRIu64, fixed_modifier); + + info->info.raw.modifier = fixed_modifier; + info->info.raw.flags &= ~SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED; + *modifier_fixed = true; + } + + return 0; +} + +static int port_set_format(struct impl *this, + enum spa_direction direction, + uint32_t port_id, + uint32_t flags, + const struct spa_pod *format) +{ + struct port *port; + int res; + + port = GET_PORT(this, direction, port_id); + + spa_log_debug(this->log, "%p: set format", this); + + if (format == NULL) { + port->have_format = false; + clear_buffers(this, port); + } else { + struct spa_video_info info = { 0 }; + + if ((res = spa_format_parse(format, &info.media_type, &info.media_subtype)) < 0) { + spa_log_error(this->log, "can't parse format %s", spa_strerror(res)); + return res; + } + + if (info.media_type != SPA_MEDIA_TYPE_video) + return -EINVAL; + + bool has_modifier = false; + bool modifier_fixed = false; + if (info.media_subtype == SPA_MEDIA_SUBTYPE_dsp) { + if ((res = port_set_dsp_format(this, port, flags, &info, &has_modifier, &modifier_fixed, format)) < 0) + return res; + } else if (info.media_subtype == SPA_MEDIA_SUBTYPE_raw) { + if ((res = port_set_raw_format(this, port, flags, &info, &has_modifier, &modifier_fixed, format)) < 0) + return res; + } else { + return -EINVAL; + } + + if (has_modifier) { + SPA_FLAG_SET(port->info.flags, SPA_PORT_FLAG_CAN_ALLOC_BUFFERS); + } else { + SPA_FLAG_CLEAR(port->info.flags, SPA_PORT_FLAG_CAN_ALLOC_BUFFERS); + } + port->info.change_mask |= SPA_PORT_CHANGE_MASK_FLAGS; + + this->dir[direction].format = info; + this->dir[direction].have_format = true; + port->current_format = info; + port->have_format = true; + + if (modifier_fixed) { + port->info.change_mask |= SPA_PORT_CHANGE_MASK_PARAMS; + port->params[0].flags ^= SPA_PARAM_INFO_SERIAL; + emit_port_info(this, port, false); + return 0; + } + } + + port->info.change_mask |= SPA_PORT_CHANGE_MASK_PARAMS; + if (port->have_format) { + port->params[3] = SPA_PARAM_INFO(SPA_PARAM_Format, SPA_PARAM_INFO_READWRITE); + port->params[4] = SPA_PARAM_INFO(SPA_PARAM_Buffers, SPA_PARAM_INFO_READ); + } else { + port->params[3] = SPA_PARAM_INFO(SPA_PARAM_Format, SPA_PARAM_INFO_WRITE); + port->params[4] = SPA_PARAM_INFO(SPA_PARAM_Buffers, 0); + } + emit_port_info(this, port, false); + + return 0; +} + +static int +impl_node_port_set_param(void *object, + enum spa_direction direction, uint32_t port_id, + uint32_t id, uint32_t flags, + const struct spa_pod *param) +{ + struct impl *this = object; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + spa_log_debug(this->log, "%p: set param port %d.%d %u", + this, direction, port_id, id); + + spa_return_val_if_fail(CHECK_PORT(this, direction, port_id), -EINVAL); + + switch (id) { + case SPA_PARAM_Format: + return port_set_format(this, direction, port_id, flags, param); + default: + return -ENOENT; + } +} + +static int +impl_node_port_use_buffers(void *object, + enum spa_direction direction, + uint32_t port_id, + uint32_t flags, + struct spa_buffer **buffers, + uint32_t n_buffers) +{ + struct impl *this = object; + struct port *port; + uint32_t i; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + spa_return_val_if_fail(CHECK_PORT(this, direction, port_id), -EINVAL); + + port = GET_PORT(this, direction, port_id); + + spa_log_debug(this->log, "%p: use buffers %d on port %d:%d", + this, n_buffers, direction, port_id); + + clear_buffers(this, port); + + if (n_buffers > 0 && !port->have_format) + return -EIO; + if (n_buffers > MAX_BUFFERS) + return -ENOSPC; + + for (i = 0; i < n_buffers; i++) { + struct buffer *b; + + b = &port->buffers[i]; + b->id = i; + b->flags = 0; + b->outbuf = buffers[i]; + b->h = spa_buffer_find_meta_data(buffers[i], SPA_META_Header, sizeof(*b->h)); + + spa_log_info(this->log, "%p: %d:%d add buffer %p", port, direction, port_id, b); + spa_list_append(&port->empty, &b->link); + } + spa_vulkan_blit_use_buffers(&this->state, &this->state.streams[port->stream_id], flags, &port->current_format, n_buffers, buffers); + port->n_buffers = n_buffers; + + return 0; +} + +static int +impl_node_port_set_io(void *object, + enum spa_direction direction, uint32_t port_id, + uint32_t id, void *data, size_t size) +{ + struct impl *this = object; + struct port *port; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + spa_log_debug(this->log, "%p: set io %d on port %d:%d %p", + this, id, direction, port_id, data); + + spa_return_val_if_fail(CHECK_PORT(this, direction, port_id), -EINVAL); + + port = GET_PORT(this, direction, port_id); + + switch (id) { + case SPA_IO_Buffers: + port->io = data; + break; + default: + return -ENOENT; + } + return 0; +} + +static int impl_node_port_reuse_buffer(void *object, uint32_t port_id, uint32_t buffer_id) +{ + struct impl *this = object; + struct port *port; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(CHECK_PORT(this, SPA_DIRECTION_OUTPUT, port_id), -EINVAL); + + port = GET_OUT_PORT(this, port_id); + reuse_buffer(this, port, buffer_id); + + return 0; +} + +static int impl_node_process(void *object) +{ + struct impl *this = object; + struct port *inport, *outport; + struct spa_io_buffers *inio, *outio; + struct buffer *b; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + inport = GET_IN_PORT(this, 0); + if ((inio = inport->io) == NULL) + return -EIO; + + if (inio->status != SPA_STATUS_HAVE_DATA) + return inio->status; + + if (inio->buffer_id >= inport->n_buffers) { + inio->status = -EINVAL; + return -EINVAL; + } + + outport = GET_OUT_PORT(this, 0); + if ((outio = outport->io) == NULL) + return -EIO; + + if (outio->status == SPA_STATUS_HAVE_DATA) + return SPA_STATUS_HAVE_DATA; + + if (outio->buffer_id < outport->n_buffers) { + reuse_buffer(this, outport, outio->buffer_id); + outio->buffer_id = SPA_ID_INVALID; + } + + if (spa_list_is_empty(&outport->empty)) { + spa_log_debug(this->log, NAME " %p: out of buffers", this); + return -EPIPE; + } + b = &inport->buffers[inio->buffer_id]; + this->state.streams[inport->stream_id].pending_buffer_id = b->id; + inio->status = SPA_STATUS_NEED_DATA; + + b = spa_list_first(&outport->empty, struct buffer, link); + spa_list_remove(&b->link); + SPA_FLAG_SET(b->flags, BUFFER_FLAG_OUT); + this->state.streams[outport->stream_id].pending_buffer_id = b->id; + + spa_log_debug(this->log, "filter into %d", b->id); + + spa_vulkan_blit_process(&this->state); + + b->outbuf->datas[0].chunk->offset = 0; + b->outbuf->datas[0].chunk->size = b->outbuf->datas[0].maxsize; + if (outport->current_format.media_subtype == SPA_MEDIA_SUBTYPE_raw) { + b->outbuf->datas[0].chunk->stride = + this->state.streams[outport->stream_id].bpp * outport->current_format.info.raw.size.width; + } else { + b->outbuf->datas[0].chunk->stride = this->position->video.stride; + } + + outio->buffer_id = b->id; + outio->status = SPA_STATUS_HAVE_DATA; + + return SPA_STATUS_NEED_DATA | SPA_STATUS_HAVE_DATA; +} + +static const struct spa_node_methods impl_node = { + SPA_VERSION_NODE_METHODS, + .add_listener = impl_node_add_listener, + .set_callbacks = impl_node_set_callbacks, + .enum_params = impl_node_enum_params, + .set_param = impl_node_set_param, + .set_io = impl_node_set_io, + .send_command = impl_node_send_command, + .add_port = impl_node_add_port, + .remove_port = impl_node_remove_port, + .port_enum_params = impl_node_port_enum_params, + .port_set_param = impl_node_port_set_param, + .port_use_buffers = impl_node_port_use_buffers, + .port_set_io = impl_node_port_set_io, + .port_reuse_buffer = impl_node_port_reuse_buffer, + .process = impl_node_process, +}; + +static int impl_get_interface(struct spa_handle *handle, const char *type, void **interface) +{ + struct impl *this; + + spa_return_val_if_fail(handle != NULL, -EINVAL); + spa_return_val_if_fail(interface != NULL, -EINVAL); + + this = (struct impl *) handle; + + if (spa_streq(type, SPA_TYPE_INTERFACE_Node)) + *interface = &this->node; + else + return -ENOENT; + + return 0; +} + +static int impl_clear(struct spa_handle *handle) +{ + struct impl *this; + + spa_return_val_if_fail(handle != NULL, -EINVAL); + + this = (struct impl *) handle; + + spa_vulkan_blit_unprepare(&this->state); + spa_vulkan_blit_deinit(&this->state); + return 0; +} + +static size_t +impl_get_size(const struct spa_handle_factory *factory, + const struct spa_dict *params) +{ + return sizeof(struct impl); +} + +static int +impl_init(const struct spa_handle_factory *factory, + struct spa_handle *handle, + const struct spa_dict *info, + const struct spa_support *support, + uint32_t n_support) +{ + struct impl *this; + struct dir *dir; + + spa_return_val_if_fail(factory != NULL, -EINVAL); + spa_return_val_if_fail(handle != NULL, -EINVAL); + + handle->get_interface = impl_get_interface; + handle->clear = impl_clear; + + this = (struct impl *) handle; + + this->log = spa_support_find(support, n_support, SPA_TYPE_INTERFACE_Log); + spa_log_topic_init(this->log, log_topic); + this->state.log = this->log; + + // props_reset(&this->props); + + this->node.iface = SPA_INTERFACE_INIT( + SPA_TYPE_INTERFACE_Node, + SPA_VERSION_NODE, + &impl_node, this); + spa_hook_list_init(&this->hooks); + + this->info_all = SPA_NODE_CHANGE_MASK_FLAGS | + SPA_NODE_CHANGE_MASK_PROPS | + SPA_NODE_CHANGE_MASK_PARAMS; + this->info = SPA_NODE_INFO_INIT(); + this->info.max_output_ports = 1; + this->info.max_input_ports = 1; + this->info.flags = SPA_NODE_FLAG_RT | + SPA_NODE_FLAG_IN_PORT_CONFIG | + SPA_NODE_FLAG_OUT_PORT_CONFIG | + SPA_NODE_FLAG_NEED_CONFIGURE; + this->params[IDX_EnumPortConfig] = SPA_PARAM_INFO(SPA_PARAM_EnumPortConfig, SPA_PARAM_INFO_READ); + this->params[IDX_PortConfig] = SPA_PARAM_INFO(SPA_PARAM_PortConfig, SPA_PARAM_INFO_READWRITE); + this->params[IDX_PropInfo] = SPA_PARAM_INFO(SPA_PARAM_PropInfo, SPA_PARAM_INFO_READ); + this->params[IDX_Props] = SPA_PARAM_INFO(SPA_PARAM_Props, SPA_PARAM_INFO_READWRITE); + this->info.params = this->params; + this->info.n_params = N_NODE_PARAMS; + + dir = &this->dir[SPA_DIRECTION_INPUT]; + dir->direction = SPA_DIRECTION_INPUT; + // init_port(this, SPA_DIRECTION_INPUT, 0, false); + + dir = &this->dir[SPA_DIRECTION_OUTPUT]; + dir->direction = SPA_DIRECTION_OUTPUT; + // init_port(this, SPA_DIRECTION_OUTPUT, 0, false); + + this->state.n_streams = 2; + spa_vulkan_blit_init(&this->state); + spa_vulkan_blit_prepare(&this->state); + + reconfigure_mode(this, SPA_PARAM_PORT_CONFIG_MODE_convert, SPA_DIRECTION_INPUT, NULL); + reconfigure_mode(this, SPA_PARAM_PORT_CONFIG_MODE_convert, SPA_DIRECTION_OUTPUT, NULL); + return 0; +} + +static const struct spa_interface_info impl_interfaces[] = { + {SPA_TYPE_INTERFACE_Node,}, +}; + +static int +impl_enum_interface_info(const struct spa_handle_factory *factory, + const struct spa_interface_info **info, + uint32_t *index) +{ + spa_return_val_if_fail(factory != NULL, -EINVAL); + spa_return_val_if_fail(info != NULL, -EINVAL); + spa_return_val_if_fail(index != NULL, -EINVAL); + + switch (*index) { + case 0: + *info = &impl_interfaces[*index]; + break; + default: + return 0; + } + (*index)++; + return 1; +} + +static const struct spa_dict_item info_items[] = { + { SPA_KEY_FACTORY_AUTHOR, "Columbarius " }, + { SPA_KEY_FACTORY_DESCRIPTION, "Convert video frames using a vulkan blit" }, +}; + +static const struct spa_dict info = SPA_DICT_INIT_ARRAY(info_items); + +const struct spa_handle_factory spa_videoconvert_vulkan_factory = { + SPA_VERSION_HANDLE_FACTORY, + SPA_NAME_VIDEO_CONVERT_VULKAN, + &info, + impl_get_size, + impl_init, + impl_enum_interface_info, +}; diff --git a/spa/plugins/videoconvert/vulkan-blit-utils.c b/spa/plugins/videoconvert/vulkan-blit-utils.c new file mode 100644 index 000000000..4cda26458 --- /dev/null +++ b/spa/plugins/videoconvert/vulkan-blit-utils.c @@ -0,0 +1,639 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2019 Wim Taymans */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#if !defined(__FreeBSD__) && !defined(__MidnightBSD__) +#include +#endif +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "vulkan-blit-utils.h" +#include "vulkan-utils.h" +#include "utils.h" + +#define VULKAN_INSTANCE_FUNCTION(name) \ + PFN_##name name = (PFN_##name)vkGetInstanceProcAddr(s->base.instance, #name) + +static int updateBuffers(struct vulkan_blit_state *s) +{ + uint32_t i; + + for (i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + + if (p->current_buffer_id == p->pending_buffer_id || + p->pending_buffer_id == SPA_ID_INVALID) + continue; + + p->current_buffer_id = p->pending_buffer_id; + p->busy_buffer_id = p->current_buffer_id; + p->pending_buffer_id = SPA_ID_INVALID; + + } + + return 0; +} + +static int runImportSHMBuffers(struct vulkan_blit_state *s) { + struct vulkan_stream *p = &s->streams[SPA_DIRECTION_INPUT]; + + if (p->spa_buffers[p->current_buffer_id]->datas[0].type == SPA_DATA_MemPtr) { + struct vulkan_buffer *vk_buf = &p->buffers[p->current_buffer_id]; + struct spa_buffer *spa_buf = p->spa_buffers[p->current_buffer_id]; + VkBufferImageCopy copy; + struct vulkan_write_pixels_info writeInfo = { + .data = spa_buf->datas[0].data, + .offset = 0, + .stride = p->bpp * p->dim.width, + .bytes_per_pixel = p->bpp, + .size.width = p->dim.width, + .size.height = p->dim.height, + .copies = ©, + }; + CHECK(vulkan_write_pixels(&s->base, &writeInfo, &s->staging_buffer)); + + vkCmdCopyBufferToImage(s->commandBuffer, s->staging_buffer.buffer, vk_buf->image, + VK_IMAGE_LAYOUT_GENERAL, 1, ©); + } + + return 0; +} + +static int runExportSHMBuffers(struct vulkan_blit_state *s) { + struct vulkan_stream *p = &s->streams[SPA_DIRECTION_OUTPUT]; + + if (p->spa_buffers[p->current_buffer_id]->datas[0].type == SPA_DATA_MemPtr) { + struct spa_buffer *spa_buf = p->spa_buffers[p->current_buffer_id]; + struct vulkan_read_pixels_info readInfo = { + .data = spa_buf->datas[0].data, + .offset = 0, + .stride = p->bpp * p->dim.width, + .bytes_per_pixel = p->bpp, + .size.width = p->dim.width, + .size.height = p->dim.height, + }; + CHECK(vulkan_read_pixels(&s->base, &readInfo, &p->buffers[p->current_buffer_id])); + } + + return 0; +} + +/** runCommandBuffer + * The return value of this functions means the following: + * ret < 0: Error + * ret = 0: queueSubmit was succsessful, but manual synchronization is required + * ret = 1: queueSubmit was succsessful and buffers can be released without synchronization + */ +static int runCommandBuffer(struct vulkan_blit_state *s) +{ + VULKAN_INSTANCE_FUNCTION(vkQueueSubmit2KHR); + VULKAN_INSTANCE_FUNCTION(vkGetSemaphoreFdKHR); + + static const VkCommandBufferBeginInfo beginInfo = { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, + }; + VK_CHECK_RESULT(vkBeginCommandBuffer(s->commandBuffer, &beginInfo)); + + CHECK(runImportSHMBuffers(s)); + + uint32_t i; + struct vulkan_stream *stream_input = &s->streams[SPA_DIRECTION_INPUT]; + struct vulkan_stream *stream_output = &s->streams[SPA_DIRECTION_OUTPUT]; + + VkImage src_image = stream_input->buffers[stream_input->current_buffer_id].image; + VkImage dst_image = stream_output->buffers[stream_output->current_buffer_id].image; + + VkImageBlit imageBlitRegion = { + .srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .srcSubresource.layerCount = 1, + .srcOffsets[0] = { + .x = 0, + .y = 0, + .z = 0, + }, + .srcOffsets[1] = { + .x = stream_input->dim.width, + .y = stream_input->dim.height, + .z = 1, + }, + .dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .dstSubresource.layerCount = 1, + .dstOffsets[1] = { + .x = stream_output->dim.width, + .y = stream_output->dim.height, + .z = 1, + } + }; + spa_log_trace_fp(s->log, "Blitting from (%p, %d, %d) %d,%dx%d,%d to (%p, %d, %d) %d,%dx%d,%d", + stream_input, stream_input->current_buffer_id, stream_input->direction, 0, 0, stream_input->dim.width, stream_input->dim.height, + stream_output, stream_output->current_buffer_id, stream_output->direction, 0, 0, stream_output->dim.width, stream_output->dim.height); + vkCmdBlitImage(s->commandBuffer, src_image, VK_IMAGE_LAYOUT_GENERAL, + dst_image, VK_IMAGE_LAYOUT_GENERAL, + 1, &imageBlitRegion, VK_FILTER_NEAREST); + + VkImageMemoryBarrier acquire_barrier[s->n_streams]; + VkImageMemoryBarrier release_barrier[s->n_streams]; + VkSemaphoreSubmitInfo semaphore_wait_info[s->n_streams]; + uint32_t semaphore_wait_info_len = 0; + VkSemaphoreSubmitInfo semaphore_signal_info[1]; + uint32_t semaphore_signal_info_len = 0; + + for (i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + struct vulkan_buffer *current_buffer = &p->buffers[p->current_buffer_id]; + struct spa_buffer *current_spa_buffer = p->spa_buffers[p->current_buffer_id]; + + VkAccessFlags access_flags; + if (p->direction == SPA_DIRECTION_INPUT) { + access_flags = VK_ACCESS_TRANSFER_READ_BIT; + } else { + access_flags = VK_ACCESS_TRANSFER_WRITE_BIT; + } + + acquire_barrier[i]= (VkImageMemoryBarrier) { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .dstQueueFamilyIndex = s->base.queueFamilyIndex, + .image = current_buffer->image, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcAccessMask = 0, + .dstAccessMask = access_flags, + .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .subresourceRange.levelCount = 1, + .subresourceRange.layerCount = 1, + }; + + release_barrier[i]= (VkImageMemoryBarrier) { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .srcQueueFamilyIndex = s->base.queueFamilyIndex, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .image = current_buffer->image, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcAccessMask = access_flags, + .dstAccessMask = 0, + .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .subresourceRange.levelCount = 1, + .subresourceRange.layerCount = 1, + }; + + if (current_spa_buffer->datas[0].type != SPA_DATA_DmaBuf) + continue; + + if (vulkan_sync_foreign_dmabuf(&s->base, current_buffer) < 0) { + spa_log_warn(s->log, "Failed to wait for foreign buffer DMA-BUF fence"); + } else { + if (current_buffer->foreign_semaphore != VK_NULL_HANDLE) { + semaphore_wait_info[semaphore_wait_info_len++] = (VkSemaphoreSubmitInfo) { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .semaphore = current_buffer->foreign_semaphore, + .stageMask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + }; + } + } + } + + vkCmdPipelineBarrier(s->commandBuffer, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_2_TRANSFER_BIT, + 0, 0, NULL, 0, NULL, + s->n_streams, acquire_barrier); + + vkCmdPipelineBarrier(s->commandBuffer, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, + 0, 0, NULL, 0, NULL, + s->n_streams, release_barrier); + + VK_CHECK_RESULT(vkEndCommandBuffer(s->commandBuffer)); + + VK_CHECK_RESULT(vkResetFences(s->base.device, 1, &s->fence)); + + if (s->pipelineSemaphore == VK_NULL_HANDLE) { + VkExportSemaphoreCreateInfo export_info = { + .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO, + .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT, + }; + VkSemaphoreCreateInfo semaphore_info = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, + .pNext = &export_info, + }; + VK_CHECK_RESULT(vkCreateSemaphore(s->base.device, &semaphore_info, NULL, &s->pipelineSemaphore)); + } + + semaphore_signal_info[semaphore_signal_info_len++] = (VkSemaphoreSubmitInfo) { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .semaphore = s->pipelineSemaphore, + }; + + VkCommandBufferSubmitInfoKHR commandBufferInfo = { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, + .commandBuffer = s->commandBuffer, + }; + + const VkSubmitInfo2KHR submitInfo = { + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2_KHR, + .commandBufferInfoCount = 1, + .pCommandBufferInfos = &commandBufferInfo, + .waitSemaphoreInfoCount = semaphore_wait_info_len, + .pWaitSemaphoreInfos = semaphore_wait_info, + .signalSemaphoreInfoCount = semaphore_signal_info_len, + .pSignalSemaphoreInfos = semaphore_signal_info, + }; + VK_CHECK_RESULT(vkQueueSubmit2KHR(s->base.queue, 1, &submitInfo, s->fence)); + s->started = true; + + VkSemaphoreGetFdInfoKHR get_fence_fd_info = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR, + .semaphore = s->pipelineSemaphore, + .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT, + }; + int sync_file_fd = -1; + VK_CHECK_RESULT(vkGetSemaphoreFdKHR(s->base.device, &get_fence_fd_info, &sync_file_fd)); + + int ret = 1; + for (uint32_t i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + struct spa_buffer *current_spa_buffer = p->spa_buffers[p->current_buffer_id]; + + if (current_spa_buffer->datas[0].type != SPA_DATA_DmaBuf) + continue; + + if (!vulkan_sync_export_dmabuf(&s->base, &p->buffers[p->current_buffer_id], sync_file_fd)) { + ret = 0; + } + } + close(sync_file_fd); + + return ret; +} + +static void clear_buffers(struct vulkan_blit_state *s, struct vulkan_stream *p) +{ + uint32_t i; + + for (i = 0; i < p->n_buffers; i++) { + vulkan_buffer_clear(&s->base, &p->buffers[i]); + p->spa_buffers[i] = NULL; + } + p->n_buffers = 0; + if (p->direction == SPA_DIRECTION_INPUT) { + vulkan_staging_buffer_destroy(&s->base, &s->staging_buffer); + s->staging_buffer.buffer = VK_NULL_HANDLE; + } +} + +static void clear_streams(struct vulkan_blit_state *s) +{ + uint32_t i; + for (i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + clear_buffers(s, p); + } +} + +int spa_vulkan_blit_fixate_modifier(struct vulkan_blit_state *s, struct vulkan_stream *p, struct spa_video_info *info, + uint32_t modifierCount, uint64_t *modifiers, uint64_t *modifier) +{ + VkFormat format; + struct spa_rectangle size; + switch (info->media_subtype) { + case SPA_MEDIA_SUBTYPE_dsp: + format = vulkan_id_to_vkformat(info->info.dsp.format); + size.width = p->dim.width; + size.height = p->dim.height; + break; + case SPA_MEDIA_SUBTYPE_raw: + format = vulkan_id_to_vkformat(info->info.raw.format); + size.width = p->dim.width; + size.height = p->dim.height; + break; + default: + spa_log_warn(s->log, "Unsupported media subtype %d", info->media_subtype); + return -1; + } + if (format == VK_FORMAT_UNDEFINED) { + return -1; + } + + struct dmabuf_fixation_info fixation_info = { + .format = format, + .modifierCount = modifierCount, + .modifiers = modifiers, + .size = size, + .usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT, + }; + return vulkan_fixate_modifier(&s->base, &fixation_info, modifier); +} + +int spa_vulkan_blit_use_buffers(struct vulkan_blit_state *s, struct vulkan_stream *p, uint32_t flags, + struct spa_video_info *info, uint32_t n_buffers, struct spa_buffer **buffers) +{ + struct external_buffer_info externalBufferInfo = {0}; + switch (info->media_subtype) { + case SPA_MEDIA_SUBTYPE_dsp: + externalBufferInfo.format = vulkan_id_to_vkformat(info->info.dsp.format); + externalBufferInfo.size.width = p->dim.width; + externalBufferInfo.size.height = p->dim.height; + if (info->info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER) + externalBufferInfo.modifier = info->info.dsp.modifier; + break; + case SPA_MEDIA_SUBTYPE_raw: + externalBufferInfo.format = vulkan_id_to_vkformat(info->info.raw.format); + externalBufferInfo.size.width = p->dim.width; + externalBufferInfo.size.height = p->dim.height; + if (info->info.raw.flags & SPA_VIDEO_FLAG_MODIFIER) + externalBufferInfo.modifier = info->info.raw.modifier; + break; + default: + spa_log_warn(s->log, "Unsupported media subtype %d", info->media_subtype); + return -1; + } + if (externalBufferInfo.format == VK_FORMAT_UNDEFINED) + return -1; + + vulkan_wait_idle(&s->base); + clear_buffers(s, p); + + bool alloc = flags & SPA_NODE_BUFFERS_FLAG_ALLOC; + int ret; + p->n_buffers = 0; + for (uint32_t i = 0; i < n_buffers; i++) { + externalBufferInfo.usage = p->direction == SPA_DIRECTION_OUTPUT + ? VK_IMAGE_USAGE_TRANSFER_DST_BIT + : VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + externalBufferInfo.spa_buf = buffers[i]; + if (alloc) { + if (SPA_FLAG_IS_SET(buffers[i]->datas[0].type, 1<base, &externalBufferInfo, &p->buffers[i]); + } else { + spa_log_error(s->log, "Unsupported buffer type mask %d", buffers[i]->datas[0].type); + return -1; + } + } else { + switch (buffers[i]->datas[0].type) { + case SPA_DATA_DmaBuf:; + ret = vulkan_import_dmabuf(&s->base, &externalBufferInfo, &p->buffers[i]); + break; + case SPA_DATA_MemPtr:; + if (p->direction == SPA_DIRECTION_OUTPUT) { + externalBufferInfo.usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + } else { + externalBufferInfo.usage |= VK_IMAGE_USAGE_TRANSFER_DST_BIT; + } + ret = vulkan_import_memptr(&s->base, &externalBufferInfo, &p->buffers[i]); + break; + default: + spa_log_error(s->log, "Unsupported buffer type %d", buffers[i]->datas[0].type); + return -1; + } + } + if (ret != 0) { + spa_log_error(s->log, "Failed to use buffer %d", i); + return ret; + } + p->spa_buffers[i] = buffers[i]; + p->n_buffers++; + } + if (p->direction == SPA_DIRECTION_INPUT && buffers[0]->datas[0].type == SPA_DATA_MemPtr) { + ret = vulkan_staging_buffer_create(&s->base, buffers[0]->datas[0].maxsize, &s->staging_buffer); + if (ret < 0) { + spa_log_error(s->log, "Failed to create staging buffer"); + return ret; + } + } + + return 0; +} + +int spa_vulkan_blit_enumerate_raw_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder) +{ + uint32_t fmt_idx; + bool has_modifier; + if (!find_EnumFormatInfo(&s->formatInfosRaw, index, caps, &fmt_idx, &has_modifier)) + return 0; + *param = build_raw_EnumFormat(&s->formatInfosRaw.infos[fmt_idx], has_modifier, builder); + return 1; +} + +int spa_vulkan_blit_enumerate_dsp_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder) +{ + uint32_t fmt_idx; + bool has_modifier; + if (!find_EnumFormatInfo(&s->formatInfosDSP, index, caps, &fmt_idx, &has_modifier)) + return 0; + *param = build_dsp_EnumFormat(&s->formatInfosDSP.infos[fmt_idx], has_modifier, builder); + return 1; +} + +int spa_vulkan_blit_enumerate_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder) +{ + uint32_t fmt_idx; + bool has_modifier; + uint32_t raw_offset = 0; + if ((caps & VULKAN_BUFFER_TYPE_CAP_SHM) > 0) + raw_offset += s->formatInfosDSP.formatCount; + if ((caps & VULKAN_BUFFER_TYPE_CAP_DMABUF) > 0) + raw_offset += s->formatInfosDSP.formatsWithModifiersCount; + if (index < raw_offset) { + if (find_EnumFormatInfo(&s->formatInfosDSP, index, caps, &fmt_idx, &has_modifier)) { + *param = build_dsp_EnumFormat(&s->formatInfosDSP.infos[fmt_idx], has_modifier, builder); + return 1; + } + } else { + if (find_EnumFormatInfo(&s->formatInfosRaw, index - raw_offset, caps, &fmt_idx, &has_modifier)) { + *param = build_raw_EnumFormat(&s->formatInfosRaw.infos[fmt_idx], has_modifier, builder); + return 1; + } + } + return 0; +} + +static int vulkan_stream_init(struct vulkan_stream *stream, enum spa_direction direction, + struct spa_dict *props) +{ + spa_zero(*stream); + stream->direction = direction; + stream->current_buffer_id = SPA_ID_INVALID; + stream->busy_buffer_id = SPA_ID_INVALID; + stream->ready_buffer_id = SPA_ID_INVALID; + return 0; +} + +int spa_vulkan_blit_init_stream(struct vulkan_blit_state *s, struct vulkan_stream *stream, + enum spa_direction direction, struct spa_dict *props) +{ + return vulkan_stream_init(stream, direction, props); +} + +int spa_vulkan_blit_prepare(struct vulkan_blit_state *s) +{ + if (!s->prepared) { + CHECK(vulkan_fence_create(&s->base, &s->fence)); + CHECK(vulkan_commandPool_create(&s->base, &s->commandPool)); + CHECK(vulkan_commandBuffer_create(&s->base, s->commandPool, &s->commandBuffer)); + s->prepared = true; + } + return 0; +} + +int spa_vulkan_blit_unprepare(struct vulkan_blit_state *s) +{ + if (s->prepared) { + vkDestroyCommandPool(s->base.device, s->commandPool, NULL); + vkDestroyFence(s->base.device, s->fence, NULL); + s->prepared = false; + } + return 0; +} + +int spa_vulkan_blit_start(struct vulkan_blit_state *s) +{ + uint32_t i; + + for (i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + p->current_buffer_id = SPA_ID_INVALID; + p->busy_buffer_id = SPA_ID_INVALID; + p->ready_buffer_id = SPA_ID_INVALID; + } + return 0; +} + +int spa_vulkan_blit_stop(struct vulkan_blit_state *s) +{ + VK_CHECK_RESULT(vkDeviceWaitIdle(s->base.device)); + clear_streams(s); + s->started = false; + return 0; +} + +int spa_vulkan_blit_ready(struct vulkan_blit_state *s) +{ + uint32_t i; + VkResult result; + + if (!s->started) + return 0; + + result = vkGetFenceStatus(s->base.device, s->fence); + if (result == VK_NOT_READY) + return -EBUSY; + VK_CHECK_RESULT(result); + + s->started = false; + + for (i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + p->ready_buffer_id = p->busy_buffer_id; + p->busy_buffer_id = SPA_ID_INVALID; + } + return 0; +} + +int spa_vulkan_blit_process(struct vulkan_blit_state *s) +{ + if (!s->initialized) { + spa_log_warn(s->log, "Renderer not initialized"); + return -1; + } + if (!s->prepared) { + spa_log_warn(s->log, "Renderer not prepared"); + return -1; + } + CHECK(updateBuffers(s)); + CHECK(runCommandBuffer(s)); + // CHECK(vulkan_wait_fence(&s->base, s->fence)); + CHECK(vulkan_wait_idle(&s->base)); + CHECK(runExportSHMBuffers(s)); + + return 0; +} + +int spa_vulkan_blit_get_buffer_caps(struct vulkan_blit_state *s, enum spa_direction direction) +{ + switch (direction) { + case SPA_DIRECTION_INPUT: + return VULKAN_BUFFER_TYPE_CAP_DMABUF | VULKAN_BUFFER_TYPE_CAP_SHM; + case SPA_DIRECTION_OUTPUT: + return VULKAN_BUFFER_TYPE_CAP_DMABUF | VULKAN_BUFFER_TYPE_CAP_SHM; + } + return 0; +} + +struct vulkan_modifier_info *spa_vulkan_blit_get_modifier_info(struct vulkan_blit_state *s, struct spa_video_info *info) { + VkFormat format; + uint64_t modifier; + switch (info->media_subtype) { + case SPA_MEDIA_SUBTYPE_dsp: + format = vulkan_id_to_vkformat(info->info.dsp.format); + modifier = info->info.dsp.modifier; + return vulkan_modifierInfo_find(&s->formatInfosDSP, format, modifier); + case SPA_MEDIA_SUBTYPE_raw: + format = vulkan_id_to_vkformat(info->info.raw.format); + modifier = info->info.raw.modifier; + return vulkan_modifierInfo_find(&s->formatInfosRaw, format, modifier); + default: + spa_log_warn(s->log, "Unsupported media subtype %d", info->media_subtype); + return NULL; + } +} + +int spa_vulkan_blit_init(struct vulkan_blit_state *s) +{ + int ret; + s->base.log = s->log; + struct vulkan_base_info baseInfo = { + .queueFlags = VK_QUEUE_TRANSFER_BIT, + }; + if ((ret = vulkan_base_init(&s->base, &baseInfo)) < 0) + return ret; + + uint32_t dsp_formats [] = { + SPA_VIDEO_FORMAT_DSP_F32 + }; + vulkan_format_infos_init(&s->base, SPA_N_ELEMENTS(dsp_formats), dsp_formats, &s->formatInfosDSP); + uint32_t raw_formats [] = { + SPA_VIDEO_FORMAT_BGRA, + SPA_VIDEO_FORMAT_RGBA, + SPA_VIDEO_FORMAT_BGRx, + SPA_VIDEO_FORMAT_RGBx, + SPA_VIDEO_FORMAT_BGR, + SPA_VIDEO_FORMAT_RGB, + }; + vulkan_format_infos_init(&s->base, SPA_N_ELEMENTS(raw_formats), raw_formats, &s->formatInfosRaw); + s->initialized = true; + return 0; +} + +void spa_vulkan_blit_deinit(struct vulkan_blit_state *s) +{ + vulkan_format_infos_deinit(&s->formatInfosRaw); + vulkan_format_infos_deinit(&s->formatInfosDSP); + vulkan_base_deinit(&s->base); + s->initialized = false; +} diff --git a/spa/plugins/videoconvert/vulkan-blit-utils.h b/spa/plugins/videoconvert/vulkan-blit-utils.h new file mode 100644 index 000000000..9fa089c80 --- /dev/null +++ b/spa/plugins/videoconvert/vulkan-blit-utils.h @@ -0,0 +1,84 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2019 Wim Taymans */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include + +#include +#include +#include +#include + +#include "vulkan-utils.h" + +#define MAX_STREAMS 2 + +struct vulkan_stream { + enum spa_direction direction; + + uint32_t pending_buffer_id; + uint32_t current_buffer_id; + uint32_t busy_buffer_id; + uint32_t ready_buffer_id; + + struct spa_rectangle dim; + uint32_t bpp; + + struct vulkan_buffer buffers[MAX_BUFFERS]; + struct spa_buffer *spa_buffers[MAX_BUFFERS]; + uint32_t n_buffers; +}; + +struct vulkan_blit_state { + struct spa_log *log; + + struct vulkan_base base; + + struct vulkan_format_infos formatInfosRaw; + struct vulkan_format_infos formatInfosDSP; + + VkCommandPool commandPool; + VkCommandBuffer commandBuffer; + struct vulkan_staging_buffer staging_buffer; + + VkFence fence; + VkSemaphore pipelineSemaphore; + unsigned int initialized:1; + unsigned int prepared:1; + unsigned int started:1; + + //VkSampler sampler; + + uint32_t n_streams; + struct vulkan_stream streams[MAX_STREAMS]; +}; + +int spa_vulkan_blit_init_stream(struct vulkan_blit_state *s, struct vulkan_stream *stream, enum spa_direction, + struct spa_dict *props); + +int spa_vulkan_blit_fixate_modifier(struct vulkan_blit_state *s, struct vulkan_stream *p, struct spa_video_info *info, + uint32_t modifierCount, uint64_t *modifiers, uint64_t *modifier); +int spa_vulkan_blit_use_buffers(struct vulkan_blit_state *s, struct vulkan_stream *stream, uint32_t flags, + struct spa_video_info *info, uint32_t n_buffers, struct spa_buffer **buffers); +int spa_vulkan_blit_enumerate_raw_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder); +int spa_vulkan_blit_enumerate_dsp_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder); +int spa_vulkan_blit_enumerate_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder); +int spa_vulkan_blit_prepare(struct vulkan_blit_state *s); +int spa_vulkan_blit_unprepare(struct vulkan_blit_state *s); + +int spa_vulkan_blit_start(struct vulkan_blit_state *s); +int spa_vulkan_blit_stop(struct vulkan_blit_state *s); +int spa_vulkan_blit_ready(struct vulkan_blit_state *s); +int spa_vulkan_blit_process(struct vulkan_blit_state *s); +int spa_vulkan_blit_cleanup(struct vulkan_blit_state *s); + +int spa_vulkan_blit_get_buffer_caps(struct vulkan_blit_state *s, enum spa_direction direction); +struct vulkan_modifier_info *spa_vulkan_blit_get_modifier_info(struct vulkan_blit_state *s, + struct spa_video_info *info); + +int spa_vulkan_blit_init(struct vulkan_blit_state *s); +void spa_vulkan_blit_deinit(struct vulkan_blit_state *s); diff --git a/spa/plugins/videoconvert/vulkan-types.h b/spa/plugins/videoconvert/vulkan-types.h new file mode 100644 index 000000000..e8b7ff535 --- /dev/null +++ b/spa/plugins/videoconvert/vulkan-types.h @@ -0,0 +1,66 @@ +#pragma once + +#include + +#include +#include + +#define MAX_BUFFERS 16 +#define DMABUF_MAX_PLANES 1 + +enum buffer_type_caps { + VULKAN_BUFFER_TYPE_CAP_SHM = 1<<0, + VULKAN_BUFFER_TYPE_CAP_DMABUF = 1<<1, +}; + +struct vulkan_modifier_info { + VkDrmFormatModifierPropertiesEXT props; + VkExtent2D max_extent; +}; + +struct vulkan_format_info { + uint32_t spa_format; + VkFormat vk_format; + uint32_t modifierCount; + struct vulkan_modifier_info *infos; +}; + +struct vulkan_format_infos { + uint32_t formatCount; + struct vulkan_format_info *infos; + + uint32_t formatsWithModifiersCount; +}; + +struct vulkan_buffer { + int fd; + VkImage image; + VkImageView view; + VkDeviceMemory memory; + VkSemaphore foreign_semaphore; +}; + +struct vulkan_staging_buffer { + VkBuffer buffer; + VkDeviceMemory memory; +}; + +struct vulkan_base_info { + uint32_t queueFlags; +}; + +struct vulkan_base { + struct spa_log *log; + + VkInstance instance; + + VkPhysicalDevice physicalDevice; + + VkQueue queue; + uint32_t queueFamilyIndex; + VkDevice device; + + bool implicit_sync_interop; + + unsigned int initialized:1; +}; diff --git a/spa/plugins/videoconvert/vulkan-utils.c b/spa/plugins/videoconvert/vulkan-utils.c new file mode 100644 index 000000000..d312f91b4 --- /dev/null +++ b/spa/plugins/videoconvert/vulkan-utils.c @@ -0,0 +1,980 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2019 Wim Taymans */ +/* SPDX-License-Identifier: MIT */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#if !defined(__FreeBSD__) && !defined(__MidnightBSD__) +#include +#endif +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "vulkan-utils.h" +#include "dmabuf.h" + +//#define ENABLE_VALIDATION + +#define VULKAN_INSTANCE_FUNCTION(name) \ + PFN_##name name = (PFN_##name)vkGetInstanceProcAddr(s->instance, #name) + +static int vkresult_to_errno(VkResult result) +{ + switch (result) { + case VK_SUCCESS: + case VK_EVENT_SET: + case VK_EVENT_RESET: + return 0; + case VK_NOT_READY: + case VK_INCOMPLETE: + case VK_ERROR_NATIVE_WINDOW_IN_USE_KHR: + return EBUSY; + case VK_TIMEOUT: + return ETIMEDOUT; + case VK_ERROR_OUT_OF_HOST_MEMORY: + case VK_ERROR_OUT_OF_DEVICE_MEMORY: + case VK_ERROR_MEMORY_MAP_FAILED: + case VK_ERROR_OUT_OF_POOL_MEMORY: + case VK_ERROR_FRAGMENTED_POOL: +#ifdef VK_ERROR_FRAGMENTATION_EXT + case VK_ERROR_FRAGMENTATION_EXT: +#endif + return ENOMEM; + case VK_ERROR_INITIALIZATION_FAILED: + return EIO; + case VK_ERROR_DEVICE_LOST: + case VK_ERROR_SURFACE_LOST_KHR: +#ifdef VK_ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT + case VK_ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT: +#endif + return ENODEV; + case VK_ERROR_LAYER_NOT_PRESENT: + case VK_ERROR_EXTENSION_NOT_PRESENT: + case VK_ERROR_FEATURE_NOT_PRESENT: + return ENOENT; + case VK_ERROR_INCOMPATIBLE_DRIVER: + case VK_ERROR_FORMAT_NOT_SUPPORTED: + case VK_ERROR_INCOMPATIBLE_DISPLAY_KHR: + return ENOTSUP; + case VK_ERROR_TOO_MANY_OBJECTS: + return ENFILE; + case VK_SUBOPTIMAL_KHR: + case VK_ERROR_OUT_OF_DATE_KHR: + return EIO; + case VK_ERROR_INVALID_EXTERNAL_HANDLE: + case VK_ERROR_INVALID_SHADER_NV: +#ifdef VK_ERROR_VALIDATION_FAILED_EXT + case VK_ERROR_VALIDATION_FAILED_EXT: +#endif +#ifdef VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT + case VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT: +#endif +#ifdef VK_ERROR_INVALID_DEVICE_ADDRESS_EXT + case VK_ERROR_INVALID_DEVICE_ADDRESS_EXT: +#endif + return EINVAL; +#ifdef VK_ERROR_NOT_PERMITTED_EXT + case VK_ERROR_NOT_PERMITTED_EXT: + return EPERM; +#endif + default: + return EIO; + } +} + +static struct { + VkFormat format; + uint32_t id; +} vk_video_format_convs[] = { + { VK_FORMAT_R32G32B32A32_SFLOAT, SPA_VIDEO_FORMAT_RGBA_F32 }, + { VK_FORMAT_B8G8R8A8_SRGB, SPA_VIDEO_FORMAT_BGRA }, + { VK_FORMAT_R8G8B8A8_SRGB, SPA_VIDEO_FORMAT_RGBA }, + { VK_FORMAT_B8G8R8A8_SRGB, SPA_VIDEO_FORMAT_BGRx }, + { VK_FORMAT_R8G8B8A8_SRGB, SPA_VIDEO_FORMAT_RGBx }, + { VK_FORMAT_B8G8R8_SRGB, SPA_VIDEO_FORMAT_BGR }, + { VK_FORMAT_R8G8B8_SRGB, SPA_VIDEO_FORMAT_RGB }, +}; + +static int createInstance(struct vulkan_base *s) +{ + static const VkApplicationInfo applicationInfo = { + .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO, + .pApplicationName = "PipeWire", + .applicationVersion = 0, + .pEngineName = "PipeWire Vulkan Engine", + .engineVersion = 0, + .apiVersion = VK_API_VERSION_1_1 + }; + static const char * const extensions[] = { + VK_KHR_EXTERNAL_MEMORY_CAPABILITIES_EXTENSION_NAME + }; + static const char * const checkLayers[] = { +#ifdef ENABLE_VALIDATION + "VK_LAYER_KHRONOS_validation", +#endif + NULL + }; + uint32_t i, j, layerCount, n_layers = 0; + const char *layers[1]; + vkEnumerateInstanceLayerProperties(&layerCount, NULL); + + VkLayerProperties availableLayers[layerCount]; + vkEnumerateInstanceLayerProperties(&layerCount, availableLayers); + + for (i = 0; i < layerCount; i++) { + for (j = 0; j < SPA_N_ELEMENTS(checkLayers); j++) { + if (spa_streq(availableLayers[i].layerName, checkLayers[j])) + layers[n_layers++] = checkLayers[j]; + } + } + + const VkInstanceCreateInfo createInfo = { + .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, + .pApplicationInfo = &applicationInfo, + .enabledExtensionCount = 1, + .ppEnabledExtensionNames = extensions, + .enabledLayerCount = n_layers, + .ppEnabledLayerNames = layers, + }; + + VK_CHECK_RESULT(vkCreateInstance(&createInfo, NULL, &s->instance)); + + return 0; +} + +static int findPhysicalDevice(struct vulkan_base *s) +{ + uint32_t deviceCount; + VkPhysicalDevice *devices; + + vkEnumeratePhysicalDevices(s->instance, &deviceCount, NULL); + if (deviceCount == 0) + return -ENODEV; + + devices = alloca(deviceCount * sizeof(VkPhysicalDevice)); + vkEnumeratePhysicalDevices(s->instance, &deviceCount, devices); + + s->physicalDevice = devices[0]; + + return 0; +} + +static int getComputeQueueFamilyIndex(struct vulkan_base *s, uint32_t queueFlags, uint32_t *queueFamilyIndex) +{ + uint32_t i, queueFamilyCount; + VkQueueFamilyProperties *queueFamilies; + + vkGetPhysicalDeviceQueueFamilyProperties(s->physicalDevice, &queueFamilyCount, NULL); + + queueFamilies = alloca(queueFamilyCount * sizeof(VkQueueFamilyProperties)); + vkGetPhysicalDeviceQueueFamilyProperties(s->physicalDevice, &queueFamilyCount, queueFamilies); + + for (i = 0; i < queueFamilyCount; i++) { + VkQueueFamilyProperties props = queueFamilies[i]; + + if (props.queueCount > 0 && ((props.queueFlags & queueFlags) == queueFlags)) + break; + } + if (i == queueFamilyCount) + return -ENODEV; + + *queueFamilyIndex = i; + return 0; +} + +static int createDevice(struct vulkan_base *s, struct vulkan_base_info *info) +{ + + CHECK(getComputeQueueFamilyIndex(s, info->queueFlags, &s->queueFamilyIndex)); + + const VkDeviceQueueCreateInfo queueCreateInfo = { + .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, + .queueFamilyIndex = s->queueFamilyIndex, + .queueCount = 1, + .pQueuePriorities = (const float[]) { 1.0f } + }; + const VkPhysicalDeviceSynchronization2FeaturesKHR sync2_features = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SYNCHRONIZATION_2_FEATURES_KHR, + .synchronization2 = VK_TRUE, + }; + static const char * const extensions[] = { + VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME, + VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME, + VK_KHR_IMAGE_FORMAT_LIST_EXTENSION_NAME, + VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, + VK_KHR_SYNCHRONIZATION_2_EXTENSION_NAME, + VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME, + VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME, + }; + + const VkDeviceCreateInfo deviceCreateInfo = { + .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, + .queueCreateInfoCount = 1, + .pQueueCreateInfos = &queueCreateInfo, + .enabledExtensionCount = SPA_N_ELEMENTS(extensions), + .ppEnabledExtensionNames = extensions, + .pNext = &sync2_features, + }; + + VK_CHECK_RESULT(vkCreateDevice(s->physicalDevice, &deviceCreateInfo, NULL, &s->device)); + + vkGetDeviceQueue(s->device, s->queueFamilyIndex, 0, &s->queue); + + return 0; +} + +int vulkan_write_pixels(struct vulkan_base *s, struct vulkan_write_pixels_info *info, struct vulkan_staging_buffer *vk_sbuf) +{ + void *vmap; + VK_CHECK_RESULT(vkMapMemory(s->device, vk_sbuf->memory, 0, VK_WHOLE_SIZE, 0, &vmap)); + + char *map = (char *)vmap; + + // upload data + const char *pdata = info->data; + memcpy(map, pdata, info->stride * info->size.height); + + info->copies[0] = (VkBufferImageCopy) { + .imageExtent.width = info->size.width, + .imageExtent.height = info->size.height, + .imageExtent.depth = 1, + .imageOffset.x = 0, + .imageOffset.y = 0, + .imageOffset.z = 0, + .bufferOffset = 0, + .bufferRowLength = info->size.width, + .bufferImageHeight = info->size.height, + .imageSubresource.mipLevel = 0, + .imageSubresource.baseArrayLayer = 0, + .imageSubresource.layerCount = 1, + .imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + }; + + vkUnmapMemory(s->device, vk_sbuf->memory); + + return 0; +} + +int vulkan_read_pixels(struct vulkan_base *s, struct vulkan_read_pixels_info *info, struct vulkan_buffer *vk_buf) +{ + VkImageSubresource img_sub_res = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .arrayLayer = 0, + .mipLevel = 0, + }; + VkSubresourceLayout img_sub_layout; + vkGetImageSubresourceLayout(s->device, vk_buf->image, &img_sub_res, &img_sub_layout); + + void *v; + VK_CHECK_RESULT(vkMapMemory(s->device, vk_buf->memory, 0, VK_WHOLE_SIZE, 0, &v)); + + const char *d = (const char *)v + img_sub_layout.offset; + unsigned char *p = (unsigned char *)info->data + info->offset; + uint32_t pack_stride = img_sub_layout.rowPitch; + spa_log_trace_fp(s->log, "Read pixels: %p to %p, stride: %d, width %d, height %d, offset %d, pack_stride %d", d, p, info->stride, info->size.width, info->size.height, info->offset, pack_stride); + if (pack_stride == info->stride) { + memcpy(p, d, info->stride * info->size.height); + } else { + for (uint32_t i = 0; i < info->size.height; i++) { + memcpy(p + i * info->stride, d + i * pack_stride, info->size.width * info->bytes_per_pixel); + } + } + vkUnmapMemory(s->device, vk_buf->memory); + return 0; +} + +int vulkan_sync_foreign_dmabuf(struct vulkan_base *s, struct vulkan_buffer *vk_buf) +{ + VULKAN_INSTANCE_FUNCTION(vkImportSemaphoreFdKHR); + + if (!s->implicit_sync_interop) { + struct pollfd pollfd = { + .fd = vk_buf->fd, + .events = POLLIN, + }; + int timeout_ms = 1000; + int ret = poll(&pollfd, 1, timeout_ms); + if (ret < 0) { + spa_log_error(s->log, "Failed to wait for DMA-BUF fence"); + return -1; + } else if (ret == 0) { + spa_log_error(s->log, "Timed out waiting for DMA-BUF fence"); + return -1; + } + return 0; + } + + int sync_file_fd = dmabuf_export_sync_file(s->log, vk_buf->fd, DMA_BUF_SYNC_READ); + if (sync_file_fd < 0) { + spa_log_error(s->log, "Failed to extract for DMA-BUF fence"); + return -1; + } + + if (vk_buf->foreign_semaphore == VK_NULL_HANDLE) { + VkSemaphoreCreateInfo semaphore_info = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, + }; + VK_CHECK_RESULT_WITH_CLEANUP(vkCreateSemaphore(s->device, &semaphore_info, NULL, &vk_buf->foreign_semaphore), close(sync_file_fd)); + } + + VkImportSemaphoreFdInfoKHR import_info = { + .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR, + .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT, + .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT_KHR, + .semaphore = vk_buf->foreign_semaphore, + .fd = sync_file_fd, + }; + VK_CHECK_RESULT_WITH_CLEANUP(vkImportSemaphoreFdKHR(s->device, &import_info), close(sync_file_fd)); + + return 0; +} + +bool vulkan_sync_export_dmabuf(struct vulkan_base *s, struct vulkan_buffer *vk_buf, int sync_file_fd) +{ + if (!s->implicit_sync_interop) + return false; + + return dmabuf_import_sync_file(s->log, vk_buf->fd, DMA_BUF_SYNC_WRITE, sync_file_fd); +} + +int vulkan_fence_create(struct vulkan_base *s, VkFence *fence) +{ + VkFenceCreateInfo createInfo = { + .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, + .flags = 0, + }; + VK_CHECK_RESULT(vkCreateFence(s->device, &createInfo, NULL, fence)); + + return 0; +} + +int vulkan_commandPool_create(struct vulkan_base *s, VkCommandPool *commandPool) +{ + const VkCommandPoolCreateInfo commandPoolCreateInfo = { + .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, + .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, + .queueFamilyIndex = s->queueFamilyIndex, + }; + VK_CHECK_RESULT(vkCreateCommandPool(s->device, + &commandPoolCreateInfo, NULL, + commandPool)); + + return 0; +} + +int vulkan_commandBuffer_create(struct vulkan_base *s, VkCommandPool commandPool, VkCommandBuffer *commandBuffer) +{ + const VkCommandBufferAllocateInfo commandBufferAllocateInfo = { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, + .commandPool = commandPool, + .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, + .commandBufferCount = 1, + }; + VK_CHECK_RESULT(vkAllocateCommandBuffers(s->device, + &commandBufferAllocateInfo, + commandBuffer)); + + return 0; +} + +uint32_t vulkan_memoryType_find(struct vulkan_base *s, + uint32_t memoryTypeBits, VkMemoryPropertyFlags properties) +{ + uint32_t i; + VkPhysicalDeviceMemoryProperties memoryProperties; + + vkGetPhysicalDeviceMemoryProperties(s->physicalDevice, &memoryProperties); + + for (i = 0; i < memoryProperties.memoryTypeCount; i++) { + if ((memoryTypeBits & (1 << i)) && + ((memoryProperties.memoryTypes[i].propertyFlags & properties) == properties)) + return i; + } + return -1; +} + +struct vulkan_format_info *vulkan_formatInfo_find(struct vulkan_format_infos *fmtInfo, VkFormat format) +{ + for (uint32_t i = 0; i < fmtInfo->formatCount; i++) { + if (fmtInfo->infos[i].vk_format == format) + return &fmtInfo->infos[i]; + } + return NULL; +} + +struct vulkan_modifier_info *vulkan_modifierInfo_find(struct vulkan_format_infos *fmtInfo, VkFormat format, uint64_t mod) +{ + struct vulkan_format_info *f_info = vulkan_formatInfo_find(fmtInfo, format); + if (!f_info) + return NULL; + for (uint32_t i = 0; i < f_info->modifierCount; i++) { + if (f_info->infos[i].props.drmFormatModifier == mod) + return &f_info->infos[i]; + } + return NULL; +} + +void vulkan_buffer_clear(struct vulkan_base *s, struct vulkan_buffer *buffer) +{ + if (buffer->fd != -1) + close(buffer->fd); + vkFreeMemory(s->device, buffer->memory, NULL); + vkDestroyImage(s->device, buffer->image, NULL); + vkDestroyImageView(s->device, buffer->view, NULL); +} + +static VkImageAspectFlagBits mem_plane_aspect(uint32_t i) +{ + switch (i) { + case 0: return VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT; + case 1: return VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT; + case 2: return VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT; + case 3: return VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT; + default: abort(); // unreachable + } +} + +int vulkan_staging_buffer_create(struct vulkan_base *s, uint32_t size, struct vulkan_staging_buffer *s_buf) { + VkBufferCreateInfo buf_info = { + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .size = size, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + }; + VK_CHECK_RESULT(vkCreateBuffer(s->device, &buf_info, NULL, &s_buf->buffer)); + + VkMemoryRequirements memoryRequirements; + vkGetBufferMemoryRequirements(s->device, s_buf->buffer, &memoryRequirements); + + VkMemoryAllocateInfo mem_info = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .allocationSize = memoryRequirements.size, + .memoryTypeIndex = vulkan_memoryType_find(s, + memoryRequirements.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT), + }; + VK_CHECK_RESULT(vkAllocateMemory(s->device, &mem_info, NULL, &s_buf->memory)); + VK_CHECK_RESULT(vkBindBufferMemory(s->device, s_buf->buffer, s_buf->memory, 0)); + + return 0; +} + +void vulkan_staging_buffer_destroy(struct vulkan_base *s, struct vulkan_staging_buffer *s_buf) { + if (s_buf->buffer == VK_NULL_HANDLE) + return; + vkFreeMemory(s->device, s_buf->memory, NULL); + vkDestroyBuffer(s->device, s_buf->buffer, NULL); +} + +static int allocate_dmabuf(struct vulkan_base *s, VkFormat format, uint32_t modifierCount, uint64_t *modifiers, VkImageUsageFlags usage, struct spa_rectangle *size, struct vulkan_buffer *vk_buf) +{ + VkImageDrmFormatModifierListCreateInfoEXT imageDrmFormatModifierListCreateInfo = { + .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT, + .drmFormatModifierCount = modifierCount, + .pDrmFormatModifiers = modifiers, + }; + + VkExternalMemoryImageCreateInfo extMemoryImageCreateInfo = { + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO, + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, + }; + extMemoryImageCreateInfo.pNext = &imageDrmFormatModifierListCreateInfo; + + VkImageCreateInfo imageCreateInfo = { + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .imageType = VK_IMAGE_TYPE_2D, + .format = format, + .extent.width = size->width, + .extent.height = size->height, + .extent.depth = 1, + .mipLevels = 1, + .arrayLayers = 1, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT, + .usage = usage, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + }; + imageCreateInfo.pNext = &extMemoryImageCreateInfo; + + VK_CHECK_RESULT(vkCreateImage(s->device, + &imageCreateInfo, NULL, &vk_buf->image)); + + VkMemoryRequirements memoryRequirements = {0}; + vkGetImageMemoryRequirements(s->device, + vk_buf->image, &memoryRequirements); + + VkExportMemoryAllocateInfo exportAllocateInfo = { + .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO, + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, + }; + VkMemoryAllocateInfo allocateInfo = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .allocationSize = memoryRequirements.size, + .memoryTypeIndex = vulkan_memoryType_find(s, + memoryRequirements.memoryTypeBits, + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT), + }; + allocateInfo.pNext = &exportAllocateInfo; + + VK_CHECK_RESULT(vkAllocateMemory(s->device, + &allocateInfo, NULL, &vk_buf->memory)); + + VK_CHECK_RESULT(vkBindImageMemory(s->device, + vk_buf->image, vk_buf->memory, 0)); + + return 0; +} + +int vulkan_fixate_modifier(struct vulkan_base *s, struct dmabuf_fixation_info *info, uint64_t *modifier) +{ + VULKAN_INSTANCE_FUNCTION(vkGetImageDrmFormatModifierPropertiesEXT); + + struct vulkan_buffer vk_buf; + vk_buf.fd = -1; + vk_buf.view = VK_NULL_HANDLE; + VK_CHECK_RESULT(allocate_dmabuf(s, info->format, info->modifierCount, info->modifiers, info->usage, &info->size, &vk_buf)); + + VkImageDrmFormatModifierPropertiesEXT mod_prop = { + .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT, + }; + VK_CHECK_RESULT(vkGetImageDrmFormatModifierPropertiesEXT(s->device, vk_buf.image, &mod_prop)); + + *modifier = mod_prop.drmFormatModifier; + + vulkan_buffer_clear(s, &vk_buf); + + return 0; +} + +int vulkan_create_dmabuf(struct vulkan_base *s, struct external_buffer_info *info, struct vulkan_buffer *vk_buf) +{ + VULKAN_INSTANCE_FUNCTION(vkGetMemoryFdKHR); + + if (info->spa_buf->n_datas != 1) + return -1; + + VK_CHECK_RESULT(allocate_dmabuf(s, info->format, 1, &info->modifier, info->usage, &info->size, vk_buf)); + + const VkMemoryGetFdInfoKHR getFdInfo = { + .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR, + .memory = vk_buf->memory, + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT + }; + int fd = -1; + VK_CHECK_RESULT(vkGetMemoryFdKHR(s->device, &getFdInfo, &fd)); + + /* + const struct vulkan_modifier_info *modInfo = vulkan_modifierInfo_find(s, info->format, info->modifier); + + if (info->spa_buf->n_datas != modInfo->props.drmFormatModifierPlaneCount) + return -1; + */ + + VkMemoryRequirements memoryRequirements = {0}; + vkGetImageMemoryRequirements(s->device, + vk_buf->image, &memoryRequirements); + + spa_log_info(s->log, "export DMABUF %zd", memoryRequirements.size); + + for (uint32_t i = 0; i < info->spa_buf->n_datas; i++) { + VkImageSubresource subresource = { + .aspectMask = mem_plane_aspect(i), + }; + VkSubresourceLayout subresLayout = {0}; + vkGetImageSubresourceLayout(s->device, vk_buf->image, &subresource, &subresLayout); + + info->spa_buf->datas[i].type = SPA_DATA_DmaBuf; + info->spa_buf->datas[i].fd = fd; + info->spa_buf->datas[i].flags = SPA_DATA_FLAG_READABLE; + info->spa_buf->datas[i].mapoffset = 0; + info->spa_buf->datas[i].chunk->offset = subresLayout.offset; + info->spa_buf->datas[i].chunk->stride = subresLayout.rowPitch; + info->spa_buf->datas[i].chunk->size = subresLayout.size; + info->spa_buf->datas[i].maxsize = memoryRequirements.size; + } + vk_buf->fd = fd; + + VkImageViewCreateInfo viewInfo = { + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, + .image = vk_buf->image, + .viewType = VK_IMAGE_VIEW_TYPE_2D, + .format = info->format, + .components.r = VK_COMPONENT_SWIZZLE_R, + .components.g = VK_COMPONENT_SWIZZLE_G, + .components.b = VK_COMPONENT_SWIZZLE_B, + .components.a = VK_COMPONENT_SWIZZLE_A, + .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .subresourceRange.levelCount = 1, + .subresourceRange.layerCount = 1, + }; + + VK_CHECK_RESULT(vkCreateImageView(s->device, + &viewInfo, NULL, &vk_buf->view)); + return 0; +} + +int vulkan_import_dmabuf(struct vulkan_base *s, struct external_buffer_info *info, struct vulkan_buffer *vk_buf) +{ + + if (info->spa_buf->n_datas == 0 || info->spa_buf->n_datas > DMABUF_MAX_PLANES) + return -1; + + /* + struct vulkan_modifier_info *modProps = vulkan_modifierInfo_find(s, info->format, info->modifier); + if (!modProps) + return -1; + + */ + uint32_t planeCount = info->spa_buf->n_datas; + + /* + if (planeCount != modProps->props.drmFormatModifierPlaneCount) + return -1; + + if (info->size.width > modProps->max_extent.width || info->size.height > modProps->max_extent.height) + return -1; + */ + + VkSubresourceLayout planeLayouts[DMABUF_MAX_PLANES] = {0}; + for (uint32_t i = 0; i < planeCount; i++) { + planeLayouts[i].offset = info->spa_buf->datas[i].chunk->offset; + planeLayouts[i].rowPitch = info->spa_buf->datas[i].chunk->stride; + planeLayouts[i].size = 0; + } + + VkImageDrmFormatModifierExplicitCreateInfoEXT modInfo = { + .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT, + .drmFormatModifierPlaneCount = planeCount, + .drmFormatModifier = info->modifier, + .pPlaneLayouts = planeLayouts, + }; + + VkExternalMemoryImageCreateInfo extInfo = { + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO, + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, + .pNext = &modInfo, + }; + + VkImageCreateInfo imageCreateInfo = { + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .imageType = VK_IMAGE_TYPE_2D, + .format = info->format, + .extent.width = info->size.width, + .extent.height = info->size.height, + .extent.depth = 1, + .mipLevels = 1, + .arrayLayers = 1, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT, + .usage = info->usage, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .pNext = &extInfo, + }; + + VK_CHECK_RESULT(vkCreateImage(s->device, + &imageCreateInfo, NULL, &vk_buf->image)); + + VkMemoryRequirements memoryRequirements; + vkGetImageMemoryRequirements(s->device, + vk_buf->image, &memoryRequirements); + + vk_buf->fd = fcntl(info->spa_buf->datas[0].fd, F_DUPFD_CLOEXEC, 0); + VkImportMemoryFdInfoKHR importInfo = { + .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR, + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, + .fd = fcntl(info->spa_buf->datas[0].fd, F_DUPFD_CLOEXEC, 0), + }; + + VkMemoryAllocateInfo allocateInfo = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .allocationSize = memoryRequirements.size, + .memoryTypeIndex = vulkan_memoryType_find(s, + memoryRequirements.memoryTypeBits, + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT), + }; + allocateInfo.pNext = &importInfo; + + spa_log_info(s->log, "import DMABUF"); + + VK_CHECK_RESULT(vkAllocateMemory(s->device, + &allocateInfo, NULL, &vk_buf->memory)); + VK_CHECK_RESULT(vkBindImageMemory(s->device, + vk_buf->image, vk_buf->memory, 0)); + + VkImageViewCreateInfo viewInfo = { + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, + .image = vk_buf->image, + .viewType = VK_IMAGE_VIEW_TYPE_2D, + .format = info->format, + .components.r = VK_COMPONENT_SWIZZLE_R, + .components.g = VK_COMPONENT_SWIZZLE_G, + .components.b = VK_COMPONENT_SWIZZLE_B, + .components.a = VK_COMPONENT_SWIZZLE_A, + .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .subresourceRange.levelCount = 1, + .subresourceRange.layerCount = 1, + }; + + VK_CHECK_RESULT(vkCreateImageView(s->device, + &viewInfo, NULL, &vk_buf->view)); + return 0; +} + +int vulkan_import_memptr(struct vulkan_base *s, struct external_buffer_info *info, struct vulkan_buffer *vk_buf) +{ + VkImageCreateInfo imageCreateInfo = { + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .imageType = VK_IMAGE_TYPE_2D, + .format = info->format, + .extent.width = info->size.width, + .extent.height = info->size.height, + .extent.depth = 1, + .mipLevels = 1, + .arrayLayers = 1, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_LINEAR, + .usage = info->usage, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + }; + + VK_CHECK_RESULT(vkCreateImage(s->device, + &imageCreateInfo, NULL, &vk_buf->image)); + + VkMemoryRequirements memoryRequirements; + vkGetImageMemoryRequirements(s->device, + vk_buf->image, &memoryRequirements); + + VkMemoryAllocateInfo allocateInfo = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .allocationSize = memoryRequirements.size, + .memoryTypeIndex = vulkan_memoryType_find(s, + memoryRequirements.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT), + }; + + vk_buf->fd = -1; + spa_log_info(s->log, "import MemPtr"); + + VK_CHECK_RESULT(vkAllocateMemory(s->device, + &allocateInfo, NULL, &vk_buf->memory)); + VK_CHECK_RESULT(vkBindImageMemory(s->device, + vk_buf->image, vk_buf->memory, 0)); + + VkImageViewCreateInfo viewInfo = { + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, + .image = vk_buf->image, + .viewType = VK_IMAGE_VIEW_TYPE_2D, + .format = info->format, + .components.r = VK_COMPONENT_SWIZZLE_R, + .components.g = VK_COMPONENT_SWIZZLE_G, + .components.b = VK_COMPONENT_SWIZZLE_B, + .components.a = VK_COMPONENT_SWIZZLE_A, + .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .subresourceRange.levelCount = 1, + .subresourceRange.layerCount = 1, + }; + + VK_CHECK_RESULT(vkCreateImageView(s->device, + &viewInfo, NULL, &vk_buf->view)); + return 0; +} + +uint32_t vulkan_vkformat_to_id(VkFormat format) +{ + SPA_FOR_EACH_ELEMENT_VAR(vk_video_format_convs, f) { + if (f->format == format) + return f->id; + } + return SPA_VIDEO_FORMAT_UNKNOWN; +} + +VkFormat vulkan_id_to_vkformat(uint32_t id) +{ + SPA_FOR_EACH_ELEMENT_VAR(vk_video_format_convs, f) { + if (f->id == id) + return f->format; + } + return VK_FORMAT_UNDEFINED; +} + +int vulkan_vkresult_to_errno(VkResult result) +{ + return vkresult_to_errno(result); +} + +int vulkan_wait_fence(struct vulkan_base *s, VkFence fence) +{ + VK_CHECK_RESULT(vkWaitForFences(s->device, 1, &fence, VK_TRUE, UINT64_MAX)); + + return 0; +} + +int vulkan_wait_idle(struct vulkan_base *s) +{ + VK_CHECK_RESULT(vkDeviceWaitIdle(s->device)); + + return 0; +} + +int vulkan_format_infos_init(struct vulkan_base *s, uint32_t formatCount, uint32_t *formats, + struct vulkan_format_infos *info) +{ + if (info->infos) + return 0; + + + info->infos = calloc(formatCount, sizeof(struct vulkan_format_info)); + if (!info->infos) + return -ENOMEM; + + uint32_t i; + for (i = 0; i < formatCount; i++) { + VkFormat format = vulkan_id_to_vkformat(formats[i]); + if (format == VK_FORMAT_UNDEFINED) + continue; + struct vulkan_format_info *f_info = &info->infos[info->formatCount++]; + f_info->spa_format = formats[i]; + f_info->vk_format = format; + spa_log_info(s->log, "Adding format %d (spa_format %d)", format, formats[i]); + + VkDrmFormatModifierPropertiesListEXT modPropsList = { + .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT, + }; + VkFormatProperties2 fmtProps = { + .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2, + .pNext = &modPropsList, + }; + vkGetPhysicalDeviceFormatProperties2(s->physicalDevice, format, &fmtProps); + + if (modPropsList.drmFormatModifierCount == 0) { + spa_log_info(s->log, "Format has no modifiers"); + continue; + } + + modPropsList.pDrmFormatModifierProperties = calloc(modPropsList.drmFormatModifierCount, + sizeof(modPropsList.pDrmFormatModifierProperties[0])); + if (!modPropsList.pDrmFormatModifierProperties) { + spa_log_info(s->log, "Failed to allocate DrmFormatModifierProperties"); + continue; + } + vkGetPhysicalDeviceFormatProperties2(s->physicalDevice, format, &fmtProps); + + f_info->infos = calloc(modPropsList.drmFormatModifierCount, sizeof(f_info->infos[0])); + if (!f_info->infos) { + spa_log_info(s->log, "Failed to allocate modifier infos"); + free(modPropsList.pDrmFormatModifierProperties); + continue; + } + + spa_log_info(s->log, "Found %d modifiers", modPropsList.drmFormatModifierCount); + for (uint32_t j = 0; j < modPropsList.drmFormatModifierCount; j++) { + VkDrmFormatModifierPropertiesEXT props = modPropsList.pDrmFormatModifierProperties[j]; + + if (!(props.drmFormatModifierTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) + continue; + + if (props.drmFormatModifierPlaneCount > DMABUF_MAX_PLANES) + continue; + + VkPhysicalDeviceImageDrmFormatModifierInfoEXT modInfo = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT, + .drmFormatModifier = props.drmFormatModifier, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + }; + VkPhysicalDeviceExternalImageFormatInfo extImgFmtInfo = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO, + .pNext = &modInfo, + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, + }; + VkPhysicalDeviceImageFormatInfo2 imgFmtInfo = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, + .pNext = &extImgFmtInfo, + .type = VK_IMAGE_TYPE_2D, + .format = format, + .usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, + .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT, + }; + + VkExternalImageFormatProperties extImgFmtProps = { + .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES, + }; + VkImageFormatProperties2 imgFmtProps = { + .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2, + .pNext = &extImgFmtProps, + }; + + VK_CHECK_RESULT_LOOP(vkGetPhysicalDeviceImageFormatProperties2(s->physicalDevice, &imgFmtInfo, &imgFmtProps)) + + VkExternalMemoryFeatureFlags extMemFeatures = + extImgFmtProps.externalMemoryProperties.externalMemoryFeatures; + if (!(extMemFeatures & VK_EXTERNAL_MEMORY_FEATURE_EXPORTABLE_BIT)) { + continue; + } + + VkExtent3D max_extent = imgFmtProps.imageFormatProperties.maxExtent; + f_info->infos[f_info->modifierCount++] = (struct vulkan_modifier_info){ + .props = props, + .max_extent = { .width = max_extent.width, .height = max_extent.height }, + }; + spa_log_info(s->log, "Adding modifier %"PRIu64, props.drmFormatModifier); + + } + free(modPropsList.pDrmFormatModifierProperties); + } + for (i = 0; i < info->formatCount; i++) { + if (info->infos[i].modifierCount > 0) + info->formatsWithModifiersCount++; + } + return 0; +} + +void vulkan_format_infos_deinit(struct vulkan_format_infos *info) +{ + for (uint32_t i = 0; i < info->formatCount; i++) { + free(info->infos[i].infos); + } + free(info->infos); +} + +int vulkan_base_init(struct vulkan_base *s, struct vulkan_base_info *info) +{ + if (!s->initialized) { + CHECK(createInstance(s)); + CHECK(findPhysicalDevice(s)); + CHECK(createDevice(s, info)); + s->implicit_sync_interop = dmabuf_check_sync_file_import_export(s->log); + s->initialized = true; + } + return 0; +} + +void vulkan_base_deinit(struct vulkan_base *s) +{ + if (s->initialized) { + vkDestroyDevice(s->device, NULL); + vkDestroyInstance(s->instance, NULL); + s->initialized = false; + } +} diff --git a/spa/plugins/videoconvert/vulkan-utils.h b/spa/plugins/videoconvert/vulkan-utils.h new file mode 100644 index 000000000..06a18d711 --- /dev/null +++ b/spa/plugins/videoconvert/vulkan-utils.h @@ -0,0 +1,120 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2019 Wim Taymans */ +/* SPDX-License-Identifier: MIT */ + +#pragma once + +#include + +#include +#include + +#include "vulkan-types.h" + +#define VK_CHECK_RESULT(f) \ +{ \ + VkResult _result = (f); \ + int _r = -vulkan_vkresult_to_errno(_result); \ + if (_result != VK_SUCCESS) { \ + spa_log_error(s->log, "error: %d (%d %s)", _result, _r, spa_strerror(_r)); \ + return _r; \ + } \ +} +#define VK_CHECK_RESULT_WITH_CLEANUP(f, c) \ +{ \ + VkResult _result = (f); \ + int _r = -vkresult_to_errno(_result); \ + if (_result != VK_SUCCESS) { \ + spa_log_error(s->log, "error: %d (%d %s)", _result, _r, spa_strerror(_r)); \ + (c); \ + return _r; \ + } \ +} +#define VK_CHECK_RESULT_LOOP(f) \ +{ \ + VkResult _result = (f); \ + int _r = -vkresult_to_errno(_result); \ + if (_result != VK_SUCCESS) { \ + spa_log_error(s->log, "error: %d (%d %s)", _result, _r, spa_strerror(_r)); \ + continue; \ + } \ +} +#define CHECK(f) \ +{ \ + int _res = (f); \ + if (_res < 0) \ + return _res; \ +} + +struct vulkan_write_pixels_info { + struct spa_rectangle size; + uint32_t offset; + uint32_t stride; + uint32_t bytes_per_pixel; + + VkBufferImageCopy *copies; + void *data; +}; + +struct vulkan_read_pixels_info { + struct spa_rectangle size; + void *data; + uint32_t offset; + uint32_t stride; + uint32_t bytes_per_pixel; +}; + +struct dmabuf_fixation_info { + VkFormat format; + uint64_t modifierCount; + uint64_t *modifiers; + struct spa_rectangle size; + VkImageUsageFlags usage; +}; + +struct external_buffer_info { + VkFormat format; + uint64_t modifier; + struct spa_rectangle size; + VkImageUsageFlags usage; + struct spa_buffer *spa_buf; +}; + +int vulkan_write_pixels(struct vulkan_base *s, struct vulkan_write_pixels_info *info, struct vulkan_staging_buffer *vk_sbuf); +int vulkan_read_pixels(struct vulkan_base *s, struct vulkan_read_pixels_info *info, struct vulkan_buffer *vk_buf); + +int vulkan_sync_foreign_dmabuf(struct vulkan_base *s, struct vulkan_buffer *vk_buf); +bool vulkan_sync_export_dmabuf(struct vulkan_base *s, struct vulkan_buffer *vk_buf, int sync_file_fd); + +int vulkan_staging_buffer_create(struct vulkan_base *s, uint32_t size, struct vulkan_staging_buffer *s_buf); +void vulkan_staging_buffer_destroy(struct vulkan_base *s, struct vulkan_staging_buffer *s_buf); + +int vulkan_fixate_modifier(struct vulkan_base *s, struct dmabuf_fixation_info *info, uint64_t *modifier); +int vulkan_create_dmabuf(struct vulkan_base *s, struct external_buffer_info *info, struct vulkan_buffer *vk_buf); +int vulkan_import_dmabuf(struct vulkan_base *s, struct external_buffer_info *info, struct vulkan_buffer *vk_buf); +int vulkan_import_memptr(struct vulkan_base *s, struct external_buffer_info *info, struct vulkan_buffer *vk_buf); + +int vulkan_fence_create(struct vulkan_base *s, VkFence *fence); +int vulkan_commandPool_create(struct vulkan_base *s, VkCommandPool *commandPool); +int vulkan_commandBuffer_create(struct vulkan_base *s, VkCommandPool commandPool, VkCommandBuffer *commandBuffer); + +uint32_t vulkan_memoryType_find(struct vulkan_base *s, + uint32_t memoryTypeBits, VkMemoryPropertyFlags properties); +struct vulkan_format_info *vulkan_formatInfo_find(struct vulkan_format_infos *fmtInfo, VkFormat format); +struct vulkan_modifier_info *vulkan_modifierInfo_find(struct vulkan_format_infos *fmtInfo, VkFormat format, uint64_t modifier); + +void vulkan_buffer_clear(struct vulkan_base *s, struct vulkan_buffer *buffer); + +uint32_t vulkan_vkformat_to_id(VkFormat vkFormat); +VkFormat vulkan_id_to_vkformat(uint32_t id); + +int vulkan_vkresult_to_errno(VkResult result); + +int vulkan_wait_fence(struct vulkan_base *s, VkFence fence); +int vulkan_wait_idle(struct vulkan_base *s); + +int vulkan_format_infos_init(struct vulkan_base *s, uint32_t formatCount, uint32_t *formats, + struct vulkan_format_infos *info); +void vulkan_format_infos_deinit(struct vulkan_format_infos *info); +int vulkan_base_init(struct vulkan_base *s, struct vulkan_base_info *info); +void vulkan_base_deinit(struct vulkan_base *s); diff --git a/spa/plugins/vulkan/meson.build b/spa/plugins/vulkan/meson.build index 62dc32123..6f6b4dc75 100644 --- a/spa/plugins/vulkan/meson.build +++ b/spa/plugins/vulkan/meson.build @@ -1,9 +1,13 @@ spa_vulkan_sources = [ 'plugin.c', + 'pixel-formats.c', 'vulkan-compute-filter.c', 'vulkan-compute-source.c', 'vulkan-compute-utils.c', - 'vulkan-utils.c' + 'vulkan-blit-filter.c', + 'vulkan-blit-utils.c', + 'vulkan-utils.c', + 'utils.c' ] drm = dependency('libdrm') diff --git a/spa/plugins/vulkan/pixel-formats.c b/spa/plugins/vulkan/pixel-formats.c new file mode 100644 index 000000000..06144d758 --- /dev/null +++ b/spa/plugins/vulkan/pixel-formats.c @@ -0,0 +1,33 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include "pixel-formats.h" + +#include +#include + +struct pixel_info { + uint32_t format; + uint32_t bpp; +} pixel_infos[] = { + { SPA_VIDEO_FORMAT_RGBA_F32, 16 }, + { SPA_VIDEO_FORMAT_BGRA, 4 }, + { SPA_VIDEO_FORMAT_RGBA, 4 }, + { SPA_VIDEO_FORMAT_BGRx, 4 }, + { SPA_VIDEO_FORMAT_RGBx, 4 }, + { SPA_VIDEO_FORMAT_BGR, 3 }, + { SPA_VIDEO_FORMAT_RGB, 3 }, +}; + +bool get_pixel_format_info(uint32_t format, struct pixel_format_info *info) +{ + struct pixel_info *p; + SPA_FOR_EACH_ELEMENT(pixel_infos, p) { + if (p->format != format) + continue; + info->bpp = p->bpp; + return true; + } + return false; +} diff --git a/spa/plugins/vulkan/pixel-formats.h b/spa/plugins/vulkan/pixel-formats.h new file mode 100644 index 000000000..949733107 --- /dev/null +++ b/spa/plugins/vulkan/pixel-formats.h @@ -0,0 +1,12 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include +#include + +struct pixel_format_info { + uint32_t bpp; // bytes per pixel +}; + +bool get_pixel_format_info(uint32_t format, struct pixel_format_info *info); diff --git a/spa/plugins/vulkan/plugin.c b/spa/plugins/vulkan/plugin.c index 4d2608f51..8e0f86cb8 100644 --- a/spa/plugins/vulkan/plugin.c +++ b/spa/plugins/vulkan/plugin.c @@ -8,6 +8,7 @@ extern const struct spa_handle_factory spa_vulkan_compute_filter_factory; extern const struct spa_handle_factory spa_vulkan_compute_source_factory; +extern const struct spa_handle_factory spa_vulkan_blit_filter_factory; SPA_EXPORT int spa_handle_factory_enum(const struct spa_handle_factory **factory, uint32_t *index) @@ -22,6 +23,9 @@ int spa_handle_factory_enum(const struct spa_handle_factory **factory, uint32_t case 1: *factory = &spa_vulkan_compute_filter_factory; break; + case 2: + *factory = &spa_vulkan_blit_filter_factory; + break; default: return 0; } diff --git a/spa/plugins/vulkan/utils.c b/spa/plugins/vulkan/utils.c new file mode 100644 index 000000000..43bca2848 --- /dev/null +++ b/spa/plugins/vulkan/utils.c @@ -0,0 +1,88 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include "utils.h" + +#include +#include +#include + +// This function enumerates the available formats in vulkan_state::formats, announcing all formats capable to support DmaBufs +// first and then falling back to those supported with SHM buffers. +bool find_EnumFormatInfo(struct vulkan_format_infos *fmtInfos, uint32_t index, uint32_t caps, uint32_t *fmt_idx, bool *has_modifier) { + int64_t fmtIterator = 0; + int64_t maxIterator = 0; + if (caps & VULKAN_BUFFER_TYPE_CAP_SHM) + maxIterator += fmtInfos->formatCount; + if (caps & VULKAN_BUFFER_TYPE_CAP_DMABUF) + maxIterator += fmtInfos->formatCount; + // Count available formats until index underflows, while fmtIterator indexes the current format. + // Iterate twice over formats first time with modifiers, second time without if both caps are supported. + while (index < (uint32_t)-1 && fmtIterator < maxIterator) { + const struct vulkan_format_info *f_info = &fmtInfos->infos[fmtIterator%fmtInfos->formatCount]; + if (caps & VULKAN_BUFFER_TYPE_CAP_DMABUF && fmtIterator < fmtInfos->formatCount) { + // First round, check for modifiers + if (f_info->modifierCount > 0) { + index--; + } + } else if (caps & VULKAN_BUFFER_TYPE_CAP_SHM) { + // Second round, every format should be supported. + index--; + } + fmtIterator++; + } + + if (index != (uint32_t)-1) { + // No more formats available + return false; + } + // Undo end of loop increment + fmtIterator--; + *fmt_idx = fmtIterator%fmtInfos->formatCount; + // Loop finished in first round + *has_modifier = caps & VULKAN_BUFFER_TYPE_CAP_DMABUF && fmtIterator < fmtInfos->formatCount; + return true; +} + +struct spa_pod *build_dsp_EnumFormat(const struct vulkan_format_info *fmt, bool with_modifiers, struct spa_pod_builder *builder) { + struct spa_pod_frame f[2]; + uint32_t i, c; + + spa_pod_builder_push_object(builder, &f[0], SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat); + spa_pod_builder_add(builder, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), 0); + spa_pod_builder_add(builder, SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_dsp), 0); + spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_format, SPA_POD_Id(fmt->spa_format), 0); + if (with_modifiers && fmt->modifierCount > 0) { + spa_pod_builder_prop(builder, SPA_FORMAT_VIDEO_modifier, SPA_POD_PROP_FLAG_MANDATORY | SPA_POD_PROP_FLAG_DONT_FIXATE); + spa_pod_builder_push_choice(builder, &f[1], SPA_CHOICE_Enum, 0); + for (i = 0, c = 0; i < fmt->modifierCount; i++) { + spa_pod_builder_long(builder, fmt->infos[i].props.drmFormatModifier); + if (c++ == 0) + spa_pod_builder_long(builder, fmt->infos[i].props.drmFormatModifier); + } + spa_pod_builder_pop(builder, &f[1]); + } + return spa_pod_builder_pop(builder, &f[0]); +} + +struct spa_pod *build_raw_EnumFormat(const struct vulkan_format_info *fmt, bool with_modifiers, struct spa_pod_builder *builder) { + struct spa_pod_frame f[2]; + uint32_t i, c; + + spa_pod_builder_push_object(builder, &f[0], SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat); + spa_pod_builder_add(builder, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), 0); + spa_pod_builder_add(builder, SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), 0); + spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_format, SPA_POD_Id(fmt->spa_format), 0); + if (with_modifiers && fmt->modifierCount > 0) { + spa_pod_builder_prop(builder, SPA_FORMAT_VIDEO_modifier, SPA_POD_PROP_FLAG_MANDATORY | SPA_POD_PROP_FLAG_DONT_FIXATE); + spa_pod_builder_push_choice(builder, &f[1], SPA_CHOICE_Enum, 0); + for (i = 0, c = 0; i < fmt->modifierCount; i++) { + spa_pod_builder_long(builder, fmt->infos[i].props.drmFormatModifier); + if (c++ == 0) + spa_pod_builder_long(builder, fmt->infos[i].props.drmFormatModifier); + } + spa_pod_builder_pop(builder, &f[1]); + } + return spa_pod_builder_pop(builder, &f[0]); +} diff --git a/spa/plugins/vulkan/utils.h b/spa/plugins/vulkan/utils.h new file mode 100644 index 000000000..9716804a3 --- /dev/null +++ b/spa/plugins/vulkan/utils.h @@ -0,0 +1,11 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include "vulkan-types.h" +#include "spa/pod/builder.h" + +bool find_EnumFormatInfo(struct vulkan_format_infos *fmtInfos, uint32_t index, uint32_t caps, uint32_t *fmt_idx, bool *has_modifier); + +struct spa_pod *build_dsp_EnumFormat(const struct vulkan_format_info *fmt, bool with_modifiers, struct spa_pod_builder *builder); +struct spa_pod *build_raw_EnumFormat(const struct vulkan_format_info *fmt, bool with_modifiers, struct spa_pod_builder *builder); diff --git a/spa/plugins/vulkan/vulkan-blit-filter.c b/spa/plugins/vulkan/vulkan-blit-filter.c new file mode 100644 index 000000000..d5bb64cc2 --- /dev/null +++ b/spa/plugins/vulkan/vulkan-blit-filter.c @@ -0,0 +1,995 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2019 Wim Taymans */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pixel-formats.h" +#include "vulkan-blit-utils.h" + +#define NAME "vulkan-blit-filter" + +struct buffer { + uint32_t id; +#define BUFFER_FLAG_OUT (1<<0) + uint32_t flags; + struct spa_buffer *outbuf; + struct spa_meta_header *h; + struct spa_list link; +}; + +struct port { + uint64_t info_all; + struct spa_port_info info; + + enum spa_direction direction; + struct spa_param_info params[5]; + + struct spa_io_buffers *io; + + bool have_format; + struct spa_video_info current_format; + + struct buffer buffers[MAX_BUFFERS]; + uint32_t n_buffers; + + struct spa_list empty; + struct spa_list ready; + uint32_t stream_id; +}; + +struct impl { + struct spa_handle handle; + struct spa_node node; + + struct spa_log *log; + + struct spa_io_position *position; + + uint64_t info_all; + struct spa_node_info info; + struct spa_param_info params[2]; + + struct spa_hook_list hooks; + struct spa_callbacks callbacks; + + bool started; + + struct vulkan_blit_state state; + struct port port[2]; +}; + +#define CHECK_PORT(this,d,p) ((p) < 1) + +static int impl_node_enum_params(void *object, int seq, + uint32_t id, uint32_t start, uint32_t num, + const struct spa_pod *filter) +{ + struct impl *this = object; + struct spa_pod *param; + struct spa_pod_builder b = { 0 }; + uint8_t buffer[1024]; + struct spa_result_node_params result; + uint32_t count = 0; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(num != 0, -EINVAL); + + result.id = id; + result.next = start; + next: + result.index = result.next++; + + spa_pod_builder_init(&b, buffer, sizeof(buffer)); + + switch (id) { + default: + return -ENOENT; + } + + if (spa_pod_filter(&b, &result.param, param, filter) < 0) + goto next; + + spa_node_emit_result(&this->hooks, seq, 0, SPA_RESULT_TYPE_NODE_PARAMS, &result); + + if (++count != num) + goto next; + + return 0; +} + +static int impl_node_set_io(void *object, uint32_t id, void *data, size_t size) +{ + struct impl *this = object; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + switch (id) { + case SPA_IO_Position: + if (size > 0 && size < sizeof(struct spa_io_position)) + return -EINVAL; + this->position = data; + break; + default: + return -ENOENT; + } + return 0; +} +static int impl_node_set_param(void *object, uint32_t id, uint32_t flags, + const struct spa_pod *param) +{ + struct impl *this = object; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + switch (id) { + default: + return -ENOENT; + } + return 0; +} + +static inline void reuse_buffer(struct impl *this, struct port *port, uint32_t id) +{ + struct buffer *b = &port->buffers[id]; + + if (SPA_FLAG_IS_SET(b->flags, BUFFER_FLAG_OUT)) { + spa_log_debug(this->log, NAME " %p: reuse buffer %d", this, id); + + SPA_FLAG_CLEAR(b->flags, BUFFER_FLAG_OUT); + spa_list_append(&port->empty, &b->link); + } +} + +static int impl_node_send_command(void *object, const struct spa_command *command) +{ + struct impl *this = object; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(command != NULL, -EINVAL); + + switch (SPA_NODE_COMMAND_ID(command)) { + case SPA_NODE_COMMAND_Start: + if (this->started) + return 0; + + this->started = true; + spa_vulkan_blit_start(&this->state); + break; + + case SPA_NODE_COMMAND_Suspend: + case SPA_NODE_COMMAND_Pause: + if (!this->started) + return 0; + + this->started = false; + spa_vulkan_blit_stop(&this->state); + break; + default: + return -ENOTSUP; + } + return 0; +} + +static const struct spa_dict_item node_info_items[] = { + { SPA_KEY_MEDIA_CLASS, "Video/Filter" }, +}; + +static void emit_node_info(struct impl *this, bool full) +{ + uint64_t old = full ? this->info.change_mask : 0; + if (full) + this->info.change_mask = this->info_all; + if (this->info.change_mask) { + this->info.props = &SPA_DICT_INIT_ARRAY(node_info_items); + spa_node_emit_info(&this->hooks, &this->info); + this->info.change_mask = old; + } +} + +static void emit_port_info(struct impl *this, struct port *port, bool full) +{ + uint64_t old = full ? port->info.change_mask : 0; + if (full) + port->info.change_mask = port->info_all; + if (port->info.change_mask) { + struct spa_dict_item items[1]; + + items[0] = SPA_DICT_ITEM_INIT(SPA_KEY_FORMAT_DSP, "32 bit float RGBA video"); + port->info.props = &SPA_DICT_INIT(items, 1); + spa_node_emit_port_info(&this->hooks, + port->direction, 0, &port->info); + port->info.change_mask = old; + } +} + +static int +impl_node_add_listener(void *object, + struct spa_hook *listener, + const struct spa_node_events *events, + void *data) +{ + struct impl *this = object; + struct spa_hook_list save; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + spa_hook_list_isolate(&this->hooks, &save, listener, events, data); + + emit_node_info(this, true); + emit_port_info(this, &this->port[0], true); + emit_port_info(this, &this->port[1], true); + + spa_hook_list_join(&this->hooks, &save); + + return 0; +} + +static int +impl_node_set_callbacks(void *object, + const struct spa_node_callbacks *callbacks, + void *data) +{ + struct impl *this = object; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + this->callbacks = SPA_CALLBACKS_INIT(callbacks, data); + + return 0; +} + +static int impl_node_add_port(void *object, enum spa_direction direction, uint32_t port_id, + const struct spa_dict *props) +{ + return -ENOTSUP; +} + +static int +impl_node_remove_port(void *object, enum spa_direction direction, uint32_t port_id) +{ + return -ENOTSUP; +} + +static bool port_has_fixated_format(struct port *p) +{ + if (!p->have_format) + return false; + switch (p->current_format.media_subtype) { + case SPA_MEDIA_SUBTYPE_dsp: + return p->current_format.info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER + && p->current_format.info.dsp.flags ^ SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED; + case SPA_MEDIA_SUBTYPE_raw: + return p->current_format.info.raw.flags & SPA_VIDEO_FLAG_MODIFIER + && p->current_format.info.raw.flags ^ SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED; + } + return false; +} + +static int port_enum_formats(void *object, + enum spa_direction direction, uint32_t port_id, + uint32_t index, + const struct spa_pod *filter, + struct spa_pod **param, + struct spa_pod_builder *builder) +{ + struct impl *this = object; + + if (port_has_fixated_format(&this->port[port_id])) { + if (index == 0) { + if (this->port[port_id].current_format.media_subtype == SPA_MEDIA_SUBTYPE_dsp) { + spa_log_info(this->log, NAME": enum_formats fixated format idx: %d, format %d, has_modifier 1", + index, this->port[port_id].current_format.info.dsp.format); + *param = spa_format_video_dsp_build(builder, SPA_PARAM_EnumFormat, &this->port[port_id].current_format.info.dsp); + } else { + spa_log_info(this->log, NAME": enum_formats fixated format idx: %d, format %d, has_modifier 1", + index, this->port[port_id].current_format.info.raw.format); + *param = spa_format_video_raw_build(builder, SPA_PARAM_EnumFormat, &this->port[port_id].current_format.info.raw); + } + return 1; + } + return spa_vulkan_blit_enumerate_formats(&this->state, index-1, spa_vulkan_blit_get_buffer_caps(&this->state, direction), param, builder); + } else { + return spa_vulkan_blit_enumerate_formats(&this->state, index, spa_vulkan_blit_get_buffer_caps(&this->state, direction), param, builder); + } +} + +static int port_get_buffer_props(struct impl *this, struct port *port, + uint32_t *blocks, uint32_t *size, uint32_t *stride, bool *is_dmabuf) +{ + if (port->current_format.media_subtype == SPA_MEDIA_SUBTYPE_dsp) { + if (this->position == NULL) + return -EIO; + + spa_log_debug(this->log, NAME" %p: %dx%d stride %d", this, + this->position->video.size.width, + this->position->video.size.height, + this->position->video.stride); + + if (port->current_format.info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER) { + *is_dmabuf = true; + + struct vulkan_modifier_info *mod_info = spa_vulkan_blit_get_modifier_info(&this->state, + &port->current_format); + *blocks = mod_info->props.drmFormatModifierPlaneCount; + } else { + *is_dmabuf = false; + *blocks = 1; + *size = this->position->video.stride * this->position->video.size.height; + *stride = this->position->video.stride; + } + return 0; + } else if (port->current_format.media_subtype == SPA_MEDIA_SUBTYPE_raw) { + spa_log_debug(this->log, NAME" %p: %dx%d", this, + port->current_format.info.raw.size.width, + port->current_format.info.raw.size.height); + + if (port->current_format.info.raw.flags & SPA_VIDEO_FLAG_MODIFIER) { + *is_dmabuf = true; + + struct vulkan_modifier_info *mod_info = spa_vulkan_blit_get_modifier_info(&this->state, + &port->current_format); + *blocks = mod_info->props.drmFormatModifierPlaneCount; + } else { + struct pixel_format_info pInfo = {0}; + if (!get_pixel_format_info(port->current_format.info.raw.format, &pInfo)) + return -EINVAL; + uint32_t buffer_stride = pInfo.bpp * port->current_format.info.raw.size.width; + *is_dmabuf = false; + *blocks = 1; + *size = buffer_stride * port->current_format.info.raw.size.height; + *stride = buffer_stride; + } + return 0; + } else { + return -EINVAL; + } +} + +static int +impl_node_port_enum_params(void *object, int seq, + enum spa_direction direction, uint32_t port_id, + uint32_t id, uint32_t start, uint32_t num, + const struct spa_pod *filter) +{ + struct impl *this = object; + struct port *port; + struct spa_pod_builder b = { 0 }; + uint8_t buffer[1024]; + struct spa_pod *param; + struct spa_result_node_params result; + uint32_t count = 0; + int res; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(num != 0, -EINVAL); + + spa_return_val_if_fail(CHECK_PORT(this, direction, port_id), -EINVAL); + port = &this->port[direction]; + + result.id = id; + result.next = start; + next: + result.index = result.next++; + + spa_pod_builder_init(&b, buffer, sizeof(buffer)); + + switch (id) { + case SPA_PARAM_EnumFormat: + if ((res = port_enum_formats(this, direction, port_id, + result.index, filter, ¶m, &b)) <= 0) + return res; + break; + + case SPA_PARAM_Format: + if (!port->have_format) + return -EIO; + if (result.index > 0) + return 0; + + if (port->current_format.media_subtype == SPA_MEDIA_SUBTYPE_dsp) { + param = spa_format_video_dsp_build(&b, id, &port->current_format.info.dsp); + } else if (port->current_format.media_subtype == SPA_MEDIA_SUBTYPE_raw) { + param = spa_format_video_raw_build(&b, id, &port->current_format.info.raw); + } else { + return -EINVAL; + } + break; + + case SPA_PARAM_Buffers: + { + if (!port->have_format) + return -EIO; + if (result.index > 0) + return 0; + + int ret; + uint32_t blocks, size, stride; + bool is_dmabuf; + + if ((ret = port_get_buffer_props(this, port, &blocks, &size, &stride, &is_dmabuf)) < 0) + return ret; + + if (is_dmabuf) { + param = spa_pod_builder_add_object(&b, + SPA_TYPE_OBJECT_ParamBuffers, id, + SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(2, 1, MAX_BUFFERS), + SPA_PARAM_BUFFERS_blocks, SPA_POD_Int(blocks), + SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int(1<hooks, seq, 0, SPA_RESULT_TYPE_NODE_PARAMS, &result); + + if (++count != num) + goto next; + + return 0; +} + +static int clear_buffers(struct impl *this, struct port *port) +{ + if (port->n_buffers > 0) { + spa_log_debug(this->log, NAME " %p: clear buffers", this); + spa_vulkan_blit_stop(&this->state); + spa_vulkan_blit_use_buffers(&this->state, &this->state.streams[port->stream_id], 0, &port->current_format, 0, NULL); + port->n_buffers = 0; + spa_list_init(&port->empty); + spa_list_init(&port->ready); + this->started = false; + } + return 0; +} + +static int port_set_dsp_format(struct impl *this, struct port *port, + uint32_t flags, struct spa_video_info *info, + bool *has_modifier, bool *modifier_fixed, + const struct spa_pod *format) +{ + if (spa_format_video_dsp_parse(format, &info->info.dsp) < 0) + return -EINVAL; + + if (info->info.dsp.format != SPA_VIDEO_FORMAT_DSP_F32) + return -EINVAL; + + this->state.streams[port->stream_id].dim.width = this->position->video.size.width; + this->state.streams[port->stream_id].dim.height = this->position->video.size.height; + this->state.streams[port->stream_id].bpp = 16; + *has_modifier = SPA_FLAG_IS_SET(info->info.dsp.flags, SPA_VIDEO_FLAG_MODIFIER); + + // fixate modifier + if (port->direction == SPA_DIRECTION_OUTPUT + && info->info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER + && info->info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED) { + const struct spa_pod_prop *mod_prop; + if ((mod_prop = spa_pod_find_prop(format, NULL, SPA_FORMAT_VIDEO_modifier)) == NULL) + return -EINVAL; + + const struct spa_pod *mod_pod = &mod_prop->value; + uint32_t modifierCount = SPA_POD_CHOICE_N_VALUES(mod_pod); + uint64_t *modifiers = SPA_POD_CHOICE_VALUES(mod_pod); + if (modifierCount <= 1) + return -EINVAL; + // SPA_POD_CHOICE carries the "preferred" value at position 0 + modifierCount -= 1; + modifiers++; + uint64_t fixed_modifier; + if (spa_vulkan_blit_fixate_modifier(&this->state, &this->state.streams[port->stream_id], info, modifierCount, modifiers, &fixed_modifier) != 0) + return -EINVAL; + + spa_log_info(this->log, NAME ": modifier fixated %"PRIu64, fixed_modifier); + + info->info.dsp.modifier = fixed_modifier; + info->info.dsp.flags &= ~SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED; + *modifier_fixed = true; + } + + return 0; +} + +static int port_set_raw_format(struct impl *this, struct port *port, + uint32_t flags, struct spa_video_info *info, + bool *has_modifier, bool *modifier_fixed, + const struct spa_pod *format) +{ + if (spa_format_video_raw_parse(format, &info->info.raw) < 0) + return -EINVAL; + + struct pixel_format_info pInfo; + if (!get_pixel_format_info(info->info.raw.format, &pInfo)) + return -EINVAL; + this->state.streams[port->stream_id].dim = info->info.raw.size; + this->state.streams[port->stream_id].bpp = pInfo.bpp; + *has_modifier = SPA_FLAG_IS_SET(info->info.raw.flags, SPA_VIDEO_FLAG_MODIFIER); + + // fixate modifier + if (port->direction == SPA_DIRECTION_OUTPUT + && info->info.raw.flags & SPA_VIDEO_FLAG_MODIFIER + && info->info.raw.flags & SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED) { + const struct spa_pod_prop *mod_prop; + if ((mod_prop = spa_pod_find_prop(format, NULL, SPA_FORMAT_VIDEO_modifier)) == NULL) + return -EINVAL; + + const struct spa_pod *mod_pod = &mod_prop->value; + uint32_t modifierCount = SPA_POD_CHOICE_N_VALUES(mod_pod); + uint64_t *modifiers = SPA_POD_CHOICE_VALUES(mod_pod); + if (modifierCount <= 1) + return -EINVAL; + // SPA_POD_CHOICE carries the "preferred" value at position 0 + modifierCount -= 1; + modifiers++; + uint64_t fixed_modifier; + if (spa_vulkan_blit_fixate_modifier(&this->state, &this->state.streams[port->stream_id], info, modifierCount, modifiers, &fixed_modifier) != 0) + return -EINVAL; + + spa_log_info(this->log, NAME ": modifier fixated %"PRIu64, fixed_modifier); + + info->info.raw.modifier = fixed_modifier; + info->info.raw.flags &= ~SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED; + *modifier_fixed = true; + } + + return 0; +} + +static int port_set_format(struct impl *this, struct port *port, + uint32_t flags, + const struct spa_pod *format) +{ + int res; + + if (format == NULL) { + port->have_format = false; + clear_buffers(this, port); + spa_vulkan_blit_unprepare(&this->state); + } else { + struct spa_video_info info = { 0 }; + + if ((res = spa_format_parse(format, &info.media_type, &info.media_subtype)) < 0) + return res; + + if (info.media_type != SPA_MEDIA_TYPE_video) + return -EINVAL; + + bool has_modifier = false; + bool modifier_fixed = false; + if (info.media_subtype == SPA_MEDIA_SUBTYPE_dsp) { + if ((res = port_set_dsp_format(this, port, flags, &info, &has_modifier, &modifier_fixed, format)) < 0) + return res; + } else if (info.media_subtype == SPA_MEDIA_SUBTYPE_raw) { + if ((res = port_set_raw_format(this, port, flags, &info, &has_modifier, &modifier_fixed, format)) < 0) + return res; + } else { + return -EINVAL; + } + + if (has_modifier) { + SPA_FLAG_SET(port->info.flags, SPA_PORT_FLAG_CAN_ALLOC_BUFFERS); + } else { + SPA_FLAG_CLEAR(port->info.flags, SPA_PORT_FLAG_CAN_ALLOC_BUFFERS); + } + port->info.change_mask |= SPA_PORT_CHANGE_MASK_FLAGS; + + port->current_format = info; + port->have_format = true; + + if (modifier_fixed) { + port->info.change_mask |= SPA_PORT_CHANGE_MASK_PARAMS; + port->params[0].flags ^= SPA_PARAM_INFO_SERIAL; + emit_port_info(this, port, false); + return 0; + } + } + + port->info.change_mask |= SPA_PORT_CHANGE_MASK_PARAMS; + if (port->have_format) { + port->params[3] = SPA_PARAM_INFO(SPA_PARAM_Format, SPA_PARAM_INFO_READWRITE); + port->params[4] = SPA_PARAM_INFO(SPA_PARAM_Buffers, SPA_PARAM_INFO_READ); + } else { + port->params[3] = SPA_PARAM_INFO(SPA_PARAM_Format, SPA_PARAM_INFO_WRITE); + port->params[4] = SPA_PARAM_INFO(SPA_PARAM_Buffers, 0); + } + emit_port_info(this, port, false); + + return 0; +} + +static int +impl_node_port_set_param(void *object, + enum spa_direction direction, uint32_t port_id, + uint32_t id, uint32_t flags, + const struct spa_pod *param) +{ + struct impl *this = object; + struct port *port; + int res; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(CHECK_PORT(node, direction, port_id), -EINVAL); + port = &this->port[direction]; + + switch (id) { + case SPA_PARAM_Format: + res = port_set_format(this, port, flags, param); + break; + default: + return -ENOENT; + } + return res; +} + +static int +impl_node_port_use_buffers(void *object, + enum spa_direction direction, + uint32_t port_id, + uint32_t flags, + struct spa_buffer **buffers, + uint32_t n_buffers) +{ + struct impl *this = object; + struct port *port; + uint32_t i; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(CHECK_PORT(this, direction, port_id), -EINVAL); + port = &this->port[direction]; + + clear_buffers(this, port); + + if (n_buffers > 0 && !port->have_format) + return -EIO; + if (n_buffers > MAX_BUFFERS) + return -ENOSPC; + + for (i = 0; i < n_buffers; i++) { + struct buffer *b; + + b = &port->buffers[i]; + b->id = i; + b->outbuf = buffers[i]; + b->flags = 0; + b->h = spa_buffer_find_meta_data(buffers[i], SPA_META_Header, sizeof(*b->h)); + + spa_log_info(this->log, "%p: %d:%d add buffer %p", port, direction, port_id, b); + spa_list_append(&port->empty, &b->link); + } + spa_vulkan_blit_use_buffers(&this->state, &this->state.streams[port->stream_id], flags, &port->current_format, n_buffers, buffers); + port->n_buffers = n_buffers; + + return 0; +} + +static int +impl_node_port_set_io(void *object, + enum spa_direction direction, + uint32_t port_id, + uint32_t id, + void *data, size_t size) +{ + struct impl *this = object; + struct port *port; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(CHECK_PORT(this, direction, port_id), -EINVAL); + port = &this->port[direction]; + + switch (id) { + case SPA_IO_Buffers: + port->io = data; + break; + default: + return -ENOENT; + } + return 0; +} + +static int impl_node_port_reuse_buffer(void *object, uint32_t port_id, uint32_t buffer_id) +{ + struct impl *this = object; + struct port *port; + + spa_return_val_if_fail(this != NULL, -EINVAL); + spa_return_val_if_fail(port_id == 0, -EINVAL); + + port = &this->port[SPA_DIRECTION_OUTPUT]; + spa_return_val_if_fail(buffer_id < port->n_buffers, -EINVAL); + + reuse_buffer(this, port, buffer_id); + + return 0; +} + +static int impl_node_process(void *object) +{ + struct impl *this = object; + struct port *inport, *outport; + struct spa_io_buffers *inio, *outio; + struct buffer *b; + + spa_return_val_if_fail(this != NULL, -EINVAL); + + inport = &this->port[SPA_DIRECTION_INPUT]; + if ((inio = inport->io) == NULL) + return -EIO; + + if (inio->status != SPA_STATUS_HAVE_DATA) + return inio->status; + + if (inio->buffer_id >= inport->n_buffers) { + inio->status = -EINVAL; + return -EINVAL; + } + + outport = &this->port[SPA_DIRECTION_OUTPUT]; + if ((outio = outport->io) == NULL) + return -EIO; + + if (outio->status == SPA_STATUS_HAVE_DATA) + return SPA_STATUS_HAVE_DATA; + + if (outio->buffer_id < outport->n_buffers) { + reuse_buffer(this, outport, outio->buffer_id); + outio->buffer_id = SPA_ID_INVALID; + } + + if (spa_list_is_empty(&outport->empty)) { + spa_log_debug(this->log, NAME " %p: out of buffers", this); + return -EPIPE; + } + b = &inport->buffers[inio->buffer_id]; + this->state.streams[inport->stream_id].pending_buffer_id = b->id; + inio->status = SPA_STATUS_NEED_DATA; + + b = spa_list_first(&outport->empty, struct buffer, link); + spa_list_remove(&b->link); + SPA_FLAG_SET(b->flags, BUFFER_FLAG_OUT); + this->state.streams[outport->stream_id].pending_buffer_id = b->id; + + spa_log_debug(this->log, "filter into %d", b->id); + + spa_vulkan_blit_process(&this->state); + + b->outbuf->datas[0].chunk->offset = 0; + b->outbuf->datas[0].chunk->size = b->outbuf->datas[0].maxsize; + if (outport->current_format.media_subtype == SPA_MEDIA_SUBTYPE_raw) { + b->outbuf->datas[0].chunk->stride = + this->state.streams[outport->stream_id].bpp * outport->current_format.info.raw.size.width; + } else { + b->outbuf->datas[0].chunk->stride = this->position->video.stride; + } + + outio->buffer_id = b->id; + outio->status = SPA_STATUS_HAVE_DATA; + + return SPA_STATUS_NEED_DATA | SPA_STATUS_HAVE_DATA; +} + +static const struct spa_node_methods impl_node = { + SPA_VERSION_NODE_METHODS, + .add_listener = impl_node_add_listener, + .set_callbacks = impl_node_set_callbacks, + .enum_params = impl_node_enum_params, + .set_param = impl_node_set_param, + .set_io = impl_node_set_io, + .send_command = impl_node_send_command, + .add_port = impl_node_add_port, + .remove_port = impl_node_remove_port, + .port_enum_params = impl_node_port_enum_params, + .port_set_param = impl_node_port_set_param, + .port_use_buffers = impl_node_port_use_buffers, + .port_set_io = impl_node_port_set_io, + .port_reuse_buffer = impl_node_port_reuse_buffer, + .process = impl_node_process, +}; + +static int impl_get_interface(struct spa_handle *handle, const char *type, void **interface) +{ + struct impl *this; + + spa_return_val_if_fail(handle != NULL, -EINVAL); + spa_return_val_if_fail(interface != NULL, -EINVAL); + + this = (struct impl *) handle; + + if (spa_streq(type, SPA_TYPE_INTERFACE_Node)) + *interface = &this->node; + else + return -ENOENT; + + return 0; +} + +static int impl_clear(struct spa_handle *handle) +{ + struct impl *this; + + spa_return_val_if_fail(handle != NULL, -EINVAL); + + this = (struct impl *) handle; + + spa_vulkan_blit_deinit(&this->state); + return 0; +} + +static size_t +impl_get_size(const struct spa_handle_factory *factory, + const struct spa_dict *params) +{ + return sizeof(struct impl); +} + +static int +impl_init(const struct spa_handle_factory *factory, + struct spa_handle *handle, + const struct spa_dict *info, + const struct spa_support *support, + uint32_t n_support) +{ + struct impl *this; + struct port *port; + + spa_return_val_if_fail(factory != NULL, -EINVAL); + spa_return_val_if_fail(handle != NULL, -EINVAL); + + handle->get_interface = impl_get_interface; + handle->clear = impl_clear; + + this = (struct impl *) handle; + + this->log = spa_support_find(support, n_support, SPA_TYPE_INTERFACE_Log); + this->state.log = this->log; + + spa_hook_list_init(&this->hooks); + + this->node.iface = SPA_INTERFACE_INIT( + SPA_TYPE_INTERFACE_Node, + SPA_VERSION_NODE, + &impl_node, this); + + this->info_all = SPA_NODE_CHANGE_MASK_FLAGS | + SPA_NODE_CHANGE_MASK_PROPS | + SPA_NODE_CHANGE_MASK_PARAMS; + this->info = SPA_NODE_INFO_INIT(); + this->info.max_output_ports = 1; + this->info.max_input_ports = 1; + this->info.flags = SPA_NODE_FLAG_RT; + this->params[0] = SPA_PARAM_INFO(SPA_PARAM_PropInfo, SPA_PARAM_INFO_READ); + this->params[1] = SPA_PARAM_INFO(SPA_PARAM_Props, SPA_PARAM_INFO_READWRITE); + this->info.params = this->params; + this->info.n_params = 2; + + port = &this->port[SPA_DIRECTION_INPUT]; + port->stream_id = SPA_DIRECTION_INPUT; + port->direction = SPA_DIRECTION_INPUT; + port->info_all = SPA_PORT_CHANGE_MASK_FLAGS | + SPA_PORT_CHANGE_MASK_PARAMS | + SPA_PORT_CHANGE_MASK_PROPS; + port->info = SPA_PORT_INFO_INIT(); + port->info.flags = SPA_PORT_FLAG_NO_REF; + port->params[0] = SPA_PARAM_INFO(SPA_PARAM_EnumFormat, SPA_PARAM_INFO_READ); + port->params[1] = SPA_PARAM_INFO(SPA_PARAM_Meta, SPA_PARAM_INFO_READ); + port->params[2] = SPA_PARAM_INFO(SPA_PARAM_IO, SPA_PARAM_INFO_READ); + port->params[3] = SPA_PARAM_INFO(SPA_PARAM_Format, SPA_PARAM_INFO_WRITE); + port->params[4] = SPA_PARAM_INFO(SPA_PARAM_Buffers, 0); + port->info.params = port->params; + port->info.n_params = 5; + spa_vulkan_blit_init_stream(&this->state, &this->state.streams[port->stream_id], + SPA_DIRECTION_INPUT, NULL); + spa_list_init(&port->empty); + spa_list_init(&port->ready); + + port = &this->port[SPA_DIRECTION_OUTPUT]; + port->stream_id = SPA_DIRECTION_OUTPUT; + port->direction = SPA_DIRECTION_OUTPUT; + port->info_all = SPA_PORT_CHANGE_MASK_FLAGS | + SPA_PORT_CHANGE_MASK_PARAMS | + SPA_PORT_CHANGE_MASK_PROPS; + port->info = SPA_PORT_INFO_INIT(); + port->info.flags = SPA_PORT_FLAG_NO_REF | SPA_PORT_FLAG_CAN_ALLOC_BUFFERS; + port->params[0] = SPA_PARAM_INFO(SPA_PARAM_EnumFormat, SPA_PARAM_INFO_READ); + port->params[1] = SPA_PARAM_INFO(SPA_PARAM_Meta, SPA_PARAM_INFO_READ); + port->params[2] = SPA_PARAM_INFO(SPA_PARAM_IO, SPA_PARAM_INFO_READ); + port->params[3] = SPA_PARAM_INFO(SPA_PARAM_Format, SPA_PARAM_INFO_WRITE); + port->params[4] = SPA_PARAM_INFO(SPA_PARAM_Buffers, 0); + port->info.params = port->params; + port->info.n_params = 5; + spa_list_init(&port->empty); + spa_list_init(&port->ready); + spa_vulkan_blit_init_stream(&this->state, &this->state.streams[port->stream_id], + SPA_DIRECTION_OUTPUT, NULL); + + this->state.n_streams = 2; + spa_vulkan_blit_init(&this->state); + spa_vulkan_blit_prepare(&this->state); + + return 0; +} + +static const struct spa_interface_info impl_interfaces[] = { + {SPA_TYPE_INTERFACE_Node,}, +}; + +static int +impl_enum_interface_info(const struct spa_handle_factory *factory, + const struct spa_interface_info **info, + uint32_t *index) +{ + spa_return_val_if_fail(factory != NULL, -EINVAL); + spa_return_val_if_fail(info != NULL, -EINVAL); + spa_return_val_if_fail(index != NULL, -EINVAL); + + switch (*index) { + case 0: + *info = &impl_interfaces[*index]; + break; + default: + return 0; + } + (*index)++; + return 1; +} + +static const struct spa_dict_item info_items[] = { + { SPA_KEY_FACTORY_AUTHOR, "Columbarius " }, + { SPA_KEY_FACTORY_DESCRIPTION, "Convert video frames using a vulkan blit" }, +}; + +static const struct spa_dict info = SPA_DICT_INIT_ARRAY(info_items); + +const struct spa_handle_factory spa_vulkan_blit_filter_factory = { + SPA_VERSION_HANDLE_FACTORY, + SPA_NAME_API_VULKAN_BLIT_FILTER, + &info, + impl_get_size, + impl_init, + impl_enum_interface_info, +}; diff --git a/spa/plugins/vulkan/vulkan-blit-utils.c b/spa/plugins/vulkan/vulkan-blit-utils.c new file mode 100644 index 000000000..4cda26458 --- /dev/null +++ b/spa/plugins/vulkan/vulkan-blit-utils.c @@ -0,0 +1,639 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2019 Wim Taymans */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#if !defined(__FreeBSD__) && !defined(__MidnightBSD__) +#include +#endif +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "vulkan-blit-utils.h" +#include "vulkan-utils.h" +#include "utils.h" + +#define VULKAN_INSTANCE_FUNCTION(name) \ + PFN_##name name = (PFN_##name)vkGetInstanceProcAddr(s->base.instance, #name) + +static int updateBuffers(struct vulkan_blit_state *s) +{ + uint32_t i; + + for (i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + + if (p->current_buffer_id == p->pending_buffer_id || + p->pending_buffer_id == SPA_ID_INVALID) + continue; + + p->current_buffer_id = p->pending_buffer_id; + p->busy_buffer_id = p->current_buffer_id; + p->pending_buffer_id = SPA_ID_INVALID; + + } + + return 0; +} + +static int runImportSHMBuffers(struct vulkan_blit_state *s) { + struct vulkan_stream *p = &s->streams[SPA_DIRECTION_INPUT]; + + if (p->spa_buffers[p->current_buffer_id]->datas[0].type == SPA_DATA_MemPtr) { + struct vulkan_buffer *vk_buf = &p->buffers[p->current_buffer_id]; + struct spa_buffer *spa_buf = p->spa_buffers[p->current_buffer_id]; + VkBufferImageCopy copy; + struct vulkan_write_pixels_info writeInfo = { + .data = spa_buf->datas[0].data, + .offset = 0, + .stride = p->bpp * p->dim.width, + .bytes_per_pixel = p->bpp, + .size.width = p->dim.width, + .size.height = p->dim.height, + .copies = ©, + }; + CHECK(vulkan_write_pixels(&s->base, &writeInfo, &s->staging_buffer)); + + vkCmdCopyBufferToImage(s->commandBuffer, s->staging_buffer.buffer, vk_buf->image, + VK_IMAGE_LAYOUT_GENERAL, 1, ©); + } + + return 0; +} + +static int runExportSHMBuffers(struct vulkan_blit_state *s) { + struct vulkan_stream *p = &s->streams[SPA_DIRECTION_OUTPUT]; + + if (p->spa_buffers[p->current_buffer_id]->datas[0].type == SPA_DATA_MemPtr) { + struct spa_buffer *spa_buf = p->spa_buffers[p->current_buffer_id]; + struct vulkan_read_pixels_info readInfo = { + .data = spa_buf->datas[0].data, + .offset = 0, + .stride = p->bpp * p->dim.width, + .bytes_per_pixel = p->bpp, + .size.width = p->dim.width, + .size.height = p->dim.height, + }; + CHECK(vulkan_read_pixels(&s->base, &readInfo, &p->buffers[p->current_buffer_id])); + } + + return 0; +} + +/** runCommandBuffer + * The return value of this functions means the following: + * ret < 0: Error + * ret = 0: queueSubmit was succsessful, but manual synchronization is required + * ret = 1: queueSubmit was succsessful and buffers can be released without synchronization + */ +static int runCommandBuffer(struct vulkan_blit_state *s) +{ + VULKAN_INSTANCE_FUNCTION(vkQueueSubmit2KHR); + VULKAN_INSTANCE_FUNCTION(vkGetSemaphoreFdKHR); + + static const VkCommandBufferBeginInfo beginInfo = { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, + }; + VK_CHECK_RESULT(vkBeginCommandBuffer(s->commandBuffer, &beginInfo)); + + CHECK(runImportSHMBuffers(s)); + + uint32_t i; + struct vulkan_stream *stream_input = &s->streams[SPA_DIRECTION_INPUT]; + struct vulkan_stream *stream_output = &s->streams[SPA_DIRECTION_OUTPUT]; + + VkImage src_image = stream_input->buffers[stream_input->current_buffer_id].image; + VkImage dst_image = stream_output->buffers[stream_output->current_buffer_id].image; + + VkImageBlit imageBlitRegion = { + .srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .srcSubresource.layerCount = 1, + .srcOffsets[0] = { + .x = 0, + .y = 0, + .z = 0, + }, + .srcOffsets[1] = { + .x = stream_input->dim.width, + .y = stream_input->dim.height, + .z = 1, + }, + .dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .dstSubresource.layerCount = 1, + .dstOffsets[1] = { + .x = stream_output->dim.width, + .y = stream_output->dim.height, + .z = 1, + } + }; + spa_log_trace_fp(s->log, "Blitting from (%p, %d, %d) %d,%dx%d,%d to (%p, %d, %d) %d,%dx%d,%d", + stream_input, stream_input->current_buffer_id, stream_input->direction, 0, 0, stream_input->dim.width, stream_input->dim.height, + stream_output, stream_output->current_buffer_id, stream_output->direction, 0, 0, stream_output->dim.width, stream_output->dim.height); + vkCmdBlitImage(s->commandBuffer, src_image, VK_IMAGE_LAYOUT_GENERAL, + dst_image, VK_IMAGE_LAYOUT_GENERAL, + 1, &imageBlitRegion, VK_FILTER_NEAREST); + + VkImageMemoryBarrier acquire_barrier[s->n_streams]; + VkImageMemoryBarrier release_barrier[s->n_streams]; + VkSemaphoreSubmitInfo semaphore_wait_info[s->n_streams]; + uint32_t semaphore_wait_info_len = 0; + VkSemaphoreSubmitInfo semaphore_signal_info[1]; + uint32_t semaphore_signal_info_len = 0; + + for (i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + struct vulkan_buffer *current_buffer = &p->buffers[p->current_buffer_id]; + struct spa_buffer *current_spa_buffer = p->spa_buffers[p->current_buffer_id]; + + VkAccessFlags access_flags; + if (p->direction == SPA_DIRECTION_INPUT) { + access_flags = VK_ACCESS_TRANSFER_READ_BIT; + } else { + access_flags = VK_ACCESS_TRANSFER_WRITE_BIT; + } + + acquire_barrier[i]= (VkImageMemoryBarrier) { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .dstQueueFamilyIndex = s->base.queueFamilyIndex, + .image = current_buffer->image, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcAccessMask = 0, + .dstAccessMask = access_flags, + .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .subresourceRange.levelCount = 1, + .subresourceRange.layerCount = 1, + }; + + release_barrier[i]= (VkImageMemoryBarrier) { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .srcQueueFamilyIndex = s->base.queueFamilyIndex, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .image = current_buffer->image, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcAccessMask = access_flags, + .dstAccessMask = 0, + .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .subresourceRange.levelCount = 1, + .subresourceRange.layerCount = 1, + }; + + if (current_spa_buffer->datas[0].type != SPA_DATA_DmaBuf) + continue; + + if (vulkan_sync_foreign_dmabuf(&s->base, current_buffer) < 0) { + spa_log_warn(s->log, "Failed to wait for foreign buffer DMA-BUF fence"); + } else { + if (current_buffer->foreign_semaphore != VK_NULL_HANDLE) { + semaphore_wait_info[semaphore_wait_info_len++] = (VkSemaphoreSubmitInfo) { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .semaphore = current_buffer->foreign_semaphore, + .stageMask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + }; + } + } + } + + vkCmdPipelineBarrier(s->commandBuffer, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_2_TRANSFER_BIT, + 0, 0, NULL, 0, NULL, + s->n_streams, acquire_barrier); + + vkCmdPipelineBarrier(s->commandBuffer, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, + 0, 0, NULL, 0, NULL, + s->n_streams, release_barrier); + + VK_CHECK_RESULT(vkEndCommandBuffer(s->commandBuffer)); + + VK_CHECK_RESULT(vkResetFences(s->base.device, 1, &s->fence)); + + if (s->pipelineSemaphore == VK_NULL_HANDLE) { + VkExportSemaphoreCreateInfo export_info = { + .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO, + .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT, + }; + VkSemaphoreCreateInfo semaphore_info = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, + .pNext = &export_info, + }; + VK_CHECK_RESULT(vkCreateSemaphore(s->base.device, &semaphore_info, NULL, &s->pipelineSemaphore)); + } + + semaphore_signal_info[semaphore_signal_info_len++] = (VkSemaphoreSubmitInfo) { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .semaphore = s->pipelineSemaphore, + }; + + VkCommandBufferSubmitInfoKHR commandBufferInfo = { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, + .commandBuffer = s->commandBuffer, + }; + + const VkSubmitInfo2KHR submitInfo = { + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2_KHR, + .commandBufferInfoCount = 1, + .pCommandBufferInfos = &commandBufferInfo, + .waitSemaphoreInfoCount = semaphore_wait_info_len, + .pWaitSemaphoreInfos = semaphore_wait_info, + .signalSemaphoreInfoCount = semaphore_signal_info_len, + .pSignalSemaphoreInfos = semaphore_signal_info, + }; + VK_CHECK_RESULT(vkQueueSubmit2KHR(s->base.queue, 1, &submitInfo, s->fence)); + s->started = true; + + VkSemaphoreGetFdInfoKHR get_fence_fd_info = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR, + .semaphore = s->pipelineSemaphore, + .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT, + }; + int sync_file_fd = -1; + VK_CHECK_RESULT(vkGetSemaphoreFdKHR(s->base.device, &get_fence_fd_info, &sync_file_fd)); + + int ret = 1; + for (uint32_t i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + struct spa_buffer *current_spa_buffer = p->spa_buffers[p->current_buffer_id]; + + if (current_spa_buffer->datas[0].type != SPA_DATA_DmaBuf) + continue; + + if (!vulkan_sync_export_dmabuf(&s->base, &p->buffers[p->current_buffer_id], sync_file_fd)) { + ret = 0; + } + } + close(sync_file_fd); + + return ret; +} + +static void clear_buffers(struct vulkan_blit_state *s, struct vulkan_stream *p) +{ + uint32_t i; + + for (i = 0; i < p->n_buffers; i++) { + vulkan_buffer_clear(&s->base, &p->buffers[i]); + p->spa_buffers[i] = NULL; + } + p->n_buffers = 0; + if (p->direction == SPA_DIRECTION_INPUT) { + vulkan_staging_buffer_destroy(&s->base, &s->staging_buffer); + s->staging_buffer.buffer = VK_NULL_HANDLE; + } +} + +static void clear_streams(struct vulkan_blit_state *s) +{ + uint32_t i; + for (i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + clear_buffers(s, p); + } +} + +int spa_vulkan_blit_fixate_modifier(struct vulkan_blit_state *s, struct vulkan_stream *p, struct spa_video_info *info, + uint32_t modifierCount, uint64_t *modifiers, uint64_t *modifier) +{ + VkFormat format; + struct spa_rectangle size; + switch (info->media_subtype) { + case SPA_MEDIA_SUBTYPE_dsp: + format = vulkan_id_to_vkformat(info->info.dsp.format); + size.width = p->dim.width; + size.height = p->dim.height; + break; + case SPA_MEDIA_SUBTYPE_raw: + format = vulkan_id_to_vkformat(info->info.raw.format); + size.width = p->dim.width; + size.height = p->dim.height; + break; + default: + spa_log_warn(s->log, "Unsupported media subtype %d", info->media_subtype); + return -1; + } + if (format == VK_FORMAT_UNDEFINED) { + return -1; + } + + struct dmabuf_fixation_info fixation_info = { + .format = format, + .modifierCount = modifierCount, + .modifiers = modifiers, + .size = size, + .usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT, + }; + return vulkan_fixate_modifier(&s->base, &fixation_info, modifier); +} + +int spa_vulkan_blit_use_buffers(struct vulkan_blit_state *s, struct vulkan_stream *p, uint32_t flags, + struct spa_video_info *info, uint32_t n_buffers, struct spa_buffer **buffers) +{ + struct external_buffer_info externalBufferInfo = {0}; + switch (info->media_subtype) { + case SPA_MEDIA_SUBTYPE_dsp: + externalBufferInfo.format = vulkan_id_to_vkformat(info->info.dsp.format); + externalBufferInfo.size.width = p->dim.width; + externalBufferInfo.size.height = p->dim.height; + if (info->info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER) + externalBufferInfo.modifier = info->info.dsp.modifier; + break; + case SPA_MEDIA_SUBTYPE_raw: + externalBufferInfo.format = vulkan_id_to_vkformat(info->info.raw.format); + externalBufferInfo.size.width = p->dim.width; + externalBufferInfo.size.height = p->dim.height; + if (info->info.raw.flags & SPA_VIDEO_FLAG_MODIFIER) + externalBufferInfo.modifier = info->info.raw.modifier; + break; + default: + spa_log_warn(s->log, "Unsupported media subtype %d", info->media_subtype); + return -1; + } + if (externalBufferInfo.format == VK_FORMAT_UNDEFINED) + return -1; + + vulkan_wait_idle(&s->base); + clear_buffers(s, p); + + bool alloc = flags & SPA_NODE_BUFFERS_FLAG_ALLOC; + int ret; + p->n_buffers = 0; + for (uint32_t i = 0; i < n_buffers; i++) { + externalBufferInfo.usage = p->direction == SPA_DIRECTION_OUTPUT + ? VK_IMAGE_USAGE_TRANSFER_DST_BIT + : VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + externalBufferInfo.spa_buf = buffers[i]; + if (alloc) { + if (SPA_FLAG_IS_SET(buffers[i]->datas[0].type, 1<base, &externalBufferInfo, &p->buffers[i]); + } else { + spa_log_error(s->log, "Unsupported buffer type mask %d", buffers[i]->datas[0].type); + return -1; + } + } else { + switch (buffers[i]->datas[0].type) { + case SPA_DATA_DmaBuf:; + ret = vulkan_import_dmabuf(&s->base, &externalBufferInfo, &p->buffers[i]); + break; + case SPA_DATA_MemPtr:; + if (p->direction == SPA_DIRECTION_OUTPUT) { + externalBufferInfo.usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + } else { + externalBufferInfo.usage |= VK_IMAGE_USAGE_TRANSFER_DST_BIT; + } + ret = vulkan_import_memptr(&s->base, &externalBufferInfo, &p->buffers[i]); + break; + default: + spa_log_error(s->log, "Unsupported buffer type %d", buffers[i]->datas[0].type); + return -1; + } + } + if (ret != 0) { + spa_log_error(s->log, "Failed to use buffer %d", i); + return ret; + } + p->spa_buffers[i] = buffers[i]; + p->n_buffers++; + } + if (p->direction == SPA_DIRECTION_INPUT && buffers[0]->datas[0].type == SPA_DATA_MemPtr) { + ret = vulkan_staging_buffer_create(&s->base, buffers[0]->datas[0].maxsize, &s->staging_buffer); + if (ret < 0) { + spa_log_error(s->log, "Failed to create staging buffer"); + return ret; + } + } + + return 0; +} + +int spa_vulkan_blit_enumerate_raw_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder) +{ + uint32_t fmt_idx; + bool has_modifier; + if (!find_EnumFormatInfo(&s->formatInfosRaw, index, caps, &fmt_idx, &has_modifier)) + return 0; + *param = build_raw_EnumFormat(&s->formatInfosRaw.infos[fmt_idx], has_modifier, builder); + return 1; +} + +int spa_vulkan_blit_enumerate_dsp_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder) +{ + uint32_t fmt_idx; + bool has_modifier; + if (!find_EnumFormatInfo(&s->formatInfosDSP, index, caps, &fmt_idx, &has_modifier)) + return 0; + *param = build_dsp_EnumFormat(&s->formatInfosDSP.infos[fmt_idx], has_modifier, builder); + return 1; +} + +int spa_vulkan_blit_enumerate_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder) +{ + uint32_t fmt_idx; + bool has_modifier; + uint32_t raw_offset = 0; + if ((caps & VULKAN_BUFFER_TYPE_CAP_SHM) > 0) + raw_offset += s->formatInfosDSP.formatCount; + if ((caps & VULKAN_BUFFER_TYPE_CAP_DMABUF) > 0) + raw_offset += s->formatInfosDSP.formatsWithModifiersCount; + if (index < raw_offset) { + if (find_EnumFormatInfo(&s->formatInfosDSP, index, caps, &fmt_idx, &has_modifier)) { + *param = build_dsp_EnumFormat(&s->formatInfosDSP.infos[fmt_idx], has_modifier, builder); + return 1; + } + } else { + if (find_EnumFormatInfo(&s->formatInfosRaw, index - raw_offset, caps, &fmt_idx, &has_modifier)) { + *param = build_raw_EnumFormat(&s->formatInfosRaw.infos[fmt_idx], has_modifier, builder); + return 1; + } + } + return 0; +} + +static int vulkan_stream_init(struct vulkan_stream *stream, enum spa_direction direction, + struct spa_dict *props) +{ + spa_zero(*stream); + stream->direction = direction; + stream->current_buffer_id = SPA_ID_INVALID; + stream->busy_buffer_id = SPA_ID_INVALID; + stream->ready_buffer_id = SPA_ID_INVALID; + return 0; +} + +int spa_vulkan_blit_init_stream(struct vulkan_blit_state *s, struct vulkan_stream *stream, + enum spa_direction direction, struct spa_dict *props) +{ + return vulkan_stream_init(stream, direction, props); +} + +int spa_vulkan_blit_prepare(struct vulkan_blit_state *s) +{ + if (!s->prepared) { + CHECK(vulkan_fence_create(&s->base, &s->fence)); + CHECK(vulkan_commandPool_create(&s->base, &s->commandPool)); + CHECK(vulkan_commandBuffer_create(&s->base, s->commandPool, &s->commandBuffer)); + s->prepared = true; + } + return 0; +} + +int spa_vulkan_blit_unprepare(struct vulkan_blit_state *s) +{ + if (s->prepared) { + vkDestroyCommandPool(s->base.device, s->commandPool, NULL); + vkDestroyFence(s->base.device, s->fence, NULL); + s->prepared = false; + } + return 0; +} + +int spa_vulkan_blit_start(struct vulkan_blit_state *s) +{ + uint32_t i; + + for (i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + p->current_buffer_id = SPA_ID_INVALID; + p->busy_buffer_id = SPA_ID_INVALID; + p->ready_buffer_id = SPA_ID_INVALID; + } + return 0; +} + +int spa_vulkan_blit_stop(struct vulkan_blit_state *s) +{ + VK_CHECK_RESULT(vkDeviceWaitIdle(s->base.device)); + clear_streams(s); + s->started = false; + return 0; +} + +int spa_vulkan_blit_ready(struct vulkan_blit_state *s) +{ + uint32_t i; + VkResult result; + + if (!s->started) + return 0; + + result = vkGetFenceStatus(s->base.device, s->fence); + if (result == VK_NOT_READY) + return -EBUSY; + VK_CHECK_RESULT(result); + + s->started = false; + + for (i = 0; i < s->n_streams; i++) { + struct vulkan_stream *p = &s->streams[i]; + p->ready_buffer_id = p->busy_buffer_id; + p->busy_buffer_id = SPA_ID_INVALID; + } + return 0; +} + +int spa_vulkan_blit_process(struct vulkan_blit_state *s) +{ + if (!s->initialized) { + spa_log_warn(s->log, "Renderer not initialized"); + return -1; + } + if (!s->prepared) { + spa_log_warn(s->log, "Renderer not prepared"); + return -1; + } + CHECK(updateBuffers(s)); + CHECK(runCommandBuffer(s)); + // CHECK(vulkan_wait_fence(&s->base, s->fence)); + CHECK(vulkan_wait_idle(&s->base)); + CHECK(runExportSHMBuffers(s)); + + return 0; +} + +int spa_vulkan_blit_get_buffer_caps(struct vulkan_blit_state *s, enum spa_direction direction) +{ + switch (direction) { + case SPA_DIRECTION_INPUT: + return VULKAN_BUFFER_TYPE_CAP_DMABUF | VULKAN_BUFFER_TYPE_CAP_SHM; + case SPA_DIRECTION_OUTPUT: + return VULKAN_BUFFER_TYPE_CAP_DMABUF | VULKAN_BUFFER_TYPE_CAP_SHM; + } + return 0; +} + +struct vulkan_modifier_info *spa_vulkan_blit_get_modifier_info(struct vulkan_blit_state *s, struct spa_video_info *info) { + VkFormat format; + uint64_t modifier; + switch (info->media_subtype) { + case SPA_MEDIA_SUBTYPE_dsp: + format = vulkan_id_to_vkformat(info->info.dsp.format); + modifier = info->info.dsp.modifier; + return vulkan_modifierInfo_find(&s->formatInfosDSP, format, modifier); + case SPA_MEDIA_SUBTYPE_raw: + format = vulkan_id_to_vkformat(info->info.raw.format); + modifier = info->info.raw.modifier; + return vulkan_modifierInfo_find(&s->formatInfosRaw, format, modifier); + default: + spa_log_warn(s->log, "Unsupported media subtype %d", info->media_subtype); + return NULL; + } +} + +int spa_vulkan_blit_init(struct vulkan_blit_state *s) +{ + int ret; + s->base.log = s->log; + struct vulkan_base_info baseInfo = { + .queueFlags = VK_QUEUE_TRANSFER_BIT, + }; + if ((ret = vulkan_base_init(&s->base, &baseInfo)) < 0) + return ret; + + uint32_t dsp_formats [] = { + SPA_VIDEO_FORMAT_DSP_F32 + }; + vulkan_format_infos_init(&s->base, SPA_N_ELEMENTS(dsp_formats), dsp_formats, &s->formatInfosDSP); + uint32_t raw_formats [] = { + SPA_VIDEO_FORMAT_BGRA, + SPA_VIDEO_FORMAT_RGBA, + SPA_VIDEO_FORMAT_BGRx, + SPA_VIDEO_FORMAT_RGBx, + SPA_VIDEO_FORMAT_BGR, + SPA_VIDEO_FORMAT_RGB, + }; + vulkan_format_infos_init(&s->base, SPA_N_ELEMENTS(raw_formats), raw_formats, &s->formatInfosRaw); + s->initialized = true; + return 0; +} + +void spa_vulkan_blit_deinit(struct vulkan_blit_state *s) +{ + vulkan_format_infos_deinit(&s->formatInfosRaw); + vulkan_format_infos_deinit(&s->formatInfosDSP); + vulkan_base_deinit(&s->base); + s->initialized = false; +} diff --git a/spa/plugins/vulkan/vulkan-blit-utils.h b/spa/plugins/vulkan/vulkan-blit-utils.h new file mode 100644 index 000000000..9fa089c80 --- /dev/null +++ b/spa/plugins/vulkan/vulkan-blit-utils.h @@ -0,0 +1,84 @@ +/* Spa */ +/* SPDX-FileCopyrightText: Copyright © 2019 Wim Taymans */ +/* SPDX-FileCopyrightText: Copyright © 2023 columbarius */ +/* SPDX-License-Identifier: MIT */ + +#include + +#include +#include +#include +#include + +#include "vulkan-utils.h" + +#define MAX_STREAMS 2 + +struct vulkan_stream { + enum spa_direction direction; + + uint32_t pending_buffer_id; + uint32_t current_buffer_id; + uint32_t busy_buffer_id; + uint32_t ready_buffer_id; + + struct spa_rectangle dim; + uint32_t bpp; + + struct vulkan_buffer buffers[MAX_BUFFERS]; + struct spa_buffer *spa_buffers[MAX_BUFFERS]; + uint32_t n_buffers; +}; + +struct vulkan_blit_state { + struct spa_log *log; + + struct vulkan_base base; + + struct vulkan_format_infos formatInfosRaw; + struct vulkan_format_infos formatInfosDSP; + + VkCommandPool commandPool; + VkCommandBuffer commandBuffer; + struct vulkan_staging_buffer staging_buffer; + + VkFence fence; + VkSemaphore pipelineSemaphore; + unsigned int initialized:1; + unsigned int prepared:1; + unsigned int started:1; + + //VkSampler sampler; + + uint32_t n_streams; + struct vulkan_stream streams[MAX_STREAMS]; +}; + +int spa_vulkan_blit_init_stream(struct vulkan_blit_state *s, struct vulkan_stream *stream, enum spa_direction, + struct spa_dict *props); + +int spa_vulkan_blit_fixate_modifier(struct vulkan_blit_state *s, struct vulkan_stream *p, struct spa_video_info *info, + uint32_t modifierCount, uint64_t *modifiers, uint64_t *modifier); +int spa_vulkan_blit_use_buffers(struct vulkan_blit_state *s, struct vulkan_stream *stream, uint32_t flags, + struct spa_video_info *info, uint32_t n_buffers, struct spa_buffer **buffers); +int spa_vulkan_blit_enumerate_raw_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder); +int spa_vulkan_blit_enumerate_dsp_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder); +int spa_vulkan_blit_enumerate_formats(struct vulkan_blit_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder); +int spa_vulkan_blit_prepare(struct vulkan_blit_state *s); +int spa_vulkan_blit_unprepare(struct vulkan_blit_state *s); + +int spa_vulkan_blit_start(struct vulkan_blit_state *s); +int spa_vulkan_blit_stop(struct vulkan_blit_state *s); +int spa_vulkan_blit_ready(struct vulkan_blit_state *s); +int spa_vulkan_blit_process(struct vulkan_blit_state *s); +int spa_vulkan_blit_cleanup(struct vulkan_blit_state *s); + +int spa_vulkan_blit_get_buffer_caps(struct vulkan_blit_state *s, enum spa_direction direction); +struct vulkan_modifier_info *spa_vulkan_blit_get_modifier_info(struct vulkan_blit_state *s, + struct spa_video_info *info); + +int spa_vulkan_blit_init(struct vulkan_blit_state *s); +void spa_vulkan_blit_deinit(struct vulkan_blit_state *s); diff --git a/spa/plugins/vulkan/vulkan-compute-filter.c b/spa/plugins/vulkan/vulkan-compute-filter.c index cd9288092..cc9e2fcab 100644 --- a/spa/plugins/vulkan/vulkan-compute-filter.c +++ b/spa/plugins/vulkan/vulkan-compute-filter.c @@ -172,7 +172,7 @@ static int impl_node_send_command(void *object, const struct spa_command *comman return 0; this->started = true; - spa_vulkan_start(&this->state); + spa_vulkan_compute_start(&this->state); break; case SPA_NODE_COMMAND_Suspend: @@ -181,7 +181,7 @@ static int impl_node_send_command(void *object, const struct spa_command *comman return 0; this->started = false; - spa_vulkan_stop(&this->state); + spa_vulkan_compute_stop(&this->state); break; default: return -ENOTSUP; @@ -269,64 +269,6 @@ impl_node_remove_port(void *object, enum spa_direction direction, uint32_t port_ return -ENOTSUP; } -static struct spa_pod *build_EnumFormat(uint32_t fmt, const struct vulkan_format_info *fmtInfo, struct spa_pod_builder *builder) { - struct spa_pod_frame f[2]; - uint32_t i, c; - - spa_pod_builder_push_object(builder, &f[0], SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat); - spa_pod_builder_add(builder, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), 0); - spa_pod_builder_add(builder, SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_dsp), 0); - spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_format, SPA_POD_Id(fmt), 0); - if (fmtInfo && fmtInfo->modifierCount > 0) { - spa_pod_builder_prop(builder, SPA_FORMAT_VIDEO_modifier, SPA_POD_PROP_FLAG_MANDATORY | SPA_POD_PROP_FLAG_DONT_FIXATE); - spa_pod_builder_push_choice(builder, &f[1], SPA_CHOICE_Enum, 0); - for (i = 0, c = 0; i < fmtInfo->modifierCount; i++) { - spa_pod_builder_long(builder, fmtInfo->infos[i].props.drmFormatModifier); - if (c++ == 0) - spa_pod_builder_long(builder, fmtInfo->infos[i].props.drmFormatModifier); - } - spa_pod_builder_pop(builder, &f[1]); - } - return spa_pod_builder_pop(builder, &f[0]); -} - -// This function enumerates the available formats in vulkan_state::formats, announcing all formats capable to support DmaBufs -// first and then falling back to those supported with SHM buffers. -static bool find_EnumFormatInfo(struct vulkan_base *s, uint32_t index, uint32_t caps, uint32_t *fmt_idx, bool *has_modifier) { - int64_t fmtIterator = 0; - int64_t maxIterator = 0; - if (caps & VULKAN_BUFFER_TYPE_CAP_SHM) - maxIterator += s->formatInfoCount; - if (caps & VULKAN_BUFFER_TYPE_CAP_DMABUF) - maxIterator += s->formatInfoCount; - // Count available formats until index underflows, while fmtIterator indexes the current format. - // Iterate twice over formats first time with modifiers, second time without if both caps are supported. - while (index < (uint32_t)-1 && fmtIterator < maxIterator) { - const struct vulkan_format_info *f_info = &s->formatInfos[fmtIterator%s->formatInfoCount]; - if (caps & VULKAN_BUFFER_TYPE_CAP_DMABUF && fmtIterator < s->formatInfoCount) { - // First round, check for modifiers - if (f_info->modifierCount > 0) { - index--; - } - } else if (caps & VULKAN_BUFFER_TYPE_CAP_SHM) { - // Second round, every format should be supported. - index--; - } - fmtIterator++; - } - - if (index != (uint32_t)-1) { - // No more formats available - return false; - } - // Undo end of loop increment - fmtIterator--; - *fmt_idx = fmtIterator%s->formatInfoCount; - // Loop finished in first round - *has_modifier = caps & VULKAN_BUFFER_TYPE_CAP_DMABUF && fmtIterator < s->formatInfoCount; - return true; -} - static int port_enum_formats(void *object, enum spa_direction direction, uint32_t port_id, uint32_t index, @@ -336,8 +278,6 @@ static int port_enum_formats(void *object, { struct impl *this = object; - uint32_t fmt_index; - bool has_modifier; if (this->port[port_id].have_format && this->port[port_id].current_format.info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER && this->port[port_id].current_format.info.dsp.flags ^ SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED) { @@ -347,18 +287,10 @@ static int port_enum_formats(void *object, *param = spa_format_video_dsp_build(builder, SPA_PARAM_EnumFormat, &this->port[port_id].current_format.info.dsp); return 1; } - if (!find_EnumFormatInfo(&this->state.base, index-1, spa_vulkan_get_buffer_caps(&this->state, direction), &fmt_index, &has_modifier)) - return 0; + return spa_vulkan_compute_enumerate_formats(&this->state, index-1, spa_vulkan_compute_get_buffer_caps(&this->state, direction), param, builder); } else { - if (!find_EnumFormatInfo(&this->state.base, index, spa_vulkan_get_buffer_caps(&this->state, direction), &fmt_index, &has_modifier)) - return 0; + return spa_vulkan_compute_enumerate_formats(&this->state, index, spa_vulkan_compute_get_buffer_caps(&this->state, direction), param, builder); } - - const struct vulkan_format_info *f_info = &this->state.base.formatInfos[fmt_index]; - spa_log_info(this->log, "vulkan-compute-filter: enum_formats idx: %d, format %d, has_modifier %d", index, f_info->spa_format, has_modifier); - *param = build_EnumFormat(f_info->spa_format, has_modifier ? f_info : NULL, builder); - - return 1; } static int @@ -420,7 +352,7 @@ impl_node_port_enum_params(void *object, int seq, this->position->video.stride); if (port->current_format.info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER) { - struct vulkan_modifier_info *mod_info = spa_vulkan_get_modifier_info(&this->state, + struct vulkan_modifier_info *mod_info = spa_vulkan_compute_get_modifier_info(&this->state, &port->current_format.info.dsp); param = spa_pod_builder_add_object(&b, SPA_TYPE_OBJECT_ParamBuffers, id, @@ -472,8 +404,8 @@ static int clear_buffers(struct impl *this, struct port *port) { if (port->n_buffers > 0) { spa_log_debug(this->log, NAME " %p: clear buffers", this); - spa_vulkan_stop(&this->state); - spa_vulkan_use_buffers(&this->state, &this->state.streams[port->stream_id], 0, &port->current_format.info.dsp, 0, NULL); + spa_vulkan_compute_stop(&this->state); + spa_vulkan_compute_use_buffers(&this->state, &this->state.streams[port->stream_id], 0, &port->current_format.info.dsp, 0, NULL); port->n_buffers = 0; spa_list_init(&port->empty); spa_list_init(&port->ready); @@ -491,7 +423,7 @@ static int port_set_format(struct impl *this, struct port *port, if (format == NULL) { port->have_format = false; clear_buffers(this, port); - spa_vulkan_unprepare(&this->state); + spa_vulkan_compute_unprepare(&this->state); } else { struct spa_video_info info = { 0 }; @@ -528,7 +460,7 @@ static int port_set_format(struct impl *this, struct port *port, modifierCount -= 1; modifiers++; uint64_t fixed_modifier; - if (spa_vulkan_fixate_modifier(&this->state, &this->state.streams[port->stream_id], &info.info.dsp, modifierCount, modifiers, &fixed_modifier) != 0) + if (spa_vulkan_compute_fixate_modifier(&this->state, &this->state.streams[port->stream_id], &info.info.dsp, modifierCount, modifiers, &fixed_modifier) != 0) return -EINVAL; spa_log_info(this->log, NAME ": modifier fixated %"PRIu64, fixed_modifier); @@ -628,7 +560,7 @@ impl_node_port_use_buffers(void *object, spa_log_info(this->log, "%p: %d:%d add buffer %p", port, direction, port_id, b); spa_list_append(&port->empty, &b->link); } - spa_vulkan_use_buffers(&this->state, &this->state.streams[port->stream_id], flags, &port->current_format.info.dsp, n_buffers, buffers); + spa_vulkan_compute_use_buffers(&this->state, &this->state.streams[port->stream_id], flags, &port->current_format.info.dsp, n_buffers, buffers); port->n_buffers = n_buffers; return 0; @@ -725,7 +657,7 @@ static int impl_node_process(void *object) spa_log_debug(this->log, "filter into %d", b->id); - spa_vulkan_process(&this->state); + spa_vulkan_compute_process(&this->state); b->outbuf->datas[0].chunk->offset = 0; b->outbuf->datas[0].chunk->size = b->outbuf->datas[0].maxsize; @@ -780,7 +712,7 @@ static int impl_clear(struct spa_handle *handle) this = (struct impl *) handle; - spa_vulkan_deinit(&this->state); + spa_vulkan_compute_deinit(&this->state); return 0; } @@ -847,7 +779,7 @@ impl_init(const struct spa_handle_factory *factory, port->params[4] = SPA_PARAM_INFO(SPA_PARAM_Buffers, 0); port->info.params = port->params; port->info.n_params = 5; - spa_vulkan_init_stream(&this->state, &this->state.streams[port->stream_id], + spa_vulkan_compute_init_stream(&this->state, &this->state.streams[port->stream_id], SPA_DIRECTION_INPUT, NULL); spa_list_init(&port->empty); spa_list_init(&port->ready); @@ -869,12 +801,12 @@ impl_init(const struct spa_handle_factory *factory, port->info.n_params = 5; spa_list_init(&port->empty); spa_list_init(&port->ready); - spa_vulkan_init_stream(&this->state, &this->state.streams[port->stream_id], + spa_vulkan_compute_init_stream(&this->state, &this->state.streams[port->stream_id], SPA_DIRECTION_OUTPUT, NULL); this->state.n_streams = 2; - spa_vulkan_init(&this->state); - spa_vulkan_prepare(&this->state); + spa_vulkan_compute_init(&this->state); + spa_vulkan_compute_prepare(&this->state); return 0; } diff --git a/spa/plugins/vulkan/vulkan-compute-source.c b/spa/plugins/vulkan/vulkan-compute-source.c index 765dfd6ec..95531f9c6 100644 --- a/spa/plugins/vulkan/vulkan-compute-source.c +++ b/spa/plugins/vulkan/vulkan-compute-source.c @@ -274,7 +274,7 @@ static int make_buffer(struct impl *this) if (read_timer(this) < 0) return 0; - if ((res = spa_vulkan_ready(&this->state)) < 0) { + if ((res = spa_vulkan_compute_ready(&this->state)) < 0) { res = SPA_STATUS_OK; goto next; } @@ -295,7 +295,7 @@ static int make_buffer(struct impl *this) this->state.constants.frame = this->frame_count; this->state.streams[0].pending_buffer_id = b->id; - spa_vulkan_process(&this->state); + spa_vulkan_compute_process(&this->state); if (this->state.streams[0].ready_buffer_id != SPA_ID_INVALID) { struct buffer *b = &port->buffers[this->state.streams[0].ready_buffer_id]; @@ -405,7 +405,7 @@ static int impl_node_send_command(void *object, const struct spa_command *comman this->started = true; set_timer(this, true); - spa_vulkan_start(&this->state); + spa_vulkan_compute_start(&this->state); break; } case SPA_NODE_COMMAND_Suspend: @@ -415,7 +415,7 @@ static int impl_node_send_command(void *object, const struct spa_command *comman this->started = false; set_timer(this, false); - spa_vulkan_stop(&this->state); + spa_vulkan_compute_stop(&this->state); break; default: return -ENOTSUP; @@ -503,64 +503,6 @@ impl_node_remove_port(void *object, enum spa_direction direction, uint32_t port_ return -ENOTSUP; } -static struct spa_pod *build_EnumFormat(uint32_t fmt, const struct vulkan_format_info *fmtInfo, struct spa_pod_builder *builder) { - struct spa_pod_frame f[2]; - uint32_t i, c; - - spa_pod_builder_push_object(builder, &f[0], SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat); - spa_pod_builder_add(builder, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), 0); - spa_pod_builder_add(builder, SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_dsp), 0); - spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_format, SPA_POD_Id(fmt), 0); - if (fmtInfo && fmtInfo->modifierCount > 0) { - spa_pod_builder_prop(builder, SPA_FORMAT_VIDEO_modifier, SPA_POD_PROP_FLAG_MANDATORY | SPA_POD_PROP_FLAG_DONT_FIXATE); - spa_pod_builder_push_choice(builder, &f[1], SPA_CHOICE_Enum, 0); - for (i = 0, c = 0; i < fmtInfo->modifierCount; i++) { - spa_pod_builder_long(builder, fmtInfo->infos[i].props.drmFormatModifier); - if (c++ == 0) - spa_pod_builder_long(builder, fmtInfo->infos[i].props.drmFormatModifier); - } - spa_pod_builder_pop(builder, &f[1]); - } - return spa_pod_builder_pop(builder, &f[0]); -} - -// This function enumerates the available formats in vulkan_state::formats, announcing all formats capable to support DmaBufs -// first and then falling back to those supported with SHM buffers. -static bool find_EnumFormatInfo(struct vulkan_base *s, uint32_t index, uint32_t caps, uint32_t *fmt_idx, bool *has_modifier) { - int64_t fmtIterator = 0; - int64_t maxIterator = 0; - if (caps & VULKAN_BUFFER_TYPE_CAP_SHM) - maxIterator += s->formatInfoCount; - if (caps & VULKAN_BUFFER_TYPE_CAP_DMABUF) - maxIterator += s->formatInfoCount; - // Count available formats until index underflows, while fmtIterator indexes the current format. - // Iterate twice over formats first time with modifiers, second time without if both caps are supported. - while (index < (uint32_t)-1 && fmtIterator < maxIterator) { - const struct vulkan_format_info *f_info = &s->formatInfos[fmtIterator%s->formatInfoCount]; - if (caps & VULKAN_BUFFER_TYPE_CAP_DMABUF && fmtIterator < s->formatInfoCount) { - // First round, check for modifiers - if (f_info->modifierCount > 0) { - index--; - } - } else if (caps & VULKAN_BUFFER_TYPE_CAP_SHM) { - // Second round, every format should be supported. - index--; - } - fmtIterator++; - } - - if (index != (uint32_t)-1) { - // No more formats available - return false; - } - // Undo end of loop increment - fmtIterator--; - *fmt_idx = fmtIterator%s->formatInfoCount; - // Loop finished in first round - *has_modifier = caps & VULKAN_BUFFER_TYPE_CAP_DMABUF && fmtIterator < s->formatInfoCount; - return true; -} - static int port_enum_formats(void *object, enum spa_direction direction, uint32_t port_id, uint32_t index, @@ -570,8 +512,6 @@ static int port_enum_formats(void *object, { struct impl *this = object; - uint32_t fmt_index; - bool has_modifier; if (this->port.have_format && this->port.current_format.info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER && this->port.current_format.info.dsp.flags ^ SPA_VIDEO_FLAG_MODIFIER_FIXATION_REQUIRED) { @@ -581,18 +521,10 @@ static int port_enum_formats(void *object, *param = spa_format_video_dsp_build(builder, SPA_PARAM_EnumFormat, &this->port.current_format.info.dsp); return 1; } - if (!find_EnumFormatInfo(&this->state.base, index-1, spa_vulkan_get_buffer_caps(&this->state, direction), &fmt_index, &has_modifier)) - return 0; + return spa_vulkan_compute_enumerate_formats(&this->state, index-1, spa_vulkan_compute_get_buffer_caps(&this->state, direction), param, builder); } else { - if (!find_EnumFormatInfo(&this->state.base, index, spa_vulkan_get_buffer_caps(&this->state, direction), &fmt_index, &has_modifier)) - return 0; + return spa_vulkan_compute_enumerate_formats(&this->state, index, spa_vulkan_compute_get_buffer_caps(&this->state, direction), param, builder); } - - const struct vulkan_format_info *f_info = &this->state.base.formatInfos[fmt_index]; - spa_log_info(this->log, "vulkan-compute-source: enum_formats idx: %d, format %d, has_modifier %d", index, f_info->spa_format, has_modifier); - *param = build_EnumFormat(f_info->spa_format, has_modifier ? f_info : NULL, builder); - - return 1; } static int @@ -655,7 +587,7 @@ impl_node_port_enum_params(void *object, int seq, if (port->current_format.info.dsp.flags & SPA_VIDEO_FLAG_MODIFIER) { - struct vulkan_modifier_info *mod_info = spa_vulkan_get_modifier_info(&this->state, + struct vulkan_modifier_info *mod_info = spa_vulkan_compute_get_modifier_info(&this->state, &port->current_format.info.dsp); param = spa_pod_builder_add_object(&b, SPA_TYPE_OBJECT_ParamBuffers, id, @@ -707,7 +639,7 @@ static int clear_buffers(struct impl *this, struct port *port) { if (port->n_buffers > 0) { spa_log_debug(this->log, NAME " %p: clear buffers", this); - spa_vulkan_use_buffers(&this->state, &this->state.streams[0], 0, &port->current_format.info.dsp, 0, NULL); + spa_vulkan_compute_use_buffers(&this->state, &this->state.streams[0], 0, &port->current_format.info.dsp, 0, NULL); port->n_buffers = 0; spa_list_init(&port->empty); spa_list_init(&port->ready); @@ -726,7 +658,7 @@ static int port_set_format(struct impl *this, struct port *port, if (format == NULL) { port->have_format = false; clear_buffers(this, port); - spa_vulkan_unprepare(&this->state); + spa_vulkan_compute_unprepare(&this->state); } else { struct spa_video_info info = { 0 }; @@ -763,7 +695,7 @@ static int port_set_format(struct impl *this, struct port *port, modifiers++; uint64_t fixed_modifier; - if (spa_vulkan_fixate_modifier(&this->state, &this->state.streams[0], &info.info.dsp, modifierCount, modifiers, &fixed_modifier) != 0) + if (spa_vulkan_compute_fixate_modifier(&this->state, &this->state.streams[0], &info.info.dsp, modifierCount, modifiers, &fixed_modifier) != 0) return -EINVAL; spa_log_info(this->log, NAME ": modifier fixated %"PRIu64, fixed_modifier); @@ -782,7 +714,7 @@ static int port_set_format(struct impl *this, struct port *port, port->current_format = info; port->have_format = true; - spa_vulkan_prepare(&this->state); + spa_vulkan_compute_prepare(&this->state); if (modifier_fixed) { port->info.change_mask |= SPA_PORT_CHANGE_MASK_PARAMS; @@ -864,7 +796,7 @@ impl_node_port_use_buffers(void *object, spa_log_info(this->log, "%p: %d:%d add buffer %p", port, direction, port_id, b); spa_list_append(&port->empty, &b->link); } - spa_vulkan_use_buffers(&this->state, &this->state.streams[0], flags, &port->current_format.info.dsp, n_buffers, buffers); + spa_vulkan_compute_use_buffers(&this->state, &this->state.streams[0], flags, &port->current_format.info.dsp, n_buffers, buffers); port->n_buffers = n_buffers; return 0; @@ -985,7 +917,7 @@ static int impl_clear(struct spa_handle *handle) this = (struct impl *) handle; - spa_vulkan_deinit(&this->state); + spa_vulkan_compute_deinit(&this->state); if (this->data_loop) spa_loop_invoke(this->data_loop, do_remove_timer, 0, NULL, 0, true, this); @@ -1075,11 +1007,11 @@ impl_init(const struct spa_handle_factory *factory, spa_list_init(&port->ready); this->state.log = this->log; - spa_vulkan_init_stream(&this->state, &this->state.streams[0], + spa_vulkan_compute_init_stream(&this->state, &this->state.streams[0], SPA_DIRECTION_OUTPUT, NULL); this->state.shaderName = "spa/plugins/vulkan/shaders/main.spv"; this->state.n_streams = 1; - spa_vulkan_init(&this->state); + spa_vulkan_compute_init(&this->state); return 0; } diff --git a/spa/plugins/vulkan/vulkan-compute-utils.c b/spa/plugins/vulkan/vulkan-compute-utils.c index 22f276fa3..18de6773c 100644 --- a/spa/plugins/vulkan/vulkan-compute-utils.c +++ b/spa/plugins/vulkan/vulkan-compute-utils.c @@ -27,20 +27,11 @@ #include "vulkan-compute-utils.h" #include "vulkan-utils.h" +#include "utils.h" #define VULKAN_INSTANCE_FUNCTION(name) \ PFN_##name name = (PFN_##name)vkGetInstanceProcAddr(s->base.instance, #name) -static int createFence(struct vulkan_compute_state *s) { - VkFenceCreateInfo createInfo = { - .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, - .flags = 0, - }; - VK_CHECK_RESULT(vkCreateFence(s->base.device, &createInfo, NULL, &s->fence)); - - return 0; -}; - static int createDescriptors(struct vulkan_compute_state *s) { uint32_t i; @@ -157,6 +148,11 @@ static int updateDescriptors(struct vulkan_compute_state *s) }; descriptorSetLen++; } + // Abort process if any buffer is not valid + if (s->streams[0].current_buffer_id == SPA_ID_INVALID + || (s->n_streams == 2 && s->streams[1].current_buffer_id == SPA_ID_INVALID)) + return -1; + vkUpdateDescriptorSets(s->base.device, descriptorSetLen, writeDescriptorSet, 0, NULL); @@ -460,7 +456,7 @@ static void clear_streams(struct vulkan_compute_state *s) } } -int spa_vulkan_fixate_modifier(struct vulkan_compute_state *s, struct vulkan_stream *p, struct spa_video_info_dsp *dsp_info, +int spa_vulkan_compute_fixate_modifier(struct vulkan_compute_state *s, struct vulkan_stream *p, struct spa_video_info_dsp *dsp_info, uint32_t modifierCount, uint64_t *modifiers, uint64_t *modifier) { VkFormat format = vulkan_id_to_vkformat(dsp_info->format); @@ -479,7 +475,7 @@ int spa_vulkan_fixate_modifier(struct vulkan_compute_state *s, struct vulkan_str return vulkan_fixate_modifier(&s->base, &fixation_info, modifier); } -int spa_vulkan_use_buffers(struct vulkan_compute_state *s, struct vulkan_stream *p, uint32_t flags, +int spa_vulkan_compute_use_buffers(struct vulkan_compute_state *s, struct vulkan_stream *p, uint32_t flags, struct spa_video_info_dsp *dsp_info, uint32_t n_buffers, struct spa_buffer **buffers) { VkFormat format = vulkan_id_to_vkformat(dsp_info->format); @@ -553,16 +549,38 @@ int spa_vulkan_use_buffers(struct vulkan_compute_state *s, struct vulkan_stream return 0; } -int spa_vulkan_init_stream(struct vulkan_compute_state *s, struct vulkan_stream *stream, +int spa_vulkan_compute_enumerate_formats(struct vulkan_compute_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder) +{ + uint32_t fmt_idx; + bool has_modifier; + if (!find_EnumFormatInfo(&s->formatInfos, index, caps, &fmt_idx, &has_modifier)) + return 0; + *param = build_dsp_EnumFormat(&s->formatInfos.infos[fmt_idx], has_modifier, builder); + return 1; +} + +static int vulkan_stream_init(struct vulkan_stream *stream, enum spa_direction direction, + struct spa_dict *props) +{ + spa_zero(*stream); + stream->direction = direction; + stream->current_buffer_id = SPA_ID_INVALID; + stream->busy_buffer_id = SPA_ID_INVALID; + stream->ready_buffer_id = SPA_ID_INVALID; + return 0; +} + +int spa_vulkan_compute_init_stream(struct vulkan_compute_state *s, struct vulkan_stream *stream, enum spa_direction direction, struct spa_dict *props) { return vulkan_stream_init(stream, direction, props); } -int spa_vulkan_prepare(struct vulkan_compute_state *s) +int spa_vulkan_compute_prepare(struct vulkan_compute_state *s) { if (!s->prepared) { - CHECK(createFence(s)); + CHECK(vulkan_fence_create(&s->base, &s->fence)); CHECK(createDescriptors(s)); CHECK(createComputePipeline(s, s->shaderName)); CHECK(createCommandBuffer(s)); @@ -571,7 +589,7 @@ int spa_vulkan_prepare(struct vulkan_compute_state *s) return 0; } -int spa_vulkan_unprepare(struct vulkan_compute_state *s) +int spa_vulkan_compute_unprepare(struct vulkan_compute_state *s) { if (s->prepared) { vkDestroyShaderModule(s->base.device, s->computeShaderModule, NULL); @@ -587,7 +605,7 @@ int spa_vulkan_unprepare(struct vulkan_compute_state *s) return 0; } -int spa_vulkan_start(struct vulkan_compute_state *s) +int spa_vulkan_compute_start(struct vulkan_compute_state *s) { uint32_t i; @@ -600,7 +618,7 @@ int spa_vulkan_start(struct vulkan_compute_state *s) return 0; } -int spa_vulkan_stop(struct vulkan_compute_state *s) +int spa_vulkan_compute_stop(struct vulkan_compute_state *s) { VK_CHECK_RESULT(vkDeviceWaitIdle(s->base.device)); clear_streams(s); @@ -608,7 +626,7 @@ int spa_vulkan_stop(struct vulkan_compute_state *s) return 0; } -int spa_vulkan_ready(struct vulkan_compute_state *s) +int spa_vulkan_compute_ready(struct vulkan_compute_state *s) { uint32_t i; VkResult result; @@ -631,7 +649,7 @@ int spa_vulkan_ready(struct vulkan_compute_state *s) return 0; } -int spa_vulkan_process(struct vulkan_compute_state *s) +int spa_vulkan_compute_process(struct vulkan_compute_state *s) { CHECK(updateDescriptors(s)); CHECK(runCommandBuffer(s)); @@ -641,7 +659,7 @@ int spa_vulkan_process(struct vulkan_compute_state *s) return 0; } -int spa_vulkan_get_buffer_caps(struct vulkan_compute_state *s, enum spa_direction direction) +int spa_vulkan_compute_get_buffer_caps(struct vulkan_compute_state *s, enum spa_direction direction) { switch (direction) { case SPA_DIRECTION_INPUT: @@ -652,24 +670,29 @@ int spa_vulkan_get_buffer_caps(struct vulkan_compute_state *s, enum spa_directio return 0; } -struct vulkan_modifier_info *spa_vulkan_get_modifier_info(struct vulkan_compute_state *s, struct spa_video_info_dsp *info) { +struct vulkan_modifier_info *spa_vulkan_compute_get_modifier_info(struct vulkan_compute_state *s, struct spa_video_info_dsp *info) { VkFormat vk_format = vulkan_id_to_vkformat(info->format); - return vulkan_modifierInfo_find(&s->base, vk_format, info->modifier); + return vulkan_modifierInfo_find(&s->formatInfos, vk_format, info->modifier); } -int spa_vulkan_init(struct vulkan_compute_state *s) +int spa_vulkan_compute_init(struct vulkan_compute_state *s) { + int ret; s->base.log = s->log; - uint32_t dsp_format = SPA_VIDEO_FORMAT_DSP_F32; + uint32_t dsp_formats[] = { + SPA_VIDEO_FORMAT_DSP_F32 + }; struct vulkan_base_info baseInfo = { .queueFlags = VK_QUEUE_COMPUTE_BIT, - .formatInfo.formatCount = 1, - .formatInfo.formats = &dsp_format, }; - return vulkan_base_init(&s->base, &baseInfo); + if ((ret = vulkan_base_init(&s->base, &baseInfo)) < 0) + return ret; + return vulkan_format_infos_init(&s->base, SPA_N_ELEMENTS(dsp_formats), dsp_formats, &s->formatInfos); + } -void spa_vulkan_deinit(struct vulkan_compute_state *s) +void spa_vulkan_compute_deinit(struct vulkan_compute_state *s) { + vulkan_format_infos_deinit(&s->formatInfos); vulkan_base_deinit(&s->base); } diff --git a/spa/plugins/vulkan/vulkan-compute-utils.h b/spa/plugins/vulkan/vulkan-compute-utils.h index d24a68e77..4e1ea1cf8 100644 --- a/spa/plugins/vulkan/vulkan-compute-utils.h +++ b/spa/plugins/vulkan/vulkan-compute-utils.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "vulkan-utils.h" @@ -24,6 +25,18 @@ struct push_constants { int height; }; +struct vulkan_stream { + enum spa_direction direction; + + uint32_t pending_buffer_id; + uint32_t current_buffer_id; + uint32_t busy_buffer_id; + uint32_t ready_buffer_id; + + struct vulkan_buffer buffers[MAX_BUFFERS]; + struct spa_buffer *spa_buffers[MAX_BUFFERS]; + uint32_t n_buffers; +}; struct vulkan_compute_state { struct spa_log *log; @@ -32,6 +45,8 @@ struct vulkan_compute_state { struct vulkan_base base; + struct vulkan_format_infos formatInfos; + VkPipeline pipeline; VkPipelineLayout pipelineLayout; const char *shaderName; @@ -56,25 +71,27 @@ struct vulkan_compute_state { struct vulkan_stream streams[MAX_STREAMS]; }; -int spa_vulkan_init_stream(struct vulkan_compute_state *s, struct vulkan_stream *stream, enum spa_direction, +int spa_vulkan_compute_init_stream(struct vulkan_compute_state *s, struct vulkan_stream *stream, enum spa_direction, struct spa_dict *props); -int spa_vulkan_fixate_modifier(struct vulkan_compute_state *s, struct vulkan_stream *p, struct spa_video_info_dsp *dsp_info, +int spa_vulkan_compute_fixate_modifier(struct vulkan_compute_state *s, struct vulkan_stream *p, struct spa_video_info_dsp *dsp_info, uint32_t modifierCount, uint64_t *modifiers, uint64_t *modifier); -int spa_vulkan_prepare(struct vulkan_compute_state *s); -int spa_vulkan_use_buffers(struct vulkan_compute_state *s, struct vulkan_stream *stream, uint32_t flags, +int spa_vulkan_compute_use_buffers(struct vulkan_compute_state *s, struct vulkan_stream *stream, uint32_t flags, struct spa_video_info_dsp *dsp_info, uint32_t n_buffers, struct spa_buffer **buffers); -int spa_vulkan_unprepare(struct vulkan_compute_state *s); +int spa_vulkan_compute_enumerate_formats(struct vulkan_compute_state *s, uint32_t index, uint32_t caps, + struct spa_pod **param, struct spa_pod_builder *builder); +int spa_vulkan_compute_prepare(struct vulkan_compute_state *s); +int spa_vulkan_compute_unprepare(struct vulkan_compute_state *s); -int spa_vulkan_start(struct vulkan_compute_state *s); -int spa_vulkan_stop(struct vulkan_compute_state *s); -int spa_vulkan_ready(struct vulkan_compute_state *s); -int spa_vulkan_process(struct vulkan_compute_state *s); -int spa_vulkan_cleanup(struct vulkan_compute_state *s); +int spa_vulkan_compute_start(struct vulkan_compute_state *s); +int spa_vulkan_compute_stop(struct vulkan_compute_state *s); +int spa_vulkan_compute_ready(struct vulkan_compute_state *s); +int spa_vulkan_compute_process(struct vulkan_compute_state *s); +int spa_vulkan_compute_cleanup(struct vulkan_compute_state *s); -int spa_vulkan_get_buffer_caps(struct vulkan_compute_state *s, enum spa_direction direction); -struct vulkan_modifier_info *spa_vulkan_get_modifier_info(struct vulkan_compute_state *s, +int spa_vulkan_compute_get_buffer_caps(struct vulkan_compute_state *s, enum spa_direction direction); +struct vulkan_modifier_info *spa_vulkan_compute_get_modifier_info(struct vulkan_compute_state *s, struct spa_video_info_dsp *dsp_info); -int spa_vulkan_init(struct vulkan_compute_state *s); -void spa_vulkan_deinit(struct vulkan_compute_state *s); +int spa_vulkan_compute_init(struct vulkan_compute_state *s); +void spa_vulkan_compute_deinit(struct vulkan_compute_state *s); diff --git a/spa/plugins/vulkan/vulkan-types.h b/spa/plugins/vulkan/vulkan-types.h index 8fd4be363..e8b7ff535 100644 --- a/spa/plugins/vulkan/vulkan-types.h +++ b/spa/plugins/vulkan/vulkan-types.h @@ -25,6 +25,13 @@ struct vulkan_format_info { struct vulkan_modifier_info *infos; }; +struct vulkan_format_infos { + uint32_t formatCount; + struct vulkan_format_info *infos; + + uint32_t formatsWithModifiersCount; +}; + struct vulkan_buffer { int fd; VkImage image; @@ -33,26 +40,13 @@ struct vulkan_buffer { VkSemaphore foreign_semaphore; }; -struct vulkan_stream { - enum spa_direction direction; - - uint32_t pending_buffer_id; - uint32_t current_buffer_id; - uint32_t busy_buffer_id; - uint32_t ready_buffer_id; - - struct vulkan_buffer buffers[MAX_BUFFERS]; - struct spa_buffer *spa_buffers[MAX_BUFFERS]; - uint32_t n_buffers; +struct vulkan_staging_buffer { + VkBuffer buffer; + VkDeviceMemory memory; }; struct vulkan_base_info { uint32_t queueFlags; - - struct { - uint32_t formatCount; - uint32_t *formats; - } formatInfo; }; struct vulkan_base { @@ -66,9 +60,6 @@ struct vulkan_base { uint32_t queueFamilyIndex; VkDevice device; - uint32_t formatInfoCount; - struct vulkan_format_info *formatInfos; - bool implicit_sync_interop; unsigned int initialized:1; diff --git a/spa/plugins/vulkan/vulkan-utils.c b/spa/plugins/vulkan/vulkan-utils.c index 3dfdc0d85..d312f91b4 100644 --- a/spa/plugins/vulkan/vulkan-utils.c +++ b/spa/plugins/vulkan/vulkan-utils.c @@ -103,6 +103,12 @@ static struct { uint32_t id; } vk_video_format_convs[] = { { VK_FORMAT_R32G32B32A32_SFLOAT, SPA_VIDEO_FORMAT_RGBA_F32 }, + { VK_FORMAT_B8G8R8A8_SRGB, SPA_VIDEO_FORMAT_BGRA }, + { VK_FORMAT_R8G8B8A8_SRGB, SPA_VIDEO_FORMAT_RGBA }, + { VK_FORMAT_B8G8R8A8_SRGB, SPA_VIDEO_FORMAT_BGRx }, + { VK_FORMAT_R8G8B8A8_SRGB, SPA_VIDEO_FORMAT_RGBx }, + { VK_FORMAT_B8G8R8_SRGB, SPA_VIDEO_FORMAT_BGR }, + { VK_FORMAT_R8G8B8_SRGB, SPA_VIDEO_FORMAT_RGB }, }; static int createInstance(struct vulkan_base *s) @@ -213,8 +219,10 @@ static int createDevice(struct vulkan_base *s, struct vulkan_base_info *info) VK_KHR_IMAGE_FORMAT_LIST_EXTENSION_NAME, VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, VK_KHR_SYNCHRONIZATION_2_EXTENSION_NAME, - VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME + VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME, + VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME, }; + const VkDeviceCreateInfo deviceCreateInfo = { .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, .queueCreateInfoCount = 1, @@ -231,113 +239,38 @@ static int createDevice(struct vulkan_base *s, struct vulkan_base_info *info) return 0; } -static int queryFormatInfo(struct vulkan_base *s, struct vulkan_base_info *info) +int vulkan_write_pixels(struct vulkan_base *s, struct vulkan_write_pixels_info *info, struct vulkan_staging_buffer *vk_sbuf) { - if (s->formatInfos) - return 0; + void *vmap; + VK_CHECK_RESULT(vkMapMemory(s->device, vk_sbuf->memory, 0, VK_WHOLE_SIZE, 0, &vmap)); - s->formatInfos = calloc(info->formatInfo.formatCount, sizeof(struct vulkan_format_info)); - if (!s->formatInfos) - return -ENOMEM; + char *map = (char *)vmap; - for (uint32_t i = 0; i < info->formatInfo.formatCount; i++) { - VkFormat format = vulkan_id_to_vkformat(info->formatInfo.formats[i]); - if (format == VK_FORMAT_UNDEFINED) - continue; - struct vulkan_format_info *f_info = &s->formatInfos[s->formatInfoCount++]; - f_info->spa_format = info->formatInfo.formats[i]; - f_info->vk_format = format; + // upload data + const char *pdata = info->data; + memcpy(map, pdata, info->stride * info->size.height); - VkDrmFormatModifierPropertiesListEXT modPropsList = { - .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT, - }; - VkFormatProperties2 fmtProps = { - .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2, - .pNext = &modPropsList, - }; - vkGetPhysicalDeviceFormatProperties2(s->physicalDevice, format, &fmtProps); + info->copies[0] = (VkBufferImageCopy) { + .imageExtent.width = info->size.width, + .imageExtent.height = info->size.height, + .imageExtent.depth = 1, + .imageOffset.x = 0, + .imageOffset.y = 0, + .imageOffset.z = 0, + .bufferOffset = 0, + .bufferRowLength = info->size.width, + .bufferImageHeight = info->size.height, + .imageSubresource.mipLevel = 0, + .imageSubresource.baseArrayLayer = 0, + .imageSubresource.layerCount = 1, + .imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + }; - if (!modPropsList.drmFormatModifierCount) - continue; + vkUnmapMemory(s->device, vk_sbuf->memory); - modPropsList.pDrmFormatModifierProperties = calloc(modPropsList.drmFormatModifierCount, - sizeof(modPropsList.pDrmFormatModifierProperties[0])); - if (!modPropsList.pDrmFormatModifierProperties) - continue; - vkGetPhysicalDeviceFormatProperties2(s->physicalDevice, format, &fmtProps); - - f_info->infos = calloc(modPropsList.drmFormatModifierCount, sizeof(f_info->infos[0])); - if (!f_info->infos) { - free(modPropsList.pDrmFormatModifierProperties); - continue; - } - - for (uint32_t j = 0; j < modPropsList.drmFormatModifierCount; j++) { - VkDrmFormatModifierPropertiesEXT props = modPropsList.pDrmFormatModifierProperties[j]; - - if (!(props.drmFormatModifierTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) - continue; - - if (props.drmFormatModifierPlaneCount > DMABUF_MAX_PLANES) - continue; - - VkPhysicalDeviceImageDrmFormatModifierInfoEXT modInfo = { - .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT, - .drmFormatModifier = props.drmFormatModifier, - .sharingMode = VK_SHARING_MODE_EXCLUSIVE, - }; - VkPhysicalDeviceExternalImageFormatInfo extImgFmtInfo = { - .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO, - .pNext = &modInfo, - .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, - }; - VkPhysicalDeviceImageFormatInfo2 imgFmtInfo = { - .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, - .pNext = &extImgFmtInfo, - .type = VK_IMAGE_TYPE_2D, - .format = format, - .usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, - .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT, - }; - - VkExternalImageFormatProperties extImgFmtProps = { - .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES, - }; - VkImageFormatProperties2 imgFmtProps = { - .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2, - .pNext = &extImgFmtProps, - }; - - VK_CHECK_RESULT_LOOP(vkGetPhysicalDeviceImageFormatProperties2(s->physicalDevice, &imgFmtInfo, &imgFmtProps)) - - VkExternalMemoryFeatureFlags extMemFeatures = - extImgFmtProps.externalMemoryProperties.externalMemoryFeatures; - if (!(extMemFeatures & VK_EXTERNAL_MEMORY_FEATURE_EXPORTABLE_BIT)) { - continue; - } - - VkExtent3D max_extent = imgFmtProps.imageFormatProperties.maxExtent; - f_info->infos[f_info->modifierCount++] = (struct vulkan_modifier_info){ - .props = props, - .max_extent = { .width = max_extent.width, .height = max_extent.height }, - }; - - } - free(modPropsList.pDrmFormatModifierProperties); - } return 0; } -static void destroyFormatInfo(struct vulkan_base *s) -{ - for (uint32_t i = 0; i < s->formatInfoCount; i++) { - free(s->formatInfos[i].infos); - } - free(s->formatInfos); - s->formatInfos = NULL; - s->formatInfoCount = 0; -} - int vulkan_read_pixels(struct vulkan_base *s, struct vulkan_read_pixels_info *info, struct vulkan_buffer *vk_buf) { VkImageSubresource img_sub_res = { @@ -353,13 +286,13 @@ int vulkan_read_pixels(struct vulkan_base *s, struct vulkan_read_pixels_info *in const char *d = (const char *)v + img_sub_layout.offset; unsigned char *p = (unsigned char *)info->data + info->offset; - uint32_t bytes_per_pixel = 16; uint32_t pack_stride = img_sub_layout.rowPitch; + spa_log_trace_fp(s->log, "Read pixels: %p to %p, stride: %d, width %d, height %d, offset %d, pack_stride %d", d, p, info->stride, info->size.width, info->size.height, info->offset, pack_stride); if (pack_stride == info->stride) { memcpy(p, d, info->stride * info->size.height); } else { for (uint32_t i = 0; i < info->size.height; i++) { - memcpy(p + i * info->stride, d + i * pack_stride, info->size.width * bytes_per_pixel); + memcpy(p + i * info->stride, d + i * pack_stride, info->size.width * info->bytes_per_pixel); } } vkUnmapMemory(s->device, vk_buf->memory); @@ -420,6 +353,17 @@ bool vulkan_sync_export_dmabuf(struct vulkan_base *s, struct vulkan_buffer *vk_b return dmabuf_import_sync_file(s->log, vk_buf->fd, DMA_BUF_SYNC_WRITE, sync_file_fd); } +int vulkan_fence_create(struct vulkan_base *s, VkFence *fence) +{ + VkFenceCreateInfo createInfo = { + .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, + .flags = 0, + }; + VK_CHECK_RESULT(vkCreateFence(s->device, &createInfo, NULL, fence)); + + return 0; +} + int vulkan_commandPool_create(struct vulkan_base *s, VkCommandPool *commandPool) { const VkCommandPoolCreateInfo commandPoolCreateInfo = { @@ -465,18 +409,18 @@ uint32_t vulkan_memoryType_find(struct vulkan_base *s, return -1; } -struct vulkan_format_info *vulkan_formatInfo_find(struct vulkan_base *s, VkFormat format) +struct vulkan_format_info *vulkan_formatInfo_find(struct vulkan_format_infos *fmtInfo, VkFormat format) { - for (uint32_t i = 0; i < s->formatInfoCount; i++) { - if (s->formatInfos[i].vk_format == format) - return &s->formatInfos[i]; + for (uint32_t i = 0; i < fmtInfo->formatCount; i++) { + if (fmtInfo->infos[i].vk_format == format) + return &fmtInfo->infos[i]; } return NULL; } -struct vulkan_modifier_info *vulkan_modifierInfo_find(struct vulkan_base *s, VkFormat format, uint64_t mod) +struct vulkan_modifier_info *vulkan_modifierInfo_find(struct vulkan_format_infos *fmtInfo, VkFormat format, uint64_t mod) { - struct vulkan_format_info *f_info = vulkan_formatInfo_find(s, format); + struct vulkan_format_info *f_info = vulkan_formatInfo_find(fmtInfo, format); if (!f_info) return NULL; for (uint32_t i = 0; i < f_info->modifierCount; i++) { @@ -506,6 +450,40 @@ static VkImageAspectFlagBits mem_plane_aspect(uint32_t i) } } +int vulkan_staging_buffer_create(struct vulkan_base *s, uint32_t size, struct vulkan_staging_buffer *s_buf) { + VkBufferCreateInfo buf_info = { + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .size = size, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + }; + VK_CHECK_RESULT(vkCreateBuffer(s->device, &buf_info, NULL, &s_buf->buffer)); + + VkMemoryRequirements memoryRequirements; + vkGetBufferMemoryRequirements(s->device, s_buf->buffer, &memoryRequirements); + + VkMemoryAllocateInfo mem_info = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .allocationSize = memoryRequirements.size, + .memoryTypeIndex = vulkan_memoryType_find(s, + memoryRequirements.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT), + }; + VK_CHECK_RESULT(vkAllocateMemory(s->device, &mem_info, NULL, &s_buf->memory)); + VK_CHECK_RESULT(vkBindBufferMemory(s->device, s_buf->buffer, s_buf->memory, 0)); + + return 0; +} + +void vulkan_staging_buffer_destroy(struct vulkan_base *s, struct vulkan_staging_buffer *s_buf) { + if (s_buf->buffer == VK_NULL_HANDLE) + return; + vkFreeMemory(s->device, s_buf->memory, NULL); + vkDestroyBuffer(s->device, s_buf->buffer, NULL); +} + static int allocate_dmabuf(struct vulkan_base *s, VkFormat format, uint32_t modifierCount, uint64_t *modifiers, VkImageUsageFlags usage, struct spa_rectangle *size, struct vulkan_buffer *vk_buf) { VkImageDrmFormatModifierListCreateInfoEXT imageDrmFormatModifierListCreateInfo = { @@ -604,10 +582,12 @@ int vulkan_create_dmabuf(struct vulkan_base *s, struct external_buffer_info *inf int fd = -1; VK_CHECK_RESULT(vkGetMemoryFdKHR(s->device, &getFdInfo, &fd)); + /* const struct vulkan_modifier_info *modInfo = vulkan_modifierInfo_find(s, info->format, info->modifier); if (info->spa_buf->n_datas != modInfo->props.drmFormatModifierPlaneCount) return -1; + */ VkMemoryRequirements memoryRequirements = {0}; vkGetImageMemoryRequirements(s->device, @@ -658,17 +638,21 @@ int vulkan_import_dmabuf(struct vulkan_base *s, struct external_buffer_info *inf if (info->spa_buf->n_datas == 0 || info->spa_buf->n_datas > DMABUF_MAX_PLANES) return -1; + /* struct vulkan_modifier_info *modProps = vulkan_modifierInfo_find(s, info->format, info->modifier); if (!modProps) return -1; + */ uint32_t planeCount = info->spa_buf->n_datas; + /* if (planeCount != modProps->props.drmFormatModifierPlaneCount) return -1; if (info->size.width > modProps->max_extent.width || info->size.height > modProps->max_extent.height) return -1; + */ VkSubresourceLayout planeLayouts[DMABUF_MAX_PLANES] = {0}; for (uint32_t i = 0; i < planeCount; i++) { @@ -817,17 +801,6 @@ int vulkan_import_memptr(struct vulkan_base *s, struct external_buffer_info *inf return 0; } -int vulkan_stream_init(struct vulkan_stream *stream, enum spa_direction direction, - struct spa_dict *props) -{ - spa_zero(*stream); - stream->direction = direction; - stream->current_buffer_id = SPA_ID_INVALID; - stream->busy_buffer_id = SPA_ID_INVALID; - stream->ready_buffer_id = SPA_ID_INVALID; - return 0; -} - uint32_t vulkan_vkformat_to_id(VkFormat format) { SPA_FOR_EACH_ELEMENT_VAR(vk_video_format_convs, f) { @@ -865,13 +838,132 @@ int vulkan_wait_idle(struct vulkan_base *s) return 0; } +int vulkan_format_infos_init(struct vulkan_base *s, uint32_t formatCount, uint32_t *formats, + struct vulkan_format_infos *info) +{ + if (info->infos) + return 0; + + + info->infos = calloc(formatCount, sizeof(struct vulkan_format_info)); + if (!info->infos) + return -ENOMEM; + + uint32_t i; + for (i = 0; i < formatCount; i++) { + VkFormat format = vulkan_id_to_vkformat(formats[i]); + if (format == VK_FORMAT_UNDEFINED) + continue; + struct vulkan_format_info *f_info = &info->infos[info->formatCount++]; + f_info->spa_format = formats[i]; + f_info->vk_format = format; + spa_log_info(s->log, "Adding format %d (spa_format %d)", format, formats[i]); + + VkDrmFormatModifierPropertiesListEXT modPropsList = { + .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT, + }; + VkFormatProperties2 fmtProps = { + .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2, + .pNext = &modPropsList, + }; + vkGetPhysicalDeviceFormatProperties2(s->physicalDevice, format, &fmtProps); + + if (modPropsList.drmFormatModifierCount == 0) { + spa_log_info(s->log, "Format has no modifiers"); + continue; + } + + modPropsList.pDrmFormatModifierProperties = calloc(modPropsList.drmFormatModifierCount, + sizeof(modPropsList.pDrmFormatModifierProperties[0])); + if (!modPropsList.pDrmFormatModifierProperties) { + spa_log_info(s->log, "Failed to allocate DrmFormatModifierProperties"); + continue; + } + vkGetPhysicalDeviceFormatProperties2(s->physicalDevice, format, &fmtProps); + + f_info->infos = calloc(modPropsList.drmFormatModifierCount, sizeof(f_info->infos[0])); + if (!f_info->infos) { + spa_log_info(s->log, "Failed to allocate modifier infos"); + free(modPropsList.pDrmFormatModifierProperties); + continue; + } + + spa_log_info(s->log, "Found %d modifiers", modPropsList.drmFormatModifierCount); + for (uint32_t j = 0; j < modPropsList.drmFormatModifierCount; j++) { + VkDrmFormatModifierPropertiesEXT props = modPropsList.pDrmFormatModifierProperties[j]; + + if (!(props.drmFormatModifierTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) + continue; + + if (props.drmFormatModifierPlaneCount > DMABUF_MAX_PLANES) + continue; + + VkPhysicalDeviceImageDrmFormatModifierInfoEXT modInfo = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT, + .drmFormatModifier = props.drmFormatModifier, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + }; + VkPhysicalDeviceExternalImageFormatInfo extImgFmtInfo = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO, + .pNext = &modInfo, + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, + }; + VkPhysicalDeviceImageFormatInfo2 imgFmtInfo = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, + .pNext = &extImgFmtInfo, + .type = VK_IMAGE_TYPE_2D, + .format = format, + .usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, + .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT, + }; + + VkExternalImageFormatProperties extImgFmtProps = { + .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES, + }; + VkImageFormatProperties2 imgFmtProps = { + .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2, + .pNext = &extImgFmtProps, + }; + + VK_CHECK_RESULT_LOOP(vkGetPhysicalDeviceImageFormatProperties2(s->physicalDevice, &imgFmtInfo, &imgFmtProps)) + + VkExternalMemoryFeatureFlags extMemFeatures = + extImgFmtProps.externalMemoryProperties.externalMemoryFeatures; + if (!(extMemFeatures & VK_EXTERNAL_MEMORY_FEATURE_EXPORTABLE_BIT)) { + continue; + } + + VkExtent3D max_extent = imgFmtProps.imageFormatProperties.maxExtent; + f_info->infos[f_info->modifierCount++] = (struct vulkan_modifier_info){ + .props = props, + .max_extent = { .width = max_extent.width, .height = max_extent.height }, + }; + spa_log_info(s->log, "Adding modifier %"PRIu64, props.drmFormatModifier); + + } + free(modPropsList.pDrmFormatModifierProperties); + } + for (i = 0; i < info->formatCount; i++) { + if (info->infos[i].modifierCount > 0) + info->formatsWithModifiersCount++; + } + return 0; +} + +void vulkan_format_infos_deinit(struct vulkan_format_infos *info) +{ + for (uint32_t i = 0; i < info->formatCount; i++) { + free(info->infos[i].infos); + } + free(info->infos); +} + int vulkan_base_init(struct vulkan_base *s, struct vulkan_base_info *info) { if (!s->initialized) { CHECK(createInstance(s)); CHECK(findPhysicalDevice(s)); CHECK(createDevice(s, info)); - CHECK(queryFormatInfo(s, info)); s->implicit_sync_interop = dmabuf_check_sync_file_import_export(s->log); s->initialized = true; } @@ -881,7 +973,6 @@ int vulkan_base_init(struct vulkan_base *s, struct vulkan_base_info *info) void vulkan_base_deinit(struct vulkan_base *s) { if (s->initialized) { - destroyFormatInfo(s); vkDestroyDevice(s->device, NULL); vkDestroyInstance(s->instance, NULL); s->initialized = false; diff --git a/spa/plugins/vulkan/vulkan-utils.h b/spa/plugins/vulkan/vulkan-utils.h index 9f9611f95..06a18d711 100644 --- a/spa/plugins/vulkan/vulkan-utils.h +++ b/spa/plugins/vulkan/vulkan-utils.h @@ -46,6 +46,16 @@ return _res; \ } +struct vulkan_write_pixels_info { + struct spa_rectangle size; + uint32_t offset; + uint32_t stride; + uint32_t bytes_per_pixel; + + VkBufferImageCopy *copies; + void *data; +}; + struct vulkan_read_pixels_info { struct spa_rectangle size; void *data; @@ -70,29 +80,31 @@ struct external_buffer_info { struct spa_buffer *spa_buf; }; +int vulkan_write_pixels(struct vulkan_base *s, struct vulkan_write_pixels_info *info, struct vulkan_staging_buffer *vk_sbuf); int vulkan_read_pixels(struct vulkan_base *s, struct vulkan_read_pixels_info *info, struct vulkan_buffer *vk_buf); int vulkan_sync_foreign_dmabuf(struct vulkan_base *s, struct vulkan_buffer *vk_buf); bool vulkan_sync_export_dmabuf(struct vulkan_base *s, struct vulkan_buffer *vk_buf, int sync_file_fd); +int vulkan_staging_buffer_create(struct vulkan_base *s, uint32_t size, struct vulkan_staging_buffer *s_buf); +void vulkan_staging_buffer_destroy(struct vulkan_base *s, struct vulkan_staging_buffer *s_buf); + int vulkan_fixate_modifier(struct vulkan_base *s, struct dmabuf_fixation_info *info, uint64_t *modifier); int vulkan_create_dmabuf(struct vulkan_base *s, struct external_buffer_info *info, struct vulkan_buffer *vk_buf); int vulkan_import_dmabuf(struct vulkan_base *s, struct external_buffer_info *info, struct vulkan_buffer *vk_buf); int vulkan_import_memptr(struct vulkan_base *s, struct external_buffer_info *info, struct vulkan_buffer *vk_buf); +int vulkan_fence_create(struct vulkan_base *s, VkFence *fence); int vulkan_commandPool_create(struct vulkan_base *s, VkCommandPool *commandPool); int vulkan_commandBuffer_create(struct vulkan_base *s, VkCommandPool commandPool, VkCommandBuffer *commandBuffer); uint32_t vulkan_memoryType_find(struct vulkan_base *s, uint32_t memoryTypeBits, VkMemoryPropertyFlags properties); -struct vulkan_format_info *vulkan_formatInfo_find(struct vulkan_base *s, VkFormat format); -struct vulkan_modifier_info *vulkan_modifierInfo_find(struct vulkan_base *s, VkFormat format, uint64_t modifier); +struct vulkan_format_info *vulkan_formatInfo_find(struct vulkan_format_infos *fmtInfo, VkFormat format); +struct vulkan_modifier_info *vulkan_modifierInfo_find(struct vulkan_format_infos *fmtInfo, VkFormat format, uint64_t modifier); void vulkan_buffer_clear(struct vulkan_base *s, struct vulkan_buffer *buffer); -int vulkan_stream_init(struct vulkan_stream *stream, enum spa_direction direction, - struct spa_dict *props); - uint32_t vulkan_vkformat_to_id(VkFormat vkFormat); VkFormat vulkan_id_to_vkformat(uint32_t id); @@ -101,5 +113,8 @@ int vulkan_vkresult_to_errno(VkResult result); int vulkan_wait_fence(struct vulkan_base *s, VkFence fence); int vulkan_wait_idle(struct vulkan_base *s); +int vulkan_format_infos_init(struct vulkan_base *s, uint32_t formatCount, uint32_t *formats, + struct vulkan_format_infos *info); +void vulkan_format_infos_deinit(struct vulkan_format_infos *info); int vulkan_base_init(struct vulkan_base *s, struct vulkan_base_info *info); void vulkan_base_deinit(struct vulkan_base *s); diff --git a/src/daemon/pipewire-vulkan.conf.in b/src/daemon/pipewire-vulkan.conf.in index 794827b20..bdd689a69 100644 --- a/src/daemon/pipewire-vulkan.conf.in +++ b/src/daemon/pipewire-vulkan.conf.in @@ -95,4 +95,5 @@ context.objects = [ #{ factory = adapter args = { factory.name = audiotestsrc node.name = my-test node.description = audiotestsrc } } { factory = spa-node-factory args = { factory.name = api.vulkan.compute.source node.name = vulkan-compute-source object.export = true } } { factory = spa-node-factory args = { factory.name = api.vulkan.compute.filter node.name = vulkan-compute-filter object.export = true } } + { factory = spa-node-factory args = { factory.name = api.vulkan.blit.filter node.name = vulkan-blit-filter object.export = true } } ] diff --git a/src/daemon/pipewire.conf.in b/src/daemon/pipewire.conf.in index a9fb1c35d..bedf1a90b 100644 --- a/src/daemon/pipewire.conf.in +++ b/src/daemon/pipewire.conf.in @@ -23,7 +23,7 @@ context.properties = { #cpu.zero.denormals = false core.daemon = true # listening for socket connections - core.name = pipewire-0 # core name and socket name + core.name = pipewire-1 # core name and socket name ## Properties for the DSP configuration. #default.clock.rate = 48000 diff --git a/src/examples/video-play.c b/src/examples/video-play.c index a98a95c55..0eb6dca07 100644 --- a/src/examples/video-play.c +++ b/src/examples/video-play.c @@ -407,6 +407,7 @@ static int build_format(struct data *data, struct spa_pod_builder *b, const stru fprintf(stderr, "supported SDL formats:\n"); spa_debug_format(2, NULL, params[0]); + return 1; params[1] = spa_pod_builder_add_object(b, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), diff --git a/src/pipewire/stream.c b/src/pipewire/stream.c index c6965cfd0..e09f886cc 100644 --- a/src/pipewire/stream.c +++ b/src/pipewire/stream.c @@ -2096,7 +2096,8 @@ pw_stream_connect(struct pw_stream *stream, pw_properties_set(props, PW_KEY_PORT_IGNORE_LATENCY, "true"); } - if (impl->media_type == SPA_MEDIA_TYPE_audio) { + if (impl->media_type == SPA_MEDIA_TYPE_audio + || impl->media_type == SPA_MEDIA_TYPE_video) { factory = pw_context_find_factory(impl->context, "adapter"); if (factory == NULL) { pw_log_error("%p: no adapter factory found", stream);