From 1904521a4dcf5bd56827a688cf8862c9fa9ab4d6 Mon Sep 17 00:00:00 2001 From: Wim Taymans Date: Mon, 5 May 2025 10:49:35 +0200 Subject: [PATCH] videoconvert: add PeerFormats support Make a new PeerFormats param that can be set on ports to let it know about the possible peer formats. This can be used by converters to calculate an optimum conversion. make the videoadpter query the follower formats, simplify them and then set them as PeerFormats on the converter. Implement peerformats in videoconvert. This makes EnumFormat on the port depend on the negotiated format of the peer. It will suggest a Format that most closely matches the current negotiated format with the available PeerFormats. This then makes it possible to negotiate to the format that would require the least amount of conversions. --- spa/include/spa/param/param-types.h | 1 + spa/include/spa/param/param.h | 1 + spa/include/spa/pod/simplify.h | 188 ++++++++ spa/plugins/videoconvert/videoadapter.c | 112 +++-- .../videoconvert/videoconvert-ffmpeg.c | 449 ++++++++++++++++-- 5 files changed, 643 insertions(+), 108 deletions(-) create mode 100644 spa/include/spa/pod/simplify.h diff --git a/spa/include/spa/param/param-types.h b/spa/include/spa/param/param-types.h index ebb8d988b..62ffe7da9 100644 --- a/spa/include/spa/param/param-types.h +++ b/spa/include/spa/param/param-types.h @@ -41,6 +41,7 @@ static const struct spa_type_info spa_type_param[] = { { SPA_PARAM_Latency, SPA_TYPE_OBJECT_ParamLatency, SPA_TYPE_INFO_PARAM_ID_BASE "Latency", NULL }, { SPA_PARAM_ProcessLatency, SPA_TYPE_OBJECT_ParamProcessLatency, SPA_TYPE_INFO_PARAM_ID_BASE "ProcessLatency", NULL }, { SPA_PARAM_Tag, SPA_TYPE_OBJECT_ParamTag, SPA_TYPE_INFO_PARAM_ID_BASE "Tag", NULL }, + { SPA_PARAM_PeerFormats, SPA_TYPE_Struct, SPA_TYPE_INFO_PARAM_ID_BASE "PeerFormats", NULL }, { 0, 0, NULL, NULL }, }; diff --git a/spa/include/spa/param/param.h b/spa/include/spa/param/param.h index 51c442c35..be8deb54c 100644 --- a/spa/include/spa/param/param.h +++ b/spa/include/spa/param/param.h @@ -40,6 +40,7 @@ enum spa_param_type { SPA_PARAM_Latency, /**< latency reporting, a SPA_TYPE_OBJECT_ParamLatency */ SPA_PARAM_ProcessLatency, /**< processing latency, a SPA_TYPE_OBJECT_ParamProcessLatency */ SPA_PARAM_Tag, /**< tag reporting, a SPA_TYPE_OBJECT_ParamTag. Since 0.3.79 */ + SPA_PARAM_PeerFormats, /**< peer formats, a SPA_TYPE_Struct of SPA_TYPE_OBJECT_Format. Since 1.5.0 */ }; /** information about a parameter */ diff --git a/spa/include/spa/pod/simplify.h b/spa/include/spa/pod/simplify.h new file mode 100644 index 000000000..dc7803194 --- /dev/null +++ b/spa/include/spa/pod/simplify.h @@ -0,0 +1,188 @@ +/* Simple Plugin API */ +/* SPDX-FileCopyrightText: Copyright © 2018 Wim Taymans */ +/* SPDX-License-Identifier: MIT */ + +#ifndef SPA_POD_SIMPLIFY_H +#define SPA_POD_SIMPLIFY_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifndef SPA_API_POD_SIMPLIFY + #ifdef SPA_API_IMPL + #define SPA_API_POD_SIMPLIFY SPA_API_IMPL + #else + #define SPA_API_POD_SIMPLIFY static inline + #endif +#endif + +/** + * \addtogroup spa_pod + * \{ + */ + +SPA_API_POD_SIMPLIFY int +spa_pod_simplify_merge(struct spa_pod_builder *b, const struct spa_pod *pod1, const struct spa_pod *pod2) +{ + const struct spa_pod_object *o1, *o2; + const struct spa_pod_prop *p1, *p2; + struct spa_pod_frame f[2]; + int res = 0, count = 0; + + if (pod1->type != pod2->type) + return -ENOTSUP; + if (pod1->type != SPA_TYPE_Object) + return -ENOTSUP; + + o1 = (const struct spa_pod_object*) pod1; + o2 = (const struct spa_pod_object*) pod2; + + spa_pod_builder_push_object(b, &f[0], o1->body.type, o1->body.id); + p2 = NULL; + SPA_POD_OBJECT_FOREACH(o1, p1) { + p2 = spa_pod_object_find_prop(o2, p2, p1->key); + if (p2 == NULL) + goto error_enoent; + + if (spa_pod_compare(&p1->value, &p2->value) == 0) { + spa_pod_builder_raw_padded(b, p1, SPA_POD_PROP_SIZE(p1)); + } + else { + uint32_t i, n_vals1, n_vals2, choice1, choice2, size; + const struct spa_pod *vals1, *vals2; + void *alt1, *alt2, *a1, *a2; + + count++; + if (count > 1) + goto error_einval; + + vals1 = spa_pod_get_values(&p1->value, &n_vals1, &choice1); + vals2 = spa_pod_get_values(&p2->value, &n_vals2, &choice2); + + if (vals1->type != vals2->type) + goto error_einval; + + size = vals1->size; + + alt1 = SPA_POD_BODY(vals1); + alt2 = SPA_POD_BODY(vals2); + + if ((choice1 == SPA_CHOICE_None && choice2 == SPA_CHOICE_None) || + (choice1 == SPA_CHOICE_None && choice2 == SPA_CHOICE_Enum) || + (choice1 == SPA_CHOICE_Enum && choice2 == SPA_CHOICE_None) || + (choice1 == SPA_CHOICE_Enum && choice2 == SPA_CHOICE_Enum)) { + spa_pod_builder_prop(b, p1->key, p1->flags); + spa_pod_builder_push_choice(b, &f[1], SPA_CHOICE_Enum, 0); + spa_pod_builder_child(b, size, vals1->type); + for (i = 0, a1 = alt1; i < n_vals1; i++, a1 = SPA_PTROFF(a1,size,void)) { + if (i == 0 && n_vals1 == 1) + spa_pod_builder_raw(b, a1, size); + spa_pod_builder_raw(b, a1, size); + } + for (i = 0, a2 = alt2; i < n_vals2; i++, a2 = SPA_PTROFF(a2,size,void)) { + spa_pod_builder_raw(b, a2, size); + } + spa_pod_builder_pop(b, &f[1]); + } else { + goto error_einval; + } + } + } + p1 = NULL; + SPA_POD_OBJECT_FOREACH(o2, p2) { + p1 = spa_pod_object_find_prop(o1, p1, p2->key); + if (p1 == NULL) + goto error_enoent; + } +done: + spa_pod_builder_pop(b, &f[0]); + return res; + +error_einval: + res = -EINVAL; + goto done; +error_enoent: + res = -ENOENT; + goto done; +} + +SPA_API_POD_SIMPLIFY int +spa_pod_simplify_struct(struct spa_pod_builder *b, const struct spa_pod *pod, uint32_t pod_size) +{ + struct spa_pod *p1 = NULL, *p2; + struct spa_pod_frame f; + struct spa_pod_builder_state state; + uint32_t p1offs; + + spa_pod_builder_push_struct(b, &f); + SPA_POD_STRUCT_FOREACH(pod, p2) { + spa_pod_builder_get_state(b, &state); + if (p1 == NULL || spa_pod_simplify_merge(b, p1, p2) < 0) { + spa_pod_builder_reset(b, &state); + spa_pod_builder_raw_padded(b, p2, SPA_POD_SIZE(p2)); + p1offs = state.offset; + p1 = SPA_PTROFF(b->data, p1offs, struct spa_pod); + } else { + void *pnew = SPA_PTROFF(b->data, state.offset, void); + p1 = SPA_PTROFF(b->data, p1offs, struct spa_pod); + spa_pod_builder_remove(b, SPA_POD_SIZE(p1)); + memmove(p1, pnew, SPA_POD_SIZE(pnew)); + } + } + spa_pod_builder_pop(b, &f); + return 0; +} + +SPA_API_POD_SIMPLIFY int +spa_pod_simplify(struct spa_pod_builder *b, struct spa_pod **result, const struct spa_pod *pod) +{ + int res = 0; + struct spa_pod_builder_state state; + + spa_return_val_if_fail(pod != NULL, -EINVAL); + spa_return_val_if_fail(b != NULL, -EINVAL); + + spa_pod_builder_get_state(b, &state); + + if (!spa_pod_is_struct(pod)) { + res = spa_pod_builder_raw_padded(b, pod, SPA_POD_SIZE(pod)); + } else { + struct spa_pod_dynamic_builder db; + spa_pod_dynamic_builder_continue(&db, b); + res = spa_pod_simplify_struct(&db.b, pod, SPA_POD_SIZE(pod)); + if (res >= 0) + res = spa_pod_builder_raw_padded(b, db.b.data, db.b.state.offset); + spa_pod_dynamic_builder_clean(&db); + } + + if (res >= 0 && result) { + *result = (struct spa_pod*)spa_pod_builder_deref(b, state.offset); + if (*result == NULL) + res = -ENOSPC; + } + return res; +} + +/** + * \} + */ + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* SPA_POD_SIMPLIFY_H */ diff --git a/spa/plugins/videoconvert/videoadapter.c b/spa/plugins/videoconvert/videoadapter.c index d706239e6..1dadf2f14 100644 --- a/spa/plugins/videoconvert/videoadapter.c +++ b/spa/plugins/videoconvert/videoadapter.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -907,6 +908,54 @@ static int impl_node_set_io(void *object, uint32_t id, void *data, size_t size) return res; } +static int update_param_peer_formats(struct impl *impl) +{ + uint8_t buffer[4096]; + spa_auto(spa_pod_dynamic_builder) b = { 0 }; + uint32_t state = 0; + struct spa_pod *param; + struct spa_pod_frame f; + int res; + + if (!impl->recheck_format) + return 0; + + spa_log_debug(impl->log, "updating peer formats"); + + spa_node_send_command(impl->follower, + &SPA_NODE_COMMAND_INIT(SPA_NODE_COMMAND_ParamBegin)); + + spa_pod_dynamic_builder_init(&b, buffer, sizeof(buffer), 4096); + spa_pod_builder_push_struct(&b.b, &f); + + while (true) { + res = node_port_enum_params_sync(impl, impl->follower, + impl->direction, 0, + SPA_PARAM_EnumFormat, &state, + NULL, ¶m, &b.b); + if (res != 1) + break; + } + param = spa_pod_builder_pop(&b.b, &f); + + spa_node_send_command(impl->follower, + &SPA_NODE_COMMAND_INIT(SPA_NODE_COMMAND_ParamEnd)); + + spa_pod_simplify(&b.b, ¶m, param); + spa_debug_log_pod(impl->log, SPA_LOG_LEVEL_DEBUG, 0, NULL, param); + + res = spa_node_port_set_param(impl->target, + SPA_DIRECTION_REVERSE(impl->direction), 0, + SPA_PARAM_PeerFormats, 0, param); + + impl->recheck_format = false; + + spa_log_debug(impl->log, "done updating peer formats: %d", res); + + return 0; +} + + static struct spa_pod *merge_objects(struct impl *this, struct spa_pod_builder *b, uint32_t id, struct spa_pod_object *o1, struct spa_pod_object *o2) { @@ -958,7 +1007,7 @@ static int negotiate_format(struct impl *this) if (this->have_format && !this->recheck_format) return 0; - this->recheck_format = false; + update_param_peer_formats(this); spa_pod_builder_init(&b, buffer, sizeof(buffer)); @@ -1618,55 +1667,6 @@ impl_node_remove_port(void *object, enum spa_direction direction, uint32_t port_ return spa_node_remove_port(this->target, direction, port_id); } -static int -port_enum_formats_for_convert(struct impl *this, int seq, enum spa_direction direction, - uint32_t port_id, uint32_t id, uint32_t start, uint32_t num, - const struct spa_pod *filter) -{ - uint8_t buffer[4096]; - struct spa_pod_builder b = { 0 }; - int res; - uint32_t count = 0; - struct spa_result_node_params result; - - result.id = id; - result.next = start; -next: - result.index = result.next; - - spa_pod_builder_init(&b, buffer, sizeof(buffer)); - - if (result.next < 0x100000) { - /* Enumerate follower formats first, until we have enough or we run out */ - if ((res = node_port_enum_params_sync(this, this->follower, direction, port_id, id, - &result.next, filter, &result.param, &b)) != 1) { - if (res == 0 || res == -ENOENT) { - result.next = 0x100000; - goto next; - } else { - spa_log_error(this->log, "could not enum follower format: %s", spa_strerror(res)); - return res; - } - } - } else if (result.next < 0x200000) { - /* Then enumerate converter formats */ - result.next &= 0xfffff; - if ((res = node_port_enum_params_sync(this, this->convert, direction, port_id, id, - &result.next, filter, &result.param, &b)) != 1) { - return res; - } else { - result.next |= 0x100000; - } - } - - spa_node_emit_result(&this->hooks, seq, 0, SPA_RESULT_TYPE_NODE_PARAMS, &result); - - if (++count < num) - goto next; - - return 0; -} - static int impl_node_port_enum_params(void *object, int seq, enum spa_direction direction, uint32_t port_id, @@ -1683,12 +1683,7 @@ impl_node_port_enum_params(void *object, int seq, spa_log_debug(this->log, "%p: %d %u %u %u", this, seq, id, start, num); - /* We only need special handling for EnumFormat in convert mode */ - if (id == SPA_PARAM_EnumFormat && this->mode == SPA_PARAM_PORT_CONFIG_MODE_convert) - return port_enum_formats_for_convert(this, seq, direction, port_id, id, - start, num, filter); - else - return spa_node_port_enum_params(this->target, seq, direction, port_id, id, + return spa_node_port_enum_params(this->target, seq, direction, port_id, id, start, num, filter); } @@ -1903,7 +1898,7 @@ static int load_converter(struct impl *this, const struct spa_dict *info, factory_name = spa_dict_lookup(&cinfo, "video.adapt.converter"); if (factory_name == NULL) - return 0; + factory_name = "video.convert.ffmpeg"; if (this->ploader) { hnd_convert = spa_plugin_loader_load(this->ploader, factory_name, &cinfo); @@ -2061,6 +2056,9 @@ impl_init(const struct spa_handle_factory *factory, &this->follower_listener, &follower_node_events, this); spa_node_set_callbacks(this->follower, &follower_node_callbacks, this); + if (this->convert != NULL) + update_param_peer_formats(this); + // TODO: adapt port bootstrap for arbitrary converter (incl. dummy) if (this->convert) { spa_node_add_listener(this->convert, diff --git a/spa/plugins/videoconvert/videoconvert-ffmpeg.c b/spa/plugins/videoconvert/videoconvert-ffmpeg.c index 2c8f70097..3852ce187 100644 --- a/spa/plugins/videoconvert/videoconvert-ffmpeg.c +++ b/spa/plugins/videoconvert/videoconvert-ffmpeg.c @@ -35,6 +35,8 @@ #include #include #include +#include +#include #include #undef SPA_LOG_TOPIC_DEFAULT @@ -74,6 +76,7 @@ struct port { uint64_t info_all; struct spa_port_info info; + #define IDX_EnumFormat 0 #define IDX_Meta 1 #define IDX_IO 2 @@ -81,9 +84,14 @@ struct port { #define IDX_Buffers 4 #define IDX_Latency 5 #define IDX_Tag 6 -#define N_PORT_PARAMS 7 +#define IDX_PeerFormats 7 +#define N_PORT_PARAMS 8 struct spa_param_info params[N_PORT_PARAMS]; + struct spa_pod *peer_format_pod; + const struct spa_pod **peer_formats; + uint32_t n_peer_formats; + struct buffer buffers[MAX_BUFFERS]; uint32_t n_buffers; @@ -463,6 +471,7 @@ static int init_port(struct impl *this, enum spa_direction direction, uint32_t p port->params[IDX_Buffers] = SPA_PARAM_INFO(SPA_PARAM_Buffers, 0); port->params[IDX_Latency] = SPA_PARAM_INFO(SPA_PARAM_Latency, SPA_PARAM_INFO_READWRITE); port->params[IDX_Tag] = SPA_PARAM_INFO(SPA_PARAM_Tag, SPA_PARAM_INFO_READWRITE); + port->params[IDX_PeerFormats] = SPA_PARAM_INFO(SPA_PARAM_PeerFormats, SPA_PARAM_INFO_WRITE); port->info.params = port->params; port->info.n_params = N_PORT_PARAMS; @@ -499,6 +508,11 @@ static int deinit_port(struct impl *this, enum spa_direction direction, uint32_t if (port == NULL || !port->valid) return -ENOENT; port->valid = false; + free(port->peer_formats); + port->peer_formats = NULL; + port->n_peer_formats = 0; + free(port->peer_format_pod); + port->peer_format_pod = NULL; spa_node_emit_port_info(&this->hooks, direction, port_id, NULL); return 0; } @@ -763,6 +777,7 @@ static int reconfigure_mode(struct impl *this, enum spa_param_port_config_mode m this->info.change_mask |= SPA_NODE_CHANGE_MASK_FLAGS | SPA_NODE_CHANGE_MASK_PARAMS; this->info.flags &= ~SPA_NODE_FLAG_NEED_CONFIGURE; this->params[IDX_PortConfig].user++; + return 0; } @@ -959,7 +974,6 @@ static int setup_convert(struct impl *this) av_frame_free(&this->decoder.frame); if ((this->decoder.frame = av_frame_alloc()) == NULL) return -EIO; - if (encoder_id) { if ((codec = avcodec_find_encoder(encoder_id)) == NULL) { spa_log_error(this->log, "failed to find %d encoder", encoder_id); @@ -1074,73 +1088,264 @@ impl_node_remove_port(void *object, enum spa_direction direction, uint32_t port_ return -ENOTSUP; } -static int port_param_enum_format(struct impl *this, struct port *port, uint32_t id, - uint32_t index, const struct spa_pod **param, struct spa_pod_builder *b) +static void add_video_formats(struct spa_pod_builder *b, uint32_t def) { - struct dir *other = &this->dir[SPA_DIRECTION_REVERSE(port->direction)]; + uint32_t i; struct spa_pod_frame f[1]; + + spa_pod_builder_prop(b, SPA_FORMAT_VIDEO_format, 0); + spa_pod_builder_push_choice(b, &f[0], SPA_CHOICE_Enum, 0); + if (def == SPA_ID_INVALID) + def = format_info[0].format; + spa_pod_builder_id(b, def); + for (i = 0; i < SPA_N_ELEMENTS(format_info); i++) { + if (format_info[i].pix_fmt != AV_PIX_FMT_NONE) + spa_pod_builder_id(b, format_info[i].format); + } + spa_pod_builder_pop(b, &f[0]); +} + +static struct spa_pod *transform_format(struct impl *this, struct port *port, const struct spa_pod *format, + uint32_t id, struct spa_pod_builder *b) +{ + uint32_t media_type, media_subtype; + struct spa_pod_object *obj; + const struct spa_pod_prop *prop; + struct spa_pod_frame f[2]; + + if (!spa_format_parse(format, &media_type, &media_subtype) || + media_type != SPA_MEDIA_TYPE_video) { + return NULL; + } + + obj = (struct spa_pod_object*)format; + spa_pod_builder_push_object(b, &f[0], obj->body.type, id); + SPA_POD_OBJECT_FOREACH(obj, prop) { + switch (prop->key) { + case SPA_FORMAT_mediaType: + spa_pod_builder_prop(b, prop->key, prop->flags); + spa_pod_builder_id(b, SPA_MEDIA_TYPE_video); + break; + case SPA_FORMAT_mediaSubtype: + spa_pod_builder_prop(b, prop->key, prop->flags); + spa_pod_builder_id(b, SPA_MEDIA_SUBTYPE_raw); + switch (media_subtype) { + case SPA_MEDIA_SUBTYPE_raw: + break; + case SPA_MEDIA_SUBTYPE_mjpg: + add_video_formats(b, SPA_VIDEO_FORMAT_I420); + break; + case SPA_MEDIA_SUBTYPE_h264: + add_video_formats(b, SPA_VIDEO_FORMAT_I420); + break; + default: + return NULL; + } + break; + case SPA_FORMAT_VIDEO_format: + { + uint32_t i, j, n_vals, choice, *id_vals; + struct spa_pod *val = spa_pod_get_values(&prop->value, &n_vals, &choice); + + if (!spa_pod_is_id(val)) + return 0; + + spa_pod_builder_prop(b, SPA_FORMAT_VIDEO_format, 0); + spa_pod_builder_push_choice(b, &f[1], SPA_CHOICE_Enum, 0); + id_vals = SPA_POD_BODY(val); + spa_pod_builder_id(b, id_vals[0]); + /* first add all supported formats */ + for (i = 1; i < n_vals; i++) { + for (j = 0; j < i; j++) { + if (id_vals[j] == id_vals[i]) + break; + } + if (j == i && format_to_pix_fmt(id_vals[i]) != AV_PIX_FMT_NONE) + spa_pod_builder_id(b, id_vals[i]); + } + /* then add all other supported formats */ + for (i = 0; i < SPA_N_ELEMENTS(format_info); i++) { + if (format_info[i].pix_fmt == AV_PIX_FMT_NONE) + continue; + for (j = 1; j < n_vals; j++) { + if (format_info[i].format == id_vals[j]) + break; + } + if (j == n_vals) + spa_pod_builder_id(b, format_info[i].format); + } + spa_pod_builder_pop(b, &f[1]); + break; + } + default: + spa_pod_builder_raw_padded(b, prop, SPA_POD_PROP_SIZE(prop)); + break; + } + } + return spa_pod_builder_pop(b, &f[0]); +} + +static int diff_value(struct impl *impl, uint32_t type, uint32_t size, const void *v1, const void *v2) +{ + switch (type) { + case SPA_TYPE_None: + return 0; + case SPA_TYPE_Bool: + return (!!*(int32_t *)v1) - (!!*(int32_t *)v2); + case SPA_TYPE_Id: + return (*(uint32_t *)v1) != (*(uint32_t *)v2); + case SPA_TYPE_Int: + return *(int32_t *)v1 - *(int32_t *)v2; + case SPA_TYPE_Long: + return *(int64_t *)v1 - *(int64_t *)v2; + case SPA_TYPE_Float: + return (int)(*(float *)v1 - *(float *)v2); + case SPA_TYPE_Double: + return (int)(*(double *)v1 - *(double *)v2); + case SPA_TYPE_String: + return strcmp((char *)v1, (char *)v2); + case SPA_TYPE_Bytes: + return memcmp((char *)v1, (char *)v2, size); + case SPA_TYPE_Rectangle: + { + const struct spa_rectangle *rec1 = (struct spa_rectangle *) v1, + *rec2 = (struct spa_rectangle *) v2; + uint64_t n1 = ((uint64_t) rec1->width) * rec1->height; + uint64_t n2 = ((uint64_t) rec2->width) * rec2->height; + if (rec1->width == rec2->width && rec1->height == rec2->height) + return 0; + else if (n1 < n2) + return -(n2 - n1); + else if (n1 > n2) + return n1 - n2; + else if (rec1->width == rec2->width) + return (int)rec1->height - (int)rec2->height; + else + return (int)rec1->width - (int)rec2->width; + } + case SPA_TYPE_Fraction: + { + const struct spa_fraction *f1 = (struct spa_fraction *) v1, + *f2 = (struct spa_fraction *) v2; + uint64_t n1, n2; + n1 = ((uint64_t) f1->num) * f2->denom; + n2 = ((uint64_t) f2->num) * f1->denom; + return (int) (n1 - n2); + } + default: + break; + } + return 0; +} + +static int diff_prop(struct impl *impl, struct spa_pod_prop *prop, + uint32_t type, const void *target, bool fix) +{ + uint32_t i, n_vals, choice, size; + struct spa_pod *val = spa_pod_get_values(&prop->value, &n_vals, &choice); + void *vals, *v, *best = NULL; + int res = INT_MAX; + + if (SPA_POD_TYPE(val) != type) + return -EINVAL; + + size = SPA_POD_BODY_SIZE(val); + vals = SPA_POD_BODY(val); + + switch (choice) { + case SPA_CHOICE_None: + case SPA_CHOICE_Enum: + for (i = 0, v = vals; i < n_vals; i++, v = SPA_PTROFF(v, size, void)) { + int diff = SPA_ABS(diff_value(impl, type, size, v, target)); + if (diff < res) { + res = diff; + best = v; + } + } + if (fix) { + if (best != NULL && best != vals) + memcpy(vals, best, size); + if (spa_pod_is_choice(&prop->value)) + SPA_POD_CHOICE_TYPE(&prop->value) = SPA_CHOICE_None; + } + break; + default: + return res; + } + return res; +} + +static int calc_diff(struct impl *impl, struct spa_pod *param, struct dir *dir, bool fix) +{ + struct spa_pod_object *obj = (struct spa_pod_object*)param; + struct spa_pod_prop *prop; struct spa_rectangle size; struct spa_fraction framerate; - uint32_t format = 0; + uint32_t format; + int diff = 0; - get_format(other, &format, &size, &framerate); + if (!dir->have_format) + return -1; + + get_format(dir, &format, &size, &framerate); + + SPA_POD_OBJECT_FOREACH(obj, prop) { + switch (prop->key) { + case SPA_FORMAT_VIDEO_format: + diff += diff_prop(impl, prop, SPA_TYPE_Id, &format, fix); + break; + case SPA_FORMAT_VIDEO_size: + diff += diff_prop(impl, prop, SPA_TYPE_Rectangle, &size, fix); + break; + case SPA_FORMAT_VIDEO_framerate: + diff += diff_prop(impl, prop, SPA_TYPE_Fraction, &framerate, fix); + break; + default: + break; + } + } + return diff; +} + +static int all_formats(struct impl *this, struct port *port, uint32_t id, + uint32_t index, const struct spa_pod **param, struct spa_pod_builder *b) +{ + struct spa_pod_frame f[1]; switch (index) { case 0: - if (port->is_dsp) { - struct spa_video_info_dsp info = SPA_VIDEO_INFO_DSP_INIT( - .format = SPA_VIDEO_FORMAT_DSP_F32); - *param = spa_format_video_dsp_build(b, id, &info); - } else if (port->is_control) { - *param = spa_pod_builder_add_object(b, - SPA_TYPE_OBJECT_Format, id, - SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_application), - SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_control), - SPA_FORMAT_CONTROL_types, SPA_POD_CHOICE_FLAGS_Int( - (1u<have_format) { - *param = spa_format_video_build(b, id, &other->format); - } else { - *param = NULL; - } - } - break; - case 1: - if (port->is_dsp || port->is_control) - return 0; - - spa_pod_builder_push_object(b, &f[0], SPA_TYPE_OBJECT_Format, id); + spa_pod_builder_push_object(b, &f[0], + SPA_TYPE_OBJECT_Format, id); spa_pod_builder_add(b, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), - SPA_FORMAT_VIDEO_format, SPA_POD_CHOICE_ENUM_Id(7, - format ? format : SPA_VIDEO_FORMAT_YUY2, - SPA_VIDEO_FORMAT_YUY2, - SPA_VIDEO_FORMAT_I420, - SPA_VIDEO_FORMAT_UYVY, - SPA_VIDEO_FORMAT_YVYU, - SPA_VIDEO_FORMAT_RGBA, - SPA_VIDEO_FORMAT_BGRx), - SPA_FORMAT_VIDEO_size, SPA_POD_CHOICE_RANGE_Rectangle( - &SPA_RECTANGLE(size.width, size.height), - &SPA_RECTANGLE(1, 1), - &SPA_RECTANGLE(INT32_MAX, INT32_MAX)), + 0); + add_video_formats(b, SPA_ID_INVALID); + *param = spa_pod_builder_pop(b, &f[0]); + break; + case 1: + if (this->direction != port->direction) + return 0; + + /* JPEG */ + spa_pod_builder_push_object(b, &f[0], + SPA_TYPE_OBJECT_Format, id); + spa_pod_builder_add(b, + SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), + SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_mjpg), 0); *param = spa_pod_builder_pop(b, &f[0]); break; case 2: - if (port->is_dsp || port->is_control) + if (this->direction != port->direction) return 0; - spa_pod_builder_push_object(b, &f[0], SPA_TYPE_OBJECT_Format, id); + /* H264 */ + spa_pod_builder_push_object(b, &f[0], + SPA_TYPE_OBJECT_Format, id); spa_pod_builder_add(b, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), - SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_mjpg), - SPA_FORMAT_VIDEO_size, SPA_POD_CHOICE_RANGE_Rectangle( - &SPA_RECTANGLE(size.width, size.height), - &SPA_RECTANGLE(1, 1), - &SPA_RECTANGLE(INT32_MAX, INT32_MAX)), + SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_h264), 0); *param = spa_pod_builder_pop(b, &f[0]); break; @@ -1150,6 +1355,71 @@ static int port_param_enum_format(struct impl *this, struct port *port, uint32_t return 1; } +static int port_param_enum_format(struct impl *this, struct port *port, uint32_t id, + uint32_t index, const struct spa_pod **param, struct spa_pod_builder *b) +{ + struct dir *other = &this->dir[SPA_DIRECTION_REVERSE(port->direction)]; + + spa_log_debug(this->log, "%p %d %d %d %d", port, port->valid, this->direction, port->direction, index); + + if (index == 0) { + if (port->is_dsp) { + struct spa_video_info_dsp info = SPA_VIDEO_INFO_DSP_INIT( + .format = SPA_VIDEO_FORMAT_DSP_F32); + *param = spa_format_video_dsp_build(b, id, &info); + return 1; + } else if (port->is_control) { + *param = spa_pod_builder_add_object(b, + SPA_TYPE_OBJECT_Format, id, + SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_application), + SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_control), + SPA_FORMAT_CONTROL_types, SPA_POD_CHOICE_FLAGS_Int( + (1u<have_format) { + /* peer format */ + *param = spa_format_video_build(b, id, &other->format); + } + return 1; + } else if (this->direction != port->direction) { + struct port *oport = GET_PORT(this, SPA_DIRECTION_REVERSE(port->direction), port->id); + if (oport != NULL && oport->valid && + index - 1 < oport->n_peer_formats) { + const struct spa_pod *p = oport->peer_formats[index-1]; + *param = transform_format(this, port, p, id, b); + } else { + return all_formats(this, port, id, index - 1 - + (oport ? oport->n_peer_formats : 0), param, b); + + } + } else if (index == 1) { + const struct spa_pod *best = NULL; + int best_diff = INT_MAX; + uint32_t i; + + for (i = 0; i < port->n_peer_formats; i++) { + const struct spa_pod *p = port->peer_formats[i]; + int diff = calc_diff(this, (struct spa_pod*)p, other, false); + if (diff < 0) + break; + if (diff < best_diff) { + best_diff = diff; + best = p; + } + } + if (best) { + uint32_t offset = b->state.offset; + struct spa_pod *p; + spa_pod_builder_primitive(b, best); + p = spa_pod_builder_deref(b, offset); + calc_diff(this, p, other, true); + *param = p; + } + } else { + return all_formats(this, port, id, index - 2, param, b); + } + return 1; +} static int port_param_format(struct impl *this, struct port *port, uint32_t id, uint32_t index, const struct spa_pod **param, struct spa_pod_builder *b) @@ -1263,6 +1533,16 @@ static int port_param_tag(struct impl *this, struct port *port, uint32_t id, return 0; } +static int port_param_peer_formats(struct impl *this, struct port *port, uint32_t index, + const struct spa_pod **param, struct spa_pod_builder *b) +{ + if (index >= port->n_peer_formats) + return 0; + + *param = port->peer_formats[port->n_peer_formats - index]; + return 1; +} + static int impl_node_port_enum_params(void *object, int seq, enum spa_direction direction, uint32_t port_id, @@ -1318,6 +1598,9 @@ impl_node_port_enum_params(void *object, int seq, case SPA_PARAM_Tag: res = port_param_tag(this, port, id, result.index, ¶m, &b); break; + case SPA_PARAM_PeerFormats: + res = port_param_peer_formats(this, port, result.index, ¶m, &b); + break; default: return -ENOENT; } @@ -1631,6 +1914,68 @@ static int port_set_format(void *object, } +static int port_set_peer_formats(void *object, + enum spa_direction direction, + uint32_t port_id, + uint32_t flags, + const struct spa_pod *formats) +{ + struct impl *this = object; + struct port *port, *oport; + int res = 0; + uint32_t i; + const struct spa_pod *format; + enum spa_direction other = SPA_DIRECTION_REVERSE(direction); + static uint32_t subtypes[] = { + SPA_MEDIA_SUBTYPE_raw, + SPA_MEDIA_SUBTYPE_mjpg, + SPA_MEDIA_SUBTYPE_h264 }; + + spa_return_val_if_fail(spa_pod_is_struct(formats), -EINVAL); + + port = GET_PORT(this, direction, port_id); + oport = GET_PORT(this, other, port_id); + + free(port->peer_formats); + port->peer_formats = NULL; + free(port->peer_format_pod); + port->peer_format_pod = NULL; + port->n_peer_formats = 0; + + if (formats) { + uint32_t count = 0; + port->peer_format_pod = spa_pod_copy(formats); + + for (i = 0; i < SPA_N_ELEMENTS(subtypes); i++) { + SPA_POD_STRUCT_FOREACH(port->peer_format_pod, format) { + uint32_t media_type, media_subtype; + if (!spa_format_parse(format, &media_type, &media_subtype) || + media_type != SPA_MEDIA_TYPE_video || + media_subtype != subtypes[i]) + continue; + count++; + } + } + port->peer_formats = calloc(count, sizeof(struct spa_pod *)); + for (i = 0; i < SPA_N_ELEMENTS(subtypes); i++) { + SPA_POD_STRUCT_FOREACH(port->peer_format_pod, format) { + uint32_t media_type, media_subtype; + if (!spa_format_parse(format, &media_type, &media_subtype) || + media_type != SPA_MEDIA_TYPE_video || + media_subtype != subtypes[i]) + continue; + port->peer_formats[port->n_peer_formats++] = format; + } + } + } + oport->info.change_mask |= SPA_PORT_CHANGE_MASK_PARAMS; + oport->params[IDX_EnumFormat].user++; + port->info.change_mask |= SPA_PORT_CHANGE_MASK_PARAMS; + port->params[IDX_EnumFormat].user++; + port->params[IDX_PeerFormats].user++; + return res; +} + static int impl_node_port_set_param(void *object, enum spa_direction direction, uint32_t port_id, @@ -1657,6 +2002,9 @@ impl_node_port_set_param(void *object, case SPA_PARAM_Format: res = port_set_format(this, direction, port_id, flags, param); break; + case SPA_PARAM_PeerFormats: + res = port_set_peer_formats(this, direction, port_id, flags, param); + break; default: return -ENOENT; } @@ -1760,7 +2108,6 @@ impl_node_port_use_buffers(void *object, spa_log_debug(this->log, "buffer %d: mem:%d passthrough:%p maxsize:%d", i, j, b->datas[j], d[j].maxsize); } - } else { for (j = 0; j < n_datas; j++) { void *data = d[j].data;