render/color: replace COLOR_TRANSFORM_LUT_3D with COLOR_TRANSFORM_LCMS2

Converting the LCMS2 transform to a 3D LUT early causes issues:

- It's a lossy process, the consumer will not be able to pick a
  3D LUT size on their own.
- It requires unnecessary conversions and allocations: an intermediate
  3D LUT is allocated, but the renderer already allocates one.
- It makes it harder to support arbitrary color transforms in the
  renderer, because each type needs to be handled differently.

Instead, expose a function to evaluate a color transform, and use
that to build the 3D LUT in the renderer.

include/render/color.h

@@ -7,7 +7,7 @@
 
 enum wlr_color_transform_type {
 	COLOR_TRANSFORM_SRGB,
-	COLOR_TRANSFORM_LUT_3D,
+	COLOR_TRANSFORM_LCMS2,
 };
 
 struct wlr_color_transform {
@@ -17,37 +17,24 @@ struct wlr_color_transform {
 	enum wlr_color_transform_type type;
 };
 
-/**
- * The formula is approximated via a 3D look-up table. A 3D LUT is a
- * three-dimensional array where each element is an RGB triplet. The flat lut_3d
- * array has a length of dim_len³.
- *
- * Color channel values in the range [0.0, 1.0] are mapped linearly to
- * 3D LUT indices such that 0.0 maps exactly to the first element and 1.0 maps
- * exactly to the last element in each dimension.
- *
- * The offset of the RGB triplet given red, green and blue indices r_index,
- * g_index and b_index is:
- *
- *     offset = 3 * (r_index + dim_len * g_index + dim_len * dim_len * b_index)
- */
-struct wlr_color_transform_lut3d {
-	struct wlr_color_transform base;
-
-	float *lut_3d;
-	size_t dim_len;
-};
-
 void wlr_color_transform_init(struct wlr_color_transform *tr,
 	enum wlr_color_transform_type type);
 
 /**
- * Gets a wlr_color_transform_lut3d from a generic wlr_color_transform.
+ * Get a struct wlr_color_transform_lcms2 from a generic struct wlr_color_transform.
- * Asserts that the base type is COLOR_TRANSFORM_LUT_3D
+ * Asserts that the base type is COLOR_TRANSFORM_LCMS2.
  */
-struct wlr_color_transform_lut3d *wlr_color_transform_lut3d_from_base(
+struct wlr_color_transform_lcms2 *color_transform_lcms2_from_base(
 	struct wlr_color_transform *tr);
 
+void color_transform_lcms2_finish(struct wlr_color_transform_lcms2 *tr);
+
+/**
+ * Evaluate a LCMS2 color transform for a given RGB triplet.
+ */
+void color_transform_lcms2_eval(struct wlr_color_transform_lcms2 *tr,
+	float out[static 3], const float in[static 3]);
+
 /**
  * Obtain primaries values from a well-known primaries name.
  */

include/render/vulkan.h

@@ -516,6 +516,7 @@ struct wlr_vk_color_transform {
 	struct wl_list link; // wlr_vk_renderer, list of all color transforms
 
 	struct {
+		size_t dim;
 		VkImage image;
 		VkImageView image_view;
 		VkDeviceMemory memory;

render/color.c

@@ -42,10 +42,8 @@ static void color_transform_destroy(struct wlr_color_transform *tr) {
 	switch (tr->type) {
 	case COLOR_TRANSFORM_SRGB:
 		break;
-	case COLOR_TRANSFORM_LUT_3D:;
+	case COLOR_TRANSFORM_LCMS2:
-		struct wlr_color_transform_lut3d *lut3d =
+		color_transform_lcms2_finish(color_transform_lcms2_from_base(tr));
-			wlr_color_transform_lut3d_from_base(tr);
-		free(lut3d->lut_3d);
 		break;
 	}
 	wlr_addon_set_finish(&tr->addons);
@@ -68,13 +66,6 @@ void wlr_color_transform_unref(struct wlr_color_transform *tr) {
 	}
 }
 
-struct wlr_color_transform_lut3d *wlr_color_transform_lut3d_from_base(
-		struct wlr_color_transform *tr) {
-	assert(tr->type == COLOR_TRANSFORM_LUT_3D);
-	struct wlr_color_transform_lut3d *lut3d = wl_container_of(tr, lut3d, base);
-	return lut3d;
-}
-
 void wlr_color_primaries_from_named(struct wlr_color_primaries *out,
 		enum wlr_color_named_primaries named) {
 	switch (named) {

render/color_fallback.c

@@ -1,5 +1,6 @@
-#include <wlr/render/color.h>
+#include <stdlib.h>
 #include <wlr/util/log.h>
+#include "render/color.h"
 
 struct wlr_color_transform *wlr_color_transform_init_linear_to_icc(
 		const void *data, size_t size) {
@@ -7,3 +8,17 @@ struct wlr_color_transform *wlr_color_transform_init_linear_to_icc(
 		"LCMS2 is compile-time disabled");
 	return NULL;
 }
+
+struct wlr_color_transform_lcms2 *color_transform_lcms2_from_base(
+		struct wlr_color_transform *tr) {
+	abort(); // unreachable
+}
+
+void color_transform_lcms2_finish(struct wlr_color_transform_lcms2 *tr) {
+	abort(); // unreachable
+}
+
+void color_transform_lcms2_eval(struct wlr_color_transform_lcms2 *tr,
+		float out[static 3], const float in[static 3]) {
+	abort(); // unreachable
+}

render/color_lcms2.c

@@ -1,9 +1,17 @@
+#include <assert.h>
 #include <lcms2.h>
 #include <stdlib.h>
 #include <wlr/util/log.h>
 #include <wlr/render/color.h>
 #include "render/color.h"
 
+struct wlr_color_transform_lcms2 {
+	struct wlr_color_transform base;
+
+	cmsContext ctx;
+	cmsHTRANSFORM lcms;
+};
+
 static const cmsCIExyY srgb_whitepoint = { 0.3127, 0.3291, 1 };
 
 static const cmsCIExyYTRIPLE srgb_primaries = {
@@ -18,7 +26,7 @@ static void handle_lcms_error(cmsContext ctx, cmsUInt32Number code, const char *
 
 struct wlr_color_transform *wlr_color_transform_init_linear_to_icc(
 		const void *data, size_t size) {
-	struct wlr_color_transform_lut3d *tx = NULL;
+	struct wlr_color_transform_lcms2 *tx = NULL;
 
 	cmsContext ctx = cmsCreateContext(NULL, NULL);
 	if (ctx == NULL) {
@@ -31,18 +39,18 @@ struct wlr_color_transform *wlr_color_transform_init_linear_to_icc(
 	cmsHPROFILE icc_profile = cmsOpenProfileFromMemTHR(ctx, data, size);
 	if (icc_profile == NULL) {
 		wlr_log(WLR_ERROR, "cmsOpenProfileFromMemTHR failed");
-		goto out_ctx;
+		goto error_ctx;
 	}
 
 	if (cmsGetDeviceClass(icc_profile) != cmsSigDisplayClass) {
 		wlr_log(WLR_ERROR, "ICC profile must have the Display device class");
-		goto out_icc_profile;
+		goto error_icc_profile;
 	}
 
 	cmsToneCurve *linear_tone_curve = cmsBuildGamma(ctx, 1);
 	if (linear_tone_curve == NULL) {
 		wlr_log(WLR_ERROR, "cmsBuildGamma failed");
-		goto out_icc_profile;
+		goto error_icc_profile;
 	}
 
 	cmsToneCurve *linear_tf[] = {
@@ -52,66 +60,54 @@ struct wlr_color_transform *wlr_color_transform_init_linear_to_icc(
 	};
 	cmsHPROFILE srgb_profile = cmsCreateRGBProfileTHR(ctx, &srgb_whitepoint,
 		&srgb_primaries, linear_tf);
+	cmsFreeToneCurve(linear_tone_curve);
 	if (srgb_profile == NULL) {
 		wlr_log(WLR_ERROR, "cmsCreateRGBProfileTHR failed");
-		goto out_linear_tone_curve;
+		goto error_icc_profile;
 	}
 
 	cmsHTRANSFORM lcms_tr = cmsCreateTransformTHR(ctx,
 		srgb_profile, TYPE_RGB_FLT, icc_profile, TYPE_RGB_FLT,
 		INTENT_RELATIVE_COLORIMETRIC, 0);
+	cmsCloseProfile(srgb_profile);
+	cmsCloseProfile(icc_profile);
 	if (lcms_tr == NULL) {
 		wlr_log(WLR_ERROR, "cmsCreateTransformTHR failed");
-		goto out_srgb_profile;
+		goto error_ctx;
-	}
-
-	size_t dim_len = 33;
-	float *lut_3d = calloc(3 * dim_len * dim_len * dim_len, sizeof(float));
-	if (lut_3d == NULL) {
-		wlr_log_errno(WLR_ERROR, "Allocation failed");
-		goto out_lcms_tr;
-	}
-
-	float factor = 1.0f / (dim_len - 1);
-	for (size_t b_index = 0; b_index < dim_len; b_index++) {
-		for (size_t g_index = 0; g_index < dim_len; g_index++) {
-			for (size_t r_index = 0; r_index < dim_len; r_index++) {
-				float rgb_in[3] = {
-					r_index * factor,
-					g_index * factor,
-					b_index * factor,
-				};
-				float rgb_out[3];
-				// TODO: use a single call to cmsDoTransform for the entire calculation?
-				// this does require allocating an extra temp buffer
-				cmsDoTransform(lcms_tr, rgb_in, rgb_out, 1);
-
-				size_t offset = 3 * (r_index + dim_len * g_index + dim_len * dim_len * b_index);
-				// TODO: maybe clamp values to [0.0, 1.0] here?
-				lut_3d[offset] = rgb_out[0];
-				lut_3d[offset + 1] = rgb_out[1];
-				lut_3d[offset + 2] = rgb_out[2];
-			}
-		}
 	}
 
 	tx = calloc(1, sizeof(*tx));
 	if (!tx) {
-		goto out_lcms_tr;
+		cmsDeleteTransform(lcms_tr);
+		goto error_ctx;
 	}
-	wlr_color_transform_init(&tx->base, COLOR_TRANSFORM_LUT_3D);
+	wlr_color_transform_init(&tx->base, COLOR_TRANSFORM_LCMS2);
-	tx->dim_len = dim_len;
+
-	tx->lut_3d = lut_3d;
+	tx->ctx = ctx;
+	tx->lcms = lcms_tr;
 
-out_lcms_tr:
-	cmsDeleteTransform(lcms_tr);
-out_linear_tone_curve:
-	cmsFreeToneCurve(linear_tone_curve);
-out_srgb_profile:
-	cmsCloseProfile(srgb_profile);
-out_icc_profile:
-	cmsCloseProfile(icc_profile);
-out_ctx:
-	cmsDeleteContext(ctx);
 	return &tx->base;
+
+error_icc_profile:
+	cmsCloseProfile(icc_profile);
+error_ctx:
+	cmsDeleteContext(ctx);
+	return NULL;
+}
+
+void color_transform_lcms2_finish(struct wlr_color_transform_lcms2 *tr) {
+	cmsDeleteTransform(tr->lcms);
+	cmsDeleteContext(tr->ctx);
+}
+
+struct wlr_color_transform_lcms2 *color_transform_lcms2_from_base(
+		struct wlr_color_transform *tr) {
+	assert(tr->type == COLOR_TRANSFORM_LCMS2);
+	struct wlr_color_transform_lcms2 *lcms2 = wl_container_of(tr, lcms2, base);
+	return lcms2;
+}
+
+void color_transform_lcms2_eval(struct wlr_color_transform_lcms2 *tr,
+		float out[static 3], const float in[static 3]) {
+	cmsDoTransform(tr->lcms, in, out, 1);
 }

render/vulkan/pass.c

@@ -179,11 +179,12 @@ static bool render_pass_submit(struct wlr_render_pass *wlr_pass) {
 			.uv_size = { 1, 1 },
 		};
 
+		struct wlr_vk_color_transform *transform = NULL;
 		size_t dim = 1;
-		if (pass->color_transform && pass->color_transform->type == COLOR_TRANSFORM_LUT_3D) {
+		if (pass->color_transform && pass->color_transform->type != COLOR_TRANSFORM_SRGB) {
-			struct wlr_color_transform_lut3d *lut3d =
+			transform = get_color_transform(pass->color_transform, renderer);
-				wlr_color_transform_lut3d_from_base(pass->color_transform);
+			assert(transform);
-			dim = lut3d->dim_len;
+			dim = transform->lut_3d.dim;
 		}
 
 		struct wlr_vk_frag_output_pcr_data frag_pcr_data = {
@@ -204,10 +205,7 @@ static bool render_pass_submit(struct wlr_render_pass *wlr_pass) {
 			sizeof(frag_pcr_data), &frag_pcr_data);
 
 		VkDescriptorSet lut_ds;
-		if (pass->color_transform && pass->color_transform->type == COLOR_TRANSFORM_LUT_3D) {
+		if (transform != NULL) {
-			struct wlr_vk_color_transform *transform =
-				get_color_transform(pass->color_transform, renderer);
-			assert(transform);
 			lut_ds = transform->lut_3d.ds;
 		} else {
 			lut_ds = renderer->output_ds_lut3d_dummy;
@@ -840,9 +838,9 @@ void vk_color_transform_destroy(struct wlr_addon *addon) {
 }
 
 static bool create_3d_lut_image(struct wlr_vk_renderer *renderer,
-		const struct wlr_color_transform_lut3d *lut_3d,
+		struct wlr_color_transform_lcms2 *tr, size_t dim_len,
 		VkImage *image, VkImageView *image_view,
-		VkDeviceMemory *memory,	VkDescriptorSet *ds,
+		VkDeviceMemory *memory, VkDescriptorSet *ds,
 		struct wlr_vk_descriptor_pool **ds_pool) {
 	VkDevice dev = renderer->dev->dev;
 	VkResult res;
@@ -866,7 +864,7 @@ static bool create_3d_lut_image(struct wlr_vk_renderer *renderer,
 		.samples = VK_SAMPLE_COUNT_1_BIT,
 		.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
 		.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
-		.extent = (VkExtent3D) { lut_3d->dim_len, lut_3d->dim_len, lut_3d->dim_len },
+		.extent = (VkExtent3D) { dim_len, dim_len, dim_len },
 		.tiling = VK_IMAGE_TILING_OPTIMAL,
 		.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT,
 	};
@@ -927,7 +925,7 @@ static bool create_3d_lut_image(struct wlr_vk_renderer *renderer,
 	}
 
 	size_t bytes_per_block = 4 * sizeof(float);
-	size_t size = lut_3d->dim_len * lut_3d->dim_len * lut_3d->dim_len * bytes_per_block;
+	size_t size = dim_len * dim_len * dim_len * bytes_per_block;
 	struct wlr_vk_buffer_span span = vulkan_get_stage_span(renderer,
 		size, bytes_per_block);
 	if (!span.buffer || span.alloc.size != size) {
@@ -935,18 +933,26 @@ static bool create_3d_lut_image(struct wlr_vk_renderer *renderer,
 		goto fail_imageview;
 	}
 
-	char *map = (char*)span.buffer->cpu_mapping + span.alloc.start;
+	float sample_range = 1.0f / (dim_len - 1);
-	float *dst = (float*)map;
+	char *map = (char *)span.buffer->cpu_mapping + span.alloc.start;
-	size_t dim_len = lut_3d->dim_len;
+	float *dst = (float *)map;
 	for (size_t b_index = 0; b_index < dim_len; b_index++) {
 		for (size_t g_index = 0; g_index < dim_len; g_index++) {
 			for (size_t r_index = 0; r_index < dim_len; r_index++) {
 				size_t sample_index = r_index + dim_len * g_index + dim_len * dim_len * b_index;
-				size_t src_offset = 3 * sample_index;
 				size_t dst_offset = 4 * sample_index;
-				dst[dst_offset] = lut_3d->lut_3d[src_offset];
+
-				dst[dst_offset + 1] = lut_3d->lut_3d[src_offset + 1];
+				float rgb_in[3] = {
-				dst[dst_offset + 2] = lut_3d->lut_3d[src_offset + 2];
+					r_index * sample_range,
+					g_index * sample_range,
+					b_index * sample_range,
+				};
+				float rgb_out[3];
+				color_transform_lcms2_eval(tr, rgb_out, rgb_in);
+
+				dst[dst_offset] = rgb_out[0];
+				dst[dst_offset + 1] = rgb_out[1];
+				dst[dst_offset + 2] = rgb_out[2];
 				dst[dst_offset + 3] = 1.0;
 			}
 		}
@@ -959,9 +965,9 @@ static bool create_3d_lut_image(struct wlr_vk_renderer *renderer,
 		VK_ACCESS_TRANSFER_WRITE_BIT);
 	VkBufferImageCopy copy = {
 		.bufferOffset = span.alloc.start,
-		.imageExtent.width = lut_3d->dim_len,
+		.imageExtent.width = dim_len,
-		.imageExtent.height = lut_3d->dim_len,
+		.imageExtent.height = dim_len,
-		.imageExtent.depth = lut_3d->dim_len,
+		.imageExtent.depth = dim_len,
 		.imageSubresource.layerCount = 1,
 		.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 	};
@@ -1012,9 +1018,11 @@ static struct wlr_vk_color_transform *vk_color_transform_create(
 		return NULL;
 	}
 
-	if (transform->type == COLOR_TRANSFORM_LUT_3D) {
+	if (transform->type == COLOR_TRANSFORM_LCMS2) {
+		vk_transform->lut_3d.dim = 33;
 		if (!create_3d_lut_image(renderer,
-				wlr_color_transform_lut3d_from_base(transform),
+				color_transform_lcms2_from_base(transform),
+				vk_transform->lut_3d.dim,
 				&vk_transform->lut_3d.image,
 				&vk_transform->lut_3d.image_view,
 				&vk_transform->lut_3d.memory,