render/pass: implement antialiasing for fractional coordinates

Signed-off-by: Loukas Agorgianitis <loukas@agorgianitis.com>

include/render/gles2.h

@@ -35,6 +35,7 @@ struct wlr_gles2_tex_shader {
 	GLint tex;
 	GLint alpha;
 	GLint pos_attrib;
+	GLint dst_bounds;
 };
 
 struct wlr_gles2_renderer {
@@ -79,6 +80,7 @@ struct wlr_gles2_renderer {
 			GLint proj;
 			GLint color;
 			GLint pos_attrib;
+			GLint dst_bounds;
 		} quad;
 		struct wlr_gles2_tex_shader tex_rgba;
 		struct wlr_gles2_tex_shader tex_rgbx;

include/render/vulkan.h

@@ -368,6 +368,13 @@ struct wlr_vk_frag_texture_pcr_data {
 	float matrix[4][4]; // only a 3x3 subset is used
 	float alpha;
 	float luminance_multiplier;
+	float _pad[2]; // padding for vec4 alignment
+	float dst_bounds[4]; // x, y, x+width, y+height
+};
+
+struct wlr_vk_frag_quad_pcr_data {
+	float color[4];
+	float dst_bounds[4]; // x, y, x+width, y+height
 };
 
 struct wlr_vk_frag_output_pcr_data {

render/gles2/pass.c

@@ -1,6 +1,8 @@
 #include <stdlib.h>
 #include <assert.h>
+#include <math.h>
 #include <pixman.h>
+#include <stdbool.h>
 #include <time.h>
 #include <unistd.h>
 #include <wlr/render/drm_syncobj.h>
@@ -77,7 +79,12 @@ out:
 
 static void render(const struct wlr_fbox *box, const pixman_region32_t *clip, GLint attrib) {
 	pixman_region32_t region;
-	pixman_region32_init_rect(&region, round(box->x), round(box->y), round(box->width), round(box->height));
+	// Expand region to include edge fragments for AA
+	pixman_region32_init_rect(&region,
+		floor(box->x),
+		floor(box->y),
+		ceil(box->x + box->width) - floor(box->x),
+		ceil(box->y + box->height) - floor(box->y));
 
 	if (clip) {
 		pixman_region32_intersect(&region, &region, clip);
@@ -247,6 +254,12 @@ static void render_pass_add_texture(struct wlr_render_pass *wlr_pass,
 	set_proj_matrix(shader->proj, pass->projection_matrix, &dst_box);
 	set_tex_matrix(shader->tex_proj, options->transform, &src_box);
 
+	glUniform4f(shader->dst_bounds,
+		(float)dst_box.x,
+		(float)dst_box.y,
+		(float)(dst_box.x + dst_box.width),
+		(float)(dst_box.y + dst_box.height));
+
 	render(&dst_box, options->clip, shader->pos_attrib);
 
 	glBindTexture(texture->target, 0);
@@ -270,6 +283,12 @@ static void render_pass_add_rect(struct wlr_render_pass *wlr_pass,
 	set_proj_matrix(renderer->shaders.quad.proj, pass->projection_matrix, &box);
 	glUniform4f(renderer->shaders.quad.color, color->r, color->g, color->b, color->a);
 
+	glUniform4f(renderer->shaders.quad.dst_bounds,
+		(float)box.x,
+		(float)box.y,
+		(float)(box.x + box.width),
+		(float)(box.y + box.height));
+
 	render(&box, options->clip, renderer->shaders.quad.pos_attrib);
 
 	pop_gles2_debug(renderer);

render/gles2/renderer.c

@@ -641,6 +641,7 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	renderer->shaders.quad.proj = glGetUniformLocation(prog, "proj");
 	renderer->shaders.quad.color = glGetUniformLocation(prog, "color");
 	renderer->shaders.quad.pos_attrib = glGetAttribLocation(prog, "pos");
+	renderer->shaders.quad.dst_bounds = glGetUniformLocation(prog, "dst_bounds");
 
 	renderer->shaders.tex_rgba.program = prog =
 		link_program(renderer, common_vert_src, tex_rgba_frag_src);
@@ -652,6 +653,7 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	renderer->shaders.tex_rgba.tex = glGetUniformLocation(prog, "tex");
 	renderer->shaders.tex_rgba.alpha = glGetUniformLocation(prog, "alpha");
 	renderer->shaders.tex_rgba.pos_attrib = glGetAttribLocation(prog, "pos");
+	renderer->shaders.tex_rgba.dst_bounds = glGetUniformLocation(prog, "dst_bounds");
 
 	renderer->shaders.tex_rgbx.program = prog =
 		link_program(renderer, common_vert_src, tex_rgbx_frag_src);
@@ -663,6 +665,7 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	renderer->shaders.tex_rgbx.tex = glGetUniformLocation(prog, "tex");
 	renderer->shaders.tex_rgbx.alpha = glGetUniformLocation(prog, "alpha");
 	renderer->shaders.tex_rgbx.pos_attrib = glGetAttribLocation(prog, "pos");
+	renderer->shaders.tex_rgbx.dst_bounds = glGetUniformLocation(prog, "dst_bounds");
 
 	if (renderer->exts.OES_egl_image_external) {
 		renderer->shaders.tex_ext.program = prog =
@@ -675,6 +678,7 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 		renderer->shaders.tex_ext.tex = glGetUniformLocation(prog, "tex");
 		renderer->shaders.tex_ext.alpha = glGetUniformLocation(prog, "alpha");
 		renderer->shaders.tex_ext.pos_attrib = glGetAttribLocation(prog, "pos");
+		renderer->shaders.tex_ext.dst_bounds = glGetUniformLocation(prog, "dst_bounds");
 	}
 
 	pop_gles2_debug(renderer);

render/gles2/shaders/quad.frag

@@ -7,7 +7,19 @@ precision mediump float;
 varying vec4 v_color;
 varying vec2 v_texcoord;
 uniform vec4 color;
+uniform vec4 dst_bounds; // x, y, x+width, y+height in output coordinates
+
+float compute_coverage() {
+	vec4 d = vec4(
+		gl_FragCoord.x - dst_bounds.x,
+		gl_FragCoord.y - dst_bounds.y,
+		dst_bounds.z - gl_FragCoord.x,
+		dst_bounds.w - gl_FragCoord.y
+	);
+	vec4 cov = clamp(d + 0.5, 0.0, 1.0);
+	return min(min(cov.x, cov.y), min(cov.z, cov.w));
+}
 
 void main() {
-	gl_FragColor = color;
+	gl_FragColor = color * compute_coverage();
 }

render/gles2/shaders/tex_external.frag

@@ -9,7 +9,19 @@ precision mediump float;
 varying vec2 v_texcoord;
 uniform samplerExternalOES texture0;
 uniform float alpha;
+uniform vec4 dst_bounds; // x, y, x+width, y+height in output coordinates
+
+float compute_coverage() {
+	vec4 d = vec4(
+		gl_FragCoord.x - dst_bounds.x,
+		gl_FragCoord.y - dst_bounds.y,
+		dst_bounds.z - gl_FragCoord.x,
+		dst_bounds.w - gl_FragCoord.y
+	);
+	vec4 cov = clamp(d + 0.5, 0.0, 1.0);
+	return min(min(cov.x, cov.y), min(cov.z, cov.w));
+}
 
 void main() {
-	gl_FragColor = texture2D(texture0, v_texcoord) * alpha;
+	gl_FragColor = texture2D(texture0, v_texcoord) * alpha * compute_coverage();
 }

render/gles2/shaders/tex_rgba.frag

@@ -7,7 +7,19 @@ precision mediump float;
 varying vec2 v_texcoord;
 uniform sampler2D tex;
 uniform float alpha;
+uniform vec4 dst_bounds; // x, y, x+width, y+height in output coordinates
+
+float compute_coverage() {
+	vec4 d = vec4(
+		gl_FragCoord.x - dst_bounds.x,
+		gl_FragCoord.y - dst_bounds.y,
+		dst_bounds.z - gl_FragCoord.x,
+		dst_bounds.w - gl_FragCoord.y
+	);
+	vec4 cov = clamp(d + 0.5, 0.0, 1.0);
+	return min(min(cov.x, cov.y), min(cov.z, cov.w));
+}
 
 void main() {
-	gl_FragColor = texture2D(tex, v_texcoord) * alpha;
+	gl_FragColor = texture2D(tex, v_texcoord) * alpha * compute_coverage();
 }

render/gles2/shaders/tex_rgbx.frag

@@ -7,7 +7,19 @@ precision mediump float;
 varying vec2 v_texcoord;
 uniform sampler2D tex;
 uniform float alpha;
+uniform vec4 dst_bounds; // x, y, x+width, y+height in output coordinates
+
+float compute_coverage() {
+	vec4 d = vec4(
+		gl_FragCoord.x - dst_bounds.x,
+		gl_FragCoord.y - dst_bounds.y,
+		dst_bounds.z - gl_FragCoord.x,
+		dst_bounds.w - gl_FragCoord.y
+	);
+	vec4 cov = clamp(d + 0.5, 0.0, 1.0);
+	return min(min(cov.x, cov.y), min(cov.z, cov.w));
+}
 
 void main() {
-	gl_FragColor = vec4(texture2D(tex, v_texcoord).rgb, 1.0) * alpha;
+	gl_FragColor = vec4(texture2D(tex, v_texcoord).rgb, 1.0) * alpha * compute_coverage();
 }

render/vulkan/pass.c

@@ -1,5 +1,7 @@
 #include <assert.h>
 #include <drm_fourcc.h>
+#include <math.h>
+#include <stdbool.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <wlr/util/log.h>
@@ -704,9 +706,17 @@ static void render_pass_add_rect(struct wlr_render_pass *wlr_pass,
 
 	switch (options->blend_mode) {
 	case WLR_RENDER_BLEND_MODE_PREMULTIPLIED:;
+		// Expand the rendered region to include edge pixels for AA
+		struct wlr_fbox render_box = {
+			.x = floor(box.x),
+			.y = floor(box.y),
+			.width = ceil(box.x + box.width) - floor(box.x),
+			.height = ceil(box.y + box.height) - floor(box.y),
+		};
+
 		float proj[9], matrix[9];
 		wlr_matrix_identity(proj);
-		wlr_matrix_project_box(matrix, &box, WL_OUTPUT_TRANSFORM_NORMAL, proj);
+		wlr_matrix_project_box(matrix, &render_box, WL_OUTPUT_TRANSFORM_NORMAL, proj);
 		wlr_matrix_multiply(matrix, pass->projection, matrix);
 
 		struct wlr_vk_pipeline *pipe = setup_get_or_create_pipeline(
@@ -726,12 +736,22 @@ static void render_pass_add_rect(struct wlr_render_pass *wlr_pass,
 		};
 		encode_proj_matrix(matrix, vert_pcr_data.mat4);
 
+		struct wlr_vk_frag_quad_pcr_data frag_pcr_data = {
+			.color = { linear_color[0], linear_color[1], linear_color[2], linear_color[3] },
+			.dst_bounds = {
+				(float)box.x,
+				(float)box.y,
+				(float)(box.x + box.width),
+				(float)(box.y + box.height),
+			},
+		};
+
 		bind_pipeline(pass, pipe->vk);
 		vkCmdPushConstants(cb, pipe->layout->vk,
 			VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(vert_pcr_data), &vert_pcr_data);
 		vkCmdPushConstants(cb, pipe->layout->vk,
-			VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(vert_pcr_data), sizeof(float) * 4,
-			linear_color);
+			VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(vert_pcr_data),
+			sizeof(frag_pcr_data), &frag_pcr_data);
 
 		for (int i = 0; i < clip_rects_len; i++) {
 			VkRect2D rect;
@@ -792,20 +812,39 @@ static void render_pass_add_texture(struct wlr_render_pass *wlr_pass,
 	wlr_render_texture_options_get_dst_box(options, &dst_box);
 	float alpha = wlr_render_texture_options_get_alpha(options);
 
+	// Expand the rendered region to include edge pixels for AA
+	struct wlr_fbox render_box = {
+		.x = floor(dst_box.x),
+		.y = floor(dst_box.y),
+		.width = ceil(dst_box.x + dst_box.width) - floor(dst_box.x),
+		.height = ceil(dst_box.y + dst_box.height) - floor(dst_box.y),
+	};
+
 	float proj[9], matrix[9];
 	wlr_matrix_identity(proj);
-	wlr_matrix_project_box(matrix, &dst_box, options->transform, proj);
+	wlr_matrix_project_box(matrix, &render_box, options->transform, proj);
 	wlr_matrix_multiply(matrix, pass->projection, matrix);
 
+	// Base UV coordinates for sampling src_box from texture
+	float uv_off_x = src_box.x / options->texture->width;
+	float uv_off_y = src_box.y / options->texture->height;
+	float uv_size_x = src_box.width / options->texture->width;
+	float uv_size_y = src_box.height / options->texture->height;
+
+	// Adjust UV offset and size to compensate for expanded render region
+	float x_expand = (render_box.x - dst_box.x) / dst_box.width;
+	float y_expand = (render_box.y - dst_box.y) / dst_box.height;
+	float x_scale = render_box.width / dst_box.width;
+	float y_scale = render_box.height / dst_box.height;
+
+	uv_off_x += x_expand * uv_size_x;
+	uv_off_y += y_expand * uv_size_y;
+	uv_size_x *= x_scale;
+	uv_size_y *= y_scale;
+
 	struct wlr_vk_vert_pcr_data vert_pcr_data = {
-		.uv_off = {
-			src_box.x / options->texture->width,
-			src_box.y / options->texture->height,
-		},
-		.uv_size = {
-			src_box.width / options->texture->width,
-			src_box.height / options->texture->height,
-		},
+		.uv_off = { uv_off_x, uv_off_y },
+		.uv_size = { uv_size_x, uv_size_y },
 	};
 	encode_proj_matrix(matrix, vert_pcr_data.mat4);
 
@@ -901,6 +940,12 @@ static void render_pass_add_texture(struct wlr_render_pass *wlr_pass,
 	struct wlr_vk_frag_texture_pcr_data frag_pcr_data = {
 		.alpha = alpha,
 		.luminance_multiplier = luminance_multiplier,
+		.dst_bounds = {
+			(float)dst_box.x,
+			(float)dst_box.y,
+			(float)(dst_box.x + dst_box.width),
+			(float)(dst_box.y + dst_box.height),
+		},
 	};
 	encode_color_matrix(color_matrix, frag_pcr_data.matrix);
 

render/vulkan/shaders/quad.frag

@@ -3,8 +3,20 @@
 layout(location = 0) out vec4 out_color;
 layout(push_constant) uniform UBO {
 	layout(offset = 80) vec4 color;
+	vec4 dst_bounds; // x, y, x+width, y+height in output coordinates
 } data;
 
-void main() {
-	out_color = data.color;
+float compute_coverage() {
+	vec4 d = vec4(
+		gl_FragCoord.x - data.dst_bounds.x,
+		gl_FragCoord.y - data.dst_bounds.y,
+		data.dst_bounds.z - gl_FragCoord.x,
+		data.dst_bounds.w - gl_FragCoord.y
+	);
+	vec4 cov = clamp(d + 0.5, 0.0, 1.0);
+	return min(min(cov.x, cov.y), min(cov.z, cov.w));
+}
+
+void main() {
+	out_color = data.color * compute_coverage();
 }

render/vulkan/shaders/texture.frag

@@ -10,6 +10,8 @@ layout(push_constant, row_major) uniform UBO {
 	layout(offset = 80) mat4 matrix;
 	float alpha;
 	float luminance_multiplier;
+	vec2 _pad; // padding for vec4 alignment
+	vec4 dst_bounds; // x, y, x+width, y+height in output coordinates
 } data;
 
 layout (constant_id = 0) const int TEXTURE_TRANSFORM = 0;
@@ -57,6 +59,17 @@ vec3 bt1886_color_to_linear(vec3 color) {
 	return (L - Lmin) / (Lmax - Lmin);
 }
 
+float compute_coverage() {
+	vec4 d = vec4(
+		gl_FragCoord.x - data.dst_bounds.x,
+		gl_FragCoord.y - data.dst_bounds.y,
+		data.dst_bounds.z - gl_FragCoord.x,
+		data.dst_bounds.w - gl_FragCoord.y
+	);
+	vec4 cov = clamp(d + 0.5, 0.0, 1.0);
+	return min(min(cov.x, cov.y), min(cov.z, cov.w));
+}
+
 void main() {
 	vec4 in_color = textureLod(tex, uv, 0);
 
@@ -89,5 +102,5 @@ void main() {
 	// Back to pre-multiplied alpha
 	out_color = vec4(rgb * alpha, alpha);
 
-	out_color *= data.alpha;
+	out_color *= data.alpha * compute_coverage();
 }