render/pass: implement antialiasing for fractional coordinates

Signed-off-by: Loukas Agorgianitis <loukas@agorgianitis.com>

render/gles2/pass.c

@@ -1,6 +1,8 @@
 #include <stdlib.h>
 #include <assert.h>
+#include <math.h>
 #include <pixman.h>
+#include <stdbool.h>
 #include <time.h>
 #include <unistd.h>
 #include <wlr/render/drm_syncobj.h>
@@ -77,7 +79,12 @@ out:
 
 static void render(const struct wlr_fbox *box, const pixman_region32_t *clip, GLint attrib) {
 	pixman_region32_t region;
-	pixman_region32_init_rect(&region, round(box->x), round(box->y), round(box->width), round(box->height));
+	// Expand region to include edge fragments for AA
+	pixman_region32_init_rect(&region,
+		floor(box->x),
+		floor(box->y),
+		ceil(box->x + box->width) - floor(box->x),
+		ceil(box->y + box->height) - floor(box->y));
 
 	if (clip) {
 		pixman_region32_intersect(&region, &region, clip);
@@ -247,6 +254,12 @@ static void render_pass_add_texture(struct wlr_render_pass *wlr_pass,
 	set_proj_matrix(shader->proj, pass->projection_matrix, &dst_box);
 	set_tex_matrix(shader->tex_proj, options->transform, &src_box);
 
+	glUniform4f(shader->dst_bounds,
+		(float)dst_box.x,
+		(float)dst_box.y,
+		(float)(dst_box.x + dst_box.width),
+		(float)(dst_box.y + dst_box.height));
+
 	render(&dst_box, options->clip, shader->pos_attrib);
 
 	glBindTexture(texture->target, 0);
@@ -270,6 +283,12 @@ static void render_pass_add_rect(struct wlr_render_pass *wlr_pass,
 	set_proj_matrix(renderer->shaders.quad.proj, pass->projection_matrix, &box);
 	glUniform4f(renderer->shaders.quad.color, color->r, color->g, color->b, color->a);
 
+	glUniform4f(renderer->shaders.quad.dst_bounds,
+		(float)box.x,
+		(float)box.y,
+		(float)(box.x + box.width),
+		(float)(box.y + box.height));
+
 	render(&box, options->clip, renderer->shaders.quad.pos_attrib);
 
 	pop_gles2_debug(renderer);

render/gles2/renderer.c

@@ -641,6 +641,7 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	renderer->shaders.quad.proj = glGetUniformLocation(prog, "proj");
 	renderer->shaders.quad.color = glGetUniformLocation(prog, "color");
 	renderer->shaders.quad.pos_attrib = glGetAttribLocation(prog, "pos");
+	renderer->shaders.quad.dst_bounds = glGetUniformLocation(prog, "dst_bounds");
 
 	renderer->shaders.tex_rgba.program = prog =
 		link_program(renderer, common_vert_src, tex_rgba_frag_src);
@@ -652,6 +653,7 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	renderer->shaders.tex_rgba.tex = glGetUniformLocation(prog, "tex");
 	renderer->shaders.tex_rgba.alpha = glGetUniformLocation(prog, "alpha");
 	renderer->shaders.tex_rgba.pos_attrib = glGetAttribLocation(prog, "pos");
+	renderer->shaders.tex_rgba.dst_bounds = glGetUniformLocation(prog, "dst_bounds");
 
 	renderer->shaders.tex_rgbx.program = prog =
 		link_program(renderer, common_vert_src, tex_rgbx_frag_src);
@@ -663,6 +665,7 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 	renderer->shaders.tex_rgbx.tex = glGetUniformLocation(prog, "tex");
 	renderer->shaders.tex_rgbx.alpha = glGetUniformLocation(prog, "alpha");
 	renderer->shaders.tex_rgbx.pos_attrib = glGetAttribLocation(prog, "pos");
+	renderer->shaders.tex_rgbx.dst_bounds = glGetUniformLocation(prog, "dst_bounds");
 
 	if (renderer->exts.OES_egl_image_external) {
 		renderer->shaders.tex_ext.program = prog =
@@ -675,6 +678,7 @@ struct wlr_renderer *wlr_gles2_renderer_create(struct wlr_egl *egl) {
 		renderer->shaders.tex_ext.tex = glGetUniformLocation(prog, "tex");
 		renderer->shaders.tex_ext.alpha = glGetUniformLocation(prog, "alpha");
 		renderer->shaders.tex_ext.pos_attrib = glGetAttribLocation(prog, "pos");
+		renderer->shaders.tex_ext.dst_bounds = glGetUniformLocation(prog, "dst_bounds");
 	}
 
 	pop_gles2_debug(renderer);

render/gles2/shaders/quad.frag

@@ -7,7 +7,19 @@ precision mediump float;
 varying vec4 v_color;
 varying vec2 v_texcoord;
 uniform vec4 color;
+uniform vec4 dst_bounds; // x, y, x+width, y+height in output coordinates
+
+float compute_coverage() {
+	vec4 d = vec4(
+		gl_FragCoord.x - dst_bounds.x,
+		gl_FragCoord.y - dst_bounds.y,
+		dst_bounds.z - gl_FragCoord.x,
+		dst_bounds.w - gl_FragCoord.y
+	);
+	vec4 cov = clamp(d + 0.5, 0.0, 1.0);
+	return min(min(cov.x, cov.y), min(cov.z, cov.w));
+}
 
 void main() {
-	gl_FragColor = color;
+	gl_FragColor = color * compute_coverage();
 }

render/gles2/shaders/tex_external.frag

@@ -9,7 +9,19 @@ precision mediump float;
 varying vec2 v_texcoord;
 uniform samplerExternalOES texture0;
 uniform float alpha;
+uniform vec4 dst_bounds; // x, y, x+width, y+height in output coordinates
+
+float compute_coverage() {
+	vec4 d = vec4(
+		gl_FragCoord.x - dst_bounds.x,
+		gl_FragCoord.y - dst_bounds.y,
+		dst_bounds.z - gl_FragCoord.x,
+		dst_bounds.w - gl_FragCoord.y
+	);
+	vec4 cov = clamp(d + 0.5, 0.0, 1.0);
+	return min(min(cov.x, cov.y), min(cov.z, cov.w));
+}
 
 void main() {
-	gl_FragColor = texture2D(texture0, v_texcoord) * alpha;
+	gl_FragColor = texture2D(texture0, v_texcoord) * alpha * compute_coverage();
 }

render/gles2/shaders/tex_rgba.frag

@@ -7,7 +7,19 @@ precision mediump float;
 varying vec2 v_texcoord;
 uniform sampler2D tex;
 uniform float alpha;
+uniform vec4 dst_bounds; // x, y, x+width, y+height in output coordinates
+
+float compute_coverage() {
+	vec4 d = vec4(
+		gl_FragCoord.x - dst_bounds.x,
+		gl_FragCoord.y - dst_bounds.y,
+		dst_bounds.z - gl_FragCoord.x,
+		dst_bounds.w - gl_FragCoord.y
+	);
+	vec4 cov = clamp(d + 0.5, 0.0, 1.0);
+	return min(min(cov.x, cov.y), min(cov.z, cov.w));
+}
 
 void main() {
-	gl_FragColor = texture2D(tex, v_texcoord) * alpha;
+	gl_FragColor = texture2D(tex, v_texcoord) * alpha * compute_coverage();
 }

render/gles2/shaders/tex_rgbx.frag

@@ -7,7 +7,19 @@ precision mediump float;
 varying vec2 v_texcoord;
 uniform sampler2D tex;
 uniform float alpha;
+uniform vec4 dst_bounds; // x, y, x+width, y+height in output coordinates
+
+float compute_coverage() {
+	vec4 d = vec4(
+		gl_FragCoord.x - dst_bounds.x,
+		gl_FragCoord.y - dst_bounds.y,
+		dst_bounds.z - gl_FragCoord.x,
+		dst_bounds.w - gl_FragCoord.y
+	);
+	vec4 cov = clamp(d + 0.5, 0.0, 1.0);
+	return min(min(cov.x, cov.y), min(cov.z, cov.w));
+}
 
 void main() {
-	gl_FragColor = vec4(texture2D(tex, v_texcoord).rgb, 1.0) * alpha;
+	gl_FragColor = vec4(texture2D(tex, v_texcoord).rgb, 1.0) * alpha * compute_coverage();
 }