From c181eb2bf68e2e776cff96b36489de5d45e67fe9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= <daniel@ekloef.se>
Date: Sun, 7 Mar 2021 14:44:16 +0100
Subject: [PATCH] =?UTF-8?q?sixel:=20empty=20pixels=20in=20the=20last=20six?=
 =?UTF-8?q?el=20row=20doesn=E2=80=99t=20contribute=20to=20the=20image=20he?=
 =?UTF-8?q?ight?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

All-empty pixels rows in the last sixel row should not be included in
the final sixel image.

This allows applications to emit sixels whose height is not a multiple
of 6, and is how XTerm works.

This is done by tracking the largest row number that contains
non-empty pixels.

In unhook, when emitting the image, the image height is adjusted based
on this value.
---
 sixel.c    | 18 +++++++++++++++++-
 terminal.h |  1 +
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/sixel.c b/sixel.c
index 55586df5..6216523a 100644
--- a/sixel.c
+++ b/sixel.c
@@ -37,6 +37,7 @@ sixel_init(struct terminal *term)
 
     term->sixel.state = SIXEL_DECSIXEL;
     term->sixel.pos = (struct coord){0, 0};
+    term->sixel.max_non_empty_row_no = 0;
     term->sixel.row_byte_ofs = 0;
     term->sixel.color_idx = 0;
     term->sixel.param = 0;
@@ -695,6 +696,13 @@ sixel_reflow(struct terminal *term)
 void
 sixel_unhook(struct terminal *term)
 {
+    if (term->sixel.image.height > term->sixel.max_non_empty_row_no + 1) {
+        LOG_DBG(
+            "last row only partially filled, reducing image height: %d -> %d",
+            term->sixel.image.height, term->sixel.max_non_empty_row_no + 1);
+        term->sixel.image.height = term->sixel.max_non_empty_row_no + 1;
+    }
+
     int pixel_row_idx = 0;
     int pixel_rows_left = term->sixel.image.height;
     const int stride = term->sixel.image.width * sizeof(uint32_t);
@@ -846,6 +854,7 @@ resize(struct terminal *term, int new_width, int new_height)
     if (new_width > term->sixel.max_width ||
         new_height > term->sixel.max_height)
     {
+        LOG_WARN("maximum image dimensions reached");
         return false;
     }
 
@@ -921,13 +930,20 @@ sixel_add(struct terminal *term, uint32_t color, uint8_t sixel)
     size_t ofs = term->sixel.row_byte_ofs + term->sixel.pos.col;
     uint32_t *data = term->sixel.image.data;
 
+    int max_non_empty_row = 0;
     for (int i = 0; i < 6; i++, sixel >>= 1, ofs += width) {
-        if (sixel & 1)
+        if (sixel & 1) {
             data[ofs] = color;
+            max_non_empty_row = i;
+        }
     }
 
     xassert(sixel == 0);
     term->sixel.pos.col++;
+
+    term->sixel.max_non_empty_row_no = max(
+        term->sixel.max_non_empty_row_no,
+        term->sixel.pos.row + max_non_empty_row);
 }
 
 static void
diff --git a/terminal.h b/terminal.h
index 81893e34..8be5dd74 100644
--- a/terminal.h
+++ b/terminal.h
@@ -521,6 +521,7 @@ struct terminal {
         } state;
 
         struct coord pos;    /* Current sixel coordinate */
+        int max_non_empty_row_no;
         size_t row_byte_ofs; /* Byte position into image, for current row */
         int color_idx;       /* Current palette index */
         uint32_t *private_palette;   /* Private palette, used when private mode 1070 is enabled */