Merge branch 'unicode-combining-for-real'

CHANGELOG.md

@@ -22,14 +22,12 @@
 * Right mouse button extends the current selection.
 * `CSI Ps ; Ps ; Ps t` escape sequences for the following parameters:
   `11t`, `13t`, `13;2t`, `14t`, `14;2t`, `15t`, `19t`.
-* (Optional) spport for unicode combining characters. For example,
-  `a\u0301` will be combined to `á` (`\u00e1`). Note that copying the
-  printed character to the clipboard/primary selection will copy the
-  byte `\u00e1` and **not** `\u0061\u0301`. It requires
-  [utf8proc](https://github.com/JuliaStrings/utf8proc). By default,
-  the feature is enabled if utf8proc is found. However, it can also be
-  explicitly disabled (or enabled) with `meson
-  -Dunicode-combining=enabled|disabled`)
+* Unicode combining characters. This feature is optional. By default,
+  it is enabled if
+  [utf8proc](https://github.com/JuliaStrings/utf8proc) is available,
+  but can be explicitly disabled or enabled at compile time with
+  `meson -Dunicode-combining=disabled|enabled`.
+
 
 ### Changed
 

commands.c

@@ -6,9 +6,7 @@
 #include "terminal.h"
 #include "render.h"
 #include "grid.h"
-
-#define max(x, y) ((x) > (y) ? (x) : (y))
-#define min(x, y) ((x) < (y) ? (x) : (y))
+#include "util.h"
 
 void
 cmd_scrollback_up(struct terminal *term, int rows)

config.c

@@ -19,11 +19,9 @@
 #define LOG_ENABLE_DBG 0
 #include "log.h"
 #include "input.h"
+#include "util.h"
 #include "wayland.h"
 
-#define ALEN(v) (sizeof(v) / sizeof(v[0]))
-#define min(x, y) ((x) < (y) ? (x) : (y))
-
 static const uint32_t default_foreground = 0xdcdccc;
 static const uint32_t default_background = 0x111111;
 

csi.c

@@ -16,8 +16,7 @@
 #include "vt.h"
 #include "selection.h"
 #include "sixel.h"
-
-#define min(x, y) ((x) < (y) ? (x) : (y))
+#include "util.h"
 
 #define UNHANDLED()        LOG_DBG("unhandled: %s", csi_as_string(term, final, -1))
 #define UNHANDLED_SGR(idx) LOG_DBG("unhandled: %s", csi_as_string(term, 'm', idx))

grid.c

@@ -7,8 +7,7 @@
 #define LOG_ENABLE_DBG 0
 #include "log.h"
 #include "sixel.h"
-
-#define max(x, y) ((x) > (y) ? (x) : (y))
+#include "util.h"
 
 void
 grid_swap_row(struct grid *grid, int row_a, int row_b, bool initialize)
@@ -35,10 +34,17 @@ grid_row_alloc(int cols, bool initialize)
 
     if (initialize) {
         row->cells = calloc(cols, sizeof(row->cells[0]));
+#if FOOT_UNICODE_COMBINING
+        row->comb_chars = calloc(cols, sizeof(row->comb_chars[0]));
+#endif
         for (size_t c = 0; c < cols; c++)
             row->cells[c].attrs.clean = 1;
-    } else
+    } else {
         row->cells = malloc(cols * sizeof(row->cells[0]));
+#if FOOT_UNICODE_COMBINING
+        row->comb_chars = malloc(cols * sizeof(row->comb_chars[0]));
+#endif
+    }
 
     return row;
 }
@@ -49,6 +55,9 @@ grid_row_free(struct row *row)
     if (row == NULL)
         return;
 
+#if FOOT_UNICODE_COMBINING
+    free(row->comb_chars);
+#endif
     free(row->cells);
     free(row);
 }
@@ -205,6 +214,17 @@ grid_reflow(struct grid *grid, int new_rows, int new_cols,
                 new_row->cells[new_col_idx] = *old_cell;
                 new_row->cells[new_col_idx].attrs.clean = 1;
 
+#if FOOT_UNICODE_COMBINING
+                struct combining_chars *old_comb_chars
+                    = &old_row->comb_chars[c - empty_count + i];
+                struct combining_chars *new_comb_chars
+                    = &new_row->comb_chars[new_col_idx];
+
+                new_comb_chars->count = old_comb_chars->count;
+                for (size_t j = 0; j < ALEN(new_comb_chars->chars); j++)
+                    new_comb_chars->chars[j] = old_comb_chars->chars[j];
+#endif
+
                 /* Translate tracking point(s) */
                 if (is_tracking_point && i >= empty_count) {
                     tll_foreach(tracking_points, it) {

input.c

@@ -29,10 +29,9 @@
 #include "search.h"
 #include "selection.h"
 #include "terminal.h"
+#include "util.h"
 #include "vt.h"
 
-#define ALEN(v) (sizeof(v) / sizeof(v[0]))
-
 static void
 execute_binding(struct terminal *term, enum bind_action_normal action,
                 uint32_t serial)

quirks.c

@@ -7,8 +7,7 @@
 #define LOG_MODULE "quirks"
 #define LOG_ENABLE_DBG 0
 #include "log.h"
-
-#define ALEN(v) (sizeof(v) / sizeof(v[0]))
+#include "util.h"
 
 static bool
 is_weston(void)

render.c

@@ -21,14 +21,11 @@
 #include "quirks.h"
 #include "selection.h"
 #include "shm.h"
+#include "util.h"
 
 #define TIME_FRAME_RENDERING 0
 #define TIME_SCROLL_DAMAGE 0
 
-#define ALEN(v) (sizeof(v) / sizeof((v)[0]))
-#define min(x, y) ((x) < (y) ? (x) : (y))
-#define max(x, y) ((x) > (y) ? (x) : (y))
-
 struct renderer {
     struct fdm *fdm;
     struct wayland *wayl;
@@ -359,8 +356,9 @@ draw_cursor(const struct terminal *term, const struct cell *cell,
 
 static int
 render_cell(struct terminal *term, pixman_image_t *pix,
-            struct cell *cell, int col, int row, bool has_cursor)
+            struct row *row, int col, int row_no, bool has_cursor)
 {
+    struct cell *cell = &row->cells[col];
     if (cell->attrs.clean)
         return 0;
 
@@ -369,7 +367,7 @@ render_cell(struct terminal *term, pixman_image_t *pix,
     int width = term->cell_width;
     int height = term->cell_height;
     int x = term->margins.left + col * width;
-    int y = term->margins.top + row * height;
+    int y = term->margins.top + row_no * height;
 
     assert(cell->attrs.selected == 0 || cell->attrs.selected == 1);
     bool is_selected = cell->attrs.selected;
@@ -444,6 +442,28 @@ render_cell(struct terminal *term, pixman_image_t *pix,
         }
     }
 
+    #if FOOT_UNICODE_COMBINING
+    /* Combining characters */
+    const struct combining_chars *comb_chars = &row->comb_chars[col];
+    for (size_t i = 0; i < comb_chars->count; i++) {
+        const struct fcft_glyph *g = fcft_glyph_rasterize(
+            font, comb_chars->chars[i], term->font_subpixel);
+
+        if (g == NULL)
+            continue;
+
+        pixman_image_t *src = pixman_image_create_solid_fill(&fg);
+        pixman_image_composite32(
+            PIXMAN_OP_OVER, src, g->pix, pix, 0, 0, 0, 0,
+            /* Some fonts use a negative offset, while others use a
+             * "normal" offset */
+            x + (g->x < 0 ? term->cell_width : 0) + g->x,
+            y + font_baseline(term) - g->y,
+            g->width, g->height);
+        pixman_image_unref(src);
+    }
+#endif
+
     /* Underline */
     if (cell->attrs.underline) {
         draw_underline(term, pix, attrs_to_font(term, &cell->attrs),
@@ -763,7 +783,7 @@ static void
 render_row(struct terminal *term, pixman_image_t *pix, struct row *row, int row_no)
 {
     for (int col = term->cols - 1; col >= 0; col--)
-        render_cell(term, pix, &row->cells[col], col, row_no, false);
+        render_cell(term, pix, row, col, row_no, false);
 }
 
 int
@@ -1274,16 +1294,18 @@ grid_render(struct terminal *term)
     pixman_image_set_clip_region(buf->pix, &clip);
 
     /* Erase old cursor (if we rendered a cursor last time) */
-    if (term->render.last_cursor.cell != NULL) {
+    if (term->render.last_cursor.row != NULL) {
 
-        struct cell *cell = term->render.last_cursor.cell;
+        struct row *row = term->render.last_cursor.row;
         struct coord at = term->render.last_cursor.in_view;
-        term->render.last_cursor.cell = NULL;
+        term->render.last_cursor.row = NULL;
+
+        struct cell *cell = &row->cells[at.col];
 
         /* If cell is already dirty, it will be rendered anyway */
         if (cell->attrs.clean) {
             cell->attrs.clean = 0;
-            int cols = render_cell(term, buf->pix, cell, at.col, at.row, false);
+            int cols = render_cell(term, buf->pix, row, at.col, at.row, false);
 
             wl_surface_damage_buffer(
                 term->window->surface,
@@ -1411,9 +1433,9 @@ grid_render(struct terminal *term)
         struct cell *cell = &row->cells[term->grid->cursor.point.col];
 
         cell->attrs.clean = 0;
-        term->render.last_cursor.cell = cell;
+        term->render.last_cursor.row = row;
         int cols_updated = render_cell(
-            term, buf->pix, cell, term->grid->cursor.point.col, view_aligned_row, true);
+            term, buf->pix, row, term->grid->cursor.point.col, view_aligned_row, true);
 
         wl_surface_damage_buffer(
             term->window->surface,
@@ -1807,7 +1829,7 @@ maybe_resize(struct terminal *term, int width, int height, bool force)
     if (term->scroll_region.end >= old_rows)
         term->scroll_region.end = term->rows;
 
-    term->render.last_cursor.cell = NULL;
+    term->render.last_cursor.row = NULL;
 
 damage_view:
     if (!term->window->is_maximized && !term->window->is_fullscreen) {

search.c

@@ -16,8 +16,7 @@
 #include "render.h"
 #include "selection.h"
 #include "shm.h"
-
-#define max(x, y) ((x) > (y) ? (x) : (y))
+#include "util.h"
 
 static bool
 search_ensure_size(struct terminal *term, size_t wanted_size)

selection.c

@@ -17,11 +17,9 @@
 #include "grid.h"
 #include "misc.h"
 #include "render.h"
+#include "util.h"
 #include "vt.h"
 
-#define min(x, y) ((x) < (y) ? (x) : (y))
-#define max(x, y) ((x) > (y) ? (x) : (y))
-
 bool
 selection_enabled(const struct terminal *term)
 {
@@ -48,7 +46,7 @@ selection_on_row_in_view(const struct terminal *term, int row_no)
 static void
 foreach_selected_normal(
     struct terminal *term, struct coord _start, struct coord _end,
-    void (*cb)(struct terminal *term, struct row *row, struct cell *cell, void *data),
+    void (*cb)(struct terminal *term, struct row *row, struct cell *cell, int col, void *data),
     void *data)
 {
     const struct coord *start = &_start;
@@ -82,7 +80,7 @@ foreach_selected_normal(
              c <= (r == end_row ? end_col : term->cols - 1);
              c++)
         {
-            cb(term, row, &row->cells[c], data);
+            cb(term, row, &row->cells[c], c, data);
         }
 
         start_col = 0;
@@ -92,7 +90,7 @@ foreach_selected_normal(
 static void
 foreach_selected_block(
     struct terminal *term, struct coord _start, struct coord _end,
-    void (*cb)(struct terminal *term, struct row *row, struct cell *cell, void *data),
+    void (*cb)(struct terminal *term, struct row *row, struct cell *cell, int col, void *data),
     void *data)
 {
     const struct coord *start = &_start;
@@ -114,14 +112,14 @@ foreach_selected_block(
         assert(row != NULL);
 
         for (int c = top_left.col; c <= bottom_right.col; c++)
-            cb(term, row, &row->cells[c], data);
+            cb(term, row, &row->cells[c], c, data);
     }
 }
 
 static void
 foreach_selected(
     struct terminal *term, struct coord start, struct coord end,
-    void (*cb)(struct terminal *term, struct row *row, struct cell *cell, void *data),
+    void (*cb)(struct terminal *term, struct row *row, struct cell *cell, int col, void *data),
     void *data)
 {
     switch (term->selection.kind) {
@@ -144,6 +142,12 @@ min_bufsize_for_extraction(const struct terminal *term)
 {
     const struct coord *start = &term->selection.start;
     const struct coord *end = &term->selection.end;
+    const size_t chars_per_cell =
+#if FOOT_UNICODE_COMBINING
+        1 + ALEN(term->grid->cur_row->comb_chars[0].chars);
+#else
+        1;
+#endif
 
     switch (term->selection.kind) {
     case SELECTION_NONE:
@@ -162,7 +166,7 @@ min_bufsize_for_extraction(const struct terminal *term)
         }
 
         if (start->row == end->row)
-            return end->col - start->col + 1;
+            return (end->col - start->col + 1) * chars_per_cell;
         else {
             size_t cells = 0;
 
@@ -171,7 +175,7 @@ min_bufsize_for_extraction(const struct terminal *term)
             cells += term->cols - start->col + 1;
             cells += (term->cols + 1) * (end->row - start->row - 1);
             cells += end->col + 1 + 1;
-            return cells;
+            return cells * chars_per_cell;
         }
 
     case SELECTION_BLOCK: {
@@ -188,7 +192,7 @@ min_bufsize_for_extraction(const struct terminal *term)
         /* Add one extra column on each row, for \n */
         int cols = bottom_right.col - top_left.col + 1 + 1;
         int rows = bottom_right.row - top_left.row + 1;
-        return rows * cols;
+        return rows * cols * chars_per_cell;
     }
     }
 
@@ -201,13 +205,13 @@ struct extract {
     size_t size;
     size_t idx;
     size_t empty_count;
-    struct row *last_row;
-    struct cell *last_cell;
+    const struct row *last_row;
+    const struct cell *last_cell;
 };
 
 static void
 extract_one(struct terminal *term, struct row *row, struct cell *cell,
-            void *data)
+            int col, void *data)
 {
     struct extract *ctx = data;
 
@@ -237,6 +241,15 @@ extract_one(struct terminal *term, struct row *row, struct cell *cell,
     assert(ctx->idx + 1 <= ctx->size);
     ctx->buf[ctx->idx++] = cell->wc;
 
+#if FOOT_UNICODE_COMBINING
+    const struct combining_chars *comb_chars = &row->comb_chars[col];
+
+    assert(cell->wc != 0);
+    assert(ctx->idx + comb_chars->count <= ctx->size);
+    for (size_t i = 0; i < comb_chars->count; i++)
+        ctx->buf[ctx->idx++] = comb_chars->chars[i];
+#endif
+
     ctx->last_row = row;
     ctx->last_cell = cell;
 }
@@ -301,7 +314,7 @@ selection_start(struct terminal *term, int col, int row,
 
 static void
 unmark_selected(struct terminal *term, struct row *row, struct cell *cell,
-                void *data)
+                int col, void *data)
 {
     if (cell->attrs.selected == 0 || (cell->attrs.selected & 2)) {
         /* Ignore if already deselected, or if premarked for updated selection */
@@ -315,7 +328,7 @@ unmark_selected(struct terminal *term, struct row *row, struct cell *cell,
 
 static void
 premark_selected(struct terminal *term, struct row *row, struct cell *cell,
-              void *data)
+                 int col, void *data)
 {
     /* Tell unmark to leave this be */
     cell->attrs.selected |= 2;
@@ -323,7 +336,7 @@ premark_selected(struct terminal *term, struct row *row, struct cell *cell,
 
 static void
 mark_selected(struct terminal *term, struct row *row, struct cell *cell,
-              void *data)
+              int col, void *data)
 {
     if (cell->attrs.selected & 1) {
         cell->attrs.selected = 1;  /* Clear the pre-mark bit */

sixel.c

@@ -7,11 +7,7 @@
 #include "log.h"
 #include "render.h"
 #include "sixel-hls.h"
-
-#define ALEN(v) (sizeof(v) / sizeof(v[0]))
-#define max(x, y) ((x) > (y) ? (x) : (y))
-#define min(x, y) ((x) < (y) ? (x) : (y))
-
+#include "util.h"
 
 static size_t count;
 

terminal.c

@@ -27,12 +27,9 @@
 #include "selection.h"
 #include "sixel.h"
 #include "slave.h"
+#include "util.h"
 #include "vt.h"
 
-#define ALEN(v) (sizeof(v) / sizeof(v[0]))
-#define min(x, y) ((x) < (y) ? (x) : (y))
-#define max(x, y) ((x) > (y) ? (x) : (y))
-
 #define PTMX_TIMING 0
 
 static const char *const XCURSOR_LEFT_PTR = "left_ptr";
@@ -1322,7 +1319,7 @@ term_reset(struct terminal *term, bool hard)
     tll_free(term->normal.scroll_damage);
     tll_free(term->alt.damage);
     tll_free(term->alt.scroll_damage);
-    term->render.last_cursor.cell = NULL;
+    term->render.last_cursor.row = NULL;
     term->render.was_flashing = false;
     term_damage_all(term);
 }
@@ -2298,6 +2295,10 @@ term_print(struct terminal *term, wchar_t wc, int width)
     cell->wc = term->vt.last_printed = wc;
     cell->attrs = term->vt.attrs;
 
+#if FOOT_UNICODE_COMBINING
+    row->comb_chars[term->grid->cursor.point.col].count = 0;
+#endif
+
     row->dirty = true;
     cell->attrs.clean = 0;
 

terminal.h

@@ -77,10 +77,21 @@ struct damage {
     int lines;
 };
 
+#if FOOT_UNICODE_COMBINING
+struct combining_chars {
+    uint8_t count;
+    wchar_t chars[2];  /* TODO: how many do we need? */
+};
+#endif
+
 struct row {
     struct cell *cells;
     bool dirty;
     bool linebreak;
+
+#if FOOT_UNICODE_COMBINING
+    struct combining_chars *comb_chars;
+#endif
 };
 
 struct sixel {
@@ -368,7 +379,7 @@ struct terminal {
         struct {
             struct coord actual;     /* Absolute */
             struct coord in_view;    /* Offset by view */
-            struct cell *cell; /* For easy access to content */
+            struct row *row;         /* Actual row TODO: remove */
         } last_cursor;
 
         struct buffer *last_buf;     /* Buffer we rendered to last time */

util.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#define ALEN(v) (sizeof(v) / sizeof((v)[0]))
+#define min(x, y) ((x) < (y) ? (x) : (y))
+#define max(x, y) ((x) > (y) ? (x) : (y))

vt.c

@@ -16,9 +16,7 @@
 #include "dcs.h"
 #include "grid.h"
 #include "osc.h"
-
-#define min(x, y) ((x) < (y) ? (x) : (y))
-#define max(x, y) ((x) > (y) ? (x) : (y))
+#include "util.h"
 
 #define UNHANDLED() LOG_DBG("unhandled: %s", esc_as_string(term, final))
 
@@ -548,10 +546,6 @@ action_utf8_print(struct terminal *term, uint8_t c)
      * We _could_ try regardless of what 'wc' is. However, for
      * performance reasons, we only do it when 'wc' is in a known
      * 'combining' range.
-     *
-     * TODO:
-     *  - doesn't work when base character is multi-column (we'll only
-     *    see an empty "null" character)
      */
 
     if (((wc >= 0x0300 && wc <= 0x036F) || /* diacritical marks */
@@ -561,31 +555,57 @@ action_utf8_print(struct terminal *term, uint8_t c)
          (wc >= 0xFE20 && wc <= 0xFE2F))   /* half marks */
         && term->grid->cursor.point.col > 0)
     {
+        const struct row *row = term->grid->cur_row;
+
         int base_col = term->grid->cursor.point.col;
         if (!term->grid->cursor.lcf)
             base_col--;
 
         assert(base_col >= 0 && base_col < term->cols);
-        wchar_t base = term->grid->cur_row->cells[base_col].wc;
+        wchar_t base = row->cells[base_col].wc;
+
+        /* Handle double-column glyphs */
+        if (base == 0 && base_col > 0) {
+            base_col--;
+            base = row->cells[base_col].wc;
+        }
+
         int base_width = wcwidth(base);
 
-        if (base_width > 0) {
+        if (base != 0 && base_width > 0) {
+
+            /*
+             * First, see if there's a pre-composed character of this
+             * combo, with the same column width as the base
+             * character. If there is, replace the base character with
+             * the pre-composed character, as that is likely to
+             * produce a better looking result.
+             */
+
             wchar_t composed[] = {base, wc};
             ssize_t composed_length = utf8proc_normalize_utf32(
-                composed, sizeof(composed) / sizeof(composed[0]),
-                UTF8PROC_COMPOSE | UTF8PROC_STABLE);
+                composed, ALEN(composed), UTF8PROC_COMPOSE | UTF8PROC_STABLE);
+            int composed_width = wcwidth(composed[0]);
 
-            LOG_DBG("composed = 0x%04x, 0x%04x (length = %zd)",
-                    composed[0], composed[1], composed_length);
-
-            if (composed_length == 1) {
-                /* Compose succeess - overwrite last cell with
-                 * combined character */
+            if (composed_length == 1 && composed_width == base_width) {
                 term->grid->cursor.point.col = base_col;
                 term->grid->cursor.lcf = false;
-                term_print(term, composed[0], wcwidth(composed[0]));
+                term_print(term, composed[0], composed_width);
                 return;
             }
+
+            struct combining_chars *comb_chars = &row->comb_chars[base_col];
+
+            if (comb_chars->count < ALEN(comb_chars->chars))
+                comb_chars->chars[comb_chars->count++] = wc;
+            else {
+                LOG_WARN("combining character overflow:");
+                LOG_WARN("  0x%04x", base);
+                for (size_t i = 0; i < comb_chars->count; i++)
+                    LOG_WARN("  0x%04x", comb_chars->chars[i]);
+                LOG_ERR("  0x%04x", wc);
+            }
+            return;
         }
     }
 #endif /* FOOT_UNICODE_COMBINING */

wayland.c

@@ -28,10 +28,7 @@
 #include "input.h"
 #include "render.h"
 #include "selection.h"
-
-#define ALEN(v) (sizeof(v) / sizeof(v[0]))
-#define min(x, y) ((x) < (y) ? (x) : (y))
-#define max(x, y) ((x) > (y) ? (x) : (y))
+#include "util.h"
 
 static bool wayl_reload_cursor_theme(
     struct wayland *wayl, struct terminal *term);