From b9ef703eb14ba2357b847b62a715b4e97f690209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 20 Aug 2020 19:25:35 +0200 Subject: [PATCH 01/30] wip: grapheme shaping --- .builds/alpine-x64.yml | 5 +- .builds/alpine-x86.yml.disabled | 5 +- .builds/freebsd-x64.yml | 5 +- .gitlab-ci.yml | 24 +++- config.c | 30 ++++ config.h | 2 + doc/foot.ini.5.scd | 22 +++ extract.c | 5 +- meson.build | 8 +- meson_options.txt | 3 + render.c | 128 +++++++++++------ search.c | 6 +- selection.c | 8 +- terminal.h | 17 ++- util.h | 8 ++ vt.c | 242 +++++++++++++++++--------------- 16 files changed, 340 insertions(+), 178 deletions(-) diff --git a/.builds/alpine-x64.yml b/.builds/alpine-x64.yml index 3884e66b..6823c834 100644 --- a/.builds/alpine-x64.yml +++ b/.builds/alpine-x64.yml @@ -13,6 +13,7 @@ packages: - freetype-dev - fontconfig-dev - harfbuzz-dev + - utf8proc-dev - pixman-dev - libxkbcommon-dev - ncurses @@ -33,12 +34,12 @@ sources: tasks: - debug: | mkdir -p bld/debug - meson --buildtype=debug -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/debug + meson --buildtype=debug -Dgrapheme-clustering=enabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/debug ninja -C bld/debug -k0 meson test -C bld/debug --print-errorlogs - release: | mkdir -p bld/release - meson --buildtype=minsize -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/release + meson --buildtype=minsize -Dgrapheme-clustering=enabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/release ninja -C bld/release -k0 meson test -C bld/release --print-errorlogs - codespell: | diff --git a/.builds/alpine-x86.yml.disabled b/.builds/alpine-x86.yml.disabled index 52bb6e26..1866e15a 100644 --- a/.builds/alpine-x86.yml.disabled +++ b/.builds/alpine-x86.yml.disabled @@ -14,6 +14,7 @@ packages: - freetype-dev - fontconfig-dev - harfbuzz-dev + - utf8proc-dev - pixman-dev - libxkbcommon-dev - ncurses @@ -32,11 +33,11 @@ sources: tasks: - debug: | mkdir -p bld/debug - meson --buildtype=debug -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/debug + meson --buildtype=debug -Dgrapheme-clustering=enabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/debug ninja -C bld/debug -k0 meson test -C bld/debug --print-errorlogs - release: | mkdir -p bld/release - meson --buildtype=minsize -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/release + meson --buildtype=minsize -Dgrapheme-clustering=enabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/release ninja -C bld/release -k0 meson test -C bld/release --print-errorlogs diff --git a/.builds/freebsd-x64.yml b/.builds/freebsd-x64.yml index d14ae645..7fc65233 100644 --- a/.builds/freebsd-x64.yml +++ b/.builds/freebsd-x64.yml @@ -11,6 +11,7 @@ packages: - freetype2 - fontconfig - harfbuzz + - utf8proc - pixman - libxkbcommon - check @@ -28,11 +29,11 @@ sources: tasks: - debug: | mkdir -p bld/debug - meson --buildtype=debug -Dterminfo=disabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/debug + meson --buildtype=debug -Dterminfo=disabled -Dgrapheme-clustering=enabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/debug ninja -C bld/debug -k0 meson test -C bld/debug --print-errorlogs - release: | mkdir -p bld/release - meson --buildtype=minsize -Dterminfo=disabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/release + meson --buildtype=minsize -Dterminfo=disabled -Dgrapheme-clustering=enabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true foot bld/release ninja -C bld/release -k0 meson test -C bld/release --print-errorlogs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf61ada4..05992cf8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ variables: before_script: - apk update - apk add musl-dev linux-headers meson ninja gcc scdoc ncurses - - apk add libxkbcommon-dev pixman-dev freetype-dev fontconfig-dev harfbuzz-dev + - apk add libxkbcommon-dev pixman-dev freetype-dev fontconfig-dev harfbuzz-dev utf8proc-dev - apk add wayland-dev wayland-protocols - apk add git - apk add check-dev @@ -19,7 +19,21 @@ debug-x64: script: - mkdir -p bld/debug - cd bld/debug - - meson --buildtype=debug -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true ../../ + - meson --buildtype=debug -Dgrapheme-clustering=enabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true ../../ + - ninja -v -k0 + - ninja -v test + artifacts: + reports: + junit: bld/debug/meson-logs/testlog.junit.xml + +debug-x64-no-grapheme-clustering: + image: alpine:edge + stage: build + script: + - apk del harfbuzz harfbuzz-dev utf8proc utf8proc-dev + - mkdir -p bld/debug + - cd bld/debug + - meson --buildtype=debug -Dgrapheme-clustering=disabled -Dfcft:text-shaping=disabled -Dfcft:test-text-shaping=false ../../ - ninja -v -k0 - ninja -v test artifacts: @@ -32,7 +46,7 @@ release-x64: script: - mkdir -p bld/release - cd bld/release - - meson --buildtype=release -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true ../../ + - meson --buildtype=release -Dgrapheme-clustering=enabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true ../../ - ninja -v -k0 - ninja -v test artifacts: @@ -45,7 +59,7 @@ debug-x86: script: - mkdir -p bld/debug - cd bld/debug - - meson --buildtype=debug -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true ../../ + - meson --buildtype=debug -Dgrapheme-clustering=enabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true ../../ - ninja -v -k0 - ninja -v test artifacts: @@ -58,7 +72,7 @@ release-x86: script: - mkdir -p bld/release - cd bld/release - - meson --buildtype=release -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true ../../ + - meson --buildtype=release -Dgrapheme-clustering=enabled -Dfcft:text-shaping=enabled -Dfcft:test-text-shaping=true ../../ - ninja -v -k0 - ninja -v test artifacts: diff --git a/config.c b/config.c index 81392043..a95f8485 100644 --- a/config.c +++ b/config.c @@ -2129,6 +2129,33 @@ parse_section_tweak( LOG_WARN("tweak: damage whole window"); } + else if (strcmp(key, "grapheme-shaping") == 0) { + conf->tweak.grapheme_shaping = str_to_bool(value); + +#if !defined(FOOT_GRAPHEME_CLUSTERING) + if (conf->tweak.grapheme_shaping) { + LOG_AND_NOTIFY_WARN( + "%s:%d: [tweak]: " + "grapheme-shaping enabled but foot was not compiled with " + "support for it", path, lineno); + conf->tweak.grapheme_shaping = false; + } +#endif + + if (conf->tweak.grapheme_shaping && !conf->can_shape_grapheme) { + LOG_WARN( + "%s:%d [tweak]: " + "grapheme-shaping enabled but fcft was not compiled with " + "support for it", path, lineno); + + /* Keep it enabled though - this will cause us to do + * grapheme-clustering at least */ + } + + if (conf->tweak.grapheme_shaping) + LOG_WARN("tweak: grapheme shaping"); + } + else if (strcmp(key, "render-timer") == 0) { if (strcmp(value, "none") == 0) { conf->tweak.render_timer_osd = false; @@ -2580,6 +2607,7 @@ config_load(struct config *conf, const char *conf_path, config_override_t *overrides, bool errors_are_fatal) { bool ret = false; + enum fcft_capabilities fcft_caps = fcft_capabilities(); *conf = (struct config) { .term = xstrdup(DEFAULT_TERM), @@ -2620,6 +2648,7 @@ config_load(struct config *conf, const char *conf_path, .label_letters = xwcsdup(L"sadfjklewcmpgh"), .osc8_underline = OSC8_UNDERLINE_URL_MODE, }, + .can_shape_grapheme = fcft_caps & FCFT_CAPABILITY_GRAPHEME_SHAPING, .scrollback = { .lines = 1000, .indicator = { @@ -2694,6 +2723,7 @@ config_load(struct config *conf, const char *conf_path, .tweak = { .fcft_filter = FCFT_SCALING_FILTER_LANCZOS3, .allow_overflowing_double_width_glyphs = true, + .grapheme_shaping = false, .delayed_render_lower_ns = 500000, /* 0.5ms */ .delayed_render_upper_ns = 16666666 / 2, /* half a frame period (60Hz) */ .max_shm_pool_size = 512 * 1024 * 1024, diff --git a/config.h b/config.h index c3c36b6a..ee427267 100644 --- a/config.h +++ b/config.h @@ -111,6 +111,7 @@ struct config { struct pt_or_px underline_offset; bool box_drawings_uses_font_glyphs; + bool can_shape_grapheme; struct { bool urgent; @@ -244,6 +245,7 @@ struct config { struct { enum fcft_scaling_filter fcft_filter; bool allow_overflowing_double_width_glyphs; + bool grapheme_shaping; bool render_timer_osd; bool render_timer_log; bool damage_whole_window; diff --git a/doc/foot.ini.5.scd b/doc/foot.ini.5.scd index 58134060..94ad42f6 100644 --- a/doc/foot.ini.5.scd +++ b/doc/foot.ini.5.scd @@ -975,6 +975,28 @@ any of these options. Default: _no_. +*grapheme-shaping* + Boolean. When enabled, foot will use _utf8proc_ to do grapheme + cluster segmentation while parsing "printed" text. Then, when + rendering, it will use _fcft_ (if compiled with _HarfBuzz_ + support) to shape the grapheme clusters. + + This is required to render e.g. flag (emoji) sequences, keycap + sequences, modifier sequences, zero-width-joiner (ZWJ) sequences + andn emoji tag sequences. + + This is an experimental feature with the following requirements and limitations: + + - foot must have been compiled with utf8proc support + - fcft must have been compiled with HarfBuzz support + - This option must be set to true + - Foot will use *wcswidth*(3) to calculate a cluster's display + width. This will typically _not_ match the shaped glyph's width, + but is necessary to not break cursor synchronization with the + application running in foot. + + Default: _no_ + *max-shm-pool-size-mb* This option controls the amount of virtual address space used by the pixmap memory to which the terminal screen content is diff --git a/extract.c b/extract.c index 07144597..8aa1acf3 100644 --- a/extract.c +++ b/extract.c @@ -229,12 +229,11 @@ extract_one(const struct terminal *term, const struct row *row, const struct composed *composed = &term->composed[cell->wc - CELL_COMB_CHARS_LO]; - if (!ensure_size(ctx, 1 + composed->count)) + if (!ensure_size(ctx, composed->count)) goto err; - ctx->buf[ctx->idx++] = composed->base; for (size_t i = 0; i < composed->count; i++) - ctx->buf[ctx->idx++] = composed->combining[i]; + ctx->buf[ctx->idx++] = composed->chars[i]; } else { diff --git a/meson.build b/meson.build index 14d23975..107fd905 100644 --- a/meson.build +++ b/meson.build @@ -71,6 +71,11 @@ wayland_client = dependency('wayland-client') wayland_cursor = dependency('wayland-cursor') xkb = dependency('xkbcommon', version: '>=1.0.0') fontconfig = dependency('fontconfig') +utf8proc = dependency('libutf8proc', required: get_option('grapheme-clustering')) + +if utf8proc.found() + add_project_arguments('-DFOOT_GRAPHEME_CLUSTERING=1', language: 'c') +endif tllist = dependency('tllist', version: '>=1.0.4', fallback: 'tllist') fcft = dependency('fcft', version: ['>=2.4.0', '<3.0.0'], fallback: 'fcft') @@ -149,7 +154,7 @@ vtlib = static_library( 'vt.c', 'vt.h', wl_proto_src + wl_proto_headers, version, - dependencies: [libepoll, pixman, fcft, tllist, wayland_client, xkb], + dependencies: [libepoll, pixman, fcft, tllist, wayland_client, xkb, utf8proc], link_with: [common, misc], ) @@ -247,6 +252,7 @@ subdir('icons') summary( { 'IME': get_option('ime'), + 'Grapheme clustering': utf8proc.found(), 'Terminfo': tic.found(), 'Terminfo install location': terminfo_install_location, }, diff --git a/meson_options.txt b/meson_options.txt index 52625b8d..57b5721a 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -1,5 +1,8 @@ option('ime', type: 'boolean', value: true, description: 'IME (Input Method Editor) support') +option('grapheme-clustering', type: 'feature', + description: 'Enables grapheme clustering using libutf8proc. Requires fcft with harfbuzz support to be useful.') + option('terminfo', type: 'feature', description: 'Build terminfo. When disabled, foot\'s terminfo will not be built, and foot will default to \'xterm-256color\' instead of \'foot\'.') option('terminfo-install-location', type: 'string', description: 'Where to install the foot terminfo files, relative to the installation prefix. If set to \'disabled\', the terminfo files are not installed at all (useful when packaging the terminfo files in a separate package). Defaults to $datadir/terminfo.') diff --git a/render.c b/render.c index f96ddb9e..54269bf3 100644 --- a/render.c +++ b/render.c @@ -500,19 +500,15 @@ render_cell(struct terminal *term, pixman_image_t *pix, } struct fcft_font *font = attrs_to_font(term, &cell->attrs); - const struct fcft_glyph *glyph = NULL; const struct composed *composed = NULL; + const struct fcft_grapheme *grapheme = NULL; + const struct fcft_glyph *single = NULL; + const struct fcft_glyph **glyphs = NULL; + unsigned glyph_count = 0; wchar_t base = cell->wc; if (base != 0) { - if (base >= CELL_COMB_CHARS_LO && - base < (CELL_COMB_CHARS_LO + term->composed_count)) - { - composed = &term->composed[base - CELL_COMB_CHARS_LO]; - base = composed->base; - } - if (unlikely( /* Classic box drawings */ (base >= 0x2500 && base <= 0x259f) || @@ -528,7 +524,7 @@ render_cell(struct terminal *term, pixman_image_t *pix, (base >= 0x1fb00 && base <= 0x1fb3b) || /* Unicode 13 partial blocks */ - /* TODO: there's more here! */ + /* TODO: there's more here! */ (base >= 0x1fb70 && base <= 0x1fb8b)) && likely(!term->conf->box_drawings_uses_font_glyphs)) @@ -542,7 +538,7 @@ render_cell(struct terminal *term, pixman_image_t *pix, xassert(idx < ALEN(term->box_drawing)); if (likely(term->box_drawing[idx] != NULL)) - glyph = term->box_drawing[idx]; + single = term->box_drawing[idx]; else { mtx_lock(&term->render.workers.lock); @@ -551,15 +547,45 @@ render_cell(struct terminal *term, pixman_image_t *pix, term->box_drawing[idx] = box_drawing(term, base); mtx_unlock(&term->render.workers.lock); - glyph = term->box_drawing[idx]; - xassert(glyph != NULL); + single = term->box_drawing[idx]; + xassert(single != NULL); } - } else - glyph = fcft_glyph_rasterize(font, base, term->font_subpixel); + + glyph_count = 1; + glyphs = &single; + } + + else if (base >= CELL_COMB_CHARS_LO && + base < (CELL_COMB_CHARS_LO + term->composed_count)) + { + composed = &term->composed[base - CELL_COMB_CHARS_LO]; + base = composed->chars[0]; + + if (term->conf->can_shape_grapheme && term->conf->tweak.grapheme_shaping) { + grapheme = fcft_grapheme_rasterize( + font, composed->count, composed->chars, + 0, NULL, term->font_subpixel); + } + + if (grapheme != NULL) { + composed = NULL; + glyphs = grapheme->glyphs; + glyph_count = grapheme->count; + } + } + + + if (single == NULL && grapheme == NULL) { + xassert(base != 0); + single = fcft_glyph_rasterize(font, base, term->font_subpixel); + glyph_count = 1; + glyphs = &single; + } } + assert(glyph_count == 0 || glyphs != NULL); const int cols_left = term->cols - col; - int cell_cols = glyph != NULL ? max(1, min(glyph->cols, cols_left)) : 1; + int cell_cols = glyph_count > 0 ? max(1, min(glyphs[0]->cols, cols_left)) : 1; /* * Hack! @@ -580,15 +606,15 @@ render_cell(struct terminal *term, pixman_image_t *pix, * - *this* cells is followed by an empty cell, or a space */ if (term->conf->tweak.allow_overflowing_double_width_glyphs && - ((glyph != NULL && - glyph->cols == 1 && - glyph->width >= term->cell_width * 15 / 10 && - glyph->width < 3 * term->cell_width && - col < term->cols - 1) || - (term->conf->tweak.pua_double_width && - ((base >= 0x00e000 && base <= 0x00f8ff) || - (base >= 0x0f0000 && base <= 0x0ffffd) || - (base >= 0x100000 && base <= 0x10fffd)))) && + ((glyph_count > 0 && + glyphs[0]->cols == 1 && + glyphs[0]->width >= term->cell_width * 15 / 10 && + glyphs[0]->width < 3 * term->cell_width && + col < term->cols - 1 || + (term->conf->tweak.pua_double_width && + ((base >= 0x00e000 && base <= 0x00f8ff) || + (base >= 0x0f0000 && base <= 0x0ffffd) || + (base >= 0x100000 && base <= 0x10fffd))))) && (row->cells[col + 1].wc == 0 || row->cells[col + 1].wc == L' ')) { cell_cols = 2; @@ -632,33 +658,43 @@ render_cell(struct terminal *term, pixman_image_t *pix, pixman_image_t *clr_pix = pixman_image_create_solid_fill(&fg); - if (glyph != NULL) { - const int letter_x_ofs = term->font_x_ofs; + for (unsigned i = 0; i < glyph_count; i++) { + const int letter_x_ofs = i == 0 ? term->font_x_ofs : 0; + + const struct fcft_glyph *glyph = glyphs[i]; + if (glyph == NULL) + continue; + + int g_x = glyph->x; + int g_y = glyph->y; + + if (i > 0 && glyph->x >= 0) + g_x -= term->cell_width; if (unlikely(pixman_image_get_format(glyph->pix) == PIXMAN_a8r8g8b8)) { /* Glyph surface is a pre-rendered image (typically a color emoji...) */ if (!(cell->attrs.blink && term->blink.state == BLINK_OFF)) { pixman_image_composite32( PIXMAN_OP_OVER, glyph->pix, NULL, pix, 0, 0, 0, 0, - x + letter_x_ofs + glyph->x, y + font_baseline(term) - glyph->y, + x + letter_x_ofs + g_x, y + font_baseline(term) - g_y, glyph->width, glyph->height); } } else { pixman_image_composite32( PIXMAN_OP_OVER, clr_pix, glyph->pix, pix, 0, 0, 0, 0, - x + letter_x_ofs + glyph->x, y + font_baseline(term) - glyph->y, + x + letter_x_ofs + g_x, y + font_baseline(term) - g_y, glyph->width, glyph->height); - } + /* Combining characters */ + if (composed != NULL) { + assert(glyph_count == 1); - /* Combining characters */ - if (composed != NULL) { - for (size_t i = 0; i < composed->count; i++) { - const struct fcft_glyph *g = fcft_glyph_rasterize( - font, composed->combining[i], term->font_subpixel); + for (size_t i = 1; i < composed->count; i++) { + const struct fcft_glyph *g = fcft_glyph_rasterize( + font, composed->chars[i], term->font_subpixel); - if (g == NULL) - continue; + if (g == NULL) + continue; /* * Fonts _should_ assume the pen position is now @@ -677,16 +713,22 @@ render_cell(struct terminal *term, pixman_image_t *pix, * somewhat deal with double-width glyphs we use * an offset of *one* cell. */ - int x_ofs = g->x < 0 - ? cell_cols * term->cell_width - : (cell_cols - 1) * term->cell_width; + int x_ofs = g->x < 0 + ? cell_cols * term->cell_width + : (cell_cols - 1) * term->cell_width; - pixman_image_composite32( - PIXMAN_OP_OVER, clr_pix, g->pix, pix, 0, 0, 0, 0, - x + letter_x_ofs + x_ofs + g->x, y + font_baseline(term) - g->y, - g->width, g->height); + pixman_image_composite32( + PIXMAN_OP_OVER, clr_pix, g->pix, pix, 0, 0, 0, 0, + /* Some fonts use a negative offset, while others use a + * "normal" offset */ + x + x_ofs + g->x, + y + font_baseline(term) - g->y, + g->width, g->height); + } } } + + x += glyph->advance.x; } pixman_image_unref(clr_pix); diff --git a/search.c b/search.c index 9fc1ab2a..859608f5 100644 --- a/search.c +++ b/search.c @@ -249,7 +249,7 @@ matches_cell(const struct terminal *term, const struct cell *cell, size_t search base < (CELL_COMB_CHARS_LO + term->composed_count)) { composed = &term->composed[base - CELL_COMB_CHARS_LO]; - base = composed->base; + base = composed->chars[0]; } if (composed == NULL && base == 0 && term->search.buf[search_ofs] == L' ') @@ -262,8 +262,8 @@ matches_cell(const struct terminal *term, const struct cell *cell, size_t search if (search_ofs + 1 + composed->count > term->search.len) return -1; - for (size_t j = 0; j < composed->count; j++) { - if (composed->combining[j] != term->search.buf[search_ofs + 1 + j]) + for (size_t j = 1; j < composed->count; j++) { + if (composed->chars[j] != term->search.buf[search_ofs + 1 + j]) return -1; } } diff --git a/selection.c b/selection.c index 555b32f2..cafb161e 100644 --- a/selection.c +++ b/selection.c @@ -249,7 +249,7 @@ selection_find_word_boundary_left(struct terminal *term, struct coord *pos, if (c >= CELL_COMB_CHARS_LO && c < (CELL_COMB_CHARS_LO + term->composed_count)) { - c = term->composed[c - CELL_COMB_CHARS_LO].base; + c = term->composed[c - CELL_COMB_CHARS_LO].chars[0]; } bool initial_is_space = c == 0 || iswspace(c); @@ -289,7 +289,7 @@ selection_find_word_boundary_left(struct terminal *term, struct coord *pos, if (c >= CELL_COMB_CHARS_LO && c < (CELL_COMB_CHARS_LO + term->composed_count)) { - c = term->composed[c - CELL_COMB_CHARS_LO].base; + c = term->composed[c - CELL_COMB_CHARS_LO].chars[0]; } bool is_space = c == 0 || iswspace(c); @@ -328,7 +328,7 @@ selection_find_word_boundary_right(struct terminal *term, struct coord *pos, if (c >= CELL_COMB_CHARS_LO && c < (CELL_COMB_CHARS_LO + term->composed_count)) { - c = term->composed[c - CELL_COMB_CHARS_LO].base; + c = term->composed[c - CELL_COMB_CHARS_LO].chars[0]; } bool initial_is_space = c == 0 || iswspace(c); @@ -370,7 +370,7 @@ selection_find_word_boundary_right(struct terminal *term, struct coord *pos, if (c >= CELL_COMB_CHARS_LO && c < (CELL_COMB_CHARS_LO + term->composed_count)) { - c = term->composed[c - CELL_COMB_CHARS_LO].base; + c = term->composed[c - CELL_COMB_CHARS_LO].chars[0]; } bool is_space = c == 0 || iswspace(c); diff --git a/terminal.h b/terminal.h index e6bcfa40..45c6c98b 100644 --- a/terminal.h +++ b/terminal.h @@ -8,6 +8,10 @@ #include #include +#if defined(FOOT_GRAPHEME_CLUSTERING) + #include +#endif + #include #include @@ -81,8 +85,7 @@ struct damage { }; struct composed { - wchar_t base; - wchar_t combining[5]; + wchar_t chars[20]; uint8_t count; }; @@ -152,6 +155,9 @@ struct vt_param { struct vt { int state; /* enum state */ wchar_t last_printed; +#if defined(FOOT_GRAPHEME_CLUSTERING) + utf8proc_int32_t grapheme_state; +#endif wchar_t utf8; struct { struct vt_param v[16]; @@ -720,3 +726,10 @@ void term_collect_urls(struct terminal *term); void term_osc8_open(struct terminal *term, uint64_t id, const char *uri); void term_osc8_close(struct terminal *term); + +static inline void term_reset_grapheme_state(struct terminal *term) +{ +#if defined(FOOT_GRAPHEME_CLUSTERING) + term->vt.grapheme_state = 0; +#endif +} diff --git a/util.h b/util.h index aa9fc8ba..af215111 100644 --- a/util.h +++ b/util.h @@ -35,3 +35,11 @@ sdbm_hash(const char *s) return hash; } + +#include +static inline int +my_wcswidth(const wchar_t *s, size_t n) +{ + int ret = wcswidth(s, n); + return max(0, ret); +} diff --git a/vt.c b/vt.c index 723f0450..496333d7 100644 --- a/vt.c +++ b/vt.c @@ -4,9 +4,14 @@ #include #include +#if defined(FOOT_GRAPHEME_CLUSTERING) + #include +#endif + #define LOG_MODULE "vt" #define LOG_ENABLE_DBG 0 #include "log.h" +#include "config.h" #include "csi.h" #include "dcs.h" #include "debug.h" @@ -283,6 +288,7 @@ action_execute(struct terminal *term, uint8_t c) static void action_print(struct terminal *term, uint8_t c) { + term_reset_grapheme_state(term); term->ascii_printer(term, c); } @@ -583,152 +589,166 @@ static void action_utf8_print(struct terminal *term, wchar_t wc) { int width = wcwidth(wc); + const bool grapheme_clustering = term->conf->tweak.grapheme_shaping; - /* - * Is this is combining character? The basic assumption is that if - * wcwdith() returns 0, then it *is* a combining character. - * - * We hen optimize this by ignoring all characters before 0x0300, - * since there aren't any zero-width characters there. This means - * all "normal" western characters will quickly be categorized as - * *not* being combining characters. - * - * TODO: xterm does more or less the same, but also filters a - * small subset of BIDI control characters. Should we too? I think - * what we have here is good enough - a control character - * shouldn't have a glyph associated with it, so rendering - * shouldn't be affected. - * - * TODO: handle line-wrap when locating the base character. - */ - if (width == 0 && wc >= 0x0300 && term->grid->cursor.point.col > 0) { - const struct row *row = term->grid->cur_row; +#if !defined(FOOT_GRAPHEME_CLUSTERING) + xassert(!grapheme_clustering); +#endif - int base_col = term->grid->cursor.point.col; + if (term->grid->cursor.point.col > 0 && + (grapheme_clustering || + (!grapheme_clustering && width == 0 && wc >= 0x300))) + { + int col = term->grid->cursor.point.col; if (!term->grid->cursor.lcf) - base_col--; + col--; - while (row->cells[base_col].wc >= CELL_SPACER && base_col > 0) - base_col--; + /* Skip past spacers */ + struct row *row = term->grid->cur_row; + while (row->cells[col].wc >= CELL_SPACER && col > 0) + col--; - xassert(base_col >= 0 && base_col < term->cols); - wchar_t base = row->cells[base_col].wc; + xassert(col >= 0 && col < term->cols); + wchar_t base = row->cells[col].wc; + wchar_t UNUSED last = base; + /* Is base cell already a cluster? */ const struct composed *composed = (base >= CELL_COMB_CHARS_LO && base < (CELL_COMB_CHARS_LO + term->composed_count)) ? &term->composed[base - CELL_COMB_CHARS_LO] : NULL; - if (composed != NULL) - base = composed->base; + if (composed != NULL) { + base = composed->chars[0]; + last = composed->chars[composed->count - 1]; + } + +#if defined(FOOT_GRAPHEME_CLUSTERING) + if (grapheme_clustering) { + /* Check if we're on a grapheme cluster break */ + /* Note: utf8proc fails to ZWJ */ + if (utf8proc_grapheme_break_stateful(last, wc, &term->vt.grapheme_state) && + last != 0x200d /* ZWJ */) + { + term_reset_grapheme_state(term); + if (width > 0) + term_print(term, wc, width); + return; + } + } +#endif int base_width = wcwidth(base); + term->grid->cursor.point.col = col; + term->grid->cursor.lcf = false; - if (base != 0 && base_width > 0) { + if (composed == NULL) { + bool base_from_primary; + bool comb_from_primary; + bool pre_from_primary; + + wchar_t precomposed = fcft_precompose( + term->fonts[0], base, wc, &base_from_primary, + &comb_from_primary, &pre_from_primary); + + int precomposed_width = wcwidth(precomposed); /* - * If this is the *first* combining characger, see if - * there's a pre-composed character of this combo, with - * the same column width as the base character. + * Only use the pre-composed character if: * - * If there is, replace the base character with the - * pre-composed character, as that is likely to produce a - * better looking result. + * 1. we *have* a pre-composed character + * 2. the width matches the base characters width + * 3. it's in the primary font, OR one of the base or + * combining characters are *not* from the primary + * font */ - term->grid->cursor.point.col = base_col; - term->grid->cursor.lcf = false; - if (composed == NULL) { - bool base_from_primary; - bool comb_from_primary; - bool pre_from_primary; - - wchar_t precomposed = fcft_precompose( - term->fonts[0], base, wc, &base_from_primary, - &comb_from_primary, &pre_from_primary); - - int precomposed_width = wcwidth(precomposed); - - /* - * Only use the pre-composed character if: - * - * 1. we *have* a pre-composed character - * 2. the width matches the base characters width - * 3. it's in the primary font, OR one of the base or - * combining characters are *not* from the primary - * font - */ - - if (precomposed != (wchar_t)-1 && - precomposed_width == base_width && - (pre_from_primary || - !base_from_primary || - !comb_from_primary)) - { - term_print(term, precomposed, precomposed_width); - return; - } + if (precomposed != (wchar_t)-1 && + precomposed_width == base_width && + (pre_from_primary || + !base_from_primary || + !comb_from_primary)) + { + term_reset_grapheme_state(term); + term_print(term, precomposed, precomposed_width); + return; } + } - size_t wanted_count = composed != NULL ? composed->count + 1 : 1; - if (wanted_count > ALEN(composed->combining)) { - xassert(composed != NULL); + size_t wanted_count = composed != NULL ? composed->count + 1 : 2; + if (wanted_count > ALEN(composed->chars)) { + xassert(composed != NULL); #if defined(LOG_ENABLE_DBG) && LOG_ENABLE_DBG - LOG_WARN("combining character overflow:"); - LOG_WARN(" base: 0x%04x", composed->base); - for (size_t i = 0; i < composed->count; i++) - LOG_WARN(" cc: 0x%04x", composed->combining[i]); - LOG_ERR(" new: 0x%04x", wc); + LOG_WARN("combining character overflow:"); + LOG_WARN(" base: 0x%04x", composed->chars[0]); + for (size_t i = 1; i < composed->count; i++) + LOG_WARN(" cc: 0x%04x", composed->chars[i]); + LOG_ERR(" new: 0x%04x", wc); #endif - /* This are going to break anyway... */ - wanted_count--; + /* This is going to break anyway... */ + wanted_count--; + } + + xassert(wanted_count <= ALEN(composed->chars)); + + /* Look for existing combining chain */ + for (size_t i = 0; i < term->composed_count; i++) { + const struct composed *cc = &term->composed[i]; + + if (cc->count != wanted_count) + continue; + + if (cc->chars[0] != base) + continue; + + bool match = true; + for (size_t j = 1; j < wanted_count - 1; j++) { + if (cc->chars[j] != composed->chars[j]) { + match = false; + break; + } } + if (!match) + continue; - xassert(wanted_count <= ALEN(composed->combining)); + if (cc->chars[wanted_count - 1] != wc) + continue; - /* Look for existing combining chain */ - for (size_t i = 0; i < term->composed_count; i++) { - const struct composed *cc = &term->composed[i]; - if (cc->base != base) - continue; + int grapheme_width = my_wcswidth(cc->chars, cc->count); + if (grapheme_width > 0) + term_print(term, CELL_COMB_CHARS_LO + i, grapheme_width); + return; + } - if (cc->count != wanted_count) - continue; + /* Allocate new chain */ - if (cc->combining[wanted_count - 1] != wc) - continue; + struct composed new_cc; + new_cc.count = wanted_count; + new_cc.chars[0] = base; + for (size_t i = 1; i < wanted_count - 1; i++) + new_cc.chars[i] = composed->chars[i]; + new_cc.chars[wanted_count - 1] = wc; - term_print(term, CELL_COMB_CHARS_LO + i, base_width); - return; - } + if (term->composed_count < CELL_COMB_CHARS_HI) { + term->composed_count++; + term->composed = xrealloc(term->composed, term->composed_count * sizeof(term->composed[0])); + term->composed[term->composed_count - 1] = new_cc; - /* Allocate new chain */ - - struct composed new_cc; - new_cc.base = base; - new_cc.count = wanted_count; - for (size_t i = 0; i < wanted_count - 1; i++) - new_cc.combining[i] = composed->combining[i]; - new_cc.combining[wanted_count - 1] = wc; - - if (term->composed_count < CELL_COMB_CHARS_HI) { - term->composed_count++; - term->composed = xrealloc(term->composed, term->composed_count * sizeof(term->composed[0])); - term->composed[term->composed_count - 1] = new_cc; - - term_print(term, CELL_COMB_CHARS_LO + term->composed_count - 1, base_width); - return; - } else { - /* We reached our maximum number of allowed composed - * character chains. Fall through here and print the - * current zero-width character to the current cell */ - LOG_WARN("maximum number of composed characters reached"); - } + int grapheme_width = my_wcswidth(new_cc.chars, new_cc.count); + if (grapheme_width > 0) + term_print(term, CELL_COMB_CHARS_LO + term->composed_count - 1, grapheme_width); + return; + } else { + /* We reached our maximum number of allowed composed + * character chains. Fall through here and print the + * current zero-width character to the current cell */ + LOG_WARN("maximum number of composed characters reached"); } } + term_reset_grapheme_state(term); if (width > 0) term_print(term, wc, width); } From 0a9531ac6caff2ad6abfbb804ec27cade4a92e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 27 May 2021 20:07:28 +0200 Subject: [PATCH 02/30] vt: cache grapheme cluster width in composed struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use regular wcswidth() to calculate the width * Explicitly set to ‘2’ if we see a emoji variant selector * Cache the result in the composed struct --- terminal.h | 1 + util.h | 8 -------- vt.c | 14 ++++++++++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/terminal.h b/terminal.h index 45c6c98b..83406f41 100644 --- a/terminal.h +++ b/terminal.h @@ -87,6 +87,7 @@ struct damage { struct composed { wchar_t chars[20]; uint8_t count; + int width; }; struct row_uri_range { diff --git a/util.h b/util.h index af215111..aa9fc8ba 100644 --- a/util.h +++ b/util.h @@ -35,11 +35,3 @@ sdbm_hash(const char *s) return hash; } - -#include -static inline int -my_wcswidth(const wchar_t *s, size_t n) -{ - int ret = wcswidth(s, n); - return max(0, ret); -} diff --git a/vt.c b/vt.c index 496333d7..6ad539d7 100644 --- a/vt.c +++ b/vt.c @@ -716,9 +716,8 @@ action_utf8_print(struct terminal *term, wchar_t wc) if (cc->chars[wanted_count - 1] != wc) continue; - int grapheme_width = my_wcswidth(cc->chars, cc->count); - if (grapheme_width > 0) - term_print(term, CELL_COMB_CHARS_LO + i, grapheme_width); + if (cc->width > 0) + term_print(term, CELL_COMB_CHARS_LO + i, cc->width); return; } @@ -732,11 +731,18 @@ action_utf8_print(struct terminal *term, wchar_t wc) new_cc.chars[wanted_count - 1] = wc; if (term->composed_count < CELL_COMB_CHARS_HI) { + /* TODO: grapheme cluster width */ + int grapheme_width = wcswidth(new_cc.chars, new_cc.count); + if (new_cc.chars[new_cc.count - 1] == 0xfe0f) { + /* Emoji selector */ + grapheme_width = 2; + } + new_cc.width = grapheme_width; + term->composed_count++; term->composed = xrealloc(term->composed, term->composed_count * sizeof(term->composed[0])); term->composed[term->composed_count - 1] = new_cc; - int grapheme_width = my_wcswidth(new_cc.chars, new_cc.count); if (grapheme_width > 0) term_print(term, CELL_COMB_CHARS_LO + term->composed_count - 1, grapheme_width); return; From 6c70cd9366db76f10d822252151a782f638d1f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Sun, 30 May 2021 19:36:59 +0200 Subject: [PATCH 03/30] =?UTF-8?q?vt:=20don=E2=80=99t=20force=20cols=3D2=20?= =?UTF-8?q?when=20we=20see=20an=20emoji=20variant=20selector?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fish appears to be the only shell expecting this. The rest probably just does wcswidth(), like usual. --- vt.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vt.c b/vt.c index 6ad539d7..aff3131b 100644 --- a/vt.c +++ b/vt.c @@ -733,10 +733,12 @@ action_utf8_print(struct terminal *term, wchar_t wc) if (term->composed_count < CELL_COMB_CHARS_HI) { /* TODO: grapheme cluster width */ int grapheme_width = wcswidth(new_cc.chars, new_cc.count); +#if 0 /* Fish expects this, but nothing else */ if (new_cc.chars[new_cc.count - 1] == 0xfe0f) { /* Emoji selector */ - grapheme_width = 2; + grapheme_width = max(2, grapheme_width); } +#endif new_cc.width = grapheme_width; term->composed_count++; From bd98cb6a73d225e2dc15678d8fe06d54cc802c7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Sun, 30 May 2021 19:37:53 +0200 Subject: [PATCH 04/30] render: use column count from grapheme instead of first glyph, when we have one --- render.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/render.c b/render.c index 54269bf3..894c74db 100644 --- a/render.c +++ b/render.c @@ -507,6 +507,7 @@ render_cell(struct terminal *term, pixman_image_t *pix, unsigned glyph_count = 0; wchar_t base = cell->wc; + int cell_cols; if (base != 0) { if (unlikely( @@ -553,6 +554,7 @@ render_cell(struct terminal *term, pixman_image_t *pix, glyph_count = 1; glyphs = &single; + cell_cols = single->cols; } else if (base >= CELL_COMB_CHARS_LO && @@ -571,7 +573,9 @@ render_cell(struct terminal *term, pixman_image_t *pix, composed = NULL; glyphs = grapheme->glyphs; glyph_count = grapheme->count; - } + cell_cols = grapheme->cols; + } else + base = composed->chars[0]; } @@ -580,12 +584,14 @@ render_cell(struct terminal *term, pixman_image_t *pix, single = fcft_glyph_rasterize(font, base, term->font_subpixel); glyph_count = 1; glyphs = &single; + cell_cols = single->cols; } } assert(glyph_count == 0 || glyphs != NULL); + const int cols_left = term->cols - col; - int cell_cols = glyph_count > 0 ? max(1, min(glyphs[0]->cols, cols_left)) : 1; + cell_cols = max(1, min(cell_cols, cols_left)); /* * Hack! From 50be9242852756aac6fd82b0531ed92eeb44c754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Mon, 31 May 2021 17:11:58 +0200 Subject: [PATCH 05/30] render: handle fcft_glyph_rasterize() failure correctly --- render.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/render.c b/render.c index 894c74db..259d4499 100644 --- a/render.c +++ b/render.c @@ -582,9 +582,14 @@ render_cell(struct terminal *term, pixman_image_t *pix, if (single == NULL && grapheme == NULL) { xassert(base != 0); single = fcft_glyph_rasterize(font, base, term->font_subpixel); - glyph_count = 1; - glyphs = &single; - cell_cols = single->cols; + if (single == NULL) { + glyph_count = 0; + cell_cols = 1; + } else { + glyph_count = 1; + glyphs = &single; + cell_cols = single->cols; + } } } From b471fe31b1a65c0eb11422beb8b9e119f771925d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Mon, 31 May 2021 17:50:49 +0200 Subject: [PATCH 06/30] =?UTF-8?q?render:=20ensure=20=E2=80=98cell=5Fcols?= =?UTF-8?q?=E2=80=99=20have=20been=20initialized?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- render.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/render.c b/render.c index 259d4499..b0ce83d0 100644 --- a/render.c +++ b/render.c @@ -507,7 +507,7 @@ render_cell(struct terminal *term, pixman_image_t *pix, unsigned glyph_count = 0; wchar_t base = cell->wc; - int cell_cols; + int cell_cols = 1; if (base != 0) { if (unlikely( From 96ff29bbd3968759f3ebb3c83a9c2c7a8ea96134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Tue, 15 Jun 2021 06:00:00 +0200 Subject: [PATCH 07/30] render: repair parenthesis after rebase --- render.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/render.c b/render.c index b0ce83d0..c764975c 100644 --- a/render.c +++ b/render.c @@ -621,11 +621,11 @@ render_cell(struct terminal *term, pixman_image_t *pix, glyphs[0]->cols == 1 && glyphs[0]->width >= term->cell_width * 15 / 10 && glyphs[0]->width < 3 * term->cell_width && - col < term->cols - 1 || - (term->conf->tweak.pua_double_width && - ((base >= 0x00e000 && base <= 0x00f8ff) || - (base >= 0x0f0000 && base <= 0x0ffffd) || - (base >= 0x100000 && base <= 0x10fffd))))) && + col < term->cols - 1) || + (term->conf->tweak.pua_double_width && + ((base >= 0x00e000 && base <= 0x00f8ff) || + (base >= 0x0f0000 && base <= 0x0ffffd) || + (base >= 0x100000 && base <= 0x10fffd)))) && (row->cells[col + 1].wc == 0 || row->cells[col + 1].wc == L' ')) { cell_cols = 2; From 09431dd15c0e5d67adde766deb5f685a2918bb18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Tue, 15 Jun 2021 07:25:38 +0200 Subject: [PATCH 08/30] vt: presentation selectors may be anywhere in the cluster --- vt.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/vt.c b/vt.c index aff3131b..c343ba14 100644 --- a/vt.c +++ b/vt.c @@ -726,20 +726,22 @@ action_utf8_print(struct terminal *term, wchar_t wc) struct composed new_cc; new_cc.count = wanted_count; new_cc.chars[0] = base; + for (size_t i = 1; i < wanted_count - 1; i++) new_cc.chars[i] = composed->chars[i]; new_cc.chars[wanted_count - 1] = wc; if (term->composed_count < CELL_COMB_CHARS_HI) { - /* TODO: grapheme cluster width */ - int grapheme_width = wcswidth(new_cc.chars, new_cc.count); -#if 0 /* Fish expects this, but nothing else */ - if (new_cc.chars[new_cc.count - 1] == 0xfe0f) { - /* Emoji selector */ - grapheme_width = max(2, grapheme_width); + int grapheme_width = wcwidth(base); + int min_grapheme_width = 0; + for (size_t i = 0; i < wanted_count; i++) { + wchar_t c = new_cc.chars[i]; + if (c == 0xfe0f) + min_grapheme_width = 2; + grapheme_width += wcwidth(c); } -#endif - new_cc.width = grapheme_width; + + new_cc.width = max(grapheme_width, min_grapheme_width); term->composed_count++; term->composed = xrealloc(term->composed, term->composed_count * sizeof(term->composed[0])); From 57e636dd8e3be0e75492f226e036943989422b32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Tue, 15 Jun 2021 07:38:27 +0200 Subject: [PATCH 09/30] =?UTF-8?q?vt:=20don=E2=80=99t=20call=20wcwidth()=20?= =?UTF-8?q?on=20all=20combining=20characters=20every=20time=20we=20add?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We already have all the widths needed to calculate the new one; it’s the base characters width (base_width), or the previous combining chain’s width (composed->width) plus the new characters’s width (width). --- vt.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/vt.c b/vt.c index c343ba14..b99b4fef 100644 --- a/vt.c +++ b/vt.c @@ -700,7 +700,7 @@ action_utf8_print(struct terminal *term, wchar_t wc) if (cc->count != wanted_count) continue; - if (cc->chars[0] != base) + if (cc->chars[0] != base) continue; bool match = true; @@ -732,16 +732,12 @@ action_utf8_print(struct terminal *term, wchar_t wc) new_cc.chars[wanted_count - 1] = wc; if (term->composed_count < CELL_COMB_CHARS_HI) { - int grapheme_width = wcwidth(base); - int min_grapheme_width = 0; - for (size_t i = 0; i < wanted_count; i++) { - wchar_t c = new_cc.chars[i]; - if (c == 0xfe0f) - min_grapheme_width = 2; - grapheme_width += wcwidth(c); - } - - new_cc.width = max(grapheme_width, min_grapheme_width); + int grapheme_width = composed != NULL ? composed->width : base_width; + if (wc == 0xfe0f && grapheme_width < 2) + grapheme_width = 2; + else + grapheme_width += width; + new_cc.width = grapheme_width; term->composed_count++; term->composed = xrealloc(term->composed, term->composed_count * sizeof(term->composed[0])); From 51295cd7a27641d0452db6465e852f66bfd67118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Tue, 15 Jun 2021 07:58:41 +0200 Subject: [PATCH 10/30] =?UTF-8?q?render:=20we=E2=80=99ve=20already=20assig?= =?UTF-8?q?ned=20=E2=80=98base=E2=80=99=20a=20couple=20of=20lines=20higher?= =?UTF-8?q?=20up?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- render.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/render.c b/render.c index c764975c..6799f23a 100644 --- a/render.c +++ b/render.c @@ -574,8 +574,7 @@ render_cell(struct terminal *term, pixman_image_t *pix, glyphs = grapheme->glyphs; glyph_count = grapheme->count; cell_cols = grapheme->cols; - } else - base = composed->chars[0]; + } } From f865612667d090b584c1ea6cc20f32fcffd828d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Tue, 15 Jun 2021 08:40:23 +0200 Subject: [PATCH 11/30] vt: utf8-print: check base character before count when looking for existing compose chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Count is more likely to be the same for many chains. Thus we’re likely to fail sooner by checking the base character first. --- vt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vt.c b/vt.c index b99b4fef..c2e72e71 100644 --- a/vt.c +++ b/vt.c @@ -697,10 +697,10 @@ action_utf8_print(struct terminal *term, wchar_t wc) for (size_t i = 0; i < term->composed_count; i++) { const struct composed *cc = &term->composed[i]; - if (cc->count != wanted_count) + if (cc->chars[0] != base) continue; - if (cc->chars[0] != base) + if (cc->count != wanted_count) continue; bool match = true; From 6187aa0b1bfcba7e5d956e05685ad1de42666221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Tue, 15 Jun 2021 08:44:52 +0200 Subject: [PATCH 12/30] term: lower maximum number of characters in a compose chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before the grapheme cluster segmentation work, we limited the number of combining characters to base+5. I.e. 6 in total. For a while now, we’ve had it bumped all the way up to 20. This was the reason the unicode-random benchmark ran so much slower (i.e. cache contention). Looking at emoji’s, there are a couple that need 6 code points, and *three* that needs 7. Now, with the limit at 7 chars, and the new ‘width’ member, the composed struct is 8 bytes larger than before. --- terminal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terminal.h b/terminal.h index 83406f41..307e2b0b 100644 --- a/terminal.h +++ b/terminal.h @@ -85,7 +85,7 @@ struct damage { }; struct composed { - wchar_t chars[20]; + wchar_t chars[7]; uint8_t count; int width; }; From dc5019a535f3d678332ca1d68865356c970a200c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Tue, 15 Jun 2021 09:00:18 +0200 Subject: [PATCH 13/30] =?UTF-8?q?vt:=20utf8-print:=20don=E2=80=99t=20build?= =?UTF-8?q?=20a=20compose=20chain=20on=20a=20zero-width=20base=20character?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vt.c | 194 ++++++++++++++++++++++++++++++----------------------------- 1 file changed, 98 insertions(+), 96 deletions(-) diff --git a/vt.c b/vt.c index c2e72e71..09a2188a 100644 --- a/vt.c +++ b/vt.c @@ -640,117 +640,119 @@ action_utf8_print(struct terminal *term, wchar_t wc) #endif int base_width = wcwidth(base); - term->grid->cursor.point.col = col; - term->grid->cursor.lcf = false; + if (base_width > 0) { + term->grid->cursor.point.col = col; + term->grid->cursor.lcf = false; - if (composed == NULL) { - bool base_from_primary; - bool comb_from_primary; - bool pre_from_primary; + if (composed == NULL) { + bool base_from_primary; + bool comb_from_primary; + bool pre_from_primary; - wchar_t precomposed = fcft_precompose( - term->fonts[0], base, wc, &base_from_primary, - &comb_from_primary, &pre_from_primary); + wchar_t precomposed = fcft_precompose( + term->fonts[0], base, wc, &base_from_primary, + &comb_from_primary, &pre_from_primary); - int precomposed_width = wcwidth(precomposed); + int precomposed_width = wcwidth(precomposed); - /* - * Only use the pre-composed character if: - * - * 1. we *have* a pre-composed character - * 2. the width matches the base characters width - * 3. it's in the primary font, OR one of the base or - * combining characters are *not* from the primary - * font - */ + /* + * Only use the pre-composed character if: + * + * 1. we *have* a pre-composed character + * 2. the width matches the base characters width + * 3. it's in the primary font, OR one of the base or + * combining characters are *not* from the primary + * font + */ - if (precomposed != (wchar_t)-1 && - precomposed_width == base_width && - (pre_from_primary || - !base_from_primary || - !comb_from_primary)) - { - term_reset_grapheme_state(term); - term_print(term, precomposed, precomposed_width); - return; - } - } - - size_t wanted_count = composed != NULL ? composed->count + 1 : 2; - if (wanted_count > ALEN(composed->chars)) { - xassert(composed != NULL); - -#if defined(LOG_ENABLE_DBG) && LOG_ENABLE_DBG - LOG_WARN("combining character overflow:"); - LOG_WARN(" base: 0x%04x", composed->chars[0]); - for (size_t i = 1; i < composed->count; i++) - LOG_WARN(" cc: 0x%04x", composed->chars[i]); - LOG_ERR(" new: 0x%04x", wc); -#endif - /* This is going to break anyway... */ - wanted_count--; - } - - xassert(wanted_count <= ALEN(composed->chars)); - - /* Look for existing combining chain */ - for (size_t i = 0; i < term->composed_count; i++) { - const struct composed *cc = &term->composed[i]; - - if (cc->chars[0] != base) - continue; - - if (cc->count != wanted_count) - continue; - - bool match = true; - for (size_t j = 1; j < wanted_count - 1; j++) { - if (cc->chars[j] != composed->chars[j]) { - match = false; - break; + if (precomposed != (wchar_t)-1 && + precomposed_width == base_width && + (pre_from_primary || + !base_from_primary || + !comb_from_primary)) + { + term_reset_grapheme_state(term); + term_print(term, precomposed, precomposed_width); + return; } } - if (!match) - continue; - if (cc->chars[wanted_count - 1] != wc) - continue; + size_t wanted_count = composed != NULL ? composed->count + 1 : 2; + if (wanted_count > ALEN(composed->chars)) { + xassert(composed != NULL); - if (cc->width > 0) - term_print(term, CELL_COMB_CHARS_LO + i, cc->width); - return; - } +#if defined(LOG_ENABLE_DBG) && LOG_ENABLE_DBG + LOG_WARN("combining character overflow:"); + LOG_WARN(" base: 0x%04x", composed->chars[0]); + for (size_t i = 1; i < composed->count; i++) + LOG_WARN(" cc: 0x%04x", composed->chars[i]); + LOG_ERR(" new: 0x%04x", wc); +#endif + /* This is going to break anyway... */ + wanted_count--; + } - /* Allocate new chain */ + xassert(wanted_count <= ALEN(composed->chars)); - struct composed new_cc; - new_cc.count = wanted_count; - new_cc.chars[0] = base; + /* Look for existing combining chain */ + for (size_t i = 0; i < term->composed_count; i++) { + const struct composed *cc = &term->composed[i]; - for (size_t i = 1; i < wanted_count - 1; i++) - new_cc.chars[i] = composed->chars[i]; - new_cc.chars[wanted_count - 1] = wc; + if (cc->chars[0] != base) + continue; - if (term->composed_count < CELL_COMB_CHARS_HI) { - int grapheme_width = composed != NULL ? composed->width : base_width; - if (wc == 0xfe0f && grapheme_width < 2) - grapheme_width = 2; - else - grapheme_width += width; - new_cc.width = grapheme_width; + if (cc->count != wanted_count) + continue; - term->composed_count++; - term->composed = xrealloc(term->composed, term->composed_count * sizeof(term->composed[0])); - term->composed[term->composed_count - 1] = new_cc; + bool match = true; + for (size_t j = 1; j < wanted_count - 1; j++) { + if (cc->chars[j] != composed->chars[j]) { + match = false; + break; + } + } + if (!match) + continue; - if (grapheme_width > 0) - term_print(term, CELL_COMB_CHARS_LO + term->composed_count - 1, grapheme_width); - return; - } else { - /* We reached our maximum number of allowed composed - * character chains. Fall through here and print the - * current zero-width character to the current cell */ - LOG_WARN("maximum number of composed characters reached"); + if (cc->chars[wanted_count - 1] != wc) + continue; + + if (cc->width > 0) + term_print(term, CELL_COMB_CHARS_LO + i, cc->width); + return; + } + + /* Allocate new chain */ + + struct composed new_cc; + new_cc.count = wanted_count; + new_cc.chars[0] = base; + + for (size_t i = 1; i < wanted_count - 1; i++) + new_cc.chars[i] = composed->chars[i]; + new_cc.chars[wanted_count - 1] = wc; + + if (term->composed_count < CELL_COMB_CHARS_HI) { + int grapheme_width = composed != NULL ? composed->width : base_width; + if (wc == 0xfe0f && grapheme_width < 2) + grapheme_width = 2; + else + grapheme_width += width; + new_cc.width = grapheme_width; + + term->composed_count++; + term->composed = xrealloc(term->composed, term->composed_count * sizeof(term->composed[0])); + term->composed[term->composed_count - 1] = new_cc; + + if (grapheme_width > 0) + term_print(term, CELL_COMB_CHARS_LO + term->composed_count - 1, grapheme_width); + return; + } else { + /* We reached our maximum number of allowed composed + * character chains. Fall through here and print the + * current zero-width character to the current cell */ + LOG_WARN("maximum number of composed characters reached"); + } } } From c0d9f92e1a5aae2f0963d62f9fd480a5926f2d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Tue, 15 Jun 2021 17:52:45 +0200 Subject: [PATCH 14/30] =?UTF-8?q?render:=20don=E2=80=99t=20modify=20the=20?= =?UTF-8?q?cell=E2=80=99s=20x=20position.=20Fixes=20broken=20underlines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- render.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/render.c b/render.c index 6799f23a..d46cb90b 100644 --- a/render.c +++ b/render.c @@ -449,8 +449,8 @@ render_cell(struct terminal *term, pixman_image_t *pix, int width = term->cell_width; int height = term->cell_height; - int x = term->margins.left + col * width; - int y = term->margins.top + row_no * height; + const int x = term->margins.left + col * width; + const int y = term->margins.top + row_no * height; xassert(cell->attrs.selected == 0 || cell->attrs.selected == 1); bool is_selected = cell->attrs.selected; @@ -668,6 +668,7 @@ render_cell(struct terminal *term, pixman_image_t *pix, pixman_image_t *clr_pix = pixman_image_create_solid_fill(&fg); + int pen_x = x; for (unsigned i = 0; i < glyph_count; i++) { const int letter_x_ofs = i == 0 ? term->font_x_ofs : 0; @@ -686,13 +687,13 @@ render_cell(struct terminal *term, pixman_image_t *pix, if (!(cell->attrs.blink && term->blink.state == BLINK_OFF)) { pixman_image_composite32( PIXMAN_OP_OVER, glyph->pix, NULL, pix, 0, 0, 0, 0, - x + letter_x_ofs + g_x, y + font_baseline(term) - g_y, + pen_x + letter_x_ofs + g_x, y + font_baseline(term) - g_y, glyph->width, glyph->height); } } else { pixman_image_composite32( PIXMAN_OP_OVER, clr_pix, glyph->pix, pix, 0, 0, 0, 0, - x + letter_x_ofs + g_x, y + font_baseline(term) - g_y, + pen_x + letter_x_ofs + g_x, y + font_baseline(term) - g_y, glyph->width, glyph->height); /* Combining characters */ @@ -731,14 +732,14 @@ render_cell(struct terminal *term, pixman_image_t *pix, PIXMAN_OP_OVER, clr_pix, g->pix, pix, 0, 0, 0, 0, /* Some fonts use a negative offset, while others use a * "normal" offset */ - x + x_ofs + g->x, + pen_x + x_ofs + g->x, y + font_baseline(term) - g->y, g->width, g->height); } } } - x += glyph->advance.x; + pen_x += glyph->advance.x; } pixman_image_unref(clr_pix); From e81d1845bf765055672b76028525221edefd4aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Fri, 18 Jun 2021 17:40:24 +0200 Subject: [PATCH 15/30] vt: utf8: de-duplicate; jump to end of function to print to grid --- vt.c | 51 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/vt.c b/vt.c index 09a2188a..e516735d 100644 --- a/vt.c +++ b/vt.c @@ -631,10 +631,7 @@ action_utf8_print(struct terminal *term, wchar_t wc) if (utf8proc_grapheme_break_stateful(last, wc, &term->vt.grapheme_state) && last != 0x200d /* ZWJ */) { - term_reset_grapheme_state(term); - if (width > 0) - term_print(term, wc, width); - return; + goto out; } } #endif @@ -671,9 +668,9 @@ action_utf8_print(struct terminal *term, wchar_t wc) !base_from_primary || !comb_from_primary)) { - term_reset_grapheme_state(term); - term_print(term, precomposed, precomposed_width); - return; + wc = precomposed; + width = precomposed_width; + goto out; } } @@ -717,9 +714,9 @@ action_utf8_print(struct terminal *term, wchar_t wc) if (cc->chars[wanted_count - 1] != wc) continue; - if (cc->width > 0) - term_print(term, CELL_COMB_CHARS_LO + i, cc->width); - return; + wc = CELL_COMB_CHARS_LO + i; + width = cc->width; + goto out; } /* Allocate new chain */ @@ -732,30 +729,32 @@ action_utf8_print(struct terminal *term, wchar_t wc) new_cc.chars[i] = composed->chars[i]; new_cc.chars[wanted_count - 1] = wc; - if (term->composed_count < CELL_COMB_CHARS_HI) { - int grapheme_width = composed != NULL ? composed->width : base_width; - if (wc == 0xfe0f && grapheme_width < 2) - grapheme_width = 2; - else - grapheme_width += width; - new_cc.width = grapheme_width; - - term->composed_count++; - term->composed = xrealloc(term->composed, term->composed_count * sizeof(term->composed[0])); - term->composed[term->composed_count - 1] = new_cc; - - if (grapheme_width > 0) - term_print(term, CELL_COMB_CHARS_LO + term->composed_count - 1, grapheme_width); - return; - } else { + if (unlikely(term->composed_count >= CELL_COMB_CHARS_HI)) { /* We reached our maximum number of allowed composed * character chains. Fall through here and print the * current zero-width character to the current cell */ LOG_WARN("maximum number of composed characters reached"); + goto out; } + + int grapheme_width = composed != NULL ? composed->width : base_width; + if (wc == 0xfe0f && grapheme_width < 2) + grapheme_width = 2; + else + grapheme_width += width; + new_cc.width = grapheme_width; + + term->composed_count++; + term->composed = xrealloc(term->composed, term->composed_count * sizeof(term->composed[0])); + term->composed[term->composed_count - 1] = new_cc; + + wc = CELL_COMB_CHARS_LO + term->composed_count - 1; + width = grapheme_width; + goto out; } } +out: term_reset_grapheme_state(term); if (width > 0) term_print(term, wc, width); From 81131e3a8757658be264e2069c75406993b8e876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Fri, 18 Jun 2021 17:53:15 +0200 Subject: [PATCH 16/30] =?UTF-8?q?vt:=20utf8:=20don=E2=80=99t=20scan=20*all?= =?UTF-8?q?*=20previous=20chains?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When checking if we already have a compose chain for the current sequence of characters, don’t search the list from the beginning, unless we have to. Taking the following things into consideration: * New compose chains are always appended at the end of the list * If the current sequence is 3 or more characters, it *must* consist of an existing compose chain, plus the new character. Thus, when searching, start at index 0 if we only have two characters, since then the base cell originally contained a regular base character, and not a compose chain. I.e. the new chain may be _anywhere_ in the chain list. If however we have a sequence of three or more characters, start at the index the *base* chain was at. If the chain we’re searching for exists, it *must* have been added *after* the base chain, and thus it *must* be located *after* the base chain in the chain list. --- vt.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vt.c b/vt.c index e516735d..bbe6fc9a 100644 --- a/vt.c +++ b/vt.c @@ -611,6 +611,7 @@ action_utf8_print(struct terminal *term, wchar_t wc) xassert(col >= 0 && col < term->cols); wchar_t base = row->cells[col].wc; wchar_t UNUSED last = base; + size_t search_start_index = 0; /* Is base cell already a cluster? */ const struct composed *composed = @@ -620,6 +621,7 @@ action_utf8_print(struct terminal *term, wchar_t wc) : NULL; if (composed != NULL) { + search_start_index = base - CELL_COMB_CHARS_LO; base = composed->chars[0]; last = composed->chars[composed->count - 1]; } @@ -692,7 +694,7 @@ action_utf8_print(struct terminal *term, wchar_t wc) xassert(wanted_count <= ALEN(composed->chars)); /* Look for existing combining chain */ - for (size_t i = 0; i < term->composed_count; i++) { + for (size_t i = search_start_index; i < term->composed_count; i++) { const struct composed *cc = &term->composed[i]; if (cc->chars[0] != base) From 34e85e7726fa73c0847058585f94f8d886b14e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Wed, 23 Jun 2021 18:55:30 +0200 Subject: [PATCH 17/30] scripts: generate-alt-random: add emoji sequences --- scripts/generate-alt-random-writes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate-alt-random-writes.py b/scripts/generate-alt-random-writes.py index 4e50bd8c..812b0213 100755 --- a/scripts/generate-alt-random-writes.py +++ b/scripts/generate-alt-random-writes.py @@ -88,7 +88,7 @@ def main(): count = 256 * 1024**1 # Characters to choose from - alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRTSTUVWXYZ0123456789 öäå 👨👩🧒' + alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRTSTUVWXYZ0123456789 öäå 👨👩🧒👩🏽‍🔬🇸🇪' color_variants = ([ColorVariant.NONE] + ([ColorVariant.REGULAR] if opts.colors_regular else []) + From b14524215ba2119c1f37787bbd4201d6d00d13df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 10:08:58 +0200 Subject: [PATCH 18/30] render: use cell cols from compose chain, not grapheme MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fcft’s view of how many columns a grapheme cluster is may differ from our own. Make sure the rendered glyph matches the number of columns that were allocated when the cluster was printed. --- render.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/render.c b/render.c index d46cb90b..5bb1023e 100644 --- a/render.c +++ b/render.c @@ -570,10 +570,11 @@ render_cell(struct terminal *term, pixman_image_t *pix, } if (grapheme != NULL) { + cell_cols = composed->width; + composed = NULL; glyphs = grapheme->glyphs; glyph_count = grapheme->count; - cell_cols = grapheme->cols; } } From f3e5c3deb943bde60364e129a63bea5d56cb6159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 10:10:38 +0200 Subject: [PATCH 19/30] doc: foot.ini: grapheme-shaping: mention regular compose characters --- doc/foot.ini.5.scd | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/foot.ini.5.scd b/doc/foot.ini.5.scd index 94ad42f6..2e408b3a 100644 --- a/doc/foot.ini.5.scd +++ b/doc/foot.ini.5.scd @@ -983,9 +983,11 @@ any of these options. This is required to render e.g. flag (emoji) sequences, keycap sequences, modifier sequences, zero-width-joiner (ZWJ) sequences - andn emoji tag sequences. + andn emoji tag sequences. It might also improve rendering of + composed characters, depending on font. - This is an experimental feature with the following requirements and limitations: + This is an experimental feature with the following requirements + and limitations: - foot must have been compiled with utf8proc support - fcft must have been compiled with HarfBuzz support From fd7005879577c08b9e5920755fd2f3a5a2274d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 10:51:43 +0200 Subject: [PATCH 20/30] =?UTF-8?q?changelog:=20add=20a=20=E2=80=9Cgrapheme?= =?UTF-8?q?=20shaping=E2=80=9D=20section?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8e7f333..1c0f1443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,59 @@ ## Unreleased +### Grapheme shaping + +This release adds _experimental_, opt-in support for grapheme cluster +segmentation and grapheme shaping. + +(note: several of the examples below may not render correctly in your +browser, viewer or editor). + +Grapheme cluster segmentation is the art of splitting up text into +grapheme clusters, where a cluster may consist of more than one +Unicode codepoint. For example, 🙂 is a single codepoint, while 👩🏽‍🚀 +consists of 5 codepoints (_Woman_ + _Medium skin tone_ + _Zero width +joiner_ + _Rocket_). The goal is to _cluster_ codepoints belonging to +the same grapheme in the same cell in the terminal. + +Previous versions of foot implemented a simple grapheme cluster +segmentation technique that **only** handled zero-width +codepoints. This allowed us to cluster composed characters (e.g. q́ - +_q_ + _COMBINING ACUTE ACCENT_). + +Once we have a grapheme cluster, we need to _shape_ it. + +Composed characters are simple: they are typically rendered as +multiple glyphs layered on top of each other. This is why previous +versions of foot got away with it without any actual text shaping +support. + +Beyond that, support from the font library is needed. Foot now depends +on fcft-2.4, which added support for grapheme and text shaping. When +rendering a cell, we ask the font library: give us the glyph(s) for +this sequence of codepoints. + +Fancy emoji sequences aside, using libutf8proc for grapheme cluster +segmentation means **improved correctness**. + +For full support, the following is required: + +* fcft compiled with HarfBuzz support +* foot compiled with libutf8proc support +* `tweak.grapheme-shaping=yes` in `foot.ini` + +This feature is _experimental_ mostly due to the “wcwidth” problem; +how many cells should foot allocate for a grapheme cluster? While the +answer may seem simple, the problem is that, whatever the answer is, +the client application **must** have come up with the **same** +answer. Otherwise we get cursor synchronization issues. + +In this release, foot simply adds together the `wcwidth()` of all +codepoints in the grapheme cluster. This is equivalent to running +`wcswidth()` on the entire cluster. **This is likely to change in the +future**. + + ### Added * Support for DECSET/DECRST 2026, as an alternative to the existing From fcd632729787b181eb3ec3d2bd839ed2395f1e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 11:24:30 +0200 Subject: [PATCH 21/30] term: bump compose chain char array to 10 chars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are some very long sequences out there... e.g. 👩🏼‍❤️‍💋‍👨🏽 --- terminal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terminal.h b/terminal.h index 307e2b0b..c9f5aee8 100644 --- a/terminal.h +++ b/terminal.h @@ -85,7 +85,7 @@ struct damage { }; struct composed { - wchar_t chars[7]; + wchar_t chars[10]; uint8_t count; int width; }; From fe8ca23cfee77a6cac433892755dd7455e354a9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 13:17:07 +0200 Subject: [PATCH 22/30] composed: store compose chains in a binary search tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous implementation stored compose chains in a dynamically allocated array. Adding a chain was easy: resize the array and append the new chain at the end. Looking up a compose chain given a compose chain key/index was also easy: just index into the array. However, searching for a pre-existing chain given a codepoint sequence was very slow. Since the array wasn’t sorted, we typically had to scan through the entire array, just to realize that there is no pre-existing chain, and that we need to add a new one. Since this happens for *each* codepoint in a grapheme cluster, things quickly became really slow. Things were ok:ish as long as the compose chain struct was small, as that made it possible to hold all the chains in the cache. Once the number of chains reached a certain point, or when we were forced to bump maximum number of allowed codepoints in a chain, we started thrashing the cache and things got much much worse. So what can we do? We can’t sort the array, because a) that would invalidate all existing chain keys in the grid (and iterating the entire scrollback and updating compose keys is *not* an option). b) inserting a chain becomes slow as we need to first find _where_ to insert it, and then memmove() the rest of the array. This patch uses a binary search tree to store the chains instead of a simple array. The tree is sorted on a “key”, which is the XOR of all codepoints, truncated to the CELL_COMB_CHARS_HI-CELL_COMB_CHARS_LO range. The grid now stores CELL_COMB_CHARS_LO+key, instead of CELL_COMB_CHARS_LO+index. Since the key is truncated, collisions may occur. This is handled by incrementing the key by 1. Lookup is of course slower than before, O(log n) instead of O(1). Insertion is slightly slower as well: technically it’s O(log n) instead of O(1). However, we also need to take into account the re-allocating the array will occasionally force a full copy of the array when it cannot simply be growed. But finding a pre-existing chain is now *much* faster: O(log n) instead of O(n). In most cases, the first lookup will either succeed (return a true match), or fail (return NULL). However, since key collisions are possible, it may also return false matches. This means we need to verify the contents of the chain before deciding to use it instead of inserting a new chain. But remember that this comparison was being done for each and every chain in the previous implementation. With lookups being much faster, and in particular, no longer requiring us to check the chain contents for every singlec chain, we can now use a dynamically allocated ‘chars’ array in the chain. This was previously a hardcoded array of 10 chars. Using a dynamic allocated array means looking in the array is slower, since we now need two loads: one to load the pointer, and a second to load _from_ the pointer. As a result, the base size of a compose chain (i.e. an “empty” chain) has now been reduced from 48 bytes to 32. A chain with two codepoints is 40 bytes. This means we have up to 4 codepoints while still using less, or the same amount, of memory as before. Furthermore, the Unicode random test (i.e. write random “unicode” chars) is now **faster** than current master (i.e. before text-shaping support was added), **with** test-shaping enabled. With text-shaping disabled, we’re _even_ faster. --- composed.c | 76 +++++++++++++++++++++++++++++++++++++++++++ composed.h | 18 ++++++++++ extract.c | 7 ++-- grid.c | 4 +-- grid.h | 4 +-- meson.build | 1 + render.c | 9 +++-- search.c | 5 ++- selection.c | 28 +++++----------- terminal.c | 3 +- terminal.h | 7 +--- vt.c | 94 ++++++++++++++++++++++++++++++----------------------- 12 files changed, 170 insertions(+), 86 deletions(-) create mode 100644 composed.c create mode 100644 composed.h diff --git a/composed.c b/composed.c new file mode 100644 index 00000000..f969c8f1 --- /dev/null +++ b/composed.c @@ -0,0 +1,76 @@ +#include "composed.h" + +#include +#include + +#include "debug.h" + +struct composed * +composed_lookup(struct composed *root, uint32_t key) +{ + struct composed *node = root; + + while (node != NULL) { + if (key == node->key) + return node; + + node = key < node->key ? node->left : node->right; + } + + return NULL; +} + +uint32_t +composed_insert(struct composed **root, struct composed *node) +{ + node->left = node->right = NULL; + + if (*root == NULL) { + *root = node; + return node->key; + } + + uint32_t key = node->key; + + struct composed *prev = NULL; + struct composed *n = *root; + + while (n != NULL) { + if (n->key == node->key) { + /* TODO: wrap around at (CELL_COMB_CHARS_HI - CELL_COMB_CHARS_LO) */ + key++; + } + + prev = n; + n = key < n->key ? n->left : n->right; + } + + xassert(prev != NULL); + xassert(n == NULL); + + /* May have been changed */ + node->key = key; + + if (key < prev->key) { + xassert(prev->left == NULL); + prev->left = node; + } else { + xassert(prev->right == NULL); + prev->right = node; + } + + return key; +} + +void +composed_free(struct composed *root) +{ + if (root == NULL) + return; + + composed_free(root->left); + composed_free(root->right); + + free(root->chars); + free(root); +} diff --git a/composed.h b/composed.h new file mode 100644 index 00000000..3511ad30 --- /dev/null +++ b/composed.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +struct composed { + wchar_t *chars; + struct composed *left; + struct composed *right; + uint32_t key; + uint8_t count; + uint8_t width; +}; + +struct composed *composed_lookup(struct composed *root, uint32_t key); +uint32_t composed_insert(struct composed **root, struct composed *node); + +void composed_free(struct composed *root); diff --git a/extract.c b/extract.c index 8aa1acf3..4acd481c 100644 --- a/extract.c +++ b/extract.c @@ -223,11 +223,10 @@ extract_one(const struct terminal *term, const struct row *row, ctx->newline_count = 0; ctx->empty_count = 0; - if (cell->wc >= CELL_COMB_CHARS_LO && - cell->wc < (CELL_COMB_CHARS_LO + term->composed_count)) + if (cell->wc >= CELL_COMB_CHARS_LO && cell->wc <= CELL_COMB_CHARS_HI) { - const struct composed *composed - = &term->composed[cell->wc - CELL_COMB_CHARS_LO]; + const struct composed *composed = composed_lookup( + term->composed, cell->wc - CELL_COMB_CHARS_LO); if (!ensure_size(ctx, composed->count)) goto err; diff --git a/grid.c b/grid.c index 4419434c..733d8ded 100644 --- a/grid.c +++ b/grid.c @@ -416,9 +416,7 @@ grid_resize_and_reflow( struct grid *grid, int new_rows, int new_cols, int old_screen_rows, int new_screen_rows, size_t tracking_points_count, - struct coord *const _tracking_points[static tracking_points_count], - size_t compose_count, const struct - composed composed[static compose_count]) + struct coord *const _tracking_points[static tracking_points_count]) { #if defined(TIME_REFLOW) && TIME_REFLOW struct timeval start; diff --git a/grid.h b/grid.h index c41a5ba5..3537ddb5 100644 --- a/grid.h +++ b/grid.h @@ -19,9 +19,7 @@ void grid_resize_and_reflow( struct grid *grid, int new_rows, int new_cols, int old_screen_rows, int new_screen_rows, size_t tracking_points_count, - struct coord *const _tracking_points[static tracking_points_count], - size_t compose_count, - const struct composed composed[static compose_count]); + struct coord *const _tracking_points[static tracking_points_count]); static inline int grid_row_absolute(const struct grid *grid, int row_no) diff --git a/meson.build b/meson.build index 107fd905..33b54c9c 100644 --- a/meson.build +++ b/meson.build @@ -147,6 +147,7 @@ misc = static_library( vtlib = static_library( 'vtlib', 'base64.c', 'base64.h', + 'composed.c', 'composed.h', 'csi.c', 'csi.h', 'dcs.c', 'dcs.h', 'osc.c', 'osc.h', diff --git a/render.c b/render.c index 5bb1023e..d3e03b63 100644 --- a/render.c +++ b/render.c @@ -557,10 +557,9 @@ render_cell(struct terminal *term, pixman_image_t *pix, cell_cols = single->cols; } - else if (base >= CELL_COMB_CHARS_LO && - base < (CELL_COMB_CHARS_LO + term->composed_count)) + else if (base >= CELL_COMB_CHARS_LO && base <= CELL_COMB_CHARS_HI) { - composed = &term->composed[base - CELL_COMB_CHARS_LO]; + composed = composed_lookup(term->composed, base - CELL_COMB_CHARS_LO); base = composed->chars[0]; if (term->conf->can_shape_grapheme && term->conf->tweak.grapheme_shaping) { @@ -3339,8 +3338,8 @@ maybe_resize(struct terminal *term, int width, int height, bool force) /* Resize grids */ grid_resize_and_reflow( &term->normal, new_normal_grid_rows, new_cols, old_rows, new_rows, - term->selection.end.row >= 0 ? ALEN(tracking_points) : 0, tracking_points, - term->composed_count, term->composed); + term->selection.end.row >= 0 ? ALEN(tracking_points) : 0, + tracking_points); grid_resize_without_reflow( &term->alt, new_alt_grid_rows, new_cols, old_rows, new_rows); diff --git a/search.c b/search.c index 859608f5..8f28a656 100644 --- a/search.c +++ b/search.c @@ -245,10 +245,9 @@ matches_cell(const struct terminal *term, const struct cell *cell, size_t search wchar_t base = cell->wc; const struct composed *composed = NULL; - if (base >= CELL_COMB_CHARS_LO && - base < (CELL_COMB_CHARS_LO + term->composed_count)) + if (base >= CELL_COMB_CHARS_LO && base <= CELL_COMB_CHARS_HI) { - composed = &term->composed[base - CELL_COMB_CHARS_LO]; + composed = composed_lookup(term->composed, base - CELL_COMB_CHARS_LO); base = composed->chars[0]; } diff --git a/selection.c b/selection.c index cafb161e..57ab8bf6 100644 --- a/selection.c +++ b/selection.c @@ -246,11 +246,8 @@ selection_find_word_boundary_left(struct terminal *term, struct coord *pos, c = r->cells[pos->col].wc; } - if (c >= CELL_COMB_CHARS_LO && - c < (CELL_COMB_CHARS_LO + term->composed_count)) - { - c = term->composed[c - CELL_COMB_CHARS_LO].chars[0]; - } + if (c >= CELL_COMB_CHARS_LO && c <= CELL_COMB_CHARS_HI) + c = composed_lookup(term->composed, c - CELL_COMB_CHARS_LO)->chars[0]; bool initial_is_space = c == 0 || iswspace(c); bool initial_is_delim = @@ -286,11 +283,8 @@ selection_find_word_boundary_left(struct terminal *term, struct coord *pos, c = row->cells[next_col].wc; } - if (c >= CELL_COMB_CHARS_LO && - c < (CELL_COMB_CHARS_LO + term->composed_count)) - { - c = term->composed[c - CELL_COMB_CHARS_LO].chars[0]; - } + if (c >= CELL_COMB_CHARS_LO && c <= CELL_COMB_CHARS_HI) + c = composed_lookup(term->composed, c - CELL_COMB_CHARS_LO)->chars[0]; bool is_space = c == 0 || iswspace(c); bool is_delim = @@ -325,11 +319,8 @@ selection_find_word_boundary_right(struct terminal *term, struct coord *pos, c = r->cells[pos->col].wc; } - if (c >= CELL_COMB_CHARS_LO && - c < (CELL_COMB_CHARS_LO + term->composed_count)) - { - c = term->composed[c - CELL_COMB_CHARS_LO].chars[0]; - } + if (c >= CELL_COMB_CHARS_LO && c <= CELL_COMB_CHARS_HI) + c = composed_lookup(term->composed, c - CELL_COMB_CHARS_LO)->chars[0]; bool initial_is_space = c == 0 || iswspace(c); bool initial_is_delim = @@ -367,11 +358,8 @@ selection_find_word_boundary_right(struct terminal *term, struct coord *pos, c = row->cells[next_col].wc; } - if (c >= CELL_COMB_CHARS_LO && - c < (CELL_COMB_CHARS_LO + term->composed_count)) - { - c = term->composed[c - CELL_COMB_CHARS_LO].chars[0]; - } + if (c >= CELL_COMB_CHARS_LO && c <= CELL_COMB_CHARS_HI) + c = composed_lookup(term->composed, c - CELL_COMB_CHARS_LO)->chars[0]; bool is_space = c == 0 || iswspace(c); bool is_delim = diff --git a/terminal.c b/terminal.c index 9e0795a1..cb836fef 100644 --- a/terminal.c +++ b/terminal.c @@ -1132,7 +1132,6 @@ term_init(const struct config *conf, struct fdm *fdm, struct reaper *reaper, .normal = {.scroll_damage = tll_init(), .sixel_images = tll_init()}, .alt = {.scroll_damage = tll_init(), .sixel_images = tll_init()}, .grid = &term->normal, - .composed_count = 0, .composed = NULL, .alt_scrolling = conf->mouse.alternate_scroll_mode, .meta = { @@ -1431,7 +1430,7 @@ term_destroy(struct terminal *term) grid_free(&term->normal); grid_free(&term->alt); - free(term->composed); + composed_free(term->composed); free(term->window_title); tll_free_and_free(term->window_title_stack, free); diff --git a/terminal.h b/terminal.h index c9f5aee8..00c4221f 100644 --- a/terminal.h +++ b/terminal.h @@ -16,6 +16,7 @@ #include //#include "config.h" +#include "composed.h" #include "debug.h" #include "fdm.h" #include "macros.h" @@ -84,12 +85,6 @@ struct damage { int lines; }; -struct composed { - wchar_t chars[10]; - uint8_t count; - int width; -}; - struct row_uri_range { int start; int end; diff --git a/vt.c b/vt.c index bbe6fc9a..158ac539 100644 --- a/vt.c +++ b/vt.c @@ -611,21 +611,22 @@ action_utf8_print(struct terminal *term, wchar_t wc) xassert(col >= 0 && col < term->cols); wchar_t base = row->cells[col].wc; wchar_t UNUSED last = base; - size_t search_start_index = 0; + uint32_t key = base ^ wc; /* Is base cell already a cluster? */ const struct composed *composed = - (base >= CELL_COMB_CHARS_LO && - base < (CELL_COMB_CHARS_LO + term->composed_count)) - ? &term->composed[base - CELL_COMB_CHARS_LO] + (base >= CELL_COMB_CHARS_LO && base <= CELL_COMB_CHARS_HI) + ? composed_lookup(term->composed, base - CELL_COMB_CHARS_LO) : NULL; if (composed != NULL) { - search_start_index = base - CELL_COMB_CHARS_LO; base = composed->chars[0]; last = composed->chars[composed->count - 1]; + key = composed->key ^ wc; } + key &= CELL_COMB_CHARS_HI - CELL_COMB_CHARS_LO; + #if defined(FOOT_GRAPHEME_CLUSTERING) if (grapheme_clustering) { /* Check if we're on a grapheme cluster break */ @@ -677,7 +678,7 @@ action_utf8_print(struct terminal *term, wchar_t wc) } size_t wanted_count = composed != NULL ? composed->count + 1 : 2; - if (wanted_count > ALEN(composed->chars)) { + if (wanted_count > 255) { xassert(composed != NULL); #if defined(LOG_ENABLE_DBG) && LOG_ENABLE_DBG @@ -691,47 +692,46 @@ action_utf8_print(struct terminal *term, wchar_t wc) wanted_count--; } - xassert(wanted_count <= ALEN(composed->chars)); + xassert(wanted_count <= 255); /* Look for existing combining chain */ - for (size_t i = search_start_index; i < term->composed_count; i++) { - const struct composed *cc = &term->composed[i]; + while (true) { + const struct composed *cc = composed_lookup(term->composed, key); + if (cc == NULL) + break; - if (cc->chars[0] != base) + /* + * We may have a key collisison, so need to check that + * it’s a true match. If not, bumb the key and try + * again. + */ + + if (cc->chars[0] != base || + cc->count != wanted_count || + cc->chars[wanted_count - 1] != wc) + { + key++; continue; - - if (cc->count != wanted_count) - continue; - - bool match = true; - for (size_t j = 1; j < wanted_count - 1; j++) { - if (cc->chars[j] != composed->chars[j]) { - match = false; - break; - } } - if (!match) - continue; - if (cc->chars[wanted_count - 1] != wc) - continue; + bool match = composed != NULL + ? memcmp(&cc->chars[1], &composed->chars[1], + (wanted_count - 2) * sizeof(cc->chars[0])) == 0 + : true; - wc = CELL_COMB_CHARS_LO + i; + if (!match) { + key++; + continue; + } + + wc = CELL_COMB_CHARS_LO + cc->key; width = cc->width; goto out; } - /* Allocate new chain */ - - struct composed new_cc; - new_cc.count = wanted_count; - new_cc.chars[0] = base; - - for (size_t i = 1; i < wanted_count - 1; i++) - new_cc.chars[i] = composed->chars[i]; - new_cc.chars[wanted_count - 1] = wc; - - if (unlikely(term->composed_count >= CELL_COMB_CHARS_HI)) { + if (unlikely(term->composed_count >= + (CELL_COMB_CHARS_HI - CELL_COMB_CHARS_LO))) + { /* We reached our maximum number of allowed composed * character chains. Fall through here and print the * current zero-width character to the current cell */ @@ -739,18 +739,32 @@ action_utf8_print(struct terminal *term, wchar_t wc) goto out; } + /* Allocate new chain */ + struct composed *new_cc = xmalloc(sizeof(*new_cc)); + new_cc->chars = xmalloc(wanted_count * sizeof(new_cc->chars[0])); + new_cc->key = key; + new_cc->count = wanted_count; + new_cc->chars[0] = base; + new_cc->chars[wanted_count - 1] = wc; + + if (composed != NULL) { + memcpy(&new_cc->chars[1], &composed->chars[1], + (wanted_count - 2) * sizeof(new_cc->chars[0])); + } + int grapheme_width = composed != NULL ? composed->width : base_width; + if (wc == 0xfe0f && grapheme_width < 2) grapheme_width = 2; else grapheme_width += width; - new_cc.width = grapheme_width; + new_cc->width = grapheme_width; term->composed_count++; - term->composed = xrealloc(term->composed, term->composed_count * sizeof(term->composed[0])); - term->composed[term->composed_count - 1] = new_cc; + key = composed_insert(&term->composed, new_cc); + wc = CELL_COMB_CHARS_LO + key; + xassert(wc <= CELL_COMB_CHARS_HI); - wc = CELL_COMB_CHARS_LO + term->composed_count - 1; width = grapheme_width; goto out; } From 415ecfc6fa0d0d7c2d4f8c455b770b781f2a5ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 13:53:19 +0200 Subject: [PATCH 23/30] vt: codespell: bumb -> bump --- vt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vt.c b/vt.c index 158ac539..94bd6685 100644 --- a/vt.c +++ b/vt.c @@ -702,7 +702,7 @@ action_utf8_print(struct terminal *term, wchar_t wc) /* * We may have a key collisison, so need to check that - * it’s a true match. If not, bumb the key and try + * it’s a true match. If not, bump the key and try * again. */ From f19797a5af71112393f0fad6c12061432a29ca69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 14:04:08 +0200 Subject: [PATCH 24/30] =?UTF-8?q?changelog:=20updates=20to=20=E2=80=9Cgrap?= =?UTF-8?q?heme=20shaping=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c0f1443..6b84b1f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,14 +39,14 @@ browser, viewer or editor). Grapheme cluster segmentation is the art of splitting up text into grapheme clusters, where a cluster may consist of more than one Unicode codepoint. For example, 🙂 is a single codepoint, while 👩🏽‍🚀 -consists of 5 codepoints (_Woman_ + _Medium skin tone_ + _Zero width +consists of 4 codepoints (_Woman_ + _Medium skin tone_ + _Zero width joiner_ + _Rocket_). The goal is to _cluster_ codepoints belonging to the same grapheme in the same cell in the terminal. Previous versions of foot implemented a simple grapheme cluster segmentation technique that **only** handled zero-width -codepoints. This allowed us to cluster composed characters (e.g. q́ - -_q_ + _COMBINING ACUTE ACCENT_). +codepoints. This allowed us to cluster composed characters, like q́ +(_q_ + _COMBINING ACUTE ACCENT_). Once we have a grapheme cluster, we need to _shape_ it. @@ -72,7 +72,7 @@ For full support, the following is required: This feature is _experimental_ mostly due to the “wcwidth” problem; how many cells should foot allocate for a grapheme cluster? While the answer may seem simple, the problem is that, whatever the answer is, -the client application **must** have come up with the **same** +the client application **must** come up with the **same** answer. Otherwise we get cursor synchronization issues. In this release, foot simply adds together the `wcwidth()` of all @@ -80,6 +80,11 @@ codepoints in the grapheme cluster. This is equivalent to running `wcswidth()` on the entire cluster. **This is likely to change in the future**. +Finally, note that grapheme shaping is not the same thing as text (or +text run) shaping. In this version, foot only shapes individual +graphemes, not entire text runs. That means e.g. ligatures are **not** +supported. + ### Added From 09c4d162329c3141caf059d05f7c3ec9c5f4456b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 17:33:09 +0200 Subject: [PATCH 25/30] =?UTF-8?q?changelog:=20put=20emphasis=20on=20?= =?UTF-8?q?=E2=80=98opt-in=E2=80=99=20as=20well?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b84b1f8..d8c4c628 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ ### Grapheme shaping -This release adds _experimental_, opt-in support for grapheme cluster +This release adds _experimenta, opt-in_ support for grapheme cluster segmentation and grapheme shaping. (note: several of the examples below may not render correctly in your From d5d57c1b20ee7caa6ac8461cdf5aedb1da5e60bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 17:36:52 +0200 Subject: [PATCH 26/30] changelog: composed -> combining --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8c4c628..4613ea35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,12 +45,12 @@ the same grapheme in the same cell in the terminal. Previous versions of foot implemented a simple grapheme cluster segmentation technique that **only** handled zero-width -codepoints. This allowed us to cluster composed characters, like q́ +codepoints. This allowed us to cluster combining characters, like q́ (_q_ + _COMBINING ACUTE ACCENT_). Once we have a grapheme cluster, we need to _shape_ it. -Composed characters are simple: they are typically rendered as +Combining characters are simple: they are typically rendered as multiple glyphs layered on top of each other. This is why previous versions of foot got away with it without any actual text shaping support. From cf101ea3004691614081c3e9f079829a67f3136c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 17:36:57 +0200 Subject: [PATCH 27/30] changelog: describe what (does not) happens when grapheme-shaping=no --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4613ea35..b68183fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,11 @@ For full support, the following is required: * foot compiled with libutf8proc support * `tweak.grapheme-shaping=yes` in `foot.ini` +If `tweak.grapheme-shaping` has **not** been enabled, foot will +neither use libutf8proc to do grapheme cluster segmentation, nor will +it use fcft’s grapheme shaping capabilities to shape combining +characters. + This feature is _experimental_ mostly due to the “wcwidth” problem; how many cells should foot allocate for a grapheme cluster? While the answer may seem simple, the problem is that, whatever the answer is, From 4ea7c5b63f890aa1c91aee074c85d482fffab1c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 17:50:04 +0200 Subject: [PATCH 28/30] features: add feature_graphemes() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Returns true if we’re compiled with grapheme shaping support, false otherwise. --- foot-features.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/foot-features.h b/foot-features.h index ae00c564..cdc8056f 100644 --- a/foot-features.h +++ b/foot-features.h @@ -19,3 +19,12 @@ static inline bool feature_pgo(void) return false; #endif } + +static inline bool feature_graphemes(void) +{ +#if defined(FOOT_GRAPHEME_CLUSTERING) && FOOT_GRAPHEME_CLUSTERING + return true; +#else + return false; +#endif +} From a319ddf094cd015673bb80e8137cbebaeaa2c26e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 17:50:30 +0200 Subject: [PATCH 29/30] foot: add +/-graphemes to version output --- main.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.c b/main.c index 2c392fca..bbe89052 100644 --- a/main.c +++ b/main.c @@ -45,10 +45,11 @@ static const char * version_and_features(void) { static char buf[256]; - snprintf(buf, sizeof(buf), "version: %s %cime %cpgo", + snprintf(buf, sizeof(buf), "version: %s %cpgo %cime %cgraphemes", FOOT_VERSION, + feature_pgo() ? '+' : '-', feature_ime() ? '+' : '-', - feature_pgo() ? '+' : '-'); + feature_graphemes() ? '+' : '-'); return buf; } From 3f0f5ec3b7ee5bdbdc085f133920b83a611601b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Ekl=C3=B6f?= Date: Thu, 24 Jun 2021 17:50:44 +0200 Subject: [PATCH 30/30] client: add +/-graphemes to version output --- client.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client.c b/client.c index 95c57656..9764da11 100644 --- a/client.c +++ b/client.c @@ -43,10 +43,11 @@ static const char * version_and_features(void) { static char buf[256]; - snprintf(buf, sizeof(buf), "version: %s %cime %cpgo", + snprintf(buf, sizeof(buf), "version: %s %cpgo %cime %cgraphemes", FOOT_VERSION, + feature_pgo() ? '+' : '-', feature_ime() ? '+' : '-', - feature_pgo() ? '+' : '-'); + feature_graphemes() ? '+' : '-'); return buf; }