diff --git a/PKGBUILD b/PKGBUILD
index c09028d3..e8307ef6 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -5,7 +5,7 @@ arch=('x86_64' 'aarch64')
 url=https://codeberg.org/dnkl/foot
 license=(mit)
 makedepends=('meson' 'ninja' 'scdoc' 'python' 'ncurses' 'wayland-protocols' 'tllist>=1.0.1')
-depends=('libxkbcommon' 'wayland' 'pixman' 'fcft>=2.1.1')
+depends=('libxkbcommon' 'wayland' 'pixman' 'fontconfig' 'fcft>=2.1.1')
 source=()
 
 pkgver() {
diff --git a/config.c b/config.c
index bf539442..ab9bb08b 100644
--- a/config.c
+++ b/config.c
@@ -14,6 +14,7 @@
 
 #include <linux/input-event-codes.h>
 #include <xkbcommon/xkbcommon.h>
+#include <fontconfig/fontconfig.h>
 
 #define LOG_MODULE "config"
 #define LOG_ENABLE_DBG 0
@@ -265,7 +266,7 @@ parse_section_main(const char *key, const char *value, struct config *conf,
             while (*font != '\0' && isspace(*font))
                 font++;
             if (*font != '\0')
-                tll_push_back(conf->fonts, strdup(font));
+                tll_push_back(conf->fonts, config_font_parse(font));
         }
         free(copy);
     }
@@ -963,7 +964,7 @@ config_load(struct config *conf, const char *conf_path)
 
 out:
     if (ret && tll_length(conf->fonts) == 0)
-        tll_push_back(conf->fonts, strdup("monospace"));
+        tll_push_back(conf->fonts, config_font_parse("monospace"));
 
     free(default_path);
     return ret;
@@ -976,7 +977,9 @@ config_free(struct config conf)
     free(conf.shell);
     free(conf.title);
     free(conf.app_id);
-    tll_free_and_free(conf.fonts, free);
+    tll_foreach(conf.fonts, it)
+        config_font_destroy(&it->item);
+    tll_free(conf.fonts);
     free(conf.server_socket_path);
 
     for (enum bind_action_normal i = 0; i < BIND_ACTION_COUNT; i++)
@@ -984,3 +987,36 @@ config_free(struct config conf)
     for (enum bind_action_search i = 0; i < BIND_ACTION_SEARCH_COUNT; i++)
         free(conf.bindings.search[i]);
 }
+
+struct config_font
+config_font_parse(const char *pattern)
+{
+    FcPattern *pat = FcNameParse((const FcChar8 *)pattern);
+
+    double pt_size = -1.0;
+    FcPatternGetDouble(pat, FC_SIZE, 0, &pt_size);
+    FcPatternRemove(pat, FC_SIZE, 0);
+
+    int px_size = -1;
+    FcPatternGetInteger(pat, FC_PIXEL_SIZE, 0, &px_size);
+    FcPatternRemove(pat, FC_PIXEL_SIZE, 0);
+
+    if (pt_size == -1. && px_size == -1)
+        pt_size = 8.0;
+
+    char *stripped_pattern = (char *)FcNameUnparse(pat);
+    FcPatternDestroy(pat);
+
+    return (struct config_font){
+        .pattern = stripped_pattern,
+        .pt_size = pt_size,
+        .px_size = px_size};
+}
+
+void
+config_font_destroy(struct config_font *font)
+{
+    if (font == NULL)
+        return;
+    free(font->pattern);
+}
diff --git a/config.h b/config.h
index bb8ad1f4..caf269b5 100644
--- a/config.h
+++ b/config.h
@@ -7,6 +7,12 @@
 
 #include "terminal.h"
 
+struct config_font {
+    char *pattern;
+    double pt_size;
+    int px_size;
+};
+
 struct config {
     char *term;
     char *shell;
@@ -19,7 +25,7 @@ struct config {
     unsigned pad_y;
     enum { STARTUP_WINDOWED, STARTUP_MAXIMIZED, STARTUP_FULLSCREEN } startup_mode;
 
-    tll(char *) fonts;
+    tll(struct config_font) fonts;
 
     int scrollback_lines;
 
@@ -87,3 +93,6 @@ struct config {
 
 bool config_load(struct config *conf, const char *path);
 void config_free(struct config conf);
+
+struct config_font config_font_parse(const char *pattern);
+void config_font_destroy(struct config_font *font);
diff --git a/main.c b/main.c
index 210179d3..02bd7fcd 100644
--- a/main.c
+++ b/main.c
@@ -224,7 +224,7 @@ main(int argc, char *const *argv)
                 if (strlen(font) == 0)
                     continue;
 
-                tll_push_back(conf_fonts, strdup(font));
+                tll_push_back(conf_fonts, font);
             }
             break;
 
@@ -342,9 +342,11 @@ main(int argc, char *const *argv)
     if (login_shell)
         conf.login_shell = true;
     if (tll_length(conf_fonts) > 0) {
-        tll_free_and_free(conf.fonts, free);
+        tll_foreach(conf.fonts, it)
+            config_font_destroy(&it->item);
+        tll_free(conf.fonts);
         tll_foreach(conf_fonts, it)
-            tll_push_back(conf.fonts, it->item);
+            tll_push_back(conf.fonts, config_font_parse(it->item));
         tll_free(conf_fonts);
     }
     if (conf_width > 0)
diff --git a/meson.build b/meson.build
index af9f834a..eda8d06c 100644
--- a/meson.build
+++ b/meson.build
@@ -56,6 +56,7 @@ wayland_protocols = dependency('wayland-protocols')
 wayland_client = dependency('wayland-client')
 wayland_cursor = dependency('wayland-cursor')
 xkb = dependency('xkbcommon')
+fontconfig = dependency('fontconfig')
 
 tllist = dependency('tllist', version: '>=1.0.1', fallback: 'tllist')
 fcft = dependency('fcft', version: ['>=2.1.1', '<3.0.0'], fallback: 'fcft')
@@ -126,7 +127,8 @@ executable(
   'vt.c', 'vt.h',
   'wayland.c', 'wayland.h',
   wl_proto_src + wl_proto_headers, version,
-  dependencies: [math, threads, pixman, wayland_client, wayland_cursor, xkb, tllist, fcft],
+  dependencies: [math, threads, pixman, wayland_client, wayland_cursor, xkb, fontconfig,
+                 tllist, fcft],
   install: true)
 
 executable(
diff --git a/terminal.c b/terminal.c
index c6e03c24..0bb545e2 100644
--- a/terminal.c
+++ b/terminal.c
@@ -664,27 +664,41 @@ font_loader_thread(void *_data)
 }
 
 static bool
-load_fonts_from_conf(const struct terminal *term, const struct config *conf,
-                     struct fcft_font *fonts[static 4])
+reload_fonts(struct terminal *term)
 {
-    const size_t count = tll_length(conf->fonts);
-    const char *names[count];
+    const size_t count = tll_length(term->conf->fonts);
+    char *names[count];
 
     size_t i = 0;
-    tll_foreach(conf->fonts, it)
-        names[i++] = it->item;
+    tll_foreach(term->conf->fonts, it) {
+        bool use_px_size = term->font_sizes[i].px_size > 0;
+        char size[64];
 
-    char attrs0[64], attrs1[64], attrs2[64], attrs3[64];
+        if (use_px_size)
+            snprintf(size, sizeof(size), ":pixelsize=%d", term->font_sizes[i].px_size);
+        else
+            snprintf(size, sizeof(size), ":size=%.2f", term->font_sizes[i].pt_size);
+
+        size_t len = strlen(it->item.pattern) + strlen(size) + 1;
+        names[i] = malloc(len);
+
+        strcpy(names[i], it->item.pattern);
+        strcat(names[i], size);
+        i++;
+    }
+
+    char attrs0[256], attrs1[256], attrs2[256], attrs3[256];
     snprintf(attrs0, sizeof(attrs0), "dpi=%u", term->font_dpi);
     snprintf(attrs1, sizeof(attrs1), "dpi=%u:weight=bold", term->font_dpi);
     snprintf(attrs2, sizeof(attrs2), "dpi=%u:slant=italic", term->font_dpi);
     snprintf(attrs3, sizeof(attrs3), "dpi=%u:weight=bold:slant=italic", term->font_dpi);
 
+    struct fcft_font *fonts[4];
     struct font_load_data data[4] = {
-        {count, names, attrs0, &fonts[0]},
-        {count, names, attrs1, &fonts[1]},
-        {count, names, attrs2, &fonts[2]},
-        {count, names, attrs3, &fonts[3]},
+        {count, (const char **)names, attrs0, &fonts[0]},
+        {count, (const char **)names, attrs1, &fonts[1]},
+        {count, (const char **)names, attrs2, &fonts[2]},
+        {count, (const char **)names, attrs3, &fonts[3]},
     };
 
     thrd_t tids[4] = {};
@@ -715,7 +729,22 @@ load_fonts_from_conf(const struct terminal *term, const struct config *conf,
         }
     }
 
-    return success;
+    for (size_t i = 0; i < count; i++)
+        free(names[i]);
+
+    return success ? term_set_fonts(term, fonts) : success;
+}
+
+static bool
+load_fonts_from_conf(struct terminal *term)
+{
+    size_t i = 0;
+    tll_foreach(term->conf->fonts, it) {
+        term->font_sizes[i++] = (struct config_font){
+            .pt_size = it->item.pt_size, .px_size = it->item.px_size};
+    }
+
+    return reload_fonts(term);
 }
 
 struct terminal *
@@ -802,8 +831,8 @@ term_init(const struct config *conf, struct fdm *fdm, struct reaper *reaper,
         .quit = false,
         .ptmx = ptmx,
         .ptmx_buffer = tll_init(),
+        .font_sizes = malloc(sizeof(term->font_sizes[0]) * tll_length(conf->fonts)),
         .font_dpi = 0,
-        .font_adjustments = 0,
         .font_subpixel = (conf->colors.alpha == 0xffff  /* Can't do subpixel rendering on transparent background */
                           ? FCFT_SUBPIXEL_DEFAULT
                           : FCFT_SUBPIXEL_NONE),
@@ -900,6 +929,14 @@ term_init(const struct config *conf, struct fdm *fdm, struct reaper *reaper,
         .cwd = strdup(cwd),
     };
 
+    {
+        size_t i = 0;
+        tll_foreach(conf->fonts, it) {
+            term->font_sizes[i++] = (struct config_font){
+                .pt_size = it->item.pt_size, .px_size = it->item.px_size};
+        }
+    }
+
     /* Start the slave/client */
     if ((term->slave = slave_spawn(
              term->ptmx, argc, term->cwd, argv,
@@ -1144,6 +1181,7 @@ term_destroy(struct terminal *term)
 
     for (size_t i = 0; i < sizeof(term->fonts) / sizeof(term->fonts[0]); i++)
         fcft_destroy(term->fonts[i]);
+    free(term->font_sizes);
 
     free(term->search.buf);
 
@@ -1377,57 +1415,31 @@ term_reset(struct terminal *term, bool hard)
     term_damage_all(term);
 }
 
-struct font_adjust_data {
-    struct fcft_font *font_in;
-    double amount;
-    struct fcft_font *font_out;
-};
-
-static int
-font_size_adjust_thread(void *_data)
-{
-    struct font_adjust_data *data = _data;
-    data->font_out = fcft_size_adjust(data->font_in, data->amount);
-    return data->font_out != NULL;
-}
-
 static bool
 term_font_size_adjust(struct terminal *term, double amount)
 {
-    struct font_adjust_data data[4] = {
-        {term->fonts[0], amount},
-        {term->fonts[1], amount},
-        {term->fonts[2], amount},
-        {term->fonts[3], amount},
-    };
+    for (size_t i = 0; i < tll_length(term->conf->fonts); i++) {
+        double old_pt_size = term->font_sizes[i].pt_size;
 
-    thrd_t tids[4] = {};
-    for (size_t i = 0; i < 4; i++) {
-        int ret = thrd_create(&tids[i], &font_size_adjust_thread, &data[i]);
-        if (ret != thrd_success) {
-            LOG_ERR("failed to create font adjustmen thread: %s (%d)",
-                    thrd_err_as_string(ret), ret);
-            break;
+        /*
+         * To ensure primary and user-configured fallback fonts are
+         * resizes by the same amount, convert pixel sizes to point
+         * sizes, and to the adjustment on point sizes only.
+         */
+
+        if (term->font_sizes[i].px_size > 0) {
+            double dpi = term->font_dpi;
+            old_pt_size = term->font_sizes[i].px_size * 72. / dpi;
         }
-    }
 
-    for (size_t i = 0; i < 4; i++) {
-        if (tids[i] != 0)
-            thrd_join(tids[i], NULL);
-    }
-
-    if (data[0].font_out == NULL || data[1].font_out == NULL ||
-        data[2].font_out == NULL || data[3].font_out == NULL)
-    {
-        for (size_t i = 0; i < 4; i++)
-            fcft_destroy(data[i].font_out);
-        return false;
+        term->font_sizes[i].pt_size = old_pt_size + amount;
+        term->font_sizes[i].px_size = -1;
     }
 
     const int old_cell_width = term->cell_width;
     const int old_cell_height = term->cell_height;
 
-    if (!term_set_fonts(term, (struct fcft_font *[]){data[0].font_out, data[1].font_out, data[2].font_out, data[3].font_out}))
+    if (!reload_fonts(term))
         return false;
 
     if (term->cell_width < old_cell_width ||
@@ -1457,7 +1469,6 @@ term_font_size_increase(struct terminal *term)
     if (!term_font_size_adjust(term, 0.5))
         return false;
 
-    term->font_adjustments++;
     return true;
 }
 
@@ -1467,20 +1478,13 @@ term_font_size_decrease(struct terminal *term)
     if (!term_font_size_adjust(term, -0.5))
         return false;
 
-    term->font_adjustments--;
     return true;
 }
 
 bool
 term_font_size_reset(struct terminal *term)
 {
-    struct fcft_font *fonts[4];
-    if (!load_fonts_from_conf(term, term->conf, fonts))
-        return false;
-
-    term_set_fonts(term, fonts);
-    term->font_adjustments = 0;
-    return true;
+    return load_fonts_from_conf(term);
 }
 
 bool
@@ -1493,41 +1497,7 @@ term_font_dpi_changed(struct terminal *term)
     LOG_DBG("DPI changed (%u -> %u): reloading fonts", term->font_dpi, dpi);
     term->font_dpi = dpi;
 
-    struct fcft_font *fonts[4];
-    if (!load_fonts_from_conf(term, term->conf, fonts))
-        return false;
-
-    if (term->font_adjustments == 0)
-        return term_set_fonts(term, fonts);
-
-    /* User has adjusted the font size run-time, re-apply */
-
-    double amount = term->font_adjustments * 0.5;
-
-    struct fcft_font *adjusted_fonts[4] = {
-        fcft_size_adjust(fonts[0], amount),
-        fcft_size_adjust(fonts[1], amount),
-        fcft_size_adjust(fonts[2], amount),
-        fcft_size_adjust(fonts[3], amount),
-    };
-
-    if (adjusted_fonts[0] == NULL || adjusted_fonts[1] == NULL ||
-        adjusted_fonts[2] == NULL || adjusted_fonts[3] == NULL)
-    {
-        for (size_t i = 0; i < 4; i++)
-            fcft_destroy(adjusted_fonts[i]);
-
-        /* At least use the newly re-loaded default fonts */
-        term->font_adjustments = 0;
-        return term_set_fonts(term, fonts);
-    } else {
-        for (size_t i = 0; i < 4; i++)
-            fcft_destroy(fonts[i]);
-        return term_set_fonts(term, adjusted_fonts);
-    }
-
-    assert(false);
-    return false;
+    return reload_fonts(term);
 }
 
 void
diff --git a/terminal.h b/terminal.h
index a9b8c5ca..bcbfbf4d 100644
--- a/terminal.h
+++ b/terminal.h
@@ -222,8 +222,8 @@ struct terminal {
     struct composed *composed;
 
     struct fcft_font *fonts[4];
+    struct config_font *font_sizes;
     int font_dpi;
-    int font_adjustments;
     enum fcft_subpixel font_subpixel;
 
     tll(struct ptmx_buffer) ptmx_buffer;