diff --git a/csi.c b/csi.c
index becfc142..0224dd80 100644
--- a/csi.c
+++ b/csi.c
@@ -115,7 +115,7 @@ csi_sgr(struct terminal *term)
         case 35:
         case 36:
         case 37:
-            term->vt.attrs.have_fg = 1;
+            term->vt.attrs.fg_src = COLOR_BASE16;
             term->vt.attrs.fg = term->colors.table[param - 30];
             break;
 
@@ -125,7 +125,7 @@ csi_sgr(struct terminal *term)
                 term->vt.params.v[i + 1].value == 5)
             {
                 uint8_t idx = term->vt.params.v[i + 2].value;
-                term->vt.attrs.have_fg = 1;
+                term->vt.attrs.fg_src = COLOR_BASE256;
                 term->vt.attrs.fg = term->colors.table[idx];
                 i += 2;
 
@@ -138,7 +138,7 @@ csi_sgr(struct terminal *term)
                 uint8_t r = term->vt.params.v[i + 2].value;
                 uint8_t g = term->vt.params.v[i + 3].value;
                 uint8_t b = term->vt.params.v[i + 4].value;
-                term->vt.attrs.have_fg = 1;
+                term->vt.attrs.fg_src = COLOR_RGB;
                 term->vt.attrs.fg = r << 16 | g << 8 | b;
                 i += 4;
             }
@@ -150,7 +150,7 @@ csi_sgr(struct terminal *term)
                 const struct vt_param *param = &term->vt.params.v[i];
 
                 uint8_t idx = param->sub.value[1];
-                term->vt.attrs.have_fg = 1;
+                term->vt.attrs.fg_src = COLOR_BASE256;
                 term->vt.attrs.fg = term->colors.table[idx];
             }
 
@@ -180,7 +180,7 @@ csi_sgr(struct terminal *term)
                 uint8_t g = param->sub.value[g_idx];
                 uint8_t b = param->sub.value[b_idx];
 
-                term->vt.attrs.have_fg = 1;
+                term->vt.attrs.fg_src = COLOR_RGB;
                 term->vt.attrs.fg = r << 16 | g << 8 | b;
             }
 
@@ -196,7 +196,7 @@ csi_sgr(struct terminal *term)
         }
 
         case 39:
-            term->vt.attrs.have_fg = 0;
+            term->vt.attrs.fg_src = COLOR_DEFAULT;
             break;
 
         /* Regular background colors */
@@ -208,7 +208,7 @@ csi_sgr(struct terminal *term)
         case 45:
         case 46:
         case 47:
-            term->vt.attrs.have_bg = 1;
+            term->vt.attrs.bg_src = COLOR_BASE16;
             term->vt.attrs.bg = term->colors.table[param - 40];
             break;
 
@@ -218,7 +218,7 @@ csi_sgr(struct terminal *term)
                 term->vt.params.v[i + 1].value == 5)
             {
                 uint8_t idx = term->vt.params.v[i + 2].value;
-                term->vt.attrs.have_bg = 1;
+                term->vt.attrs.bg_src = COLOR_BASE256;
                 term->vt.attrs.bg = term->colors.table[idx];
                 i += 2;
 
@@ -231,7 +231,7 @@ csi_sgr(struct terminal *term)
                 uint8_t r = term->vt.params.v[i + 2].value;
                 uint8_t g = term->vt.params.v[i + 3].value;
                 uint8_t b = term->vt.params.v[i + 4].value;
-                term->vt.attrs.have_bg = 1;
+                term->vt.attrs.bg_src = COLOR_RGB;
                 term->vt.attrs.bg = r << 16 | g << 8 | b;
                 i += 4;
             }
@@ -243,7 +243,7 @@ csi_sgr(struct terminal *term)
                 const struct vt_param *param = &term->vt.params.v[i];
 
                 uint8_t idx = param->sub.value[1];
-                term->vt.attrs.have_bg = 1;
+                term->vt.attrs.bg_src = COLOR_BASE256;
                 term->vt.attrs.bg = term->colors.table[idx];
             }
 
@@ -273,7 +273,7 @@ csi_sgr(struct terminal *term)
                 uint8_t g = param->sub.value[g_idx];
                 uint8_t b = param->sub.value[b_idx];
 
-                term->vt.attrs.have_bg = 1;
+                term->vt.attrs.bg_src = COLOR_RGB;
                 term->vt.attrs.bg = r << 16 | g << 8 | b;
             }
 
@@ -287,7 +287,7 @@ csi_sgr(struct terminal *term)
             break;
         }
         case 49:
-            term->vt.attrs.have_bg = 0;
+            term->vt.attrs.bg_src = COLOR_DEFAULT;
             break;
 
         /* Bright foreground colors */
@@ -299,7 +299,7 @@ csi_sgr(struct terminal *term)
         case 95:
         case 96:
         case 97:
-            term->vt.attrs.have_fg = 1;
+            term->vt.attrs.fg_src = COLOR_BASE16;
             term->vt.attrs.fg = term->colors.table[param - 90 + 8];
             break;
 
@@ -312,7 +312,7 @@ csi_sgr(struct terminal *term)
         case 105:
         case 106:
         case 107:
-            term->vt.attrs.have_bg = 1;
+            term->vt.attrs.bg_src = COLOR_BASE16;
             term->vt.attrs.bg = term->colors.table[param - 100 + 8];
             break;
 
diff --git a/osc.c b/osc.c
index ecb908ca..910c1e02 100644
--- a/osc.c
+++ b/osc.c
@@ -503,7 +503,7 @@ osc_notify(struct terminal *term, char *string)
      * (https://pub.phyks.me/scripts/urxvt/notify) is very simple:
      *
      * #!/usr/bin/perl
-     * 
+     *
      * sub on_osc_seq_perl {
      *   my ($term, $osc, $resp) = @_;
      *   if ($osc =~ /^notify;(\S+);(.*)$/) {
@@ -560,7 +560,7 @@ update_color_in_grids(struct terminal *term, uint32_t old_color,
 
             for (size_t c = 0; c < term->grid->num_cols; c++) {
                 struct cell *cell = &row->cells[c];
-                if (cell->attrs.have_fg &&
+                if (cell->attrs.fg_src != COLOR_DEFAULT &&
                     cell->attrs.fg == old_color)
                 {
                     cell->attrs.fg = new_color;
@@ -568,7 +568,7 @@ update_color_in_grids(struct terminal *term, uint32_t old_color,
                     row->dirty = true;
                 }
 
-                if ( cell->attrs.have_bg &&
+                if (cell->attrs.bg_src != COLOR_DEFAULT &&
                     cell->attrs.bg == old_color)
                 {
                     cell->attrs.bg = new_color;
diff --git a/render.c b/render.c
index 46cc0f75..2710c0a2 100644
--- a/render.c
+++ b/render.c
@@ -494,14 +494,14 @@ render_cell(struct terminal *term, pixman_image_t *pix,
         _bg = term->colors.selection_bg;
     } else {
         /* Use cell specific color, if set, otherwise the default colors (possible reversed) */
-        _fg = cell->attrs.have_fg ? cell->attrs.fg : term->reverse ? term->colors.bg : term->colors.fg;
-        _bg = cell->attrs.have_bg ? cell->attrs.bg : term->reverse ? term->colors.fg : term->colors.bg;
+        _fg = cell->attrs.fg_src != COLOR_DEFAULT ? cell->attrs.fg : term->reverse ? term->colors.bg : term->colors.fg;
+        _bg = cell->attrs.bg_src != COLOR_DEFAULT ? cell->attrs.bg : term->reverse ? term->colors.fg : term->colors.bg;
 
         if (cell->attrs.reverse ^ is_selected) {
             uint32_t swap = _fg;
             _fg = _bg;
             _bg = swap;
-        } else if (!cell->attrs.have_bg)
+        } else if (cell->attrs.bg_src == COLOR_DEFAULT)
             alpha = term->colors.alpha;
     }
 
diff --git a/sixel.c b/sixel.c
index acb3f31b..e50fa511 100644
--- a/sixel.c
+++ b/sixel.c
@@ -70,7 +70,7 @@ sixel_init(struct terminal *term, int p1, int p2, int p3)
 
     term->sixel.default_bg = term->sixel.transparent_bg
         ? 0x00000000u
-        : 0xffu << 24 | (term->vt.attrs.have_bg
+        : 0xffu << 24 | (term->vt.attrs.bg_src != COLOR_DEFAULT
                          ? term->vt.attrs.bg
                          : term->colors.bg);
 
diff --git a/terminal.c b/terminal.c
index df8128fc..bf13dd46 100644
--- a/terminal.c
+++ b/terminal.c
@@ -1770,11 +1770,13 @@ erase_cell_range(struct terminal *term, struct row *row, int start, int end)
 
     row->dirty = true;
 
-    if (unlikely(term->vt.attrs.have_bg)) {
+    const enum color_source bg_src = term->vt.attrs.bg_src;
+
+    if (unlikely(bg_src != COLOR_DEFAULT)) {
         for (int col = start; col <= end; col++) {
             struct cell *c = &row->cells[col];
             c->wc = 0;
-            c->attrs = (struct attributes){.have_bg = 1, .bg = term->vt.attrs.bg};
+            c->attrs = (struct attributes){.bg_src = bg_src, .bg = term->vt.attrs.bg};
         }
     } else
         memset(&row->cells[start], 0, (end - start + 1) * sizeof(row->cells[0]));
diff --git a/terminal.h b/terminal.h
index 3516551f..b0526803 100644
--- a/terminal.h
+++ b/terminal.h
@@ -24,6 +24,13 @@
 #include "shm.h"
 #include "wayland.h"
 
+enum color_source {
+    COLOR_DEFAULT,
+    COLOR_BASE16,
+    COLOR_BASE256,
+    COLOR_RGB,
+};
+
 /*
  *  Note: we want the cells to be as small as possible. Larger cells
  *  means fewer scrollback lines (or performance drops due to cache
@@ -43,12 +50,11 @@ struct attributes {
     uint32_t fg:24;
 
     bool clean:1;
+    enum color_source fg_src:2;
+    enum color_source bg_src:2;
     bool confined:1;
-    bool have_fg:1;
-    bool have_bg:1;
     bool selected:1;
     bool url:1;
-    uint32_t reserved:2;
     uint32_t bg:24;
 };
 static_assert(sizeof(struct attributes) == 8, "VT attribute struct too large");