diff --git a/sixel.c b/sixel.c
index dede6e10..63ce493b 100644
--- a/sixel.c
+++ b/sixel.c
@@ -50,17 +50,9 @@ sixel_init(struct terminal *term)
     /* TODO: default palette */
 
     if (term->sixel.use_private_palette) {
-        if (term->sixel.private_palette == NULL) {
-            term->sixel.private_palette = xcalloc(
-                term->sixel.palette_size, sizeof(term->sixel.private_palette[0]));
-        } else {
-            /* Private palette - i.e. each sixel starts with a clean palette */
-            memset(
-                term->sixel.private_palette,
-                0,
-                term->sixel.palette_size * sizeof(term->sixel.private_palette[0]));
-        }
-
+        xassert(term->sixel.private_palette == NULL);
+        term->sixel.private_palette = xcalloc(
+            term->sixel.palette_size, sizeof(term->sixel.private_palette[0]));
         term->sixel.palette = term->sixel.private_palette;
     } else {
         if (term->sixel.shared_palette == NULL) {
@@ -794,6 +786,9 @@ sixel_unhook(struct terminal *term)
     term->sixel.max_col = 0;
     term->sixel.pos = (struct coord){0, 0};
 
+    free(term->sixel.private_palette);
+    term->sixel.private_palette = NULL;
+
     LOG_DBG("you now have %zu sixels in current grid",
             tll_length(term->grid->sixel_images));