diff --git a/examples/output-layers.c b/examples/output-layers.c
index cd05d9f0e..1a60be3f0 100644
--- a/examples/output-layers.c
+++ b/examples/output-layers.c
@@ -110,12 +110,8 @@ static void output_handle_frame(struct wl_listener *listener, void *data) {
 		output_surface->layer_accepted = layers[i].accepted;
 		i++;
 
-		if (wlr_surface->buffer == NULL || output_surface->layer_accepted) {
-			continue;
-		}
-
 		struct wlr_texture *texture = wlr_surface_get_texture(wlr_surface);
-		if (texture == NULL) {
+		if (texture == NULL || output_surface->layer_accepted) {
 			continue;
 		}
 
@@ -136,7 +132,7 @@ static void output_handle_frame(struct wl_listener *listener, void *data) {
 	wl_list_for_each(output_surface, &output->surfaces, link) {
 		wlr_surface_send_frame_done(output_surface->wlr_surface, &now);
 
-		if (output_surface->wlr_surface->buffer == NULL) {
+		if (wlr_surface_get_texture(output_surface->wlr_surface) == NULL) {
 			continue;
 		}
 
@@ -203,10 +199,11 @@ static void output_surface_handle_commit(struct wl_listener *listener,
 		void *data) {
 	struct output_surface *output_surface =
 		wl_container_of(listener, output_surface, commit);
+	struct wlr_surface *surface = output_surface->wlr_surface;
 
 	struct wlr_buffer *buffer = NULL;
-	if (output_surface->wlr_surface->buffer != NULL) {
-		buffer = wlr_buffer_lock(&output_surface->wlr_surface->buffer->base);
+	if (surface->current.buffer != NULL) {
+		buffer = wlr_buffer_lock(surface->current.buffer);
 	}
 
 	wlr_buffer_unlock(output_surface->buffer);
diff --git a/include/types/wlr_buffer.h b/include/types/wlr_buffer.h
index 947ccde7b..f34b03a88 100644
--- a/include/types/wlr_buffer.h
+++ b/include/types/wlr_buffer.h
@@ -59,19 +59,4 @@ bool dmabuf_buffer_drop(struct wlr_dmabuf_buffer *buffer);
  */
 bool buffer_is_opaque(struct wlr_buffer *buffer);
 
-/**
- * Creates a struct wlr_client_buffer from a given struct wlr_buffer by creating
- * a texture from it, and copying its struct wl_resource.
- */
-struct wlr_client_buffer *wlr_client_buffer_create(struct wlr_buffer *buffer,
-	struct wlr_renderer *renderer);
-/**
- * Try to update the buffer's content.
- *
- * Fails if there's more than one reference to the buffer or if the texture
- * isn't mutable.
- */
-bool wlr_client_buffer_apply_damage(struct wlr_client_buffer *client_buffer,
-	struct wlr_buffer *next, const pixman_region32_t *damage);
-
 #endif
diff --git a/include/types/wlr_scene.h b/include/types/wlr_scene.h
index 80dcfd1bd..db2436fcb 100644
--- a/include/types/wlr_scene.h
+++ b/include/types/wlr_scene.h
@@ -7,4 +7,8 @@ struct wlr_scene *scene_node_get_root(struct wlr_scene_node *node);
 
 void scene_surface_set_clip(struct wlr_scene_surface *surface, struct wlr_box *clip);
 
+void scene_buffer_set_buffer_and_texture(struct wlr_scene_buffer *scene_buffer,
+	struct wlr_buffer *buffer, struct wlr_texture *texture,
+	const pixman_region32_t *damage);
+
 #endif
diff --git a/include/wlr/render/wlr_texture.h b/include/wlr/render/wlr_texture.h
index 1e352c6e6..2f3e6948b 100644
--- a/include/wlr/render/wlr_texture.h
+++ b/include/wlr/render/wlr_texture.h
@@ -24,6 +24,10 @@ struct wlr_texture {
 	uint32_t width, height;
 
 	struct wlr_renderer *renderer;
+
+	struct {
+		struct wl_signal destroy;
+	} events;
 };
 
 struct wlr_texture_read_pixels_options {
diff --git a/include/wlr/types/wlr_buffer.h b/include/wlr/types/wlr_buffer.h
index de3aeec3d..063015395 100644
--- a/include/wlr/types/wlr_buffer.h
+++ b/include/wlr/types/wlr_buffer.h
@@ -132,33 +132,4 @@ bool wlr_buffer_begin_data_ptr_access(struct wlr_buffer *buffer, uint32_t flags,
 	void **data, uint32_t *format, size_t *stride);
 void wlr_buffer_end_data_ptr_access(struct wlr_buffer *buffer);
 
-/**
- * A client buffer.
- */
-struct wlr_client_buffer {
-	struct wlr_buffer base;
-
-	/**
-	 * The buffer's texture, if any. A buffer will not have a texture if the
-	 * client destroys the buffer before it has been released.
-	 */
-	struct wlr_texture *texture;
-	/**
-	 * The buffer this client buffer was created from. NULL if destroyed.
-	 */
-	struct wlr_buffer *source;
-
-	// private state
-
-	struct wl_listener source_destroy;
-
-	size_t n_ignore_locks;
-};
-
-/**
- * Get a client buffer from a generic buffer. If the buffer isn't a client
- * buffer, returns NULL.
- */
-struct wlr_client_buffer *wlr_client_buffer_get(struct wlr_buffer *buffer);
-
 #endif
diff --git a/include/wlr/types/wlr_compositor.h b/include/wlr/types/wlr_compositor.h
index dc95b3037..b3ae0b488 100644
--- a/include/wlr/types/wlr_compositor.h
+++ b/include/wlr/types/wlr_compositor.h
@@ -119,16 +119,29 @@ struct wlr_surface_output {
 	struct wl_listener destroy;
 };
 
+struct wlr_surface_texture {
+	struct wlr_texture *texture;
+	// The buffer this surface texture was created from. NULL if released.
+	struct wlr_buffer *buffer;
+	// True if the texture won't be mutated by client surface commits.
+	bool locked;
+
+	// private state
+
+	struct wl_listener buffer_release;
+	bool dropped;
+};
+
 struct wlr_surface {
 	struct wl_resource *resource;
 	struct wlr_renderer *renderer; // may be NULL
 	/**
-	 * The surface's buffer, if any. A surface has an attached buffer when it
+	 * The surface's texture, if any. A surface has an attached buffer when it
 	 * commits with a non-null buffer in its pending state. A surface will not
-	 * have a buffer if it has never committed one, has committed a null buffer,
-	 * or something went wrong with uploading the buffer.
+	 * have a texture if it has never committed one, has committed a null
+	 * buffer, or something went wrong with uploading the buffer.
 	 */
-	struct wlr_client_buffer *buffer;
+	struct wlr_surface_texture *texture;
 	/**
 	 * The last commit's buffer damage, in buffer-local coordinates. This
 	 * contains both the damage accumulated by the client via
@@ -257,6 +270,9 @@ struct wlr_compositor {
 typedef void (*wlr_surface_iterator_func_t)(struct wlr_surface *surface,
 	int sx, int sy, void *data);
 
+void wlr_surface_texture_lock(struct wlr_surface_texture *surf_tex);
+void wlr_surface_texture_unlock(struct wlr_surface_texture *surf_tex);
+
 /**
  * Set the lifetime role for this surface.
  *
diff --git a/include/wlr/types/wlr_scene.h b/include/wlr/types/wlr_scene.h
index 0d6d97a31..5defb40cf 100644
--- a/include/wlr/types/wlr_scene.h
+++ b/include/wlr/types/wlr_scene.h
@@ -184,6 +184,7 @@ struct wlr_scene_buffer {
 
 	uint64_t active_outputs;
 	struct wlr_texture *texture;
+	bool own_texture;
 	struct wlr_linux_dmabuf_feedback_v1_init_options prev_feedback_options;
 
 	bool own_buffer;
@@ -191,6 +192,7 @@ struct wlr_scene_buffer {
 	bool buffer_is_opaque;
 
 	struct wl_listener buffer_release;
+	struct wl_listener texture_destroy;
 };
 
 /** A viewport for an output in the scene-graph */
@@ -399,6 +401,12 @@ void wlr_scene_buffer_set_buffer(struct wlr_scene_buffer *scene_buffer,
 void wlr_scene_buffer_set_buffer_with_damage(struct wlr_scene_buffer *scene_buffer,
 	struct wlr_buffer *buffer, const pixman_region32_t *region);
 
+/**
+ * Sets the buffer's backing texture.
+ */
+void wlr_scene_buffer_set_texture(struct wlr_scene_buffer *scene_buffer,
+	struct wlr_texture *texture);
+
 /**
  * Sets the buffer's opaque region. This is an optimization hint used to
  * determine if buffers which reside under this one need to be rendered or not.
diff --git a/render/wlr_texture.c b/render/wlr_texture.c
index 3526ee140..2dfe56de3 100644
--- a/render/wlr_texture.c
+++ b/render/wlr_texture.c
@@ -18,9 +18,12 @@ void wlr_texture_init(struct wlr_texture *texture, struct wlr_renderer *renderer
 		.width = width,
 		.height = height,
 	};
+
+	wl_signal_init(&texture->events.destroy);
 }
 
 void wlr_texture_destroy(struct wlr_texture *texture) {
+	wl_signal_emit_mutable(&texture->events.destroy, NULL);
 	if (texture && texture->impl && texture->impl->destroy) {
 		texture->impl->destroy(texture);
 	} else {
diff --git a/types/buffer/client.c b/types/buffer/client.c
deleted file mode 100644
index 4cfa57a89..000000000
--- a/types/buffer/client.c
+++ /dev/null
@@ -1,93 +0,0 @@
-#include <assert.h>
-#include <stdlib.h>
-#include <wlr/interfaces/wlr_buffer.h>
-#include <wlr/render/wlr_renderer.h>
-#include <wlr/util/log.h>
-#include "types/wlr_buffer.h"
-
-static const struct wlr_buffer_impl client_buffer_impl;
-
-struct wlr_client_buffer *wlr_client_buffer_get(struct wlr_buffer *wlr_buffer) {
-	if (wlr_buffer->impl != &client_buffer_impl) {
-		return NULL;
-	}
-	struct wlr_client_buffer *buffer = wl_container_of(wlr_buffer, buffer, base);
-	return buffer;
-}
-
-static struct wlr_client_buffer *client_buffer_from_buffer(
-		struct wlr_buffer *buffer) {
-	struct wlr_client_buffer *client_buffer = wlr_client_buffer_get(buffer);
-	assert(client_buffer != NULL);
-	return client_buffer;
-}
-
-static void client_buffer_destroy(struct wlr_buffer *buffer) {
-	struct wlr_client_buffer *client_buffer = client_buffer_from_buffer(buffer);
-	wl_list_remove(&client_buffer->source_destroy.link);
-	wlr_texture_destroy(client_buffer->texture);
-	free(client_buffer);
-}
-
-static bool client_buffer_get_dmabuf(struct wlr_buffer *buffer,
-		struct wlr_dmabuf_attributes *attribs) {
-	struct wlr_client_buffer *client_buffer = client_buffer_from_buffer(buffer);
-
-	if (client_buffer->source == NULL) {
-		return false;
-	}
-
-	return wlr_buffer_get_dmabuf(client_buffer->source, attribs);
-}
-
-static const struct wlr_buffer_impl client_buffer_impl = {
-	.destroy = client_buffer_destroy,
-	.get_dmabuf = client_buffer_get_dmabuf,
-};
-
-static void client_buffer_handle_source_destroy(struct wl_listener *listener,
-		void *data) {
-	struct wlr_client_buffer *client_buffer =
-		wl_container_of(listener, client_buffer, source_destroy);
-	wl_list_remove(&client_buffer->source_destroy.link);
-	wl_list_init(&client_buffer->source_destroy.link);
-	client_buffer->source = NULL;
-}
-
-struct wlr_client_buffer *wlr_client_buffer_create(struct wlr_buffer *buffer,
-		struct wlr_renderer *renderer) {
-	struct wlr_texture *texture = wlr_texture_from_buffer(renderer, buffer);
-	if (texture == NULL) {
-		wlr_log(WLR_ERROR, "Failed to create texture");
-		return NULL;
-	}
-
-	struct wlr_client_buffer *client_buffer = calloc(1, sizeof(*client_buffer));
-	if (client_buffer == NULL) {
-		wlr_texture_destroy(texture);
-		return NULL;
-	}
-	wlr_buffer_init(&client_buffer->base, &client_buffer_impl,
-		texture->width, texture->height);
-	client_buffer->source = buffer;
-	client_buffer->texture = texture;
-
-	wl_signal_add(&buffer->events.destroy, &client_buffer->source_destroy);
-	client_buffer->source_destroy.notify = client_buffer_handle_source_destroy;
-
-	// Ensure the buffer will be released before being destroyed
-	wlr_buffer_lock(&client_buffer->base);
-	wlr_buffer_drop(&client_buffer->base);
-
-	return client_buffer;
-}
-
-bool wlr_client_buffer_apply_damage(struct wlr_client_buffer *client_buffer,
-		struct wlr_buffer *next, const pixman_region32_t *damage) {
-	if (client_buffer->base.n_locks - client_buffer->n_ignore_locks > 1) {
-		// Someone else still has a reference to the buffer
-		return false;
-	}
-
-	return wlr_texture_update_from_buffer(client_buffer->texture, next, damage);
-}
diff --git a/types/meson.build b/types/meson.build
index 8962a390e..97236d631 100644
--- a/types/meson.build
+++ b/types/meson.build
@@ -29,7 +29,6 @@ wlr_files += files(
 	'xdg_shell/wlr_xdg_surface.c',
 	'xdg_shell/wlr_xdg_toplevel.c',
 	'buffer/buffer.c',
-	'buffer/client.c',
 	'buffer/dmabuf.c',
 	'buffer/readonly_data.c',
 	'buffer/resource.c',
diff --git a/types/scene/surface.c b/types/scene/surface.c
index 11396e2fd..9567ee4bb 100644
--- a/types/scene/surface.c
+++ b/types/scene/surface.c
@@ -72,28 +72,6 @@ static void scene_surface_handle_surface_destroy(
 	wlr_scene_node_destroy(&surface->buffer->node);
 }
 
-// This is used for wlr_scene where it unconditionally locks buffers preventing
-// reuse of the existing texture for shm clients. With the usage pattern of
-// wlr_scene surface handling, we can mark its locked buffer as safe
-// for mutation.
-static void client_buffer_mark_next_can_damage(struct wlr_client_buffer *buffer) {
-	buffer->n_ignore_locks++;
-}
-
-static void scene_buffer_unmark_client_buffer(struct wlr_scene_buffer *scene_buffer) {
-	if (!scene_buffer->buffer) {
-		return;
-	}
-
-	struct wlr_client_buffer *buffer = wlr_client_buffer_get(scene_buffer->buffer);
-	if (!buffer) {
-		return;
-	}
-
-	assert(buffer->n_ignore_locks > 0);
-	buffer->n_ignore_locks--;
-}
-
 static int min(int a, int b) {
 	return a < b ? a : b;
 }
@@ -148,13 +126,10 @@ static void surface_reconfigure(struct wlr_scene_surface *scene_surface) {
 	wlr_scene_buffer_set_dest_size(scene_buffer, width, height);
 	wlr_scene_buffer_set_transform(scene_buffer, state->transform);
 
-	scene_buffer_unmark_client_buffer(scene_buffer);
-
-	if (surface->buffer) {
-		client_buffer_mark_next_can_damage(surface->buffer);
-
-		wlr_scene_buffer_set_buffer_with_damage(scene_buffer,
-			&surface->buffer->base, &surface->buffer_damage);
+	if (surface->texture) {
+		scene_buffer_set_buffer_and_texture(scene_buffer,
+			surface->texture->buffer, surface->texture->texture,
+			&surface->buffer_damage);
 	} else {
 		wlr_scene_buffer_set_buffer(scene_buffer, NULL);
 	}
@@ -198,7 +173,7 @@ static bool scene_buffer_point_accepts_input(struct wlr_scene_buffer *scene_buff
 static void surface_addon_destroy(struct wlr_addon *addon) {
 	struct wlr_scene_surface *surface = wl_container_of(addon, surface, addon);
 
-	scene_buffer_unmark_client_buffer(surface->buffer);
+	wlr_scene_buffer_set_buffer(surface->buffer, NULL);
 
 	wlr_addon_finish(&surface->addon);
 
diff --git a/types/scene/wlr_scene.c b/types/scene/wlr_scene.c
index 9b7e1e980..81a31238f 100644
--- a/types/scene/wlr_scene.c
+++ b/types/scene/wlr_scene.c
@@ -112,7 +112,9 @@ void wlr_scene_node_destroy(struct wlr_scene_node *node) {
 			}
 		}
 
-		wlr_texture_destroy(scene_buffer->texture);
+		if (scene_buffer->own_texture) {
+			wlr_texture_destroy(scene_buffer->texture);
+		}
 		scene_buffer_set_buffer(scene_buffer, NULL);
 		pixman_region32_fini(&scene_buffer->opaque_region);
 	} else if (node->type == WLR_SCENE_NODE_TREE) {
@@ -635,6 +637,37 @@ static void scene_buffer_set_buffer(struct wlr_scene_buffer *scene_buffer,
 	wl_signal_add(&buffer->events.release, &scene_buffer->buffer_release);
 }
 
+static void scene_buffer_set_texture(struct wlr_scene_buffer *scene_buffer,
+	struct wlr_texture *texture, bool own_texture);
+
+static void scene_buffer_handle_texture_destroy(struct wl_listener *listener,
+		void *data) {
+	struct wlr_scene_buffer *scene_buffer =
+		wl_container_of(listener, scene_buffer, texture_destroy);
+	scene_buffer_set_texture(scene_buffer, NULL, false);
+}
+
+static void scene_buffer_set_texture(struct wlr_scene_buffer *scene_buffer,
+		struct wlr_texture *texture, bool own_texture) {
+	wl_list_remove(&scene_buffer->texture_destroy.link);
+	wl_list_init(&scene_buffer->texture_destroy.link);
+	if (scene_buffer->own_texture) {
+		wlr_texture_destroy(scene_buffer->texture);
+	}
+	scene_buffer->texture = NULL;
+	scene_buffer->own_texture = false;
+
+	if (!texture) {
+		return;
+	}
+
+	scene_buffer->own_texture = own_texture;
+	scene_buffer->texture = texture;
+
+	scene_buffer->texture_destroy.notify = scene_buffer_handle_texture_destroy;
+	wl_signal_add(&texture->events.destroy, &scene_buffer->texture_destroy);
+}
+
 struct wlr_scene_buffer *wlr_scene_buffer_create(struct wlr_scene_tree *parent,
 		struct wlr_buffer *buffer) {
 	struct wlr_scene_buffer *scene_buffer = calloc(1, sizeof(*scene_buffer));
@@ -659,14 +692,28 @@ struct wlr_scene_buffer *wlr_scene_buffer_create(struct wlr_scene_tree *parent,
 	return scene_buffer;
 }
 
-void wlr_scene_buffer_set_buffer_with_damage(struct wlr_scene_buffer *scene_buffer,
-		struct wlr_buffer *buffer, const pixman_region32_t *damage) {
+void scene_buffer_set_buffer_and_texture(struct wlr_scene_buffer *scene_buffer,
+		struct wlr_buffer *buffer, struct wlr_texture *texture,
+		const pixman_region32_t *damage) {
 	// specifying a region for a NULL buffer doesn't make sense. We need to know
 	// about the buffer to scale the buffer local coordinates down to scene
 	// coordinates.
-	assert(buffer || !damage);
+	assert(buffer || texture || !damage);
 
-	bool mapped = buffer != NULL;
+	int buffer_width = 0, buffer_height = 0;
+	if (buffer != NULL && texture != NULL) {
+		assert(buffer->width == (int)texture->width &&
+			buffer->height == (int)texture->height);
+	}
+	if (buffer != NULL) {
+		buffer_width = buffer->width;
+		buffer_height = buffer->height;
+	} else if (texture != NULL) {
+		buffer_width = texture->width;
+		buffer_height = texture->height;
+	}
+
+	bool mapped = buffer != NULL || texture != NULL;
 	bool prev_mapped = scene_buffer->buffer != NULL || scene_buffer->texture != NULL;
 
 	if (!mapped && !prev_mapped) {
@@ -678,15 +725,15 @@ void wlr_scene_buffer_set_buffer_with_damage(struct wlr_scene_buffer *scene_buff
 	// buffer region will be different from what the new buffer would
 	// produce we need to update the node.
 	bool update = mapped != prev_mapped;
-	if (buffer != NULL && scene_buffer->dst_width == 0 && scene_buffer->dst_height == 0) {
-		update = update || scene_buffer->buffer_width != buffer->width ||
-			scene_buffer->buffer_height != buffer->height;
+	if (scene_buffer->dst_width == 0 && scene_buffer->dst_height == 0) {
+		update = update || scene_buffer->buffer_width != buffer_width ||
+			scene_buffer->buffer_height != buffer_height;
 	}
 
-	wlr_texture_destroy(scene_buffer->texture);
-	scene_buffer->texture = NULL;
-
 	scene_buffer_set_buffer(scene_buffer, buffer);
+	scene_buffer_set_texture(scene_buffer, texture, false);
+	scene_buffer->buffer_width = buffer_width;
+	scene_buffer->buffer_height = buffer_height;
 
 	if (update) {
 		scene_node_update(&scene_buffer->node, NULL);
@@ -701,7 +748,7 @@ void wlr_scene_buffer_set_buffer_with_damage(struct wlr_scene_buffer *scene_buff
 	}
 
 	pixman_region32_t fallback_damage;
-	pixman_region32_init_rect(&fallback_damage, 0, 0, buffer->width, buffer->height);
+	pixman_region32_init_rect(&fallback_damage, 0, 0, buffer_width, buffer_height);
 	if (!damage) {
 		damage = &fallback_damage;
 	}
@@ -710,26 +757,26 @@ void wlr_scene_buffer_set_buffer_with_damage(struct wlr_scene_buffer *scene_buff
 	if (wlr_fbox_empty(&box)) {
 		box.x = 0;
 		box.y = 0;
-		box.width = buffer->width;
-		box.height = buffer->height;
+		box.width = buffer_width;
+		box.height = buffer_height;
 	}
 
 	wlr_fbox_transform(&box, &box, scene_buffer->transform,
-		buffer->width, buffer->height);
+		buffer_width, buffer_height);
 
 	float scale_x, scale_y;
 	if (scene_buffer->dst_width || scene_buffer->dst_height) {
 		scale_x = scene_buffer->dst_width / box.width;
 		scale_y = scene_buffer->dst_height / box.height;
 	} else {
-		scale_x = buffer->width / box.width;
-		scale_y = buffer->height / box.height;
+		scale_x = buffer_width / box.width;
+		scale_y = buffer_height / box.height;
 	}
 
 	pixman_region32_t trans_damage;
 	pixman_region32_init(&trans_damage);
 	wlr_region_transform(&trans_damage, damage,
-		scene_buffer->transform, buffer->width, buffer->height);
+		scene_buffer->transform, buffer_width, buffer_height);
 	pixman_region32_intersect_rect(&trans_damage, &trans_damage,
 		box.x, box.y, box.width, box.height);
 	pixman_region32_translate(&trans_damage, -box.x, -box.y);
@@ -785,11 +832,21 @@ void wlr_scene_buffer_set_buffer_with_damage(struct wlr_scene_buffer *scene_buff
 	pixman_region32_fini(&fallback_damage);
 }
 
+void wlr_scene_buffer_set_buffer_with_damage(struct wlr_scene_buffer *scene_buffer,
+		struct wlr_buffer *buffer, const pixman_region32_t *damage) {
+	scene_buffer_set_buffer_and_texture(scene_buffer, buffer, NULL, damage);
+}
+
 void wlr_scene_buffer_set_buffer(struct wlr_scene_buffer *scene_buffer,
 		struct wlr_buffer *buffer)  {
 	wlr_scene_buffer_set_buffer_with_damage(scene_buffer, buffer, NULL);
 }
 
+void wlr_scene_buffer_set_texture(struct wlr_scene_buffer *scene_buffer,
+		struct wlr_texture *texture) {
+	scene_buffer_set_buffer_and_texture(scene_buffer, NULL, texture, NULL);
+}
+
 void wlr_scene_buffer_set_opaque_region(struct wlr_scene_buffer *scene_buffer,
 		const pixman_region32_t *region) {
 	if (pixman_region32_equal(&scene_buffer->opaque_region, region)) {
@@ -879,15 +936,10 @@ static struct wlr_texture *scene_buffer_get_texture(
 		return scene_buffer->texture;
 	}
 
-	struct wlr_client_buffer *client_buffer =
-		wlr_client_buffer_get(scene_buffer->buffer);
-	if (client_buffer != NULL) {
-		return client_buffer->texture;
-	}
-
-	scene_buffer->texture =
+	struct wlr_texture *texture =
 		wlr_texture_from_buffer(renderer, scene_buffer->buffer);
-	if (scene_buffer->texture != NULL && scene_buffer->own_buffer) {
+	scene_buffer_set_texture(scene_buffer, texture, true);
+	if (texture != NULL && scene_buffer->own_buffer) {
 		scene_buffer->own_buffer = false;
 		wlr_buffer_unlock(scene_buffer->buffer);
 	}
diff --git a/types/wlr_compositor.c b/types/wlr_compositor.c
index 791eb77cb..298a2eb37 100644
--- a/types/wlr_compositor.c
+++ b/types/wlr_compositor.c
@@ -53,6 +53,76 @@ static void pending_buffer_resource_handle_destroy(struct wl_listener *listener,
 	set_pending_buffer_resource(surface, NULL);
 }
 
+static void surface_texture_handle_buffer_release(struct wl_listener *listener,
+		void *data) {
+	struct wlr_surface_texture *surf_tex = wl_container_of(listener, surf_tex, buffer_release);
+	surf_tex->buffer = NULL;
+	wl_list_remove(&surf_tex->buffer_release.link);
+	wl_list_init(&surf_tex->buffer_release.link);
+}
+
+static struct wlr_surface_texture *surface_texture_create(struct wlr_buffer *buffer,
+		struct wlr_renderer *renderer) {
+	struct wlr_texture *texture = wlr_texture_from_buffer(renderer, buffer);
+	if (texture == NULL) {
+		return NULL;
+	}
+
+	struct wlr_surface_texture *surf_tex = calloc(1, sizeof(*surf_tex));
+	if (surf_tex == NULL) {
+		wlr_texture_destroy(texture);
+		return NULL;
+	}
+
+	surf_tex->buffer = buffer;
+	surf_tex->texture = texture;
+
+	surf_tex->buffer_release.notify = surface_texture_handle_buffer_release;
+	wl_signal_add(&buffer->events.release, &surf_tex->buffer_release);
+
+	return surf_tex;
+}
+
+static void surface_texture_consider_destroy(struct wlr_surface_texture *surf_tex) {
+	if (!surf_tex->dropped || surf_tex->locked) {
+		return;
+	}
+	wl_list_remove(&surf_tex->buffer_release.link);
+	wlr_texture_destroy(surf_tex->texture);
+	free(surf_tex);
+}
+
+static bool surface_texture_apply_damage(struct wlr_surface_texture *surf_tex,
+		struct wlr_buffer *next, const pixman_region32_t *damage) {
+	if (surf_tex->locked) {
+		return false;
+	}
+	return wlr_texture_update_from_buffer(surf_tex->texture, next, damage);
+}
+
+static void surface_texture_drop(struct wlr_surface_texture *surf_tex) {
+	if (surf_tex == NULL) {
+		return;
+	}
+	assert(!surf_tex->dropped);
+	surf_tex->dropped = true;
+	surface_texture_consider_destroy(surf_tex);
+}
+
+void wlr_surface_texture_lock(struct wlr_surface_texture *surf_tex) {
+	assert(!surf_tex->locked);
+	surf_tex->locked = true;
+}
+
+void wlr_surface_texture_unlock(struct wlr_surface_texture *surf_tex) {
+	if (surf_tex == NULL) {
+		return;
+	}
+	assert(surf_tex->locked);
+	surf_tex->locked = false;
+	surface_texture_consider_destroy(surf_tex);
+}
+
 static void surface_handle_destroy(struct wl_client *client,
 		struct wl_resource *resource) {
 	struct wlr_surface *surface = wlr_surface_from_resource(resource);
@@ -407,18 +477,16 @@ static void surface_state_move(struct wlr_surface_state *state,
 static void surface_apply_damage(struct wlr_surface *surface) {
 	if (surface->current.buffer == NULL) {
 		// NULL commit
-		if (surface->buffer != NULL) {
-			wlr_buffer_unlock(&surface->buffer->base);
-		}
-		surface->buffer = NULL;
+		surface_texture_drop(surface->texture);
+		surface->texture = NULL;
 		surface->opaque = false;
 		return;
 	}
 
 	surface->opaque = buffer_is_opaque(surface->current.buffer);
 
-	if (surface->buffer != NULL) {
-		if (wlr_client_buffer_apply_damage(surface->buffer,
+	if (surface->texture != NULL) {
+		if (surface_texture_apply_damage(surface->texture,
 				surface->current.buffer, &surface->buffer_damage)) {
 			wlr_buffer_unlock(surface->current.buffer);
 			surface->current.buffer = NULL;
@@ -430,18 +498,15 @@ static void surface_apply_damage(struct wlr_surface *surface) {
 		return;
 	}
 
-	struct wlr_client_buffer *buffer = wlr_client_buffer_create(
-			surface->current.buffer, surface->renderer);
-
-	if (buffer == NULL) {
+	struct wlr_surface_texture *texture =
+		surface_texture_create(surface->current.buffer, surface->renderer);
+	if (texture == NULL) {
 		wlr_log(WLR_ERROR, "Failed to upload buffer");
 		return;
 	}
 
-	if (surface->buffer != NULL) {
-		wlr_buffer_unlock(&surface->buffer->base);
-	}
-	surface->buffer = buffer;
+	surface_texture_drop(surface->texture);
+	surface->texture = texture;
 }
 
 static void surface_update_opaque_region(struct wlr_surface *surface) {
@@ -746,9 +811,7 @@ static void surface_handle_resource_destroy(struct wl_resource *resource) {
 	pixman_region32_fini(&surface->buffer_damage);
 	pixman_region32_fini(&surface->opaque_region);
 	pixman_region32_fini(&surface->input_region);
-	if (surface->buffer != NULL) {
-		wlr_buffer_unlock(&surface->buffer->base);
-	}
+	surface_texture_drop(surface->texture);
 	free(surface);
 }
 
@@ -814,10 +877,10 @@ static struct wlr_surface *surface_create(struct wl_client *client,
 }
 
 struct wlr_texture *wlr_surface_get_texture(struct wlr_surface *surface) {
-	if (surface->buffer == NULL) {
+	if (surface->texture == NULL) {
 		return NULL;
 	}
-	return surface->buffer->texture;
+	return surface->texture->texture;
 }
 
 bool wlr_surface_has_buffer(struct wlr_surface *surface) {