From 3b3640dda4242ad0150f3175a9c52c7c29376873 Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Mon, 21 Aug 2023 16:34:13 +0200 Subject: [PATCH 1/8] backend: add wlr_backend.event_loop Many objects depending on wlr_backend need access to the event loop. Storing it here will simplify wlr_output when combined with [1], and will allow wlr_renderer_autocreate() to have access to the event loop. [1]: https://gitlab.freedesktop.org/wlroots/wlroots/-/merge_requests/4310 --- backend/backend.c | 3 ++- backend/drm/backend.c | 2 +- backend/headless/backend.c | 2 +- backend/libinput/backend.c | 2 +- backend/multi/backend.c | 2 +- backend/wayland/backend.c | 2 +- backend/x11/backend.c | 2 +- include/wlr/backend.h | 1 + include/wlr/backend/interface.h | 2 +- 9 files changed, 10 insertions(+), 8 deletions(-) diff --git a/backend/backend.c b/backend/backend.c index 1e57e8321..fb2446ca3 100644 --- a/backend/backend.c +++ b/backend/backend.c @@ -39,9 +39,10 @@ #define WAIT_SESSION_TIMEOUT 10000 // ms void wlr_backend_init(struct wlr_backend *backend, - const struct wlr_backend_impl *impl) { + const struct wlr_backend_impl *impl, struct wl_event_loop *loop) { *backend = (struct wlr_backend){ .impl = impl, + .event_loop = loop, }; wl_signal_init(&backend->events.destroy); wl_signal_init(&backend->events.new_input); diff --git a/backend/drm/backend.c b/backend/drm/backend.c index 4697234db..aa23e11d9 100644 --- a/backend/drm/backend.c +++ b/backend/drm/backend.c @@ -208,7 +208,7 @@ struct wlr_backend *wlr_drm_backend_create(struct wl_display *display, wlr_log_errno(WLR_ERROR, "Allocation failed"); return NULL; } - wlr_backend_init(&drm->backend, &backend_impl); + wlr_backend_init(&drm->backend, &backend_impl, wl_display_get_event_loop(display)); drm->session = session; wl_list_init(&drm->fbs); diff --git a/backend/headless/backend.c b/backend/headless/backend.c index 8ab7d039e..a3f1a7ca1 100644 --- a/backend/headless/backend.c +++ b/backend/headless/backend.c @@ -72,7 +72,7 @@ struct wlr_backend *wlr_headless_backend_create(struct wl_display *display) { return NULL; } - wlr_backend_init(&backend->backend, &backend_impl); + wlr_backend_init(&backend->backend, &backend_impl, wl_display_get_event_loop(display)); backend->display = display; wl_list_init(&backend->outputs); diff --git a/backend/libinput/backend.c b/backend/libinput/backend.c index cbdf8b72e..2a0b405ca 100644 --- a/backend/libinput/backend.c +++ b/backend/libinput/backend.c @@ -199,7 +199,7 @@ struct wlr_backend *wlr_libinput_backend_create(struct wl_display *display, wlr_log(WLR_ERROR, "Allocation failed: %s", strerror(errno)); return NULL; } - wlr_backend_init(&backend->backend, &backend_impl); + wlr_backend_init(&backend->backend, &backend_impl, wl_display_get_event_loop(display)); wl_list_init(&backend->devices); diff --git a/backend/multi/backend.c b/backend/multi/backend.c index 8c36834e4..d3ebfea9a 100644 --- a/backend/multi/backend.c +++ b/backend/multi/backend.c @@ -119,7 +119,7 @@ struct wlr_backend *wlr_multi_backend_create(struct wl_display *display) { } wl_list_init(&backend->backends); - wlr_backend_init(&backend->backend, &backend_impl); + wlr_backend_init(&backend->backend, &backend_impl, wl_display_get_event_loop(display)); wl_signal_init(&backend->events.backend_add); wl_signal_init(&backend->events.backend_remove); diff --git a/backend/wayland/backend.c b/backend/wayland/backend.c index 9f78b6f1b..5e56ca85b 100644 --- a/backend/wayland/backend.c +++ b/backend/wayland/backend.c @@ -577,7 +577,7 @@ struct wlr_backend *wlr_wl_backend_create(struct wl_display *display, return NULL; } - wlr_backend_init(&wl->backend, &backend_impl); + wlr_backend_init(&wl->backend, &backend_impl, wl_display_get_event_loop(display)); wl->local_display = display; wl_list_init(&wl->outputs); diff --git a/backend/x11/backend.c b/backend/x11/backend.c index fcaab6186..5102921aa 100644 --- a/backend/x11/backend.c +++ b/backend/x11/backend.c @@ -400,7 +400,7 @@ struct wlr_backend *wlr_x11_backend_create(struct wl_display *display, return NULL; } - wlr_backend_init(&x11->backend, &backend_impl); + wlr_backend_init(&x11->backend, &backend_impl, wl_display_get_event_loop(display)); x11->wl_display = display; wl_list_init(&x11->outputs); diff --git a/include/wlr/backend.h b/include/wlr/backend.h index 9a8a2d874..2ea7621d6 100644 --- a/include/wlr/backend.h +++ b/include/wlr/backend.h @@ -19,6 +19,7 @@ struct wlr_backend_impl; */ struct wlr_backend { const struct wlr_backend_impl *impl; + struct wl_event_loop *event_loop; struct { /** Raised when destroyed */ diff --git a/include/wlr/backend/interface.h b/include/wlr/backend/interface.h index da57cae90..c5d8bbf43 100644 --- a/include/wlr/backend/interface.h +++ b/include/wlr/backend/interface.h @@ -24,7 +24,7 @@ struct wlr_backend_impl { * to the provided struct wlr_backend_impl reference. */ void wlr_backend_init(struct wlr_backend *backend, - const struct wlr_backend_impl *impl); + const struct wlr_backend_impl *impl, struct wl_event_loop *loop); /** * Emit the destroy event and clean up common backend state. */ From 7abb227617f4dc5563ece0e37b8d143f901a64d6 Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Wed, 11 Oct 2023 11:27:59 +0200 Subject: [PATCH 2/8] render: take event loop as input when creating renderer Some renderers will need access to the event loop for async GPU uploads. --- backend/drm/renderer.c | 3 ++- include/render/wlr_renderer.h | 2 +- render/wlr_renderer.c | 10 +++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/drm/renderer.c b/backend/drm/renderer.c index 321d583c7..cc7033c8d 100644 --- a/backend/drm/renderer.c +++ b/backend/drm/renderer.c @@ -13,7 +13,8 @@ bool init_drm_renderer(struct wlr_drm_backend *drm, struct wlr_drm_renderer *renderer) { - renderer->wlr_rend = renderer_autocreate_with_drm_fd(drm->fd); + renderer->wlr_rend = renderer_autocreate_with_drm_fd(drm->fd, + wl_display_get_event_loop(drm->display)); if (!renderer->wlr_rend) { wlr_log(WLR_ERROR, "Failed to create renderer"); return false; diff --git a/include/render/wlr_renderer.h b/include/render/wlr_renderer.h index a8777bc01..83508a99f 100644 --- a/include/render/wlr_renderer.h +++ b/include/render/wlr_renderer.h @@ -6,7 +6,7 @@ /** * Automatically select and create a renderer suitable for the DRM FD. */ -struct wlr_renderer *renderer_autocreate_with_drm_fd(int drm_fd); +struct wlr_renderer *renderer_autocreate_with_drm_fd(int drm_fd, struct wl_event_loop *loop); /** * Get the supported render formats. Buffers allocated with a format from this * list may be attached via wlr_renderer_begin_with_buffer. diff --git a/render/wlr_renderer.c b/render/wlr_renderer.c index 7f571286a..89f81abd8 100644 --- a/render/wlr_renderer.c +++ b/render/wlr_renderer.c @@ -229,7 +229,8 @@ static bool has_render_node(struct wlr_backend *backend) { return has_render_node; } -static struct wlr_renderer *renderer_autocreate(struct wlr_backend *backend, int drm_fd) { +static struct wlr_renderer *renderer_autocreate(struct wlr_backend *backend, int drm_fd, + struct wl_event_loop *loop) { const char *renderer_options[] = { "auto", "gles2", @@ -297,14 +298,13 @@ out: return renderer; } -struct wlr_renderer *renderer_autocreate_with_drm_fd(int drm_fd) { +struct wlr_renderer *renderer_autocreate_with_drm_fd(int drm_fd, struct wl_event_loop *loop) { assert(drm_fd >= 0); - - return renderer_autocreate(NULL, drm_fd); + return renderer_autocreate(NULL, drm_fd, loop); } struct wlr_renderer *wlr_renderer_autocreate(struct wlr_backend *backend) { - return renderer_autocreate(backend, -1); + return renderer_autocreate(backend, -1, backend->event_loop); } int wlr_renderer_get_drm_fd(struct wlr_renderer *r) { From 361804c727ecd6b4ec5513097bd9135c1f04303e Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Tue, 28 Nov 2023 17:14:48 +0100 Subject: [PATCH 3/8] buffer: allow concurrent data ptr accesses --- include/wlr/types/wlr_buffer.h | 7 ++++++- types/buffer/buffer.c | 34 ++++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/include/wlr/types/wlr_buffer.h b/include/wlr/types/wlr_buffer.h index de3aeec3d..82d3249b1 100644 --- a/include/wlr/types/wlr_buffer.h +++ b/include/wlr/types/wlr_buffer.h @@ -51,7 +51,12 @@ struct wlr_buffer { bool dropped; size_t n_locks; - bool accessing_data_ptr; + + size_t n_data_ptr_accesses; + uint32_t data_ptr_access_flags; // bitfield of wlr_buffer_data_ptr_access_flag + void *data_ptr_access_data; + uint32_t data_ptr_access_format; + size_t data_ptr_access_stride; struct { struct wl_signal destroy; diff --git a/types/buffer/buffer.c b/types/buffer/buffer.c index a8e9c54d7..058cecfa3 100644 --- a/types/buffer/buffer.c +++ b/types/buffer/buffer.c @@ -26,7 +26,7 @@ static void buffer_consider_destroy(struct wlr_buffer *buffer) { return; } - assert(!buffer->accessing_data_ptr); + assert(buffer->n_data_ptr_accesses == 0); wl_signal_emit_mutable(&buffer->events.destroy, NULL); wlr_addon_set_finish(&buffer->addons); @@ -74,21 +74,39 @@ bool wlr_buffer_get_dmabuf(struct wlr_buffer *buffer, bool wlr_buffer_begin_data_ptr_access(struct wlr_buffer *buffer, uint32_t flags, void **data, uint32_t *format, size_t *stride) { - assert(!buffer->accessing_data_ptr); if (!buffer->impl->begin_data_ptr_access) { return false; } - if (!buffer->impl->begin_data_ptr_access(buffer, flags, data, format, stride)) { - return false; + if (buffer->n_data_ptr_accesses == 0) { + if (!buffer->impl->begin_data_ptr_access(buffer, flags, data, format, stride)) { + return false; + } + buffer->data_ptr_access_flags = flags; + buffer->data_ptr_access_data = *data; + buffer->data_ptr_access_format = *format; + buffer->data_ptr_access_stride = *stride; + } else { + if (buffer->data_ptr_access_flags != flags) { + return false; + } + *data = buffer->data_ptr_access_data; + *format = buffer->data_ptr_access_format; + *stride = buffer->data_ptr_access_stride; } - buffer->accessing_data_ptr = true; + buffer->n_data_ptr_accesses++; return true; } void wlr_buffer_end_data_ptr_access(struct wlr_buffer *buffer) { - assert(buffer->accessing_data_ptr); - buffer->impl->end_data_ptr_access(buffer); - buffer->accessing_data_ptr = false; + assert(buffer->n_data_ptr_accesses > 0); + buffer->n_data_ptr_accesses--; + if (buffer->n_data_ptr_accesses == 0) { + buffer->impl->end_data_ptr_access(buffer); + buffer->data_ptr_access_flags = 0; + buffer->data_ptr_access_data = NULL; + buffer->data_ptr_access_format = 0; + buffer->data_ptr_access_stride = 0; + } } bool wlr_buffer_get_shm(struct wlr_buffer *buffer, From a59dd1d567bfdac639c1045d2e0ecd58ead3c986 Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Fri, 24 Nov 2023 18:08:36 +0100 Subject: [PATCH 4/8] render/vulkan: add upload timeline semaphore Doesn't do much at the moment, since it's always signalled when we submit the stage buffer. --- include/render/vulkan.h | 4 ++++ render/vulkan/pass.c | 35 +++++++++++++++++++++++------------ render/vulkan/renderer.c | 7 +++++++ render/vulkan/texture.c | 14 ++++++++++++++ render/vulkan/vulkan.c | 2 ++ 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/include/render/vulkan.h b/include/render/vulkan.h index a518ee4a8..1f12ab4da 100644 --- a/include/render/vulkan.h +++ b/include/render/vulkan.h @@ -51,6 +51,7 @@ struct wlr_vk_device { PFN_vkGetMemoryFdPropertiesKHR vkGetMemoryFdPropertiesKHR; PFN_vkWaitSemaphoresKHR vkWaitSemaphoresKHR; PFN_vkGetSemaphoreCounterValueKHR vkGetSemaphoreCounterValueKHR; + PFN_vkSignalSemaphoreKHR vkSignalSemaphoreKHR; PFN_vkGetSemaphoreFdKHR vkGetSemaphoreFdKHR; PFN_vkImportSemaphoreFdKHR vkImportSemaphoreFdKHR; PFN_vkQueueSubmit2KHR vkQueueSubmit2KHR; @@ -252,6 +253,9 @@ struct wlr_vk_renderer { VkSemaphore timeline_semaphore; uint64_t timeline_point; + VkSemaphore upload_timeline_semaphore; + uint64_t upload_timeline_point; + size_t last_pool_size; struct wl_list descriptor_pools; // wlr_vk_descriptor_pool.link struct wl_list render_format_setups; // wlr_vk_render_format_setup.link diff --git a/render/vulkan/pass.c b/render/vulkan/pass.c index 42bd6b4e1..4fe8ce5ac 100644 --- a/render/vulkan/pass.c +++ b/render/vulkan/pass.c @@ -296,27 +296,38 @@ static bool render_pass_submit(struct wlr_render_pass *wlr_pass) { .semaphore = renderer->timeline_semaphore, .value = stage_timeline_point, }; - VkSubmitInfo2KHR stage_submit = { - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2_KHR, - .commandBufferInfoCount = 1, - .pCommandBufferInfos = &stage_cb_info, - .signalSemaphoreInfoCount = 1, - .pSignalSemaphoreInfos = &stage_signal, - }; - VkSemaphoreSubmitInfoKHR stage_wait; + VkSemaphoreSubmitInfoKHR stage_wait[2]; + uint32_t stage_wait_len = 0; + + if (renderer->upload_timeline_point > 0) { + stage_wait[stage_wait_len++] = (VkSemaphoreSubmitInfoKHR){ + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO_KHR, + .semaphore = renderer->upload_timeline_semaphore, + .value = renderer->upload_timeline_point, + .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT_KHR, + }; + } + if (renderer->stage.last_timeline_point > 0) { - stage_wait = (VkSemaphoreSubmitInfoKHR){ + stage_wait[stage_wait_len++] = (VkSemaphoreSubmitInfoKHR){ .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO_KHR, .semaphore = renderer->timeline_semaphore, .value = renderer->stage.last_timeline_point, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT_KHR, }; - - stage_submit.waitSemaphoreInfoCount = 1; - stage_submit.pWaitSemaphoreInfos = &stage_wait; } + VkSubmitInfo2KHR stage_submit = { + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2_KHR, + .commandBufferInfoCount = 1, + .pCommandBufferInfos = &stage_cb_info, + .waitSemaphoreInfoCount = stage_wait_len, + .pWaitSemaphoreInfos = stage_wait, + .signalSemaphoreInfoCount = 1, + .pSignalSemaphoreInfos = &stage_signal, + }; + renderer->stage.last_timeline_point = stage_timeline_point; uint64_t render_timeline_point = vulkan_end_command_buffer(render_cb, renderer); diff --git a/render/vulkan/renderer.c b/render/vulkan/renderer.c index b3f764017..63fad5e3b 100644 --- a/render/vulkan/renderer.c +++ b/render/vulkan/renderer.c @@ -1047,6 +1047,7 @@ static void vulkan_destroy(struct wlr_renderer *wlr_renderer) { } vkDestroySemaphore(dev->dev, renderer->timeline_semaphore, NULL); + vkDestroySemaphore(dev->dev, renderer->upload_timeline_semaphore, NULL); vkDestroyPipelineLayout(dev->dev, renderer->output_pipe_layout, NULL); vkDestroyDescriptorSetLayout(dev->dev, renderer->output_ds_layout, NULL); vkDestroyCommandPool(dev->dev, renderer->command_pool, NULL); @@ -2193,6 +2194,12 @@ struct wlr_renderer *vulkan_renderer_create_for_device(struct wlr_vk_device *dev wlr_vk_error("vkCreateSemaphore", res); goto error; } + res = vkCreateSemaphore(dev->dev, &semaphore_info, NULL, + &renderer->upload_timeline_semaphore); + if (res != VK_SUCCESS) { + wlr_vk_error("vkCreateSemaphore", res); + goto error; + } return &renderer->wlr_renderer; diff --git a/render/vulkan/texture.c b/render/vulkan/texture.c index 20aa12f1e..35dd8e50f 100644 --- a/render/vulkan/texture.c +++ b/render/vulkan/texture.c @@ -81,6 +81,8 @@ static bool write_pixels(struct wlr_vk_texture *texture, return false; } + uint64_t timeline_point = ++renderer->upload_timeline_point; + void *vmap; res = vkMapMemory(dev, span.buffer->memory, span.alloc.start, bsize, 0, &vmap); @@ -141,6 +143,18 @@ static bool write_pixels(struct wlr_vk_texture *texture, assert((uint32_t)(map - (char *)vmap) == bsize); vkUnmapMemory(dev, span.buffer->memory); + VkSemaphoreSignalInfoKHR signal_info = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO_KHR, + .semaphore = renderer->upload_timeline_semaphore, + .value = timeline_point, + }; + res = renderer->dev->api.vkSignalSemaphoreKHR(renderer->dev->dev, &signal_info); + if (res != VK_SUCCESS) { + free(copies); + wlr_vk_error("vkMapMemory", res); + return false; + } + // record staging cb // will be executed before next frame VkCommandBuffer cb = vulkan_record_stage_cb(renderer); diff --git a/render/vulkan/vulkan.c b/render/vulkan/vulkan.c index b8ba0b480..67a720e00 100644 --- a/render/vulkan/vulkan.c +++ b/render/vulkan/vulkan.c @@ -617,6 +617,8 @@ struct wlr_vk_device *vulkan_device_create(struct wlr_vk_instance *ini, load_device_proc(dev, "vkWaitSemaphoresKHR", &dev->api.vkWaitSemaphoresKHR); load_device_proc(dev, "vkGetSemaphoreCounterValueKHR", &dev->api.vkGetSemaphoreCounterValueKHR); + load_device_proc(dev, "vkSignalSemaphoreKHR", + &dev->api.vkSignalSemaphoreKHR); load_device_proc(dev, "vkGetSemaphoreFdKHR", &dev->api.vkGetSemaphoreFdKHR); load_device_proc(dev, "vkImportSemaphoreFdKHR", &dev->api.vkImportSemaphoreFdKHR); load_device_proc(dev, "vkQueueSubmit2KHR", &dev->api.vkQueueSubmit2KHR); From 50b471e03583ed8af29a4f2d448f394ee62a416a Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Fri, 24 Nov 2023 19:23:32 +0100 Subject: [PATCH 5/8] render/vulkan: extract pixel copy into separate function --- render/vulkan/texture.c | 78 +++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/render/vulkan/texture.c b/render/vulkan/texture.c index 35dd8e50f..487f36174 100644 --- a/render/vulkan/texture.c +++ b/render/vulkan/texture.c @@ -36,6 +36,40 @@ static VkImageAspectFlagBits mem_plane_aspect(unsigned i) { } } +static void copy_pixels(char *vmap, const char *vdata, uint32_t tex_width, + uint32_t stride, uint32_t size, const pixman_region32_t *region, + const struct wlr_pixel_format_info *format_info) { + int rects_len = 0; + const pixman_box32_t *rects = pixman_region32_rectangles(region, &rects_len); + + char *map = vmap; + for (int i = 0; i < rects_len; i++) { + pixman_box32_t rect = rects[i]; + uint32_t width = rect.x2 - rect.x1; + uint32_t height = rect.y2 - rect.y1; + uint32_t src_x = rect.x1; + uint32_t src_y = rect.y1; + uint32_t packed_stride = (uint32_t)pixel_format_info_min_stride(format_info, width); + + // write data into staging buffer span + const char *pdata = vdata; // data iterator + pdata += stride * src_y; + pdata += format_info->bytes_per_block * src_x; + if (src_x == 0 && width == tex_width && stride == packed_stride) { + memcpy(map, pdata, packed_stride * height); + map += packed_stride * height; + } else { + for (unsigned i = 0u; i < height; ++i) { + memcpy(map, pdata, packed_stride); + pdata += stride; + map += packed_stride; + } + } + } + + assert((uint32_t)(map - vmap) == size); +} + // Will transition the texture to shaderReadOnlyOptimal layout for reading // from fragment shader later on static bool write_pixels(struct wlr_vk_texture *texture, @@ -83,19 +117,7 @@ static bool write_pixels(struct wlr_vk_texture *texture, uint64_t timeline_point = ++renderer->upload_timeline_point; - void *vmap; - res = vkMapMemory(dev, span.buffer->memory, span.alloc.start, - bsize, 0, &vmap); - if (res != VK_SUCCESS) { - wlr_vk_error("vkMapMemory", res); - free(copies); - return false; - } - char *map = (char *)vmap; - - // upload data - - uint32_t buf_off = span.alloc.start + (map - (char *)vmap); + uint32_t buf_off = span.alloc.start; for (int i = 0; i < rects_len; i++) { pixman_box32_t rect = rects[i]; uint32_t width = rect.x2 - rect.x1; @@ -104,22 +126,6 @@ static bool write_pixels(struct wlr_vk_texture *texture, uint32_t src_y = rect.y1; uint32_t packed_stride = (uint32_t)pixel_format_info_min_stride(format_info, width); - // write data into staging buffer span - const char *pdata = vdata; // data iterator - pdata += stride * src_y; - pdata += format_info->bytes_per_block * src_x; - if (src_x == 0 && width == texture->wlr_texture.width && - stride == packed_stride) { - memcpy(map, pdata, packed_stride * height); - map += packed_stride * height; - } else { - for (unsigned i = 0u; i < height; ++i) { - memcpy(map, pdata, packed_stride); - pdata += stride; - map += packed_stride; - } - } - copies[i] = (VkBufferImageCopy) { .imageExtent.width = width, .imageExtent.height = height, @@ -136,11 +142,21 @@ static bool write_pixels(struct wlr_vk_texture *texture, .imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, }; - buf_off += height * packed_stride; } - assert((uint32_t)(map - (char *)vmap) == bsize); + void *vmap; + res = vkMapMemory(dev, span.buffer->memory, span.alloc.start, + bsize, 0, &vmap); + if (res != VK_SUCCESS) { + wlr_vk_error("vkMapMemory", res); + free(copies); + return false; + } + + copy_pixels(vmap, vdata, texture->wlr_texture.width, + stride, bsize, region, format_info); + vkUnmapMemory(dev, span.buffer->memory); VkSemaphoreSignalInfoKHR signal_info = { From 79cbbfb36637886281fdb7226a03a29c959dc690 Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Fri, 24 Nov 2023 20:13:31 +0100 Subject: [PATCH 6/8] render/vulkan: keep wlr_vk_shared_buffer mapped --- include/render/vulkan.h | 1 + render/vulkan/renderer.c | 9 +++++++++ render/vulkan/texture.c | 15 ++------------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/include/render/vulkan.h b/include/render/vulkan.h index 1f12ab4da..2434b00d5 100644 --- a/include/render/vulkan.h +++ b/include/render/vulkan.h @@ -436,6 +436,7 @@ struct wlr_vk_shared_buffer { VkBuffer buffer; VkDeviceMemory memory; VkDeviceSize buf_size; + void *map; struct wl_array allocs; // struct wlr_vk_allocation }; diff --git a/render/vulkan/renderer.c b/render/vulkan/renderer.c index 63fad5e3b..0b7f23e70 100644 --- a/render/vulkan/renderer.c +++ b/render/vulkan/renderer.c @@ -178,6 +178,9 @@ static void shared_buffer_destroy(struct wlr_vk_renderer *r, } wl_array_release(&buffer->allocs); + if (buffer->map) { + vkUnmapMemory(r->dev->dev, buffer->memory); + } if (buffer->buffer) { vkDestroyBuffer(r->dev->dev, buffer->buffer, NULL); } @@ -302,6 +305,12 @@ struct wlr_vk_buffer_span vulkan_get_stage_span(struct wlr_vk_renderer *r, goto error; } + res = vkMapMemory(r->dev->dev, buf->memory, 0, VK_WHOLE_SIZE, 0, &buf->map); + if (res != VK_SUCCESS) { + wlr_vk_error("vkMapMemory", res); + goto error; + } + struct wlr_vk_allocation *a = wl_array_add(&buf->allocs, sizeof(*a)); if (a == NULL) { wlr_log_errno(WLR_ERROR, "Allocation failed"); diff --git a/render/vulkan/texture.c b/render/vulkan/texture.c index 487f36174..b26397989 100644 --- a/render/vulkan/texture.c +++ b/render/vulkan/texture.c @@ -78,7 +78,6 @@ static bool write_pixels(struct wlr_vk_texture *texture, VkAccessFlags src_access) { VkResult res; struct wlr_vk_renderer *renderer = texture->renderer; - VkDevice dev = texture->renderer->dev->dev; const struct wlr_pixel_format_info *format_info = drm_get_pixel_format_info(texture->format->drm); assert(format_info); @@ -145,20 +144,10 @@ static bool write_pixels(struct wlr_vk_texture *texture, buf_off += height * packed_stride; } - void *vmap; - res = vkMapMemory(dev, span.buffer->memory, span.alloc.start, - bsize, 0, &vmap); - if (res != VK_SUCCESS) { - wlr_vk_error("vkMapMemory", res); - free(copies); - return false; - } - - copy_pixels(vmap, vdata, texture->wlr_texture.width, + char *dst = (char *)span.buffer->map + span.alloc.start; + copy_pixels(dst, vdata, texture->wlr_texture.width, stride, bsize, region, format_info); - vkUnmapMemory(dev, span.buffer->memory); - VkSemaphoreSignalInfoKHR signal_info = { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO_KHR, .semaphore = renderer->upload_timeline_semaphore, From fee89699557f0214d4798ee1a25054fdde1256b4 Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Fri, 24 Nov 2023 20:01:40 +0100 Subject: [PATCH 7/8] render/vulkan: add upload thread TODO: - Update blocking stage submission as well - Allow multiple data ptr accesses - Stop abusing FDs for multi-thread queue --- include/render/vulkan.h | 24 ++++- include/wlr/render/vulkan.h | 3 +- render/vulkan/meson.build | 1 + render/vulkan/renderer.c | 13 ++- render/vulkan/texture.c | 196 +++++++++++++++++++++++++++++++----- render/wlr_renderer.c | 2 +- 6 files changed, 208 insertions(+), 31 deletions(-) diff --git a/include/render/vulkan.h b/include/render/vulkan.h index 2434b00d5..3afa3104a 100644 --- a/include/render/vulkan.h +++ b/include/render/vulkan.h @@ -1,6 +1,7 @@ #ifndef RENDER_VULKAN_H #define RENDER_VULKAN_H +#include #include #include #include @@ -270,6 +271,12 @@ struct wlr_vk_renderer { // Pool of command buffers struct wlr_vk_command_buffer command_buffers[VULKAN_COMMAND_BUFFERS_CAP]; + struct { + pthread_t thread; + int worker_fd, control_fd; + struct wl_event_source *event_source; + } upload; + struct { struct wlr_vk_command_buffer *cb; uint64_t last_timeline_point; @@ -301,6 +308,17 @@ struct wlr_vk_texture_view { struct wlr_vk_descriptor_pool *ds_pool; }; +struct wlr_vk_upload_task { + struct wlr_buffer *buffer; + VkDeviceMemory memory; + uint64_t timeline_point; + char *dst; + const char *src; + uint32_t src_stride, dst_size; + pixman_region32_t region; + const struct wlr_pixel_format_info *format_info; +}; + struct wlr_vk_pipeline *setup_get_or_create_pipeline( struct wlr_vk_render_format_setup *setup, const struct wlr_vk_pipeline_key *key); @@ -312,7 +330,8 @@ struct wlr_vk_texture_view *vulkan_texture_get_or_create_view( const struct wlr_vk_pipeline_layout *layout); // Creates a vulkan renderer for the given device. -struct wlr_renderer *vulkan_renderer_create_for_device(struct wlr_vk_device *dev); +struct wlr_renderer *vulkan_renderer_create_for_device(struct wlr_vk_device *dev, + struct wl_event_loop *loop); // stage utility - for uploading/retrieving data // Gets an command buffer in recording state which is guaranteed to be @@ -382,6 +401,9 @@ bool vulkan_read_pixels(struct wlr_vk_renderer *vk_renderer, uint32_t width, uint32_t height, uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y, void *data); +bool vulkan_init_upload_worker(struct wlr_vk_renderer *renderer, + struct wl_event_loop *loop); + // State (e.g. image texture) associated with a surface. struct wlr_vk_texture { struct wlr_texture wlr_texture; diff --git a/include/wlr/render/vulkan.h b/include/wlr/render/vulkan.h index 50f8c558d..3c4eac77b 100644 --- a/include/wlr/render/vulkan.h +++ b/include/wlr/render/vulkan.h @@ -18,7 +18,8 @@ struct wlr_vk_image_attribs { VkFormat format; }; -struct wlr_renderer *wlr_vk_renderer_create_with_drm_fd(int drm_fd); +struct wlr_renderer *wlr_vk_renderer_create_with_drm_fd( + struct wl_event_loop *loop, int drm_fd); VkInstance wlr_vk_renderer_get_instance(struct wlr_renderer *renderer); VkPhysicalDevice wlr_vk_renderer_get_physical_device(struct wlr_renderer *renderer); diff --git a/render/vulkan/meson.build b/render/vulkan/meson.build index 36a822f9a..d4c81367f 100644 --- a/render/vulkan/meson.build +++ b/render/vulkan/meson.build @@ -46,6 +46,7 @@ wlr_files += files( ) wlr_deps += dep_vulkan +wlr_deps += dependency('threads') features += { 'vulkan-renderer': true } subdir('shaders') diff --git a/render/vulkan/renderer.c b/render/vulkan/renderer.c index 0b7f23e70..e4646e7f6 100644 --- a/render/vulkan/renderer.c +++ b/render/vulkan/renderer.c @@ -369,6 +369,7 @@ bool vulkan_submit_stage_wait(struct wlr_vk_renderer *renderer) { return false; } + // TODO VkTimelineSemaphoreSubmitInfoKHR timeline_submit_info = { .sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR, .signalSemaphoreValueCount = 1, @@ -2153,7 +2154,8 @@ error: return NULL; } -struct wlr_renderer *vulkan_renderer_create_for_device(struct wlr_vk_device *dev) { +struct wlr_renderer *vulkan_renderer_create_for_device(struct wlr_vk_device *dev, + struct wl_event_loop *loop) { struct wlr_vk_renderer *renderer; VkResult res; if (!(renderer = calloc(1, sizeof(*renderer)))) { @@ -2210,6 +2212,10 @@ struct wlr_renderer *vulkan_renderer_create_for_device(struct wlr_vk_device *dev goto error; } + if (!vulkan_init_upload_worker(renderer, loop)) { + goto error; + } + return &renderer->wlr_renderer; error: @@ -2217,7 +2223,8 @@ error: return NULL; } -struct wlr_renderer *wlr_vk_renderer_create_with_drm_fd(int drm_fd) { +struct wlr_renderer *wlr_vk_renderer_create_with_drm_fd(struct wl_event_loop *loop, + int drm_fd) { wlr_log(WLR_INFO, "The vulkan renderer is only experimental and " "not expected to be ready for daily use"); wlr_log(WLR_INFO, "Run with VK_INSTANCE_LAYERS=VK_LAYER_KHRONOS_validation " @@ -2252,7 +2259,7 @@ struct wlr_renderer *wlr_vk_renderer_create_with_drm_fd(int drm_fd) { return NULL; } - return vulkan_renderer_create_for_device(dev); + return vulkan_renderer_create_for_device(dev, loop); } VkInstance wlr_vk_renderer_get_instance(struct wlr_renderer *renderer) { diff --git a/render/vulkan/texture.c b/render/vulkan/texture.c index b26397989..8b984e58a 100644 --- a/render/vulkan/texture.c +++ b/render/vulkan/texture.c @@ -2,9 +2,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -70,13 +72,150 @@ static void copy_pixels(char *vmap, const char *vdata, uint32_t tex_width, assert((uint32_t)(map - vmap) == size); } +static bool read_upload_task(struct wlr_vk_upload_task *task, int fd) { + while (true) { + errno = 0; + ssize_t n = read(fd, task, sizeof(*task)); + if (errno == EINTR) { + continue; + } + if (n == sizeof(*task)) { + return true; + } else if (n < 0) { + wlr_log_errno(WLR_ERROR, "read() failed"); + } else if (n > 0) { + wlr_log(WLR_ERROR, "Unexpected partial read"); + } + return false; + } +} + +static bool write_upload_task(const struct wlr_vk_upload_task *task, int fd) { + while (true) { + errno = 0; + ssize_t n = write(fd, task, sizeof(*task)); + if (errno == EINTR) { + continue; + } + if (n == sizeof(*task)) { + return true; + } else if (n < 0) { + wlr_log_errno(WLR_ERROR, "write() failed"); + } else if (n > 0) { + wlr_log(WLR_ERROR, "Unexpected partial write"); + } + return false; + } +} + +static void process_upload_task(struct wlr_vk_renderer *renderer, + struct wlr_vk_upload_task *task) { + copy_pixels(task->dst, task->src, task->buffer->width, task->src_stride, + task->dst_size, &task->region, task->format_info); + + VkSemaphoreSignalInfoKHR signal_info = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO_KHR, + .semaphore = renderer->upload_timeline_semaphore, + .value = task->timeline_point, + }; + VkResult res = renderer->dev->api.vkSignalSemaphoreKHR(renderer->dev->dev, &signal_info); + if (res != VK_SUCCESS) { + wlr_vk_error("vkMapMemory", res); + } +} + +static void *run_uploads(void *data) { + struct wlr_vk_renderer *renderer = data; + + while (true) { + struct wlr_vk_upload_task task = {0}; + if (!read_upload_task(&task, renderer->upload.worker_fd)) { + break; + } + process_upload_task(renderer, &task); + if (!write_upload_task(&task, renderer->upload.worker_fd)) { + break; + } + } + + close(renderer->upload.worker_fd); + return NULL; +} + +static void handle_upload_task_complete(struct wlr_vk_renderer *renderer, + struct wlr_vk_upload_task *task) { + wlr_buffer_end_data_ptr_access(task->buffer); + wlr_buffer_unlock(task->buffer); + pixman_region32_fini(&task->region); +} + +static int handle_upload_fd_event(int fd, uint32_t mask, void *data) { + struct wlr_vk_renderer *renderer = data; + + if (mask & WL_EVENT_ERROR) { + wlr_log(WLR_ERROR, "Upload worker FD error"); + return 0; + } + if (mask & WL_EVENT_HANGUP) { + return 0; + } + + if (mask & WL_EVENT_READABLE) { + struct wlr_vk_upload_task task = {0}; + if (!read_upload_task(&task, fd)) { + return 0; + } + handle_upload_task_complete(renderer, &task); + } + + return 0; +} + +bool vulkan_init_upload_worker(struct wlr_vk_renderer *renderer, + struct wl_event_loop *loop) { + int sockets[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sockets) != 0) { + wlr_log_errno(WLR_ERROR, "pipe() failed"); + return false; + } + renderer->upload.worker_fd = sockets[0]; + renderer->upload.control_fd = sockets[1]; + + renderer->upload.event_source = wl_event_loop_add_fd(loop, + renderer->upload.control_fd, WL_EVENT_READABLE, + handle_upload_fd_event, renderer); + if (renderer->upload.event_source == NULL) { + wlr_log(WLR_ERROR, "wl_event_loop_add_fd() failed"); + goto error_fds; + } + + // Block all signals in the new thread: let the main thread handle these + sigset_t saved_sigset, new_sigset; + sigfillset(&new_sigset); + pthread_sigmask(SIG_BLOCK, &new_sigset, &saved_sigset); + int ret = pthread_create(&renderer->upload.thread, NULL, run_uploads, renderer); + pthread_sigmask(SIG_SETMASK, &saved_sigset, NULL); + if (ret != 0) { + wlr_log_errno(WLR_ERROR, "pthread_create() failed"); + goto error_event_source; + } + + return true; + +error_event_source: + wl_event_source_remove(renderer->upload.event_source); +error_fds: + close(renderer->upload.worker_fd); + close(renderer->upload.control_fd); + return false; +} + // Will transition the texture to shaderReadOnlyOptimal layout for reading // from fragment shader later on -static bool write_pixels(struct wlr_vk_texture *texture, +static bool start_upload(struct wlr_vk_texture *texture, struct wlr_buffer *buffer, uint32_t stride, const pixman_region32_t *region, const void *vdata, VkImageLayout old_layout, VkPipelineStageFlags src_stage, VkAccessFlags src_access) { - VkResult res; struct wlr_vk_renderer *renderer = texture->renderer; const struct wlr_pixel_format_info *format_info = drm_get_pixel_format_info(texture->format->drm); @@ -144,19 +283,20 @@ static bool write_pixels(struct wlr_vk_texture *texture, buf_off += height * packed_stride; } - char *dst = (char *)span.buffer->map + span.alloc.start; - copy_pixels(dst, vdata, texture->wlr_texture.width, - stride, bsize, region, format_info); - - VkSemaphoreSignalInfoKHR signal_info = { - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO_KHR, - .semaphore = renderer->upload_timeline_semaphore, - .value = timeline_point, + struct wlr_vk_upload_task task = { + .buffer = wlr_buffer_lock(buffer), + .memory = span.buffer->memory, + .timeline_point = timeline_point, + .dst = (char *)span.buffer->map + span.alloc.start, + .src = vdata, + .src_stride = stride, + .dst_size = bsize, + .format_info = format_info, }; - res = renderer->dev->api.vkSignalSemaphoreKHR(renderer->dev->dev, &signal_info); - if (res != VK_SUCCESS) { + pixman_region32_init(&task.region); + pixman_region32_copy(&task.region, region); + if (!write_upload_task(&task, renderer->upload.control_fd)) { free(copies); - wlr_vk_error("vkMapMemory", res); return false; } @@ -199,19 +339,21 @@ static bool vulkan_texture_update_from_buffer(struct wlr_texture *wlr_texture, return false; } - bool ok = true; - if (format != texture->format->drm) { - ok = false; - goto out; + goto error; } - ok = write_pixels(texture, stride, damage, data, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_ACCESS_SHADER_READ_BIT); + if (!start_upload(texture, buffer, stride, damage, data, + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_ACCESS_SHADER_READ_BIT)) { + goto error; + } -out: + return true; + +error: wlr_buffer_end_data_ptr_access(buffer); - return ok; + return false; } void vulkan_texture_destroy(struct wlr_vk_texture *texture) { @@ -407,7 +549,8 @@ static void texture_set_format(struct wlr_vk_texture *texture, } static struct wlr_texture *vulkan_texture_from_pixels( - struct wlr_vk_renderer *renderer, uint32_t drm_fmt, uint32_t stride, + struct wlr_vk_renderer *renderer, struct wlr_buffer *buffer, + uint32_t drm_fmt, uint32_t stride, uint32_t width, uint32_t height, const void *data) { VkResult res; VkDevice dev = renderer->dev->dev; @@ -480,7 +623,8 @@ static struct wlr_texture *vulkan_texture_from_pixels( pixman_region32_t region; pixman_region32_init_rect(®ion, 0, 0, width, height); - if (!write_pixels(texture, stride, ®ion, data, VK_IMAGE_LAYOUT_UNDEFINED, + if (!start_upload(texture, buffer, stride, ®ion, data, + VK_IMAGE_LAYOUT_UNDEFINED, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0)) { goto error; } @@ -816,8 +960,10 @@ struct wlr_texture *vulkan_texture_from_buffer(struct wlr_renderer *wlr_renderer } else if (wlr_buffer_begin_data_ptr_access(buffer, WLR_BUFFER_DATA_PTR_ACCESS_READ, &data, &format, &stride)) { struct wlr_texture *tex = vulkan_texture_from_pixels(renderer, - format, stride, buffer->width, buffer->height, data); - wlr_buffer_end_data_ptr_access(buffer); + buffer, format, stride, buffer->width, buffer->height, data); + if (tex == NULL) { + wlr_buffer_end_data_ptr_access(buffer); + } return tex; } else { return NULL; diff --git a/render/wlr_renderer.c b/render/wlr_renderer.c index 89f81abd8..aad0ec760 100644 --- a/render/wlr_renderer.c +++ b/render/wlr_renderer.c @@ -267,7 +267,7 @@ static struct wlr_renderer *renderer_autocreate(struct wlr_backend *backend, int log_creation_failure(is_auto, "Cannot create Vulkan renderer: no DRM FD available"); } else { #if WLR_HAS_VULKAN_RENDERER - renderer = wlr_vk_renderer_create_with_drm_fd(drm_fd); + renderer = wlr_vk_renderer_create_with_drm_fd(loop, drm_fd); #else wlr_log(WLR_ERROR, "Cannot create Vulkan renderer: disabled at compile-time"); #endif From 2c0fdb7e659b4c54b8f2a8d622557c8c1bb5f2e7 Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Fri, 24 Nov 2023 21:24:46 +0100 Subject: [PATCH 8/8] hack: measure timings --- include/render/vulkan.h | 1 + render/vulkan/texture.c | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/include/render/vulkan.h b/include/render/vulkan.h index 3afa3104a..1de0d9dce 100644 --- a/include/render/vulkan.h +++ b/include/render/vulkan.h @@ -317,6 +317,7 @@ struct wlr_vk_upload_task { uint32_t src_stride, dst_size; pixman_region32_t region; const struct wlr_pixel_format_info *format_info; + int64_t start; }; struct wlr_vk_pipeline *setup_get_or_create_pipeline( diff --git a/render/vulkan/texture.c b/render/vulkan/texture.c index 8b984e58a..58f421cda 100644 --- a/render/vulkan/texture.c +++ b/render/vulkan/texture.c @@ -16,6 +16,8 @@ #include "render/pixel_format.h" #include "render/vulkan.h" +#include "util/time.h" + static const struct wlr_texture_impl texture_impl; bool wlr_texture_is_vk(struct wlr_texture *wlr_texture) { @@ -110,9 +112,17 @@ static bool write_upload_task(const struct wlr_vk_upload_task *task, int fd) { static void process_upload_task(struct wlr_vk_renderer *renderer, struct wlr_vk_upload_task *task) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + int64_t start = timespec_to_nsec(&ts); + copy_pixels(task->dst, task->src, task->buffer->width, task->src_stride, task->dst_size, &task->region, task->format_info); + clock_gettime(CLOCK_MONOTONIC, &ts); + int64_t dur_ns = timespec_to_nsec(&ts) - start; + wlr_log(WLR_INFO, "UPLOAD: %f ms", (double)dur_ns / 1000 / 1000); + VkSemaphoreSignalInfoKHR signal_info = { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO_KHR, .semaphore = renderer->upload_timeline_semaphore, @@ -147,6 +157,11 @@ static void handle_upload_task_complete(struct wlr_vk_renderer *renderer, wlr_buffer_end_data_ptr_access(task->buffer); wlr_buffer_unlock(task->buffer); pixman_region32_fini(&task->region); + + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + int64_t dur_ns = timespec_to_nsec(&ts) - task->start; + wlr_log(WLR_INFO, "TOTAL: %f ms", (double)dur_ns / 1000 / 1000); } static int handle_upload_fd_event(int fd, uint32_t mask, void *data) { @@ -218,6 +233,10 @@ static bool start_upload(struct wlr_vk_texture *texture, struct wlr_buffer *buff VkAccessFlags src_access) { struct wlr_vk_renderer *renderer = texture->renderer; + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + int64_t start = timespec_to_nsec(&ts); + const struct wlr_pixel_format_info *format_info = drm_get_pixel_format_info(texture->format->drm); assert(format_info); @@ -292,13 +311,22 @@ static bool start_upload(struct wlr_vk_texture *texture, struct wlr_buffer *buff .src_stride = stride, .dst_size = bsize, .format_info = format_info, + .start = start, }; pixman_region32_init(&task.region); pixman_region32_copy(&task.region, region); +#if 1 if (!write_upload_task(&task, renderer->upload.control_fd)) { free(copies); return false; } +#else + process_upload_task(renderer, &task); + handle_upload_task_complete(renderer, &task); +#endif + + clock_gettime(CLOCK_MONOTONIC, &ts); + start = timespec_to_nsec(&ts); // record staging cb // will be executed before next frame @@ -308,6 +336,10 @@ static bool start_upload(struct wlr_vk_texture *texture, struct wlr_buffer *buff return false; } + clock_gettime(CLOCK_MONOTONIC, &ts); + int64_t dur_ns = timespec_to_nsec(&ts) - start; + wlr_log(WLR_INFO, "STARTUP: %f ms", (double)dur_ns / 1000 / 1000); + vulkan_change_layout(cb, texture->image, old_layout, src_stage, src_access, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TRANSFER_BIT, @@ -315,6 +347,7 @@ static bool start_upload(struct wlr_vk_texture *texture, struct wlr_buffer *buff vkCmdCopyBufferToImage(cb, span.buffer->buffer, texture->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, (uint32_t)rects_len, copies); + vulkan_change_layout(cb, texture->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,