[PATCH 5/5] wined3d: Introduce a slab allocator for small buffers.
Henri Verbeet
hverbeet at codeweavers.com
Mon Apr 20 14:00:26 CDT 2020
Signed-off-by: Henri Verbeet <hverbeet at codeweavers.com>
---
dlls/wined3d/adapter_vk.c | 29 ++++-
dlls/wined3d/context_vk.c | 192 +++++++++++++++++++++++++++++++++
dlls/wined3d/texture.c | 4 +-
dlls/wined3d/wined3d_private.h | 29 +++++
include/wine/rbtree.h | 16 +++
5 files changed, 264 insertions(+), 6 deletions(-)
diff --git a/dlls/wined3d/adapter_vk.c b/dlls/wined3d/adapter_vk.c
index 35b4e8da6aa..8a6bc2b01cf 100644
--- a/dlls/wined3d/adapter_vk.c
+++ b/dlls/wined3d/adapter_vk.c
@@ -553,13 +553,23 @@ static void *wined3d_bo_vk_map(struct wined3d_bo_vk *bo, struct wined3d_context_
{
const struct wined3d_vk_info *vk_info;
struct wined3d_device_vk *device_vk;
+ struct wined3d_bo_slab_vk *slab;
void *map_ptr;
VkResult vr;
vk_info = context_vk->vk_info;
device_vk = wined3d_device_vk(context_vk->c.device);
- if (bo->memory)
+ if ((slab = bo->slab))
+ {
+ if (!(map_ptr = slab->map_ptr) && !(map_ptr = wined3d_bo_vk_map(&slab->bo, context_vk)))
+ {
+ ERR("Failed to map slab.\n");
+ return NULL;
+ }
+ ++slab->map_count;
+ }
+ else if (bo->memory)
{
struct wined3d_allocator_chunk_vk *chunk_vk = wined3d_allocator_chunk_vk(bo->memory->chunk);
@@ -582,6 +592,17 @@ static void wined3d_bo_vk_unmap(struct wined3d_bo_vk *bo, struct wined3d_context
{
const struct wined3d_vk_info *vk_info;
struct wined3d_device_vk *device_vk;
+ struct wined3d_bo_slab_vk *slab;
+
+ if ((slab = bo->slab))
+ {
+ if (--slab->map_count)
+ return;
+
+ wined3d_bo_vk_unmap(&slab->bo, context_vk);
+ slab->map_ptr = NULL;
+ return;
+ }
vk_info = context_vk->vk_info;
device_vk = wined3d_device_vk(context_vk->c.device);
@@ -628,7 +649,7 @@ static void *adapter_vk_map_bo_address(struct wined3d_context *context,
vk_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
vk_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
vk_barrier.buffer = bo->vk_buffer;
- vk_barrier.offset = (uintptr_t)data->addr;
+ vk_barrier.offset = bo->buffer_offset + (uintptr_t)data->addr;
vk_barrier.size = size;
VK_CALL(vkCmdPipelineBarrier(vk_command_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_HOST_BIT, 0, 0, NULL, 1, &vk_barrier, 0, NULL));
@@ -718,8 +739,8 @@ static void adapter_vk_copy_bo_address(struct wined3d_context *context,
return;
}
- region.srcOffset = (uintptr_t)src->addr;
- region.dstOffset = (uintptr_t)dst->addr;
+ region.srcOffset = src_bo->buffer_offset + (uintptr_t)src->addr;
+ region.dstOffset = dst_bo->buffer_offset + (uintptr_t)dst->addr;
region.size = size;
vk_barrier[0].sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER;
diff --git a/dlls/wined3d/context_vk.c b/dlls/wined3d/context_vk.c
index 0d90c40fe64..76aafa4c110 100644
--- a/dlls/wined3d/context_vk.c
+++ b/dlls/wined3d/context_vk.c
@@ -110,6 +110,97 @@ struct wined3d_allocator_block *wined3d_context_vk_allocate_memory(struct wined3
return block;
}
+static bool wined3d_context_vk_create_slab_bo(struct wined3d_context_vk *context_vk,
+ VkDeviceSize size, VkBufferUsageFlags usage, VkMemoryPropertyFlags memory_type, struct wined3d_bo_vk *bo)
+{
+ const struct wined3d_adapter_vk *adapter_vk = wined3d_adapter_vk(context_vk->c.device->adapter);
+ const VkPhysicalDeviceLimits *limits = &adapter_vk->device_limits;
+ struct wined3d_bo_slab_vk_key key;
+ struct wined3d_bo_slab_vk *slab;
+ struct wine_rb_entry *entry;
+ size_t object_size, idx;
+ size_t alignment;
+
+ if (size > WINED3D_ALLOCATOR_MIN_BLOCK_SIZE / 2)
+ return false;
+
+ alignment = WINED3D_SLAB_BO_MIN_OBJECT_ALIGN;
+ if ((usage & (VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT))
+ && limits->minTexelBufferOffsetAlignment > alignment)
+ alignment = limits->minTexelBufferOffsetAlignment;
+ if ((usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) && limits->minUniformBufferOffsetAlignment)
+ alignment = limits->minUniformBufferOffsetAlignment;
+ if ((usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) && limits->minStorageBufferOffsetAlignment)
+ alignment = limits->minStorageBufferOffsetAlignment;
+
+ object_size = (size + (alignment - 1)) & ~(alignment - 1);
+ if (object_size < WINED3D_ALLOCATOR_MIN_BLOCK_SIZE / 32)
+ object_size = WINED3D_ALLOCATOR_MIN_BLOCK_SIZE / 32;
+ key.memory_type = memory_type;
+ key.usage = usage;
+ key.size = 32 * object_size;
+
+ if ((entry = wine_rb_get(&context_vk->bo_slab_available, &key)))
+ {
+ slab = WINE_RB_ENTRY_VALUE(entry, struct wined3d_bo_slab_vk, entry);
+ TRACE("Using existing bo slab %p.\n", slab);
+ }
+ else
+ {
+ if (!(slab = heap_alloc_zero(sizeof(*slab))))
+ {
+ ERR("Failed to allocate bo slab.\n");
+ return false;
+ }
+
+ if (!wined3d_context_vk_create_bo(context_vk, key.size, usage, memory_type, &slab->bo))
+ {
+ ERR("Failed to create slab bo.\n");
+ heap_free(slab);
+ return false;
+ }
+ slab->map = ~0u;
+
+ if (wine_rb_put(&context_vk->bo_slab_available, &key, &slab->entry) < 0)
+ {
+ ERR("Failed to add slab to available tree.\n");
+ wined3d_context_vk_destroy_bo(context_vk, &slab->bo);
+ heap_free(slab);
+ return false;
+ }
+
+ TRACE("Created new bo slab %p.\n", slab);
+ }
+
+ idx = wined3d_bit_scan(&slab->map);
+ if (!slab->map)
+ {
+ if (slab->next)
+ {
+ wine_rb_replace(&context_vk->bo_slab_available, &slab->entry, &slab->next->entry);
+ slab->next = NULL;
+ }
+ else
+ {
+ wine_rb_remove(&context_vk->bo_slab_available, &slab->entry);
+ }
+ }
+
+ *bo = slab->bo;
+ bo->memory = NULL;
+ bo->slab = slab;
+ bo->buffer_offset = idx * object_size;
+ bo->memory_offset = slab->bo.memory_offset + bo->buffer_offset;
+ bo->size = size;
+ bo->command_buffer_id = 0;
+
+ TRACE("Using buffer 0x%s, memory 0x%s, offset 0x%s for bo %p.\n",
+ wine_dbgstr_longlong(bo->vk_buffer), wine_dbgstr_longlong(bo->vk_memory),
+ wine_dbgstr_longlong(bo->buffer_offset), bo);
+
+ return true;
+}
+
BOOL wined3d_context_vk_create_bo(struct wined3d_context_vk *context_vk, VkDeviceSize size,
VkBufferUsageFlags usage, VkMemoryPropertyFlags memory_type, struct wined3d_bo_vk *bo)
{
@@ -121,6 +212,9 @@ BOOL wined3d_context_vk_create_bo(struct wined3d_context_vk *context_vk, VkDevic
unsigned int memory_type_idx;
VkResult vr;
+ if (wined3d_context_vk_create_slab_bo(context_vk, size, usage, memory_type, bo))
+ return TRUE;
+
adapter_vk = wined3d_adapter_vk(device_vk->d.adapter);
create_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
@@ -170,8 +264,12 @@ BOOL wined3d_context_vk_create_bo(struct wined3d_context_vk *context_vk, VkDevic
return FALSE;
}
+ bo->buffer_offset = 0;
+ bo->size = size;
+ bo->usage = usage;
bo->memory_type = adapter_vk->memory_properties.memoryTypes[memory_type_idx].propertyFlags;
bo->command_buffer_id = 0;
+ bo->slab = NULL;
TRACE("Created buffer 0x%s, memory 0x%s for bo %p.\n",
wine_dbgstr_longlong(bo->vk_buffer), wine_dbgstr_longlong(bo->vk_memory), bo);
@@ -246,6 +344,56 @@ void wined3d_context_vk_destroy_allocator_block(struct wined3d_context_vk *conte
o->command_buffer_id = command_buffer_id;
}
+static void wined3d_bo_slab_vk_free_slice(struct wined3d_bo_slab_vk *slab,
+ size_t idx, struct wined3d_context_vk *context_vk)
+{
+ struct wined3d_bo_slab_vk_key key;
+ struct wine_rb_entry *entry;
+
+ TRACE("slab %p, idx %zu, context_vk %p.\n", slab, idx, context_vk);
+
+ if (!slab->map)
+ {
+ key.memory_type = slab->bo.memory_type;
+ key.usage = slab->bo.usage;
+ key.size = slab->bo.size;
+
+ if ((entry = wine_rb_get(&context_vk->bo_slab_available, &key)))
+ {
+ slab->next = WINE_RB_ENTRY_VALUE(entry, struct wined3d_bo_slab_vk, entry);
+ wine_rb_replace(&context_vk->bo_slab_available, entry, &slab->entry);
+ }
+ else if (wine_rb_put(&context_vk->bo_slab_available, &key, &slab->entry) < 0)
+ {
+ ERR("Unable to return slab %p (map 0x%08x) to available tree.\n", slab, slab->map);
+ }
+ }
+ slab->map |= 1u << idx;
+}
+
+static void wined3d_context_vk_destroy_bo_slab_slice(struct wined3d_context_vk *context_vk,
+ struct wined3d_bo_slab_vk *slab, size_t idx, uint64_t command_buffer_id)
+{
+ struct wined3d_retired_object_vk *o;
+
+ if (context_vk->completed_command_buffer_id > command_buffer_id)
+ {
+ wined3d_bo_slab_vk_free_slice(slab, idx, context_vk);
+ return;
+ }
+
+ if (!(o = wined3d_context_vk_get_retired_object_vk(context_vk)))
+ {
+ ERR("Leaking slab %p, slice %#zx.\n", slab, idx);
+ return;
+ }
+
+ o->type = WINED3D_RETIRED_BO_SLAB_SLICE_VK;
+ o->u.slice.slab = slab;
+ o->u.slice.idx = idx;
+ o->command_buffer_id = command_buffer_id;
+}
+
static void wined3d_context_vk_destroy_buffer(struct wined3d_context_vk *context_vk,
VkBuffer vk_buffer, uint64_t command_buffer_id)
{
@@ -298,8 +446,18 @@ void wined3d_context_vk_destroy_image(struct wined3d_context_vk *context_vk,
void wined3d_context_vk_destroy_bo(struct wined3d_context_vk *context_vk, const struct wined3d_bo_vk *bo)
{
+ size_t object_size, idx;
+
TRACE("context_vk %p, bo %p.\n", context_vk, bo);
+ if (bo->slab)
+ {
+ object_size = bo->slab->bo.size / 32;
+ idx = bo->buffer_offset / object_size;
+ wined3d_context_vk_destroy_bo_slab_slice(context_vk, bo->slab, idx, bo->command_buffer_id);
+ return;
+ }
+
wined3d_context_vk_destroy_buffer(context_vk, bo->vk_buffer, bo->command_buffer_id);
if (bo->memory)
{
@@ -365,6 +523,10 @@ static void wined3d_context_vk_cleanup_resources(struct wined3d_context_vk *cont
wined3d_allocator_block_free(o->u.block);
break;
+ case WINED3D_RETIRED_BO_SLAB_SLICE_VK:
+ wined3d_bo_slab_vk_free_slice(o->u.slice.slab, o->u.slice.idx, context_vk);
+ break;
+
case WINED3D_RETIRED_BUFFER_VK:
VK_CALL(vkDestroyBuffer(device_vk->vk_device, o->u.vk_buffer, NULL));
TRACE("Destroyed buffer 0x%s.\n", wine_dbgstr_longlong(o->u.vk_buffer));
@@ -392,6 +554,21 @@ static void wined3d_context_vk_cleanup_resources(struct wined3d_context_vk *cont
}
}
+static void wined3d_context_vk_destroy_bo_slab(struct wine_rb_entry *entry, void *ctx)
+{
+ struct wined3d_context_vk *context_vk = ctx;
+ struct wined3d_bo_slab_vk *slab, *next;
+
+ slab = WINE_RB_ENTRY_VALUE(entry, struct wined3d_bo_slab_vk, entry);
+ while (slab)
+ {
+ next = slab->next;
+ wined3d_context_vk_destroy_bo(context_vk, &slab->bo);
+ heap_free(slab);
+ slab = next;
+ }
+}
+
void wined3d_context_vk_cleanup(struct wined3d_context_vk *context_vk)
{
struct wined3d_command_buffer_vk *buffer = &context_vk->current_command_buffer;
@@ -409,6 +586,7 @@ void wined3d_context_vk_cleanup(struct wined3d_context_vk *context_vk)
wined3d_context_vk_wait_command_buffer(context_vk, buffer->id - 1);
context_vk->completed_command_buffer_id = buffer->id;
wined3d_context_vk_cleanup_resources(context_vk);
+ wine_rb_destroy(&context_vk->bo_slab_available, wined3d_context_vk_destroy_bo_slab, context_vk);
heap_free(context_vk->submitted.buffers);
heap_free(context_vk->retired.objects);
@@ -576,6 +754,18 @@ void wined3d_context_vk_image_barrier(struct wined3d_context_vk *context_vk,
VK_CALL(vkCmdPipelineBarrier(vk_command_buffer, src_stage_mask, dst_stage_mask, 0, 0, NULL, 0, NULL, 1, &barrier));
}
+static int wined3d_bo_slab_vk_compare(const void *key, const struct wine_rb_entry *entry)
+{
+ const struct wined3d_bo_slab_vk *slab = WINE_RB_ENTRY_VALUE(entry, const struct wined3d_bo_slab_vk, entry);
+ const struct wined3d_bo_slab_vk_key *k = key;
+
+ if (k->memory_type != slab->bo.memory_type)
+ return k->memory_type - slab->bo.memory_type;
+ if (k->usage != slab->bo.usage)
+ return k->usage - slab->bo.usage;
+ return k->size - slab->bo.size;
+}
+
HRESULT wined3d_context_vk_init(struct wined3d_context_vk *context_vk, struct wined3d_swapchain *swapchain)
{
VkCommandPoolCreateInfo command_pool_info;
@@ -604,5 +794,7 @@ HRESULT wined3d_context_vk_init(struct wined3d_context_vk *context_vk, struct wi
}
context_vk->current_command_buffer.id = 1;
+ wine_rb_init(&context_vk->bo_slab_available, wined3d_bo_slab_vk_compare);
+
return WINED3D_OK;
}
diff --git a/dlls/wined3d/texture.c b/dlls/wined3d/texture.c
index 9adb978b1c7..89d7096f3c1 100644
--- a/dlls/wined3d/texture.c
+++ b/dlls/wined3d/texture.c
@@ -4326,7 +4326,7 @@ static void wined3d_texture_vk_upload_data(struct wined3d_context *context,
dst_texture_vk->layout, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
dst_texture_vk->vk_image, aspect_mask);
- region.bufferOffset = dst_offset;
+ region.bufferOffset = staging_bo.buffer_offset + dst_offset;
region.bufferRowLength = (dst_row_pitch / src_format->block_byte_count) * src_format->block_width;
if (dst_row_pitch)
region.bufferImageHeight = (dst_slice_pitch / dst_row_pitch) * src_format->block_height;
@@ -4461,7 +4461,7 @@ static void wined3d_texture_vk_download_data(struct wined3d_context *context,
src_texture_vk->layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
src_texture_vk->vk_image, aspect_mask);
- region.bufferOffset = 0;
+ region.bufferOffset = staging_bo.buffer_offset;
region.bufferRowLength = 0;
region.bufferImageHeight = 0;
region.imageSubresource.aspectMask = aspect_mask;
diff --git a/dlls/wined3d/wined3d_private.h b/dlls/wined3d/wined3d_private.h
index fee61266c16..244c320734c 100644
--- a/dlls/wined3d/wined3d_private.h
+++ b/dlls/wined3d/wined3d_private.h
@@ -1523,15 +1523,36 @@ struct wined3d_bo_vk
{
VkBuffer vk_buffer;
struct wined3d_allocator_block *memory;
+ struct wined3d_bo_slab_vk *slab;
VkDeviceMemory vk_memory;
+ VkDeviceSize buffer_offset;
VkDeviceSize memory_offset;
+ VkDeviceSize size;
+ VkBufferUsageFlags usage;
VkMemoryPropertyFlags memory_type;
uint64_t command_buffer_id;
};
+struct wined3d_bo_slab_vk_key
+{
+ VkMemoryPropertyFlags memory_type;
+ VkBufferUsageFlags usage;
+ VkDeviceSize size;
+};
+
+struct wined3d_bo_slab_vk
+{
+ struct wine_rb_entry entry;
+ struct wined3d_bo_slab_vk *next;
+ struct wined3d_bo_vk bo;
+ unsigned int map_count;
+ void *map_ptr;
+ uint32_t map;
+};
+
struct wined3d_bo_address
{
UINT_PTR buffer_object;
@@ -2194,6 +2215,7 @@ enum wined3d_retired_object_type_vk
WINED3D_RETIRED_FREE_VK,
WINED3D_RETIRED_MEMORY_VK,
WINED3D_RETIRED_ALLOCATOR_BLOCK_VK,
+ WINED3D_RETIRED_BO_SLAB_SLICE_VK,
WINED3D_RETIRED_BUFFER_VK,
WINED3D_RETIRED_IMAGE_VK,
};
@@ -2206,6 +2228,11 @@ struct wined3d_retired_object_vk
struct wined3d_retired_object_vk *next;
VkDeviceMemory vk_memory;
struct wined3d_allocator_block *block;
+ struct
+ {
+ struct wined3d_bo_slab_vk *slab;
+ size_t idx;
+ } slice;
VkBuffer vk_buffer;
VkImage vk_image;
} u;
@@ -2238,6 +2265,7 @@ struct wined3d_context_vk
} submitted;
struct wined3d_retired_objects_vk retired;
+ struct wine_rb_tree bo_slab_available;
};
static inline struct wined3d_context_vk *wined3d_context_vk(struct wined3d_context *context)
@@ -3448,6 +3476,7 @@ static inline struct wined3d_device_gl *wined3d_device_gl(struct wined3d_device
#define WINED3D_ALLOCATOR_CHUNK_SIZE (64 * 1024 * 1024)
#define WINED3D_ALLOCATOR_CHUNK_ORDER_COUNT 15
#define WINED3D_ALLOCATOR_MIN_BLOCK_SIZE (WINED3D_ALLOCATOR_CHUNK_SIZE >> (WINED3D_ALLOCATOR_CHUNK_ORDER_COUNT - 1))
+#define WINED3D_SLAB_BO_MIN_OBJECT_ALIGN 16
struct wined3d_allocator_chunk
{
diff --git a/include/wine/rbtree.h b/include/wine/rbtree.h
index dc50b5e712f..8aae29c8c10 100644
--- a/include/wine/rbtree.h
+++ b/include/wine/rbtree.h
@@ -389,4 +389,20 @@ static inline void wine_rb_remove_key(struct wine_rb_tree *tree, const void *key
if (entry) wine_rb_remove(tree, entry);
}
+static inline void wine_rb_replace(struct wine_rb_tree *tree, struct wine_rb_entry *dst, struct wine_rb_entry *src)
+{
+ if (!(src->parent = dst->parent))
+ tree->root = src;
+ else if (dst->parent->left == dst)
+ dst->parent->left = src;
+ else
+ dst->parent->right = src;
+
+ if ((src->left = dst->left))
+ src->left->parent = src;
+ if ((src->right = dst->right))
+ src->right->parent = src;
+ src->flags = dst->flags;
+}
+
#endif /* __WINE_WINE_RBTREE_H */
--
2.20.1
More information about the wine-devel
mailing list