[PATCH vkd3d 02/10] vkd3d: Clamp RTV/DSV array sizes to their real array lengths.

[Conor McCarthy]

Avoids overflowing vkCreateFramebuffer() later with UINT_MAX layers.

Based on a vkd3d-proton patch by Hans-Kristian Arntzen.

Signed-off-by: Conor McCarthy <cmccarthy at codeweavers.com>
---
 libs/vkd3d/resource.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/libs/vkd3d/resource.c b/libs/vkd3d/resource.c
index 1ca23a90..a562130d 100644
--- a/libs/vkd3d/resource.c
+++ b/libs/vkd3d/resource.c
@@ -3154,6 +3154,7 @@ void d3d12_rtv_desc_create_rtv(struct d3d12_rtv_desc *rtv_desc, struct d3d12_dev
         {
             case D3D12_RTV_DIMENSION_TEXTURE2D:
                 vkd3d_desc.miplevel_idx = desc->u.Texture2D.MipSlice;
+                vkd3d_desc.layer_count = 1;
                 if (desc->u.Texture2D.PlaneSlice)
                     FIXME("Ignoring plane slice %u.\n", desc->u.Texture2D.PlaneSlice);
                 break;
@@ -3167,6 +3168,7 @@ void d3d12_rtv_desc_create_rtv(struct d3d12_rtv_desc *rtv_desc, struct d3d12_dev
                 break;
             case D3D12_RTV_DIMENSION_TEXTURE2DMS:
                 vkd3d_desc.view_type = VK_IMAGE_VIEW_TYPE_2D;
+                vkd3d_desc.layer_count = 1;
                 break;
             case D3D12_RTV_DIMENSION_TEXTURE2DMSARRAY:
                 vkd3d_desc.view_type = VK_IMAGE_VIEW_TYPE_2D_ARRAY;
@@ -3182,6 +3184,9 @@ void d3d12_rtv_desc_create_rtv(struct d3d12_rtv_desc *rtv_desc, struct d3d12_dev
             default:
                 FIXME("Unhandled view dimension %#x.\n", desc->ViewDimension);
         }
+
+        /* Avoid passing down UINT_MAX here since that makes framebuffer logic later rather awkward. */
+        vkd3d_desc.layer_count = min(vkd3d_desc.layer_count, resource->desc.DepthOrArraySize - vkd3d_desc.layer_idx);
     }
     else if (resource->desc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE3D)
     {
@@ -3253,6 +3258,7 @@ void d3d12_dsv_desc_create_dsv(struct d3d12_dsv_desc *dsv_desc, struct d3d12_dev
         {
             case D3D12_DSV_DIMENSION_TEXTURE2D:
                 vkd3d_desc.miplevel_idx = desc->u.Texture2D.MipSlice;
+                vkd3d_desc.layer_count = 1;
                 break;
             case D3D12_DSV_DIMENSION_TEXTURE2DARRAY:
                 vkd3d_desc.view_type = VK_IMAGE_VIEW_TYPE_2D_ARRAY;
@@ -3262,6 +3268,7 @@ void d3d12_dsv_desc_create_dsv(struct d3d12_dsv_desc *dsv_desc, struct d3d12_dev
                 break;
             case D3D12_DSV_DIMENSION_TEXTURE2DMS:
                 vkd3d_desc.view_type = VK_IMAGE_VIEW_TYPE_2D;
+                vkd3d_desc.layer_count = 1;
                 break;
             case D3D12_DSV_DIMENSION_TEXTURE2DMSARRAY:
                 vkd3d_desc.view_type = VK_IMAGE_VIEW_TYPE_2D_ARRAY;
@@ -3271,6 +3278,9 @@ void d3d12_dsv_desc_create_dsv(struct d3d12_dsv_desc *dsv_desc, struct d3d12_dev
             default:
                 FIXME("Unhandled view dimension %#x.\n", desc->ViewDimension);
         }
+
+        /* Avoid passing down UINT_MAX here since that makes framebuffer logic later rather awkward. */
+        vkd3d_desc.layer_count = min(vkd3d_desc.layer_count, resource->desc.DepthOrArraySize - vkd3d_desc.layer_idx);
     }
 
     assert(d3d12_resource_is_texture(resource));
-- 
2.32.0