[PATCH] wined3d: Fix ATI1 buffer size, try 2

[Patrick Rudolph]

Apitrace crashes on WINE when trying to playback a trace containing ATI1 textures.

Add a tests to show that "unknown" formats like ATI1 are backed by a buffer
of size pitch * height. Without the attaced fix the tests will crash.

Calculate correct buffer size in wined3d_format_calculate_pitch for formats that
have WINED3DFMT_FLAG_BROKEN_PITCH flag set.
Remove no longer needed check in texture_resource_sub_resource_map.

Signed-off-by: Patrick Rudolph <siro at das-labor.org>
---
 dlls/d3d9/tests/device.c |  8 ++++++++
 dlls/wined3d/texture.c   | 10 +---------
 dlls/wined3d/utils.c     |  9 ++++++++-
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/dlls/d3d9/tests/device.c b/dlls/d3d9/tests/device.c
index fbafa14..7f61560 100644
--- a/dlls/d3d9/tests/device.c
+++ b/dlls/d3d9/tests/device.c
@@ -11049,6 +11049,8 @@ static void test_miptree_layout(void)
         {D3DFMT_A8R8G8B8, "D3DFMT_A8R8G8B8"},
         {D3DFMT_A8,       "D3DFMT_A8"},
         {D3DFMT_L8,       "D3DFMT_L8"},
+        {MAKEFOURCC('A','T','I','1'), "D3DFMT_ATI1"},
+        {MAKEFOURCC('A','T','I','2'), "D3DFMT_ATI2"},
     };
     static const struct
     {
@@ -11111,6 +11113,9 @@ static void test_miptree_layout(void)
                     ok(map_desc.pBits == base + offset,
                             "%s, %s, level %u: Got unexpected pBits %p, expected %p.\n",
                             pools[pool_idx].name, formats[format_idx].name, i, map_desc.pBits, base + offset);
+
+                memset(base, 0, (base_dimension >> i) * (base_dimension >> i));
+
                 offset += (base_dimension >> i) * map_desc.Pitch;
 
                 hr = IDirect3DTexture9_UnlockRect(texture_2d, i);
@@ -11150,6 +11155,9 @@ static void test_miptree_layout(void)
                         ok(map_desc.pBits == base + offset,
                                 "%s, %s, face %u, level %u: Got unexpected pBits %p, expected %p.\n",
                                 pools[pool_idx].name, formats[format_idx].name, i, j, map_desc.pBits, base + offset);
+
+                    memset(base, 0, (base_dimension >> j) * (base_dimension >> j));
+
                     offset += (base_dimension >> j) * map_desc.Pitch;
 
                     hr = IDirect3DCubeTexture9_UnlockRect(texture_cube, i, j);
diff --git a/dlls/wined3d/texture.c b/dlls/wined3d/texture.c
index d2b1be5..20e53c1 100644
--- a/dlls/wined3d/texture.c
+++ b/dlls/wined3d/texture.c
@@ -1741,15 +1741,7 @@ static HRESULT texture_resource_sub_resource_map(struct wined3d_resource *resour
     if (context)
         context_release(context);
 
-    if (fmt_flags & WINED3DFMT_FLAG_BROKEN_PITCH)
-    {
-        map_desc->row_pitch = wined3d_texture_get_level_width(texture, texture_level) * format->byte_count;
-        map_desc->slice_pitch = wined3d_texture_get_level_height(texture, texture_level) * map_desc->row_pitch;
-    }
-    else
-    {
-        wined3d_texture_get_pitch(texture, texture_level, &map_desc->row_pitch, &map_desc->slice_pitch);
-    }
+    wined3d_texture_get_pitch(texture, texture_level, &map_desc->row_pitch, &map_desc->slice_pitch);
 
     if (!box)
     {
diff --git a/dlls/wined3d/utils.c b/dlls/wined3d/utils.c
index f02220c..4def590 100644
--- a/dlls/wined3d/utils.c
+++ b/dlls/wined3d/utils.c
@@ -3518,9 +3518,16 @@ const struct wined3d_format *wined3d_get_format(const struct wined3d_gl_info *gl
 void wined3d_format_calculate_pitch(const struct wined3d_format *format, unsigned int alignment,
         unsigned int width, unsigned int height, unsigned int *row_pitch, unsigned int *slice_pitch)
 {
+    /* For "unknown" formats like ATIx the pitch equals width.
+     * Tests show that width * height bytes of memory are allocated. */
+    if (format->flags[WINED3D_GL_RES_TYPE_TEX_2D] & WINED3DFMT_FLAG_BROKEN_PITCH)
+    {
+        *row_pitch = width;
+        *slice_pitch = *row_pitch * height;
+    }
     /* For block based formats, pitch means the amount of bytes to the next
      * row of blocks rather than the next row of pixels. */
-    if (format->flags[WINED3D_GL_RES_TYPE_TEX_2D] & WINED3DFMT_FLAG_BLOCKS)
+    else if (format->flags[WINED3D_GL_RES_TYPE_TEX_2D] & WINED3DFMT_FLAG_BLOCKS)
     {
         unsigned int row_block_count = (width + format->block_width - 1) / format->block_width;
         unsigned int slice_block_count = (height + format->block_height - 1) / format->block_height;
-- 
2.7.4