Stefan Dösinger : d3d8: Reject broken normal declaration types with fixed function.

[Alexandre Julliard]

Module: wine
Branch: master
Commit: 0fa0df7843d4b11eaee9f0611944f10774d4fbca
URL:    http://source.winehq.org/git/wine.git/?a=commit;h=0fa0df7843d4b11eaee9f0611944f10774d4fbca

Author: Stefan Dösinger <stefan at codeweavers.com>
Date:   Thu Mar 27 20:11:36 2008 +0100

d3d8: Reject broken normal declaration types with fixed function.

---

 dlls/d3d8/d3d8_private.h      |    1 +
 dlls/d3d8/device.c            |   17 +++++++++++++++++
 dlls/d3d8/tests/device.c      |   38 ++++++++++++++++++++++++++++++++++++++
 dlls/d3d8/vertexdeclaration.c |    2 +-
 4 files changed, 57 insertions(+), 1 deletions(-)

diff --git a/dlls/d3d8/d3d8_private.h b/dlls/d3d8/d3d8_private.h
index 2601b47..0da18ca 100644
--- a/dlls/d3d8/d3d8_private.h
+++ b/dlls/d3d8/d3d8_private.h
@@ -635,6 +635,7 @@ typedef struct IDirect3DPixelShader8Impl {
  */
 void load_local_constants(const DWORD *d3d8_elements, IWineD3DVertexShader *wined3d_vertex_shader);
 UINT convert_to_wined3d_declaration(const DWORD *d3d8_elements, DWORD *d3d8_elements_size, WINED3DVERTEXELEMENT **wined3d_elements);
+size_t parse_token(const DWORD* pToken);
 
 /* Callbacks */
 extern HRESULT WINAPI D3D8CB_CreateSurface(IUnknown *device, IUnknown *pSuperior, UINT Width, UINT Height,
diff --git a/dlls/d3d8/device.c b/dlls/d3d8/device.c
index 43e748e..a846bf6 100644
--- a/dlls/d3d8/device.c
+++ b/dlls/d3d8/device.c
@@ -1522,6 +1522,23 @@ static HRESULT WINAPI IDirect3DDevice8Impl_CreateVertexShader(LPDIRECT3DDEVICE8
     HRESULT hrc = D3D_OK;
     IDirect3DVertexShader8Impl *object;
     IWineD3DVertexDeclaration *wined3d_vertex_declaration;
+    const DWORD *token = pDeclaration;
+
+    /* Test if the vertex declaration is valid */
+    while (D3DVSD_END() != *token) {
+        D3DVSD_TOKENTYPE token_type = ((*token & D3DVSD_TOKENTYPEMASK) >> D3DVSD_TOKENTYPESHIFT);
+
+        if (token_type == D3DVSD_TOKEN_STREAMDATA && !(token_type & 0x10000000)) {
+            DWORD type = ((*token & D3DVSD_DATATYPEMASK) >> D3DVSD_DATATYPESHIFT);
+            DWORD reg  = ((*token & D3DVSD_VERTEXREGMASK) >> D3DVSD_VERTEXREGSHIFT);
+
+            if(reg == D3DVSDE_NORMAL && type != D3DVSDT_FLOAT3 && !pFunction) {
+                WARN("Attempt to use a non-FLOAT3 normal with the fixed function function\n");
+                return D3DERR_INVALIDCALL;
+            }
+        }
+        token += parse_token(token);
+    }
 
     /* Setup a stub object for now */
     object = HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, sizeof(*object));
diff --git a/dlls/d3d8/tests/device.c b/dlls/d3d8/tests/device.c
index 0dd4e92..8de4ec7 100644
--- a/dlls/d3d8/tests/device.c
+++ b/dlls/d3d8/tests/device.c
@@ -880,6 +880,27 @@ static void test_shader(void)
         D3DVSD_REG(D3DVSDE_POSITION,  D3DVSDT_FLOAT3),
         D3DVSD_END()
     };
+    DWORD decl_normal_float2[] =
+    {
+        D3DVSD_STREAM(0),
+        D3DVSD_REG(D3DVSDE_POSITION, D3DVSDT_FLOAT3),  /* D3DVSDE_POSITION, Register v0 */
+        D3DVSD_REG(D3DVSDE_NORMAL,   D3DVSDT_FLOAT2),  /* D3DVSDE_NORMAL,   Register v1 */
+        D3DVSD_END()
+    };
+    DWORD decl_normal_float4[] =
+    {
+        D3DVSD_STREAM(0),
+        D3DVSD_REG(D3DVSDE_POSITION, D3DVSDT_FLOAT3),  /* D3DVSDE_POSITION, Register v0 */
+        D3DVSD_REG(D3DVSDE_NORMAL,   D3DVSDT_FLOAT4),  /* D3DVSDE_NORMAL,   Register v1 */
+        D3DVSD_END()
+    };
+    DWORD decl_normal_d3dcolor[] =
+    {
+        D3DVSD_STREAM(0),
+        D3DVSD_REG(D3DVSDE_POSITION, D3DVSDT_FLOAT3),  /* D3DVSDE_POSITION, Register v0 */
+        D3DVSD_REG(D3DVSDE_NORMAL,   D3DVSDT_D3DCOLOR),/* D3DVSDE_NORMAL,   Register v1 */
+        D3DVSD_END()
+    };
     const DWORD vertex_decl_size = sizeof(dwVertexDecl);
     const DWORD simple_vs_size = sizeof(simple_vs);
     const DWORD simple_ps_size = sizeof(simple_ps);
@@ -962,6 +983,23 @@ static void test_shader(void)
     ok(hr == D3D_OK, "IDirect3DDevice8_GetVertexShader returned %#08x\n", hr);
     ok(hTempHandle == 0, "Vertex Shader %d is set, expected shader %d\n", hTempHandle, 0);
 
+    /* Test a broken declaration. 3DMark2001 tries to use normals with 2 components
+     * First try the fixed function shader function, then a custom one
+     */
+    hr = IDirect3DDevice8_CreateVertexShader(pDevice, decl_normal_float2, 0, &hVertexShader, 0);
+    ok(hr == D3DERR_INVALIDCALL, "IDirect3DDevice8_CreateVertexShader returned %#08x\n", hr);
+    if(SUCCEEDED(hr)) IDirect3DDevice8_DeleteVertexShader(pDevice, hVertexShader);
+    hr = IDirect3DDevice8_CreateVertexShader(pDevice, decl_normal_float4, 0, &hVertexShader, 0);
+    ok(hr == D3DERR_INVALIDCALL, "IDirect3DDevice8_CreateVertexShader returned %#08x\n", hr);
+    if(SUCCEEDED(hr)) IDirect3DDevice8_DeleteVertexShader(pDevice, hVertexShader);
+    hr = IDirect3DDevice8_CreateVertexShader(pDevice, decl_normal_d3dcolor, 0, &hVertexShader, 0);
+    ok(hr == D3DERR_INVALIDCALL, "IDirect3DDevice8_CreateVertexShader returned %#08x\n", hr);
+    if(SUCCEEDED(hr)) IDirect3DDevice8_DeleteVertexShader(pDevice, hVertexShader);
+
+    hr = IDirect3DDevice8_CreateVertexShader(pDevice, decl_normal_float2, simple_vs, &hVertexShader, 0);
+    ok(hr == D3D_OK, "IDirect3DDevice8_CreateVertexShader returned %#08x\n", hr);
+    if(SUCCEEDED(hr)) IDirect3DDevice8_DeleteVertexShader(pDevice, hVertexShader);
+
     if (caps.PixelShaderVersion >= D3DPS_VERSION(1, 0))
     {
         /* The same with a pixel shader */
diff --git a/dlls/d3d8/vertexdeclaration.c b/dlls/d3d8/vertexdeclaration.c
index 57d0337..debeca0 100644
--- a/dlls/d3d8/vertexdeclaration.c
+++ b/dlls/d3d8/vertexdeclaration.c
@@ -118,7 +118,7 @@ static const char *debug_d3dvsde_register(D3DVSDE_REGISTER d3dvsde_register)
     }
 }
 
-static size_t parse_token(const DWORD* pToken)
+size_t parse_token(const DWORD* pToken)
 {
     const DWORD token = *pToken;
     size_t tokenlen = 1;