wined3d: Reserve two varyings for ATI DX10 cards

[Robert Key]

Changelog:
* This reserves 2 varyings for gl_FrontColor and gl_FrontSecondaryColor
   for ATI DX10 cards. This should fix bug #17437 once the new fglrx
   drivers get released this week.
-------------- next part --------------
From 5255421f6b43b8481f91b776b18d5e37a2b66223 Mon Sep 17 00:00:00 2001
From: Robert Key <hpestilence at gmail.com>
Date: Mon, 15 Jun 2009 11:28:01 -0400
Subject: wined3d: Reserve two varyings for ATI DX10 cards

---
 dlls/wined3d/directx.c     |   15 +++++++++++++++
 dlls/wined3d/glsl_shader.c |    3 +++
 dlls/wined3d/wined3d_gl.h  |    1 +
 3 files changed, 19 insertions(+), 0 deletions(-)

diff --git a/dlls/wined3d/directx.c b/dlls/wined3d/directx.c
index 05443fc..47b3b3a 100644
--- a/dlls/wined3d/directx.c
+++ b/dlls/wined3d/directx.c
@@ -4036,6 +4036,12 @@ static BOOL match_ati_r300_to_500(const WineD3D_GL_Info *gl_info) {
     return FALSE;
 }
 
+static BOOL match_ati_r600_to_700(const WineD3D_GL_Info *gl_info) {
+    if(gl_info->gl_vendor != VENDOR_ATI) return FALSE;
+    if(strstr(gl_info->gl_renderer, "DRI")) return FALSE; /* Filter out Mesa DRI drivers */
+    return gl_info->max_glsl_varyings > 44;
+}
+
 static BOOL match_geforce5(const WineD3D_GL_Info *gl_info) {
     if(gl_info->gl_vendor == VENDOR_NVIDIA) {
         if(gl_info->gl_card == CARD_NVIDIA_GEFORCEFX_5800 || gl_info->gl_card == CARD_NVIDIA_GEFORCEFX_5600) {
@@ -4138,6 +4144,10 @@ static void quirk_ati_dx9(WineD3D_GL_Info *gl_info) {
     gl_info->reserved_glsl_constants = max(gl_info->reserved_glsl_constants, 8);
 }
 
+static void quirk_ati_dx10(WineD3D_GL_Info *gl_info) {
+    gl_info->glsl_ati_reserve_varyings = TRUE;
+}
+
 static void quirk_no_np2(WineD3D_GL_Info *gl_info) {
     /*  The nVidia GeForceFX series reports OpenGL 2.0 capabilities with the latest drivers versions, but
      *  doesn't explicitly advertise the ARB_tex_npot extension in the GL extension string.
@@ -4188,6 +4198,11 @@ struct driver_quirk quirk_table[] = {
         quirk_ati_dx9,
         "ATI GLSL constant and normalized texrect quirk"
     },
+    {
+        match_ati_r600_to_700,
+        quirk_ati_dx10,
+        "ATI GLSL constant and varying quirk"
+    },
     /* MacOS advertises more GLSL vertex shader uniforms than supported by the hardware, and if more are
      * used it falls back to software. While the compiler can detect if the shader uses all declared
      * uniforms, the optimization fails if the shader uses relative addressing. So any GLSL shader
diff --git a/dlls/wined3d/glsl_shader.c b/dlls/wined3d/glsl_shader.c
index 846044f..207b99b 100644
--- a/dlls/wined3d/glsl_shader.c
+++ b/dlls/wined3d/glsl_shader.c
@@ -767,6 +767,9 @@ static int vec4_varyings(DWORD shader_major, const WineD3D_GL_Info *gl_info)
     /* 4.0 shaders do not write clip coords because d3d10 does not support user clipplanes */
     if(shader_major > 3) return ret;
 
+    /* ATI DX10 cards have to reserve 2 varyings for gl_FrontColor and gl_FrontSecondaryColor */
+    if(gl_info->glsl_ati_reserve_varyings) ret -= 2;
+
     /* 3.0 shaders may need an extra varying for the clip coord on some cards(mostly dx10 ones) */
     if(gl_info->glsl_clip_varying) ret -= 1;
     return ret;
diff --git a/dlls/wined3d/wined3d_gl.h b/dlls/wined3d/wined3d_gl.h
index aa59893..30adc12 100644
--- a/dlls/wined3d/wined3d_gl.h
+++ b/dlls/wined3d/wined3d_gl.h
@@ -3993,6 +3993,7 @@ typedef struct _WineD3D_GL_Info {
   BOOL set_texcoord_w;
   DWORD reserved_glsl_constants;
   BOOL glsl_clip_varying;
+  BOOL glsl_ati_reserve_varyings;
 
   BOOL supported[WINED3D_GL_EXT_COUNT];
 
-- 
1.6.3.1