wined3d: Correctly display fog for right-handed projection matrix

[Joachim Priesner]

When using a right-handed projection matrix, z coordinates of vertices may be negative. This is not considered in the code generating the GLSL fog shader. The patch fixes that and adds a test.

Tested on openSUSE 13.1 and Windows 8.1.

---
 dlls/d3d9/tests/visual.c   | 60 ++++++++++++++++++++++++++++++++++++++++++++++
 dlls/wined3d/glsl_shader.c |  2 +-
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/dlls/d3d9/tests/visual.c b/dlls/d3d9/tests/visual.c
index 117c8eb..cfefe8d 100644
--- a/dlls/d3d9/tests/visual.c
+++ b/dlls/d3d9/tests/visual.c
@@ -1106,6 +1106,24 @@ static void fog_test(void)
         0.0f, 0.0f,  1.0f, 0.0f,
         0.0f, 0.0f, -1.0f, 1.0f
     }}};
+    // A right-handed perspective projection matrix designed so that vertices
+    // (x in [-1..1], y in [-1..1], z = 0.1) will be mapped to the view plane.
+    // Obtained from D3DXMatrixPerspectiveFovRH with the parameters
+    // fov=84.29deg, aspect=640/480 near=0.01, far=100
+    static const D3DMATRIX proj_mat_rh =
+    {{{
+          0.828732f, 0.0f,       0.0f,       0.0f,
+          0.0f,      1.104976f,  0.0f,       0.0f,
+          0.0f,      0.0f,      -1.0001f,   -1.0f,
+          0.0f,      0.0f,      -0.010001f,  0.0f
+    }}};
+    static const D3DMATRIX view_mat_rh =
+    {{{
+          1.0f, 0.0f,  0.0f, 0.0f,
+          0.0f, 1.0f,  0.0f, 0.0f,
+          0.0f, 0.0f, -1.0f, 0.0f,
+          0.0f, 0.0f,  0.0f, 1.0f
+    }}};
     static const WORD Indices[] = {0, 1, 2, 2, 3, 0};
     static const WORD Indices2[] =
     {
@@ -1221,7 +1239,49 @@ static void fog_test(void)
     }
     IDirect3DDevice9_Present(device, NULL, NULL, NULL, NULL);
 
+    /* Test that fog works correctly even for right-handed projection matrices */
+    hr = IDirect3DDevice9_SetTransform(device, D3DTS_PROJECTION, &proj_mat_rh);
+    ok(SUCCEEDED(hr), "Failed to set projection transform, hr %#x.\n", hr);
+    hr = IDirect3DDevice9_SetTransform(device, D3DTS_VIEW, &view_mat_rh);
+    ok(SUCCEEDED(hr), "Failed to set view transform, hr %#x.\n", hr);
+
+    hr = IDirect3DDevice9_Clear(device, 0, NULL, D3DCLEAR_TARGET, 0xffff00ff, 0.0, 0);
+    ok(hr == D3D_OK, "IDirect3DDevice9_Clear returned %08x\n", hr);
+
+    hr = IDirect3DDevice9_BeginScene(device);
+    ok(SUCCEEDED(hr), "Failed to begin scene, hr %#x.\n", hr);
+
+    start = 0.0;
+    end = 0.1;
+    hr = IDirect3DDevice9_SetRenderState(device, D3DRS_FOGSTART, *((DWORD *)&start));
+    ok(SUCCEEDED(hr), "Failed to set fog start, hr %#x.\n", hr);
+    hr = IDirect3DDevice9_SetRenderState(device, D3DRS_FOGEND, *((DWORD *)&end));
+    ok(SUCCEEDED(hr), "Failed to set fog end, hr %#x.\n", hr);
+
+    hr = IDirect3DDevice9_SetFVF(device, D3DFVF_XYZ | D3DFVF_DIFFUSE | D3DFVF_SPECULAR);
+    ok(SUCCEEDED(hr), "Failed to set FVF, hr %#x.\n", hr);
+    hr = IDirect3DDevice9_SetRenderState(device, D3DRS_FOGVERTEXMODE, D3DFOG_LINEAR);
+    ok(SUCCEEDED(hr), "Failed to set D3DFOG_LINEAR fog vertex mode, hr %#x.\n", hr);
+    hr = IDirect3DDevice9_SetRenderState(device, D3DRS_FOGTABLEMODE, D3DFOG_NONE);
+    ok(SUCCEEDED(hr), "Failed to set D3DFOG_NONE fog table mode, hr %#x.\n", hr);
+
+    hr = IDirect3DDevice9_DrawIndexedPrimitiveUP(device, D3DPT_TRIANGLELIST, 0 /* MinIndex */, 4 /* NumVerts */,
+            2 /* PrimCount */, Indices, D3DFMT_INDEX16, untransformed_1, sizeof(untransformed_1[0]));
+    ok(SUCCEEDED(hr), "Failed to draw, hr %#x.\n", hr);
+
+    hr = IDirect3DDevice9_EndScene(device);
+    ok(SUCCEEDED(hr), "Failed to end scene, hr %#x.\n", hr);
+
+    color = getPixelColor(device, 160, 360);
+    ok(color_match(color, 0x0000ff00, 1), "Untransformed vertex with vertex fog and z = 0.1 has color %08x\n", color);
+    IDirect3DDevice9_Present(device, NULL, NULL, NULL, NULL);
+
     /* Now test the special case fogstart == fogend */
+    hr = IDirect3DDevice9_SetTransform(device, D3DTS_PROJECTION, &ident_mat);
+    ok(SUCCEEDED(hr), "Failed to set projection transform, hr %#x.\n", hr);
+    hr = IDirect3DDevice9_SetTransform(device, D3DTS_VIEW, &ident_mat);
+    ok(SUCCEEDED(hr), "Failed to set view transform, hr %#x.\n", hr);
+
     hr = IDirect3DDevice9_Clear(device, 0, NULL, D3DCLEAR_TARGET, 0xff0000ff, 0.0, 0);
     ok(hr == D3D_OK, "IDirect3DDevice9_Clear returned %08x\n", hr);
 
diff --git a/dlls/wined3d/glsl_shader.c b/dlls/wined3d/glsl_shader.c
index 34cc567..f158ee2 100644
--- a/dlls/wined3d/glsl_shader.c
+++ b/dlls/wined3d/glsl_shader.c
@@ -5005,7 +5005,7 @@ static GLhandleARB shader_glsl_generate_ffp_vertex_shader(struct wined3d_shader_
                 /* Need to undo the [0.0 - 1.0] -> [-1.0 - 1.0] transformation from D3D to GL coordinates. */
                 shader_addline(buffer, "gl_FogFragCoord = gl_Position.z * 0.5 + 0.5;\n");
             else
-                shader_addline(buffer, "gl_FogFragCoord = ec_pos.z;\n");
+                shader_addline(buffer, "gl_FogFragCoord = abs(ec_pos.z);\n");
             break;
 
         default:
-- 
1.8.4.5