[PATCH 3/4] wined3d: Add a slop for color keying.

[Stefan Dösinger]

This fixes the O2Jam Part of bug 39014.

Yeah, the 0.003 is terribly large, but it takes care of the imprecision
of all cards without using GL_RGB_565 while not breaking any other
format/value combination. The exception are r300 and r600g where GL_RGB5
produces false positive and false negative matches at the same time no
matter what slop we use, but on these cards we have GL_RGB_565 so it
doesn't matter.

i965 on Linux supports GL_ARB_ES2_compatibility, but GL_RGB_565 is still
awful. The slop added by this patch makes color keying work for all
values though. The newer Intel HD 4000 has proper 565 support. The same
applies to Geforce 7 GPUs with the blob.

Tested on r500 (OSX, r300g), Geforce 7 (Linux blob), Geforce GT 650M
(Linux blob, OSX), Evergreen (r600g), Intel HD4000 (OSX, Mesa), Intel
GMA 965 (OSX, Mesa).

Not tested on r200, we don't do shader color keying in atifs.
---
 dlls/wined3d/arb_program_shader.c | 4 ++++
 dlls/wined3d/glsl_shader.c        | 3 ++-
 dlls/wined3d/shader.c             | 8 ++++++++
 dlls/wined3d/wined3d_private.h    | 1 +
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/dlls/wined3d/arb_program_shader.c b/dlls/wined3d/arb_program_shader.c
index 378900e..64fbd9f 100644
--- a/dlls/wined3d/arb_program_shader.c
+++ b/dlls/wined3d/arb_program_shader.c
@@ -6493,6 +6493,8 @@ static GLuint gen_arbfp_ffp_shader(const struct ffp_frag_settings *settings, con
     if (settings->color_key_enabled)
     {
         shader_addline(&buffer, "SUB TMP, tex0, color_key;\n");
+        shader_addline(&buffer, "ABS TMP, TMP;\n");
+        shader_addline(&buffer, "SGE TMP, TMP, %s;\n", wined3d_color_key_precision);
         shader_addline(&buffer, "DP4 TMP.b, TMP, TMP;\n");
         shader_addline(&buffer, "SGE TMP, -TMP.b, 0.0;\n");
         shader_addline(&buffer, "KIL -TMP;\n");
@@ -7590,6 +7592,8 @@ static GLuint arbfp_gen_plain_shader(struct arbfp_blit_priv *priv,
         shader_addline(&buffer, "PARAM color_key = program.local[%u];\n", ARBFP_BLIT_PARAM_COLOR_KEY);
         shader_addline(&buffer, "TEX color, fragment.texcoord[0], texture[0], %s;\n", tex_target);
         shader_addline(&buffer, "SUB compare, color, color_key;\n");
+        shader_addline(&buffer, "ABS compare, compare;\n");
+        shader_addline(&buffer, "SGE compare, compare, %s;\n", wined3d_color_key_precision);
         shader_addline(&buffer, "DP4 compare.r, compare, compare;\n");
         shader_addline(&buffer, "SGE compare, -compare.r, 0.0;\n");
         shader_addline(&buffer, "KIL -compare;\n");
diff --git a/dlls/wined3d/glsl_shader.c b/dlls/wined3d/glsl_shader.c
index 7bcf102..e1a464b 100644
--- a/dlls/wined3d/glsl_shader.c
+++ b/dlls/wined3d/glsl_shader.c
@@ -6332,7 +6332,8 @@ static GLuint shader_glsl_generate_ffp_fragment_shader(struct shader_glsl_priv *
     }
 
     if (settings->color_key_enabled)
-        shader_addline(buffer, "if (all(equal(tex0, color_key))) discard;\n");
+        shader_addline(buffer, "if (all(lessThan(abs(tex0 - color_key), vec4(%s)))) discard;\n",
+                wined3d_color_key_precision);
 
     /* Generate the main shader */
     for (stage = 0; stage < MAX_TEXTURES; ++stage)
diff --git a/dlls/wined3d/shader.c b/dlls/wined3d/shader.c
index e9be51e..8c3535c 100644
--- a/dlls/wined3d/shader.c
+++ b/dlls/wined3d/shader.c
@@ -37,6 +37,14 @@ WINE_DEFAULT_DEBUG_CHANNEL(d3d_shader);
 const float wined3d_srgb_const0[] = {0.41666f, 1.055f, 0.055f, 12.92f};
 /* cmp */
 const float wined3d_srgb_const1[] = {0.0031308f, 0.0f, 0.0f, 0.0f};
+/* Normalization of B5G6R5_UNORM textures is horribly imprecise if we don't have
+ * GL_RGB_565 support. 1 / 256 (~0.0039) works in practice, but is awfully close
+ * to the next possible value in 8 bit formats. 1 / 384 is too precise for some
+ * 5 and 6 bit channel values at least on Nvidia.
+ *
+ * An exact comparison isn't reliable for any format, except for the normalized
+ * values 0.0 and 1.0. */
+const char *wined3d_color_key_precision = "0.003";
 
 static const char * const shader_opcode_names[] =
 {
diff --git a/dlls/wined3d/wined3d_private.h b/dlls/wined3d/wined3d_private.h
index 7364728..596a4ba 100644
--- a/dlls/wined3d/wined3d_private.h
+++ b/dlls/wined3d/wined3d_private.h
@@ -1887,6 +1887,7 @@ void wined3d_ftoa(float value, char *s) DECLSPEC_HIDDEN;
 
 extern const float wined3d_srgb_const0[] DECLSPEC_HIDDEN;
 extern const float wined3d_srgb_const1[] DECLSPEC_HIDDEN;
+extern const char *wined3d_color_key_precision DECLSPEC_HIDDEN;
 
 enum wined3d_ffp_vs_fog_mode
 {
-- 
2.4.6