[PATCH] oleaut32: Fix some corner cases in VarBstrCmp. (try 2)

[Lei Zhang]

Hi,

This should fix bug 14703. Took a while to resubmit this patch. Here's
where we left off:
http://www.winehq.org/pipermail/wine-devel/2008-August/068044.html

- Lei
-------------- next part --------------
From 6a7dc81b93cbd9579eae3c545310d0c2877546c8 Mon Sep 17 00:00:00 2001
From: Lei Zhang <thestig at google.com>
Date: Fri, 20 Mar 2009 15:37:07 -0700
Subject: [PATCH] oleaut32: Fix some corner cases in VarBstrCmp.

---
 dlls/oleaut32/tests/vartype.c |   20 ++++++++++++++++++++
 dlls/oleaut32/vartype.c       |   17 ++++++++++++-----
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/dlls/oleaut32/tests/vartype.c b/dlls/oleaut32/tests/vartype.c
index eba0bc1..0b4f301 100644
--- a/dlls/oleaut32/tests/vartype.c
+++ b/dlls/oleaut32/tests/vartype.c
@@ -4983,6 +4983,8 @@ static void test_VarBstrCmp(void)
     static const WCHAR s2[] = { 'a',0,'b' };
     static const char sb1[] = {1,0,1};
     static const char sb2[] = {1,0,2};
+    static const char sbchr0[] = {0,0};
+    static const char sbchr00[] = {0,0,0};
     BSTR bstr, bstrempty, bstr2;
 
     CHECKPTR(VarBstrCmp);
@@ -5021,6 +5023,15 @@ static void test_VarBstrCmp(void)
 
     SysFreeString(bstr);
 
+    bstr = SysAllocStringByteLen(sbchr0, sizeof(sbchr0));
+    bstr2 = SysAllocStringByteLen(sbchr0, sizeof(sbchr00));
+    VARBSTRCMP(bstr,bstrempty,0,VARCMP_GT);
+    VARBSTRCMP(bstrempty,bstr,0,VARCMP_LT);
+    VARBSTRCMP(bstr2,bstrempty,0,VARCMP_GT);
+    VARBSTRCMP(bstr2,bstr,0,VARCMP_EQ);
+    SysFreeString(bstr2);
+    SysFreeString(bstr);
+
     /* When (LCID == 0) it should be a binary comparison
      * so these two strings could not match.
      */
@@ -5030,6 +5041,15 @@ static void test_VarBstrCmp(void)
     VARBSTRCMP(bstr,bstr2,0,VARCMP_LT);
     SysFreeString(bstr2);
     SysFreeString(bstr);
+
+    bstr = SysAllocStringByteLen(sbchr0, sizeof(sbchr0));
+    bstr2 = SysAllocStringByteLen(sbchr0, sizeof(sbchr00));
+    VARBSTRCMP(bstr,bstrempty,0,VARCMP_GT);
+    VARBSTRCMP(bstrempty,bstr,0,VARCMP_LT);
+    VARBSTRCMP(bstr2,bstrempty,0,VARCMP_GT);
+    VARBSTRCMP(bstr2,bstr,0,VARCMP_GT);
+    SysFreeString(bstr2);
+    SysFreeString(bstr);
 }
 
 /* Get the internal representation of a BSTR */
diff --git a/dlls/oleaut32/vartype.c b/dlls/oleaut32/vartype.c
index 6d972a2..8de39c3 100644
--- a/dlls/oleaut32/vartype.c
+++ b/dlls/oleaut32/vartype.c
@@ -6957,9 +6957,8 @@ HRESULT WINAPI VarBstrCmp(BSTR pbstrLeft, BSTR pbstrRight, LCID lcid, DWORD dwFl
 
     if (!pbstrLeft || !*pbstrLeft)
     {
-      if (!pbstrRight || !*pbstrRight)
-        return VARCMP_EQ;
-      return VARCMP_LT;
+      if (pbstrRight && *pbstrRight)
+        return VARCMP_LT;
     }
     else if (!pbstrRight || !*pbstrRight)
         return VARCMP_GT;
@@ -6981,8 +6980,16 @@ HRESULT WINAPI VarBstrCmp(BSTR pbstrLeft, BSTR pbstrRight, LCID lcid, DWORD dwFl
     }
     else
     {
-      hres = CompareStringW(lcid, dwFlags, pbstrLeft, SysStringLen(pbstrLeft),
-              pbstrRight, SysStringLen(pbstrRight)) - 1;
+      unsigned int lenLeft = SysStringLen(pbstrLeft);
+      unsigned int lenRight = SysStringLen(pbstrRight);
+
+      if (lenLeft == 0 && lenRight == 0)
+          return VARCMP_EQ;
+      if (lenLeft == 0 || lenRight == 0)
+        return lenLeft < lenRight ? VARCMP_LT : VARCMP_GT;
+
+      hres = CompareStringW(lcid, dwFlags, pbstrLeft, lenLeft,
+              pbstrRight, lenRight) - 1;
       TRACE("%d\n", hres);
       return hres;
     }
-- 
1.5.4.3