Nikolay Sivov : rsaenh: Fail on unsupported flag values only in CryptHashData().

[Alexandre Julliard]

Module: wine
Branch: master
Commit: bfd2c533beef454a61b24f92790f91099e607365
URL:    http://source.winehq.org/git/wine.git/?a=commit;h=bfd2c533beef454a61b24f92790f91099e607365

Author: Nikolay Sivov <nsivov at codeweavers.com>
Date:   Thu Oct 17 22:36:36 2013 +0400

rsaenh: Fail on unsupported flag values only in CryptHashData().

---

 dlls/rsaenh/rsaenh.c       |    2 +-
 dlls/rsaenh/tests/rsaenh.c |   12 +++++++++---
 include/wincrypt.h         |    2 ++
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/dlls/rsaenh/rsaenh.c b/dlls/rsaenh/rsaenh.c
index 7d4cdb9..b5ab7a9 100644
--- a/dlls/rsaenh/rsaenh.c
+++ b/dlls/rsaenh/rsaenh.c
@@ -4114,7 +4114,7 @@ BOOL WINAPI RSAENH_CPHashData(HCRYPTPROV hProv, HCRYPTHASH hHash, const BYTE *pb
     TRACE("(hProv=%08lx, hHash=%08lx, pbData=%p, dwDataLen=%d, dwFlags=%08x)\n",
           hProv, hHash, pbData, dwDataLen, dwFlags);
 
-    if (dwFlags)
+    if (dwFlags & ~CRYPT_USERDATA)
     {
         SetLastError(NTE_BAD_FLAGS);
         return FALSE;
diff --git a/dlls/rsaenh/tests/rsaenh.c b/dlls/rsaenh/tests/rsaenh.c
index f43a4e4..84505f8 100644
--- a/dlls/rsaenh/tests/rsaenh.c
+++ b/dlls/rsaenh/tests/rsaenh.c
@@ -446,7 +446,10 @@ static void test_hashes(void)
     result = CryptCreateHash(hProv, CALG_MD4, 0, 0, &hHash);
     ok(result, "%08x\n", GetLastError());
 
-    result = CryptHashData(hHash, pbData, sizeof(pbData), 0);
+    result = CryptHashData(hHash, pbData, sizeof(pbData), ~0);
+    ok(!result && GetLastError() == NTE_BAD_FLAGS, "%08x\n", GetLastError());
+
+    result = CryptHashData(hHash, pbData, sizeof(pbData), CRYPT_USERDATA);
     ok(result, "%08x\n", GetLastError());
 
     len = sizeof(DWORD);
@@ -470,7 +473,10 @@ static void test_hashes(void)
     result = CryptGetHashParam(hHash, HP_HASHSIZE, (BYTE*)&hashlen, &len, 0);
     ok(result && (hashlen == 16), "%08x, hashlen: %d\n", GetLastError(), hashlen);
 
-    result = CryptHashData(hHash, pbData, sizeof(pbData), 0);
+    result = CryptHashData(hHash, pbData, sizeof(pbData), ~0);
+    ok(!result && GetLastError() == NTE_BAD_FLAGS, "%08x\n", GetLastError());
+
+    result = CryptHashData(hHash, pbData, sizeof(pbData), CRYPT_USERDATA);
     ok(result, "%08x\n", GetLastError());
 
     len = 16;
@@ -519,7 +525,7 @@ static void test_hashes(void)
     result = CryptCreateHash(hProv, CALG_SHA, 0, 0, &hHash);
     ok(result, "%08x\n", GetLastError());
 
-    result = CryptHashData(hHash, pbData, 5, 0);
+    result = CryptHashData(hHash, pbData, 5, CRYPT_USERDATA);
     ok(result, "%08x\n", GetLastError());
 
     if(pCryptDuplicateHash) {
diff --git a/include/wincrypt.h b/include/wincrypt.h
index 64c932c..324007d 100644
--- a/include/wincrypt.h
+++ b/include/wincrypt.h
@@ -3890,6 +3890,8 @@ typedef BOOL (WINAPI *PFN_CMSG_IMPORT_KEY_TRANS)(
 #define EXPORT_PRIVATE_KEYS                   0x00000004
 #define PKCS12_EXPORT_RESERVED_MASK           0xffff0000
 
+#define CRYPT_USERDATA    0x00000001
+
 /* function declarations */
 /* advapi32.dll */
 WINADVAPI BOOL WINAPI CryptAcquireContextA(HCRYPTPROV *, LPCSTR, LPCSTR, DWORD, DWORD);