Paul Gofman : rsaenh: Handle uneven hash data updates sizes for CALG_MAC.

[Alexandre Julliard]

Module: wine
Branch: master
Commit: 6bb21c771b41655f4e3b4c653c9c3a7ef1fc1346
URL:    https://source.winehq.org/git/wine.git/?a=commit;h=6bb21c771b41655f4e3b4c653c9c3a7ef1fc1346

Author: Paul Gofman <pgofman at codeweavers.com>
Date:   Fri May 13 10:25:08 2022 -0500

rsaenh: Handle uneven hash data updates sizes for CALG_MAC.

Signed-off-by: Paul Gofman <pgofman at codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard at winehq.org>

---

 dlls/rsaenh/rsaenh.c       | 59 +++++++++++++++++++++++++++++++++++++++-------
 dlls/rsaenh/tests/rsaenh.c | 30 ++++++++++++++++++++++-
 2 files changed, 79 insertions(+), 10 deletions(-)

diff --git a/dlls/rsaenh/rsaenh.c b/dlls/rsaenh/rsaenh.c
index f9513d0533e..140fdec40d5 100644
--- a/dlls/rsaenh/rsaenh.c
+++ b/dlls/rsaenh/rsaenh.c
@@ -63,6 +63,7 @@ typedef struct tagCRYPTHASH
     BYTE         abHashValue[RSAENH_MAX_HASH_SIZE];
     PHMAC_INFO   pHMACInfo;
     RSAENH_TLS1PRF_PARAMS tpPRFParams;
+    DWORD        buffered_hash_bytes;
 } CRYPTHASH;
 
 /******************************************************************************
@@ -661,6 +662,7 @@ static inline BOOL init_hash(CRYPTHASH *pCryptHash) {
 static inline void update_hash(CRYPTHASH *pCryptHash, const BYTE *pbData, DWORD dwDataLen)
 {
     BYTE *pbTemp;
+    DWORD len;
 
     switch (pCryptHash->aiAlgid)
     {
@@ -670,12 +672,49 @@ static inline void update_hash(CRYPTHASH *pCryptHash, const BYTE *pbData, DWORD
             break;
 
         case CALG_MAC:
-            pbTemp = malloc(dwDataLen);
-            if (!pbTemp) return;
-            memcpy(pbTemp, pbData, dwDataLen);
-            RSAENH_CPEncrypt(pCryptHash->hProv, pCryptHash->hKey, 0, FALSE, 0,
-                             pbTemp, &dwDataLen, dwDataLen);
-            free(pbTemp);
+            if (pCryptHash->buffered_hash_bytes)
+            {
+                len = min(pCryptHash->dwHashSize - pCryptHash->buffered_hash_bytes, dwDataLen);
+                memcpy(pCryptHash->abHashValue + pCryptHash->buffered_hash_bytes, pbData, len);
+                pbData += len;
+                dwDataLen -= len;
+                pCryptHash->buffered_hash_bytes += len;
+                if (pCryptHash->buffered_hash_bytes < pCryptHash->dwHashSize)
+                    return;
+                pCryptHash->buffered_hash_bytes = 0;
+                len = pCryptHash->dwHashSize;
+                if (!RSAENH_CPEncrypt(pCryptHash->hProv, pCryptHash->hKey, 0, FALSE, 0,
+                                      pCryptHash->abHashValue, &len, len))
+                {
+                    FIXME("RSAENH_CPEncrypt failed.\n");
+                    return;
+                }
+            }
+            len = dwDataLen - dwDataLen % pCryptHash->dwHashSize;
+            if (len)
+            {
+                pbTemp = malloc(len);
+                if (!pbTemp)
+                {
+                    ERR("No memory.\n");
+                    return;
+                }
+                memcpy(pbTemp, pbData, len);
+                pbData += len;
+                dwDataLen -= len;
+                if (!RSAENH_CPEncrypt(pCryptHash->hProv, pCryptHash->hKey, 0, FALSE, 0,
+                                      pbTemp, &len, len))
+                {
+                    FIXME("RSAENH_CPEncrypt failed.\n");
+                    return;
+                }
+                free(pbTemp);
+            }
+            if (dwDataLen)
+            {
+                memcpy(pCryptHash->abHashValue, pbData, dwDataLen);
+                pCryptHash->buffered_hash_bytes = dwDataLen;
+            }
             break;
 
         default:
@@ -715,9 +754,10 @@ static inline void finalize_hash(CRYPTHASH *pCryptHash) {
             break;
 
         case CALG_MAC:
-            dwDataLen = 0;
-            RSAENH_CPEncrypt(pCryptHash->hProv, pCryptHash->hKey, 0, TRUE, 0,
-                             pCryptHash->abHashValue, &dwDataLen, pCryptHash->dwHashSize);
+            dwDataLen = pCryptHash->buffered_hash_bytes;
+            if (!RSAENH_CPEncrypt(pCryptHash->hProv, pCryptHash->hKey, 0, TRUE, 0,
+                                  pCryptHash->abHashValue, &dwDataLen, pCryptHash->dwHashSize))
+                FIXME("RSAENH_CPEncrypt failed.\n");
             break;
 
         default:
@@ -2217,6 +2257,7 @@ BOOL WINAPI RSAENH_CPCreateHash(HCRYPTPROV hProv, ALG_ID Algid, HCRYPTKEY hKey,
     pCryptHash->pHMACInfo = NULL;
     pCryptHash->hash_handle = NULL;
     pCryptHash->dwHashSize = peaAlgidInfo->dwDefaultLen >> 3;
+    pCryptHash->buffered_hash_bytes = 0;
     init_data_blob(&pCryptHash->tpPRFParams.blobLabel);
     init_data_blob(&pCryptHash->tpPRFParams.blobSeed);
 
diff --git a/dlls/rsaenh/tests/rsaenh.c b/dlls/rsaenh/tests/rsaenh.c
index f822c29bbc0..c9dbb72f427 100644
--- a/dlls/rsaenh/tests/rsaenh.c
+++ b/dlls/rsaenh/tests/rsaenh.c
@@ -1853,6 +1853,7 @@ static void test_mac(void) {
     DWORD dwLen;
     BYTE abData[256], abEnc[264];
     static const BYTE mac_40[8] = { 0xb7, 0xa2, 0x46, 0xe9, 0x11, 0x31, 0xe0, 0xad};
+    static const BYTE mac_40_2[8] = { 0xef, 0x22, 0x0a, 0x3b, 0xd0, 0xab, 0x48, 0x49};
     int i;
 
     for (i=0; i < ARRAY_SIZE(abData); i++) abData[i] = (BYTE)i;
@@ -1876,7 +1877,34 @@ static void test_mac(void) {
     ok(result && dwLen == 8, "%08lx, dwLen: %ld\n", GetLastError(), dwLen);
 
     ok(!memcmp(abData, mac_40, sizeof(mac_40)), "MAC failed!\n");
-    
+
+    for (i = 0; i < ARRAY_SIZE(abData); ++i)
+        abData[i] = (BYTE)i;
+
+    result = CryptHashData(hHash, abData, 1, 0);
+    ok(!result && GetLastError() == NTE_BAD_HASH_STATE, "Unexpected result %d, error %#lx\n", result, GetLastError());
+
+    result = CryptDestroyHash(hHash);
+    ok(result, "%08lx\n", GetLastError());
+
+    result = CryptCreateHash(hProv, CALG_MAC, hKey, 0, &hHash);
+    ok(result, "%08lx\n", GetLastError());
+
+    result = CryptHashData(hHash, abData, 1, 0);
+    ok(result, "%08lx\n", GetLastError());
+    result = CryptHashData(hHash, abData + 1, 1, 0);
+    ok(result, "%08lx\n", GetLastError());
+    result = CryptHashData(hHash, abData + 2, 6, 0);
+    ok(result, "%08lx\n", GetLastError());
+    result = CryptHashData(hHash, abData + 8, 9, 0);
+    ok(result, "%08lx\n", GetLastError());
+
+    dwLen = ARRAY_SIZE(abData);
+    result = CryptGetHashParam(hHash, HP_HASHVAL, abData, &dwLen, 0);
+    ok(result && dwLen == 8, "%08lx, dwLen %ld\n", GetLastError(), dwLen);
+
+    ok(!memcmp(abData, mac_40_2, sizeof(mac_40)), "Hash does not match.\n");
+
     result = CryptDestroyHash(hHash);
     ok(result, "%08lx\n", GetLastError());