Juan Lang : crypt32: Use helper function to encode public keys for hashing.

[Alexandre Julliard]

Module: wine
Branch: master
Commit: d8ca5bc348b3e033c04ca4ddf9f3786706f78f0e
URL:    http://source.winehq.org/git/wine.git/?a=commit;h=d8ca5bc348b3e033c04ca4ddf9f3786706f78f0e

Author: Juan Lang <juan.lang at gmail.com>
Date:   Tue Oct 20 13:29:03 2009 -0700

crypt32: Use helper function to encode public keys for hashing.

This is necessary because the publicly callable form embeds the asn.1
NULL value when the parameters are empty, whereas the hash value
expects that it's missing.

---

 dlls/crypt32/cert.c            |   10 ++++++++--
 dlls/crypt32/crypt32_private.h |    4 ++++
 dlls/crypt32/encode.c          |    2 +-
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/dlls/crypt32/cert.c b/dlls/crypt32/cert.c
index 5b3b6e3..2115913 100644
--- a/dlls/crypt32/cert.c
+++ b/dlls/crypt32/cert.c
@@ -1855,13 +1855,19 @@ BOOL WINAPI CryptHashPublicKeyInfo(HCRYPTPROV_LEGACY hCryptProv, ALG_ID Algid,
         hCryptProv = CRYPT_GetDefaultProvider();
     if (!Algid)
         Algid = CALG_MD5;
+    if ((dwCertEncodingType & CERT_ENCODING_TYPE_MASK) != X509_ASN_ENCODING)
+    {
+        SetLastError(ERROR_FILE_NOT_FOUND);
+        return FALSE;
+    }
     if (ret)
     {
         BYTE *buf;
         DWORD size = 0;
 
-        ret = CryptEncodeObjectEx(dwCertEncodingType, X509_PUBLIC_KEY_INFO,
-         pInfo, CRYPT_ENCODE_ALLOC_FLAG, NULL, &buf, &size);
+        ret = CRYPT_AsnEncodePubKeyInfoNoNull(dwCertEncodingType,
+         X509_PUBLIC_KEY_INFO, pInfo, CRYPT_ENCODE_ALLOC_FLAG, NULL,
+         (LPBYTE)&buf, &size);
         if (ret)
         {
             ret = CryptCreateHash(hCryptProv, Algid, 0, 0, &hHash);
diff --git a/dlls/crypt32/crypt32_private.h b/dlls/crypt32/crypt32_private.h
index 07c6382..a5759a2 100644
--- a/dlls/crypt32/crypt32_private.h
+++ b/dlls/crypt32/crypt32_private.h
@@ -113,6 +113,10 @@ BOOL CRYPT_AsnDecodePKCSDigestedData(const BYTE *pbEncoded, DWORD cbEncoded,
  DWORD dwFlags, PCRYPT_DECODE_PARA pDecodePara,
  CRYPT_DIGESTED_DATA *digestedData, DWORD *pcbDigestedData);
 
+BOOL WINAPI CRYPT_AsnEncodePubKeyInfoNoNull(DWORD dwCertEncodingType,
+ LPCSTR lpszStructType, const void *pvStructInfo, DWORD dwFlags,
+ PCRYPT_ENCODE_PARA pEncodePara, BYTE *pbEncoded, DWORD *pcbEncoded);
+
 /* The following aren't defined in wincrypt.h, as they're "reserved" */
 #define CERT_CERT_PROP_ID 32
 #define CERT_CRL_PROP_ID  33
diff --git a/dlls/crypt32/encode.c b/dlls/crypt32/encode.c
index 627c354..93c4a29 100644
--- a/dlls/crypt32/encode.c
+++ b/dlls/crypt32/encode.c
@@ -464,7 +464,7 @@ static BOOL WINAPI CRYPT_AsnEncodeCert(DWORD dwCertEncodingType,
     return ret;
 }
 
-static BOOL WINAPI CRYPT_AsnEncodePubKeyInfoNoNull(DWORD dwCertEncodingType,
+BOOL WINAPI CRYPT_AsnEncodePubKeyInfoNoNull(DWORD dwCertEncodingType,
  LPCSTR lpszStructType, const void *pvStructInfo, DWORD dwFlags,
  PCRYPT_ENCODE_PARA pEncodePara, BYTE *pbEncoded, DWORD *pcbEncoded)
 {