[PATCH v3] crypt32: Implement CNG_RSA_PUBLIC_KEY_BLOB encoding/decoding

[Hans Leidekker]

On Wed, 2021-07-07 at 21:52 -0500, Aaron Hill wrote:
> -        ret = CRYPT_AsnDecodeSequence(items, ARRAY_SIZE(items),
> -         pbEncoded, cbEncoded, CRYPT_DECODE_ALLOC_FLAG, NULL, &decodedKey,
> -         &size, NULL, NULL);
> +        TRACE("%p, %d, %08x, %p, %d\n", pbEncoded, cbEncoded, dwFlags,
> +         pvStructInfo, *pcbStructInfo);
> +
> +        ret = CRYPT_raw_decode_rsa_pub_key(&decodedKey, &size, pbEncoded, cbEncoded);
> +        if (ret)
> +        {
> +            /* Header, exponent, and modulus */
> +            DWORD bytesNeeded = sizeof(BCRYPT_RSAKEY_BLOB) + sizeof(DWORD) +
> +             decodedKey->modulus.cbData;
> +
> +            if (!pvStructInfo)
> +            {
> +                *pcbStructInfo = bytesNeeded;
> +                ret = TRUE;
> +            }
> +            else if ((ret = CRYPT_DecodeEnsureSpace(dwFlags, pDecodePara,
> +             pvStructInfo, pcbStructInfo, bytesNeeded)))
> +            {
> +                BCRYPT_RSAKEY_BLOB *hdr;
> +
> +                if (dwFlags & CRYPT_DECODE_ALLOC_FLAG)
> +                    pvStructInfo = *(BYTE **)pvStructInfo;
> +
> +                hdr = pvStructInfo;
> +                hdr->Magic = BCRYPT_RSAPUBLIC_MAGIC;
> +                hdr->BitLength = decodedKey->modulus.cbData * 8;
> +                hdr->cbPublicExp = sizeof(DWORD);
> +                hdr->cbModulus = decodedKey->modulus.cbData;
> +                hdr->cbPrime1 = 0;
> +                hdr->cbPrime2 = 0;
> +                /* CNG_RSA_PUBLIC_KEY_BLOB always stores the exponent and modulus
> +                 * in big-endian format, so we need to convert from the native
> +                 * endianness
> +                 */
> +                CRYPT_memcpy_be((BYTE *)pvStructInfo + sizeof(BCRYPT_RSAKEY_BLOB),
> +                 (BYTE *)&decodedKey->pubexp, sizeof(DWORD));
> +                CRYPT_memcpy_be((BYTE *)pvStructInfo + sizeof(BCRYPT_RSAKEY_BLOB) +
> +                 sizeof(DWORD), decodedKey->modulus.pbData,
> +                 decodedKey->modulus.cbData);

Exponent and modulus are handled as a byte arrays so native endianess
is not relevant there. They should be reversed unconditionally here as
well as in the encoder and the tests.