BCryptDecrypt failes with STATUS_INVALID_PARAMETER

Question

I am trying to RSA encrypt an buffer using Windows api cng's Bcrypt library. But my first call to BCryptDecrypt failes with STATUS_INVALID_PARAMETER...

    status = BCryptDecrypt(
        hKey,
        pbEncryptedBuffer,
        cbEncryptedBuffer,
        NULL,
        NULL,
        0,
        NULL,
        0,
        pcbDecryptedBuffer,
        BCRYPT_PAD_PKCS1);

I looked at these examples:
How to use BCrypt for RSA (asymmetric encryption) https://social.msdn.microsoft.com/Forums/windowsdesktop/en-US/007a0e26-7fc0-4079-9b63-2ad23f866836/bug-in-rsa-encryptiondecryption-using-cng?forum=windowssdk
Taking in account that I would require padding. But if I remove the BCRYPT_PAD_PKCS1 parameter the first call to BCryptDecrypt is successful. Then the second call to BCryptDecrypt would fail with STATUS_INVALID_PARAMETER regardless if the BCRYPT_PAD_PKCS1 flag is present or not.

Bellow is the full code:

#include <windows.h>
#include <bcrypt.h>
#include <winternl.h>
#include <stdio.h>

#pragma comment(lib, "Bcrypt.lib")

#define NT_SUCCESS(Status)          (((NTSTATUS)(Status)) >= 0)
#define STATUS_UNSUCCESSFUL         ((NTSTATUS)0xC0000001L)

BOOL Encrypt(PBYTE pbPrivKey, DWORD cbPrivKey, PBYTE pbInputData, DWORD cbInputData,
             BYTE** ppbEncryptedBuffer, PDWORD pcbEncryptedBuffer)
{
    BOOL bRet = TRUE;
    NTSTATUS status = STATUS_UNSUCCESSFUL;
    BCRYPT_ALG_HANDLE hAlgorithm = NULL;
    BCRYPT_KEY_HANDLE hKey = NULL;

    ///
    // Open provider.
    ///
    status = BCryptOpenAlgorithmProvider(
        &hAlgorithm,
        BCRYPT_RSA_ALGORITHM,
        NULL,
        0);
    if (!NT_SUCCESS(status))
    {
        wprintf(L"**** Error 0x%x returned by BCryptOpenAlgorithmProvider\n", status);
        bRet = FALSE;
        goto LBL_CLEANUP;
    }

    ///
    // Import private key.
    ///
    status = BCryptImportKeyPair(hAlgorithm,
        NULL,
        BCRYPT_RSAPRIVATE_BLOB,
        &hKey,
        pbPrivKey,
        cbPrivKey,
        BCRYPT_NO_KEY_VALIDATION);
    if (!NT_SUCCESS(status))
    {
        wprintf(L"**** Error 0x%x returned by BCryptImportKeyPair\n", status);
        bRet = FALSE;
        goto LBL_CLEANUP;
    }

    ///
    // Get encrypted buffer size.
    ///
    status = BCryptEncrypt(
        hKey,
        pbInputData,
        cbInputData,
        NULL,
        NULL,
        0,
        NULL,
        0,
        pcbEncryptedBuffer,
        BCRYPT_PAD_PKCS1);
    if (!NT_SUCCESS(status))
    {
        wprintf(L"**** Error 0x%x returned by BCryptEncrypt\n", status);
        bRet = FALSE;
        goto LBL_CLEANUP;
    }

    ///
    // Encrypt buffer.
    ///
    *ppbEncryptedBuffer = HeapAlloc(GetProcessHeap(), 0, *pcbEncryptedBuffer);
    if (*ppbEncryptedBuffer == NULL)
    {
        wprintf(L"**** Error 0x%x returned by HeapAlloc\n", GetLastError());
        bRet = FALSE;
        goto LBL_CLEANUP;
    }

    status = BCryptEncrypt(hKey,
        pbInputData,
        cbInputData,
        NULL,
        NULL,
        0,
        *ppbEncryptedBuffer,
        *pcbEncryptedBuffer,
        pcbEncryptedBuffer,
        BCRYPT_PAD_PKCS1);
    if (!NT_SUCCESS(status))
    {
        wprintf(L"**** Error 0x%x returned by BCryptEncrypt\n", status);
        bRet = FALSE;
        goto LBL_CLEANUP;
    }

LBL_CLEANUP:

    if (hAlgorithm)
        BCryptCloseAlgorithmProvider(hAlgorithm, 0);

    if (hKey)
        BCryptDestroyKey(hKey);

    return bRet;
}

BOOL Decrypt(PBYTE pbPubKey, DWORD cbPubKey, PBYTE pbEncryptedBuffer, DWORD cbEncryptedBuffer,
             BYTE** ppbDecryptedBuffer, PDWORD pcbDecryptedBuffer)
{
    BOOL bRet = TRUE;
    NTSTATUS status = STATUS_UNSUCCESSFUL;
    BCRYPT_ALG_HANDLE hAlgorithm = NULL;
    BCRYPT_KEY_HANDLE hKey = NULL;

    ///
    // Open provider.
    ///
    status = BCryptOpenAlgorithmProvider(
        &hAlgorithm,
        BCRYPT_RSA_ALGORITHM,
        NULL,
        0);
    if (!NT_SUCCESS(status))
    {
        wprintf(L"**** Error 0x%x returned by BCryptOpenAlgorithmProvider\n", status);
        bRet = FALSE;
        goto LBL_CLEANUP;
    }

    ///
    // Import public key.
    ///
    status = BCryptImportKeyPair(hAlgorithm,
        NULL,
        BCRYPT_RSAPUBLIC_BLOB,
        &hKey,
        pbPubKey,
        cbPubKey,
        BCRYPT_NO_KEY_VALIDATION);
    if (!NT_SUCCESS(status))
    {
        wprintf(L"**** Error 0x%x returned by BCryptImportKeyPair\n", status);
        bRet = FALSE;
        goto LBL_CLEANUP;
    }
    
    ///
    // Get decrypted buffer size.
    ///
    status = BCryptDecrypt(
        hKey,
        pbEncryptedBuffer,
        cbEncryptedBuffer,
        NULL,
        NULL,
        0,
        NULL,
        0,
        pcbDecryptedBuffer,
        BCRYPT_PAD_PKCS1);
    if (!NT_SUCCESS(status))
    {
        wprintf(L"**** Error 0x%x returned by BCryptDecrypt\n", status);
        bRet = FALSE;
        goto LBL_CLEANUP;
    }

    ///
    // Decrypt buffer.
    ///
    *ppbDecryptedBuffer = HeapAlloc(GetProcessHeap(), 0, *pcbDecryptedBuffer);
    if (*ppbDecryptedBuffer == NULL)
    {
        wprintf(L"**** Error 0x%x returned by HeapAlloc\n", GetLastError());
        bRet = FALSE;
        goto LBL_CLEANUP;
    }

    status = BCryptDecrypt(
        hKey,
        pbEncryptedBuffer,
        cbEncryptedBuffer,
        NULL,
        NULL,
        0,
        *ppbDecryptedBuffer,
        *pcbDecryptedBuffer,
        pcbDecryptedBuffer,
        BCRYPT_PAD_PKCS1);
    if (!NT_SUCCESS(status))
    {
        wprintf(L"**** Error 0x%x returned by BCryptDecrypt\n", status);
        bRet = FALSE;
        goto LBL_CLEANUP;
    }

LBL_CLEANUP:

    if (hAlgorithm)
        BCryptCloseAlgorithmProvider(hAlgorithm, 0);

    if (hKey)
        BCryptDestroyKey(hKey);

    return bRet;
}

BYTE PubKey[91] = {
    0x52, 0x53, 0x41, 0x31, 0x00, 0x02, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00,
    0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
    0x01, 0x00, 0x01, 0xD5, 0x0F, 0xFD, 0x75, 0xA3, 0xDC, 0xCD, 0xCA, 0xC1,
    0x38, 0xBB, 0x3A, 0x8F, 0x6F, 0xC5, 0x53, 0xF7, 0xAC, 0x29, 0x5E, 0x14,
    0xF5, 0x95, 0xA1, 0x76, 0xAA, 0xD0, 0xAB, 0xAA, 0x4E, 0x02, 0xFE, 0x28,
    0xF5, 0xE0, 0xD7, 0xAC, 0x2E, 0x23, 0x5B, 0x20, 0x53, 0x22, 0x0D, 0x78,
    0x33, 0x2B, 0x05, 0x93, 0xF7, 0xD2, 0x28, 0x24, 0xD4, 0x48, 0xC5, 0xEE,
    0x1B, 0xE7, 0x41, 0x2E, 0x1A, 0x05, 0x7D
};

BYTE PrivKey[155] = {
    0x52, 0x53, 0x41, 0x32, 0x00, 0x02, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00,
    0x40, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00,
    0x01, 0x00, 0x01, 0xD5, 0x0F, 0xFD, 0x75, 0xA3, 0xDC, 0xCD, 0xCA, 0xC1,
    0x38, 0xBB, 0x3A, 0x8F, 0x6F, 0xC5, 0x53, 0xF7, 0xAC, 0x29, 0x5E, 0x14,
    0xF5, 0x95, 0xA1, 0x76, 0xAA, 0xD0, 0xAB, 0xAA, 0x4E, 0x02, 0xFE, 0x28,
    0xF5, 0xE0, 0xD7, 0xAC, 0x2E, 0x23, 0x5B, 0x20, 0x53, 0x22, 0x0D, 0x78,
    0x33, 0x2B, 0x05, 0x93, 0xF7, 0xD2, 0x28, 0x24, 0xD4, 0x48, 0xC5, 0xEE,
    0x1B, 0xE7, 0x41, 0x2E, 0x1A, 0x05, 0x7D, 0xF3, 0x4A, 0xE4, 0x0A, 0xF6,
    0xCA, 0xBC, 0xB6, 0xE2, 0x10, 0xF3, 0x9F, 0x8D, 0x3B, 0x07, 0x7D, 0x83,
    0x59, 0x51, 0xAA, 0x3C, 0xBA, 0x89, 0x2C, 0xCF, 0x69, 0x29, 0x8D, 0x96,
    0x2C, 0x80, 0x67, 0xE0, 0x30, 0xE3, 0x27, 0xDC, 0x2C, 0x85, 0xF3, 0x8B,
    0x47, 0xB0, 0x10, 0xC2, 0x49, 0x09, 0x14, 0x15, 0x47, 0x90, 0xAF, 0x5C,
    0xF1, 0x3D, 0x4C, 0x6D, 0x88, 0xAB, 0x98, 0x7B, 0x80, 0x8C, 0x7B
};

BYTE Msg[10] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A };

int main()
{
    BOOL bRet = TRUE;

    PBYTE pbEncryptedMessage = NULL;
    DWORD cbEncryptedMessage = 0;
    PBYTE pbDecryptedMessage = NULL;
    DWORD cbDecryptedMessage = 0;

    bRet = Encrypt(
        PrivKey, 
        sizeof(PrivKey), 
        Msg, 
        sizeof(Msg), 
        &pbEncryptedMessage, 
        &cbEncryptedMessage);
    if (bRet == FALSE)
    {
        wprintf(L"**** Error FALSE returned by Encrypt\n");
        goto LBL_CLEANUP;
    }
    wprintf(L"Encryption Success!\n");

    bRet = Decrypt(
        PubKey, 
        sizeof(PubKey), 
        pbEncryptedMessage, 
        cbEncryptedMessage, 
        &pbDecryptedMessage, 
        &cbDecryptedMessage);
    if (bRet == FALSE)
    {
        wprintf(L"**** Error FALSE returned by Decrypt\n");
        goto LBL_CLEANUP;
    }
    wprintf(L"Decryption Success!\n");

LBL_CLEANUP:

    if (pbEncryptedMessage)
        HeapFree(GetProcessHeap(), 0, pbEncryptedMessage);

    if (pbDecryptedMessage)
        HeapFree(GetProcessHeap(), 0, pbDecryptedMessage);
}```

It seems that some of the random values that you are encrypting are greater than the modulus of the key.I suggest you could refer to the thread: https://stackoverflow.com/questions/6316158/bug-in-rsa-encryption-decryption-using-cryptoapi-next-generation-cng-in-window — Jeaninez - MSFT, Feb 07 '23 at 07:14

score 1 · Answer 1 · edited Feb 19 '23 at 17:55

NCryptDecrypt returns STATUS_INVALID_PARAMETER when it can't decrypt the buffer; this doesn't mean that your parameters are invalid.

In your case, you encrypt with the private key and decrypt with the public key. Try the opposite and it will work.

Here is what happens: a handle to a private key knows in fact both the private key and the public key and if you use it to encrypt, the public key is used and not the private key as expected. Just test it: get the handle to what is supposed to be just the private key, and decrypt using the same handle, it works. It has encrypted with the public key and decrypted with the private key.

Yeaah but I need pub key decryption....... – woldgrep Feb 18 '23 at 17:20 — woldgrep, Feb 18 '23 at 17:20

BCryptDecrypt failes with STATUS_INVALID_PARAMETER

1 Answers1