BCryptEncrypt API failing for AES-128-CFB-128 when the input plaintext is size is not multiple of 16 and padding is off

Pavan 101 Reputation points
2021-10-06T11:53:51.17+00:00

I trying to encrypt the data with AES-128-CFB-128 using BCryptEncrypt API but API is failing when input plaintext is size is not multiple of 16 and padding is off with error 0xc0000206/STATUS_INVALID_BUFFER_SIZE. AES-CFB encryption is self-synchronizing stream cipher so it should run if the input plain text length is not multiple of blocksize. I tested AES-128-CFB-128 with OpenSSL for the same plain text and padding is off then also it generating the ciphertext but windows CNG is failing for the same. I asked the same question in Stackoverflow and most of the people thinking it may be a bug in Windows CNG.

I am using the below code to test the encryption with windows CNG.

#include <windows.h>
#include <stdio.h>
#include <bcrypt.h>
#pragma comment(lib, "bcrypt.lib") 

#define NT_SUCCESS(Status)          (((NTSTATUS)(Status)) >= 0)

#define STATUS_UNSUCCESSFUL         ((NTSTATUS)0xC0000001L)
void
hexdump(const char *msg, const unsigned char *in, unsigned int len)
{
    printf("%s [%d][", msg, len);
    for (; len > 0; len--, in++) {
        printf("%02X", *in);
    }
    printf("]\n");
}
const BYTE rgbPlaintext[] =  "The quick brown fox jumps over the lazy dog";

static const BYTE rgbIV[] = "1234567887654321";

static const BYTE rgbAES128Key[] = "0123456789abcdefghijklmnopqrstuv";

const int ival_len = 16;
const int key_len = 16;
const DWORD block_len = 16;

void __cdecl wmain(
                   int                      argc, 
                   __in_ecount(argc) LPWSTR *wargv)
{

    BCRYPT_ALG_HANDLE       hAesAlg                     = NULL;
    BCRYPT_KEY_HANDLE       hKey                        = NULL;
    NTSTATUS                status                      = STATUS_UNSUCCESSFUL;
    DWORD                   cbCipherText                = 0, 
                            cbData                      = 0, 
                            cbKeyObject                 = 0,
                            cbBlockLen                  = 0,
                            cbBlob                      = 0;
    PBYTE                   pbCipherText                = NULL,
                            pbKeyObject                 = NULL,
                            pbIV                        = NULL,
                            pbBlob                      = NULL;

    UNREFERENCED_PARAMETER(argc);
    UNREFERENCED_PARAMETER(wargv);

    if(!NT_SUCCESS(status = BCryptOpenAlgorithmProvider(
                                                &hAesAlg,
                                                BCRYPT_AES_ALGORITHM,
                                                NULL,
                                                0)))
    {
        wprintf(L"**** Error 0x%x returned by BCryptOpenAlgorithmProvider\n", status);
        goto Cleanup;
    }

    if(!NT_SUCCESS(status = BCryptGetProperty(
                                        hAesAlg, 
                                        BCRYPT_OBJECT_LENGTH, 
                                        (PBYTE)&cbKeyObject, 
                                        sizeof(DWORD), 
                                        &cbData, 
                                        0)))
    {
        wprintf(L"**** Error 0x%x returned by BCryptGetProperty\n", status);
        goto Cleanup;
    }

    pbKeyObject = (PBYTE)HeapAlloc (GetProcessHeap (), 0, cbKeyObject);
    if(NULL == pbKeyObject)
    {
        wprintf(L"**** memory allocation failed\n");
        goto Cleanup;
    }

    if (ival_len > sizeof (rgbIV))
    {
        wprintf (L"**** block length is longer than the provided IV length\n");
        goto Cleanup;
    }

   if(ival_len > 0)
   {       
        pbIV= (PBYTE) HeapAlloc (GetProcessHeap (), 0, ival_len + 1);
        if(NULL == pbIV)
        {
            wprintf(L"**** memory allocation failed\n");
            goto Cleanup;
        }
        strncpy(pbIV,rgbIV,ival_len);
        pbIV[ival_len]='\0';
   }    

    if(!NT_SUCCESS(status = BCryptSetProperty(
                                hAesAlg, 
                                BCRYPT_CHAINING_MODE, 
                                (PBYTE)BCRYPT_CHAIN_MODE_CFB, 
                                sizeof(BCRYPT_CHAIN_MODE_CFB), 
                                0)))
    {
        wprintf(L"**** Error 0x%x returned by BCryptSetProperty\n", status);
        goto Cleanup;
    }

    if(!NT_SUCCESS(status = BCryptGetProperty(
                                        hAesAlg, 
                                        BCRYPT_BLOCK_LENGTH, 
                                        (PBYTE)&cbBlockLen, 
                                        sizeof(DWORD), 
                                        &cbData, 
                                        0)))
    {
        wprintf(L"**** Error 0x%x returned by BCryptGetProperty\n", status);
        goto Cleanup;
    }

    if(!NT_SUCCESS(status = BCryptGenerateSymmetricKey(
                                        hAesAlg, 
                                        &hKey, 
                                        pbKeyObject, 
                                        cbKeyObject, 
                                        (PBYTE)rgbAES128Key, 
                                        key_len, 
                                        0)))
    {
        wprintf(L"**** Error 0x%x returned by BCryptGenerateSymmetricKey\n ", status);
        fprintf(stderr, "\nError number <%x>", GetLastError());
        goto Cleanup;
    }
    if (!NT_SUCCESS(status = BCryptSetProperty(hKey,
                                            BCRYPT_MESSAGE_BLOCK_LENGTH,
                                            (PBYTE)&block_len,
                                            sizeof(DWORD),
                                            0))) {
        wprintf(L"**** Error 0x%x returned by BCryptSetProperty\n ", status);
        fprintf(stderr, "\nError number <%x>", GetLastError());
        goto Cleanup;
    }

    printf("\nplan text is [%d][%s]\n", sizeof(rgbPlaintext) - 1, rgbPlaintext);
    if(!NT_SUCCESS(status = BCryptEncrypt(
                                        hKey, 
                                        rgbPlaintext, 
                                        sizeof(rgbPlaintext) - 1,
                                        NULL,
                                        pbIV,
                                        ival_len,
                                        NULL, 
                                        0, 
                                        &cbCipherText, 
                                        0)))
    {
        wprintf(L"\n**** Error 1 0x%x returned by BCryptEncrypt\n", status);
        goto Cleanup;
    }

    pbCipherText = (PBYTE)HeapAlloc (GetProcessHeap (), 0, cbCipherText);
    if(NULL == pbCipherText)
    {
        wprintf(L"**** memory allocation failed\n");
        goto Cleanup;
    }

    // Use the key to encrypt the plaintext buffer.
    // For block sized messages, block padding will add an extra block.
    if(!NT_SUCCESS(status = BCryptEncrypt(
                                        hKey, 
                                        rgbPlaintext, 
                                        sizeof(rgbPlaintext) - 1,
                                        NULL,
                                        pbIV,
                                        ival_len, 
                                        pbCipherText, 
                                        cbCipherText, 
                                        &cbCipherText, 
                                        0)))
    {
        wprintf(L"**** Error 2 0x%x returned by BCryptEncrypt\n", status);
        goto Cleanup;
    }

    hexdump("cipher is", pbCipherText, cbCipherText);

Cleanup:

    if(hAesAlg)
    {
        BCryptCloseAlgorithmProvider(hAesAlg,0);
    }

    if (hKey)    
    {
        BCryptDestroyKey(hKey);
    }

    if(pbCipherText)
    {
        HeapFree(GetProcessHeap(), 0, pbCipherText);
    }

    if(pbKeyObject)
    {
        HeapFree(GetProcessHeap(), 0, pbKeyObject);
    }

    if(pbIV)
    {
        HeapFree(GetProcessHeap(), 0, pbIV);
    }
}
Windows App SDK
Windows App SDK
A set of Microsoft open-source libraries, frameworks, components, and tools to be used in apps to access Windows platform functionality on many versions of Windows. Previously known as Project Reunion.
747 questions
Windows API - Win32
Windows API - Win32
A core set of Windows application programming interfaces (APIs) for desktop and server applications. Previously known as Win32 API.
2,499 questions
0 comments No comments
{count} vote

2 answers

Sort by: Most helpful
  1. Xiaopo Yang - MSFT 12,151 Reputation points Microsoft Vendor
    2021-10-07T02:38:26.62+00:00

    Yes. According to BCryptEncrypt,

    If BCRYPT_BLOCK_PADDING flag is not specified, the size of the plaintext specified in the cbInput parameter must be a multiple of the algorithm's block size.
    The block size can be obtained by calling the BCryptGetProperty function to get the BCRYPT_BLOCK_LENGTH property for the key. This will provide the size of a block for the algorithm.

    0 comments No comments

  2. Pavan 101 Reputation points
    2021-10-07T06:03:19.467+00:00

    @Xiaopo Yang - MSFT , Yes I saw the documentation of BCryptEncrypt API it is expecting the input plaintext size in multiple of the algorithm's block size but AES-CFB cipher algorithm is self-synchronizing stream cipher so it should run if the input plain text length is not multiple of block size then why BCryptEncrypt expecting input length in multiple bytes when padding off. I tested AES-128-CFB-128 with OpenSSL it working perfectly when input plaintext size in multiple of the algorithm's block size why the same thing is not implemented in BCryptEncrypt API.