2

For industrial purposes, I want to decrypt an AES-encrypted message with an RSA-encrypted key in C. At first, I thought doing it step-by-step by first, using OpenSSL libcrypto library, by first RSA decoding the key then AES decoding the data.

I have found out that EVP tools were commonly seen as a better way to do this since it actually does what the low-levels functions do but correctly. Here is what I see the flow of the program :

  • Initialize OpenSSL;
  • Read and store the RSA private key;
  • Initialize the decryption by specifying the decryption algorithm (AES) and the private key;
  • Update the decryption by giving the key, the data, the key and their length
  • Finally decrypt the data and return it.

I have been a lot confused by the fact that so far we do not intend to use any IV or ADD (although IV might come up later in the project). I have followed this guide it is not very clear and does not fit the way I use EVP.

So here is my actual code :

#include <openssl/evp.h>
#include <openssl/conf.h>
#include <openssl/pem.h>
#include <openssl/rsa.h>
#include <openssl/aes.h>
#include <openssl/err.h>
#include "openssl\applink.c" 
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>

const char PRIVATE_KEY_PATH[] = "C:/Users/Local_user/privateKey.pem";

EVP_PKEY* initializePrivateKey(void)
{
    FILE* privateKeyfile;
    if ((privateKeyfile = fopen(PRIVATE_KEY_PATH, "r")) == NULL) // Check PEM file opening
    {
        perror("Error while trying to access to private key.\n");
        return NULL;
    }

    RSA *rsaPrivateKey = RSA_new();
    EVP_PKEY *privateKey = EVP_PKEY_new();

    if ((rsaPrivateKey = PEM_read_RSAPrivateKey(privateKeyfile, &rsaPrivateKey, NULL, NULL)) == NULL) // Check PEM file reading
    {
        fprintf(stderr, "Error loading RSA Private Key File.\n");
        ERR_print_errors_fp(stderr);
        return NULL;
    }
    if (!EVP_PKEY_assign_RSA(privateKey, rsaPrivateKey))
    {
        fprintf(stderr, "Error when initializing EVP private key.\n");
        ERR_print_errors_fp(stderr);
        return NULL;
    }
    return privateKey;
}
const uint8_t* decodeWrappingKey(uint8_t const* data, const size_t data_len, uint8_t const* wrappingKey, const size_t wrappingKey_len)
{
    // Start Decryption
    EVP_CIPHER_CTX *ctx;
    if (!(ctx = EVP_CIPHER_CTX_new())) // Initialize context
    {
        fprintf(stderr, "Error when initializing context.\n");
        ERR_print_errors_fp(stderr);
        return NULL;
    }
    EVP_PKEY *privateKey = initializePrivateKey();
    if (1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, privateKey, NULL)) // Initialize decryption
    {
        fprintf(stderr, "Error when initializing decryption.\n");
        ERR_print_errors_fp(stderr);
        return NULL;
    } 
    uint8_t* res;
    if ((res = calloc(data_len, sizeof(uint8_t))) == NULL) // Check memory allocating
    {
        perror("Memory allocating error ");
        return NULL;
    }
    puts("Initialization done. Decoding..\n");
    size_t res_len = 0;
    if (1 != EVP_DecryptUpdate(ctx, res, &res_len, data, data_len))
    {
        fprintf(stderr, "Error when preparing decryption.\n");
        ERR_print_errors_fp(stderr);
    }


    if (1 != EVP_DecryptFinal_ex(ctx, res, &res_len))
    {
        fprintf(stderr, "Error when decrypting.\n");
        ERR_print_errors_fp(stderr);
    }
    return res;
}

void hexToBytes(uint8_t *des, char const *source, const size_t size) {

    for (int i = 0; i < size - 1; i += 2) 
        sscanf(source + i, "%02x", des + (i / 2));
}

int main(void) {
    char const *strWrap = "5f82c48f85054ef6a3b2621819dd0e969030c79cc00deb89........";
    char const *strData = "ca1518d44716e3a4588af741982f29ad0a3e7a8d67.....";

    uint8_t *wrap = calloc(strlen(strWrap), sizeof(uint8_t));
    hexToBytes(wrap, strWrap, strlen(strWrap)); // Converts string to raw data
    uint8_t *data = calloc(strlen(strData), sizeof(uint8_t));
    hexToBytes(data, strData, strlen(strData));

    /* Load the human readable error strings for libcrypto */
    ERR_load_crypto_strings();

    /* Load all digest and cipher algorithms */
    OpenSSL_add_all_algorithms();

    /* Load config file, and other important initialisation */
    OPENSSL_config(NULL);
    const uint8_t *res = decodeWrappingKey(data, strlen(strData) / 2, wrap, strlen(strWrap) / 2);
    if (res == NULL)
        return 1;
    return 0;
}

My output is the following one :

Initialization done. Decoding..

Error when preparing decryption.
Error when decrypting. 

Obviously it fails when updating and finalising the decryption but I can't figure out why and the ERR_print_errors_fp(stderr); which had always worked for me so far seems to be mute.

Badda
  • 1,329
  • 2
  • 15
  • 40
  • 1
    Can you [edit] this to instead include a *minimal, self-contained, compilable example* that illustrates your problem? I'm pretty sure the code you included has prerequisites which are *not* included in that code; certainly just feeding it to GCC doesn't compile (I get 28 lines of diagnostic output, including several that are "first use in this function"). – user Jun 07 '17 at 09:24
  • @MichaelKjörling Thanks for trying to help. Edited, I think it is ok now – Badda Jun 07 '17 at 09:36
  • AES needs symmetric key for encryption,but appears you are passing RSA key,if you have key encrypted with RSA public key, you need to decrypt it with corresponding private key,then use the decrypted key for AES decryption of data – Pras Jun 07 '17 at 09:37
  • @Pras Well then where and when would I give the RSA private key to EVP for it to decrypt the wrapping key ? I know AES needs an symetric key and RSA an asymmetric one. – Badda Jun 07 '17 at 09:41
  • 1
    Your key is encrypted with RSA, so you will decrypt it with RSA APIs like RSA_private_decrypt, not with EVP* apis, you will use them(EVP*) for data that is encrypted with AES – Pras Jun 07 '17 at 09:50
  • @Pras Thanks a lot. I have been very stupid. If there is nothing else to change in the code for it to work (I will test it this afternoon), I'd be glad if you could post this as an answer so I can accept it. – Badda Jun 07 '17 at 09:56
  • 2
    Have you considered using the EVP_Seal APIs? They provide a high level interface to do exactly what you describe and hide all the low level details. See https://www.openssl.org/docs/man1.1.0/crypto/EVP_SealInit.html – Matt Caswell Jun 07 '17 at 10:37
  • @MattCaswell Spent the afternoon trying to use it. Honestly those EVP APIs are a hell to use, no errors when it's not working, no tutorials out there.. I don't understand why `EVP_OpenUpdate` has to be called several times ? – Badda Jun 07 '17 at 15:39
  • 1
    There is some sample code available at https://wiki.openssl.org/index.php/EVP_Asymmetric_Encryption_and_Decryption_of_an_Envelope. As for errors, if the functions return an error code you can get a more detailed error message using the various error functions, e.g. see https://www.openssl.org/docs/man1.1.0/crypto/ERR_print_errors_fp.html or https://www.openssl.org/docs/man1.1.0/crypto/ERR_get_error.html. `EVP_OpenUpdate` does *not* need to be called several times - once is enough. You may choose to call it multiple times if the encrypted data is being streamed to you. – Matt Caswell Jun 07 '17 at 22:17
  • @MattCaswell Ok thanks a lot. I'd have two last questions, is it possible to `Open` a symmetric key which has been encrypted somehow else than with `Seal` ? And is it more verbose/more effective to use `ERR_get_error`than `ERR_print_errors_fp`? – Badda Jun 08 '17 at 06:50
  • 1
    Yes, as long as the key has been encrypted using RSA with PKCS#1 padding it should be fine to call `Open` with it. `ERR_print_errors_fp` is an easy call to just dump all the errors to (say) stderr. Some people want more control over the format of the error message and how it gets displayed (e.g. maybe you don't want to send it to a FILE *, but instead to some custom logging API). If you wan more control you can use `ERR_get_error` and then the various other `ERR_*` functions to get hold of the component parts and do with it as you wish. – Matt Caswell Jun 08 '17 at 08:28
  • Let us [continue this discussion in chat](http://chat.stackoverflow.com/rooms/146147/discussion-between-badda-and-matt-caswell). – Badda Jun 08 '17 at 09:02

2 Answers2

2

Your key is encrypted with RSA, so you will decrypt it with RSA APIs like RSA_private_decrypt first, not with EVP* APIs.

Once you get key decrypted you need to use it with (EVP*) APIs to decrypt the data with AES.

Badda
  • 1,329
  • 2
  • 15
  • 40
Pras
  • 4,047
  • 10
  • 20
  • What mode should I use ? I am still getting the same error after using the RSA-decrypted AES key – Badda Jun 08 '17 at 09:32
  • 1
    You need to pass correct cipher type to EVP_DecryptInit_ex, depending on length of key being used, you can use EVP_aes_128_cbc,EVP_aes_192_cbc,EVP_aes_256_cbc – Pras Jun 08 '17 at 09:42
  • Isn't cbc mode used with IV ? – Badda Jun 08 '17 at 09:44
  • 1
    The cipher type should be the same that was used during the encryption, list of cipher type you can find in man page of evp_decryptinit_ex, when you mentioned no IV is used I believe you mean all zeros as IV, it is failing in EVP_DecryptUpdatetoo or just in EVP_DecryptFinal_ex? – Pras Jun 08 '17 at 10:26
  • Except the size of the symmetric and asymmetric keys, I have no other information about how it was encrypted. It is failing only with `EVP_DecryptFinal_ex`, but `EVP_DecryptUpdate`'s result gives random values. The result is like 512 bytes long instead of something like 60. I'll try with all zeros as IV, but I thought IVs were optional. – Badda Jun 08 '17 at 11:59
  • 1
    EVP_DecryptUpdate decrypts the final block that is not multiple of cipher block, I am not sure if it returns 1 if the data is already multiple of cipher block, can you check if data you got from EVP_DecryptUpdate is the decrypted plain data that you expect,Please note you have to be in sync with the client(the one who encrypts) on iv,encryption mode,key ie use the same one while decrypting to get correct decrypted data. – Pras Jun 09 '17 at 03:23
2

Here is a complete working example of how you can encrypt a key using RSA, and encrypt a message using that key using AES, followed by the subsequent decryption of those things. It assumes AES-256-CBC is being used. If you want to use AES-256-GCM instead then you will need to make some changes to get and set the tag (ask me if you need some pointers on how to do this). It also assumes that the RSA encryption is done with PKCS#1 padding (which is all that the EVP_Seal* APIs support). If you need some other kind of padding then you will need to use a different method. Finally it assumes you are using OpenSSL 1.1.0. If you are using 1.0.2 then some changes will probably be necessary (at least you will need to explicitly init and de-init the library - that isn't required in 1.1.0).

The code reads the RSA private and public keys from files called privkey.pem and pubkey.pem which are in the current working directory. I generated these files like this:

openssl genrsa -out privkey.pem 2048
openssl rsa -in privkey.pem -pubout -out pubkey.pem

I've tested this on Linux only. The code is as follows:

#include <openssl/evp.h>
#include <openssl/pem.h>
#include <openssl/err.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

static int envelope_seal(EVP_PKEY *pub_key, unsigned char *plaintext,
                         int plaintext_len, unsigned char **encrypted_key,
                         int *encrypted_key_len, unsigned char **iv,
                         int *iv_len,  unsigned char **ciphertext,
                         int *ciphertext_len)
{
    EVP_CIPHER_CTX *ctx;
    int len, ret = 0;
    const EVP_CIPHER *type = EVP_aes_256_cbc();
    unsigned char *tmpiv = NULL, *tmpenc_key = NULL, *tmpctxt = NULL;

    if((ctx = EVP_CIPHER_CTX_new()) == NULL)
        return 0;

    *iv_len = EVP_CIPHER_iv_length(type);
    if ((tmpiv = malloc(*iv_len)) == NULL)
        goto err;

    if ((tmpenc_key = malloc(EVP_PKEY_size(pub_key))) == NULL)
        goto err;

    if ((tmpctxt = malloc(plaintext_len + EVP_CIPHER_block_size(type)))
            == NULL)
        goto err;

    if(EVP_SealInit(ctx, type, &tmpenc_key, encrypted_key_len, tmpiv, &pub_key,
                    1) != 1)
        goto err;

    if(EVP_SealUpdate(ctx, tmpctxt, &len, plaintext, plaintext_len) != 1)
        goto err;
    *ciphertext_len = len;

    if(EVP_SealFinal(ctx, tmpctxt + len, &len) != 1)
        goto err;
    *ciphertext_len += len;

    *iv = tmpiv;
    *encrypted_key = tmpenc_key;
    *ciphertext = tmpctxt;
    tmpiv = NULL;
    tmpenc_key = NULL;
    tmpctxt = NULL;
    ret = 1;
 err:
    EVP_CIPHER_CTX_free(ctx);
    free(tmpiv);
    free(tmpenc_key);
    free(tmpctxt);

    return ret;
}

int envelope_open(EVP_PKEY *priv_key, unsigned char *ciphertext,
                  int ciphertext_len, unsigned char *encrypted_key,
                  int encrypted_key_len, unsigned char *iv,
                  unsigned char **plaintext, int *plaintext_len)
{
    EVP_CIPHER_CTX *ctx;
    int len, ret = 0;
    unsigned char *tmpptxt = NULL;

    if((ctx = EVP_CIPHER_CTX_new()) == NULL)
        return 0;

    if ((tmpptxt = malloc(ciphertext_len)) == NULL)
        goto err;

    if(EVP_OpenInit(ctx, EVP_aes_256_cbc(), encrypted_key, encrypted_key_len,
                    iv, priv_key) != 1)
        return 0;

    if(EVP_OpenUpdate(ctx, tmpptxt, &len, ciphertext, ciphertext_len) != 1)
        return 0;
    *plaintext_len = len;

    if(EVP_OpenFinal(ctx, tmpptxt + len, &len) != 1)
        return 0;
    *plaintext_len += len;

    *plaintext = tmpptxt;
    tmpptxt = NULL;
    ret = 1;
 err:
    EVP_CIPHER_CTX_free(ctx);
    free(tmpptxt);

    return ret;
}

int main(void)
{
    EVP_PKEY *pubkey = NULL, *privkey = NULL;
    FILE *pubkeyfile, *privkeyfile;
    int ret = 1;
    unsigned char *iv = NULL, *message = "Hello World!\n";
    unsigned char *enc_key = NULL, *ciphertext = NULL, *plaintext = NULL;
    int iv_len = 0, enc_key_len = 0, ciphertext_len = 0, plaintext_len = 0, i;

    if ((pubkeyfile = fopen("pubkey.pem", "r")) == NULL) {
        printf("Failed to open public key for reading\n");
        goto err;
    }
    if ((pubkey = PEM_read_PUBKEY(pubkeyfile, &pubkey, NULL, NULL)) == NULL) {
        fclose(pubkeyfile);
        goto err;
    }
    fclose(pubkeyfile);

    if ((privkeyfile = fopen("privkey.pem", "r")) == NULL) {
        printf("Failed to open private key for reading\n");
        goto err;
    }
    if ((privkey = PEM_read_PrivateKey(privkeyfile, &privkey, NULL, NULL))
            == NULL) {
        fclose(privkeyfile);
        goto err;
    }
    fclose(privkeyfile);

    if (!envelope_seal(pubkey, message, strlen(message), &enc_key, &enc_key_len,
                       &iv, &iv_len, &ciphertext, &ciphertext_len))
        goto err;

    printf("Ciphertext:\n");
    for (i = 0; i < ciphertext_len; i++)
        printf("%02x", ciphertext[i]);
    printf("\n");

    printf("Encrypted Key:\n");
    for (i = 0; i < enc_key_len; i++)
        printf("%02x", enc_key[i]);
    printf("\n");

    printf("IV:\n");
    for (i = 0; i < iv_len; i++)
        printf("%02x", iv[i]);
    printf("\n");

    if (!envelope_open(privkey, ciphertext, ciphertext_len, enc_key,
                       enc_key_len, iv, &plaintext, &plaintext_len))
        goto err;

    plaintext[plaintext_len] = '\0';
    printf("Plaintext: %s\n", plaintext);

    ret = 0;
 err:
    if (ret != 0) {
        printf("Error\n");
        ERR_print_errors_fp(stdout);
    }
    EVP_PKEY_free(pubkey);
    EVP_PKEY_free(privkey);
    free(iv);
    free(enc_key);
    free(ciphertext);
    free(plaintext);

    return ret;
}
Matt Caswell
  • 8,167
  • 25
  • 28