0

I'm using openssl-1.0.1i with FIPS openssl-fips-2.0.7 (Which I download and compile dynamic dll's).

My program is very simple and I tring to understand why the result of the function RSA_generate_key_ex is changed when I enabled or disabled FIPS mode.

I running the below code the output is:

64
128
Size are diffrent
Press any key to continue . . .

#include "stdafx.h"

#include <stdio.h>
#include <openssl/rsa.h>
#include <iostream>

typedef int (__cdecl *f_FIPS_mode_set)(int);
typedef int (__cdecl *f_RSA_generate_key_ex)(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
typedef RSA* (__cdecl *f_RSA_new)(void);
typedef BIGNUM* (__cdecl *f_BN_new)(void);
typedef int (__cdecl *f_BN_set_word)(BIGNUM *a, unsigned long w);
typedef int (__cdecl *f_RSA_size)(const RSA *rsa);

#define INT_POINTER_FUNCTION( function ) \
    f_##function p_##function = (f_##function)GetProcAddress(hGetProcIDDLL, #function); \
    if (!p_##function) { \
    std::cout << "could not locate the ##function" << std::endl; \
    return EXIT_FAILURE; \
    } 

int _tmain(int argc, _TCHAR* argv[])
{
    HINSTANCE hGetProcIDDLL = LoadLibrary(L"C:\\Users\\amos_zamir\\Desktop\\openssl_project\\openssl-1.0.1i\\out32dll\\libeay32.dll");

  if (hGetProcIDDLL == NULL) {
    std::cout << "cannot locate the .dll file" << std::endl;
    return -1;
  }

  INT_POINTER_FUNCTION( FIPS_mode_set );
  INT_POINTER_FUNCTION( RSA_generate_key_ex );
  INT_POINTER_FUNCTION( RSA_new );
  INT_POINTER_FUNCTION( BN_new );
  INT_POINTER_FUNCTION( BN_set_word );
  INT_POINTER_FUNCTION( RSA_size );


  int status=p_FIPS_mode_set(1);
  if (status == 0)
  {
      std::cout << "cannot FIPS_mode_set" << std::endl;
      goto free_all;
  }


    int             ret = 0;
    RSA             *r = NULL;
    RSA             *r1 = NULL;
    BIGNUM          *bne = NULL;
    BIO             *bp_public = NULL, *bp_private = NULL;

    int             bits = 1024;
    unsigned long   e = RSA_F4;

    // 1. generate rsa key
    bne = p_BN_new();
    ret = p_BN_set_word(bne,e);
    if(ret != 1){
        std::cout << "cannot BN_set_word" << std::endl;
        goto free_all;
    }

    r = p_RSA_new();
    ret = p_RSA_generate_key_ex(r, bits, bne, NULL);
    if(ret != 1){
        std::cout << "cannot RSA_generate_key_ex" << std::endl;
        goto free_all;
    }

    int num_bytes_with_fips=p_RSA_size(r);
    std::cout << num_bytes_with_fips << std::endl;

    status=p_FIPS_mode_set(0);
    if (status == 0)
    {
        std::cout << "cannot FIPS_mode_set" << std::endl;
        goto free_all;
    }

    r1 = p_RSA_new();
    ret = p_RSA_generate_key_ex(r1, bits, bne, NULL);
    if(ret != 1){
        std::cout << "cannot RSA_generate_key_ex" << std::endl;
        goto free_all;
    }

    int num_bytes_without_fips=p_RSA_size(r1);
    std::cout << num_bytes_without_fips << std::endl;

    if(num_bytes_without_fips!= num_bytes_with_fips)
    {
        std::cout << "Size are diffrent" << std::endl;
    }


free_all:
    system("pause");
     return 0;
}

Moreover when FIPS is enabled and I encrypt(RSA_public_encrypt) with RSA object which RSA_size function returns 64 the output buffer length is 128, which I think that is a bug because the manual says

RSA_public_encrypt() returns the size of the encrypted data (i.e., RSA_size(rsa)).

https://www.openssl.org/docs/crypto/RSA_public_encrypt.html/

There is something that I'm doing wrong or it's a bug(which I very skeptical).

  • 128 bytes should be the effective size of the key and indeed the *maximum* size of the signature or encryption result (as they are both related to the size of the modulus). 64 seems a wrong value. But I cannot see at first glance if that's because of OpenSSL or a programming mistake. – Maarten Bodewes Aug 24 '14 at 23:05
  • Sorry for the many edits, going to sleep now ... – Maarten Bodewes Aug 24 '14 at 23:06

0 Answers0