Issue porting Decryption from Windows CryptoAPI to linux libmcrypt

Question

I am trying to port my program from Windows to Linux. The windows program uses Window CryptoAPI and linux is using libmcrypt.

Here is the Windows code:

#include <windows.h>
#include <iostream>
#include <iomanip>
#include <vector>
#include <string>
#include <exception>

using namespace std;

class CryptError:public exception{
    public:
        CryptError(){}
};


#define CHECK_RET(x) if(x == FALSE) {retval = GetLastError(); throw CryptError();};

LONG Decrypt(const string &key, std::vector<BYTE> &data){
    LONG retval = 0;
    try{
        HCRYPTPROV hCrypt;
        HCRYPTHASH hHash;
        HCRYPTKEY hKey;
        CHECK_RET(CryptAcquireContext(&hCrypt, NULL, NULL, PROV_RSA_FULL, 0));
        CHECK_RET(CryptCreateHash(hCrypt, CALG_MD5, 0, 0, &hHash));
        CHECK_RET(CryptHashData(hHash, reinterpret_cast<const BYTE *>(key.c_str()), key.size(), 0));

        CHECK_RET(CryptDeriveKey(hCrypt, CALG_RC2, hHash, MAKELPARAM(CRYPT_EXPORTABLE, 80), &hKey));

        BYTE tempVal[200];
        DWORD len = 200;
        CryptGetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, tempVal, &len, 0);

        len = 200;
        CryptGetKeyParam(hKey, KP_MODE, tempVal, &len, 0);

        len = 200;
        CryptExportKey(hKey, NULL, PLAINTEXTKEYBLOB, 0, tempVal, &len);

        len = 200;
        CryptGetKeyParam(hKey, KP_IV, tempVal, &len, 0);

        DWORD count = data.size();
        CHECK_RET(CryptDecrypt(hKey, 0, TRUE, 0, &(data[0]), &count));
        data.resize(count);
    }catch(CryptError &e){
    }

    return retval;
}

int main(void){
    BYTE data[9] = {0xdc,0x3d,0x96,0x23,0x29,0xdd,0x1b,0x2f, 0};
    vector<BYTE> vData(data, data + 8);

    Decrypt("PNEMAIL", vData);

    cerr << "vData: ";
    int len = vData.size();
    for(int i = 0; i < len; i++){
        if(i > 0)
            cerr << ',';
        cerr << hex << setw(2) << setfill('0') << (int)(vData[i]);
    }
    cerr << endl;

    return 0;
} 

When the program is run, it returns:

vData: 42,46,30,41,43,34,31

The Q&D linux version looks like this:

#include <mcrypt.h>
#include <iostream>
#include <iomanip>
#include <string>
#include <openssl/md5.h>
#include <stdint.h>
#include <stdexcept>
#include <vector>
#include <valarray>
#include <memory.h>

using namespace std;

class MCrypt{
    private:
        MCRYPT mcrypt;
    public:
        MCrypt(char *algorithm, char* algorithm_directory, char *mode, char* mode_directory){
            mcrypt = mcrypt_module_open(algorithm, algorithm_directory, mode, mode_directory);
            if(mcrypt == MCRYPT_FAILED)
                throw runtime_error("MCrypt init failed");
        }

        int init(void *key, int lenofkey, void *IV){
            return mcrypt_generic_init(mcrypt, key, lenofkey, IV);
        }

        int enc_get_iv_size(){
            return mcrypt_enc_get_iv_size(mcrypt);
        }

        int deinit(){
            return mcrypt_generic_deinit(mcrypt);
        }

        int decrypt(void *data, int len){
            mdecrypt_generic(mcrypt, data, len);
        }

        ~MCrypt(){
            deinit();
            mcrypt_module_close(mcrypt);
        }
};

#ifdef DEBUG
void inline printArrayFunc(const char *start, const uint8_t *data, int len){
    // DEBUG: print value of $key1
    cerr << start;
    for(int i = 0; i < len; i++){
        if(i > 0)
            cerr << ',';
        cerr << hex << setw(2) << setfill('0') << (int)(data[i]);
    }
    cerr << endl;
}
#define printArray(start, data, len) printArrayFunc(start, data, len)
#else
#define printArray(start, data, len)
#endif

int main(void){
    uint8_t data[8] = {0xdc,0x3d,0x96,0x23,0x29,0xdd,0x1b,0x2f};
    const char *sKey1 = "PNEMAIL";
    const int key1Len = 7;

    uint8_t *dataPtr = &(data[0]);

    uint8_t key1[17];
    key1[16] = 0;

    // Hash sKey1
    MD5(reinterpret_cast<const unsigned char *>(sKey1), key1Len, key1);

    MCrypt mcrypt(MCRYPT_RC2, NULL, MCRYPT_CBC, NULL);

    vector<uint8_t> iv(mcrypt.enc_get_iv_size(), 0);

    // Use the first 80-bits of key1
    mcrypt.init(key1, 10, &(iv[0]));

    mcrypt.decrypt(dataPtr, 8);

    printArray("vData: ", dataPtr, 8);

    return 0;
}

When the program is run, it returns:

vData: 4d,3d,82,71,88,d2,d5,4b

I've check that both programs are using the same data.

• CryptDeriveKey creates a key 07,f1,e2,ea,d4,c8,79,74,03,a6 (according to CryptExportKey), the same as the first 10 bytes of the md5 generated in Linux (which I shorten to match the requested 80-bit key).
• Neither are using a salt on the algorithm (or at least are not reporting as such)
• They are both using an 8-byte IV of {0,0,0,0,0,0,0,0}
• They are both using the RC2 algorithm
• They are both using CBC mode

I cannot figure out why they are returning different data. Any assistance would be greatly appreciated.