0

I am trying to decrypt data encrypted with AES in Java, using the Wincrypt API in C++. On the C++ side, I get the error code 80090005 at the CryptDecrypt stage. What exactly is the C++ Wincrypt API default algorithm? Is my algorithm selection wrong?

Java Encrypt code

public static byte[] encryptData(String keyStr, byte[] data) throws Exception {
    byte[] keyBytes = new byte[16]; // 16 bayt (128 bit)
    byte[] inputKeyBytes = keyStr.getBytes("UTF-8");

    if (inputKeyBytes.length <= keyBytes.length) {
        System.arraycopy(inputKeyBytes, 0, keyBytes, 0, inputKeyBytes.length);
    } else {
        System.arraycopy(inputKeyBytes, 0, keyBytes, 0, keyBytes.length);
    }

    SecretKeySpec keySpec = new SecretKeySpec(keyBytes, "AES");
    Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
    cipher.init(Cipher.ENCRYPT_MODE, keySpec);
    byte[] encrypted = cipher.doFinal(data);
    return encrypted;
}

public static void main(String[] args) {
    try {
        String key = "3igcZhRdWq96m3GUmTAiv9";

        File in = new File("C:\\test\\Test.class");
        File out = new File("C:\\test\\TestOUT.class");

        byte[] fileBytes = Files.readAllBytes(in.toPath());
        byte[] encryptedData = encryptData(key, fileBytes);

        FileUtils.writeByteArrayToFile(out, encryptedData);

        System.out.println("Done!");
    } catch (Exception e) {
        e.printStackTrace();
    }
}

C++ Decrypt Code

std::vector<BYTE> readFileBytes() {
    std::ifstream file("C:\\test\\TestOUT.class", std::ios::binary);

    if (!file) {
    }

    file.seekg(0, std::ios::end);
    std::streampos fileSize = file.tellg();
    file.seekg(0, std::ios::beg);

    std::vector<BYTE> byteVector(fileSize);
    file.read(reinterpret_cast<char*>(byteVector.data()), fileSize);

    file.close();

    return byteVector;
}

int decryptData(wchar_t* key_str, char* bytes, DWORD bytes_len) {
    const size_t len = lstrlenW(key_str);
    const size_t key_size = len * sizeof(key_str[0]); // size in bytes

    printf("Key: %#x\n", key_str);
    printf("Key len: %#x\n", len);
    printf("Key size: %#x\n", key_size);

    printf("key_size-len: %d", bytes_len);

    BOOL bResult = FALSE;
    //wchar_t info[] = L"Microsoft Enhanced RSA and AES Cryptographic Provider";
    HCRYPTPROV hProv;
    if (!CryptAcquireContext(&hProv,
                             NULL,
                             MS_ENH_RSA_AES_PROV,
                             PROV_RSA_AES,
                             CRYPT_VERIFYCONTEXT)) {
        printf("CryptAcquireContext failed: %x\n", GetLastError());
        CryptReleaseContext(hProv, 0);
        return 1;
    }
    HCRYPTHASH hHash;
    if (!CryptCreateHash(hProv, CALG_SHA_256, 0, 0, &hHash)) {
        printf("CryptCreateHash failed: %x\n", GetLastError());
        CryptReleaseContext(hProv, 0);
        return 1;
    }

    if (!CryptHashData(hHash, (BYTE*) key_str, key_size, 0)) {
        printf("CryptHashData Failed : %#x\n", GetLastError());
        return 1;
    }

    HCRYPTKEY hKey;
    if (!CryptDeriveKey(hProv, CALG_AES_128, hHash, 0, &hKey)) {
        printf("CryptDeriveKey failed: %x\n", GetLastError());
        CryptReleaseContext(hProv, 0);
        return 1;
    }

    const size_t chunk_size = 16 * 10;
    BYTE* chunk = new BYTE[chunk_size];
    DWORD out_len = 0;

    BOOL isFinal = FALSE;
    DWORD readTotalSize = 0;

    int total_chunks = bytes_len / chunk_size;
    int last_chunk_size = 0;
    if ((last_chunk_size = bytes_len % chunk_size) != 0) total_chunks++;
    printf("total chunks: %d, last chunk size: %d\n", total_chunks, last_chunk_size);

    std::vector<BYTE> decryptedData;

    for (int i = 0; i < total_chunks; i++) {
        std::memcpy(chunk, bytes + (i * chunk_size), chunk_size);
        out_len = chunk_size;

        printf("current chunk: %d\n", i);
        if (i == (total_chunks - 1)) {
            isFinal = TRUE;
            out_len = last_chunk_size;
            printf("Final chunk set, len: %d = %x\n", out_len, out_len);
        }

        printf("  decrypting %d\n", i);
        if (!CryptDecrypt(hKey, NULL, isFinal, 0, chunk, &out_len)) {
            printf("[-] CryptDecrypt failed: %x\n", GetLastError());
            break;
        }

        // byte array'e yaz
        for (int l = 0; l < out_len; l++) decryptedData.push_back(chunk[l]);

        memset(chunk, 0, chunk_size);
    }

    delete[]chunk; chunk = NULL;

    CryptDestroyHash(hHash);
    CryptDestroyKey(hKey);
    CryptReleaseContext(hProv, 0);

    writeBytesToFile(decryptedData);
}

int main()
{
        wchar_t default_key[] = L"3igcZhRdWq96m3GUmTAiv9";
        wchar_t* key_str = default_key;

        std::vector<BYTE> encrypted_bytes = readFileBytes();
        decryptData(key_str, (char*)encrypted_bytes.data(), encrypted_bytes.size());

        return 0;
}

C++ Console output

Key len: 0x16                                                                                                           Key len: 0x16
Key size: 0x2c
key_size-len: 1568total chunks: 10, last chunk size: 128
current chunk: 0
  decrypting 0
current chunk: 1
  decrypting 1
current chunk: 2
  decrypting 2
current chunk: 3
  decrypting 3
current chunk: 4
  decrypting 4
current chunk: 5
  decrypting 5
current chunk: 6
  decrypting 6
current chunk: 7
  decrypting 7
current chunk: 8
  decrypting 8
current chunk: 9
Final chunk set, len: 128 = 80
  decrypting 9
[-] CryptDecrypt failed: 80090005

I have tried byte[] keyBytes = keyStr.getBytes("UTF-16LE") but it's still not working.

Jan Schultke
  • 17,446
  • 6
  • 47
  • 96
Content
  • 53
  • 8
  • So.. where in your encryption do you actually hash the key string to generate the encryption key? Better still, why aren't you using a proper PBKD (password-based key derivation) operation instead of a single-hash on the decrypt, and none at all on the encrypt (that I can see)? – WhozCraig Jun 11 '23 at 21:19

0 Answers0