Using OpenSSL to verify a file signed with Java

Question

A signature that I created with Java fails to be verified with OpenSSL. I would be thankful for any hint.

Here is what I did:

1) Generated a key pair with the Java key tool:

>> keytool -genkeypair -alias signing_test -keyalg DSA -keysize 2048 -keypass test123 
   -dname 'CN=tilo, OU=dev, O=company, L=SF, ST=CA, C=US' 
   -keystore test_store.jks -storepass test123

2) Used Java to sign a file:

import java.io.*;
import java.nio.file.*;
import java.security.*;

public class SignatureTest {
  public static void main(String[] args) throws Exception {
    String path = "..."; // Local Path
    String keyStoreFile = path + "test_store.jks";
    String dataFile = path + "picture.jpg";
    String signatureFile = path + "signature.data";

    KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
    try (FileInputStream inputStream = new FileInputStream(keyStoreFile)) {
      keyStore.load(inputStream, "test123".toCharArray());
    }

    Signature sig = Signature.getInstance("SHA256withDSA"); // requires Java 8
    sig.initSign((PrivateKey)keyStore.getKey("signing_test", "test123".toCharArray()));
    byte[] data = Files.readAllBytes(Paths.get(dataFile));
    sig.update(data);
    Files.write(Paths.get(signatureFile), sig.sign());  
  }
}

3) Exported the certificate with the Java key tool:

>> keytool -exportcert -alias signing_test -keypass test123 -keystore test_store.jks 
   -storepass test123 -rfc -file public_key.pem

4) Use OpenSSL to verify the signature:

#include <fstream>
#include <string>
#include <vector>
#include <stdio.h>
#include <openssl/pem.h>
#include <openssl/x509.h>
#include <openssl/err.h>

void ReadData(const std::string& fileName, std::vector<unsigned char>& data) {
  std::ifstream inFile{ fileName.c_str(), 
         std::ios::in | std::ios::binary | std::ios::ate };
  auto size = inFile.tellg();
  data.resize(size);
  inFile.seekg(0, std::ios::beg);
  inFile.read(reinterpret_cast<char*>(data.data()), size);
  inFile.close();
}

int main(int argc, char** argv) {
  std::string path = "..."; // Local Path
  std::string publicKeyFileName = path + "public_key.pem";
  std::string dataFileName = path + "picture.jpg";
  std::string signatureFileName = path + "signature.data";

  // Read certificate
  FILE *publicKeyFile = fopen(publicKeyFileName.c_str(), "rb");
  X509 *x509 = PEM_read_X509(publicKeyFile, nullptr, nullptr, nullptr);
  fclose(publicKeyFile);

  // Read signature & data
  std::vector<unsigned char> data;
  ReadData(dataFileName, data);
  std::vector<unsigned char> sig;
  ReadData(signatureFileName, sig);

  // Verify
  EVP_PKEY *publicKey = X509_get_pubkey(x509);

  EVP_MD_CTX messageDigest;
  EVP_MD_CTX_init(&messageDigest);

  EVP_VerifyInit(&messageDigest, EVP_sha256());
  EVP_VerifyUpdate(&messageDigest, data.data(), data.size());
  int result = EVP_VerifyFinal(&messageDigest, sig.data(), sig.size(), publicKey);

  if (result > 0) {
    std::cout << "Verifed\n";
  } else if (result == 0) {
    std::cout << "Failure\n";
  } else {
    ERR_print_errors_fp(stderr);
  }

  // Cleanup
  EVP_MD_CTX_cleanup(&messageDigest);
  EVP_PKEY_free(publicKey);
  X509_free(x509);
}

Unfortunately, this prints Failure.