OpenSSL in FIPS mode: connection error (elliptic curve routines:ec_GFp_simple_oct2point:invalid encoding)

Question

I built OpenSSL 1.0.2p on Windows (64bit) with the FIPS module. I followed the standard build procedure for the FIPS module, then I built the OpenSSL dynamic libraries using a custom procedure based on Scons.

At runtime, when I enable the FIPS mode, the connection to any secure web server fails with error:

error:10067066:elliptic curve routines:ec_GFp_simple_oct2point:invalid encoding

I wrote a simple code that reproduces the failure (see below). Note that if I build OpenSSL using the standard procedure, the same code connects successfully.

Maybe the error is due to some compile setting I use in my Scons scripts:

['OPENSSL_SYSNAME_WIN32', 'WIN32_LEAN_AND_MEAN', 'L_ENDIAN', 'DSO_WIN32', 'OPENSSL_USE_APPLINK', 'OPENSSL_NO_CAMELLIA', 'OPENSSL_NO_SEED', 'OPENSSL_NO_RC5', 'OPENSSL_NO_MDC2', 'OPENSSL_NO_CMS', 'OPENSSL_NO_JPAKE', 'OPENSSL_NO_CAPIENG', 'OPENSSL_NO_KRB5', 'OPENSSL_NO_DYNAMIC_ENGINE', '_WINDLL', 'OPENSSL_BUILD_SHLIBCRYPTO', 'MK1MF_BUILD', 'OPENSSL_FIPS', 'UNICODE', '_UNICODE', 'OPENSSL_SYSNAME_WINNT', 'MK1MF_PLATFORM_VC_WIN64A']

Any idea of what could be causing the error?

My testing application:

#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <openssl/ssl.h>
#include <openssl/err.h>

int g_fips = 1;

SSL_CTX *g_ctx = NULL;
const SSL_METHOD *g_meth = NULL;

int initSSL()
{
  SSLeay_add_ssl_algorithms();
  SSL_load_error_strings();

  int current_fips = FIPS_mode();
  if (current_fips != g_fips) {
    printf("Setting FIPS mode to %d\n", g_fips);
    int nRet = FIPS_mode_set(g_fips);
    if (nRet == 0) {
      unsigned long nErr = ERR_get_error();
      char sErr[1024];
      ERR_error_string_n(nErr, sErr, sizeof(sErr));
      printf("FIPS_mode_set returned error %lu (%s)\n", nErr, sErr);
      return -1;
    }
    else
      printf("FIPS mode correctly set\n");
  }
  else
    printf("FIPS mode already set to %d\n", g_fips);

  if ((g_meth = SSLv23_client_method()) == NULL) {
    printf("SSLv23_client_method failed\n");
    return -1;
  }
  if ((g_ctx = SSL_CTX_new(g_meth)) == NULL) {
    printf("SSL_CTX_new failed\n");
    return -1;
  }
  SSL_CTX_set_options(g_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
  return 0;
}

SOCKET createSocket(struct sockaddr_in *psocketaddr)
{
  SOCKET socketfd;

  WSADATA wsaData;
  int iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
  if (iResult != NO_ERROR) {
    printf("WSAStartup function failed with error: %d\n", iResult);
    return 0;
  }

  socketfd = socket(AF_INET, SOCK_STREAM, 0);

  psocketaddr->sin_family = AF_INET;
  psocketaddr->sin_addr.s_addr = inet_addr("10.100.90.100");
  psocketaddr->sin_port = htons(443);
  return socketfd;
}

int main()
{
  initSSL();

  sockaddr_in socketaddr;
  SOCKET socketfd = createSocket(&socketaddr);

  SSL *ssl = SSL_new(g_ctx);
  if (!ssl) {
    printf("Error creating SSL structure.\n");
    SSL_CTX_free(g_ctx);
    WSACleanup();
    return -1;
  }

  int err = connect(socketfd, (sockaddr*)&socketaddr, sizeof(socketaddr));
  if (err < 0) {
    printf("Socket returned error #%d,program terminated\n", WSAGetLastError());
    SSL_free(ssl);
    SSL_CTX_free(g_ctx);
    WSACleanup();
    return -1;
  }

  SSL_set_fd(ssl, (int)socketfd);
  err = SSL_connect(ssl);
  if (err < 1) {
    unsigned long nErr = ERR_get_error();
    char sErr[1024];
    ERR_error_string_n(nErr, sErr, sizeof(sErr));
    printf("SSL error #%d in SSL_connect (%d), %s\n", nErr, err, sErr);
    closesocket(socketfd);
    SSL_free(ssl);
    SSL_CTX_free(g_ctx);
    WSACleanup();
    return -1;
  }

  printf("SSL connection on socket %x,Version: %s, Cipher: %s\n",
    (unsigned int)socketfd,
    SSL_get_version(ssl),
    SSL_get_cipher(ssl));

  closesocket(socketfd);
  SSL_free(ssl);
  SSL_CTX_free(g_ctx);
  WSACleanup();
  return 0;
}

This is the output I get:

Setting FIPS mode to 1
FIPS mode correctly set
SSL error #268857446 in SSL_connect (-1), error:10067066:elliptic curve routines:ec_GFp_simple_oct2point:invalid encoding