I have encounted a strange communication problem when using winhttp to program http client and using openssl to program http server.
I use wireshark to analyze the communication. Everything seems ok when handshake between the two, and the handshake procedure is really quick. But after the handshake, winhttp client taken a really long time send a http request, then server received the request (the received packet is right) then server attempted to send response with ssl_write, but client cannot receive this packet.
please see notes in my code for advanced info.
Below is client's main communication code.
BOOL CHttpClient :: _Synchronize(BYTE* pbRequest, DWORD dwRequestCb,
BYTE** pbResponse, DWORD* dwResponseCb)
{
DWORD dwErrorCode = 0;
DWORD dwStatusCode = 0;
DWORD dwSize = sizeof(DWORD);
DWORD dwLastStatusCode = 0;
BOOL bDone = FALSE;
if (FALSE == DoConnectAndOpenRequest(dwRequestCb == 0))
{
return FALSE;
}
while (!bDone)
{
if (!WinHttpSendRequest(m_hRequest, NULL, 0, pbRequest, dwRequestCb, dwRequestCb, NULL))
{
dwErrorCode = GetLastError();
switch (dwErrorCode)
{
case ERROR_WINHTTP_CANNOT_CONNECT:
case ERROR_WINHTTP_TIMEOUT:
m_dwDefaultAddrIndex = (m_dwDefaultAddrIndex + 1) % m_lpConfig->dwHttpAddrNum;
case ERROR_WINHTTP_CONNECTION_ERROR:
if (FALSE == DoConnectAndOpenRequest(dwRequestCb == 0))
{
return FALSE;
}
continue;
case 12175:
continue;
case ERROR_WINHTTP_CLIENT_AUTH_CERT_NEEDED:
continue;
default:
return FALSE;
}
}
// client always wait here for server's response and get a error code as 12002--ERROR_INTERNET_TIMEOUT
if (!WinHttpReceiveResponse(m_hRequest, NULL))
{
dwErrorCode = GetLastError(); // get error code 12002--ERROR_INTERNET_TIMEOUT
switch (dwErrorCode)
{
case ERROR_WINHTTP_RESEND_REQUEST:
continue;
case ERROR_WINHTTP_CLIENT_AUTH_CERT_NEEDED:
continue;
default:
if (FALSE == DoConnectAndOpenRequest(dwRequestCb == 0))
{
return FALSE;
}
continue;
}
}
if (!DoServerAuth())
{
m_dwDefaultAddrIndex = (m_dwDefaultAddrIndex + 1) % m_lpConfig->dwHttpAddrNum;
if (FALSE == DoConnectAndOpenRequest(dwRequestCb == 0))
{
return FALSE;
}
continue;
}
if (!WinHttpQueryHeaders(m_hRequest, WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER,
NULL, &dwStatusCode, &dwSize, NULL))
{
return FALSE;
}
switch (dwStatusCode)
{
// success
case 200:
bDone = TRUE;
break;
// proxy need authentication
case 407:
if (dwLastStatusCode == 407)
{
bDone = TRUE;
}
else if (!DoProxyAuth())
return FALSE;
break;
default:
return FALSE;
}
dwLastStatusCode = dwStatusCode;
}
DWORD dwBytesAvailable = 0;
DWORD dwBytesTransfered = 0;
BYTE* pbTmpBuf = new BYTE [GetMaxTransferCb()];
// read http server reply
*dwResponseCb = 0;
do
{
if (!WinHttpQueryDataAvailable(m_hRequest, &dwBytesAvailable))
{
return FALSE;
}
if (!WinHttpReadData(m_hRequest, pbTmpBuf + *dwResponseCb, dwBytesAvailable,
&dwBytesTransfered))
{
return FALSE;
}
*dwResponseCb += dwBytesTransfered;
} while (dwBytesAvailable > 0);
*pbResponse = new BYTE [*dwResponseCb];
memcpy(*pbResponse, pbTmpBuf, *dwResponseCb);
RELEASE_ARRAY(pbTmpBuf);
// all done
return TRUE;
}
BOOL CHttpClient :: DoConnectAndOpenRequest(BOOL bGetVerb)
{
BOOL bOK = FALSE;
DWORD dwReqObjNameCch = (DWORD)rand() % (20 - 5) + 5;
BYTE* lpReqObjName = new BYTE [(dwReqObjNameCch + 6) * sizeof(TCHAR)];
DWORD dwOptionCode = 0;
__try
{
// release old handles
RELEASE_HINTERNET(m_hConnect);
RELEASE_HINTERNET(m_hRequest);
// connect to default address, change of default address is not my duty
if (NULL == (m_hConnect = WinHttpConnect(m_hSession,
m_lpConfig->lpHttpAddrs[m_dwDefaultAddrIndex].tszAddr,
m_lpConfig->lpHttpAddrs[m_dwDefaultAddrIndex].wPort, 0)))
__leave;
// random generate request object name
// now this method is slow and bad, we can improve later
if (!CryptGenRandom(m_hCryptProv, dwReqObjNameCch * sizeof(TCHAR), lpReqObjName))
return FALSE;
for (DWORD i = 0; i < dwReqObjNameCch * sizeof(TCHAR); i += sizeof(TCHAR))
{
lpReqObjName[i] = (lpReqObjName[i] % 26) + 97;
lpReqObjName[i + 1] = 0;
}
((LPTSTR)lpReqObjName)[dwReqObjNameCch] = _T('.');
((LPTSTR)lpReqObjName)[dwReqObjNameCch + 1] = _T('h');
((LPTSTR)lpReqObjName)[dwReqObjNameCch + 2] = _T('t');
((LPTSTR)lpReqObjName)[dwReqObjNameCch + 3] = _T('m');
((LPTSTR)lpReqObjName)[dwReqObjNameCch + 4] = _T('l');
((LPTSTR)lpReqObjName)[dwReqObjNameCch + 5] = 0;
// open request
if (NULL == (m_hRequest = WinHttpOpenRequest(m_hConnect, bGetVerb ? L"GET" : L"POST",
(LPTSTR)lpReqObjName, L"HTTP/1.1", WINHTTP_NO_REFERER,
WINHTTP_DEFAULT_ACCEPT_TYPES, WINHTTP_FLAG_SECURE)))
__leave;
// set cert options
if (m_lpConfig->bSsl)
{
dwOptionCode = SECURITY_FLAG_IGNORE_CERT_CN_INVALID |
SECURITY_FLAG_IGNORE_UNKNOWN_CA;
if (!WinHttpSetOption(m_hRequest, WINHTTP_OPTION_SECURITY_FLAGS, &dwOptionCode,
sizeof(DWORD)))
__leave;
}
// all done
bOK = TRUE;
}
__finally
{
if (!bOK)
{
RELEASE_HINTERNET(m_hConnect);
RELEASE_HINTERNET(m_hRequest);
}
RELEASE_ARRAY(lpReqObjName);
}
return bOK;
}
Below is server's main communication code
#include <strsafe.h>
BOOL CSslServer :: StartService(SSL_CONFIG* lpSslServerConfig)
{
WSADATA lpWSAData;
WSAStartup(MAKEWORD(2, 2), &lpWSAData);
CRYPTO_malloc_init(); // Initialize malloc, free, etc for OpenSSL's use
SSL_library_init(); // Initialize OpenSSL's SSL libraries
SSL_load_error_strings(); // Load SSL error strings
OpenSSL_add_all_algorithms(); // Load all available encryption algorithms
m_lpConfig = lpSslServerConfig;
m_SslCtx = SSL_CTX_new(SSLv3_server_method());
SSL_CTX_set_default_passwd_cb_userdata(m_SslCtx, (void*)"123456");
if (1 > SSL_CTX_use_certificate_file(m_SslCtx, m_lpConfig->szCertFileDir, SSL_FILETYPE_PEM))
{
return FALSE;
}
if (1 > SSL_CTX_use_PrivateKey_file(m_SslCtx, m_lpConfig->szSkeyFileDir, SSL_FILETYPE_PEM))
{
return FALSE;
}
SSL_CTX_set_cipher_list(m_SslCtx, "ALL");
SSL_CTX_set_verify(m_SslCtx, SSL_VERIFY_NONE, NULL);
if (NULL == (m_hServerQuitEvt = CreateEvent(NULL, FALSE, FALSE, NULL)))
return FALSE;
if (NULL == (m_hWorkThread = CreateThread(NULL, 0, SslServerWorkThread, this, 0, NULL)))
{
StopService();
return FALSE;
}
return TRUE;
}
BOOL CSslServer :: StopService()
{
DWORD dwExitCode = 0;
if (m_hWorkThread != NULL)
{
SetEvent(m_hServerQuitEvt);
WaitForSingleObject(m_hWorkThread, 30000);
if(GetExitCodeThread(m_hWorkThread, &dwExitCode))
{
if(dwExitCode == STILL_ACTIVE)
TerminateThread(m_hWorkThread, 0);
}
}
SSL_CTX_free(m_SslCtx);
RELEASE_HANDLE(m_hServerQuitEvt);
RELEASE_HANDLE(m_hWorkThread);
m_WorkThreadList.empty();
return TRUE;
}
DWORD WINAPI SslServerWorkThread(LPVOID lpParam)
{
CSslServer* lpServer = (CSslServer*)lpParam;
SOCKET hListenSocket = INVALID_SOCKET;
TIMEVAL* lpWaitTime = new TIMEVAL;
fd_set ListenFds;
DWORD dwWaitForQuit = 0;
int nSelectRel = 0;
int nSize = 0;
SESSION_CONTEXT* lpSessionCtx = NULL;
WORK_THREAD_CONTEXT* lpWorkThreadCtx = NULL;
hListenSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
bind(hListenSocket, (sockaddr*)&lpServer->m_lpConfig->SslServerAddr,
sizeof(sockaddr_in));
if (SOCKET_ERROR == listen(hListenSocket, SOMAXCONN))
{
RELEASE_SOCKET(hListenSocket);
RELEASE(lpWaitTime);
return 0;
}
FD_ZERO(&ListenFds);
FD_SET(hListenSocket, &ListenFds);
lpWaitTime->tv_sec = 30;
lpWaitTime->tv_usec = 0;
do
{
nSelectRel = select(0, &ListenFds, NULL, NULL, lpWaitTime);
switch (nSelectRel)
{
case SOCKET_ERROR:
goto END;
case 1:
lpSessionCtx = new SESSION_CONTEXT;
nSize = sizeof(sockaddr_in);
if (INVALID_SOCKET == (lpSessionCtx->hClientSocket = accept(hListenSocket,
(struct sockaddr *)&lpSessionCtx->ClientAddress, &nSize)))
{
RELEASE(lpSessionCtx);
goto END;
}
lpSessionCtx->lpServer = lpServer;
lpSessionCtx->lpWorkConfig = lpServer->m_lpConfig->lpWorkConfig;
lpSessionCtx->SslSession = SSL_new(lpServer->m_SslCtx);
SSL_set_fd(lpSessionCtx->SslSession, lpSessionCtx->hClientSocket);
SSL_accept(lpSessionCtx->SslSession);
lpWorkThreadCtx = new WORK_THREAD_CONTEXT;
lpWorkThreadCtx->hWorkThreadQuitEvt = CreateEvent(NULL, FALSE, FALSE, NULL);
lpWorkThreadCtx->hWorkThread = CreateThread(NULL, 0, WorkMain, lpSessionCtx,
0, NULL);
// WorkMain is not provided here, the function just analyze the received packet and
// invoke SendPacket to send response.
lpSessionCtx->lpWorkThreadCtx = lpWorkThreadCtx;
lpServer->m_WorkThreadList.push_back(lpWorkThreadCtx);
break;
default:
FD_SET(hListenSocket, &ListenFds);
}
dwWaitForQuit = WaitForSingleObject(lpServer->m_hServerQuitEvt, 0);
if (dwWaitForQuit == WAIT_FAILED || dwWaitForQuit == WAIT_ABANDONED)
{
goto END;
}
} while (dwWaitForQuit != WAIT_OBJECT_0);
END:
RELEASE_SOCKET(hListenSocket);
RELEASE(lpWaitTime);
return 0;
}
// I have examined the received packet. This function is ok
BOOL CSslServer :: ParseClientPacket(SSL* ssl, BYTE** lpBuf, DWORD* dwTransCb)
{
char* lpHeader = NULL;
BOOL bOk = FALSE;
int nRet = 0;
int nHeaderBufCb = 1024;
int nIndex = 0;
DWORD dwPackBufCb = 0;
__try
{
*dwTransCb = 0;
lpHeader = new char [nHeaderBufCb];
memset(lpHeader, 0, nHeaderBufCb);
if (0 >= (nRet = SSL_read(ssl, lpHeader, nHeaderBufCb)))
__leave;
nHeaderBufCb = lstrlenA(lpHeader);
for (nIndex = 20; nIndex < nHeaderBufCb - 15; nIndex++)
{
if (0 == memcmp(lpHeader + nIndex, "Content-Length: ", 16))
{
sscanf_s(lpHeader + nIndex + 16, "%d", &dwPackBufCb);
break;
}
}
if (nIndex == nHeaderBufCb - 15)
__leave;
for (nIndex += 16; nIndex < nHeaderBufCb - 4; nIndex++)
{
if (0 == memcmp(lpHeader + nIndex, "\r\n\r\n", 4))
break;
}
if (nIndex == nHeaderBufCb - 4)
__leave;
*lpBuf = new BYTE [dwPackBufCb];
if (nRet - nIndex - 4 > 0)
{
memcpy(*lpBuf, lpHeader + nIndex + 4, nRet - nIndex - 4);
*dwTransCb = nRet - nIndex - 4;
}
while (*dwTransCb < dwPackBufCb)
{
if (0 >= SSL_read(ssl, *lpBuf + *dwTransCb, dwPackBufCb - *dwTransCb))
{
bOk = TRUE;
__leave;
}
*dwTransCb += nRet;
}
bOk = TRUE;
}
__finally
{
RELEASE_ARRAY(lpHeader);
}
return bOk;
}
BOOL CSslServer :: SendPacket(SSL* ssl, BYTE* lpBuf, DWORD cb)
{
LPSTR lpHttpPacket = NULL;
DWORD dwHeaderLen = 0;
BOOL bOk = FALSE;
char szTime[50] = {0};
time_t lTime;
struct tm GmtTime;
__try
{
lpHttpPacket = new char [200 + cb];
memset(lpHttpPacket, 0, 200 + cb);
time(&lTime);
_gmtime64_s(&GmtTime, &lTime);
strftime(szTime, 50, "Date: %a, %d %b %Y %H:%M:%S GMT\r\n", &GmtTime);
StringCchPrintfA(lpHttpPacket, 200,
"HTTP/1.1 200 OK\r\nServer: Microsoft-IIS/8.0\r\nConnection: Keep-Alive\r\n%sContent-Type: text/html\r\nContent-Length: %d\r\n\r\n",
szTime, cb);
dwHeaderLen = lstrlenA(lpHttpPacket);
memcpy(lpHttpPacket + dwHeaderLen, lpBuf, cb);
if (0 >= SSL_write(ssl, lpHttpPacket, cb)) // the packet send by this sentence cannot be received by client
__leave; // observe by wireshark, this sentence send a ssl reassembled pdu
bOk = TRUE;
}
__finally
{
RELEASE_ARRAY(lpHttpPacket);
}
return bOk;
}
And below is wireshark snap.
NO. Time Source Destination Protocol Length Info
15951 3691.1 .23 .98 SSL 126 client hello
15952 3691.1 .98 .23 SSLv3 1109 server hello
15953 3691.1 .23 .98 SSLv3 386 client key exchange, change cipher spec, finished
15954 3691.1 .98 .23 SSLv3 121 change cipher spec, finished
16029 3706.6 .23 .98 http 301 POST ...... HTTP/1.1
16060 3711.9 .98 .23 SSLv3 83 [SSL segment of a ressembled PDU]
It takes me really a long time to solve this problem. I really hope someone and handle this.
best wishes
