I have added the link of two projects, exactly same dummy code but different project types, first one is "Application (.exe)" {link to download} and the second one is "Dynamic Library (dll)" {link to download}. I am using Visual Studio Enterprise 2015, with the /Gh compiler option for both projects. There is another project included in both solutions that is NOT compiled with /Gh option and has implementation of _penter hook function. Strangely, the first project compiles fine and _penter is reached and functions properly (prints on console). The second one (dll) is throwing compilation error of "LNK2019 unresolved external symbol __penter referenced in function _DllMain@12" when compiled with /Gh switch. Also, I have tried with c and c++ code in both solutions, first one works with both and second works with none.
Configuration of Library Project
Just for reference, code content is pasted below.
Dll Project -- dllmain.c
// dllmain.cpp : Defines the entry point for the DLL application.
#include "stdafx.h"
#include "Profile\Profile.h"
int a = 7;
char ch = 'a';
char y(char c) {
return c + 1;
}
int xx2(int c, int d) {
y(ch++); return c + 3;
}
BOOL APIENTRY DllMain( HMODULE hModule,
DWORD ul_reason_for_call,
LPVOID lpReserved
)
{
a = xx2(5, 100);
a = xx2(70, 200);
a = xx2(150, 300);
ch = y(ch);
ch = y(ch);
ch = y(ch);
switch (ul_reason_for_call)
{
case DLL_PROCESS_ATTACH:
case DLL_THREAD_ATTACH:
case DLL_THREAD_DETACH:
case DLL_PROCESS_DETACH:
break;
}
return TRUE;
}
Exe Project -- sample.c
// Sample.cpp : Defines the entry point for the console application.
//
#include "stdafx.h"
#include "Profile\Profile.h"
int a = 7;
char ch = 'a';
char y(char c) {
printf("%c", c); return c + 1;
}
int xx2(int c, int d) {
printf("%d", c); y(ch++); return c + 3;
}
int main() {
//PassSomething(xx2, "int xx2(int c, int d)\0");
a = xx2(5, 100);
a = xx2(70, 200);
a = xx2(150, 300);
ch = y(ch);
ch = y(ch);
ch = y(ch);
}
Included Profiler Project - header profile.h
#include <windows.h>
#include <assert.h>
#include <tchar.h>
typedef struct
{
const void * function; // function address
const TCHAR * parameter; // formatting string for parameter decoding, '?' for class method
const TCHAR * returnval; // formatting string for return value decoding,
} Signature;
typedef struct
{
void * Pointer;
char Name[100];
char Parameters[20][100];
char ReturnType[20];
} Function;
Included Profiler Project - code profile.cpp
#define STRICT
#include "profile.h"
#define MAX_DEPTH 512 // good enough for 512 levels of nesting
void * Stack[MAX_DEPTH * 2]; // NOT multi-thread safe
int SP = 0;
int depth = 0;
void _stdcall ExitFunc(unsigned * pStack)
{
TCHAR temp[MAX_PATH];
Signature * pSig;
OutputDebugString("Exit \n");
SP--;
pSig = (Signature *)Stack[SP];
SP--;
pStack[0] = (unsigned)Stack[SP]; // change return address to point to original caller
}
extern "C" __declspec(naked) void __cdecl _pexit()
{
_asm
{
push eax // function return value and placehold for return to original caller
pushad // save all general purpose registers
mov eax, esp // current stack pointer
add eax, 32 // stack pointer before pushad
push eax // push pointer to where EAX is saved as parameter to ExitFunc
call ExitFunc
popad // restore general registers
ret // return to original caller
}
}
void _stdcall EnterFunc(unsigned * pStack)
{
char temp[MAX_PATH];
Signature * pSig;
void * pCaller;
pCaller = (void *)(pStack[0] - 5); // the instruction for calling _penter is 5 bytes long
pSig = NULL;//FuncTable;
OutputDebugString("Entered \n");
Stack[SP++] = (void *)pStack[1]; // save return address to original caller
Stack[SP++] = pSig; // save functions signature
pStack[1] = (unsigned)_pexit; // HACK stack to link to _pexit
depth++;
}
void _stdcall EnterFunc0(unsigned * pStack)
{
EnterFunc(pStack); // process the call
}
extern "C" __declspec(naked) void __cdecl _penter()
{
_asm
{
pushad // save all general purpose registers
mov eax, esp // current stack pointer
add eax, 32 // stack pointer before pushad
push eax // push pointer to return address as parameter to EnterFunc0
call EnterFunc0
popad // restore general purpose registers
ret // start executing original function
}
}
NOTE: Complete solution links are provided inline.