What is an API Call?

  • An API (Application Programming Interface) is a set of functions provided by the operating system or a library — like CreateFile(), ReadProcessMemory(), or MessageBox().
  • These functions are called by programs to interact with the system, hardware, or other programs.

What is Hooking?

  • Hooking means intercepting a function call so that you can run your own code before, after, or instead of the original function.
    Think of it as placing a “trap” or “detour” on a function.---

What is API Hooking Specifically?

It’s the act of modifying or intercepting calls to standard system functions (APIs) — such as Windows API functions.
There are many ways to do it:

  • Modify function pointers (like in IAT, the Import Address Table),
  • Overwrite function code (e.g., with a jump to your own code),
  • Use system-level hooks (SetWindowsHookEx), etc.

How Antivirus (AV) Uses It.

  • Antivirus and EDR (Endpoint Detection and Response) tools hook common Windows APIs to:
  • Detect suspicious behavior (like injecting code into another process),
  • Log sensitive calls (like WriteProcessMemory, NtCreateThreadEx),
  • Block dangerous operations in real time.

So, API hooking is used both offensively (by malware) and defensively (by security software).

Simple dll Code

//pet.dll
#include <windows.h>
#pragma comment (lib, "user32.lib")
 
BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) {
	switch (ul_reason_for_call) {
		case DLL_PROCESS_ATTACH:
			break;
		case DLL_PROCESS_DETACH:
			break;
		case DLL_THREAD_ATTACH:
			break;
		case DLL_THREAD_DETACH:
			break;
	}
	return TRUE;
}
extern "C" {
	__declspec(dllexport) int _cdecl Cat(LPCTSTR say) {
		MessageBox(NULL, say, "=^..^=", MB_OK);
		return 1;
	}
}
extern "C" {
	__declspec(dllexport) int _cdecl Mouse(LPCTSTR say) {
		MessageBox(NULL, say, "<:3()~~", MB_OK);
		return 1;
	}
}
extern "C" {
	__declspec(dllexport) int _cdecl Frog(LPCTSTR say) {
		MessageBox(NULL, say, "8)~", MB_OK);
		return 1;
	}
}
extern "C" {
	__declspec(dllexport) int _cdecl Bird(LPCTSTR say) {
		MessageBox(NULL, say, "<(-)", MB_OK);
		return 1;
	}
}

We have created a DLL which exported function: Cat, Mouse, Frog, Brid with one parameter.
And in output it just pop-up a message.

Compile

x86_64-w64-mingw32-gcc -shared -o pet.dll pet.cpp -fpermissive

C Code


//cat.cpp
#include <windows.h>
 
typedef int (__cdecl *CatProc)(LPCTSTR say);
typedef int (__cdecl *BirdProc)(LPCTSTR say);
 
int main(void) {
	HINSTANCE petDll;
	CatProc catFunc;
	BirdProc birdFunc;
	BOOL freeRes;
	
	petDll = LoadLibrary("pet.dll");
	
	if (petDll != NULL) {
		catFunc = (CatProc) GetProcAddress(petDll, "Cat");
		birdFunc = (BirdProc) GetProcAddress(petDll, "Bird");
		if ((catFunc != NULL) && (birdFunc != NULL)) {
			(catFunc) ("meow-meow");
			(catFunc) ("mmmmeow");
			(birdFunc) ("tweet-tweet");
		}
		freeRes = FreeLibrary(petDll);
	}
	
	return 0;
}

Compile and Run

Compile

x86_64-w64-mingw32-g++ -O2 cat.cpp -o cat.exe \
-mconsole -I/usr/share/mingw-w64/include/ -s \
-ffunction-sections -fdata-sections -Wno-write-strings \
-fno-exceptions -fmerge-all-constants -static-libstdc++ \
-static-libgcc -fpermissive

Run

.\cat.exe



Simple hooking

Program 1

Code

//hooking.cpp
#include <windows.h>
#include <stdio.h>
 
// Function pointer type
typedef int (__cdecl *CatProc)(LPCTSTR);
 
// Buffer for saving original bytes
char originalBytes[12];
FARPROC hookedAddress;
 
// Our hook function
int __stdcall myFunc(LPCTSTR say) {
    HINSTANCE petDll;
    CatProc catFunc;
 
    // Unhook the function
    DWORD oldProtect;
    VirtualProtect((LPVOID)hookedAddress, 12, PAGE_EXECUTE_READWRITE, &oldProtect);
    memcpy((LPVOID)hookedAddress, originalBytes, 12);
    VirtualProtect((LPVOID)hookedAddress, 12, oldProtect, &oldProtect);
 
    // Call the original function with modified text
    petDll = LoadLibrary("pet.dll");
    catFunc = (CatProc) GetProcAddress(petDll, "Cat");
    return (catFunc)("meow-squeak-tweet!!!");
}
 
// Hooking logic
void setMySuperHook() {
    HINSTANCE hLib;
    void* myFuncAddress;
    uintptr_t myFuncAddr;
    CHAR patch[12] = {0};
    DWORD oldProtect;
 
    // Load pet.dll and get the Cat function address
    hLib = LoadLibraryA("pet.dll");
    hookedAddress = GetProcAddress(hLib, "Cat");
 
    // Save original bytes
    VirtualProtect((LPVOID)hookedAddress, 12, PAGE_EXECUTE_READWRITE, &oldProtect);
    memcpy(originalBytes, (LPCVOID)hookedAddress, 12);
 
    myFuncAddress = (void*)&myFunc;
    myFuncAddr = (uintptr_t)myFuncAddress;
 
    // Build the patch: mov rax, myFuncAddr; jmp rax
    patch[0] = 0x48;     // REX.W
    patch[1] = 0xB8;     // MOV RAX, imm64
    memcpy(patch + 2, &myFuncAddr, 8);
    patch[10] = 0xFF;    // JMP RAX
    patch[11] = 0xE0;
 
    // Write the patch
    memcpy((LPVOID)hookedAddress, patch, 12);
    VirtualProtect((LPVOID)hookedAddress, 12, oldProtect, &oldProtect);
}
 
int main() {
    HINSTANCE petDll;
    CatProc catFunc;
 
    // Load pet.dll and call original Cat
    petDll = LoadLibrary("pet.dll");
    catFunc = (CatProc) GetProcAddress(petDll, "Cat");
    (catFunc)("meow-meow");
 
    // Install hook
    setMySuperHook();
 
    // Call Cat again, now it will trigger myFunc
    (catFunc)("meow-meow");
 
    return 0;
}
 

Explanation

#include <windows.h>
#include <stdio.h>

Includes the Windows API headers (windows.h) and standard I/O (stdio.h).


typedef int (__cdecl *CatProc)(LPCTSTR);

Defines a function pointer type named CatProc.
This means:

  • A function that returns int
  • Takes a single LPCTSTR (string literal or pointer to a constant string)
  • Uses the __cdecl calling convention (standard on Windows)

char originalBytes[12];
FARPROC hookedAddress;
  • originalBytes[12]: Buffer to save the original first 12 bytes of the target function (Cat) before we overwrite it (so we can restore them later).
  • hookedAddress: Stores the address of the target function (Cat in pet.dll).

int __stdcall myFunc(LPCTSTR say) {
  • Defines a new function myFunc that uses the __stdcall calling convention.
  • This will replace the original Cat function after hooking!

    HINSTANCE petDll;
    CatProc catFunc;

Variables for:

  • petDll: Handle to the loaded DLL (pet.dll)
  • catFunc: Function pointer to the real Cat function

    DWORD oldProtect;
    VirtualProtect((LPVOID)hookedAddress, 12, PAGE_EXECUTE_READWRITE, &oldProtect);
    memcpy((LPVOID)hookedAddress, originalBytes, 12);
    VirtualProtect((LPVOID)hookedAddress, 12, oldProtect, &oldProtect);
  • This block restores the original first 12 bytes (undoes the hook) to safely call the real Cat function.
  • VirtualProtect temporarily changes memory protection to allow write access to the target memory region.

    petDll = LoadLibrary("pet.dll");
    catFunc = (CatProc) GetProcAddress(petDll, "Cat");
    return (catFunc)("meow-squeak-tweet!!!");
}
  • Loads the real pet.dll
  • Retrieves the real Cat function’s address
  • Calls Cat with modified input "meow-squeak-tweet!!!" instead of what was originally passed to the hook
  • Returns the result of the real Cat function

Hooking logic: setMySuperHook()

void setMySuperHook() {
  • Function to install the hook!

    HINSTANCE hLib;
    void* myFuncAddress;
    uintptr_t myFuncAddr;
    CHAR patch[12] = {0};
    DWORD oldProtect;

Local variables:

  • hLib: handle to pet.dll
  • myFuncAddress: pointer to myFunc
  • myFuncAddr: the actual address of myFunc (converted to integer type)
  • patch: buffer to store the 12-byte trampoline (hook)
  • oldProtect: for restoring memory protection

    hLib = LoadLibraryA("pet.dll");
    hookedAddress = GetProcAddress(hLib, "Cat");
  • Loads the target DLL (pet.dll)
  • Finds the address of the Cat function to hook

    VirtualProtect((LPVOID)hookedAddress, 12, PAGE_EXECUTE_READWRITE, &oldProtect);
    memcpy(originalBytes, (LPCVOID)hookedAddress, 12);
  • Temporarily makes the first 12 bytes of Cat writable
  • Copies those bytes into originalBytes buffer so we can restore them later

    myFuncAddress = (void*)&myFunc;
    myFuncAddr = (uintptr_t)myFuncAddress;
  • Gets the actual address of our hook function myFunc and stores it as an integer

    patch[0] = 0x48;     // REX.W prefix: 64-bit instruction
    patch[1] = 0xB8;     // MOV RAX, imm64
    memcpy(patch + 2, &myFuncAddr, 8);
    patch[10] = 0xFF;    // JMP RAX
    patch[11] = 0xE0;

Builds a 12-byte trampoline for 64-bit hooking:

  • 0x48 0xB8: MOV RAX, immediate 64-bit address
  • 8 bytes: the address of myFunc
  • 0xFF 0xE0: JMP RAX (jump to our myFunc)
    This effectively redirects execution to myFunc when Cat is called!

    memcpy((LPVOID)hookedAddress, patch, 12);
    VirtualProtect((LPVOID)hookedAddress, 12, oldProtect, &oldProtect);
}
  • Writes the trampoline to the start of Cat
  • Restores the original memory protection

Main function

int main() {
    HINSTANCE petDll;
    CatProc catFunc;
  • Standard main() entry point
  • Load pet.dll and prepare to call Cat normally

    petDll = LoadLibrary("pet.dll");
    catFunc = (CatProc) GetProcAddress(petDll, "Cat");
  • Loads the DLL and finds the real Cat function

    (catFunc)("meow-meow");
  • Calls the original Cat function — it shows a message box "meow-meow".

    setMySuperHook();
  • Installs the hook — overwrites the Cat function’s first 12 bytes to redirect it to myFunc.

    (catFunc)("meow-meow");
  • Calls Cat again — but now it runs myFunc instead of the original Cat.
  • myFunc restores the original bytes, loads the real Cat again, and calls it with "meow-squeak-tweet!!!"

    return 0;
}
  • Program ends.

Compile and Run

Compile

x86_64-w64-mingw32-g++ hooking.cpp -o hooking.exe -static -luser32 -lkernel32

Move hooking.exe and pet.dll to Window 7(64-bit)\

Run

Program 2

Code

//hooking2.cpp
#include <windows.h>
#include <stdio.h>
 
char originalBytes[12];
FARPROC hookedAddress;
 
// The replacement function
int __stdcall myFunc(LPCSTR lpCmdLine, UINT uCmdShow) {
    // Restore the original bytes
    DWORD oldProtect;
    VirtualProtect((LPVOID)hookedAddress, 12, PAGE_EXECUTE_READWRITE, &oldProtect);
    memcpy((LPVOID)hookedAddress, originalBytes, 12);
    VirtualProtect((LPVOID)hookedAddress, 12, oldProtect, &oldProtect);
 
    // Launch calculator instead
    return WinExec("calc.exe", uCmdShow);
}
 
// Hooking logic (64-bit safe)
void setMySuperHook() {
    HINSTANCE hLib;
    void* myFuncAddress;
    uintptr_t myFuncAddr;
    CHAR patch[12] = {0};
    DWORD oldProtect;
 
    // Get memory address of WinExec
    hLib = LoadLibraryA("kernel32.dll");
    hookedAddress = GetProcAddress(hLib, "WinExec");
 
    // Save the first 12 bytes
    VirtualProtect((LPVOID)hookedAddress, 12, PAGE_EXECUTE_READWRITE, &oldProtect);
    memcpy(originalBytes, (LPCVOID)hookedAddress, 12);
 
    myFuncAddress = (void*)&myFunc;
    myFuncAddr = (uintptr_t)myFuncAddress;
 
    // Build the patch: mov rax, myFuncAddr; jmp rax
    patch[0] = 0x48;     // REX.W
    patch[1] = 0xB8;     // MOV RAX, imm64
    memcpy(patch + 2, &myFuncAddr, 8);
    patch[10] = 0xFF;    // JMP RAX
    patch[11] = 0xE0;
 
    // Write the patch
    memcpy((LPVOID)hookedAddress, patch, 12);
    VirtualProtect((LPVOID)hookedAddress, 12, oldProtect, &oldProtect);
}
 
int main() {
    // Call original WinExec
    WinExec("notepad", SW_SHOWDEFAULT);
 
    // Install hook
    setMySuperHook();
 
    // Call again → this should trigger your hook
    WinExec("notepad", SW_SHOWDEFAULT);
 
    return 0;
}

Explanation

int __stdcall myFunc(LPCSTR lpCmdLine, UINT uCmdShow) {
  • This is the hook (replacement) function that replaces WinExec.

    return WinExec("calc.exe", uCmdShow);
}
  • Instead of running the original command, we force WinExec to launch "calc.exe" (Calculator).
  • After restoring the real WinExec, it’s safe to call it directly.

    // Get memory address of WinExec
    hLib = LoadLibraryA("kernel32.dll");
    hookedAddress = GetProcAddress(hLib, "WinExec");
  • Dynamically loads kernel32.dll and finds the address of WinExec.

Main Function

    // Call original WinExec
    WinExec("notepad", SW_SHOWDEFAULT);
  • Calls WinExec normally, launching Notepad.
    // Install hook
    setMySuperHook();
  • Installs the hook that redirects WinExec to myFunc.
    // Call again → this should trigger your hook
    WinExec("notepad", SW_SHOWDEFAULT);
  • Calls WinExec again.
  • But this time, it jumps to myFunc (hooked).
  • myFunc launches Calculator instead of Notepad.

Compile and Run

Compile

x86_64-w64-mingw32-g++ hooking2.cpp -o hooking2.exe -static -luser32 -lkernel32

Move hooking2.cpp to Windows 7 (64-bit).