admin/omath
1
0
Fork 0
mirror of https://github.com/orange-cpp/omath.git synced 2026-05-04 10:03:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

added dx12 hooking

Browse Source
This commit is contained in:
Orange
2026-05-03 21:35:08 +03:00
parent 7e55b1d00e
commit 06d2752059
4 changed files with 496 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
CMakeLists.txt
View File

@@ -31,9 +31,10 @@ option(OMATH_SUPRESS_SAFETY_CHECKS
option(OMATH_ENABLE_COVERAGE "Enable coverage" OFF)
option(OMATH_ENABLE_FORCE_INLINE
"Will for compiler to make some functions to be force inlined no matter what" ON)
option(OMATH_ENABLE_LUA
"omath bindings for lua" OFF)
option(OMATH_ENABLE_HOOKING "omath will HooksManager that can hook DirectX automatically" OFF)
if(VCPKG_MANIFEST_FEATURES)
foreach(omath_feature IN LISTS VCPKG_MANIFEST_FEATURES)
if(omath_feature STREQUAL "imgui")
@@ -48,6 +49,8 @@ if(VCPKG_MANIFEST_FEATURES)
set(OMATH_BUILD_EXAMPLES ON)
elseif(omath_feature STREQUAL "lua")
set(OMATH_ENABLE_LUA ON)
elseif(omath_feature STREQUAL "hooking")
set(OMATH_ENABLE_HOOKING ON)
endif()
endforeach()
@@ -100,6 +103,17 @@ if (OMATH_ENABLE_LUA)
target_include_directories(${PROJECT_NAME} PRIVATE ${SOL2_INCLUDE_DIRS})
endif ()
if (OMATH_ENABLE_HOOKING)
target_compile_definitions(${PROJECT_NAME} PUBLIC OMATH_ENABLE_HOOKING)
find_package(safetyhook CONFIG REQUIRED)
target_link_libraries(${PROJECT_NAME} PRIVATE safetyhook::safetyhook)
if (WIN32)
target_link_libraries(${PROJECT_NAME} PRIVATE d3d12 dxgi)
endif ()
endif ()
add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
target_compile_definitions(${PROJECT_NAME} PUBLIC OMATH_VERSION="${PROJECT_VERSION}")

96
include/omath/hooks/hooks_manager.hpp Normal file
View File

@@ -0,0 +1,96 @@
#pragma once
#ifdef OMATH_ENABLE_HOOKING
#include <functional>
#include <optional>
#include <shared_mutex>
#include <vector>
#include <cstdint>
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <Windows.h>
#include <dxgi.h>
#include <d3d12.h>
#include <safetyhook.hpp>
namespace omath::hooks
{
class HooksManager final
{
HooksManager() = default;
public:
using present_callback = std::function<void(IDXGISwapChain*, UINT, UINT)>;
using resize_buffers_callback = std::function<void(IDXGISwapChain*, UINT, UINT, UINT, DXGI_FORMAT, UINT)>;
using execute_command_lists_callback = std::function<void(ID3D12CommandQueue*, UINT, ID3D12CommandList* const*)>;
// Return nullopt to pass the message to the original WndProc; return a value to intercept it.
using wnd_proc_callback = std::function<std::optional<LRESULT>(HWND, UINT, WPARAM, LPARAM)>;
[[nodiscard]] static HooksManager& get();
HooksManager(const HooksManager&) = delete;
HooksManager& operator=(const HooksManager&) = delete;
~HooksManager();
[[nodiscard]] bool hook_dx12();
void unhook_dx12();
void set_on_present(present_callback callback);
void set_on_resize_buffers(resize_buffers_callback callback);
void set_on_execute_command_lists(execute_command_lists_callback callback);
[[nodiscard]] bool hook_wnd_proc(HWND hwnd);
void unhook_wnd_proc();
void set_on_wnd_proc(wnd_proc_callback callback);
private:
[[nodiscard]] bool build_vtable();
static HRESULT __stdcall present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags);
static HRESULT __stdcall resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count,
UINT width, UINT height, DXGI_FORMAT new_format,
UINT swap_chain_flags);
static void __stdcall execute_command_lists_detour(ID3D12CommandQueue* p_command_queue,
UINT num_command_lists,
ID3D12CommandList* const* pp_command_lists);
static LRESULT __stdcall wnd_proc_detour(HWND hwnd, UINT msg, WPARAM w_param, LPARAM l_param);
mutable std::shared_mutex m_mutex;
bool m_is_dx12_hooked = false;
bool m_is_wnd_proc_hooked = false;
std::vector<uintptr_t> m_vtable;
HWND m_hooked_hwnd = nullptr;
WNDPROC m_original_wndproc = nullptr;
safetyhook::InlineHook m_present_hook;
safetyhook::InlineHook m_resize_buffers_hook;
safetyhook::InlineHook m_execute_command_lists_hook;
present_callback m_present_cb;
resize_buffers_callback m_resize_buffers_cb;
execute_command_lists_callback m_execute_command_lists_cb;
wnd_proc_callback m_wnd_proc_cb;
};
}
#else // !OMATH_ENABLE_HOOKING
namespace omath::hooks
{
class HooksManager final
{
HooksManager() = default;
public:
[[nodiscard]] static HooksManager& get();
HooksManager(const HooksManager&) = delete;
~HooksManager();
};
}
#endif

369
source/hooks/hooks_manager.cpp Normal file
View File

@@ -0,0 +1,369 @@
#include "omath/hooks/hooks_manager.hpp"
#ifdef OMATH_ENABLE_HOOKING
#include <cstring>
namespace
{
class DummyWindow final
{
WNDCLASSEX m_window_class{};
HWND m_window_handle = nullptr;
public:
DummyWindow()
{
m_window_class.cbSize = sizeof(WNDCLASSEX);
m_window_class.style = CS_HREDRAW | CS_VREDRAW;
m_window_class.lpfnWndProc = DefWindowProc;
m_window_class.hInstance = GetModuleHandle(nullptr);
m_window_class.lpszClassName = "OM";
RegisterClassEx(&m_window_class);
m_window_handle = CreateWindow(m_window_class.lpszClassName, "Dummy", WS_OVERLAPPEDWINDOW,
0, 0, 100, 100, nullptr, nullptr, m_window_class.hInstance, nullptr);
}
~DummyWindow()
{
if (m_window_handle)
DestroyWindow(m_window_handle);
UnregisterClass(m_window_class.lpszClassName, m_window_class.hInstance);
}
[[nodiscard]] HWND handle() const { return m_window_handle; }
[[nodiscard]] bool valid() const { return m_window_handle != nullptr; }
};
// Method counts per interface (vtable slots including inherited ones)
constexpr std::size_t k_device_methods = 44;
constexpr std::size_t k_command_queue_methods = 19;
constexpr std::size_t k_command_allocator_methods = 9;
constexpr std::size_t k_command_list_methods = 60;
constexpr std::size_t k_swap_chain_methods = 18;
constexpr std::size_t k_total_methods = k_device_methods + k_command_queue_methods +
k_command_allocator_methods + k_command_list_methods +
k_swap_chain_methods; // 150
// Base offsets in the combined table
constexpr std::size_t k_cmd_queue_base = k_device_methods; // 44
constexpr std::size_t k_swap_chain_base = k_device_methods + k_command_queue_methods +
k_command_allocator_methods + k_command_list_methods; // 132
// IDXGISwapChain vtable: Present=8, ResizeBuffers=13 (from IUnknown base)
// ID3D12CommandQueue vtable: ExecuteCommandLists=8 (from IUnknown base)
constexpr std::size_t k_present_index = k_swap_chain_base + 8; // 140
constexpr std::size_t k_resize_buffers_index = k_swap_chain_base + 13; // 145
constexpr std::size_t k_execute_cmd_lists_index = k_cmd_queue_base + 8; // 52
} // namespace
namespace omath::hooks
{
HooksManager& HooksManager::get()
{
static HooksManager obj;
return obj;
}
HooksManager::~HooksManager()
{
unhook_wnd_proc();
unhook_dx12();
}
bool HooksManager::build_vtable()
{
const DummyWindow window;
if (!window.valid())
return false;
const HMODULE d3d12_module = GetModuleHandle("d3d12.dll");
const HMODULE dxgi_module = GetModuleHandle("dxgi.dll");
if (!d3d12_module || !dxgi_module)
return false;
using create_dxgi_factory_fn = HRESULT(__stdcall*)(REFIID, void**);
using d3d12_create_device_fn = HRESULT(__stdcall*)(IUnknown*, D3D_FEATURE_LEVEL, REFIID, void**);
const auto create_dxgi_factory = reinterpret_cast<create_dxgi_factory_fn>(
GetProcAddress(dxgi_module, "CreateDXGIFactory"));
const auto d3d12_create_device = reinterpret_cast<d3d12_create_device_fn>(
GetProcAddress(d3d12_module, "D3D12CreateDevice"));
if (!create_dxgi_factory || !d3d12_create_device)
return false;
IDXGIFactory* factory = nullptr;
if (FAILED(create_dxgi_factory(__uuidof(IDXGIFactory), reinterpret_cast<void**>(&factory))))
return false;
IDXGIAdapter* adapter = nullptr;
if (factory->EnumAdapters(0, &adapter) == DXGI_ERROR_NOT_FOUND)
{
factory->Release();
return false;
}
ID3D12Device* device = nullptr;
if (FAILED(d3d12_create_device(adapter, D3D_FEATURE_LEVEL_11_0,
__uuidof(ID3D12Device), reinterpret_cast<void**>(&device))))
{
adapter->Release();
factory->Release();
return false;
}
adapter->Release();
D3D12_COMMAND_QUEUE_DESC queue_desc{};
queue_desc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
ID3D12CommandQueue* command_queue = nullptr;
if (FAILED(device->CreateCommandQueue(&queue_desc, __uuidof(ID3D12CommandQueue),
reinterpret_cast<void**>(&command_queue))))
{
device->Release();
factory->Release();
return false;
}
ID3D12CommandAllocator* command_allocator = nullptr;
if (FAILED(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT,
__uuidof(ID3D12CommandAllocator),
reinterpret_cast<void**>(&command_allocator))))
{
command_queue->Release();
device->Release();
factory->Release();
return false;
}
ID3D12GraphicsCommandList* command_list = nullptr;
if (FAILED(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, command_allocator, nullptr,
__uuidof(ID3D12GraphicsCommandList),
reinterpret_cast<void**>(&command_list))))
{
command_allocator->Release();
command_queue->Release();
device->Release();
factory->Release();
return false;
}
DXGI_SWAP_CHAIN_DESC swap_chain_desc{};
swap_chain_desc.BufferDesc.Width = 100;
swap_chain_desc.BufferDesc.Height = 100;
swap_chain_desc.BufferDesc.RefreshRate = {60, 1};
swap_chain_desc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
swap_chain_desc.SampleDesc = {1, 0};
swap_chain_desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
swap_chain_desc.BufferCount = 2;
swap_chain_desc.OutputWindow = window.handle();
swap_chain_desc.Windowed = TRUE;
swap_chain_desc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD;
swap_chain_desc.Flags = DXGI_SWAP_CHAIN_FLAG_ALLOW_MODE_SWITCH;
IDXGISwapChain* swap_chain = nullptr;
if (FAILED(factory->CreateSwapChain(command_queue, &swap_chain_desc, &swap_chain)))
{
command_list->Release();
command_allocator->Release();
command_queue->Release();
device->Release();
factory->Release();
return false;
}
m_vtable.resize(k_total_methods);
const auto copy_vtable = [](uintptr_t* dst, void* com_obj, std::size_t count)
{
std::memcpy(dst, *reinterpret_cast<uintptr_t**>(com_obj), count * sizeof(uintptr_t));
};
copy_vtable(m_vtable.data(), device, k_device_methods);
copy_vtable(m_vtable.data() + k_device_methods, command_queue, k_command_queue_methods);
copy_vtable(m_vtable.data() + k_device_methods + k_command_queue_methods, command_allocator, k_command_allocator_methods);
copy_vtable(m_vtable.data() + k_device_methods + k_command_queue_methods + k_command_allocator_methods, command_list, k_command_list_methods);
copy_vtable(m_vtable.data() + k_swap_chain_base, swap_chain, k_swap_chain_methods);
swap_chain->Release();
command_list->Release();
command_allocator->Release();
command_queue->Release();
device->Release();
factory->Release();
return true;
}
bool HooksManager::hook_dx12()
{
std::unique_lock lock(m_mutex);
if (m_is_dx12_hooked)
return true;
if (!build_vtable())
return false;
m_present_hook = safetyhook::create_inline(
reinterpret_cast<void*>(m_vtable[k_present_index]),
reinterpret_cast<void*>(&present_detour));
m_resize_buffers_hook = safetyhook::create_inline(
reinterpret_cast<void*>(m_vtable[k_resize_buffers_index]),
reinterpret_cast<void*>(&resize_buffers_detour));
m_execute_command_lists_hook = safetyhook::create_inline(
reinterpret_cast<void*>(m_vtable[k_execute_cmd_lists_index]),
reinterpret_cast<void*>(&execute_command_lists_detour));
if (!m_present_hook || !m_resize_buffers_hook || !m_execute_command_lists_hook)
{
m_present_hook = {};
m_resize_buffers_hook = {};
m_execute_command_lists_hook = {};
return false;
}
m_is_dx12_hooked = true;
return true;
}
void HooksManager::unhook_dx12()
{
std::unique_lock lock(m_mutex);
m_present_hook = {};
m_resize_buffers_hook = {};
m_execute_command_lists_hook = {};
m_vtable.clear();
m_is_dx12_hooked = false;
}
void HooksManager::set_on_present(present_callback callback)
{
std::unique_lock lock(m_mutex);
m_present_cb = std::move(callback);
}
void HooksManager::set_on_resize_buffers(resize_buffers_callback callback)
{
std::unique_lock lock(m_mutex);
m_resize_buffers_cb = std::move(callback);
}
void HooksManager::set_on_execute_command_lists(execute_command_lists_callback callback)
{
std::unique_lock lock(m_mutex);
m_execute_command_lists_cb = std::move(callback);
}
bool HooksManager::hook_wnd_proc(HWND hwnd)
{
std::unique_lock lock(m_mutex);
if (m_is_wnd_proc_hooked)
return true;
const auto prev = reinterpret_cast<WNDPROC>(
SetWindowLongPtr(hwnd, GWLP_WNDPROC, reinterpret_cast<LONG_PTR>(&wnd_proc_detour)));
if (!prev)
return false;
m_hooked_hwnd = hwnd;
m_original_wndproc = prev;
m_is_wnd_proc_hooked = true;
return true;
}
void HooksManager::unhook_wnd_proc()
{
std::unique_lock lock(m_mutex);
if (!m_is_wnd_proc_hooked)
return;
SetWindowLongPtr(m_hooked_hwnd, GWLP_WNDPROC, reinterpret_cast<LONG_PTR>(m_original_wndproc));
m_hooked_hwnd = nullptr;
m_original_wndproc = nullptr;
m_is_wnd_proc_hooked = false;
}
void HooksManager::set_on_wnd_proc(wnd_proc_callback callback)
{
std::unique_lock lock(m_mutex);
m_wnd_proc_cb = std::move(callback);
}
// Detour implementations: copy callback under shared lock, call it unlocked,
// then call original. This avoids a deadlock if the callback itself calls set_on_*().
HRESULT __stdcall HooksManager::present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags)
{
auto& mgr = get();
present_callback cb;
{
std::shared_lock lock(mgr.m_mutex);
cb = mgr.m_present_cb;
}
if (cb)
cb(p_swap_chain, sync_interval, flags);
return mgr.m_present_hook.call<HRESULT>(p_swap_chain, sync_interval, flags);
}
HRESULT __stdcall HooksManager::resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count,
UINT width, UINT height, DXGI_FORMAT new_format,
UINT swap_chain_flags)
{
auto& mgr = get();
resize_buffers_callback cb;
{
std::shared_lock lock(mgr.m_mutex);
cb = mgr.m_resize_buffers_cb;
}
if (cb)
cb(p_swap_chain, buffer_count, width, height, new_format, swap_chain_flags);
return mgr.m_resize_buffers_hook.call<HRESULT>(p_swap_chain, buffer_count, width, height, new_format,
swap_chain_flags);
}
void __stdcall HooksManager::execute_command_lists_detour(ID3D12CommandQueue* p_command_queue,
UINT num_command_lists,
ID3D12CommandList* const* pp_command_lists)
{
auto& mgr = get();
execute_command_lists_callback cb;
{
std::shared_lock lock(mgr.m_mutex);
cb = mgr.m_execute_command_lists_cb;
}
if (cb)
cb(p_command_queue, num_command_lists, pp_command_lists);
mgr.m_execute_command_lists_hook.call<void>(p_command_queue, num_command_lists, pp_command_lists);
}
LRESULT __stdcall HooksManager::wnd_proc_detour(HWND hwnd, UINT msg, WPARAM w_param, LPARAM l_param)
{
auto& mgr = get();
wnd_proc_callback cb;
WNDPROC original;
{
std::shared_lock lock(mgr.m_mutex);
cb = mgr.m_wnd_proc_cb;
original = mgr.m_original_wndproc;
}
if (cb)
{
if (const auto result = cb(hwnd, msg, w_param, l_param))
return *result;
}
return CallWindowProc(original, hwnd, msg, w_param, l_param);
}
} // namespace omath::hooks
#else // !OMATH_ENABLE_HOOKING
namespace omath::hooks
{
HooksManager& HooksManager::get()
{
static HooksManager obj;
return obj;
}
HooksManager::~HooksManager() = default;
} // namespace omath::hooks
#endif

16
vcpkg.json
View File

@@ -16,6 +16,15 @@
}
],
"features": {
"all": {
"description": "Enable all additional features",
"dependencies": [
{
"name": "omath",
"features": ["imgui", "lua", "hooking"]
}
]
},
"avx2": {
"description": "omath will use AVX2 to boost performance",
"supports": "!arm"
@@ -26,6 +35,13 @@
"benchmark"
]
},
"hooking": {
"description": "Add interface for automatic hooking of DirectX",
"dependencies": [
"safetyhook"
],
"supports": "(windows | linux) & !arm & !uwp"
},
"examples": {
"description": "Build examples",
"dependencies": [