diff --git a/CMakeLists.txt b/CMakeLists.txt index 768414b..3343416 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,7 +110,7 @@ if (OMATH_ENABLE_HOOKING) target_link_libraries(${PROJECT_NAME} PRIVATE safetyhook::safetyhook) if (WIN32) - target_link_libraries(${PROJECT_NAME} PRIVATE d3d12 dxgi) + target_link_libraries(${PROJECT_NAME} PRIVATE d3d11 d3d12 dxgi) endif () endif () diff --git a/include/omath/hooks/hooks_manager.hpp b/include/omath/hooks/hooks_manager.hpp index c0c47fd..cf6aeb6 100644 --- a/include/omath/hooks/hooks_manager.hpp +++ b/include/omath/hooks/hooks_manager.hpp @@ -23,6 +23,7 @@ namespace omath::hooks { HooksManager() = default; public: + // IDXGISwapChain callbacks are shared between DX11 and DX12 — same interface, same signature. using present_callback = std::function; using resize_buffers_callback = std::function; using execute_command_lists_callback = std::function; @@ -34,39 +35,53 @@ namespace omath::hooks HooksManager& operator=(const HooksManager&) = delete; ~HooksManager(); + [[nodiscard]] bool hook_dx11(); + void unhook_dx11(); + [[nodiscard]] bool hook_dx12(); void unhook_dx12(); + // Present and ResizeBuffers callbacks fire for whichever API is hooked. void set_on_present(present_callback callback); void set_on_resize_buffers(resize_buffers_callback callback); + void set_on_execute_command_lists(execute_command_lists_callback callback); [[nodiscard]] bool hook_wnd_proc(HWND hwnd); void unhook_wnd_proc(); - void set_on_wnd_proc(wnd_proc_callback callback); private: - static HRESULT __stdcall present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags); - static HRESULT __stdcall resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count, - UINT width, UINT height, DXGI_FORMAT new_format, - UINT swap_chain_flags); - static void __stdcall execute_command_lists_detour(ID3D12CommandQueue* p_command_queue, - UINT num_command_lists, - ID3D12CommandList* const* pp_command_lists); + static HRESULT __stdcall dx11_present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags); + static HRESULT __stdcall dx11_resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count, + UINT width, UINT height, DXGI_FORMAT new_format, + UINT swap_chain_flags); + + static HRESULT __stdcall dx12_present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags); + static HRESULT __stdcall dx12_resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count, + UINT width, UINT height, DXGI_FORMAT new_format, + UINT swap_chain_flags); + static void __stdcall dx12_execute_command_lists_detour(ID3D12CommandQueue* p_command_queue, + UINT num_command_lists, + ID3D12CommandList* const* pp_command_lists); + static LRESULT __stdcall wnd_proc_detour(HWND hwnd, UINT msg, WPARAM w_param, LPARAM l_param); mutable std::shared_mutex m_mutex; + bool m_is_dx11_hooked = false; bool m_is_dx12_hooked = false; bool m_is_wnd_proc_hooked = false; HWND m_hooked_hwnd = nullptr; WNDPROC m_original_wndproc = nullptr; - safetyhook::InlineHook m_present_hook; - safetyhook::InlineHook m_resize_buffers_hook; - safetyhook::InlineHook m_execute_command_lists_hook; + safetyhook::InlineHook m_dx11_present_hook; + safetyhook::InlineHook m_dx11_resize_buffers_hook; + + safetyhook::InlineHook m_dx12_present_hook; + safetyhook::InlineHook m_dx12_resize_buffers_hook; + safetyhook::InlineHook m_dx12_execute_command_lists_hook; present_callback m_present_cb; resize_buffers_callback m_resize_buffers_cb; diff --git a/source/hooks/hooks_manager.cpp b/source/hooks/hooks_manager.cpp index af8f331..1471688 100644 --- a/source/hooks/hooks_manager.cpp +++ b/source/hooks/hooks_manager.cpp @@ -1,6 +1,7 @@ #include "omath/hooks/hooks_manager.hpp" #ifdef OMATH_ENABLE_HOOKING +#include namespace { @@ -48,9 +49,89 @@ namespace omath::hooks HooksManager::~HooksManager() { unhook_wnd_proc(); + unhook_dx11(); unhook_dx12(); } + bool HooksManager::hook_dx11() + { + std::unique_lock lock(m_mutex); + if (m_is_dx11_hooked) + return true; + + const DummyWindow window; + if (!window.valid()) + return false; + + const HMODULE d3d11_module = GetModuleHandle("d3d11.dll"); + if (!d3d11_module) + return false; + + using d3d11_create_device_and_swap_chain_fn = + HRESULT(__stdcall*)(IDXGIAdapter*, D3D_DRIVER_TYPE, HMODULE, UINT, + const D3D_FEATURE_LEVEL*, UINT, UINT, + const DXGI_SWAP_CHAIN_DESC*, IDXGISwapChain**, + ID3D11Device**, D3D_FEATURE_LEVEL*, ID3D11DeviceContext**); + + const auto create_device_and_swap_chain = reinterpret_cast( + GetProcAddress(d3d11_module, "D3D11CreateDeviceAndSwapChain")); + if (!create_device_and_swap_chain) + return false; + + DXGI_SWAP_CHAIN_DESC swap_chain_desc{}; + swap_chain_desc.BufferDesc.Width = 100; + swap_chain_desc.BufferDesc.Height = 100; + swap_chain_desc.BufferDesc.RefreshRate = {60, 1}; + swap_chain_desc.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; + swap_chain_desc.SampleDesc = {1, 0}; + swap_chain_desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; + swap_chain_desc.BufferCount = 1; + swap_chain_desc.OutputWindow = window.handle(); + swap_chain_desc.Windowed = TRUE; + swap_chain_desc.SwapEffect = DXGI_SWAP_EFFECT_DISCARD; + + constexpr D3D_FEATURE_LEVEL feature_levels[] = {D3D_FEATURE_LEVEL_11_0}; + ID3D11Device* device = nullptr; + ID3D11DeviceContext* device_context = nullptr; + IDXGISwapChain* swap_chain = nullptr; + + if (FAILED(create_device_and_swap_chain(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, + feature_levels, 1, D3D11_SDK_VERSION, + &swap_chain_desc, &swap_chain, + &device, nullptr, &device_context))) + return false; + + m_dx11_present_hook = safetyhook::create_inline( + vtable_fn(swap_chain, 8), // IDXGISwapChain::Present + reinterpret_cast(&dx11_present_detour)); + + m_dx11_resize_buffers_hook = safetyhook::create_inline( + vtable_fn(swap_chain, 13), // IDXGISwapChain::ResizeBuffers + reinterpret_cast(&dx11_resize_buffers_detour)); + + swap_chain->Release(); + device_context->Release(); + device->Release(); + + if (!m_dx11_present_hook || !m_dx11_resize_buffers_hook) + { + m_dx11_present_hook = {}; + m_dx11_resize_buffers_hook = {}; + return false; + } + + m_is_dx11_hooked = true; + return true; + } + + void HooksManager::unhook_dx11() + { + std::unique_lock lock(m_mutex); + m_dx11_present_hook = {}; + m_dx11_resize_buffers_hook = {}; + m_is_dx11_hooked = false; + } + bool HooksManager::hook_dx12() { std::unique_lock lock(m_mutex); @@ -157,19 +238,19 @@ namespace omath::hooks return false; } - // Read the 3 needed vtable slots directly — these addresses live in the DLL, + // Read the needed vtable slots directly — addresses live in the DLL, // not in the objects, so they remain valid after the objects are released. - m_present_hook = safetyhook::create_inline( - vtable_fn(swap_chain, 8), // IDXGISwapChain::Present - reinterpret_cast(&present_detour)); + m_dx12_present_hook = safetyhook::create_inline( + vtable_fn(swap_chain, 8), // IDXGISwapChain::Present + reinterpret_cast(&dx12_present_detour)); - m_resize_buffers_hook = safetyhook::create_inline( - vtable_fn(swap_chain, 13), // IDXGISwapChain::ResizeBuffers - reinterpret_cast(&resize_buffers_detour)); + m_dx12_resize_buffers_hook = safetyhook::create_inline( + vtable_fn(swap_chain, 13), // IDXGISwapChain::ResizeBuffers + reinterpret_cast(&dx12_resize_buffers_detour)); - m_execute_command_lists_hook = safetyhook::create_inline( + m_dx12_execute_command_lists_hook = safetyhook::create_inline( vtable_fn(command_queue, 8), // ID3D12CommandQueue::ExecuteCommandLists - reinterpret_cast(&execute_command_lists_detour)); + reinterpret_cast(&dx12_execute_command_lists_detour)); swap_chain->Release(); command_list->Release(); @@ -178,11 +259,11 @@ namespace omath::hooks device->Release(); factory->Release(); - if (!m_present_hook || !m_resize_buffers_hook || !m_execute_command_lists_hook) + if (!m_dx12_present_hook || !m_dx12_resize_buffers_hook || !m_dx12_execute_command_lists_hook) { - m_present_hook = {}; - m_resize_buffers_hook = {}; - m_execute_command_lists_hook = {}; + m_dx12_present_hook = {}; + m_dx12_resize_buffers_hook = {}; + m_dx12_execute_command_lists_hook = {}; return false; } @@ -193,9 +274,9 @@ namespace omath::hooks void HooksManager::unhook_dx12() { std::unique_lock lock(m_mutex); - m_present_hook = {}; - m_resize_buffers_hook = {}; - m_execute_command_lists_hook = {}; + m_dx12_present_hook = {}; + m_dx12_resize_buffers_hook = {}; + m_dx12_execute_command_lists_hook = {}; m_is_dx12_hooked = false; } @@ -254,7 +335,8 @@ namespace omath::hooks // Detour implementations: copy callback under shared lock, call it unlocked, // then call original. This avoids a deadlock if the callback itself calls set_on_*(). - HRESULT __stdcall HooksManager::present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags) + + HRESULT __stdcall HooksManager::dx11_present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags) { auto& mgr = get(); present_callback cb; @@ -264,12 +346,12 @@ namespace omath::hooks } if (cb) cb(p_swap_chain, sync_interval, flags); - return mgr.m_present_hook.call(p_swap_chain, sync_interval, flags); + return mgr.m_dx11_present_hook.call(p_swap_chain, sync_interval, flags); } - HRESULT __stdcall HooksManager::resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count, - UINT width, UINT height, DXGI_FORMAT new_format, - UINT swap_chain_flags) + HRESULT __stdcall HooksManager::dx11_resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count, + UINT width, UINT height, DXGI_FORMAT new_format, + UINT swap_chain_flags) { auto& mgr = get(); resize_buffers_callback cb; @@ -279,13 +361,42 @@ namespace omath::hooks } if (cb) cb(p_swap_chain, buffer_count, width, height, new_format, swap_chain_flags); - return mgr.m_resize_buffers_hook.call(p_swap_chain, buffer_count, width, height, new_format, - swap_chain_flags); + return mgr.m_dx11_resize_buffers_hook.call(p_swap_chain, buffer_count, width, height, + new_format, swap_chain_flags); } - void __stdcall HooksManager::execute_command_lists_detour(ID3D12CommandQueue* p_command_queue, - UINT num_command_lists, - ID3D12CommandList* const* pp_command_lists) + HRESULT __stdcall HooksManager::dx12_present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags) + { + auto& mgr = get(); + present_callback cb; + { + std::shared_lock lock(mgr.m_mutex); + cb = mgr.m_present_cb; + } + if (cb) + cb(p_swap_chain, sync_interval, flags); + return mgr.m_dx12_present_hook.call(p_swap_chain, sync_interval, flags); + } + + HRESULT __stdcall HooksManager::dx12_resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count, + UINT width, UINT height, DXGI_FORMAT new_format, + UINT swap_chain_flags) + { + auto& mgr = get(); + resize_buffers_callback cb; + { + std::shared_lock lock(mgr.m_mutex); + cb = mgr.m_resize_buffers_cb; + } + if (cb) + cb(p_swap_chain, buffer_count, width, height, new_format, swap_chain_flags); + return mgr.m_dx12_resize_buffers_hook.call(p_swap_chain, buffer_count, width, height, + new_format, swap_chain_flags); + } + + void __stdcall HooksManager::dx12_execute_command_lists_detour(ID3D12CommandQueue* p_command_queue, + UINT num_command_lists, + ID3D12CommandList* const* pp_command_lists) { auto& mgr = get(); execute_command_lists_callback cb; @@ -295,7 +406,7 @@ namespace omath::hooks } if (cb) cb(p_command_queue, num_command_lists, pp_command_lists); - mgr.m_execute_command_lists_hook.call(p_command_queue, num_command_lists, pp_command_lists); + mgr.m_dx12_execute_command_lists_hook.call(p_command_queue, num_command_lists, pp_command_lists); } LRESULT __stdcall HooksManager::wnd_proc_detour(HWND hwnd, UINT msg, WPARAM w_param, LPARAM l_param)