admin/omath
1
0
Fork 0
mirror of https://github.com/orange-cpp/omath.git synced 2026-05-04 10:03:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

added hooking of dx9

Browse Source
This commit is contained in:
Orange
2026-05-03 21:58:51 +03:00
parent 1789b1ef51
commit 71171acf36
2 changed files with 171 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
include/omath/hooks/hooks_manager.hpp
View File

@@ -14,6 +14,7 @@
#endif
#include <Windows.h>
#include <dxgi.h>
#include <d3d9.h>
#include <d3d12.h>
#include <safetyhook.hpp>
@@ -23,28 +24,39 @@ namespace omath::hooks
{
HooksManager() = default;
public:
// IDXGISwapChain callbacks are shared between DX11 and DX12 — same interface, same signature.
// IDXGISwapChain callbacks — shared between DX11 and DX12 (same interface, same signature).
using present_callback = std::function<void(IDXGISwapChain*, UINT, UINT)>;
using resize_buffers_callback = std::function<void(IDXGISwapChain*, UINT, UINT, UINT, DXGI_FORMAT, UINT)>;
using execute_command_lists_callback = std::function<void(ID3D12CommandQueue*, UINT, ID3D12CommandList* const*)>;
// IDirect3DDevice9 callbacks — DX9 only.
using dx9_present_callback = std::function<void(IDirect3DDevice9*, const RECT*, const RECT*, HWND, const RGNDATA*)>;
using dx9_reset_callback = std::function<void(IDirect3DDevice9*, D3DPRESENT_PARAMETERS*)>;
using dx9_end_scene_callback = std::function<void(IDirect3DDevice9*)>;
// Return nullopt to pass the message to the original WndProc; return a value to intercept it.
using wnd_proc_callback = std::function<std::optional<LRESULT>(HWND, UINT, WPARAM, LPARAM)>;
using wnd_proc_callback = std::function<std::optional<LRESULT>(HWND, UINT, WPARAM, LPARAM)>;
[[nodiscard]] static HooksManager& get();
HooksManager(const HooksManager&) = delete;
HooksManager& operator=(const HooksManager&) = delete;
~HooksManager();
[[nodiscard]] bool hook_dx9();
void unhook_dx9();
void set_on_dx9_present(dx9_present_callback callback);
void set_on_dx9_reset(dx9_reset_callback callback);
void set_on_dx9_end_scene(dx9_end_scene_callback callback);
[[nodiscard]] bool hook_dx11();
void unhook_dx11();
[[nodiscard]] bool hook_dx12();
void unhook_dx12();
// Present and ResizeBuffers callbacks fire for whichever API is hooked.
// Present and ResizeBuffers callbacks fire for whichever of DX11/DX12 is hooked.
void set_on_present(present_callback callback);
void set_on_resize_buffers(resize_buffers_callback callback);
void set_on_execute_command_lists(execute_command_lists_callback callback);
[[nodiscard]] bool hook_wnd_proc(HWND hwnd);
@@ -52,6 +64,13 @@ namespace omath::hooks
void set_on_wnd_proc(wnd_proc_callback callback);
private:
static HRESULT __stdcall dx9_present_detour(IDirect3DDevice9* p_device, const RECT* p_source_rect,
const RECT* p_dest_rect, HWND h_dest_window_override,
const RGNDATA* p_dirty_region);
static HRESULT __stdcall dx9_reset_detour(IDirect3DDevice9* p_device,
D3DPRESENT_PARAMETERS* p_presentation_parameters);
static HRESULT __stdcall dx9_end_scene_detour(IDirect3DDevice9* p_device);
static HRESULT __stdcall dx11_present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags);
static HRESULT __stdcall dx11_resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count,
UINT width, UINT height, DXGI_FORMAT new_format,
@@ -69,6 +88,7 @@ namespace omath::hooks
mutable std::shared_mutex m_mutex;
bool m_is_dx9_hooked = false;
bool m_is_dx11_hooked = false;
bool m_is_dx12_hooked = false;
bool m_is_wnd_proc_hooked = false;
@@ -76,6 +96,10 @@ namespace omath::hooks
HWND m_hooked_hwnd = nullptr;
WNDPROC m_original_wndproc = nullptr;
safetyhook::InlineHook m_dx9_present_hook;
safetyhook::InlineHook m_dx9_reset_hook;
safetyhook::InlineHook m_dx9_end_scene_hook;
safetyhook::InlineHook m_dx11_present_hook;
safetyhook::InlineHook m_dx11_resize_buffers_hook;
@@ -83,6 +107,10 @@ namespace omath::hooks
safetyhook::InlineHook m_dx12_resize_buffers_hook;
safetyhook::InlineHook m_dx12_execute_command_lists_hook;
dx9_present_callback m_dx9_present_cb;
dx9_reset_callback m_dx9_reset_cb;
dx9_end_scene_callback m_dx9_end_scene_cb;
present_callback m_present_cb;
resize_buffers_callback m_resize_buffers_cb;
execute_command_lists_callback m_execute_command_lists_cb;

139
source/hooks/hooks_manager.cpp
View File

@@ -49,10 +49,106 @@ namespace omath::hooks
HooksManager::~HooksManager()
{
unhook_wnd_proc();
unhook_dx9();
unhook_dx11();
unhook_dx12();
}
bool HooksManager::hook_dx9()
{
std::unique_lock lock(m_mutex);
if (m_is_dx9_hooked)
return true;
const DummyWindow window;
if (!window.valid())
return false;
const HMODULE d3d9_module = GetModuleHandle("d3d9.dll");
if (!d3d9_module)
return false;
using direct3d_create9_fn = IDirect3D9*(__stdcall*)(UINT);
const auto direct3d_create9 = reinterpret_cast<direct3d_create9_fn>(
GetProcAddress(d3d9_module, "Direct3DCreate9"));
if (!direct3d_create9)
return false;
IDirect3D9* d3d9 = direct3d_create9(D3D_SDK_VERSION);
if (!d3d9)
return false;
D3DPRESENT_PARAMETERS pp{};
pp.SwapEffect = D3DSWAPEFFECT_DISCARD;
pp.hDeviceWindow = window.handle();
pp.Windowed = TRUE;
IDirect3DDevice9* device = nullptr;
if (FAILED(d3d9->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, window.handle(),
D3DCREATE_SOFTWARE_VERTEXPROCESSING, &pp, &device)))
{
d3d9->Release();
return false;
}
// IDirect3DDevice9 vtable indices (from IUnknown base):
// Reset = 16
// Present = 17
// EndScene = 42
m_dx9_present_hook = safetyhook::create_inline(
vtable_fn(device, 17),
reinterpret_cast<void*>(&dx9_present_detour));
m_dx9_reset_hook = safetyhook::create_inline(
vtable_fn(device, 16),
reinterpret_cast<void*>(&dx9_reset_detour));
m_dx9_end_scene_hook = safetyhook::create_inline(
vtable_fn(device, 42),
reinterpret_cast<void*>(&dx9_end_scene_detour));
device->Release();
d3d9->Release();
if (!m_dx9_present_hook || !m_dx9_reset_hook || !m_dx9_end_scene_hook)
{
m_dx9_present_hook = {};
m_dx9_reset_hook = {};
m_dx9_end_scene_hook = {};
return false;
}
m_is_dx9_hooked = true;
return true;
}
void HooksManager::unhook_dx9()
{
std::unique_lock lock(m_mutex);
m_dx9_present_hook = {};
m_dx9_reset_hook = {};
m_dx9_end_scene_hook = {};
m_is_dx9_hooked = false;
}
void HooksManager::set_on_dx9_present(dx9_present_callback callback)
{
std::unique_lock lock(m_mutex);
m_dx9_present_cb = std::move(callback);
}
void HooksManager::set_on_dx9_reset(dx9_reset_callback callback)
{
std::unique_lock lock(m_mutex);
m_dx9_reset_cb = std::move(callback);
}
void HooksManager::set_on_dx9_end_scene(dx9_end_scene_callback callback)
{
std::unique_lock lock(m_mutex);
m_dx9_end_scene_cb = std::move(callback);
}
bool HooksManager::hook_dx11()
{
std::unique_lock lock(m_mutex);
@@ -336,6 +432,49 @@ namespace omath::hooks
// Detour implementations: copy callback under shared lock, call it unlocked,
// then call original. This avoids a deadlock if the callback itself calls set_on_*().
HRESULT __stdcall HooksManager::dx9_present_detour(IDirect3DDevice9* p_device, const RECT* p_source_rect,
const RECT* p_dest_rect, HWND h_dest_window_override,
const RGNDATA* p_dirty_region)
{
auto& mgr = get();
dx9_present_callback cb;
{
std::shared_lock lock(mgr.m_mutex);
cb = mgr.m_dx9_present_cb;
}
if (cb)
cb(p_device, p_source_rect, p_dest_rect, h_dest_window_override, p_dirty_region);
return mgr.m_dx9_present_hook.call<HRESULT>(p_device, p_source_rect, p_dest_rect,
h_dest_window_override, p_dirty_region);
}
HRESULT __stdcall HooksManager::dx9_reset_detour(IDirect3DDevice9* p_device,
D3DPRESENT_PARAMETERS* p_presentation_parameters)
{
auto& mgr = get();
dx9_reset_callback cb;
{
std::shared_lock lock(mgr.m_mutex);
cb = mgr.m_dx9_reset_cb;
}
if (cb)
cb(p_device, p_presentation_parameters);
return mgr.m_dx9_reset_hook.call<HRESULT>(p_device, p_presentation_parameters);
}
HRESULT __stdcall HooksManager::dx9_end_scene_detour(IDirect3DDevice9* p_device)
{
auto& mgr = get();
dx9_end_scene_callback cb;
{
std::shared_lock lock(mgr.m_mutex);
cb = mgr.m_dx9_end_scene_cb;
}
if (cb)
cb(p_device);
return mgr.m_dx9_end_scene_hook.call<HRESULT>(p_device);
}
HRESULT __stdcall HooksManager::dx11_present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags)
{
auto& mgr = get();