mirror of
https://github.com/orange-cpp/omath.git
synced 2026-05-06 12:43:26 +00:00
added hooking of dx9
This commit is contained in:
@@ -14,6 +14,7 @@
|
|||||||
#endif
|
#endif
|
||||||
#include <Windows.h>
|
#include <Windows.h>
|
||||||
#include <dxgi.h>
|
#include <dxgi.h>
|
||||||
|
#include <d3d9.h>
|
||||||
#include <d3d12.h>
|
#include <d3d12.h>
|
||||||
#include <safetyhook.hpp>
|
#include <safetyhook.hpp>
|
||||||
|
|
||||||
@@ -23,10 +24,16 @@ namespace omath::hooks
|
|||||||
{
|
{
|
||||||
HooksManager() = default;
|
HooksManager() = default;
|
||||||
public:
|
public:
|
||||||
// IDXGISwapChain callbacks are shared between DX11 and DX12 — same interface, same signature.
|
// IDXGISwapChain callbacks — shared between DX11 and DX12 (same interface, same signature).
|
||||||
using present_callback = std::function<void(IDXGISwapChain*, UINT, UINT)>;
|
using present_callback = std::function<void(IDXGISwapChain*, UINT, UINT)>;
|
||||||
using resize_buffers_callback = std::function<void(IDXGISwapChain*, UINT, UINT, UINT, DXGI_FORMAT, UINT)>;
|
using resize_buffers_callback = std::function<void(IDXGISwapChain*, UINT, UINT, UINT, DXGI_FORMAT, UINT)>;
|
||||||
using execute_command_lists_callback = std::function<void(ID3D12CommandQueue*, UINT, ID3D12CommandList* const*)>;
|
using execute_command_lists_callback = std::function<void(ID3D12CommandQueue*, UINT, ID3D12CommandList* const*)>;
|
||||||
|
|
||||||
|
// IDirect3DDevice9 callbacks — DX9 only.
|
||||||
|
using dx9_present_callback = std::function<void(IDirect3DDevice9*, const RECT*, const RECT*, HWND, const RGNDATA*)>;
|
||||||
|
using dx9_reset_callback = std::function<void(IDirect3DDevice9*, D3DPRESENT_PARAMETERS*)>;
|
||||||
|
using dx9_end_scene_callback = std::function<void(IDirect3DDevice9*)>;
|
||||||
|
|
||||||
// Return nullopt to pass the message to the original WndProc; return a value to intercept it.
|
// Return nullopt to pass the message to the original WndProc; return a value to intercept it.
|
||||||
using wnd_proc_callback = std::function<std::optional<LRESULT>(HWND, UINT, WPARAM, LPARAM)>;
|
using wnd_proc_callback = std::function<std::optional<LRESULT>(HWND, UINT, WPARAM, LPARAM)>;
|
||||||
|
|
||||||
@@ -35,16 +42,21 @@ namespace omath::hooks
|
|||||||
HooksManager& operator=(const HooksManager&) = delete;
|
HooksManager& operator=(const HooksManager&) = delete;
|
||||||
~HooksManager();
|
~HooksManager();
|
||||||
|
|
||||||
|
[[nodiscard]] bool hook_dx9();
|
||||||
|
void unhook_dx9();
|
||||||
|
void set_on_dx9_present(dx9_present_callback callback);
|
||||||
|
void set_on_dx9_reset(dx9_reset_callback callback);
|
||||||
|
void set_on_dx9_end_scene(dx9_end_scene_callback callback);
|
||||||
|
|
||||||
[[nodiscard]] bool hook_dx11();
|
[[nodiscard]] bool hook_dx11();
|
||||||
void unhook_dx11();
|
void unhook_dx11();
|
||||||
|
|
||||||
[[nodiscard]] bool hook_dx12();
|
[[nodiscard]] bool hook_dx12();
|
||||||
void unhook_dx12();
|
void unhook_dx12();
|
||||||
|
|
||||||
// Present and ResizeBuffers callbacks fire for whichever API is hooked.
|
// Present and ResizeBuffers callbacks fire for whichever of DX11/DX12 is hooked.
|
||||||
void set_on_present(present_callback callback);
|
void set_on_present(present_callback callback);
|
||||||
void set_on_resize_buffers(resize_buffers_callback callback);
|
void set_on_resize_buffers(resize_buffers_callback callback);
|
||||||
|
|
||||||
void set_on_execute_command_lists(execute_command_lists_callback callback);
|
void set_on_execute_command_lists(execute_command_lists_callback callback);
|
||||||
|
|
||||||
[[nodiscard]] bool hook_wnd_proc(HWND hwnd);
|
[[nodiscard]] bool hook_wnd_proc(HWND hwnd);
|
||||||
@@ -52,6 +64,13 @@ namespace omath::hooks
|
|||||||
void set_on_wnd_proc(wnd_proc_callback callback);
|
void set_on_wnd_proc(wnd_proc_callback callback);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
static HRESULT __stdcall dx9_present_detour(IDirect3DDevice9* p_device, const RECT* p_source_rect,
|
||||||
|
const RECT* p_dest_rect, HWND h_dest_window_override,
|
||||||
|
const RGNDATA* p_dirty_region);
|
||||||
|
static HRESULT __stdcall dx9_reset_detour(IDirect3DDevice9* p_device,
|
||||||
|
D3DPRESENT_PARAMETERS* p_presentation_parameters);
|
||||||
|
static HRESULT __stdcall dx9_end_scene_detour(IDirect3DDevice9* p_device);
|
||||||
|
|
||||||
static HRESULT __stdcall dx11_present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags);
|
static HRESULT __stdcall dx11_present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags);
|
||||||
static HRESULT __stdcall dx11_resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count,
|
static HRESULT __stdcall dx11_resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count,
|
||||||
UINT width, UINT height, DXGI_FORMAT new_format,
|
UINT width, UINT height, DXGI_FORMAT new_format,
|
||||||
@@ -69,6 +88,7 @@ namespace omath::hooks
|
|||||||
|
|
||||||
mutable std::shared_mutex m_mutex;
|
mutable std::shared_mutex m_mutex;
|
||||||
|
|
||||||
|
bool m_is_dx9_hooked = false;
|
||||||
bool m_is_dx11_hooked = false;
|
bool m_is_dx11_hooked = false;
|
||||||
bool m_is_dx12_hooked = false;
|
bool m_is_dx12_hooked = false;
|
||||||
bool m_is_wnd_proc_hooked = false;
|
bool m_is_wnd_proc_hooked = false;
|
||||||
@@ -76,6 +96,10 @@ namespace omath::hooks
|
|||||||
HWND m_hooked_hwnd = nullptr;
|
HWND m_hooked_hwnd = nullptr;
|
||||||
WNDPROC m_original_wndproc = nullptr;
|
WNDPROC m_original_wndproc = nullptr;
|
||||||
|
|
||||||
|
safetyhook::InlineHook m_dx9_present_hook;
|
||||||
|
safetyhook::InlineHook m_dx9_reset_hook;
|
||||||
|
safetyhook::InlineHook m_dx9_end_scene_hook;
|
||||||
|
|
||||||
safetyhook::InlineHook m_dx11_present_hook;
|
safetyhook::InlineHook m_dx11_present_hook;
|
||||||
safetyhook::InlineHook m_dx11_resize_buffers_hook;
|
safetyhook::InlineHook m_dx11_resize_buffers_hook;
|
||||||
|
|
||||||
@@ -83,6 +107,10 @@ namespace omath::hooks
|
|||||||
safetyhook::InlineHook m_dx12_resize_buffers_hook;
|
safetyhook::InlineHook m_dx12_resize_buffers_hook;
|
||||||
safetyhook::InlineHook m_dx12_execute_command_lists_hook;
|
safetyhook::InlineHook m_dx12_execute_command_lists_hook;
|
||||||
|
|
||||||
|
dx9_present_callback m_dx9_present_cb;
|
||||||
|
dx9_reset_callback m_dx9_reset_cb;
|
||||||
|
dx9_end_scene_callback m_dx9_end_scene_cb;
|
||||||
|
|
||||||
present_callback m_present_cb;
|
present_callback m_present_cb;
|
||||||
resize_buffers_callback m_resize_buffers_cb;
|
resize_buffers_callback m_resize_buffers_cb;
|
||||||
execute_command_lists_callback m_execute_command_lists_cb;
|
execute_command_lists_callback m_execute_command_lists_cb;
|
||||||
|
|||||||
@@ -49,10 +49,106 @@ namespace omath::hooks
|
|||||||
HooksManager::~HooksManager()
|
HooksManager::~HooksManager()
|
||||||
{
|
{
|
||||||
unhook_wnd_proc();
|
unhook_wnd_proc();
|
||||||
|
unhook_dx9();
|
||||||
unhook_dx11();
|
unhook_dx11();
|
||||||
unhook_dx12();
|
unhook_dx12();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool HooksManager::hook_dx9()
|
||||||
|
{
|
||||||
|
std::unique_lock lock(m_mutex);
|
||||||
|
if (m_is_dx9_hooked)
|
||||||
|
return true;
|
||||||
|
|
||||||
|
const DummyWindow window;
|
||||||
|
if (!window.valid())
|
||||||
|
return false;
|
||||||
|
|
||||||
|
const HMODULE d3d9_module = GetModuleHandle("d3d9.dll");
|
||||||
|
if (!d3d9_module)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
using direct3d_create9_fn = IDirect3D9*(__stdcall*)(UINT);
|
||||||
|
const auto direct3d_create9 = reinterpret_cast<direct3d_create9_fn>(
|
||||||
|
GetProcAddress(d3d9_module, "Direct3DCreate9"));
|
||||||
|
if (!direct3d_create9)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
IDirect3D9* d3d9 = direct3d_create9(D3D_SDK_VERSION);
|
||||||
|
if (!d3d9)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
D3DPRESENT_PARAMETERS pp{};
|
||||||
|
pp.SwapEffect = D3DSWAPEFFECT_DISCARD;
|
||||||
|
pp.hDeviceWindow = window.handle();
|
||||||
|
pp.Windowed = TRUE;
|
||||||
|
|
||||||
|
IDirect3DDevice9* device = nullptr;
|
||||||
|
if (FAILED(d3d9->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, window.handle(),
|
||||||
|
D3DCREATE_SOFTWARE_VERTEXPROCESSING, &pp, &device)))
|
||||||
|
{
|
||||||
|
d3d9->Release();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// IDirect3DDevice9 vtable indices (from IUnknown base):
|
||||||
|
// Reset = 16
|
||||||
|
// Present = 17
|
||||||
|
// EndScene = 42
|
||||||
|
m_dx9_present_hook = safetyhook::create_inline(
|
||||||
|
vtable_fn(device, 17),
|
||||||
|
reinterpret_cast<void*>(&dx9_present_detour));
|
||||||
|
|
||||||
|
m_dx9_reset_hook = safetyhook::create_inline(
|
||||||
|
vtable_fn(device, 16),
|
||||||
|
reinterpret_cast<void*>(&dx9_reset_detour));
|
||||||
|
|
||||||
|
m_dx9_end_scene_hook = safetyhook::create_inline(
|
||||||
|
vtable_fn(device, 42),
|
||||||
|
reinterpret_cast<void*>(&dx9_end_scene_detour));
|
||||||
|
|
||||||
|
device->Release();
|
||||||
|
d3d9->Release();
|
||||||
|
|
||||||
|
if (!m_dx9_present_hook || !m_dx9_reset_hook || !m_dx9_end_scene_hook)
|
||||||
|
{
|
||||||
|
m_dx9_present_hook = {};
|
||||||
|
m_dx9_reset_hook = {};
|
||||||
|
m_dx9_end_scene_hook = {};
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_is_dx9_hooked = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void HooksManager::unhook_dx9()
|
||||||
|
{
|
||||||
|
std::unique_lock lock(m_mutex);
|
||||||
|
m_dx9_present_hook = {};
|
||||||
|
m_dx9_reset_hook = {};
|
||||||
|
m_dx9_end_scene_hook = {};
|
||||||
|
m_is_dx9_hooked = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void HooksManager::set_on_dx9_present(dx9_present_callback callback)
|
||||||
|
{
|
||||||
|
std::unique_lock lock(m_mutex);
|
||||||
|
m_dx9_present_cb = std::move(callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
void HooksManager::set_on_dx9_reset(dx9_reset_callback callback)
|
||||||
|
{
|
||||||
|
std::unique_lock lock(m_mutex);
|
||||||
|
m_dx9_reset_cb = std::move(callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
void HooksManager::set_on_dx9_end_scene(dx9_end_scene_callback callback)
|
||||||
|
{
|
||||||
|
std::unique_lock lock(m_mutex);
|
||||||
|
m_dx9_end_scene_cb = std::move(callback);
|
||||||
|
}
|
||||||
|
|
||||||
bool HooksManager::hook_dx11()
|
bool HooksManager::hook_dx11()
|
||||||
{
|
{
|
||||||
std::unique_lock lock(m_mutex);
|
std::unique_lock lock(m_mutex);
|
||||||
@@ -336,6 +432,49 @@ namespace omath::hooks
|
|||||||
// Detour implementations: copy callback under shared lock, call it unlocked,
|
// Detour implementations: copy callback under shared lock, call it unlocked,
|
||||||
// then call original. This avoids a deadlock if the callback itself calls set_on_*().
|
// then call original. This avoids a deadlock if the callback itself calls set_on_*().
|
||||||
|
|
||||||
|
HRESULT __stdcall HooksManager::dx9_present_detour(IDirect3DDevice9* p_device, const RECT* p_source_rect,
|
||||||
|
const RECT* p_dest_rect, HWND h_dest_window_override,
|
||||||
|
const RGNDATA* p_dirty_region)
|
||||||
|
{
|
||||||
|
auto& mgr = get();
|
||||||
|
dx9_present_callback cb;
|
||||||
|
{
|
||||||
|
std::shared_lock lock(mgr.m_mutex);
|
||||||
|
cb = mgr.m_dx9_present_cb;
|
||||||
|
}
|
||||||
|
if (cb)
|
||||||
|
cb(p_device, p_source_rect, p_dest_rect, h_dest_window_override, p_dirty_region);
|
||||||
|
return mgr.m_dx9_present_hook.call<HRESULT>(p_device, p_source_rect, p_dest_rect,
|
||||||
|
h_dest_window_override, p_dirty_region);
|
||||||
|
}
|
||||||
|
|
||||||
|
HRESULT __stdcall HooksManager::dx9_reset_detour(IDirect3DDevice9* p_device,
|
||||||
|
D3DPRESENT_PARAMETERS* p_presentation_parameters)
|
||||||
|
{
|
||||||
|
auto& mgr = get();
|
||||||
|
dx9_reset_callback cb;
|
||||||
|
{
|
||||||
|
std::shared_lock lock(mgr.m_mutex);
|
||||||
|
cb = mgr.m_dx9_reset_cb;
|
||||||
|
}
|
||||||
|
if (cb)
|
||||||
|
cb(p_device, p_presentation_parameters);
|
||||||
|
return mgr.m_dx9_reset_hook.call<HRESULT>(p_device, p_presentation_parameters);
|
||||||
|
}
|
||||||
|
|
||||||
|
HRESULT __stdcall HooksManager::dx9_end_scene_detour(IDirect3DDevice9* p_device)
|
||||||
|
{
|
||||||
|
auto& mgr = get();
|
||||||
|
dx9_end_scene_callback cb;
|
||||||
|
{
|
||||||
|
std::shared_lock lock(mgr.m_mutex);
|
||||||
|
cb = mgr.m_dx9_end_scene_cb;
|
||||||
|
}
|
||||||
|
if (cb)
|
||||||
|
cb(p_device);
|
||||||
|
return mgr.m_dx9_end_scene_hook.call<HRESULT>(p_device);
|
||||||
|
}
|
||||||
|
|
||||||
HRESULT __stdcall HooksManager::dx11_present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags)
|
HRESULT __stdcall HooksManager::dx11_present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags)
|
||||||
{
|
{
|
||||||
auto& mgr = get();
|
auto& mgr = get();
|
||||||
|
|||||||
Reference in New Issue
Block a user