added dx12 hooking

CMakeLists.txt

@@ -31,9 +31,10 @@ option(OMATH_SUPRESS_SAFETY_CHECKS
 option(OMATH_ENABLE_COVERAGE "Enable coverage" OFF)
 option(OMATH_ENABLE_FORCE_INLINE
        "Will for compiler to make some functions to be force inlined no matter what" ON)
-
 option(OMATH_ENABLE_LUA
         "omath bindings for lua" OFF)
+option(OMATH_ENABLE_HOOKING "omath will HooksManager that can hook DirectX automatically" OFF)
+
 if(VCPKG_MANIFEST_FEATURES)
     foreach(omath_feature IN LISTS VCPKG_MANIFEST_FEATURES)
         if(omath_feature STREQUAL "imgui")
@@ -48,6 +49,8 @@ if(VCPKG_MANIFEST_FEATURES)
             set(OMATH_BUILD_EXAMPLES ON)
         elseif(omath_feature STREQUAL "lua")
             set(OMATH_ENABLE_LUA ON)
+        elseif(omath_feature STREQUAL "hooking")
+            set(OMATH_ENABLE_HOOKING ON)
         endif()
 
     endforeach()
@@ -100,6 +103,17 @@ if (OMATH_ENABLE_LUA)
     target_include_directories(${PROJECT_NAME} PRIVATE ${SOL2_INCLUDE_DIRS})
 endif ()
 
+if (OMATH_ENABLE_HOOKING)
+    target_compile_definitions(${PROJECT_NAME} PUBLIC OMATH_ENABLE_HOOKING)
+
+    find_package(safetyhook CONFIG REQUIRED)
+    target_link_libraries(${PROJECT_NAME} PRIVATE safetyhook::safetyhook)
+
+    if (WIN32)
+        target_link_libraries(${PROJECT_NAME} PRIVATE d3d12 dxgi)
+    endif ()
+endif ()
+
 add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
 
 target_compile_definitions(${PROJECT_NAME} PUBLIC OMATH_VERSION="${PROJECT_VERSION}")

include/omath/hooks/hooks_manager.hpp

@@ -0,0 +1,96 @@
+#pragma once
+
+#ifdef OMATH_ENABLE_HOOKING
+#include <functional>
+#include <optional>
+#include <shared_mutex>
+#include <vector>
+#include <cstdint>
+
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <Windows.h>
+#include <dxgi.h>
+#include <d3d12.h>
+#include <safetyhook.hpp>
+
+namespace omath::hooks
+{
+    class HooksManager final
+    {
+        HooksManager() = default;
+    public:
+        using present_callback              = std::function<void(IDXGISwapChain*, UINT, UINT)>;
+        using resize_buffers_callback       = std::function<void(IDXGISwapChain*, UINT, UINT, UINT, DXGI_FORMAT, UINT)>;
+        using execute_command_lists_callback = std::function<void(ID3D12CommandQueue*, UINT, ID3D12CommandList* const*)>;
+        // Return nullopt to pass the message to the original WndProc; return a value to intercept it.
+        using wnd_proc_callback             = std::function<std::optional<LRESULT>(HWND, UINT, WPARAM, LPARAM)>;
+
+        [[nodiscard]] static HooksManager& get();
+        HooksManager(const HooksManager&)            = delete;
+        HooksManager& operator=(const HooksManager&) = delete;
+        ~HooksManager();
+
+        [[nodiscard]] bool hook_dx12();
+        void unhook_dx12();
+
+        void set_on_present(present_callback callback);
+        void set_on_resize_buffers(resize_buffers_callback callback);
+        void set_on_execute_command_lists(execute_command_lists_callback callback);
+
+        [[nodiscard]] bool hook_wnd_proc(HWND hwnd);
+        void unhook_wnd_proc();
+
+        void set_on_wnd_proc(wnd_proc_callback callback);
+
+    private:
+        [[nodiscard]] bool build_vtable();
+
+        static HRESULT __stdcall present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags);
+        static HRESULT __stdcall resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count,
+                                                       UINT width, UINT height, DXGI_FORMAT new_format,
+                                                       UINT swap_chain_flags);
+        static void __stdcall execute_command_lists_detour(ID3D12CommandQueue* p_command_queue,
+                                                           UINT num_command_lists,
+                                                           ID3D12CommandList* const* pp_command_lists);
+        static LRESULT __stdcall wnd_proc_detour(HWND hwnd, UINT msg, WPARAM w_param, LPARAM l_param);
+
+        mutable std::shared_mutex m_mutex;
+
+        bool m_is_dx12_hooked     = false;
+        bool m_is_wnd_proc_hooked = false;
+
+        std::vector<uintptr_t> m_vtable;
+        HWND    m_hooked_hwnd      = nullptr;
+        WNDPROC m_original_wndproc = nullptr;
+
+        safetyhook::InlineHook m_present_hook;
+        safetyhook::InlineHook m_resize_buffers_hook;
+        safetyhook::InlineHook m_execute_command_lists_hook;
+
+        present_callback              m_present_cb;
+        resize_buffers_callback       m_resize_buffers_cb;
+        execute_command_lists_callback m_execute_command_lists_cb;
+        wnd_proc_callback             m_wnd_proc_cb;
+    };
+}
+
+#else // !OMATH_ENABLE_HOOKING
+
+namespace omath::hooks
+{
+    class HooksManager final
+    {
+        HooksManager() = default;
+    public:
+        [[nodiscard]] static HooksManager& get();
+        HooksManager(const HooksManager&) = delete;
+        ~HooksManager();
+    };
+}
+
+#endif

source/hooks/hooks_manager.cpp

@@ -0,0 +1,369 @@
+#include "omath/hooks/hooks_manager.hpp"
+
+#ifdef OMATH_ENABLE_HOOKING
+#include <cstring>
+
+namespace
+{
+    class DummyWindow final
+    {
+        WNDCLASSEX m_window_class{};
+        HWND m_window_handle = nullptr;
+
+    public:
+        DummyWindow()
+        {
+            m_window_class.cbSize        = sizeof(WNDCLASSEX);
+            m_window_class.style         = CS_HREDRAW | CS_VREDRAW;
+            m_window_class.lpfnWndProc   = DefWindowProc;
+            m_window_class.hInstance     = GetModuleHandle(nullptr);
+            m_window_class.lpszClassName = "OM";
+            RegisterClassEx(&m_window_class);
+            m_window_handle = CreateWindow(m_window_class.lpszClassName, "Dummy", WS_OVERLAPPEDWINDOW,
+                                           0, 0, 100, 100, nullptr, nullptr, m_window_class.hInstance, nullptr);
+        }
+        ~DummyWindow()
+        {
+            if (m_window_handle)
+                DestroyWindow(m_window_handle);
+            UnregisterClass(m_window_class.lpszClassName, m_window_class.hInstance);
+        }
+        [[nodiscard]] HWND handle() const { return m_window_handle; }
+        [[nodiscard]] bool valid()  const { return m_window_handle != nullptr; }
+    };
+
+    // Method counts per interface (vtable slots including inherited ones)
+    constexpr std::size_t k_device_methods           = 44;
+    constexpr std::size_t k_command_queue_methods     = 19;
+    constexpr std::size_t k_command_allocator_methods = 9;
+    constexpr std::size_t k_command_list_methods      = 60;
+    constexpr std::size_t k_swap_chain_methods        = 18;
+    constexpr std::size_t k_total_methods = k_device_methods + k_command_queue_methods +
+                                            k_command_allocator_methods + k_command_list_methods +
+                                            k_swap_chain_methods; // 150
+
+    // Base offsets in the combined table
+    constexpr std::size_t k_cmd_queue_base  = k_device_methods; // 44
+    constexpr std::size_t k_swap_chain_base = k_device_methods + k_command_queue_methods +
+                                              k_command_allocator_methods + k_command_list_methods; // 132
+
+    // IDXGISwapChain vtable: Present=8, ResizeBuffers=13 (from IUnknown base)
+    // ID3D12CommandQueue vtable: ExecuteCommandLists=8 (from IUnknown base)
+    constexpr std::size_t k_present_index          = k_swap_chain_base + 8;   // 140
+    constexpr std::size_t k_resize_buffers_index   = k_swap_chain_base + 13;  // 145
+    constexpr std::size_t k_execute_cmd_lists_index = k_cmd_queue_base + 8;   // 52
+} // namespace
+
+namespace omath::hooks
+{
+    HooksManager& HooksManager::get()
+    {
+        static HooksManager obj;
+        return obj;
+    }
+
+    HooksManager::~HooksManager()
+    {
+        unhook_wnd_proc();
+        unhook_dx12();
+    }
+
+    bool HooksManager::build_vtable()
+    {
+        const DummyWindow window;
+        if (!window.valid())
+            return false;
+
+        const HMODULE d3d12_module = GetModuleHandle("d3d12.dll");
+        const HMODULE dxgi_module  = GetModuleHandle("dxgi.dll");
+        if (!d3d12_module || !dxgi_module)
+            return false;
+
+        using create_dxgi_factory_fn = HRESULT(__stdcall*)(REFIID, void**);
+        using d3d12_create_device_fn = HRESULT(__stdcall*)(IUnknown*, D3D_FEATURE_LEVEL, REFIID, void**);
+
+        const auto create_dxgi_factory = reinterpret_cast<create_dxgi_factory_fn>(
+            GetProcAddress(dxgi_module, "CreateDXGIFactory"));
+        const auto d3d12_create_device = reinterpret_cast<d3d12_create_device_fn>(
+            GetProcAddress(d3d12_module, "D3D12CreateDevice"));
+
+        if (!create_dxgi_factory || !d3d12_create_device)
+            return false;
+
+        IDXGIFactory* factory = nullptr;
+        if (FAILED(create_dxgi_factory(__uuidof(IDXGIFactory), reinterpret_cast<void**>(&factory))))
+            return false;
+
+        IDXGIAdapter* adapter = nullptr;
+        if (factory->EnumAdapters(0, &adapter) == DXGI_ERROR_NOT_FOUND)
+        {
+            factory->Release();
+            return false;
+        }
+
+        ID3D12Device* device = nullptr;
+        if (FAILED(d3d12_create_device(adapter, D3D_FEATURE_LEVEL_11_0,
+                                       __uuidof(ID3D12Device), reinterpret_cast<void**>(&device))))
+        {
+            adapter->Release();
+            factory->Release();
+            return false;
+        }
+        adapter->Release();
+
+        D3D12_COMMAND_QUEUE_DESC queue_desc{};
+        queue_desc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
+
+        ID3D12CommandQueue* command_queue = nullptr;
+        if (FAILED(device->CreateCommandQueue(&queue_desc, __uuidof(ID3D12CommandQueue),
+                                              reinterpret_cast<void**>(&command_queue))))
+        {
+            device->Release();
+            factory->Release();
+            return false;
+        }
+
+        ID3D12CommandAllocator* command_allocator = nullptr;
+        if (FAILED(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT,
+                                                   __uuidof(ID3D12CommandAllocator),
+                                                   reinterpret_cast<void**>(&command_allocator))))
+        {
+            command_queue->Release();
+            device->Release();
+            factory->Release();
+            return false;
+        }
+
+        ID3D12GraphicsCommandList* command_list = nullptr;
+        if (FAILED(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, command_allocator, nullptr,
+                                              __uuidof(ID3D12GraphicsCommandList),
+                                              reinterpret_cast<void**>(&command_list))))
+        {
+            command_allocator->Release();
+            command_queue->Release();
+            device->Release();
+            factory->Release();
+            return false;
+        }
+
+        DXGI_SWAP_CHAIN_DESC swap_chain_desc{};
+        swap_chain_desc.BufferDesc.Width       = 100;
+        swap_chain_desc.BufferDesc.Height      = 100;
+        swap_chain_desc.BufferDesc.RefreshRate = {60, 1};
+        swap_chain_desc.BufferDesc.Format      = DXGI_FORMAT_R8G8B8A8_UNORM;
+        swap_chain_desc.SampleDesc             = {1, 0};
+        swap_chain_desc.BufferUsage            = DXGI_USAGE_RENDER_TARGET_OUTPUT;
+        swap_chain_desc.BufferCount            = 2;
+        swap_chain_desc.OutputWindow           = window.handle();
+        swap_chain_desc.Windowed               = TRUE;
+        swap_chain_desc.SwapEffect             = DXGI_SWAP_EFFECT_FLIP_DISCARD;
+        swap_chain_desc.Flags                  = DXGI_SWAP_CHAIN_FLAG_ALLOW_MODE_SWITCH;
+
+        IDXGISwapChain* swap_chain = nullptr;
+        if (FAILED(factory->CreateSwapChain(command_queue, &swap_chain_desc, &swap_chain)))
+        {
+            command_list->Release();
+            command_allocator->Release();
+            command_queue->Release();
+            device->Release();
+            factory->Release();
+            return false;
+        }
+
+        m_vtable.resize(k_total_methods);
+
+        const auto copy_vtable = [](uintptr_t* dst, void* com_obj, std::size_t count)
+        {
+            std::memcpy(dst, *reinterpret_cast<uintptr_t**>(com_obj), count * sizeof(uintptr_t));
+        };
+
+        copy_vtable(m_vtable.data(),                                                                           device,           k_device_methods);
+        copy_vtable(m_vtable.data() + k_device_methods,                                                        command_queue,    k_command_queue_methods);
+        copy_vtable(m_vtable.data() + k_device_methods + k_command_queue_methods,                              command_allocator, k_command_allocator_methods);
+        copy_vtable(m_vtable.data() + k_device_methods + k_command_queue_methods + k_command_allocator_methods, command_list,    k_command_list_methods);
+        copy_vtable(m_vtable.data() + k_swap_chain_base,                                                       swap_chain,       k_swap_chain_methods);
+
+        swap_chain->Release();
+        command_list->Release();
+        command_allocator->Release();
+        command_queue->Release();
+        device->Release();
+        factory->Release();
+
+        return true;
+    }
+
+    bool HooksManager::hook_dx12()
+    {
+        std::unique_lock lock(m_mutex);
+        if (m_is_dx12_hooked)
+            return true;
+
+        if (!build_vtable())
+            return false;
+
+        m_present_hook = safetyhook::create_inline(
+            reinterpret_cast<void*>(m_vtable[k_present_index]),
+            reinterpret_cast<void*>(&present_detour));
+
+        m_resize_buffers_hook = safetyhook::create_inline(
+            reinterpret_cast<void*>(m_vtable[k_resize_buffers_index]),
+            reinterpret_cast<void*>(&resize_buffers_detour));
+
+        m_execute_command_lists_hook = safetyhook::create_inline(
+            reinterpret_cast<void*>(m_vtable[k_execute_cmd_lists_index]),
+            reinterpret_cast<void*>(&execute_command_lists_detour));
+
+        if (!m_present_hook || !m_resize_buffers_hook || !m_execute_command_lists_hook)
+        {
+            m_present_hook               = {};
+            m_resize_buffers_hook        = {};
+            m_execute_command_lists_hook = {};
+            return false;
+        }
+
+        m_is_dx12_hooked = true;
+        return true;
+    }
+
+    void HooksManager::unhook_dx12()
+    {
+        std::unique_lock lock(m_mutex);
+        m_present_hook               = {};
+        m_resize_buffers_hook        = {};
+        m_execute_command_lists_hook = {};
+        m_vtable.clear();
+        m_is_dx12_hooked = false;
+    }
+
+    void HooksManager::set_on_present(present_callback callback)
+    {
+        std::unique_lock lock(m_mutex);
+        m_present_cb = std::move(callback);
+    }
+
+    void HooksManager::set_on_resize_buffers(resize_buffers_callback callback)
+    {
+        std::unique_lock lock(m_mutex);
+        m_resize_buffers_cb = std::move(callback);
+    }
+
+    void HooksManager::set_on_execute_command_lists(execute_command_lists_callback callback)
+    {
+        std::unique_lock lock(m_mutex);
+        m_execute_command_lists_cb = std::move(callback);
+    }
+
+    bool HooksManager::hook_wnd_proc(HWND hwnd)
+    {
+        std::unique_lock lock(m_mutex);
+        if (m_is_wnd_proc_hooked)
+            return true;
+
+        const auto prev = reinterpret_cast<WNDPROC>(
+            SetWindowLongPtr(hwnd, GWLP_WNDPROC, reinterpret_cast<LONG_PTR>(&wnd_proc_detour)));
+        if (!prev)
+            return false;
+
+        m_hooked_hwnd       = hwnd;
+        m_original_wndproc  = prev;
+        m_is_wnd_proc_hooked = true;
+        return true;
+    }
+
+    void HooksManager::unhook_wnd_proc()
+    {
+        std::unique_lock lock(m_mutex);
+        if (!m_is_wnd_proc_hooked)
+            return;
+
+        SetWindowLongPtr(m_hooked_hwnd, GWLP_WNDPROC, reinterpret_cast<LONG_PTR>(m_original_wndproc));
+        m_hooked_hwnd        = nullptr;
+        m_original_wndproc   = nullptr;
+        m_is_wnd_proc_hooked = false;
+    }
+
+    void HooksManager::set_on_wnd_proc(wnd_proc_callback callback)
+    {
+        std::unique_lock lock(m_mutex);
+        m_wnd_proc_cb = std::move(callback);
+    }
+
+    // Detour implementations: copy callback under shared lock, call it unlocked,
+    // then call original. This avoids a deadlock if the callback itself calls set_on_*().
+    HRESULT __stdcall HooksManager::present_detour(IDXGISwapChain* p_swap_chain, UINT sync_interval, UINT flags)
+    {
+        auto& mgr = get();
+        present_callback cb;
+        {
+            std::shared_lock lock(mgr.m_mutex);
+            cb = mgr.m_present_cb;
+        }
+        if (cb)
+            cb(p_swap_chain, sync_interval, flags);
+        return mgr.m_present_hook.call<HRESULT>(p_swap_chain, sync_interval, flags);
+    }
+
+    HRESULT __stdcall HooksManager::resize_buffers_detour(IDXGISwapChain* p_swap_chain, UINT buffer_count,
+                                                          UINT width, UINT height, DXGI_FORMAT new_format,
+                                                          UINT swap_chain_flags)
+    {
+        auto& mgr = get();
+        resize_buffers_callback cb;
+        {
+            std::shared_lock lock(mgr.m_mutex);
+            cb = mgr.m_resize_buffers_cb;
+        }
+        if (cb)
+            cb(p_swap_chain, buffer_count, width, height, new_format, swap_chain_flags);
+        return mgr.m_resize_buffers_hook.call<HRESULT>(p_swap_chain, buffer_count, width, height, new_format,
+                                                       swap_chain_flags);
+    }
+
+    void __stdcall HooksManager::execute_command_lists_detour(ID3D12CommandQueue* p_command_queue,
+                                                              UINT num_command_lists,
+                                                              ID3D12CommandList* const* pp_command_lists)
+    {
+        auto& mgr = get();
+        execute_command_lists_callback cb;
+        {
+            std::shared_lock lock(mgr.m_mutex);
+            cb = mgr.m_execute_command_lists_cb;
+        }
+        if (cb)
+            cb(p_command_queue, num_command_lists, pp_command_lists);
+        mgr.m_execute_command_lists_hook.call<void>(p_command_queue, num_command_lists, pp_command_lists);
+    }
+
+    LRESULT __stdcall HooksManager::wnd_proc_detour(HWND hwnd, UINT msg, WPARAM w_param, LPARAM l_param)
+    {
+        auto& mgr = get();
+        wnd_proc_callback cb;
+        WNDPROC original;
+        {
+            std::shared_lock lock(mgr.m_mutex);
+            cb       = mgr.m_wnd_proc_cb;
+            original = mgr.m_original_wndproc;
+        }
+        if (cb)
+        {
+            if (const auto result = cb(hwnd, msg, w_param, l_param))
+                return *result;
+        }
+        return CallWindowProc(original, hwnd, msg, w_param, l_param);
+    }
+} // namespace omath::hooks
+
+#else // !OMATH_ENABLE_HOOKING
+
+namespace omath::hooks
+{
+    HooksManager& HooksManager::get()
+    {
+        static HooksManager obj;
+        return obj;
+    }
+    HooksManager::~HooksManager() = default;
+} // namespace omath::hooks
+
+#endif

vcpkg.json

@@ -16,6 +16,15 @@
     }
   ],
   "features": {
+    "all": {
+      "description": "Enable all additional features",
+      "dependencies": [
+        {
+          "name": "omath",
+          "features": ["imgui", "lua", "hooking"]
+        }
+      ]
+    },
     "avx2": {
       "description": "omath will use AVX2 to boost performance",
       "supports": "!arm"
@@ -26,6 +35,13 @@
         "benchmark"
       ]
     },
+    "hooking": {
+      "description": "Add interface for automatic hooking of DirectX",
+      "dependencies": [
+        "safetyhook"
+      ],
+      "supports": "(windows | linux) & !arm & !uwp"
+    },
     "examples": {
       "description": "Build examples",
       "dependencies": [