LuaNewstate.cpp

#include "LuaNewstate.h"
#include <Windows.h>
#include <cstdint>
#include <algorithm>

#define LU_MODULE  "lua_shared.dll"
#define LU_NEWSTATE_N "lua_newstate"
#define LU_NEWSTATE_I 0xD9
#define LU_CLOSE_N "lua_close"
#define LU_CLOSE_I 0x89

#define OPCODE_JMP '\xE9'
#define TCALL_ADDR_OLD 0xFFFFFDE2u
#define UU_OPCODE_OLD '\x5F'
#define UU_ADDRESS_OLD 0xC35B5D5Eu

LuaNewstate::LuaStateList vecLuaStates;

void* pLuaStateTmp = nullptr;
void* pAddrCloseTailCall = nullptr;

void* _HandlerNewstate() {
    vecLuaStates.push_back((lua_State*)pLuaStateTmp);
    return pLuaStateTmp;
}

__declspec(naked)
void _DetourNewstate() {
    __asm {
        pop edi
        pop esi
        pop ebp
        pop ebx
        mov [pLuaStateTmp], eax
        jmp _HandlerNewstate
    }
}

void _HandlerClose() {
    auto itBegin = vecLuaStates.begin();
    auto itEnd = vecLuaStates.end();
    vecLuaStates.erase(std::remove(itBegin, itEnd, pLuaStateTmp), itEnd);
}

__declspec(naked)
void _DetourClose() {
    __asm {
        pushad
        mov eax, [esp-8]
        mov [pLuaStateTmp], eax
        call _HandlerClose
        popad
        jmp [pAddrCloseTailCall]
    }
}

bool LuaNewstate::SetupDetour() {
    auto hModule = GetModuleHandleA(LU_MODULE);
    auto pLuaOpen = GetProcAddress(hModule, LU_NEWSTATE_N);
    auto pLuaClose = GetProcAddress(hModule, LU_CLOSE_N);

    {
        char* hookPtr = (char*)pLuaOpen + LU_NEWSTATE_I;
        uint32_t hookAddr = (uint32_t)_DetourNewstate - (uint32_t)hookPtr - 5;

        if(hookPtr[0] != OPCODE_JMP) {
            DWORD dwProt;
            VirtualProtect(pLuaOpen, 0x100, PAGE_READWRITE, &dwProt);

            hookPtr[0] = OPCODE_JMP;
            *(uint32_t*)(hookPtr + 1) = hookAddr;

            VirtualProtect(pLuaOpen, 0x100, dwProt, &dwProt);
        }
    }

    {
        char* hookPtr = (char*)pLuaClose + LU_CLOSE_I;
        uint32_t hookAddr = (uint32_t)_DetourClose - (uint32_t)hookPtr - 5;
        uint32_t tmpAddr = *(uint32_t*)(hookPtr + 1);

        if(tmpAddr == TCALL_ADDR_OLD) {
            DWORD dwProt;
            VirtualProtect(pLuaClose, 0x100, PAGE_READWRITE, &dwProt);

            uint32_t tmpAddr = *(uint32_t*)(hookPtr + 1);
            pAddrCloseTailCall = (void*)(tmpAddr + (uint32_t)hookPtr + 5);

            //hookPtr[0] = (char)0xE9;
            *(uint32_t*)(hookPtr + 1) = hookAddr;

            VirtualProtect(pLuaClose, 0x100, dwProt, &dwProt);
        }

    }

    return true;
}

bool LuaNewstate::RemoveDetour() {
    auto hModule = GetModuleHandleA(LU_MODULE);
    auto pLuaOpen = GetProcAddress(hModule, LU_NEWSTATE_N);
    auto pLuaClose = GetProcAddress(hModule, LU_CLOSE_N);

    {
        char* hookPtr = (char*)pLuaOpen + LU_NEWSTATE_I;

        if(hookPtr[0] == OPCODE_JMP) {
            DWORD dwProt;
            VirtualProtect(pLuaOpen, 0xE0, PAGE_READWRITE, &dwProt);

            hookPtr[0] = UU_OPCODE_OLD;
            *(uint32_t*)(hookPtr + 1) = UU_ADDRESS_OLD;

            VirtualProtect(pLuaOpen, 0xE0, dwProt, &dwProt);
        }
    }

    {
        char* hookPtr = (char*)pLuaClose + LU_CLOSE_I;
        uint32_t tmpAddr = *(uint32_t*)(hookPtr + 1);

        if(tmpAddr != TCALL_ADDR_OLD) {
            DWORD dwProt;
            VirtualProtect(pLuaClose, 0x90, PAGE_READWRITE, &dwProt);

            //hookPtr[0] = (char)0xE9;
            *(uint32_t*)(hookPtr + 1) = TCALL_ADDR_OLD;

            VirtualProtect(pLuaClose, 0x90, dwProt, &dwProt);
        }
    }

    vecLuaStates.clear();

    return true;
}

LuaNewstate::LuaStateList LuaNewstate::Get() {
    return vecLuaStates;
}