vmcall_test.cpp

//
// vmcall_test.cpp
// VMWare hypercall backdoor quick test.
//
// sixtyvividtails, 2025
// public domain
//
// Run this on guest system under VMWare Workstation 17+.
// Guest additions not needed (but won't harm).
// Processor virtualization options don't matter.
// However, VBS support should not be enabled.
//
// Info/defines: https://github.com/vmware/open-vm-tools
// Legacy io port backdoor interface: https://wiki.osdev.org/VMware_tools
// Thread: 👉 https://x.com/sixtyvividtails/status/1904513296683639120
//
//
// Expected output:
//
// hypervisor: VMwareVMware/Hv#1, present: 1; max software leaf: 40000010
// cpuid(0x40000010): tsc rate: 3400018 kHz, lapic rate: 66000 kHz
// VMWare hypercall backdoor enabled: 1
// [+] vmcall/vmmcall succeeded; results below: rax rbx | rcx rdx | rsi rdi
// [ ] BDOOR_CMD_GETVERSION: 00000006 564D5868 | 00000004 00000000 | 00000000 00000000
// [+] backdoor interface ok, version: 6
// [ ] BDOOR_CMD_MESSAGE: 00000000 CF4C4354 | 00010000 00010000 | A9BCAD1B 7073D349
// [+] TCLO channel opened, channelId: 0001, cookie: A9BCAD1B'7073D349
//
#include <stdio.h>
#include <conio.h>
#include <stdlib.h>
#include <intrin.h>
#include <Windows.h>
#include <phnt/phnt.h>


using uint = unsigned int;
using ulong = unsigned long;
using uchar = unsigned char;
using uint64 = unsigned long long;
using wchar = wchar_t;
using puint = uint*;
using pcstr = const char*;


// from backdoor_def.h in open-vm-tools
#define   BDOOR_CMD_GETVERSION               10
#define   BDOOR_CMD_MESSAGE                  30
#define   BDOOR_CMD_SIDT                     31
#define   BDOOR_CMD_FIRMWARE_UPDATE          93 /* CPL 0 only. */


#pragma const_seg(push, ".text_hypercall_backdoor")
DECLSPEC_SELECTANY extern const uchar alignas(0x10) hypercall_backdoor_asm[] =
{
    0x81, 0xFA, 0x6E, 0x74, 0x65, 0x6C,     // cmp edx, 0x6C65746E   // "ntel", 'letn'
    0x48, 0x89, 0xE0,                       // mov rax, rsp
    0x48, 0x89, 0x68, 0x08,                 // mov [rax+8], rbp
    0x48, 0x89, 0x58, 0x10,                 // mov [rax+0x10], rbx
    0x48, 0x89, 0x70, 0x18,                 // mov [rax+0x18], rsi
    0x48, 0x89, 0x78, 0x20,                 // mov [rax+0x20], rdi
    0x48, 0x89, 0xCD,                       // mov rbp, rcx
    0x48, 0x8B, 0x01,                       // mov rax, [rcx]
    0x48, 0x8B, 0x59, 0x08,                 // mov rbx, [rcx+8]
    0x48, 0x8B, 0x51, 0x18,                 // mov rdx, [rcx+0x18]
    0x48, 0x8B, 0x71, 0x20,                 // mov rsi, [rcx+0x20]
    0x48, 0x8B, 0x79, 0x28,                 // mov rdi, [rcx+0x28]
    0x48, 0x8B, 0x49, 0x10,                 // mov rcx, [rcx+0x10]
    0x74, 0x05,                             // je .intel
    // .amd:
    0x0F, 0x01, 0xD9,                       // vmmcall
    0xEB, 0x03,                             // jmp @f
    // .intel:
    0x0F, 0x01, 0xC1,                       // vmcall
    // @@:
    0x48, 0x89, 0x45, 0x00,                 // mov [rbp], rax
    0x48, 0x89, 0x5D, 0x08,                 // mov [rbp+8], rbx
    0x48, 0x89, 0x4D, 0x10,                 // mov [rbp+0x10], rcx
    0x48, 0x89, 0x55, 0x18,                 // mov [rbp+0x18], rdx
    0x48, 0x89, 0x75, 0x20,                 // mov [rbp+0x20], rsi
    0x48, 0x89, 0x7D, 0x28,                 // mov [rbp+0x28], rdi
    0x48, 0x89, 0xE0,                       // mov rax, rsp
    0x48, 0x8B, 0x68, 0x08,                 // mov rbp, [rax+8]
    0x48, 0x8B, 0x58, 0x10,                 // mov rbx, [rax+0x10]
    0x48, 0x8B, 0x70, 0x18,                 // mov rsi, [rax+0x18]
    0x48, 0x8B, 0x78, 0x20,                 // mov rdi, [rax+0x20]
    0xC3                                    // ret
};
#pragma const_seg(pop)
#pragma comment(linker, "/merge:.text_hypercall_backdoor=.text")
#pragma comment(linker, "/alternatename:?hypercall_backdoor@@YAXAEAY05_KI@Z=?hypercall_backdoor_asm@@3QBEB")

// abcdx: rax, rbx, rcx, rdx, rsi, rdi; vendor: "ntel" to use vmcall, otherwise we'll use vmmcall
void hypercall_backdoor(_Inout_ uint64 (&abcdx)[6], uint vendor);


static NTSTATUS invoke_hypercall_backdoor(_Out_ uint64 (&abcdx)[6],
    uint64 rcxCommand, uint64 rbx = 0, uint64 rdx = 0, uint64 rsi = 0, uint64 rdi = 0)
{
    static uint cpuVendor;
    if (!cpuVendor)
    {
        int regs[4]{};
        __cpuidex(regs, 0, 0);
        cpuVendor = regs[2];    // ecx, "ntel" for Intel (vmcall), otherwise assume AMD (vmmcall)
    }
    // rax mark is "hXMV", 0x564D5868
    uint64 abcdxLocal[6]{'VMXh', rbx, rcxCommand, rdx, rsi, rdi};
    NTSTATUS exception{};
    __try
    {
        hypercall_backdoor(abcdxLocal, cpuVendor);
    }
    __except ([&](EXCEPTION_POINTERS* ex)
    {
        // we can't unwind that func, coz we don't have unwind info - gotta handle everything in filter
        NTSTATUS ecode = ex->ExceptionRecord->ExceptionCode;
        exception = ecode? ecode: -1;
        size_t& rip = ex->ContextRecord->Rip;
        printf("exception: %08X, func offset: %02I64X\n", ecode, rip - (size_t)&hypercall_backdoor_asm);
        rip += 3;   // HACKHACK: skip vmcall/vmmcall
        return EXCEPTION_CONTINUE_EXECUTION;
    }(GetExceptionInformation()))
    {
        NOTHING;    // should never be here
    }
    if (exception)
        return exception | 0x8000'0000;
    memcpy(abcdx, abcdxLocal, sizeof(abcdx));
    return STATUS_SUCCESS;
}


static void check_hypercall_backdoor()
{
    // check backdoor presense/version
    uint64 abcdx[6]{};
    NTSTATUS st = invoke_hypercall_backdoor(abcdx, BDOOR_CMD_GETVERSION);
    if (FAILED(st))
    {
        printf("[x] BDOOR_CMD_GETVERSION: failed to invoke hypercall backdoor: %08X\n", st);
        return;
    }
    printf("[+] vmcall/vmmcall succeeded; results below: rax rbx | rcx rdx | rsi rdi\n");
    printf("[ ] BDOOR_CMD_GETVERSION: %08I64X %08I64X | %08I64X %08I64X | %08I64X %08I64X\n",
        abcdx[0], abcdx[1], abcdx[2], abcdx[3], abcdx[4], abcdx[5]);
    if ((uint)abcdx[1] == 'VMXh')
        printf("[+] backdoor interface ok, version: %X\n", (uint)abcdx[0]);
    else
        printf("[x] backdoor interface NOT ok, unexpected resonse\n");

    // try to open communication channel for TCLO protocol
#define GUESTMSG_FLAG_COOKIE 0x80000000
#define MESSAGE_TYPE_OPEN 0
    memset(abcdx, 0, sizeof(abcdx));
    uint protocolId = 'OLCT';           // "TCLO", 0x4f4c4354
    st = invoke_hypercall_backdoor(abcdx,
        BDOOR_CMD_MESSAGE | (MESSAGE_TYPE_OPEN << 0x10),
        protocolId | GUESTMSG_FLAG_COOKIE);
    if (FAILED(st))
    {
        printf("[x] BDOOR_CMD_MESSAGE: failed to invoke hypercall backdoor: %08X\n", st);
        return;
    }
    printf("[ ] BDOOR_CMD_MESSAGE: %08I64X %08I64X | %08I64X %08I64X | %08I64X %08I64X\n",
        abcdx[0], abcdx[1], abcdx[2], abcdx[3], abcdx[4], abcdx[5]);
    bool succeeded = (abcdx[2] >> 0x10) & 1;
    uint channelId = (uint)abcdx[3] >> 0x10;
    if (succeeded)
        printf("[+] TCLO channel opened, channelId: %04X, cookie: %08I64X'%08I64X\n", channelId, abcdx[4], abcdx[5]);
    else
        printf("[x] TCLO channel NOT opened\n");
}


static void get_hv_info(_Out_ uint* maxLeaf)
{
    *maxLeaf = 0;
    char hvBrand[0x14]{};
    int abcd[4]{};
    __cpuidex(abcd, 0x4000'0000, 0);    // retrieves max leaf for this range, and string like 'Microsoft Hv'
    uint hvMaxLeaf = abcd[0];
    *maxLeaf = hvMaxLeaf;
    (puint(hvBrand))[0] = abcd[1];      // ebx
    (puint(hvBrand))[1] = abcd[2];      // ecx
    (puint(hvBrand))[2] = abcd[3];      // edx
    if (hvMaxLeaf >= 0x4000'0001)
    {
        __cpuidex(abcd, 0x4000'0001, 0);
        size_t len = strlen(hvBrand);
        if (len && abcd[0])             // e.g. "Hv#1"
            hvBrand[len++] = '/';
        *puint(&hvBrand[len]) = abcd[0];
    }
    for (char& c: hvBrand)
        if (c && (c < ' ' || c > 0x7F))
            c = '.';
    if (!hvBrand[0])
        *puint(hvBrand) = '-';
    __cpuidex(abcd, 1, 0);
    bool hvPresent = (abcd[2] >> 0x1F) != 0;
    printf("hypervisor: %s, present: %u; max software leaf: %08X\n", hvBrand, hvPresent, hvMaxLeaf);

    memset(abcd, 0, sizeof(abcd));
    if (hvMaxLeaf >= 0x40000010)
        __cpuidex(abcd, 0x40000010, 0);
    printf("cpuid(0x40000010): tsc rate: %u kHz, lapic rate: %u kHz\nVMWare hypercall backdoor enabled: %u\n",
        abcd[0], abcd[1], (abcd[2] & 3) != 0);      // bit0: vmmcall (AMD), bit1: vmcall (Intel)
}


static void wait_console_keypress_if_needed(_In_opt_ pcstr message)
{
    ulong dummy;
    bool needPause =
        GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &dummy)     // success, this means output is not redirected
        && GetConsoleProcessList(&dummy, 1) <= 1;                   // single console process: it's our process, no cmd
    if (!needPause)
        return;

    if (message)
        printf("%s\n", message);
    _flushall();
    int c = _getch();
    if (!c || c == 0xE0)    // arrow or function key, read one more char to drain queue
        (void)_getch();
}


static NTSTATUS set_thread_info(THREADINFOCLASS infoclass, const auto& data, HANDLE thread = NtCurrentThread())
{
    return NtSetInformationThread(thread, infoclass, (void*)&data, sizeof(data));
}


int __cdecl wmain(int argc, const wchar_t* argv[])
{
    UNREFERENCED_PARAMETER(argc);
    UNREFERENCED_PARAMETER(argv);

    set_thread_info(ThreadGroupInformation, GROUP_AFFINITY{.Mask=1});   // pin to cpu0 in group 0

    uint maxLeaf;
    get_hv_info(&maxLeaf);
    if (maxLeaf >= 0x40000010)
        check_hypercall_backdoor();

    wait_console_keypress_if_needed("Press any key to exit...");
    return 0;
}