Vulkan TLAS building problems

void AccelerationStructure::build(vk::BuildAccelerationStructureModeKHR mode, vk::ArrayProxyNoTemporaries<vk::SharedSemaphore> waitSemaphores, vk::ArrayProxyNoTemporaries<vk::SharedSemaphore> signalSemaphores) {
    device->waitForFences(*buildFinishedFence, vk::True, std::numeric_limits<uint64_t>::max());
    asBuildCmdBuffer->reset();
    asBuildCmdBuffer->begin(vk::CommandBufferBeginInfo{}.setFlags(vk::CommandBufferUsageFlagBits::eOneTimeSubmit));

    std::vector<std::unique_ptr<Buffer>> blasScratchBuffers;
    buildBLAS(blasScratchBuffers, mode);

    // Insert pipeline barrier betwee BLAS and TLAS build
    auto& memBarrier = vk::MemoryBarrier{}
        .setSrcAccessMask(vk::AccessFlagBits::eAccelerationStructureWriteKHR)
        .setDstAccessMask(vk::AccessFlagBits::eAccelerationStructureReadKHR);
    std::vector<vk::BufferMemoryBarrier> blasMemBarriers;
    blasMemBarriers.reserve(blasBuffers.size());
    for (auto& b : blasBuffers) {
        blasMemBarriers.push_back(vk::BufferMemoryBarrier{}
                                  .setBuffer(**b)
                                  .setSrcAccessMask(vk::AccessFlagBits::eAccelerationStructureWriteKHR)
                                  .setDstAccessMask(vk::AccessFlagBits::eAccelerationStructureReadKHR)
                                  .setOffset(0u)
                                  .setSize(vk::WholeSize));
    }
    asBuildCmdBuffer->pipelineBarrier(vk::PipelineStageFlagBits::eAccelerationStructureBuildKHR, vk::PipelineStageFlagBits::eAccelerationStructureBuildKHR,
                                      {}, memBarrier, blasMemBarriers, {});

    std::unique_ptr<Buffer> instancesBuffer, tlasScratchBuffer;
    buildTLAS(instancesBuffer, tlasScratchBuffer, mode);
    asBuildCmdBuffer->end();

    // Submit build commands
    std::vector<vk::PipelineStageFlags> submitWaitDstStageMask(waitSemaphores.size());
    std::vector<vk::Semaphore> submitWaitSemaphores(waitSemaphores.size());
    std::vector<vk::Semaphore> submitSignalSemaphores(signalSemaphores.size());
    std::fill(submitWaitDstStageMask.begin(), submitWaitDstStageMask.end(), vk::PipelineStageFlagBits::eAccelerationStructureBuildKHR);
    std::transform(waitSemaphores.begin(), waitSemaphores.end(), submitWaitSemaphores.begin(),
                   [](const vk::SharedSemaphore& ws) { return *ws; });
    std::transform(signalSemaphores.begin(), signalSemaphores.end(), submitSignalSemaphores.begin(),
                   [](const vk::SharedSemaphore& ss) { return *ss; });
    auto& submitInfo = vk::SubmitInfo{}
        .setCommandBuffers(*asBuildCmdBuffer)
        .setWaitDstStageMask(submitWaitDstStageMask)
        .setWaitSemaphores(submitWaitSemaphores)
        .setSignalSemaphores(submitSignalSemaphores);
    device->resetFences(*buildFinishedFence);
    std::get<vk::Queue>(computeQueue).submit(submitInfo, *buildFinishedFence);
    CHECK_VULKAN_RESULT(device->waitForFences(*buildFinishedFence, vk::True, std::numeric_limits<uint64_t>::max()));
}

void AccelerationStructure::buildTLAS(std::unique_ptr<Buffer>& instancesBuffer, std::unique_ptr<Buffer>& scratchBuffer, vk::BuildAccelerationStructureModeKHR mode) {
    std::vector<vk::AccelerationStructureInstanceKHR> instanceData;
    instanceData.reserve(scene.objectCount);

    uint32_t idx = 0;
    for (auto& it = scene.begin(); it != scene.end(); it++) {
        auto& sceneObject = (*it);
        if (sceneObject.meshIdx) {
            std::array<std::array<float, 4Ui64>, 3Ui64> transformMatrix;
            auto& affineTransform = glm::mat3x4(glm::transpose(it.transform));
            memcpy(transformMatrix.data(), &affineTransform, sizeof(transformMatrix));
            instanceData.push_back(vk::AccelerationStructureInstanceKHR{}
                                   .setTransform(vk::TransformMatrixKHR{}.setMatrix(transformMatrix))
                                   .setInstanceCustomIndex(*sceneObject.meshIdx)
                                   .setMask(0xFF)
                                   .setInstanceShaderBindingTableRecordOffset(0u)
                                   .setFlags(vk::GeometryInstanceFlagBitsKHR::eTriangleFacingCullDisable)
                                   .setAccelerationStructureReference(device->getAccelerationStructureAddressKHR(*blas[*sceneObject.meshIdx])));
            idx++;
        }
    }
    auto& instancesBufferCI = vk::BufferCreateInfo{}
        .setSize(sizeof(vk::AccelerationStructureInstanceKHR) * instanceData.size())
        .setUsage(vk::BufferUsageFlagBits::eAccelerationStructureBuildInputReadOnlyKHR | vk::BufferUsageFlagBits::eShaderDeviceAddress);
    instancesBuffer = std::make_unique<Buffer>(device, dmm, rch, instancesBufferCI,
                                               vk::ArrayProxyNoTemporaries{ static_cast<uint32_t>(sizeof(vk::AccelerationStructureInstanceKHR) * instanceData.size()), (char*)instanceData.data() },
                                               MemoryStorage::DeviceDynamic);

    auto& accelerationStructureGeometry = vk::AccelerationStructureGeometryKHR{}
        .setGeometryType(vk::GeometryTypeKHR::eInstances)
        .setFlags(vk::GeometryFlagBitsKHR::eOpaque)
        .setGeometry(vk::AccelerationStructureGeometryInstancesDataKHR{}
                     .setArrayOfPointers(vk::False)
                     .setData(device->getBufferAddress(**instancesBuffer)));
    auto& accelerationStructureBGI = vk::AccelerationStructureBuildGeometryInfoKHR{}
        .setType(vk::AccelerationStructureTypeKHR::eTopLevel)
        .setFlags(vk::BuildAccelerationStructureFlagBitsKHR::ePreferFastTrace)
        .setGeometries(accelerationStructureGeometry);
    auto& accelerationStructureBSI = device->getAccelerationStructureBuildSizesKHR(vk::AccelerationStructureBuildTypeKHR::eDevice, accelerationStructureBGI, instanceData.size());

    if (mode == vk::BuildAccelerationStructureModeKHR::eBuild) {
        tlasBuffer = std::make_unique<Buffer>(device, dmm, rch, vk::BufferCreateInfo{}
                                              .setSize(accelerationStructureBSI.accelerationStructureSize)
                                              .setUsage(vk::BufferUsageFlagBits::eAccelerationStructureStorageKHR | vk::BufferUsageFlagBits::eShaderDeviceAddress),
                                              nullptr, MemoryStorage::DevicePersistent);
        auto& accelerationStructureCI = vk::AccelerationStructureCreateInfoKHR{}
            .setBuffer(**tlasBuffer)
            .setSize(accelerationStructureBSI.accelerationStructureSize)
            .setType(vk::AccelerationStructureTypeKHR::eTopLevel);
        tlas = device->createAccelerationStructureKHRUnique(accelerationStructureCI);
    }

    auto& scratchBufferCI = vk::BufferCreateInfo{}
        .setSize(mode == vk::BuildAccelerationStructureModeKHR::eBuild ? accelerationStructureBSI.buildScratchSize : accelerationStructureBSI.updateScratchSize)
        .setUsage(vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eShaderDeviceAddress);
    scratchBuffer = std::make_unique<Buffer>(device, dmm, rch, scratchBufferCI,
                                             nullptr, MemoryStorage::DeviceDynamic);

    accelerationStructureBGI
        .setMode(mode)
        .setSrcAccelerationStructure(mode == vk::BuildAccelerationStructureModeKHR::eUpdate ? *tlas : nullptr)
        .setDstAccelerationStructure(*tlas)
        .setScratchData(device->getBufferAddress(**scratchBuffer));
    auto& accelerationStructureBRI = vk::AccelerationStructureBuildRangeInfoKHR{}
        .setPrimitiveCount(instanceData.size())
        .setPrimitiveOffset(0u)
        .setFirstVertex(0u)
        .setTransformOffset(0u);

    asBuildCmdBuffer->buildAccelerationStructuresKHR(accelerationStructureBGI, &accelerationStructureBRI);
}