NDISK_defrag

-- ndisk_defrag.lua
-- NDisk Defragmentation Tool with RAID10 Support
-- Run this WHILE SERVER IS STOPPED to safely redistribute files across all nodes

-- SAFETY: This script uses copy-first, delete-last approach
-- Original data is never deleted until new data is verified

local modem = peripheral.find("modem")
if not modem then error("No modem found") end
modem.open(5) -- NDisk channel

-- Load NDisk secret for authentication
local NDISK_SECRET = nil
if fs.exists(".ndisk_secret") then
    local f = fs.open(".ndisk_secret", "r")
    NDISK_SECRET = f.readAll()
    f.close()
end

-- Configuration
local MAX_FILE_SIZE = 64 * 1024  -- 64KB safety limit
local NODE_TIMEOUT = 3           -- Seconds to wait for node responses
local REQUEST_TIMEOUT = 2        -- Seconds for individual requests

-- State
local nodes = {}           -- hierarchicalNum -> nodeId
local chunkMap = {}        -- filename -> {chunkNum -> {primary, mirror} or nodeId}
local stats = {
    filesProcessed = 0,
    filesSkipped = 0,
    filesUpgraded = 0,     -- Files upgraded to RAID10
    bytesProcessed = 0,
    errors = {}
}

-- ============================================
-- RAID10 HELPERS
-- ============================================

local function getMirrorIndex(primaryIdx, nodeCount)
    if nodeCount < 2 then return nil end
    local offset = math.floor(nodeCount / 2)
    return ((primaryIdx - 1 + offset) % nodeCount) + 1
end

local function getSortedNodes()
    local sorted = {}
    for idx, id in pairs(nodes) do
        table.insert(sorted, {idx = idx, id = id})
    end
    table.sort(sorted, function(a, b) return a.idx < b.idx end)
    return sorted
end

-- ============================================
-- UTILITY FUNCTIONS
-- ============================================

local function log(msg)
    local timestamp = os.date("%H:%M:%S")
    print("[" .. timestamp .. "] " .. msg)
end

local function logError(msg)
    table.insert(stats.errors, msg)
    term.setTextColor(colors.red)
    log("ERROR: " .. msg)
    term.setTextColor(colors.white)
end

local function logSuccess(msg)
    term.setTextColor(colors.lime)
    log(msg)
    term.setTextColor(colors.white)
end

local function logWarn(msg)
    term.setTextColor(colors.yellow)
    log("WARN: " .. msg)
    term.setTextColor(colors.white)
end

-- ============================================
-- NODE COMMUNICATION
-- ============================================

local function generateRequestId()
    return os.getComputerID() .. "_" .. os.epoch("utc") .. "_" .. math.random(10000)
end

local function sendRequest(nodeId, request, fallbackNodeId)
    request.targetNode = nodeId
    request.secret = NDISK_SECRET  -- Include auth secret
    modem.transmit(5, 5, request)
    local timer = os.startTimer(REQUEST_TIMEOUT)

    while true do
        local event, p1, p2, p3, p4 = os.pullEvent()
        if event == "modem_message" and type(p4) == "table" and p4.id == request.id then
            return p4.response
        elseif event == "timer" and p1 == timer then
            -- Primary timeout, try fallback
            if fallbackNodeId and fallbackNodeId ~= nodeId then
                request.targetNode = fallbackNodeId
                modem.transmit(5, 5, request)
                local timer2 = os.startTimer(REQUEST_TIMEOUT)

                while true do
                    local ev2, pp1, pp2, pp3, pp4 = os.pullEvent()
                    if ev2 == "modem_message" and type(pp4) == "table" and pp4.id == request.id then
                        return pp4.response
                    elseif ev2 == "timer" and pp1 == timer2 then
                        return nil, "Both nodes timeout"
                    end
                end
            end
            return nil, "Timeout"
        end
    end
end

-- ============================================
-- NODE DISCOVERY
-- ============================================

local function discoverNodes()
    log("Discovering storage nodes...")
    modem.transmit(5, 5, {type = "discovery", secret = NDISK_SECRET})

    local discovered = {}
    local timer = os.startTimer(NODE_TIMEOUT)

    while true do
        local event, p1, p2, p3, p4 = os.pullEvent()
        if event == "modem_message" and type(p4) == "table" and p4.type == "discovery_response" then
            discovered[p4.id] = true
        elseif event == "timer" and p1 == timer then
            break
        end
    end

    -- Convert to sorted list for consistent ordering
    local nodeList = {}
    for id, _ in pairs(discovered) do
        table.insert(nodeList, id)
    end
    table.sort(nodeList)

    -- Assign hierarchical numbers
    nodes = {}
    for i, id in ipairs(nodeList) do
        nodes[i] = id
    end

    return #nodeList
end

local function saveNodeIds()
    local file = fs.open("ids.txt", "w")
    for hierarchicalNum, nodeId in pairs(nodes) do
        file.writeLine(nodeId .. " " .. hierarchicalNum)
    end
    file.close()
    log("Saved " .. #nodes .. " nodes to ids.txt")
end

-- ============================================
-- CHUNK MAP (RAID10 FORMAT)
-- ============================================

local function loadChunkMap()
    chunkMap = {}
    if fs.exists("chunklocation.txt") then
        local file = fs.open("chunklocation.txt", "r")
        for line in file.readLine do
            local parts = {}
            for part in line:gmatch("%S+") do
                table.insert(parts, part)
            end

            if #parts >= 3 then
                local filename = parts[1]
                local chunkMatch = parts[2]:match("chunk(%d+)")
                local chunkNum = chunkMatch and tonumber(chunkMatch) or nil
                local primaryId = tonumber(parts[3])
                local mirrorId = parts[4] and tonumber(parts[4]) or nil

                -- Skip malformed lines
                if filename and chunkNum and primaryId then
                    chunkMap[filename] = chunkMap[filename] or {}

                    if mirrorId and mirrorId ~= primaryId then
                        chunkMap[filename][chunkNum] = {primary = primaryId, mirror = mirrorId}
                    else
                        chunkMap[filename][chunkNum] = {primary = primaryId, mirror = nil}
                    end
                end
            end
        end
        file.close()
    end
end

local function saveChunkMap()
    local file = fs.open("chunklocation.txt", "w")
    for filename, chunks in pairs(chunkMap) do
        for chunkNum, info in pairs(chunks) do
            if type(info) == "table" then
                local mirrorId = info.mirror or info.primary
                file.writeLine(filename .. " chunk" .. chunkNum .. " " .. info.primary .. " " .. mirrorId)
            else
                file.writeLine(filename .. " chunk" .. chunkNum .. " " .. info)
            end
        end
    end
    file.close()
    log("Saved chunk map")
end

local function backupChunkMap()
    if fs.exists("chunklocation.txt") then
        if fs.exists("chunklocation.txt.backup") then
            fs.delete("chunklocation.txt.backup")
        end
        fs.copy("chunklocation.txt", "chunklocation.txt.backup")
        log("Backed up chunklocation.txt")
    end
end

-- ============================================
-- FILE OPERATIONS (RAID10)
-- ============================================

local function readFileFromNodes(path)
    if not chunkMap[path] then
        return nil, "File not found in chunk map"
    end

    local contentChunks = {}
    local chunkCount = 0
    for _ in pairs(chunkMap[path]) do chunkCount = chunkCount + 1 end

    for chunkNum = 1, chunkCount do
        local chunkInfo = chunkMap[path][chunkNum]
        if not chunkInfo then
            return nil, "Missing chunk " .. chunkNum .. " info"
        end

        local primaryId, mirrorId
        if type(chunkInfo) == "table" then
            primaryId = chunkInfo.primary
            mirrorId = chunkInfo.mirror
        else
            primaryId = chunkInfo
            mirrorId = nil
        end

        local response = sendRequest(primaryId, {
            id = generateRequestId(),
            action = "read",
            path = path,
            chunk = chunkNum
        }, mirrorId)

        if response and response.success then
            contentChunks[chunkNum] = response.content or ""
        else
            return nil, "Failed to read chunk " .. chunkNum
        end
    end

    -- Ensure no nil values before concat
    for i = 1, chunkCount do
        if contentChunks[i] == nil then
            return nil, "Missing content for chunk " .. i
        end
    end

    return table.concat(contentChunks)
end

local function writeFileToNodesRaid10(path, content)
    local sortedNodes = getSortedNodes()
    local nodeCount = #sortedNodes

    if nodeCount == 0 then
        return false, "No nodes available"
    end

    local chunkSize = math.ceil(#content / nodeCount)
    if chunkSize == 0 then chunkSize = 1 end

    local newChunks = {}

    for i, nodeInfo in ipairs(sortedNodes) do
        local start = (i - 1) * chunkSize + 1
        if start > #content and i > 1 then break end

        local chunk = content:sub(start, math.min(start + chunkSize - 1, #content))
        if #content == 0 and i == 1 then chunk = "" end

        local primaryId = nodeInfo.id
        local mirrorIdx = getMirrorIndex(i, nodeCount)
        local mirrorId = mirrorIdx and sortedNodes[mirrorIdx] and sortedNodes[mirrorIdx].id or nil

        -- Write to primary
        local response = sendRequest(primaryId, {
            id = generateRequestId(),
            action = "write",
            path = path,
            chunk = i,
            content = chunk
        })

        if not response or not response.success then
            return false, "Failed to write chunk " .. i .. " to primary"
        end

        -- Write to mirror
        if mirrorId and mirrorId ~= primaryId then
            sendRequest(mirrorId, {
                id = generateRequestId(),
                action = "write",
                path = path,
                chunk = i,
                content = chunk
            })
        end

        newChunks[i] = {primary = primaryId, mirror = mirrorId}

        if start + chunkSize > #content then break end
    end

    return true, newChunks
end

local function deleteChunksFromNodes(path, chunks)
    for chunkNum, chunkInfo in pairs(chunks) do
        local primaryId, mirrorId
        if type(chunkInfo) == "table" then
            primaryId = chunkInfo.primary
            mirrorId = chunkInfo.mirror
        else
            primaryId = chunkInfo
            mirrorId = nil
        end

        sendRequest(primaryId, {
            id = generateRequestId(),
            action = "delete",
            path = path,
            chunk = chunkNum
        })

        if mirrorId and mirrorId ~= primaryId then
            sendRequest(mirrorId, {
                id = generateRequestId(),
                action = "delete",
                path = path,
                chunk = chunkNum
            })
        end
    end
end

-- ============================================
-- DEFRAG LOGIC (RAID10 AWARE)
-- ============================================

local function isRaid10(chunkInfo)
    return type(chunkInfo) == "table" and chunkInfo.mirror and chunkInfo.mirror ~= chunkInfo.primary
end

local function defragFile(path)
    -- Step 1: Read original content
    local content, err = readFileFromNodes(path)
    if not content then
        return false, "Read failed: " .. (err or "unknown")
    end

    -- Step 2: Check size limit
    if #content > MAX_FILE_SIZE then
        return false, "File too large (" .. math.floor(#content/1024) .. "KB > 64KB limit)"
    end

    -- Step 3: Check current state
    local currentChunkCount = 0
    local hasRaid10 = true
    for _, chunkInfo in pairs(chunkMap[path]) do
        currentChunkCount = currentChunkCount + 1
        if not isRaid10(chunkInfo) then
            hasRaid10 = false
        end
    end

    local nodeCount = #getSortedNodes()

    -- Already optimal if: correct chunk count AND has RAID10
    if currentChunkCount == nodeCount and hasRaid10 then
        return true, "Already optimal (RAID10)"
    end

    -- Step 4: Store original chunks for rollback
    local originalChunks = {}
    for k, v in pairs(chunkMap[path]) do
        if type(v) == "table" then
            originalChunks[k] = {primary = v.primary, mirror = v.mirror}
        else
            originalChunks[k] = v
        end
    end

    -- Step 5: Write with RAID10
    local writeSuccess, newChunks = writeFileToNodesRaid10(path, content)
    if not writeSuccess then
        chunkMap[path] = originalChunks
        return false, "Write failed: " .. (newChunks or "unknown")
    end

    chunkMap[path] = newChunks

    -- Step 6: Verify
    local verifyContent, verifyErr = readFileFromNodes(path)
    if verifyContent ~= content then
        chunkMap[path] = originalChunks
        return false, "Verification failed: " .. (verifyErr or "content mismatch")
    end

    -- Step 7: Delete orphaned chunks
    local usedNodes = {}
    for _, info in pairs(newChunks) do
        if type(info) == "table" then
            usedNodes[info.primary] = true
            if info.mirror then usedNodes[info.mirror] = true end
        else
            usedNodes[info] = true
        end
    end

    for chunkNum, chunkInfo in pairs(originalChunks) do
        local oldPrimary, oldMirror
        if type(chunkInfo) == "table" then
            oldPrimary = chunkInfo.primary
            oldMirror = chunkInfo.mirror
        else
            oldPrimary = chunkInfo
            oldMirror = nil
        end

        if oldPrimary and not usedNodes[oldPrimary] then
            sendRequest(oldPrimary, {
                id = generateRequestId(),
                action = "delete",
                path = path,
                chunk = chunkNum
            })
        end
        if oldMirror and oldMirror ~= oldPrimary and not usedNodes[oldMirror] then
            sendRequest(oldMirror, {
                id = generateRequestId(),
                action = "delete",
                path = path,
                chunk = chunkNum
            })
        end
    end

    stats.filesProcessed = stats.filesProcessed + 1
    stats.bytesProcessed = stats.bytesProcessed + #content

    if not hasRaid10 then
        stats.filesUpgraded = stats.filesUpgraded + 1
        return true, "Upgraded to RAID10 (" .. nodeCount .. " nodes)"
    end

    return true, "Redistributed to " .. nodeCount .. " nodes (RAID10)"
end

local function defragAll()
    local files = {}
    for path, _ in pairs(chunkMap) do
        table.insert(files, path)
    end
    table.sort(files)

    log("Starting defrag of " .. #files .. " files...")
    print("")

    for i, path in ipairs(files) do
        local progress = math.floor((i / #files) * 100)
        term.clearLine()
        term.setCursorPos(1, select(2, term.getCursorPos()))
        write("[" .. progress .. "%] " .. path)

        local success, msg = defragFile(path)

        term.clearLine()
        term.setCursorPos(1, select(2, term.getCursorPos()))

        if success then
            if msg == "Already optimal (RAID10)" then
                -- Don't spam for already-optimal files
            else
                logSuccess("OK: " .. path .. " - " .. msg)
            end
        else
            logError(path .. " - " .. msg)
            stats.filesSkipped = stats.filesSkipped + 1
        end
    end

    print("")
end

-- ============================================
-- UI
-- ============================================

local function drawHeader()
    term.clear()
    term.setCursorPos(1, 1)
    term.setTextColor(colors.cyan)
    print("========================================")
    print("   NDisk Defrag Tool (RAID10 Edition)   ")
    print("========================================")
    term.setTextColor(colors.white)
    print("")
end

local function showStatus()
    local fileCount = 0
    local totalChunks = 0
    local needsDefrag = 0
    local needsRaid = 0

    local sortedNodes = getSortedNodes()
    local nodeCount = #sortedNodes

    for path, chunks in pairs(chunkMap) do
        fileCount = fileCount + 1
        local chunkCount = 0
        local hasRaid = true
        for _, info in pairs(chunks) do
            chunkCount = chunkCount + 1
            totalChunks = totalChunks + 1
            if not isRaid10(info) then hasRaid = false end
        end
        if chunkCount ~= nodeCount then
            needsDefrag = needsDefrag + 1
        elseif not hasRaid then
            needsRaid = needsRaid + 1
        end
    end

    print("Current Status:")
    print("  Nodes discovered: " .. nodeCount)
    print("  Files tracked:    " .. fileCount)
    print("  Total chunks:     " .. totalChunks)

    if nodeCount >= 2 then
        term.setTextColor(colors.lime)
        print("  RAID10:           Enabled (can lose " .. math.floor(nodeCount/2) .. " nodes)")
        term.setTextColor(colors.white)
    else
        term.setTextColor(colors.yellow)
        print("  RAID10:           Disabled (need 2+ nodes)")
        term.setTextColor(colors.white)
    end
    print("")

    if needsDefrag > 0 or needsRaid > 0 then
        term.setTextColor(colors.yellow)
        if needsDefrag > 0 then
            print("  Files needing redistribution: " .. needsDefrag)
        end
        if needsRaid > 0 then
            print("  Files needing RAID10 upgrade: " .. needsRaid)
        end
        term.setTextColor(colors.white)
    else
        term.setTextColor(colors.lime)
        print("  All files optimal with RAID10!")
        term.setTextColor(colors.white)
    end
    print("")
end

local function showNodes()
    print("Discovered Nodes:")
    local sortedNodes = getSortedNodes()

    for i, nodeInfo in ipairs(sortedNodes) do
        local mirrorIdx = getMirrorIndex(i, #sortedNodes)
        local mirrorId = mirrorIdx and sortedNodes[mirrorIdx] and sortedNodes[mirrorIdx].id or "N/A"
        print("  [" .. i .. "] ID: " .. nodeInfo.id .. " -> mirrors with [" .. (mirrorIdx or "N/A") .. "]")
    end
    print("")
end

local function showSummary()
    print("")
    print("========================================")
    print("           Defrag Summary")
    print("========================================")
    print("")
    print("Files processed:   " .. stats.filesProcessed)
    print("Files upgraded:    " .. stats.filesUpgraded .. " (to RAID10)")
    print("Files skipped:     " .. stats.filesSkipped)
    print("Bytes processed:   " .. math.floor(stats.bytesProcessed / 1024) .. " KB")
    print("")

    if #stats.errors > 0 then
        term.setTextColor(colors.red)
        print("Errors encountered:")
        for i, err in ipairs(stats.errors) do
            if i <= 5 then
                print("  - " .. err)
            end
        end
        if #stats.errors > 5 then
            print("  ... and " .. (#stats.errors - 5) .. " more")
        end
        term.setTextColor(colors.white)
    else
        term.setTextColor(colors.lime)
        print("No errors!")
        term.setTextColor(colors.white)
    end
    print("")
end

local function confirm(msg)
    term.setTextColor(colors.yellow)
    write(msg .. " [y/N]: ")
    term.setTextColor(colors.white)
    local input = read()
    return input:lower() == "y" or input:lower() == "yes"
end

-- ============================================
-- ORPHAN CLEANUP (SCAN ONLY - NO AUTO DELETE)
-- ============================================

local function isValidChunk(filename)
    return filename:match("_chunk_%d+$") ~= nil
end

local function cleanupOrphans()
    print("")
    log("Scanning nodes for potential orphans...")
    term.setTextColor(colors.orange)
    print("NOTE: This is SCAN ONLY - nothing will be deleted automatically")
    print("To delete orphans, manually run commands on each node")
    term.setTextColor(colors.white)
    print("")

    local sortedNodes = getSortedNodes()

    for _, nodeInfo in ipairs(sortedNodes) do
        local nodeId = nodeInfo.id

        -- Request file list from node
        local response = sendRequest(nodeId, {
            id = generateRequestId(),
            action = "listFiles"
        })

        if response and response.success and response.files then
            local orphans = {}
            local valid = 0
            for _, filename in ipairs(response.files) do
                if isValidChunk(filename) then
                    valid = valid + 1
                else
                    -- Only flag as potential orphan if it's NOT a known directory
                    -- Directories like 'students' contain chunk files inside!
                    table.insert(orphans, filename)
                end
            end

            if #orphans > 0 then
                term.setTextColor(colors.yellow)
                print("Node " .. nodeId .. ": " .. valid .. " chunks, " .. #orphans .. " non-chunk items")
                term.setTextColor(colors.white)
                for _, name in ipairs(orphans) do
                    print("  [?] " .. name .. " (may contain data - check manually)")
                end
            else
                term.setTextColor(colors.lime)
                print("Node " .. nodeId .. ": " .. valid .. " chunks (clean)")
                term.setTextColor(colors.white)
            end
        else
            logWarn("Could not scan node " .. nodeId)
        end
    end

    print("")
    term.setTextColor(colors.cyan)
    print("Scan complete. NO deletions performed.")
    print("If you need to delete specific files, do it manually on each node.")
    term.setTextColor(colors.white)
end

-- ============================================
-- MAIN
-- ============================================

local function main()
    drawHeader()

    term.setTextColor(colors.orange)
    print("WARNING: Run this ONLY when the main server is STOPPED!")
    print("This tool will modify ids.txt and chunklocation.txt.")
    term.setTextColor(colors.white)
    print("")

    if not confirm("Have you stopped the main server?") then
        print("Aborted. Please stop the server first.")
        return
    end

    print("")
    log("Loading existing chunk map...")
    loadChunkMap()

    local oldFileCount = 0
    for _ in pairs(chunkMap) do oldFileCount = oldFileCount + 1 end
    log("Found " .. oldFileCount .. " files in chunk map")

    log("Backing up chunk map...")
    backupChunkMap()

    print("")
    local nodeCount = discoverNodes()

    if nodeCount == 0 then
        logError("No storage nodes found! Make sure nodes are running.")
        return
    end

    logSuccess("Discovered " .. nodeCount .. " storage nodes")
    showNodes()

    if not confirm("Save this node list and proceed with defrag?") then
        print("Aborted.")
        return
    end

    log("Saving node list...")
    saveNodeIds()

    print("")
    showStatus()

    if not confirm("Start defragmentation (will upgrade to RAID10)?") then
        print("Node list saved. Defrag skipped.")
        return
    end

    print("")
    term.setTextColor(colors.cyan)
    print("Starting RAID10 defragmentation...")
    print("This may take a while. Do not interrupt!")
    term.setTextColor(colors.white)
    print("")

    defragAll()

    log("Saving updated chunk map...")
    saveChunkMap()

    showSummary()

    term.setTextColor(colors.lime)
    print("Defragmentation complete!")
    print("All files now have RAID10 redundancy.")
    term.setTextColor(colors.white)
    print("")

    -- Offer orphan cleanup
    if confirm("Scan nodes for orphaned files/directories?") then
        cleanupOrphans()
    end

    print("")
    term.setTextColor(colors.lime)
    print("You can now restart the main server.")
    term.setTextColor(colors.white)
end
-- Run
main()