#Purpose: Gracefully drain selected VMware proxies, stop Veeam services for patching/reboot, then return them to #production.
#Requirements
# -Veeam Component: Veeam Backup & Replication 12.3.2 (Server & Console)
# -Permissions:
# Backup Administrator role in VBR
# Local Administrator on each proxy (for remote Invoke-Command)
# MFA: Disabled for the account launching PowerShell (MFA is not supported for Veeam PowerShell sessions ‑ KB4535)
# PowerShell 5.1 on the jump host
#Further Prerequisites for Success:

#Proxies 'F-1' and 'M-1' must exist in VBR configuration (if mismatch you will receive error but the script can be modified to accomodate

#WinRM enabled on both proxy servers

#Account has Backup Administrator role in VBR

#Account has local administrator rights on proxy servers

#PowerShell execution policy allows script execution

#Network connectivity between script host and proxies



<#
Veeam Confidential - Internal Use Only
Validated against Veeam Backup & Replication 12.3.2 (build 12.0.3.x) on 2025-07-24

Purpose  :  Gracefully drain selected VMware proxies, stop Veeam services for patching/reboot, then return them to production.
Usage    : powershell.exe -ExecutionPolicy Bypass -File ProxyMaintenance.ps1 -Stage Pre|Post
ExitCodes: 0  = Success
           10 = Proxy objects not found
           20 = Disable failed          (Pre)
           30 = Task-drain timeout      (Pre)
           40 = Stop-service failure    (Pre)
           50 = Start-service failure   (Post)
           60 = Re-enable failure       (Post)
           90 = Invalid Stage argument
           99 = Unhandled error
NOTED Requirements:

Proxies 'F-1' and 'M-1' must exist in VBR configuration (if mismatch you will receive error but the script can be modified to accomodate

WinRM enabled on both proxy servers

Account has Backup Administrator role in VBR

Account has local administrator rights on proxy servers

PowerShell execution policy allows script execution (PowerShell 5.1 on jump ost)

MFA: Disabled for the account launching PowerShell (MFA is not supported for Veeam PowerShell sessions ‑ KB4535)

Network connectivity between script host and proxies
--
Troubleshooting:

Task Drain Delay:
Use Suspend-VBRJob to pause jobs feeding these proxies.
Check Instant Recovery sessions — they may hold proxy resources.
Service Stop Failures

Misc:
Run Get-Service Veeam* manually to confirm service names.
Ensure no orphaned TCP connections (Get-NetTCPConnection).
#>


param(
    [ValidateSet('Pre','Post')]
    [string]$Stage = 'Pre'
)

Write-Output "Process begins (Stage: $Stage)..."

$ProxyBatch           = @('F-1','M-1')      # proxies under maintenance
$PollDelay            = 30                  # seconds between task-drain checks
$DrainTimeoutMinutes  = 30                  # max wait time for drain

Import-Module Veeam.Backup.PowerShell -ErrorAction Stop

try {
    #------------------------------------------------------------
    # Common: get proxy objects
    #------------------------------------------------------------
    $ProxyObjs = Get-VBRViProxy -Name $ProxyBatch
    if (-not $ProxyObjs) {
        Write-Error "No matching proxies found!"
        exit 10
    }

    #------------------------------------------------------------
    if ($Stage -eq 'Pre') {
    #------------------------------------------------------------
        # Disable proxies
        try { $ProxyObjs | Disable-VBRViProxy }
        catch { Write-Error "Failed to disable proxies: $_"; exit 20 }

        # Drain running tasks
        Write-Host ">>> Waiting for active tasks to drain…" -ForegroundColor Cyan
        $StartTime = Get-Date
        do {
            $Busy = Get-VBRTaskSession | Where-Object {
                $_.IsWorking -and ($ProxyBatch -contains $_.ProxyName)
            }
            if ($Busy) {
                if ((Get-Date) - $StartTime -gt (New-TimeSpan -Minutes $DrainTimeoutMinutes)) {
                    Write-Error "Task drain timeout after $DrainTimeoutMinutes minutes"
                    exit 30
                }
                Write-Host (" {0} task(s) still running – sleeping {1}s" -f $Busy.Count, $PollDelay)
                Start-Sleep -Seconds $PollDelay
            }
        } until (-not $Busy)
        Write-Host ">>> No active tasks – stopping services." -ForegroundColor Green

        # Stop services on each proxy
        foreach ($Node in $ProxyBatch) {
            try {
                Write-Host " Stopping Veeam services on $Node …"
                Invoke-Command -ComputerName $Node -ScriptBlock {
                    Get-Service Veeam* | Stop-Service -Force
                }
            } catch {
                Write-Error "Failed to stop Veeam services on $Node: $_"
                exit 40
            }
        }

        Write-Output "Stage Pre completed successfully"
        exit 0
    }

    #------------------------------------------------------------
    elseif ($Stage -eq 'Post') {
    #------------------------------------------------------------
        # Start services on each proxy
        foreach ($Node in $ProxyBatch) {
            try {
                Write-Host " Starting Veeam services on $Node …"
                Invoke-Command -ComputerName $Node -ScriptBlock {
                    Get-Service Veeam* | Start-Service
                }
            } catch {
                Write-Error "Failed to start Veeam services on $Node: $_"
                exit 50
            }
        }

        # Re-enable proxies
        try { $ProxyObjs | Enable-VBRViProxy }
        catch { Write-Error "Failed to re-enable proxies: $_"; exit 60 }

        Write-Output "Stage Post completed successfully"
        exit 0
    }

    #------------------------------------------------------------
    else {
        Write-Error "Invalid -Stage argument. Use Pre or Post."
        exit 90
    }
}
catch {
    Write-Error "Unhandled error: $_"
    exit 99
}