SmartOS Zombie Zone Cleanup Script

#!/bin/bash

# #############################################
# SmartOS Zombie Zone Cleanup Script
# Usage: ./nuke_zone.sh <UUID>
# 20200112
# Use as ./nuke_zone.sh ecc5d57a-3d90-44ca-a89b-8869a120487d
# #############################################

UUID=$1

# 1. Validation
if [[ -z "$UUID" ]]; then
    echo "Error: No UUID provided."
    echo "Usage: $0 <zone-uuid>"
    exit 1
fi

if [[ "$UUID" == "global" ]]; then
    echo "Error: Cannot delete the global zone."
    exit 1
fi

# Basic check to see if the UUID string format looks roughly correct (simple check)
if [[ ! "$UUID" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]]; then
    echo "Warning: Format of '$UUID' does not look like a standard UUID."
    read -p "Are you sure you want to proceed? (y/n) " -n 1 -r
    echo
    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
        exit 1
    fi
fi

echo "--- Starting cleanup for Zone: $UUID ---"

# 2. Check current status
STATE=$(zoneadm -z "$UUID" list -p 2>/dev/null | cut -d: -f3)

if [[ -z "$STATE" ]]; then
    echo "Zone currently not recognized by zoneadm. Checking ZFS leftovers..."
else
    echo "Current Zone State: $STATE"
fi

# 3. Analyze and Kill Suspicious Processes
echo "Checking for stuck processes..."
# We look for zoneadmd specifically, as it is the controller.
# We also look for qemu/bhyve instances associated with the UUID.
PIDS=$(pgrep -f "$UUID")

if [[ -n "$PIDS" ]]; then
    echo "Found the following processes associated with this UUID:"
    pgrep -lf "$UUID"

    echo "Attempting to kill these processes to release locks..."
    # We use kill -9 (SIGKILL) because if we are here, the process is likely unresponsive to SIGTERM.
    echo "$PIDS" | xargs kill -9
    sleep 2

    # Verify they are gone
    REMAINING=$(pgrep -f "$UUID")
    if [[ -n "$REMAINING" ]]; then
        echo "WARNING: Some processes refused to die: $REMAINING"
        echo "This may indicate a kernel-level hung task (zombie)."
    else
        echo "Processes cleared."
    fi
else
    echo "No locking processes found."
fi

# 4. Force state to installed/halted if possible
# If the zone is technically 'running' in the kernel, we must halt it before uninstalling.
if [[ "$STATE" == "running" || "$STATE" == "shutting_down" ]]; then
    echo "Forcing zone halt..."
    zoneadm -z "$UUID" halt
    sleep 2
fi

# 5. Force Uninstall
echo "Attempting forced uninstall (zoneadm)..."
zoneadm -z "$UUID" uninstall -F
RET=$?

if [[ $RET -eq 0 ]]; then
    echo "zoneadm uninstall reported success."
else
    echo "zoneadm uninstall reported an error (exit code $RET)."
    echo "Proceeding to ZFS cleanup anyway."
fi

# 6. ZFS Dataset Cleanup
# vmadm requires the dataset 'zones/<UUID>' to be gone to consider the VM deleted.
DATASET="zones/$UUID"
if zfs list "$DATASET" > /dev/null 2>&1; then
    echo "ZFS dataset $DATASET still exists. Destroying it..."

    # Check if there are dependent snapshots or clones (unlikely for a deletion, but possible)
    # -r recursively destroys children/snapshots
    zfs destroy -r "$DATASET"

    if [[ $? -eq 0 ]]; then
        echo "Dataset destroyed."
    else
        echo "Failed to destroy dataset. Check 'zfs list' manually."
    fi
else
    echo "ZFS dataset $DATASET is already gone."
fi

# 7. Final Verification
echo "--- Final Status Check ---"
if vmadm list | grep -q "$UUID"; then
    echo "FAILURE: $UUID still appears in 'vmadm list'."
    vmadm list | grep "$UUID"
else
    echo "SUCCESS: $UUID is no longer in 'vmadm list'."
fi

if zoneadm list -cv | grep -q "$UUID"; then
    echo "WARNING: $UUID still appears in 'zoneadm list'."
else
    echo "SUCCESS: $UUID is no longer in 'zoneadm list'."
fi

# #############################################