Restic Backup Script

#!/bin/bash
#           RESTIC BACKUP
#
#   Complete local and remote backup script via restic.
#       - Includes container orchestration via komodo
#
# Set up provided env config and place in same folder as .env


### "=====================LOGGING===========================" ###

log() {
    if [[ "$QUIET" != true ]]; then
        echo "$@"
    fi
}
vlog() {
    if [[ "$VERBOSE" == true ]]; then
        echo "$@"
    fi
}
run() {
    if [[ "$DRY_RUN" == true ]]; then
        log "[DRY-RUN] $*"
    else
        "$@"
    fi
}
# Set default values for variables
STARTUP=${STARTUP:-false}
FORCE_RESTART=${FORCE_RESTART:-false}
PARALLEL=${PARALLEL:-true}
ENV_FILE=${ENV_FILE:-./.env}

### "=====================MENU==============================" ###

while getopts ":s:e:l:r:f:pqvhdt" opt; do
    case $opt in
        h)
            echo "Usage: $0 [options]"
            echo ""
            echo "Options:"
            echo "  -e <env_file>    Path to environment file to source"
            echo "  -l <local_repo>  Path to the local Restic repository"
            echo "  -r <remote_repo> Path to the remote Restic repository"
            echo "  -s true|false    Set restart behaviour for containers"
            echo "  -f               Force shutdown of all containers before restart"
            echo "  -p               Disable parallel backups - INOPERATIVE"
            echo "  -q               Enable quiet mode (minimal output)"
            echo "  -v               Enable verbose mode (debug output)"
            echo "  -h               Show this help message"
            echo ""
            echo "Example:"
            echo "  $0 -e .env -l /mnt/backup/local -r sftp:user@host:/backup -s true -fspv"
            exit 0
            ;;
        s)
            # Set startup (true/false) for containers
            log "Startup set to '${OPTARG}'"
            if [[ "$OPTARG" == "true" ]]; then
                set -a; STARTUP=true; set +a
            elif [[ "$OPTARG" == "false" ]]; then
                set -a; STARTUP=false; set +a
            else
                echo "Invalid value for startup. Use 'true' or 'false'."
                exit 1
            fi
            ;;
        f)
            # Set force restart (true/false) for containers
            log "Force restart set to '${OPTARG}'"
            if [[ "$OPTARG" == "true" ]]; then
                set -a; FORCE_RESTART=true; set +a
            elif [[ "$OPTARG" == "false" ]]; then
                set -a; FORCE_RESTART=false; set +a
            else
                echo "Invalid value for force restart. Use 'true' or 'false'."
                exit 1
            fi
            ;;
        d)
            log "Dry run enabled"
            set -a; DRY_RUN=true; set +a
            ;;
        e)
            log "Environment file set to '${OPTARG}'"
            set -a; ENV_FILE=$OPTARG; set +a
            ;;
        l)
            log "Local backup set to '${OPTARG}'"
            set -a; LOCAL_REPOSITORY=$OPTARG; set +a
            ;;
        r)
            log "Remote backup set to '${OPTARG}'"
            set -a; REMOTE_REPOSITORY=$OPTARG; set +a
            ;;
        p)
            log "Parallel backup disabled"
            set -a; PARALLEL=false; set +a
            ;;
        q)
            set +x
            set -a; QUIET=true; set +a
            ;;
        v)
            vlog "Verbose mode enabled"
            set -x
            set -a; VERBOSE=true; set +a
            ;;
        t)
            # Test mode
            log "Test mode enabled"
            set -a; TEST=true; set +a
            ;;
        \?)
            echo "Invalid option: -$OPTARG" >&2
            exit 1
            ;;
        :)
            echo "Option -$OPTARG requires an argument." >&2
            exit 1
            ;;
    esac
done

# Shift off the options so you can process positional args
shift $((OPTIND -1))

### "=====================ENVIRONMENT=======================" ###

set -o allexport && source ${ENV_FILE} && set +o allexport

# Set default repository to remote with a local lfallback
RESTIC_REPOSITORY=${REMOTE_REPOSITORY:-$LOCAL_REPOSITORY}

test()
{
    log "Testing..."

    rm -rf ./tmp-test-bup && rm -rf ./tmp-test-repo
    mkdir ./tmp-test-bup && touch ./tmp-test-bup/testfile.txt
    mkdir ./tmp-test-repo

    set -a; RESTIC_REPOSITORY=./tmp-test-repo; set +a
    set -a; DRY_RUN=true; set +a

    if [[ -z "$RESTIC_REPOSITORY" ]]; then
        log "Error: RESTIC_REPOSITORY is not set/used."
    fi
    if [[ -z "$LOCAL_REPOSITORY" ]]; then
        log "Error: LOCAL_REPOSITORY is not set/used."
    fi
    if [[ -z "$REMOTE_REPOSITORY" ]]; then
        log "Error: REMOTE_REPOSITORY is not set/used."
    fi
    if [[ -z "$KOMODO_SERVER_NAME" ]]; then
        log "Error: KOMODO_SERVER_NAME is not set/used."
    fi
    if [[ -z "$KOMODO_SERVER_URL" ]]; then
        log "Error: KOMODO_SERVER_URL is not set/used."
    fi
    if [[ -z "$KOMODO_API_KEY" ]]; then
        log "Error: KOMODO_API_KEY is not set/used."
    fi
    if [[ -z "$KOMODO_API_SECRET" ]]; then
        log "Error: KOMODO_API_SECRET is not set/used."
    fi
    if [[ -z "$RESTIC_CACHE_DIR" ]]; then
        log "Error: RESTIC_CACHE_DIR is not set/used."
    fi
    if [[ -z "$RESTIC_EXCLUDES_FILE" ]]; then
        log "Error: RESTIC_EXCLUDES_FILE is not set/used."
    fi
    if [[ -z "$BACKUP_DIRS" ]]; then
        log "Error: BACKUP_DIRS is not set/used."
    fi
    if [[ -z "$RESTIC_PASSWORD" ]]; then
        log "Error: RESTIC_PASSWORD is not set/used."
    fi
    if [[ -z "$RESTIC_PASSWORD_FILE" ]]; then
        log "Error: RESTIC_PASSWORD_FILE is not set/used."
    fi
    if [[ -z "$RESTIC_REPOSITORY" ]]; then
        log "Error: RESTIC_REPOSITORY is not set/used."
    fi
    if [[ -z "$RESTIC_REPOSITORY_PASSWORD" ]]; then
        log "Error: RESTIC_REPOSITORY_PASSWORD is not set/used."
    fi

    run backup ./tmp-test-bup ./tmp-test-repo

    rm -rf ./tmp-test-bup && rm -rf ./tmp-test-repo

    exit 1
}

# Backup commands
backup ()
{
    log "Backing up '$1'..."

    if restic cat config -r ${2:-$RESTIC_REPOSITORY} >/dev/null 2>&1; then
        log 'Restic repository initialized'
    else
        log 'Initializing restic repository...' && run restic init -r ${2:-$RESTIC_REPOSITORY}
    fi

    run restic backup $1 -r ${2:-$RESTIC_REPOSITORY} --exclude-file=${RESTIC_EXCLUDES_FILE} --no-scan --skip-if-unchanged --ignore-inode --cache-dir=${RESTIC_CACHE_DIR}
    log "Backup of '$1' Complete!"
}

komodoShutdownAll()
{
    # Check if the shutdown is already in progress
    SHUTDOWN_ACTIVE=$(curl --silent \
        --header "Content-Type: application/json" \
        --header "X-Api-Key: ${KOMODO_API_KEY}" \
        --header "X-Api-Secret: ${KOMODO_API_SECRET}" \
        --data "$(jq -n --arg server "$KOMODO_SERVER_NAME" '{type: "GetServerActionState", params: {server: $server}}')" \
        ${KOMODO_SERVER_URL}/read | jq -r '.stopping_containers')
    if [ "$SHUTDOWN_ACTIVE" == "true" ]; then
        vlog "Shutdown already in progress, skipping..."
        return
    elif [ "$DRY_RUN" == true ]; then
        log "[DRY-RUN] Shutdown command would be sent to komodo."
        return
    fi

    log "Stopping containers..."
    # Execute docker shutdown command via komodo
    curl --silent \
    --header "Content-Type: application/json" \
    --header "X-Api-Key: ${KOMODO_API_KEY}" \
    --header "X-Api-Secret: ${KOMODO_API_SECRET}" \
    --data "$(jq -n --arg server "$KOMODO_SERVER_NAME" '{type: "StopAllContainers", params: {server: $server}}')" \
    "${KOMODO_SERVER_URL}/execute" > /dev/null

    if [ $? -ne 0 ]; then
        vlog "Error: Failed to send shutdown command."
        exit 1
    fi

    # Wait for the shutdown to complete
    # Max time (seconds) is sleep time * retry limit
    local RETRY_COUNT=0
    local RETRY_LIMIT=20
    local SLEEP_TIME=3
    while :; do
        # Check if the shutdown was successful
        SHUTDOWN_ACTIVE=$(curl --silent \
            --header "Content-Type: application/json" \
            --header "X-Api-Key: ${KOMODO_API_KEY}" \
            --header "X-Api-Secret: ${KOMODO_API_SECRET}" \
            --data "$(jq -n --arg server "$KOMODO_SERVER_NAME" '{type: "GetServerActionState", params: {server: $server}}')" \
            ${KOMODO_SERVER_URL}/read | jq -r '.stopping_containers')

        # Check if the shutdown is still active
        if [ "$SHUTDOWN_ACTIVE" == "true" ] && [ $RETRY_COUNT != $RETRY_LIMIT ]; then
            vlog "Containers are stopping..."
            sleep $SLEEP_TIME
            ((RETRY_COUNT++))
        elif [ "$SHUTDOWN_ACTIVE" == "false" ]; then
            vlog "Container shutdown complete."
            break
        else
            vlog "Error: Unable to determine shutdown status."
            exit 1
        fi
    done
    log "All containers shutdown"
}

komodoStartAll()
{
    # Check if the startup is already in progress
    CONTAINERS_STARTING=$(curl --silent \
        --header "Content-Type: application/json" \
        --header "X-Api-Key: ${KOMODO_API_KEY}" \
        --header "X-Api-Secret: ${KOMODO_API_SECRET}" \
        --data "$(jq -n --arg server "$KOMODO_SERVER_NAME" '{type: "GetServerActionState", params: {server: $server}}')" \
        ${KOMODO_SERVER_URL}/read | jq -r '.starting_containers')
    if [ "$CONTAINERS_STARTING" == "true" ]; then
        vlog "Startup already in progress, skipping..."
        return
    elif [ "$DRY_RUN" == true ]; then
        log "[DRY-RUN] Startup command would be sent to komodo."
        return
    fi

    log "Restarting containers..."
    # Execute docker shutdown command via komodo
    curl --silent \
    --header "Content-Type: application/json" \
    --header "X-Api-Key: ${KOMODO_API_KEY}" \
    --header "X-Api-Secret: ${KOMODO_API_SECRET}" \
    --data "$(jq -n --arg server "$KOMODO_SERVER_NAME" '{type: "StartAllContainers", params: {server: $server}}')" \
    "${KOMODO_SERVER_URL}/execute" > /dev/null

    if [ $? -ne 0 ]; then
        vlog "Error: Failed to send startup command."
        exit 1
    fi

    # Wait for the startup to complete
    # Max time (seconds) is sleep time * retry limit
    local RETRY_COUNT=0
    local RETRY_LIMIT=5
    local SLEEP_TIME=3
    while :; do
        # Check if the shutdown was successful
        CONTAINERS_STARTING=$(curl --silent \
            --header "Content-Type: application/json" \
            --header "X-Api-Key: ${KOMODO_API_KEY}" \
            --header "X-Api-Secret: ${KOMODO_API_SECRET}" \
            --data "$(jq -n --arg server "$KOMODO_SERVER_NAME" '{type: "GetServerActionState", params: {server: $server}}')" \
            ${KOMODO_SERVER_URL}/read | jq -r '.starting_containers')

        # Check if the shutdown is still active
        if [ "$CONTAINERS_STARTING" == "true" ] && [ $RETRY_COUNT != $RETRY_LIMIT ]; then
            vlog "Containers are starting..."
            sleep $SLEEP_TIME
            ((RETRY_COUNT++))
        elif [ "$CONTAINERS_STARTING" == "false" ]; then
            vlog "Startup complete."
            break
        else
            vlog "Error: Unable to determine startup status."
            exit 1
        fi
    done
    log "All containers live !"
}

dumpDB()
{
  local common_dbs=("mysql" "mariadb" "postgres")
  local containers
  containers=$(docker ps -a --format "{{.ID}} {{.Image}} {{.Names}}")

  while read -r id image name; do
    for db in "${common_dbs[@]}"; do
      if [[ "$image" == *"$db"* ]]; then
        log "🔍 Detected DB container '$name' using image '$image'"

        # Extract env vars from the container using docker inspect
        env_output=$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$id")

        # Parse common DB environment variables
        MYSQL_ROOT_PASSWORD=$(echo "$env_output" | grep -E '^MYSQL_ROOT_PASSWORD=' | cut -d= -f2-)
        MYSQL_USER=$(echo "$env_output" | grep -E '^MYSQL_USER=' | cut -d= -f2-)
        MYSQL_PASSWORD=$(echo "$env_output" | grep -E '^MYSQL_PASSWORD=' | cut -d= -f2-)

        POSTGRES_USER=$(echo "$env_output" | grep -E '^POSTGRES_USER=' | cut -d= -f2-)
        POSTGRES_PASSWORD=$(echo "$env_output" | grep -E '^POSTGRES_PASSWORD=' | cut -d= -f2-)

        dump_file="${DB_DUMP_DIR}/dump-${name}-$(date +%F_%H-%M-%S).sql"

        if [[ $DRY_RUN == true ]]; then
          log "[DRY-RUN] Dumping DB from '$name' to '$dump_file'"
          continue
        fi

        case $db in
          mysql | mariadb)
            log "📤 Dumping MySQL/MariaDB from '$name'..."
            docker exec "$id" sh -c "mysqldump -u${MYSQL_USER:-root} -p'${MYSQL_PASSWORD:-$MYSQL_ROOT_PASSWORD}' --all-databases" > "$dump_file"
            ;;
          postgres)
            log "📤 Dumping PostgreSQL from '$name'..."
            docker exec "$id" sh -c "pg_dumpall -U '${POSTGRES_USER:-postgres}'" > "$dump_file"
            ;;
        esac

        log "✅ Dump completed: $dump_file"
        break
      fi
    done
  done <<< "$containers"
}

startBackups()
{
    # Parse ASYNC_BACKUP_DIRS into array
    eval "entries=(${BACKUP_DIRS})"

    # Associative array for key-path mapping
    declare -A path_map

    rm -f ./tmp-vlog.log
    verbose_log="./.tmp.vlog"
    # Stop any containers matching the provided key and backup respective dir
    for entry in "${entries[@]}"; do
        if [[ "$entry" == *:* && "$entry" =~ ^[^/]+: ]]; then
            # Entry with key:path format
            key="${entry%%:*}"
            path="${entry#*:}"

            # Adding a key with a matching tag in the name (eg: 'immich' matches all official containers spawned) ensures data integrity
            log "Stopping containers with matching name '${key}'..."
            run docker stop $(docker ps -a --filter name=${key} --format '{{ .Names }}') > $verbose_log 2>&1

            vlog ${verbose_log}
            rm -f ./tmp.vlog
            touch ./.tmp.vlog

            backup ${path} $1
        else
            # Keyless path — auto-generate a key
            key="path_$counter"
            path="$entry"

            log "Stopping containers with matching path '${path}'..."

            entry=${path%/}
            # Check if container mounts entry directory and stops if required
            for cid in $(docker ps -aq); do
                if docker inspect --format '{{ range .Mounts }}{{ .Source }} {{ end }}' "$cid" \
                | grep -qE "^$entry(/|$)"; then
                    run docker stop "$cid" > $verbose_log 2>&1

                    vlog ${verbose_log}
                    rm -f ./tmp.vlog
                    touch ./.tmp.vlog
                fi
            done

            backup ${path} $1
        fi
        ((counter++))
        path_map["$key"]="$path"
    done
    rm -f ./tmp.vlog
}

if [[ "$TEST" == true ]]; then
    log "=====================TESTING==========================="
    test
    log "======================================================="
    log "Test complete, exiting..."
    exit 0
fi


##### BACKUP START #####


log "=====================REPOSITORIES==========================="
log "Local repository: '${LOCAL_REPOSITORY}'"
log "Remote repository: '${REMOTE_REPOSITORY}'"
log "============================================================"


sleep 3


log "=====================BACKUP START==========================="
# Default to remote repo
RESTIC_REPOSITORY=${REMOTE_REPOSITORY:-$LOCAL_REPOSITORY}

log "======================DATABASE DUMP============================"
dumpDB
backup ${DB_DUMP_DIR}

rm -f ./tmp-log.txt > /dev/null 2>&1
touch ./tmp-log.txt
logfile="./.tmp.log"

# Check if we are running in parallel mode
if [[ "$PARALLEL" == true && -n "$LOCAL_REPOSITORY" && -n "$REMOTE_REPOSITORY" ]]; then

    startBackups ${LOCAL_REPOSITORY} > "${logfile}" 2>&1 &
    local_backup=$!
    log "INFO: Background process started for local backup"

    log "=====================REMOTE================================="
    startBackups ${REMOTE_REPOSITORY}
    log "============================================================"

    wait $local_backup

    log "======================LOCAL================================="
    log cat $logfile
    log "============================================================"
else
    startBackups
fi

log "======================BACKUP COMPLETE=========================="

log "======================CLEANUP==============================="
log "Cleaning up..."
run restic cache --cleanup > /dev/null 2>&1
rm -f $logfile
log "============================================================"

# Check if we are skipping auto-restart for this run
if [[ $STARTUP == "true" ]]; then
    log "======================KOMODO RESTART========================"

    if [[ $FORCE_RESTART == "true" ]]; then
        log "Force restart enabled, stopping all containers first..."
        komodoShutdownAll
    fi

    komodoStartAll
    log "============================================================"
fi

#
## CZ ##