sync-script

# Ensure rclone is in PATH
export PATH=$PATH:/usr/local/bin

# Sync script for remote media files with proper tracking
LOGDIR="/var/log/rclone"
REMOTE="seedbox"
REMOTE_BASE="/seedbox/torrents/downloads/completed"
LOCAL_BASE="/mnt/data/media/sync"
NTFY_URL="https://ntfy.mydomain.com/seedbox_sync"

# Download tracking file
TRACK_DIR="/var/lib/rclone"
TRACK_FILE="$TRACK_DIR/synced-files.txt"

mkdir -p "$LOGDIR"
mkdir -p "$LOCAL_BASE"
mkdir -p "$TRACK_DIR"
touch "$TRACK_FILE"

# Function to send notifications (keeping your existing one)
send_notification() {
    local title="$1"
    local message="$2"
    local priority="${3:-default}"
    local tags="${4:-}"

    curl -k -H "Title: $title" \
         -H "Priority: $priority" \
         -H "Tags: $tags" \
         -d "$(printf "$message")" \
         "$NTFY_URL"
}

echo "======================================"
echo "Starting Media Sync"
echo "$(date '+%Y-%m-%d %H:%M:%S')"
echo "======================================"

send_notification "Sync Started" "Beginning sync from remote server..." "low" "arrow_forward"

# Get list of all remote files (excluding unwanted patterns)
echo "Checking remote files..."
REMOTE_FILES=$(mktemp)
rclone ls "${REMOTE}:${REMOTE_BASE}" \
    --exclude ".stfolder/**" \
    --exclude "*.dat" \
    --exclude "Thumbs.db" \
    --exclude ".DS_Store" \
    --exclude "*~uTorrentPartFile*" \
    2>/dev/null | awk '{$1=""; print substr($0,2)}' | sort > "$REMOTE_FILES"

# Find files that haven't been downloaded before
NEW_TO_DOWNLOAD=$(mktemp)
comm -23 "$REMOTE_FILES" <(sort "$TRACK_FILE") > "$NEW_TO_DOWNLOAD"
NEW_COUNT=$(wc -l < "$NEW_TO_DOWNLOAD")

if [ $NEW_COUNT -eq 0 ]; then
    echo "No new files to download"
    send_notification "Sync Complete" "No new files to download" "low" "heavy_check_mark"
else
    echo "Found $NEW_COUNT new files to download"

    # Create include file for rclone
    INCLUDE_FILE=$(mktemp)
    cp "$NEW_TO_DOWNLOAD" "$INCLUDE_FILE"

    # Download with detailed logging to capture what actually transfers
    echo "Downloading new files..."
    TRANSFER_LOG=$(mktemp)

    rclone copy \
        "${REMOTE}:${REMOTE_BASE}" \
        "${LOCAL_BASE}" \
        --files-from-raw "$INCLUDE_FILE" \
        --transfers 10 \
        --checkers 16 \
        --buffer-size 32M \
        --drive-chunk-size 64M \
        --contimeout 60s \
        --timeout 300s \
        --retries 3 \
        --low-level-retries 10 \
        --stats 30s \
        --log-file="${LOGDIR}/sync.log" \
        --log-level INFO \
        --progress 2>&1 | tee "$TRANSFER_LOG"

    SYNC_STATUS=$?

    if [ $SYNC_STATUS -eq 0 ]; then
        # Simply check what new files are in the sync folder
        ACTUALLY_TRANSFERRED=$(mktemp)
        find "$LOCAL_BASE" -type f 2>/dev/null | \
            sed "s|${LOCAL_BASE}/||" | sort | \
            comm -23 - <(sort "$TRACK_FILE") > "$ACTUALLY_TRANSFERRED"

        TRANSFERRED_COUNT=$(wc -l < "$ACTUALLY_TRANSFERRED")

        if [ $TRANSFERRED_COUNT -gt 0 ]; then
            echo "$(date '+%Y-%m-%d %H:%M:%S') - Successfully synced $TRANSFERRED_COUNT files"

            # Add to tracking file
            cat "$ACTUALLY_TRANSFERRED" >> "$TRACK_FILE"
            sort -u "$TRACK_FILE" -o "$TRACK_FILE"

            # Simple notification
            send_notification "Sync Complete" "Transferred $TRANSFERRED_COUNT files" "default" "heavy_check_mark"
        else
            echo "$(date '+%Y-%m-%d %H:%M:%S') - Sync completed but no new files found"
            send_notification "Sync Complete" "No new files transferred" "low" "heavy_check_mark"
        fi

        rm -f "$ACTUALLY_TRANSFERRED"
    else
        echo "$(date '+%Y-%m-%d %H:%M:%S') - Sync failed"
        send_notification "Sync Failed" "Check logs for details" "high" "warning"
    fi

    # Cleanup
    rm -f "$INCLUDE_FILE" "$TRANSFER_LOG"
fi

# Cleanup
rm -f "$REMOTE_FILES" "$NEW_TO_DOWNLOAD"

echo "======================================"
echo "Total files tracked: $(wc -l < "$TRACK_FILE")"
echo "$(date '+%Y-%m-%d %H:%M:%S') - Sync complete"
echo "======================================"