rclone sync and check bash script 2

#!/bin/bash

# Source directories
SRC_DIRS=(
  "localsrc1:/mnt/tns-google-drive-smb"
  "localsrc2:/mnt/tns-maindatads-smb"
  "localsrc3:/mnt/tns-app-configs-smb"
  "localsrc4:/mnt/tns-proxbkps-ds-1"
)

# Destination directories for single storage units SSD and NVME
DEST_DIRS_SINGLE_STORE_SSD_NVME=(
  "localdest4:/mnt/sd25-4tb-bkp4-mnt4"
)

TEMP_UPLOAD_PATH="/home/usertemp/rclone_cache_temp/tmp_upload"
CHUNK_PATH="/home/usertemp/rclone_cache_temp/chunks"

# Transfers and checkers configuration
declare -A TRANSFERS_CONFIG
declare -A CHECKERS_CONFIG
# Populate TRANSFERS_CONFIG and CHECKERS_CONFIG for single storage units SSD and NVME
for dest in "${DEST_DIRS_SINGLE_STORE_SSD_NVME[@]}"; do
  TRANSFERS_CONFIG["$dest"]=4
  CHECKERS_CONFIG["$dest"]=4
done

# Function to get the current timestamp for logging
get_timestamp() {
  date '+%Y-%m-%d %H:%M:%S'
}

# Function to get the current timestamp for filenames
get_filename_timestamp() {
  date '+%Y%m%d_%H%M%S'
}

# Log file setup
LOG_DIR="./logs2"
mkdir -p "$LOG_DIR"
LOG_FILE="${LOG_DIR}/rclone_sync_$(get_filename_timestamp).log"

# Function to log messages with a timestamp
log() {
  local message="$1"
  echo "$(get_timestamp) - $message" >> "$LOG_FILE"
}

# Function to check files between source and destination
check_files_md5() {
  local src="$1"
  local dest="$2"

  # Generate file list for source in parallel
  log "<<<INFO>>> Generating file list for source..."
  find "$src" -type f -print0 | xargs -0 -P 4 -I {} md5sum {} > source_files.md5

  # Generate file list for destination in parallel
  log "<<<INFO>>> Generating file list for destination..."
  find "$dest" -type f -print0 | xargs -0 -P 4 -I {} md5sum {} > dest_files.md5

  # Compare the file lists
  log "<<<INFO>>> Comparing file lists..."
  diff source_files.md5 dest_files.md5 > rclone_file_diff.log

  if [ -s file_diff.log ]; then
    log "<<<ERROR>>> Differences found. Check file_diff.log for details."
  else
    log "<<<INFO>>> File lists match. No differences found."
  fi
}

# Function to check files between source and destination
check_files_rclone_check() {
  local src="$1"
  local dest="$2"
  local delay="${3:-0}"  # Optional delay parameter, defaults to 0 if not provided

  # Extract base path from destination, including everything before and including the colon
  local dest_base=$(echo "$dest" | sed -E 's/^([^:]+:\/[^\/]+\/[^\/]+).*$/\1/')
  local checkers="${CHECKERS_CONFIG[$dest_base]:-1}"
  log "<<<DEBUG>>> rclone check --checkers: $checkers"
  # local checkers=2;

  log "<<<INFO>>> Running rclone check from $src to $dest"
  {

    log "<<<DEBUG>>> 1 Rclone check sleep delay = $delay seconds."

    rclone check "$src" "$dest" \
      --checkers="$checkers" \
      --fast-list \
      --multi-thread-streams=0 \
      --buffer-size=0 \
      --one-way \
      --checksum \
      --log-file="$LOG_FILE" \
      --log-level=DEBUG \
      --retries 3 \
      --retries-sleep 3s \
      --progress

      log "<<<DEBUG>>> 2 Rclone check sleep delay = $delay seconds."

      # Optional sleep delay
      if [ -n "$delay" ] && [ "$delay" -gt 0 ]; then
        log "<<<INFO>>> Sleeping rclone check for $delay seconds..."
        sleep "$delay"
      fi

    if [ $? -eq 0 ]; then
      log "<<<INFO>>> Rclone check completed successfully for $src and $dest"
    else
      log "<<<ERROR>>> Rclone check failed for $src and $dest."
      # Uncomment the exit 1 below for fail fast debugging:
      # exit 1
    fi
  } >> "$LOG_FILE" 2>&1
}

# Function to check if directories exist
check_directories() {
  local src="$1"
  local dest="$2"

  if [ ! -d "${src#*:}" ]; then
    log "<<<ERROR>>> Source directory does not exist: ${src#*:}. Skipping."
    return 1
  fi
  return 0
}

# Function to perform rclone sync
rclone_sync() {
  local src="$1"
  local dest="$2"
  local use_checksum="${3:-false}"  # Defaults to false if not provided
  local buffer_size="$4"  # Optional
  local bwlimit="$5"  # Optional
  local cache_chunk_total_size="$6"  # Optional
  local cache_tmp_upload_path="$7"  # Optional
  local cache_chunk_path="$8"  # Optional
  local cache_info_age="$9"  # Optional
  local delay="${10:-0}"  # Optional delay parameter, defaults to 0 if not provided

  # Extract base path from destination, including everything before and including the colon
  local dest_base=$(echo "$dest" | sed -E 's/^([^:]+:\/[^\/]+\/[^\/]+).*$/\1/')
  # Debug output
  log "<<<DEBUG>>> Full destination path: ${dest#*:}"
  log "<<<DEBUG>>> Base destination directory: $dest_base"
  local transfers="${TRANSFERS_CONFIG[$dest_base]:-4}" # Defaults to default value if not provided
  local checkers="${CHECKERS_CONFIG[$dest_base]:-1}" # Defaults to default value if not provided
  log "<<<DEBUG>>> rclone sync --transfers: $transfers"
  log "<<<DEBUG>>> rclone sync --checkers: $checkers"

  # Determine if checksum should be used based on the use_checksum parameter
  local checksum_option=""
  if [ "$use_checksum" = true ]; then
    checksum_option="--checksum"
  fi

  # Optional parameters
  local buffer_size_option=""
  if [ -n "$buffer_size" ]; then
    buffer_size_option="--buffer-size=$buffer_size"
  fi

  local bwlimit_option=""
  if [ -n "$bwlimit" ]; then
    bwlimit_option="--bwlimit=$bwlimit"
  fi

  local cache_chunk_total_size_option=""
  if [ -n "$cache_chunk_total_size" ]; then
    cache_chunk_total_size_option="--cache-chunk-total-size=$cache_chunk_total_size"
  fi

  local cache_tmp_upload_path_option=""
  if [ -n "$cache_tmp_upload_path" ]; then
    cache_tmp_upload_path_option="--cache-tmp-upload-path=$cache_tmp_upload_path"
  fi

  local cache_chunk_path_option=""
  if [ -n "$cache_chunk_path" ]; then
    cache_chunk_path_option="--cache-chunk-path=$cache_chunk_path"
  fi

  local cache_info_age_option=""
  if [ -n "$cache_info_age" ]; then
    cache_info_age_option="--cache-info-age=$cache_info_age"
  fi

  log "<<<INFO>>> Syncing from $src to $dest"
  {
    rclone sync "$src" "$dest" \
      --transfers="$transfers" \
      --checkers="$checkers" \
      --log-level=DEBUG \
      --log-file="$LOG_FILE" \
      --retries 1 \
      --retries-sleep 1s \
      --copy-links \
      --progress \
      --fast-list \
      --use-mmap \
      $checksum_option \
      $buffer_size_option \
      $bwlimit_option \
      $cache_chunk_total_size_option \
      $cache_tmp_upload_path_option \
      $cache_chunk_path_option \
      $cache_info_age_option \
      --delete-during

      # Optional sleep delay
      if [ -n "$delay" ] && [ "$delay" -gt 0 ]; then
        log "<<<INFO>>> Sleeping rclone sync for $delay seconds..."
        sleep "$delay"
      fi

    if [ $? -eq 0 ]; then
      log "<<<INFO>>> Rclone sync completed successfully for $src to $dest"
    else
      log "<<<ERROR>>> Rclone sync failed for $src to $dest after 1 retry."
      # Uncomment the exit 1 below for fail fast debugging:
      # exit 1
    fi
  } >> "$LOG_FILE" 2>&1
}

# Function to sync source and destination directories to single storage units SSD and NVME
sync_sources_and_destinations_to_single_stores_ssd_nvme() {
  USE_CHECKSUM=false
  for DEST_DIR in "${DEST_DIRS_SINGLE_STORE_SSD_NVME[@]}"; do
    for SRC in "${SRC_DIRS[@]}"; do
      if check_directories "$SRC" "$DEST_DIR/$(basename "$SRC")"; then
        # Working 1 for SSD with below parameters + transfers=4 and checkers=1:
        # Working 1 for SSD with below parameters + transfers=2 and checkers=1:
        # rclone_sync "$SRC" "$DEST_DIR/$(basename "$SRC")" "$USE_CHECKSUM" "16M" "10M" "1G"
        rclone_sync "$SRC" "$DEST_DIR/$(basename "$SRC")" \
          "$USE_CHECKSUM" \
          "16M" \
          "1.5M" \
          "" \
          "" \
          "" \
          ""
      fi
    done
  done
}

# Function to check sources and destinations
check_sources_and_destinations_to_single_stores_ssd_nvme() {
  # Check source and destination directories to single storage units SSD and NVME
  for DEST_DIR in "${DEST_DIRS_SINGLE_STORE_SSD_NVME[@]}"; do
    for SRC in "${SRC_DIRS[@]}"; do
      check_files_rclone_check "$SRC" "$DEST_DIR/$(basename "$SRC")" "1"
      # Ensure that only one rclone check runs at a time
      wait
    done
  done
}

################################ Program execution start and function calling.

# Capture start time
START_TIME=$(date +%s)
START_TIMESTAMP=$(get_timestamp)

# Start logging
log "<<<INFO>>> Starting sync process at $START_TIMESTAMP"

# Calling function to sync source and destination directories to single storage units SSD and NVME
sync_sources_and_destinations_to_single_stores_ssd_nvme

# Calling function to check sources and destinations to single storage units SSD and NVME
check_sources_and_destinations_to_single_stores_ssd_nvme

# Capture end time
END_TIME=$(date +%s)
END_TIMESTAMP=$(get_timestamp)
ELAPSED_TIME=$((END_TIME - START_TIME))
ELAPSED_HOURS=$((ELAPSED_TIME / 3600))
ELAPSED_MINUTES=$(((ELAPSED_TIME % 3600) / 60))
ELAPSED_SECONDS=$((ELAPSED_TIME % 60))

# Log end time and elapsed time
log "<<<INFO>>> Sync process finished at $END_TIMESTAMP"
log "<<<INFO>>> Total time taken: ${ELAPSED_HOURS}h ${ELAPSED_MINUTES}m ${ELAPSED_SECONDS}s"