For MikroTik forum

#!/bin/bash

# Function call tree
#    +
#    |
#    +-- initialise
#    |
#    +-- ck_all_routers
#    |   |
#    +   +-- ck_one_router
#    |
#    +-- finalise
#
# Utility functions called from various places:
#    ck_file ck_uint fct msg run_cmd_with_timeout

#--------------------------
# Name: ck_all_routers
# Purpose: checks all routers
#--------------------------
function ck_all_routers {
    fct "${FUNCNAME[0]}" 'started'
    local i

    for ((i=0;i<${#mikrotik_routers[*]};i++))
    do
        ck_one_router "${mikrotik_routers[i]}"
    done

    fct "${FUNCNAME[0]}" 'returning'
    return 0
}  # end of function ck_all_routers

source /usr/local/lib/bash/ck_file.fun || exit 1

#--------------------------
# Name: ck_one_router
# Purpose: checks one router
#--------------------------
function ck_one_router {
    fct "${FUNCNAME[0]}" "started. Args: $*"
    local array buf cmd installed_ver latest_ver msg msg_part oldIFS
    local days_cmd_fail_before_report rc router_fqdn router_user ssh_key ssh_opts
    local -r check_OK_regex='status: (New version is available|System is already up to date)'
    local -r up_to_date_regex='status: System is already up to date'
    local -r whitespace_or_empty_regex='^[[:space:]]*$'

    # Parse the conf string
    # ~~~~~~~~~~~~~~~~~~~~~
    oldIFS=$IFS
    IFS=:
    array=($1)
    IFS=$oldIFS
    router_fqdn=${array[0]}
    router_user=${array[1]}
    ssh_key=${array[2]}
    days_cmd_fail_before_report=${array[3]}

    # Derive value
    # ~~~~~~~~~~~~
    cmd_OK_timestamp_fn=$var_lib_dir/$router_fqdn

    # Get update status
    # ~~~~~~~~~~~~~~~~~
    # ssh's ConnectTimeout must be bigger than run_cmd_with_timeout's -t
    ssh_opts="-i $ssh_key"
    ssh_opts+=' -o ConnectTimeout=15'
    ssh_opts+=' -o PasswordAuthentication=no -o StrictHostKeyChecking=no'
    ssh_opts+=' -o UserKnownHostsFile=/dev/null'
    cmd=(
        ssh $ssh_opts -o LogLevel=quiet "$router_user@$router_fqdn"
            system package update check-for-updates
    )
    out_fn=$tmp_dir/out
    rc_fn=$tmp_dir/rc
    run_cmd_with_timeout -t 10 -w '>0'
    rc=$?
    case $rc in
        0 ) ;;
        2 )
            msg I "Timed out running ssh command on $router_fqdn"
            ;&
        1 )
            [[ ! -f "$cmd_OK_timestamp_fn" ]] && touch "$cmd_OK_timestamp_fn"
            cmd=(find "$cmd_OK_timestamp_fn"
                -mtime +"$days_cmd_fail_before_report"
            )
            buf=$("${cmd[@]}" 2>&1)
            rc=$?
            if ((rc==0)); then
                if [[ $buf != '' ]]; then
                    msg="More than $days_cmd_fail_before_report days since"
                    msg+=" last ssh command on $router_fqdn set return code 0"
                    msg+=$'\nThe last successful command was:'
                    msg+=$'\n'$(tail -1 "$cmd_OK_timestamp_fn")
                    msg W "$msg"
                fi
            else
                msg="Command: ${cmd[*]}"
                msg+=$'\n'"Return code: $rc"
                msg+=$'\n'"Output: $buf"
                msg W "$msg"
            fi
            msg I 'Again with more logging for diagnostics ...`'
            cmd=(
                ssh $ssh_opts "$router_user@$router_fqdn"
                    system package update check-for-updates
            )
            run_cmd_with_timeout -t 10 -w '>0'
            if [[ ${msg:-} != '' ]]; then
                final_msg+=$'\n\n'$msg
            fi

            fct "${FUNCNAME[0]}" 'returning 1'
            return 1
    esac
    buf=$(< "$out_fn")
    msg I "$router_fqdn:"$'\n'"$buf"

    # Note time of successfully running command on this router
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    echo "$(date '+%a %-d %b %Y %H:%M:%S') Ran command on $router_fqdn" >> "$cmd_OK_timestamp_fn"

    # Parse update status
    # ~~~~~~~~~~~~~~~~~~~
    # Ignoring the output which normally ends with
    # "finding out latest version...":
    # Example output:
    #            channel: stable
    #  installed-version: 6.43.8
    #     latest-version: 6.43.8
    #             status: System is already up to date
    # Example output:
    #            channel: stable
    #  installed-version: 6.43.7
    #     latest-version: 6.43.8
    #             status: New version is available
    # Example output (bug; installed > latest but "New version is available")
    #            channel: long-term
    #  installed-version: 6.43.7
    #     latest-version: 6.42.10
    #             status: New version is available
    # Example output (no installed-version, only current-version)
    #            channel: long-term
    #    current-version: 6.42.11
    #     latest-version: 6.42.11
    #             status: System is already up to date
    current_or_installed_ver=$(echo "$buf" \
        | grep -E '(current|installed)-version:' \
        | tail -1 \
        | sed --regexp-extended 's/.*(current|installed)-version: //'
    )
    if [[ $current_or_installed_ver =~ $whitespace_or_empty_regex ]]; then
        msg='Failed to get current or installed version from output'
        msg+=$'\n'"Command: ${cmd[*]}"
        msg+=$'\n'"Output: $buf"
        msg W "$msg"
        fct "${FUNCNAME[0]}" 'returning 1'
        return 1
    fi
    msg D "current_or_installed_ver: $current_or_installed_ver"
    IFS=.
    array=($current_or_installed_ver)
    IFS=$oldIFS
    installed_major=${array[0]}
    installed_minor=${array[1]}
    installed_patch=${array[2]%%[[:space:]]}
    msg D "installed_major: '$installed_major'"
    msg D "installed_minor: '$installed_minor'"
    msg D "installed_patch: '$installed_patch'"
    msg=
    ! ck_uint $installed_major \
        && msg+=$'\n'"installed_major not uint: '$installed_major'"
    ! ck_uint $installed_minor \
        && msg+=$'\n'"installed_minor not uint: '$installed_minor'"
    ! ck_uint $installed_patch \
        && msg+=$'\n'"installed_patch not uint: '$installed_patch'"

    latest_ver=$(echo "$buf" \
        | grep 'latest-version:' \
        | sed 's/.*latest-version: //'
    )
    if [[ $latest_ver =~ $whitespace_or_empty_regex ]]; then
        msg='Failed to get latest-version from output'
        msg+=$'\n'"Command: ${cmd[*]}"
        msg+=$'\n'"Output: $buf"
        msg W "$msg"
        fct "${FUNCNAME[0]}" 'returning 1'
        return 1
    fi
    msg D "latest_ver: $latest_ver"
    IFS=.
    array=($latest_ver)
    IFS=$oldIFS
    latest_major=${array[0]}
    latest_minor=${array[1]}
    latest_patch=${array[2]%%[[:space:]]}
    msg D "latest_major: '$latest_major'"
    msg D "latest_minor: '$latest_minor'"
    msg D "latest_patch: '$latest_patch'"
    ! ck_uint $latest_major \
        && msg+=$'\n'"latest_major not uint: '$latest_major'"
    ! ck_uint $latest_minor \
        && msg+=$'\n'"latest_minor not uint: '$latest_minor'"
    ! ck_uint $latest_patch \
        && msg+=$'\n'"latest_patch not uint: '$latest_patch'"
    if [[ $msg != '' ]]; then
        msg W "update status output parse error:$msg"
        fct "${FUNCNAME[0]}" 'returning 1'
        return 1
    fi

    # Update required?
    # ~~~~~~~~~~~~~~~~
    msg_part=$'\n'"Current or installed version: $current_or_installed_ver"
    msg_part+=$'\n'"Latest version: $latest_ver"
    msg=
    if ((latest_major>installed_major)); then
        msg="$router_fqdn: manual update required"
        msg+=" (version major number has changed)"
    elif ((latest_major==installed_major)); then
        if ((latest_minor>installed_minor)); then
            msg="$router_fqdn: update required"
            msg+=" (version minor number has changed)"
        elif ((latest_minor==installed_minor)); then
            if ((latest_patch>installed_patch)); then
                msg="$router_fqdn: update required"
                msg+=" (patch number has changed)"
            fi
        fi
    fi
    if [[ $msg = '' ]]; then
        msg I "$router_fqdn: update not required$msg_part"
    else
        final_msg+=$'\n\n'$msg$msg_part
    fi

    fct "${FUNCNAME[0]}" 'returning'
    return 0
}  # end of function ck_one_router

#--------------------------
# Name: ck_uint
# Purpose: checks for a valid unsigned integer
# Usage: ck_uint <putative uint>
# Outputs: none
# Returns:
#   0 when $1 is a valid unsigned integer
#   1 otherwise
#--------------------------
function ck_uint {
    local regex='^[[:digit:]]+$'
    [[ $1 =~ $regex ]] && return 0 || return 1
}  #  end of function ck_uint

#--------------------------
# Name: fct
# Purpose: function call trace (for debugging)
# $1 - name of calling function
# $2 - message.  If it starts with "started" or "returning" then the output is prettily indented
#--------------------------
function fct {

    if [[ ! $debugging_flag ]]; then
        return 0
    fi

    fct_indent="${fct_indent:=}"

    case $2 in
        'started'* )
            fct_indent="$fct_indent  "
            msg D "$fct_indent$1: $2"
            ;;
        'returning'* )
            msg D "$fct_indent$1: $2"
            fct_indent="${fct_indent#  }"
            ;;
        * )
            msg D "$fct_indent$1: $2"
    esac

}  # end of function fct

#--------------------------
# Name: finalise
# Purpose: cleans up and exits
# Arguments:
#    $1  return value
# Return code (on exit):
#   The sum of zero plus
#      1 if any warnings
#      2 if any errors
#      4,8,16 unused
#      32 if terminated by a signal
#--------------------------
function finalise {
    fct "${FUNCNAME[0]}" "started with args $*"
    local buf cmd msg_part my_retval subject

    finalising_flag=$true

    # Email any message about updates being available
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    if [[ $final_msg != '' ]]; then
        msg I "Mailing report to $email_for_report"
        msg_part+=$'\n\n'
        cmd=(mailx -s "MikroTik routers needing update"
            "$email_for_report"
        )
        buf=$(echo "$msg_part$final_msg" | "${cmd[@]}" 2>&1)
        [[ $buf != '' ]] && msg E "Unexpected output from ${cmd[*]}: $buf"
    fi

    # Remove old logs
    # ~~~~~~~~~~~~~~~
    if [[ ${log_dir:-} != '' ]]; then
        msg I 'Removing old logs'
        buf=$(
            find "$log_dir" \
                -maxdepth 1 \
                -regextype posix-egrep \
                -regex ".*/$log_fn_pat" \
                -mtime +7 \
                -delete \
                2>&1
        )
        [[ $buf != '' ]] && msg W "Problem removing old logs: $buf"
    fi

    # Interrupted?  Message and exit return value
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    my_retval=0
    if ck_uint "${1:-}" && (($1>128)); then
        ((my_retval+=32))
        case $1 in
            129 )
                buf=SIGHUP
                ;;
            130 )
                buf=SIGINT
                ;;
            131 )
                buf=SIGQUIT
                ;;
            132 )
                buf=SIGILL
                ;;
            134 )
                buf=SIGABRT
                ;;
            135 )
                buf=SIGBUS
                ;;
            136 )
                buf=SIGFPE
                ;;
            138 )
                buf=SIGUSR1
                ;;
            139 )
                buf=SIGSEGV
                ;;
            140 )
                buf=SIGUSR2
                ;;
            141 )
                buf=SIGPIPE
                ;;
            142 )
                buf=SIGALRM
                ;;
            143 )
                buf=SIGTERM
                ;;
            145 )
                buf=SIGCHLD
                ;;
            146 )
                buf=SIGCONT
                ;;
            147 )
                buf=SIGSTOP
                ;;
            148 )
                buf=SIGTSTP
                ;;
            149 )
                buf=SIGTTIN
                ;;
            150 )
                buf=SIGTTOU
                ;;
            151 )
                buf=SIGURG
                ;;
            152 )
                buf=SIGCPU
                ;;
            153 )
                buf=SIGXFSZ
                ;;
            154 )
                buf=SIGVTALRM
                ;;
            155 )
                buf=SIGPROF
                ;;
            * )
                msg E "${FUNCNAME[0]}: programming error: \$1 ($1) not serviced"
                ;;
        esac
        interrupt_flag=$true
        msg="Finalising on $buf"
        msg E "$msg"    # Returns because finalising_flag is set
    fi

    # Temporary directory removal
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~
    local -r tmp_dir_regex="^/tmp/$my_name\+${cfg_fn##*/}\..{6}$"
    if [[ $tmp_dir_created_flag \
        && ${tmp_dir:-} =~ $tmp_dir_regex \
    ]]; then
        msg I "Removing temporary directory $tmp_dir"
        rm -fr "$tmp_dir"
    fi

    # Exit return value adjustment
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    if [[ $warning_flag ]]; then
        msg I "There was at least one WARNING"
        ((my_retval+=1))
    fi
    if [[ $error_flag ]]; then
        msg I "There was at least one ERROR"
        ((my_retval+=2))
    fi
    [[ $interrupt_flag ]] && msg I 'There was at least one interrupt'
    if ((my_retval==0)) && ((${1:-0}!=0)); then
        msg E 'There was an error not reported in detail (probably by ... || finalise 1)'
        my_retval=2
    fi
    msg I "Exiting with return value $my_retval"

    # Mail log
    # ~~~~~~~~
    if [[ -f "$log_fn" ]]; then
        subject=
        [[ $warning_flag ]] && subject=WARNING
        [[ $error_flag ]] && subject=ERROR
        if [[ $subject != '' ]]; then
            subject+="$my_name $subject"
            buf=$(
                mailx -s "$subject" < "$log_fn"
            )
        fi
    fi

    # Exit
    # ~~~~
    fct "${FUNCNAME[0]}" 'exiting'
    exit $my_retval
}  # end of function finalise

#--------------------------
# Name: initialise
# Purpose: sets up environment, parses command line, reads config file
#--------------------------
function initialise {
    local args buf emsg opt opt_l_flag usage_reports_csv_fn

    # Configure shell environment
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~
    export LANG=en_GB.UTF-8
    export LANGUAGE=en_GB.UTF-8
    for var_name in LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
        LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
        LC_TELEPHONE LC_TIME
    do
        unset $var_name
    done

    export PATH=/usr/sbin:/sbin:/usr/bin:/bin
    IFS=$' \n\t'
    set -o nounset
    shopt -s extglob            # Enable extended pattern matching operators
    unset CDPATH                # Ensure cd behaves as expected
    umask 022

    # Initialise some global logic variables
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    readonly false=
    readonly true=true

    debugging_flag=$false
    error_flag=$false
    finalising_flag=$false
    interrupt_flag=$false
    logging_flag=$false
    warning_flag=$false

    # Initialise some global string variables
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    readonly my_name=${0##*/}
    readonly msg_lf=$'\n    '              # Message linefeed and indent
    final_msg=
    var_lib_dir=~/var/lib/ck_mikrotik_routers_update

    # Parse command line
    # ~~~~~~~~~~~~~~~~~~
    args=("$@")
    args_org="$*"
    cfg_fn=/home/nc/etc/${my_name%.sh}.cfg
    emsg=
    opt_f_flag=$false
    opt_l_flag=$false
    while getopts :c:dhl: opt "$@"
    do
        case $opt in
            c )
                cfg_fn=$OPTARG
                ;;
            d )
                debugging_flag=$true
                ;;
            h )
                debugging_flag=$false
                usage verbose
                exit 0
                ;;
            l )
                opt_l_flag=$true
                log_fn=$OPTARG
                ;;
            : )
                emsg+=$msg_lf"Option $OPTARG must have an argument"
                [[ $OPTARG = c ]] && { opt_c_flag=$true; conf_fn=/bin/bash; }
                ;;
            * )
                emsg+=$msg_lf"Invalid option '-$OPTARG'"
        esac
    done

    # Check for mandatory options missing
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # There are no mandatory options

    # Test for mutually exclusive options
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # There are no mutually exclusive options

    # Validate option values
    # ~~~~~~~~~~~~~~~~~~~~~~
    [[ ! -r "$cfg_fn" ]] \
        && emsg+=$msg_lf"$cfg_fn does not exist or is not readable"

    # Test for extra arguments
    # ~~~~~~~~~~~~~~~~~~~~~~~~
    shift $(($OPTIND-1))
    if [[ $* != '' ]]; then
        emsg+=$msg_lf"Invalid extra argument(s) '$*'"
    fi

    # Read conffile
    # ~~~~~~~~~~~~~
    log_dir=/var/log/${my_name%.sh}
    source "$cfg_fn"

    # Set up logging
    # ~~~~~~~~~~~~~~
    if [[ ! $opt_l_flag ]]; then
        # Log name not specified so default it
        log_fn=$log_dir/$my_name.$(date +%Y-%m-%d@%H:%M:%S).log
    fi
    readonly log_fn_pat="$my_name.[^.]+\.log\$"
    exec &>"$log_fn"
    fct "${FUNCNAME[0]}" 'started (this message delayed until logging set up)'
    msg I "$my_name: PID: $$, PPID: $PPID, args: $args_org"

    # Create temporary directory
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~
    # If the mktemplate is changed, tmp_dir_regex in the finalise function
    # may also need to be changed.
    msg I 'Creating temporary directory'
    buf=$(mktemp -d "/tmp/$my_name+${cfg_fn##*/}.XXXXXX" 2>&1)
    if (($?==0)); then
        tmp_dir=$buf
        tmp_dir_created_flag=$true
        chmod 700 "$tmp_dir"
    else
        emsg+=$msg_lf"Unable to create temporary directory:$buf"
    fi

    # Report any errors
    # ~~~~~~~~~~~~~~~~~
    if [[ $emsg != '' ]]; then
        msg E "$emsg"
    fi

    # Set traps
    # ~~~~~~~~~
    trap 'finalise 129' 'HUP'
    trap 'finalise 130' 'INT'
    trap 'finalise 131' 'QUIT'
    trap 'finalise 132' 'ILL'
    trap 'finalise 134' 'ABRT'
    trap 'finalise 135' 'BUS'
    trap 'finalise 136' 'FPE'
    trap 'finalise 138' 'USR1'
    trap 'finalise 139' 'SEGV'
    trap 'finalise 140' 'USR2'
    trap 'finalise 141' 'PIPE'
    trap 'finalise 142' 'ALRM'
    trap 'finalise 143' 'TERM'
    trap 'finalise 145' 'CHLD'
    trap 'finalise 146' 'CONT'
    trap 'finalise 147' 'STOP'
    trap 'finalise 148' 'TSTP'
    trap 'finalise 149' 'TTIN'
    trap 'finalise 150' 'TTOU'
    trap 'finalise 151' 'URG'
    trap 'finalise 152' 'XCPU'
    trap 'finalise 153' 'XFSZ'
    trap 'finalise 154' 'VTALRM'
    trap 'finalise 155' 'PROF'

    fct "${FUNCNAME[0]}" 'returning'
}  # end of function initialise

#--------------------------
# Name: msg
# Purpose: generalised messaging interface
# Arguments:
#    $1 class: D, E, I or W indicating Debug, Error, Information or Warning
#    $2 message text
# Global variables read:
#     debugging_flag
# Global variables written:
#     error_flag
#     warning_flag
# Output: information messages to stdout; the rest to stderr
# Returns:
#   Does not return (calls finalise) when class is E for error
#   Otherwise returns 0
#--------------------------
function msg {
    local buf class logger_flag message_text prefix

    # Process arguments
    # ~~~~~~~~~~~~~~~~~
    class="${1:-}"
    message_text="${2:-}"

    # Class-dependent set-up
    # ~~~~~~~~~~~~~~~~~~~~~~
    logger_flag=$false
    case "$class" in
        D )
            [[ ! $debugging_flag ]] && return
            prefix='DEBUG: '
            ;;
        E )
            error_flag=$true
            prefix='ERROR: '
            logger_flag=$true
            ;;
        I )
            prefix=
            ;;
        W )
            warning_flag=$true
            prefix='WARN: '
            logger_flag=$true
            ;;
        * )
            msg E "msg: invalid class '$class': '$*'"
    esac
    message_text="$prefix$message_text"

    # Write to syslog
    # ~~~~~~~~~~~~~~~
    if [[ $logger_flag ]]; then
        buf=$(logger -t "$my_name[$$]" -- "$message_text" 2>&1)
        [[ $buf != '' ]] && msg W "${FUNCNAME[0]}: problem writing to syslog: $buf"
    fi

    # Write to stdout or stderr
    # ~~~~~~~~~~~~~~~~~~~~~~~~~
    message_text="$(date '+%H:%M:%S') $message_text"
    if [[ $class = I ]]; then
        echo "$message_text"
    else
        echo "$message_text" >&2
        if [[ $class = E ]]; then
            [[ ! $finalising_flag ]] && finalise 1
        fi
    fi

    return 0
}  #  end of function msg

#--------------------------
# Name: usage
# Purpose: prints usage message
#--------------------------
function usage {
    fct "${FUNCNAME[0]}" 'started'
    local msg usage

    # Build the messages
    # ~~~~~~~~~~~~~~~~~~
    usage="usage: $my_name "
    msg='  where:'
    usage+='[-c conffile] [-d] [-h] [-l log]'
    msg+=$'\n    -c names the configuration file. Default '"$cfg_fn"
    msg+=$'\n    -d debugging on'
    msg+=$'\n    -l log filename.  Use /dev/tty to display on screen'
    msg+=$'\n    -h prints this help and exits'

    # Display the message(s)
    # ~~~~~~~~~~~~~~~~~~~~~~
    echo "$usage" >&2
    if [[ ${1:-} != 'verbose' ]]; then
        echo "(use -h for help)" >&2
    else
        echo "$msg" >&2
    fi

    fct "${FUNCNAME[0]}" 'returning'
}  # end of function usage

source /usr/local/lib/bash/run_cmd_with_timeout.fun || exit 1

#--------------------------
# Name: main
# Purpose: where it all happens
#--------------------------
initialise "${@:-}"
ck_all_routers
finalise 0


#--------------------------
# Name: run_cmd_with_timeout
# Purpose:
#   * Runs a command with timeout
# Options:
#   -e <rc test list> error message control. If any of the comma separated
#      number tests are true an error message is generated
#   -E <regex> error message control. If the regex matches the output, an error
#      message is generated
#   -n <rc test list> message control. If any of the return code tests are true:
#      If -e is also used, a warning message is generated
#      Otherwise (-w is also used), an error message is generated
#   -N <regex> message control. If the regex matches the output:
#      If -E is also used, a warning message is generated
#      Otherwise (-W is also used), an error message is generated
#   -t <timeout>. Same as the timeout command's DURATION: floating point number
#      with an optional suffix: 's' for seconds (the default), 'm' for minutes,
#      'h' for hours or 'd' for days.  Default 10 (seconds)
#   -T <msg class> if the command times out, a <msg class> message is generated.
#      <msg_class> must be I W or E
#      Default: no message is generated on timeout.
#   -w <rc test list> warning message control. If any of the return code tests
#      are true: and no -E or -e option matches, a warning message
#      is generated.
#   -W <regex> warning message control. If the regex matches the output, and no
#      -E or -e option matches, a warning message is generated
#
# rc_test_list format:
#   One or more of > < >= <= != == followed by an unsigned integaer and
#   separated by commas
#
# -E, -e, -N, -n, -W and -w usage notes:
#
#   * When none of these options is used:
#     * stdout and stderr are ignored
#     * return code 1 generates a warning message
#     * return code >1 generates an error message
#     That is equivalent to -e '>1' '-w==1'
#   * When none of -E, -N and -W are used, stdout and stderr are ignored
#   * When -n is used, -e or -w must also be used
#   * When -N is used, -E or -W must also be used
#   * When none of -e, -n and -w are used and some of -E, -N and -W are
#     used, the return code is ignored
#
# Before calling this function:
#   * Array ${cmd[*]} must be loaded with the command.
#     Must be a simple command; cannot include | < > &
#   * $out_fn and $rc_fn must contain writeable file paths.
#     Any existing content will be deleted.
#
# This function's return codes:
#   0 - No problem detected with the command
#   1 - A warning message was generated (error messages are fatal)
#   2 - The command timed out
#
# After calling this function caller could:
#   * Examine this function's return code
#   * Read the command's output from $out_fn
#   * Read the command's return code from $rc_fn
#
# Example 1
#    msg I "Checking the $user@$hostname ssh connection"
#    cmd=(ssh "$user@$hostname" 'echo OK')
#    run_cmd_with_timeout -t 5
#    case $? in
#        0 )
#            msg I 'ssh connection OK'
#            ;;
#        1 | 2 )
#            msg W 'ssh connection check failed'
#            fct "${FUNCNAME[0]}" 'returning 1'
#            return 1
#    esac
#
# Global variables read:
#   cmd
#   false
#   msg_lf
#   out_fn
#   rc_fn
#   true
# Global variables set: none
# Output:
#   * stdout and stderr to log or screen, either directly or via msg function
# Returns: described above
#--------------------------
function run_cmd_with_timeout {
    fct "${FUNCNAME[0]}" "started with arguments $*"
    local OPTIND    # Required when getopts is called in a function
    local args opt
    local opt_e_flag opt_E_flag opt_n_flag opt_N_flag opt_w_flag opt_W_flag
    local emsg_rc_test_list n_msg_rc_test_list warn_msg_test_list
    local emsg_out_regex warn_msg_out_regex n_msg_out_regex
    local duration timeout_msg_class
    local -r duration_OK_regex='^[[:digit:]]+(|d|h|s)$'
    local -r timeout_msg_class_OK_regex='^(E|I|W)$'
    local buf emsg msg msg_class my_rc out rc rc_for_msg timed_out_flag

    # Parse options
    # ~~~~~~~~~~~~~
    args=("$@")
    duration=10
    emsg=
    emsg_out_regex=
    emsg_rc_test_list=
    emsg_warn_regex=
    n_msg_out_regex=
    n_msg_rc_test_list=
    opt_e_flag=$false
    opt_E_flag=$false
    opt_n_flag=$false
    opt_N_flag=$false
    opt_w_flag=$false
    opt_W_flag=$false
    timeout_msg_class=
    warn_msg_out_regex=
    warn_msg_rc_test_list=
    while getopts :e:E:n:N:w:W:t:T: opt "$@"
    do
        case $opt in
            e )
                opt_e_flag=$true
                emsg_rc_test_list=$OPTARG
                ;;
            E )
                opt_E_flag=$true
                emsg_out_regex=$OPTARG
                ;;
            n )
                opt_n_flag=$true
                n_msg_rc_test_list=$OPTARG
                ;;
            N )
                opt_N_flag=$true
                n_msg_out_regex=$OPTARG
                ;;
            t )
                duration=$OPTARG
                ;;
            T )
                timeout_msg_class=$OPTARG
                ;;
            w )
                opt_w_flag=$true
                warn_msg_rc_test_list=$OPTARG
                ;;
            W )
                opt_W_flag=$true
                warn_msg_out_regex=$OPTARG
                ;;
            : )
                emsg+=$msg_lf"Option $OPTARG must have an argument"
                ;;
            * )
                emsg+=$msg_lf"Invalid option '-$OPTARG'"
        esac
    done
    shift $(($OPTIND-1))

    # Test for mutually exclusive options
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    if [[ $opt_n_flag ]]; then
        buf=$opt_e_flag$opt_n_flag$opt_w_flag
        [[ $buf != '' && $buf != $true$true ]] \
            && emsg+=$msg_lf'When -n is used, -e or -w must also be used'
    fi
    if [[ $opt_N_flag ]]; then
        buf=$opt_E_flag$opt_N_flag$opt_W_flag
        [[ $buf != '' && $buf != $true$true ]] \
            && emsg+=$msg_lf'When -N is used, -E or -W must also be used'
    fi

    # Test for mandatory options not set
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # There are no mandatory options

    # Set derived default values
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # If none of -E -e -N -n -W and -w were used, rc 1 generates a warning and
    # rc>1 generates an error
    if [[ $opt_e_flag$opt_n_flag$opt_w_flag = ''
        && $opt_E_flag$opt_N_flag$opt_W_flag = '' ]]; then
        emsg_rc_test_list='>1'
        warn_msg_rc_test_list='==1'
    fi

    # Validate option arguments
    # ~~~~~~~~~~~~~~~~~~~~~~~~~
    [[ ! $duration =~ $duration_OK_regex ]] \
        && emsg+=$msg_lf"Invalid -t option $duration (does not match $duration_OK_regex)"
    if [[ $timeout_msg_class != '' ]] \
        && [[ ! $timeout_msg_class =~ $timeout_msg_class_OK_regex ]]; then
             emsg+=$msg_lf"Invalid -T option $timeout_msg_class "
             emsg+=$msg_lf" (does not match $timeout_msg_class_OK_regex)"
    fi

    # Validate variables to be used
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # Incidentally also ensure the files are empty
    [[ ${cmd:-} = '' ]] && emsg+=$msg_lf'$cmd is required but is unset or empty'
    if [[ ${out_fn:-} != '' ]]; then
        echo -n > "$out_fn" 2>/dev/null
        buf=$(ck_file "$out_fn" f:rw 2>&1)
        [[ $buf != '' ]] && emsg+=$msg_lf$buf
    else
        emsg+=$msg_lf'$out_fn is required but is unset or empty'
    fi
    if [[ ${rc_fn:-} != '' ]]; then
        echo -n > "$rc_fn" 2>/dev/null
        buf=$(ck_file "$rc_fn" f:rw 2>&1)
        [[ $buf != '' ]] && emsg+=$msg_lf$buf
    else
        emsg+=$msg_lf'$rc_fn is required but is unset or empty'
    fi

    # Report any errors
    # ~~~~~~~~~~~~~~~~~
    if [[ $emsg != '' ]]; then
        # These are programming error(s) so use error, not warning
        msg E "${FUNCNAME[0]} called with ${args[*]}$emsg"
    fi

    # Run the command with timeout
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    msg I "Running command with timeout: ${cmd[*]}"
    msg_class=
    timeout --signal=SIGTERM --kill-after=10 $duration "${cmd[@]}" > "$out_fn" 2>&1
    rc=$?
    if ((rc==124||rc==137)); then
        timed_out_flag=$true
        msg I 'Timed out'
        rc_for_msg='not available (timed out)'
        [[ $timeout_msg_class != '' ]] && msg_class=$timeout_msg_class
    else
        timed_out_flag=$false
        echo $rc > "$rc_fn"
        rc_for_msg=$rc
    fi

    # Examine any return code if requested
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # The return code is not available after timeout
    if [[ ! $timed_out_flag ]]; then
       if [[ $emsg_rc_test_list != '' ]] \
           && run_rc_tests $emsg_rc_test_list; then
           msg_class=E
       elif [[ $warn_msg_rc_test_list != '' ]] \
           && run_rc_tests $warn_msg_rc_test_list; then
           [[ $msg_class != E ]] && msg_class=W
       elif [[ $n_msg_rc_test_list != '' ]] \
           && run_rc_tests $n_msg_rc_test_list; then
           if [[ $emsg_rc_test_list != '' ]]; then
               [[ $msg_class != E ]] && msg_class=W
           else
               msg_class=E
           fi
       fi
    fi

    # Examine any output if requested
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    out=$(< "$out_fn")
    if [[ $out != '' ]]; then
       if [[ $emsg_out_regex != '' && $out =~ $emsg_out_regex ]]; then
           msg_class=E
       elif [[ $warn_msg_out_regex != '' && $out =~ $warn_msg_out_regex ]]; then
           [[ $msg_class != E ]] && msg_class=W
       elif [[ $n_msg_out_regex != '' && $out =~ $n_msg_out_regex ]]; then
           if [[ $emsg_out_regex != '' ]]; then
               [[ $msg_class != E ]] && msg_class=W
           else
               msg_class=E
           fi
       fi
    fi

    # Generate message if required
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    if [[ $msg_class != '' ]]; then
        msg="Command: ${cmd[*]}"
        msg+=$'\n'"rc: $rc_for_msg"
        msg+=$'\n'"Output: $out"
        msg $msg_class "$msg"
    fi

    # Set my return code
    # ~~~~~~~~~~~~~~~~~~
    #   0 - No problem detected with the command
    #   1 - A problem other than timeout was detected
    #   2 - The command timed out
    if [[ $timed_out_flag ]]; then
        my_rc=2
    elif [[ $msg_class = W ]]; then
        my_rc=1
    else
        my_rc=0
    fi

    fct "${FUNCNAME[0]}" "returning $my_rc"
    return $my_rc
}  #  end of function run_cmd_with_timeout

#--------------------------
# Name: run_rc_tests
# Purpose:
#   * Runs return code tests (for run_cmd_with_timeout)
# Arguments:
#   $1 - comma separated list of arithmentical tests
#
# Global variables read: rc
# Global variables set: none
# Output:
#   * stdout and stderr to log or screen, either directly or via msg function
# Returns:
#   0 - $rc matched one of the tests
#   1 - $rc did not match any of the tests
#--------------------------
function run_rc_tests {
    local array i numcom oldIFS one_char rhs two_char
    local -r two_char_numcom_regex='^(<|>|=|!)=$'
    local -r one_char_numcom_regex='^(<|>)$'
    local -r uint_regex='^[[:digit:]]+$'

    # Parse the argument
    # ~~~~~~~~~~~~~~~~~~
    oldIFS=$IFS
    IFS=,
    array=($1)
    IFS=$oldIFS

    # For each test
    # ~~~~~~~~~~~~~
    for ((i=0;i<${#array[*]};i++))
    do
       # Parse the test
       # ~~~~~~~~~~~~~~
       two_char=${array[i]:0:2}
       one_char=${array[i]:0:1}
       if [[ $two_char =~ $two_char_numcom_regex ]]; then
           numcom=$two_char
           rhs=${array[i]:2}
       elif [[ $one_char =~ $one_char_numcom_regex ]]; then
           numcom=$one_char
           rhs=${array[i]:1}
       else
           msg E "Programming error: invalid numerc comparison ${array[i]} in $1"
       fi
       [[ ! $rhs =~ $uint_regex ]] \
           &&  msg E "Programming error: invalid numerc comparison ${array[i]} in $1"

       # Test
       # ~~~~
       case $numcom in
           '<=' ) ((rc<=rhs)) && return 0 ;;
           '>=' ) ((rc>=rhs)) && return 0 ;;
           '==' ) ((rc==rhs)) && return 0 ;;
           '!=' ) ((rc!=rhs)) && return 0 ;;
           '<'  ) ((rc<rhs))  && return 0 ;;
           '>'  ) ((rc>rhs))  && return 0 ;;
       esac

    done

    return 1
}  #  end of function run_rc_tests


# Configuration for ck_mikrotik_routers_update.sh

# This file is sourced by bash
#   * Lines begining with # and empty lines ignored
#   * Remaining lines must be valid bash assignments

email_for_report=sysmail@example.com

# One array memmber for each MikroTik router
# Each member must be a colon separated list of:
#    * Router FQDN
#    * User name
#    * ssh private key
#    * Number of days failed to run a command on the router before
#      reporting by mail
mikrotik_routers=(
    amt02.example.com:nc:/home/nc/.ssh/nc_for_ck_mikrotik_routers_update:28
    bna02.example.com:nc:/home/nc/.ssh/nc_for_ck_mikrotik_routers_update:7
    cer02.example.com:/home/nc/.ssh/nc_for_ck_mikrotik_routers_update:1
)

# In case want to overide default of /var/log/ck_mikrotik_routers_update.log
#log_dir=/var/log/ck_mikrotik_routers_update.log