SHARE
TWEET

For MikroTik forum

catkin Jul 7th, 2019 39 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/bin/bash
  2.  
  3. # Function call tree
  4. #    +
  5. #    |
  6. #    +-- initialise
  7. #    |
  8. #    +-- ck_all_routers
  9. #    |   |
  10. #    +   +-- ck_one_router
  11. #    |
  12. #    +-- finalise
  13. #
  14. # Utility functions called from various places:
  15. #    ck_file ck_uint fct msg run_cmd_with_timeout
  16.  
  17. #--------------------------
  18. # Name: ck_all_routers
  19. # Purpose: checks all routers
  20. #--------------------------
  21. function ck_all_routers {
  22.     fct "${FUNCNAME[0]}" 'started'
  23.     local i
  24.  
  25.     for ((i=0;i<${#mikrotik_routers[*]};i++))
  26.     do
  27.         ck_one_router "${mikrotik_routers[i]}"
  28.     done
  29.  
  30.     fct "${FUNCNAME[0]}" 'returning'
  31.     return 0
  32. }  # end of function ck_all_routers
  33.  
  34. source /usr/local/lib/bash/ck_file.fun || exit 1
  35.  
  36. #--------------------------
  37. # Name: ck_one_router
  38. # Purpose: checks one router
  39. #--------------------------
  40. function ck_one_router {
  41.     fct "${FUNCNAME[0]}" "started. Args: $*"
  42.     local array buf cmd installed_ver latest_ver msg msg_part oldIFS
  43.     local days_cmd_fail_before_report rc router_fqdn router_user ssh_key ssh_opts
  44.     local -r check_OK_regex='status: (New version is available|System is already up to date)'
  45.     local -r up_to_date_regex='status: System is already up to date'
  46.     local -r whitespace_or_empty_regex='^[[:space:]]*$'
  47.  
  48.     # Parse the conf string
  49.     # ~~~~~~~~~~~~~~~~~~~~~
  50.     oldIFS=$IFS
  51.     IFS=:
  52.     array=($1)
  53.     IFS=$oldIFS
  54.     router_fqdn=${array[0]}
  55.     router_user=${array[1]}
  56.     ssh_key=${array[2]}
  57.     days_cmd_fail_before_report=${array[3]}
  58.  
  59.     # Derive value
  60.     # ~~~~~~~~~~~~
  61.     cmd_OK_timestamp_fn=$var_lib_dir/$router_fqdn
  62.  
  63.     # Get update status
  64.     # ~~~~~~~~~~~~~~~~~
  65.     # ssh's ConnectTimeout must be bigger than run_cmd_with_timeout's -t
  66.     ssh_opts="-i $ssh_key"
  67.     ssh_opts+=' -o ConnectTimeout=15'
  68.     ssh_opts+=' -o PasswordAuthentication=no -o StrictHostKeyChecking=no'
  69.     ssh_opts+=' -o UserKnownHostsFile=/dev/null'
  70.     cmd=(
  71.         ssh $ssh_opts -o LogLevel=quiet "$router_user@$router_fqdn"
  72.             system package update check-for-updates
  73.     )
  74.     out_fn=$tmp_dir/out
  75.     rc_fn=$tmp_dir/rc
  76.     run_cmd_with_timeout -t 10 -w '>0'
  77.     rc=$?
  78.     case $rc in
  79.         0 ) ;;
  80.         2 )
  81.             msg I "Timed out running ssh command on $router_fqdn"
  82.             ;&
  83.         1 )
  84.             [[ ! -f "$cmd_OK_timestamp_fn" ]] && touch "$cmd_OK_timestamp_fn"
  85.             cmd=(find "$cmd_OK_timestamp_fn"
  86.                 -mtime +"$days_cmd_fail_before_report"
  87.             )
  88.             buf=$("${cmd[@]}" 2>&1)
  89.             rc=$?
  90.             if ((rc==0)); then
  91.                 if [[ $buf != '' ]]; then
  92.                     msg="More than $days_cmd_fail_before_report days since"
  93.                     msg+=" last ssh command on $router_fqdn set return code 0"
  94.                     msg+=$'\nThe last successful command was:'
  95.                     msg+=$'\n'$(tail -1 "$cmd_OK_timestamp_fn")
  96.                     msg W "$msg"
  97.                 fi
  98.             else
  99.                 msg="Command: ${cmd[*]}"
  100.                 msg+=$'\n'"Return code: $rc"
  101.                 msg+=$'\n'"Output: $buf"
  102.                 msg W "$msg"
  103.             fi
  104.             msg I 'Again with more logging for diagnostics ...`'
  105.             cmd=(
  106.                 ssh $ssh_opts "$router_user@$router_fqdn"
  107.                     system package update check-for-updates
  108.             )
  109.             run_cmd_with_timeout -t 10 -w '>0'
  110.             if [[ ${msg:-} != '' ]]; then
  111.                 final_msg+=$'\n\n'$msg
  112.             fi
  113.            
  114.             fct "${FUNCNAME[0]}" 'returning 1'
  115.             return 1
  116.     esac
  117.     buf=$(< "$out_fn")
  118.     msg I "$router_fqdn:"$'\n'"$buf"
  119.  
  120.     # Note time of successfully running command on this router
  121.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  122.     echo "$(date '+%a %-d %b %Y %H:%M:%S') Ran command on $router_fqdn" >> "$cmd_OK_timestamp_fn"
  123.  
  124.     # Parse update status
  125.     # ~~~~~~~~~~~~~~~~~~~
  126.     # Ignoring the output which normally ends with
  127.     # "finding out latest version...":
  128.     # Example output:
  129.     #            channel: stable
  130.     #  installed-version: 6.43.8
  131.     #     latest-version: 6.43.8
  132.     #             status: System is already up to date
  133.     # Example output:
  134.     #            channel: stable
  135.     #  installed-version: 6.43.7
  136.     #     latest-version: 6.43.8
  137.     #             status: New version is available
  138.     # Example output (bug; installed > latest but "New version is available")
  139.     #            channel: long-term
  140.     #  installed-version: 6.43.7
  141.     #     latest-version: 6.42.10
  142.     #             status: New version is available
  143.     # Example output (no installed-version, only current-version)
  144.     #            channel: long-term
  145.     #    current-version: 6.42.11
  146.     #     latest-version: 6.42.11
  147.     #             status: System is already up to date
  148.     current_or_installed_ver=$(echo "$buf" \
  149.         | grep -E '(current|installed)-version:' \
  150.         | tail -1 \
  151.         | sed --regexp-extended 's/.*(current|installed)-version: //'
  152.     )
  153.     if [[ $current_or_installed_ver =~ $whitespace_or_empty_regex ]]; then
  154.         msg='Failed to get current or installed version from output'
  155.         msg+=$'\n'"Command: ${cmd[*]}"
  156.         msg+=$'\n'"Output: $buf"
  157.         msg W "$msg"
  158.         fct "${FUNCNAME[0]}" 'returning 1'
  159.         return 1
  160.     fi
  161.     msg D "current_or_installed_ver: $current_or_installed_ver"
  162.     IFS=.
  163.     array=($current_or_installed_ver)
  164.     IFS=$oldIFS
  165.     installed_major=${array[0]}
  166.     installed_minor=${array[1]}
  167.     installed_patch=${array[2]%%[[:space:]]}
  168.     msg D "installed_major: '$installed_major'"
  169.     msg D "installed_minor: '$installed_minor'"
  170.     msg D "installed_patch: '$installed_patch'"
  171.     msg=
  172.     ! ck_uint $installed_major \
  173.         && msg+=$'\n'"installed_major not uint: '$installed_major'"
  174.     ! ck_uint $installed_minor \
  175.         && msg+=$'\n'"installed_minor not uint: '$installed_minor'"
  176.     ! ck_uint $installed_patch \
  177.         && msg+=$'\n'"installed_patch not uint: '$installed_patch'"
  178.  
  179.     latest_ver=$(echo "$buf" \
  180.         | grep 'latest-version:' \
  181.         | sed 's/.*latest-version: //'
  182.     )
  183.     if [[ $latest_ver =~ $whitespace_or_empty_regex ]]; then
  184.         msg='Failed to get latest-version from output'
  185.         msg+=$'\n'"Command: ${cmd[*]}"
  186.         msg+=$'\n'"Output: $buf"
  187.         msg W "$msg"
  188.         fct "${FUNCNAME[0]}" 'returning 1'
  189.         return 1
  190.     fi
  191.     msg D "latest_ver: $latest_ver"
  192.     IFS=.
  193.     array=($latest_ver)
  194.     IFS=$oldIFS
  195.     latest_major=${array[0]}
  196.     latest_minor=${array[1]}
  197.     latest_patch=${array[2]%%[[:space:]]}
  198.     msg D "latest_major: '$latest_major'"
  199.     msg D "latest_minor: '$latest_minor'"
  200.     msg D "latest_patch: '$latest_patch'"
  201.     ! ck_uint $latest_major \
  202.         && msg+=$'\n'"latest_major not uint: '$latest_major'"
  203.     ! ck_uint $latest_minor \
  204.         && msg+=$'\n'"latest_minor not uint: '$latest_minor'"
  205.     ! ck_uint $latest_patch \
  206.         && msg+=$'\n'"latest_patch not uint: '$latest_patch'"
  207.     if [[ $msg != '' ]]; then
  208.         msg W "update status output parse error:$msg"
  209.         fct "${FUNCNAME[0]}" 'returning 1'
  210.         return 1
  211.     fi
  212.  
  213.     # Update required?
  214.     # ~~~~~~~~~~~~~~~~
  215.     msg_part=$'\n'"Current or installed version: $current_or_installed_ver"
  216.     msg_part+=$'\n'"Latest version: $latest_ver"
  217.     msg=
  218.     if ((latest_major>installed_major)); then
  219.         msg="$router_fqdn: manual update required"
  220.         msg+=" (version major number has changed)"
  221.     elif ((latest_major==installed_major)); then
  222.         if ((latest_minor>installed_minor)); then
  223.             msg="$router_fqdn: update required"
  224.             msg+=" (version minor number has changed)"
  225.         elif ((latest_minor==installed_minor)); then
  226.             if ((latest_patch>installed_patch)); then
  227.                 msg="$router_fqdn: update required"
  228.                 msg+=" (patch number has changed)"
  229.             fi
  230.         fi
  231.     fi
  232.     if [[ $msg = '' ]]; then
  233.         msg I "$router_fqdn: update not required$msg_part"
  234.     else
  235.         final_msg+=$'\n\n'$msg$msg_part
  236.     fi
  237.  
  238.     fct "${FUNCNAME[0]}" 'returning'
  239.     return 0
  240. }  # end of function ck_one_router
  241.  
  242. #--------------------------
  243. # Name: ck_uint
  244. # Purpose: checks for a valid unsigned integer
  245. # Usage: ck_uint <putative uint>
  246. # Outputs: none
  247. # Returns:
  248. #   0 when $1 is a valid unsigned integer
  249. #   1 otherwise
  250. #--------------------------
  251. function ck_uint {
  252.     local regex='^[[:digit:]]+$'
  253.     [[ $1 =~ $regex ]] && return 0 || return 1
  254. }  #  end of function ck_uint
  255.  
  256. #--------------------------
  257. # Name: fct
  258. # Purpose: function call trace (for debugging)
  259. # $1 - name of calling function
  260. # $2 - message.  If it starts with "started" or "returning" then the output is prettily indented
  261. #--------------------------
  262. function fct {
  263.  
  264.     if [[ ! $debugging_flag ]]; then
  265.         return 0
  266.     fi  
  267.  
  268.     fct_indent="${fct_indent:=}"
  269.  
  270.     case $2 in
  271.         'started'* )
  272.             fct_indent="$fct_indent  "
  273.             msg D "$fct_indent$1: $2"
  274.             ;;  
  275.         'returning'* )
  276.             msg D "$fct_indent$1: $2"
  277.             fct_indent="${fct_indent#  }"  
  278.             ;;  
  279.         * )
  280.             msg D "$fct_indent$1: $2"
  281.     esac
  282.  
  283. }  # end of function fct
  284.  
  285. #--------------------------
  286. # Name: finalise
  287. # Purpose: cleans up and exits
  288. # Arguments:
  289. #    $1  return value
  290. # Return code (on exit):
  291. #   The sum of zero plus
  292. #      1 if any warnings
  293. #      2 if any errors
  294. #      4,8,16 unused
  295. #      32 if terminated by a signal
  296. #--------------------------
  297. function finalise {
  298.     fct "${FUNCNAME[0]}" "started with args $*"
  299.     local buf cmd msg_part my_retval subject
  300.  
  301.     finalising_flag=$true
  302.  
  303.     # Email any message about updates being available
  304.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  305.     if [[ $final_msg != '' ]]; then
  306.         msg I "Mailing report to $email_for_report"
  307.         msg_part+=$'\n\n'
  308.         cmd=(mailx -s "MikroTik routers needing update"
  309.             "$email_for_report"
  310.         )
  311.         buf=$(echo "$msg_part$final_msg" | "${cmd[@]}" 2>&1)
  312.         [[ $buf != '' ]] && msg E "Unexpected output from ${cmd[*]}: $buf"
  313.     fi
  314.  
  315.     # Remove old logs
  316.     # ~~~~~~~~~~~~~~~
  317.     if [[ ${log_dir:-} != '' ]]; then
  318.         msg I 'Removing old logs'
  319.         buf=$(
  320.             find "$log_dir" \
  321.                 -maxdepth 1 \
  322.                 -regextype posix-egrep \
  323.                 -regex ".*/$log_fn_pat" \
  324.                 -mtime +7 \
  325.                 -delete \
  326.                 2>&1
  327.         )
  328.         [[ $buf != '' ]] && msg W "Problem removing old logs: $buf"
  329.     fi
  330.  
  331.     # Interrupted?  Message and exit return value
  332.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  333.     my_retval=0
  334.     if ck_uint "${1:-}" && (($1>128)); then
  335.         ((my_retval+=32))
  336.         case $1 in
  337.             129 )
  338.                 buf=SIGHUP
  339.                 ;;
  340.             130 )
  341.                 buf=SIGINT
  342.                 ;;
  343.             131 )
  344.                 buf=SIGQUIT
  345.                 ;;
  346.             132 )
  347.                 buf=SIGILL
  348.                 ;;
  349.             134 )
  350.                 buf=SIGABRT
  351.                 ;;
  352.             135 )
  353.                 buf=SIGBUS
  354.                 ;;
  355.             136 )
  356.                 buf=SIGFPE
  357.                 ;;
  358.             138 )
  359.                 buf=SIGUSR1
  360.                 ;;
  361.             139 )
  362.                 buf=SIGSEGV
  363.                 ;;
  364.             140 )
  365.                 buf=SIGUSR2
  366.                 ;;
  367.             141 )
  368.                 buf=SIGPIPE
  369.                 ;;
  370.             142 )
  371.                 buf=SIGALRM
  372.                 ;;
  373.             143 )
  374.                 buf=SIGTERM
  375.                 ;;
  376.             145 )
  377.                 buf=SIGCHLD
  378.                 ;;
  379.             146 )
  380.                 buf=SIGCONT
  381.                 ;;
  382.             147 )
  383.                 buf=SIGSTOP
  384.                 ;;
  385.             148 )
  386.                 buf=SIGTSTP
  387.                 ;;
  388.             149 )
  389.                 buf=SIGTTIN
  390.                 ;;
  391.             150 )
  392.                 buf=SIGTTOU
  393.                 ;;
  394.             151 )
  395.                 buf=SIGURG
  396.                 ;;
  397.             152 )
  398.                 buf=SIGCPU
  399.                 ;;
  400.             153 )
  401.                 buf=SIGXFSZ
  402.                 ;;
  403.             154 )
  404.                 buf=SIGVTALRM
  405.                 ;;
  406.             155 )
  407.                 buf=SIGPROF
  408.                 ;;
  409.             * )
  410.                 msg E "${FUNCNAME[0]}: programming error: \$1 ($1) not serviced"
  411.                 ;;
  412.         esac
  413.         interrupt_flag=$true
  414.         msg="Finalising on $buf"
  415.         msg E "$msg"    # Returns because finalising_flag is set
  416.     fi
  417.  
  418.     # Temporary directory removal
  419.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~
  420.     local -r tmp_dir_regex="^/tmp/$my_name\+${cfg_fn##*/}\..{6}$"
  421.     if [[ $tmp_dir_created_flag \
  422.         && ${tmp_dir:-} =~ $tmp_dir_regex \
  423.     ]]; then
  424.         msg I "Removing temporary directory $tmp_dir"
  425.         rm -fr "$tmp_dir"
  426.     fi
  427.  
  428.     # Exit return value adjustment
  429.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  430.     if [[ $warning_flag ]]; then
  431.         msg I "There was at least one WARNING"
  432.         ((my_retval+=1))
  433.     fi
  434.     if [[ $error_flag ]]; then
  435.         msg I "There was at least one ERROR"
  436.         ((my_retval+=2))
  437.     fi
  438.     [[ $interrupt_flag ]] && msg I 'There was at least one interrupt'
  439.     if ((my_retval==0)) && ((${1:-0}!=0)); then
  440.         msg E 'There was an error not reported in detail (probably by ... || finalise 1)'
  441.         my_retval=2
  442.     fi
  443.     msg I "Exiting with return value $my_retval"
  444.  
  445.     # Mail log
  446.     # ~~~~~~~~
  447.     if [[ -f "$log_fn" ]]; then
  448.         subject=
  449.         [[ $warning_flag ]] && subject=WARNING
  450.         [[ $error_flag ]] && subject=ERROR
  451.         if [[ $subject != '' ]]; then
  452.             subject+="$my_name $subject"
  453.             buf=$(
  454.                 mailx -s "$subject" < "$log_fn"
  455.             )
  456.         fi
  457.     fi
  458.  
  459.     # Exit
  460.     # ~~~~
  461.     fct "${FUNCNAME[0]}" 'exiting'
  462.     exit $my_retval
  463. }  # end of function finalise
  464.  
  465. #--------------------------
  466. # Name: initialise
  467. # Purpose: sets up environment, parses command line, reads config file
  468. #--------------------------
  469. function initialise {
  470.     local args buf emsg opt opt_l_flag usage_reports_csv_fn
  471.  
  472.     # Configure shell environment
  473.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~
  474.     export LANG=en_GB.UTF-8
  475.     export LANGUAGE=en_GB.UTF-8
  476.     for var_name in LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
  477.         LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
  478.         LC_TELEPHONE LC_TIME
  479.     do
  480.         unset $var_name
  481.     done
  482.  
  483.     export PATH=/usr/sbin:/sbin:/usr/bin:/bin
  484.     IFS=$' \n\t'
  485.     set -o nounset
  486.     shopt -s extglob            # Enable extended pattern matching operators
  487.     unset CDPATH                # Ensure cd behaves as expected
  488.     umask 022
  489.  
  490.     # Initialise some global logic variables
  491.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  492.     readonly false=
  493.     readonly true=true
  494.    
  495.     debugging_flag=$false
  496.     error_flag=$false
  497.     finalising_flag=$false
  498.     interrupt_flag=$false
  499.     logging_flag=$false
  500.     warning_flag=$false
  501.  
  502.     # Initialise some global string variables
  503.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  504.     readonly my_name=${0##*/}
  505.     readonly msg_lf=$'\n    '              # Message linefeed and indent
  506.     final_msg=
  507.     var_lib_dir=~/var/lib/ck_mikrotik_routers_update
  508.  
  509.     # Parse command line
  510.     # ~~~~~~~~~~~~~~~~~~
  511.     args=("$@")
  512.     args_org="$*"
  513.     cfg_fn=/home/nc/etc/${my_name%.sh}.cfg
  514.     emsg=
  515.     opt_f_flag=$false
  516.     opt_l_flag=$false
  517.     while getopts :c:dhl: opt "$@"
  518.     do
  519.         case $opt in
  520.             c )
  521.                 cfg_fn=$OPTARG
  522.                 ;;
  523.             d )
  524.                 debugging_flag=$true
  525.                 ;;
  526.             h )
  527.                 debugging_flag=$false
  528.                 usage verbose
  529.                 exit 0
  530.                 ;;
  531.             l )
  532.                 opt_l_flag=$true
  533.                 log_fn=$OPTARG
  534.                 ;;
  535.             : )
  536.                 emsg+=$msg_lf"Option $OPTARG must have an argument"
  537.                 [[ $OPTARG = c ]] && { opt_c_flag=$true; conf_fn=/bin/bash; }
  538.                 ;;
  539.             * )
  540.                 emsg+=$msg_lf"Invalid option '-$OPTARG'"
  541.         esac
  542.     done
  543.  
  544.     # Check for mandatory options missing
  545.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  546.     # There are no mandatory options
  547.  
  548.     # Test for mutually exclusive options
  549.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  550.     # There are no mutually exclusive options
  551.  
  552.     # Validate option values
  553.     # ~~~~~~~~~~~~~~~~~~~~~~
  554.     [[ ! -r "$cfg_fn" ]] \
  555.         && emsg+=$msg_lf"$cfg_fn does not exist or is not readable"
  556.  
  557.     # Test for extra arguments
  558.     # ~~~~~~~~~~~~~~~~~~~~~~~~
  559.     shift $(($OPTIND-1))
  560.     if [[ $* != '' ]]; then
  561.         emsg+=$msg_lf"Invalid extra argument(s) '$*'"
  562.     fi
  563.  
  564.     # Read conffile
  565.     # ~~~~~~~~~~~~~
  566.     log_dir=/var/log/${my_name%.sh}
  567.     source "$cfg_fn"
  568.  
  569.     # Set up logging
  570.     # ~~~~~~~~~~~~~~
  571.     if [[ ! $opt_l_flag ]]; then
  572.         # Log name not specified so default it
  573.         log_fn=$log_dir/$my_name.$(date +%Y-%m-%d@%H:%M:%S).log
  574.     fi
  575.     readonly log_fn_pat="$my_name.[^.]+\.log\$"
  576.     exec &>"$log_fn"
  577.     fct "${FUNCNAME[0]}" 'started (this message delayed until logging set up)'
  578.     msg I "$my_name: PID: $$, PPID: $PPID, args: $args_org"
  579.  
  580.     # Create temporary directory
  581.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~
  582.     # If the mktemplate is changed, tmp_dir_regex in the finalise function
  583.     # may also need to be changed.
  584.     msg I 'Creating temporary directory'
  585.     buf=$(mktemp -d "/tmp/$my_name+${cfg_fn##*/}.XXXXXX" 2>&1)
  586.     if (($?==0)); then
  587.         tmp_dir=$buf
  588.         tmp_dir_created_flag=$true
  589.         chmod 700 "$tmp_dir"
  590.     else
  591.         emsg+=$msg_lf"Unable to create temporary directory:$buf"
  592.     fi
  593.  
  594.     # Report any errors
  595.     # ~~~~~~~~~~~~~~~~~
  596.     if [[ $emsg != '' ]]; then
  597.         msg E "$emsg"
  598.     fi
  599.  
  600.     # Set traps
  601.     # ~~~~~~~~~
  602.     trap 'finalise 129' 'HUP'
  603.     trap 'finalise 130' 'INT'
  604.     trap 'finalise 131' 'QUIT'
  605.     trap 'finalise 132' 'ILL'
  606.     trap 'finalise 134' 'ABRT'
  607.     trap 'finalise 135' 'BUS'
  608.     trap 'finalise 136' 'FPE'
  609.     trap 'finalise 138' 'USR1'
  610.     trap 'finalise 139' 'SEGV'
  611.     trap 'finalise 140' 'USR2'
  612.     trap 'finalise 141' 'PIPE'
  613.     trap 'finalise 142' 'ALRM'
  614.     trap 'finalise 143' 'TERM'
  615.     trap 'finalise 145' 'CHLD'
  616.     trap 'finalise 146' 'CONT'
  617.     trap 'finalise 147' 'STOP'
  618.     trap 'finalise 148' 'TSTP'
  619.     trap 'finalise 149' 'TTIN'
  620.     trap 'finalise 150' 'TTOU'
  621.     trap 'finalise 151' 'URG'
  622.     trap 'finalise 152' 'XCPU'
  623.     trap 'finalise 153' 'XFSZ'
  624.     trap 'finalise 154' 'VTALRM'
  625.     trap 'finalise 155' 'PROF'
  626.  
  627.     fct "${FUNCNAME[0]}" 'returning'
  628. }  # end of function initialise
  629.  
  630. #--------------------------
  631. # Name: msg
  632. # Purpose: generalised messaging interface
  633. # Arguments:
  634. #    $1 class: D, E, I or W indicating Debug, Error, Information or Warning
  635. #    $2 message text
  636. # Global variables read:
  637. #     debugging_flag
  638. # Global variables written:
  639. #     error_flag
  640. #     warning_flag
  641. # Output: information messages to stdout; the rest to stderr
  642. # Returns:
  643. #   Does not return (calls finalise) when class is E for error
  644. #   Otherwise returns 0
  645. #--------------------------
  646. function msg {
  647.     local buf class logger_flag message_text prefix
  648.  
  649.     # Process arguments
  650.     # ~~~~~~~~~~~~~~~~~
  651.     class="${1:-}"
  652.     message_text="${2:-}"
  653.  
  654.     # Class-dependent set-up
  655.     # ~~~~~~~~~~~~~~~~~~~~~~
  656.     logger_flag=$false
  657.     case "$class" in  
  658.         D )
  659.             [[ ! $debugging_flag ]] && return
  660.             prefix='DEBUG: '
  661.             ;;  
  662.         E )
  663.             error_flag=$true
  664.             prefix='ERROR: '
  665.             logger_flag=$true
  666.             ;;  
  667.         I )
  668.             prefix=
  669.             ;;  
  670.         W )
  671.             warning_flag=$true
  672.             prefix='WARN: '
  673.             logger_flag=$true
  674.             ;;  
  675.         * )
  676.             msg E "msg: invalid class '$class': '$*'"
  677.     esac
  678.     message_text="$prefix$message_text"
  679.  
  680.     # Write to syslog
  681.     # ~~~~~~~~~~~~~~~
  682.     if [[ $logger_flag ]]; then
  683.         buf=$(logger -t "$my_name[$$]" -- "$message_text" 2>&1)
  684.         [[ $buf != '' ]] && msg W "${FUNCNAME[0]}: problem writing to syslog: $buf"
  685.     fi
  686.  
  687.     # Write to stdout or stderr
  688.     # ~~~~~~~~~~~~~~~~~~~~~~~~~
  689.     message_text="$(date '+%H:%M:%S') $message_text"
  690.     if [[ $class = I ]]; then
  691.         echo "$message_text"
  692.     else
  693.         echo "$message_text" >&2
  694.         if [[ $class = E ]]; then
  695.             [[ ! $finalising_flag ]] && finalise 1
  696.         fi
  697.     fi  
  698.  
  699.     return 0
  700. }  #  end of function msg
  701.  
  702. #--------------------------
  703. # Name: usage
  704. # Purpose: prints usage message
  705. #--------------------------
  706. function usage {
  707.     fct "${FUNCNAME[0]}" 'started'
  708.     local msg usage
  709.  
  710.     # Build the messages
  711.     # ~~~~~~~~~~~~~~~~~~
  712.     usage="usage: $my_name "
  713.     msg='  where:'
  714.     usage+='[-c conffile] [-d] [-h] [-l log]'
  715.     msg+=$'\n    -c names the configuration file. Default '"$cfg_fn"
  716.     msg+=$'\n    -d debugging on'
  717.     msg+=$'\n    -l log filename.  Use /dev/tty to display on screen'
  718.     msg+=$'\n    -h prints this help and exits'
  719.  
  720.     # Display the message(s)
  721.     # ~~~~~~~~~~~~~~~~~~~~~~
  722.     echo "$usage" >&2
  723.     if [[ ${1:-} != 'verbose' ]]; then
  724.         echo "(use -h for help)" >&2
  725.     else
  726.         echo "$msg" >&2
  727.     fi
  728.  
  729.     fct "${FUNCNAME[0]}" 'returning'
  730. }  # end of function usage
  731.  
  732. source /usr/local/lib/bash/run_cmd_with_timeout.fun || exit 1
  733.  
  734. #--------------------------
  735. # Name: main
  736. # Purpose: where it all happens
  737. #--------------------------
  738. initialise "${@:-}"
  739. ck_all_routers
  740. finalise 0
  741.  
  742.  
  743. #--------------------------
  744. # Name: run_cmd_with_timeout
  745. # Purpose:
  746. #   * Runs a command with timeout
  747. # Options:
  748. #   -e <rc test list> error message control. If any of the comma separated
  749. #      number tests are true an error message is generated
  750. #   -E <regex> error message control. If the regex matches the output, an error
  751. #      message is generated
  752. #   -n <rc test list> message control. If any of the return code tests are true:
  753. #      If -e is also used, a warning message is generated
  754. #      Otherwise (-w is also used), an error message is generated
  755. #   -N <regex> message control. If the regex matches the output:
  756. #      If -E is also used, a warning message is generated
  757. #      Otherwise (-W is also used), an error message is generated
  758. #   -t <timeout>. Same as the timeout command's DURATION: floating point number
  759. #      with an optional suffix: 's' for seconds (the default), 'm' for minutes,
  760. #      'h' for hours or 'd' for days.  Default 10 (seconds)
  761. #   -T <msg class> if the command times out, a <msg class> message is generated.
  762. #      <msg_class> must be I W or E
  763. #      Default: no message is generated on timeout.
  764. #   -w <rc test list> warning message control. If any of the return code tests
  765. #      are true: and no -E or -e option matches, a warning message
  766. #      is generated.
  767. #   -W <regex> warning message control. If the regex matches the output, and no
  768. #      -E or -e option matches, a warning message is generated
  769. #
  770. # rc_test_list format:
  771. #   One or more of > < >= <= != == followed by an unsigned integaer and
  772. #   separated by commas
  773. #
  774. # -E, -e, -N, -n, -W and -w usage notes:
  775. #
  776. #   * When none of these options is used:
  777. #     * stdout and stderr are ignored
  778. #     * return code 1 generates a warning message
  779. #     * return code >1 generates an error message
  780. #     That is equivalent to -e '>1' '-w==1'
  781. #   * When none of -E, -N and -W are used, stdout and stderr are ignored
  782. #   * When -n is used, -e or -w must also be used
  783. #   * When -N is used, -E or -W must also be used
  784. #   * When none of -e, -n and -w are used and some of -E, -N and -W are
  785. #     used, the return code is ignored
  786. #
  787. # Before calling this function:
  788. #   * Array ${cmd[*]} must be loaded with the command.
  789. #     Must be a simple command; cannot include | < > &
  790. #   * $out_fn and $rc_fn must contain writeable file paths.
  791. #     Any existing content will be deleted.
  792. #
  793. # This function's return codes:
  794. #   0 - No problem detected with the command
  795. #   1 - A warning message was generated (error messages are fatal)
  796. #   2 - The command timed out
  797. #
  798. # After calling this function caller could:
  799. #   * Examine this function's return code
  800. #   * Read the command's output from $out_fn
  801. #   * Read the command's return code from $rc_fn
  802. #
  803. # Example 1
  804. #    msg I "Checking the $user@$hostname ssh connection"
  805. #    cmd=(ssh "$user@$hostname" 'echo OK')
  806. #    run_cmd_with_timeout -t 5
  807. #    case $? in
  808. #        0 )
  809. #            msg I 'ssh connection OK'
  810. #            ;;
  811. #        1 | 2 )
  812. #            msg W 'ssh connection check failed'
  813. #            fct "${FUNCNAME[0]}" 'returning 1'
  814. #            return 1
  815. #    esac
  816. #
  817. # Global variables read:
  818. #   cmd
  819. #   false
  820. #   msg_lf
  821. #   out_fn
  822. #   rc_fn
  823. #   true
  824. # Global variables set: none
  825. # Output:
  826. #   * stdout and stderr to log or screen, either directly or via msg function
  827. # Returns: described above
  828. #--------------------------
  829. function run_cmd_with_timeout {
  830.     fct "${FUNCNAME[0]}" "started with arguments $*"
  831.     local OPTIND    # Required when getopts is called in a function
  832.     local args opt
  833.     local opt_e_flag opt_E_flag opt_n_flag opt_N_flag opt_w_flag opt_W_flag
  834.     local emsg_rc_test_list n_msg_rc_test_list warn_msg_test_list
  835.     local emsg_out_regex warn_msg_out_regex n_msg_out_regex
  836.     local duration timeout_msg_class
  837.     local -r duration_OK_regex='^[[:digit:]]+(|d|h|s)$'
  838.     local -r timeout_msg_class_OK_regex='^(E|I|W)$'
  839.     local buf emsg msg msg_class my_rc out rc rc_for_msg timed_out_flag
  840.  
  841.     # Parse options
  842.     # ~~~~~~~~~~~~~
  843.     args=("$@")
  844.     duration=10
  845.     emsg=
  846.     emsg_out_regex=
  847.     emsg_rc_test_list=
  848.     emsg_warn_regex=
  849.     n_msg_out_regex=
  850.     n_msg_rc_test_list=
  851.     opt_e_flag=$false
  852.     opt_E_flag=$false
  853.     opt_n_flag=$false
  854.     opt_N_flag=$false
  855.     opt_w_flag=$false
  856.     opt_W_flag=$false
  857.     timeout_msg_class=
  858.     warn_msg_out_regex=
  859.     warn_msg_rc_test_list=
  860.     while getopts :e:E:n:N:w:W:t:T: opt "$@"
  861.     do
  862.         case $opt in
  863.             e )
  864.                 opt_e_flag=$true
  865.                 emsg_rc_test_list=$OPTARG
  866.                 ;;
  867.             E )
  868.                 opt_E_flag=$true
  869.                 emsg_out_regex=$OPTARG
  870.                 ;;
  871.             n )
  872.                 opt_n_flag=$true
  873.                 n_msg_rc_test_list=$OPTARG
  874.                 ;;
  875.             N )
  876.                 opt_N_flag=$true
  877.                 n_msg_out_regex=$OPTARG
  878.                 ;;
  879.             t )
  880.                 duration=$OPTARG
  881.                 ;;
  882.             T )
  883.                 timeout_msg_class=$OPTARG
  884.                 ;;
  885.             w )
  886.                 opt_w_flag=$true
  887.                 warn_msg_rc_test_list=$OPTARG
  888.                 ;;
  889.             W )
  890.                 opt_W_flag=$true
  891.                 warn_msg_out_regex=$OPTARG
  892.                 ;;
  893.             : )
  894.                 emsg+=$msg_lf"Option $OPTARG must have an argument"
  895.                 ;;
  896.             * )
  897.                 emsg+=$msg_lf"Invalid option '-$OPTARG'"
  898.         esac
  899.     done
  900.     shift $(($OPTIND-1))
  901.  
  902.     # Test for mutually exclusive options
  903.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  904.     if [[ $opt_n_flag ]]; then
  905.         buf=$opt_e_flag$opt_n_flag$opt_w_flag
  906.         [[ $buf != '' && $buf != $true$true ]] \
  907.             && emsg+=$msg_lf'When -n is used, -e or -w must also be used'
  908.     fi
  909.     if [[ $opt_N_flag ]]; then
  910.         buf=$opt_E_flag$opt_N_flag$opt_W_flag
  911.         [[ $buf != '' && $buf != $true$true ]] \
  912.             && emsg+=$msg_lf'When -N is used, -E or -W must also be used'
  913.     fi
  914.  
  915.     # Test for mandatory options not set
  916.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  917.     # There are no mandatory options
  918.  
  919.     # Set derived default values
  920.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~
  921.     # If none of -E -e -N -n -W and -w were used, rc 1 generates a warning and
  922.     # rc>1 generates an error
  923.     if [[ $opt_e_flag$opt_n_flag$opt_w_flag = ''
  924.         && $opt_E_flag$opt_N_flag$opt_W_flag = '' ]]; then
  925.         emsg_rc_test_list='>1'
  926.         warn_msg_rc_test_list='==1'
  927.     fi
  928.  
  929.     # Validate option arguments
  930.     # ~~~~~~~~~~~~~~~~~~~~~~~~~
  931.     [[ ! $duration =~ $duration_OK_regex ]] \
  932.         && emsg+=$msg_lf"Invalid -t option $duration (does not match $duration_OK_regex)"
  933.     if [[ $timeout_msg_class != '' ]] \
  934.         && [[ ! $timeout_msg_class =~ $timeout_msg_class_OK_regex ]]; then
  935.              emsg+=$msg_lf"Invalid -T option $timeout_msg_class "
  936.              emsg+=$msg_lf" (does not match $timeout_msg_class_OK_regex)"
  937.     fi
  938.  
  939.     # Validate variables to be used
  940.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  941.     # Incidentally also ensure the files are empty
  942.     [[ ${cmd:-} = '' ]] && emsg+=$msg_lf'$cmd is required but is unset or empty'
  943.     if [[ ${out_fn:-} != '' ]]; then
  944.         echo -n > "$out_fn" 2>/dev/null
  945.         buf=$(ck_file "$out_fn" f:rw 2>&1)
  946.         [[ $buf != '' ]] && emsg+=$msg_lf$buf
  947.     else
  948.         emsg+=$msg_lf'$out_fn is required but is unset or empty'
  949.     fi
  950.     if [[ ${rc_fn:-} != '' ]]; then
  951.         echo -n > "$rc_fn" 2>/dev/null
  952.         buf=$(ck_file "$rc_fn" f:rw 2>&1)
  953.         [[ $buf != '' ]] && emsg+=$msg_lf$buf
  954.     else
  955.         emsg+=$msg_lf'$rc_fn is required but is unset or empty'
  956.     fi
  957.  
  958.     # Report any errors
  959.     # ~~~~~~~~~~~~~~~~~
  960.     if [[ $emsg != '' ]]; then
  961.         # These are programming error(s) so use error, not warning
  962.         msg E "${FUNCNAME[0]} called with ${args[*]}$emsg"
  963.     fi
  964.  
  965.     # Run the command with timeout
  966.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  967.     msg I "Running command with timeout: ${cmd[*]}"
  968.     msg_class=
  969.     timeout --signal=SIGTERM --kill-after=10 $duration "${cmd[@]}" > "$out_fn" 2>&1
  970.     rc=$?
  971.     if ((rc==124||rc==137)); then
  972.         timed_out_flag=$true
  973.         msg I 'Timed out'
  974.         rc_for_msg='not available (timed out)'
  975.         [[ $timeout_msg_class != '' ]] && msg_class=$timeout_msg_class
  976.     else
  977.         timed_out_flag=$false
  978.         echo $rc > "$rc_fn"
  979.         rc_for_msg=$rc
  980.     fi
  981.  
  982.     # Examine any return code if requested
  983.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  984.     # The return code is not available after timeout
  985.     if [[ ! $timed_out_flag ]]; then
  986.        if [[ $emsg_rc_test_list != '' ]] \
  987.            && run_rc_tests $emsg_rc_test_list; then
  988.            msg_class=E
  989.        elif [[ $warn_msg_rc_test_list != '' ]] \
  990.            && run_rc_tests $warn_msg_rc_test_list; then
  991.            [[ $msg_class != E ]] && msg_class=W
  992.        elif [[ $n_msg_rc_test_list != '' ]] \
  993.            && run_rc_tests $n_msg_rc_test_list; then
  994.            if [[ $emsg_rc_test_list != '' ]]; then
  995.                [[ $msg_class != E ]] && msg_class=W
  996.            else
  997.                msg_class=E
  998.            fi
  999.        fi
  1000.     fi
  1001.  
  1002.     # Examine any output if requested
  1003.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  1004.     out=$(< "$out_fn")
  1005.     if [[ $out != '' ]]; then
  1006.        if [[ $emsg_out_regex != '' && $out =~ $emsg_out_regex ]]; then
  1007.            msg_class=E
  1008.        elif [[ $warn_msg_out_regex != '' && $out =~ $warn_msg_out_regex ]]; then
  1009.            [[ $msg_class != E ]] && msg_class=W
  1010.        elif [[ $n_msg_out_regex != '' && $out =~ $n_msg_out_regex ]]; then
  1011.            if [[ $emsg_out_regex != '' ]]; then
  1012.                [[ $msg_class != E ]] && msg_class=W
  1013.            else
  1014.                msg_class=E
  1015.            fi
  1016.        fi
  1017.     fi
  1018.  
  1019.     # Generate message if required
  1020.     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  1021.     if [[ $msg_class != '' ]]; then
  1022.         msg="Command: ${cmd[*]}"
  1023.         msg+=$'\n'"rc: $rc_for_msg"
  1024.         msg+=$'\n'"Output: $out"
  1025.         msg $msg_class "$msg"
  1026.     fi
  1027.  
  1028.     # Set my return code
  1029.     # ~~~~~~~~~~~~~~~~~~
  1030.     #   0 - No problem detected with the command
  1031.     #   1 - A problem other than timeout was detected
  1032.     #   2 - The command timed out
  1033.     if [[ $timed_out_flag ]]; then
  1034.         my_rc=2
  1035.     elif [[ $msg_class = W ]]; then
  1036.         my_rc=1
  1037.     else
  1038.         my_rc=0
  1039.     fi
  1040.    
  1041.     fct "${FUNCNAME[0]}" "returning $my_rc"
  1042.     return $my_rc
  1043. }  #  end of function run_cmd_with_timeout
  1044.  
  1045. #--------------------------
  1046. # Name: run_rc_tests
  1047. # Purpose:
  1048. #   * Runs return code tests (for run_cmd_with_timeout)
  1049. # Arguments:
  1050. #   $1 - comma separated list of arithmentical tests
  1051. #
  1052. # Global variables read: rc
  1053. # Global variables set: none
  1054. # Output:
  1055. #   * stdout and stderr to log or screen, either directly or via msg function
  1056. # Returns:
  1057. #   0 - $rc matched one of the tests
  1058. #   1 - $rc did not match any of the tests
  1059. #--------------------------
  1060. function run_rc_tests {
  1061.     local array i numcom oldIFS one_char rhs two_char
  1062.     local -r two_char_numcom_regex='^(<|>|=|!)=$'
  1063.     local -r one_char_numcom_regex='^(<|>)$'
  1064.     local -r uint_regex='^[[:digit:]]+$'
  1065.  
  1066.     # Parse the argument
  1067.     # ~~~~~~~~~~~~~~~~~~
  1068.     oldIFS=$IFS
  1069.     IFS=,
  1070.     array=($1)
  1071.     IFS=$oldIFS
  1072.  
  1073.     # For each test
  1074.     # ~~~~~~~~~~~~~
  1075.     for ((i=0;i<${#array[*]};i++))
  1076.     do
  1077.        # Parse the test
  1078.        # ~~~~~~~~~~~~~~
  1079.        two_char=${array[i]:0:2}
  1080.        one_char=${array[i]:0:1}
  1081.        if [[ $two_char =~ $two_char_numcom_regex ]]; then
  1082.            numcom=$two_char
  1083.            rhs=${array[i]:2}
  1084.        elif [[ $one_char =~ $one_char_numcom_regex ]]; then
  1085.            numcom=$one_char
  1086.            rhs=${array[i]:1}
  1087.        else
  1088.            msg E "Programming error: invalid numerc comparison ${array[i]} in $1"
  1089.        fi
  1090.        [[ ! $rhs =~ $uint_regex ]] \
  1091.            &&  msg E "Programming error: invalid numerc comparison ${array[i]} in $1"
  1092.  
  1093.        # Test
  1094.        # ~~~~
  1095.        case $numcom in
  1096.            '<=' ) ((rc<=rhs)) && return 0 ;;
  1097.            '>=' ) ((rc>=rhs)) && return 0 ;;
  1098.            '==' ) ((rc==rhs)) && return 0 ;;
  1099.            '!=' ) ((rc!=rhs)) && return 0 ;;
  1100.            '<'  ) ((rc<rhs))  && return 0 ;;
  1101.            '>'  ) ((rc>rhs))  && return 0 ;;
  1102.        esac
  1103.  
  1104.     done
  1105.    
  1106.     return 1
  1107. }  #  end of function run_rc_tests
  1108.  
  1109.  
  1110. # Configuration for ck_mikrotik_routers_update.sh
  1111.  
  1112. # This file is sourced by bash
  1113. #   * Lines begining with # and empty lines ignored
  1114. #   * Remaining lines must be valid bash assignments
  1115.  
  1116. email_for_report=sysmail@example.com
  1117.  
  1118. # One array memmber for each MikroTik router
  1119. # Each member must be a colon separated list of:
  1120. #    * Router FQDN
  1121. #    * User name
  1122. #    * ssh private key
  1123. #    * Number of days failed to run a command on the router before
  1124. #      reporting by mail
  1125. mikrotik_routers=(
  1126.     amt02.example.com:nc:/home/nc/.ssh/nc_for_ck_mikrotik_routers_update:28
  1127.     bna02.example.com:nc:/home/nc/.ssh/nc_for_ck_mikrotik_routers_update:7
  1128.     cer02.example.com:/home/nc/.ssh/nc_for_ck_mikrotik_routers_update:1
  1129. )
  1130.  
  1131. # In case want to overide default of /var/log/ck_mikrotik_routers_update.log
  1132. #log_dir=/var/log/ck_mikrotik_routers_update.log
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top