#!/bin/bash

# Script to cycle nova-api workers periodically
#
# NB: TEST THIS BEFORE RUNNING IT IN PRODUCTION
#
# The expected usage of this is from cron, scheduled to run
# as often as you want to kill off a single API worker. This
# will generate a list of nova-api workers and kill off one
# worker per run of the script. When it gets to the end, it
# regenerates the list and starts over. So, if you run this
# from cron every night at midnight, one nova-api worker will
# be killed off each day. Adjust this to suit your need for
# cycling them. Workers are chosen in order, not by any other
# metric like CPU time or memory footprint.

SERVICE_REGEX='nova-api'
STATE_FILE='/tmp/cycle-nova-api.state'
LOCKFILE="${STATE_FILE}.lock"

function get_master_pid() {
    systemctl show openstack-nova-api | grep ^MainPID | cut -d = -f 2
}

function refresh_state_file() {
    local master_pid=$(get_master_pid)

    pgrep -P "$master_pid" "$SERVICE_REGEX" > $STATE_FILE
}

function kill_next_victim() {
    local victim=$(head -n1 "$STATE_FILE")
    sed -i 1d "$STATE_FILE"
    pgrep "$SERVICE_REGEX" | grep -q $victim && kill $victim
}

(
    flock -s 200
    if [ ! -f "$STATE_FILE" ]; then
        # First run
        refresh_state_file
    elif ! grep -q '[0-9]' "$STATE_FILE"; then
        # We have cycled all workers, start over
        refresh_state_file
    fi

    kill_next_victim
) 200>$LOCKFILE