#
# Monitors the and removes problematic objects
#
# To update:
#   kubectl apply -f clean-kubernetes-slaves-cronjob.yaml
#
# To view status:
#   kubectl describe cronjob/clean-kubernetes-slaves
#
# To view output from a specific cleanup run:
#   kubectl logs job/clean-kubernetes-slaves-12345
#
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: clean-kubernetes-slaves-fclausen
spec:
  schedule: "*/10 * * * *"
  concurrencyPolicy: Replace
  successfulJobsHistoryLimit: 12
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: jenkins-kubernetes-serviceaccount
          containers:
          - name: kubectl
            image: 392477962641.dkr.ecr.us-east-1.amazonaws.com/saas/kubectl-docker:1219
            command:
            - /bin/bash
            args:
            - -c
            - |
              set -u

              echo "Looking for failed pods"
              records=$(kubectl get pods --show-all --include-uninitialized | grep -e Error -e Completed -e ContainerCannotRun | grep -e h$ -e d$)
              if [[ -n "$records" ]]; then
                echo "$records"
                echo "$records" | awk '{print $1}' | xargs kubectl delete pods
              fi

              echo "Looking for orphaned ingresses"
              records=$(kubectl get svc,ing -l ci-transient=true | grep -e d$)
              if [ -n "$records" ]; then
                echo "$records"
                echo "$records" | awk '{print $1}' | xargs kubectl  delete
              fi

              echo "Looking for orphaned deployments and services"
              all_items=$(mktemp) # We'll save all items here
              old_entries=$(mktemp) # Cleanup candidates saved in this file
              max_age=43200 # 12 hours
              kubectl get -o json deployments,service -l ci-transient=true > "$all_items"
              # First find all items older than 12 hours by subtracting max_age from now, now
              # being defined as $(date +%s).
              jq --argjson max_age $(( $(date +%s) - $max_age )) -r \
                '[ .items[].metadata | .creationTimestamp | fromdate | tonumber | select( . < $max_age ) ] | unique' \
                "$all_items" > "$old_entries"
              # Then use the timestamps from above to get the names of the orphan items
              deletion_targets=$(mktemp)
              jq -r '.[] | todate' "$old_entries" | while read -r timestamp; do
                jq -r --arg ts "$timestamp" \
                  '.items[].metadata | select( .creationTimestamp == $ts ) | .name' \
                  < "$all_items"
              done > "$deletion_targets"


              cat "$deletion_targets" | while read -r item; do
                echo "kubectl delete deployment,service $item"
              done
              rm "$old_entries" "$deletion_targets" "$all_items"
          restartPolicy: OnFailure