SHARE
TWEET

Untitled

a guest May 22nd, 2019 67 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #
  2. # Monitors the and removes problematic objects
  3. #
  4. # To update:
  5. #   kubectl apply -f clean-kubernetes-slaves-cronjob.yaml
  6. #
  7. # To view status:
  8. #   kubectl describe cronjob/clean-kubernetes-slaves
  9. #
  10. # To view output from a specific cleanup run:
  11. #   kubectl logs job/clean-kubernetes-slaves-12345
  12. #
  13. apiVersion: batch/v1beta1
  14. kind: CronJob
  15. metadata:
  16.   name: clean-kubernetes-slaves-fclausen
  17. spec:
  18.   schedule: "*/10 * * * *"
  19.   concurrencyPolicy: Replace
  20.   successfulJobsHistoryLimit: 12
  21.   failedJobsHistoryLimit: 3
  22.   jobTemplate:
  23.     spec:
  24.       template:
  25.         spec:
  26.           serviceAccountName: jenkins-kubernetes-serviceaccount
  27.           containers:
  28.           - name: kubectl
  29.             image: 392477962641.dkr.ecr.us-east-1.amazonaws.com/saas/kubectl-docker:1219
  30.             command:
  31.             - /bin/bash
  32.             args:
  33.             - -c
  34.             - |
  35.               set -u
  36.  
  37.               echo "Looking for failed pods"
  38.               records=$(kubectl get pods --show-all --include-uninitialized | grep -e Error -e Completed -e ContainerCannotRun | grep -e h$ -e d$)
  39.               if [[ -n "$records" ]]; then
  40.                 echo "$records"
  41.                 echo "$records" | awk '{print $1}' | xargs kubectl delete pods
  42.               fi
  43.  
  44.               echo "Looking for orphaned ingresses"
  45.               records=$(kubectl get svc,ing -l ci-transient=true | grep -e d$)
  46.               if [ -n "$records" ]; then
  47.                 echo "$records"
  48.                 echo "$records" | awk '{print $1}' | xargs kubectl  delete
  49.               fi
  50.  
  51.               echo "Looking for orphaned deployments and services"
  52.               all_items=$(mktemp) # We'll save all items here
  53.               old_entries=$(mktemp) # Cleanup candidates saved in this file
  54.               max_age=43200 # 12 hours
  55.               kubectl get -o json deployments,service -l ci-transient=true > "$all_items"
  56.               # First find all items older than 12 hours by subtracting max_age from now, now
  57.               # being defined as $(date +%s).
  58.               jq --argjson max_age $(( $(date +%s) - $max_age )) -r \
  59.                 '[ .items[].metadata | .creationTimestamp | fromdate | tonumber | select( . < $max_age ) ] | unique' \
  60.                 "$all_items" > "$old_entries"
  61.               # Then use the timestamps from above to get the names of the orphan items
  62.               deletion_targets=$(mktemp)
  63.               jq -r '.[] | todate' "$old_entries" | while read -r timestamp; do
  64.                 jq -r --arg ts "$timestamp" \
  65.                   '.items[].metadata | select( .creationTimestamp == $ts ) | .name' \
  66.                   < "$all_items"
  67.               done > "$deletion_targets"
  68.  
  69.  
  70.               cat "$deletion_targets" | while read -r item; do
  71.                 echo "kubectl delete deployment,service $item"
  72.               done
  73.               rm "$old_entries" "$deletion_targets" "$all_items"
  74.           restartPolicy: OnFailure
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top