daily pastebin goal
3%
SHARE
TWEET

Untitled

a guest Feb 22nd, 2019 66 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/usr/bin/env python3
  2. #
  3. # Script to replace EC2 instances in an ECS cluster's auto-scaling group after
  4. # changing the AMI or instance type in the launch configuration. It
  5. # checks for instances with the incorrect AMI or type, scales up the
  6. # auto-scaling group with replacement instances, then drains the tasks
  7. # from the old instances.
  8. #
  9. # Usage: aws-vault exec profile-name -- python3 replace_ecs_cluster_instances.py --group=asg-name --cluster=ecs-cluster-name --count=3
  10. #
  11. # The count is specified  so that it knows what the "real" desired count is in
  12. # case it is interrupted and restarted after increasing the desired count.
  13. #
  14. # License: ISC
  15. # Copyright 2019 3D Robotics
  16. # Permission to use, copy, modify, and/or distribute this software for any
  17. # purpose with or without fee is hereby granted, provided that the above
  18. # copyright notice and this permission notice appear in all copies.
  19. #
  20. # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  21. # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  22. # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  23. # SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
  24. # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
  25. # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE
  26. # USE OR PERFORMANCE OF THIS SOFTWARE.
  27. #
  28.  
  29. import boto3
  30. import argparse
  31. import time
  32.  
  33. autoscaling = boto3.client('autoscaling')
  34. ec2 = boto3.client('ec2')
  35. ecs = boto3.client('ecs')
  36.  
  37. parser = argparse.ArgumentParser(description='Update the instances in the autoscaling group.')
  38. parser.add_argument('--group', metavar='NAME', required=True, help='Autoscaling group name')
  39. parser.add_argument('--cluster', metavar='NAME', required=True, help='ECS cluster name')
  40. parser.add_argument('--count', metavar='ARN', type=int, required=True, help='Desired count of instances in autoscaling group')
  41. args = parser.parse_args()
  42.  
  43. group_name = args.group
  44. cluster_name = args.cluster
  45. desired_count = args.count
  46.  
  47. describe_group = autoscaling.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0]
  48. assert(describe_group['AutoScalingGroupName'] == group_name)
  49.  
  50. desired_capacity = describe_group['DesiredCapacity']
  51. target_launch_configuration = describe_group['LaunchConfigurationName']
  52. asg_instances = [instance['InstanceId'] for instance in describe_group['Instances']]
  53. prev_desired_count = describe_group['DesiredCapacity']
  54.  
  55. print(f"Target launch configuration {target_launch_configuration}")
  56.  
  57. describe_launch_configuration = autoscaling.describe_launch_configurations(LaunchConfigurationNames=[target_launch_configuration])['LaunchConfigurations'][0]
  58.  
  59. target_ami = describe_launch_configuration['ImageId']
  60. target_instance_type = describe_launch_configuration['InstanceType']
  61.  
  62. print(f"Target AMI {target_ami} on {target_instance_type}")
  63.  
  64. describe_instances_reservations = ec2.describe_instances(InstanceIds=asg_instances)['Reservations']
  65.  
  66. instances_to_replace = []
  67.  
  68. for reservation in describe_instances_reservations:
  69.     for instance in reservation['Instances']:
  70.         instance_id = instance['InstanceId']
  71.         instance_ami = instance['ImageId']
  72.         instance_type = instance['InstanceType']
  73.         instance_launched = instance['LaunchTime']
  74.        
  75.         needs_replace = instance_ami != target_ami or instance_type !=target_instance_type
  76.        
  77.         if needs_replace:
  78.             instances_to_replace.append(instance_id)
  79.        
  80.         print(f"Instance {instance_id}, created {instance_launched.ctime()}, type {instance_type}, AMI {instance_ami} -- {'REPLACE' if needs_replace else 'OK'}")
  81.  
  82. new_desired_count = max(prev_desired_count, desired_count + len(instances_to_replace))
  83. print(f"Temporarily scaling cluster from {prev_desired_count} to {new_desired_count} instances")
  84. autoscaling.set_desired_capacity(AutoScalingGroupName=group_name, DesiredCapacity=new_desired_count)
  85.  
  86. while True:
  87.     print('\n----\n')
  88.    
  89.     list_container_instances = ecs.list_container_instances(cluster = cluster_name)['containerInstanceArns']
  90.     container_instances = ecs.describe_container_instances(cluster = cluster_name, containerInstances = list_container_instances)['containerInstances']
  91.    
  92.     container_instances.sort(key = lambda ci: ci['registeredAt'])
  93.    
  94.     available_instances = 0
  95.     remaining_tasks = 0
  96.     to_drain = []
  97.    
  98.     for ci in container_instances:
  99.         ci_ec2_id = ci['ec2InstanceId']
  100.         ci_arn = ci['containerInstanceArn']
  101.         running_tasks = ci['runningTasksCount']
  102.         status = ci['status']
  103.        
  104.         print(f"{ci_ec2_id} {status}, {running_tasks} tasks")
  105.        
  106.         if ci_ec2_id in instances_to_replace:
  107.             remaining_tasks += running_tasks
  108.             if status == 'ACTIVE':
  109.                 to_drain.append(ci_arn)
  110.         elif status == 'ACTIVE':
  111.             available_instances += 1
  112.  
  113.     if available_instances < desired_count:
  114.         print("Waiting for new instances to boot")
  115.     elif len(to_drain) > 0:
  116.         print("Draining instances:", to_drain)
  117.         ecs.update_container_instances_state(cluster = cluster_name, containerInstances = to_drain, status='DRAINING')
  118.     elif remaining_tasks == 0:
  119.         break
  120.     else:
  121.         print("Waiting for instances to drain")
  122.  
  123.     time.sleep(10)
  124.  
  125. for instance_id in instances_to_replace:
  126.     if input(f"Terminate instance {instance_id}? ") == "y":
  127.         autoscaling.terminate_instance_in_auto_scaling_group(InstanceId=instance_id, ShouldDecrementDesiredCapacity=True)
  128.  
  129. print("Done")
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top