Advertisement
Guest User

Untitled

a guest
Apr 4th, 2018
138
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.81 KB | None | 0 0
  1. def call(Map config) {
  2.  
  3. // hc - health check url
  4. // av - artifact version url
  5. if [[ "${config.ENV}" == "PROD"]]; then
  6. hc="https://ui.smrt.vip.nordstrom.com/health"
  7. av="https://ui.smrt.vip.nordstrom.com/info"
  8. else
  9. hc="https://ui-${config.ENV}.nonprod.smrt.vip.nordstrom.com/health"
  10. av="https://ui-${config.ENV}.nonprod.smrt.vip.nordstrom.com/v.html"
  11. fi
  12.  
  13.  
  14. // if [ "${new_env}" = "1" ]; then
  15. // sleep 30
  16. // cd ${WORKSPACE}
  17. // echo "Starting liquibase DB migration"
  18.  
  19. // // Apply migration
  20. // echo
  21. // echo "**** APPLYING MIGRATION smrt-liquibase-${VERSION}.jar TO ${smrt_db_url} ****"
  22. // echo
  23.  
  24. // java -jar -Dsmrt_db_url=${smrt_db_url} -Dsmrt_db_username=${smrt_db_username} -Dsmrt_db_password=${smrt_db_password} ./smrt-liquibase-${VERSION}.jar
  25.  
  26. // echo
  27. // echo "Finished liquibase DB migration"
  28. // fi
  29.  
  30. // Wait for application up
  31. // hc - health check url
  32. // av - artifact version url
  33.  
  34. echo "Starting check-loop"
  35. echo "Health check ${hc}"
  36. echo "Version page ${av}"
  37.  
  38. while sleep 20
  39. do
  40. echo "Requesting /v.html..."
  41. curl --max-time 10 -kLsD - ${av}
  42. echo
  43.  
  44. echo "Requesting /health..."
  45. curl --max-time 10 -kLsD - ${hc}
  46. echo
  47.  
  48. curl -m 10 -kLsD - ${hc} | grep "HTTP/1.1 200" && curl -m 10 -kLsD - ${hc} | grep "UP" && curl -m 10 -kLs ${av} | grep "${VERSION}" && break
  49. done
  50. echo "Check-loop finished"
  51.  
  52. echo "Final check"
  53. success=0
  54. curl -m 10 -kLsD - ${hc} | grep "HTTP/1.1 200" && curl -m 10 -kLsD - ${hc} | grep "UP" && curl -m 10 -kLs ${av} | grep "${VERSION}" && success=1
  55. echo
  56.  
  57. echo "Getting logs for the components"
  58. for SERVICE in ${config.SERVICES}; do
  59. TAG="${SERVICE}-${config.ENV}"
  60. HOST=$(aws ec2 describe-instances --query "Reservations[*].Instances[*].PrivateIpAddress" --output=text --filter Name=tag:Name,Values=${TAG})
  61. if [ -n "${HOST}" ]; then
  62. nc -w 3 ${HOST} 22 </dev/null && res="0" || res="1"
  63. if [ "$res" != "1" ]; then
  64. NAME="${SERVICE}-${config.ENV}-${HOST}-$(date +%Y-%m-%d).log.txt"
  65. echo "${TAG} ${HOST} ${NAME}"
  66. ssh -o StrictHostKeyChecking=no ec2-user@${HOST} -tt "sudo docker ps -a | grep -v ecs-agent | head -2 | tail -1 | awk '{ print \$1 }' | xargs sudo docker logs --tail 500" > ${WORKSPACE}/${NAME}
  67. if [ "${success}" != "1" ]; then
  68. ssh -o StrictHostKeyChecking=no ec2-user@${HOST} -t "sudo reboot"
  69. fi
  70. fi
  71. fi
  72. done
  73.  
  74. # Check APIGateway status
  75. echo -e "\033[33m Checking APIGateway status \033[0m"
  76. URL="https://api-${config.ENV}.nonprod.smrt.vip.nordstrom.com/smrt/v1/inventory"
  77. HTTP_RESPONSE='{"Message":"User is not authorized to access this resource with an explicit deny"}'
  78. HTTP_STATUS=$(curl -s -k -X POST -d '{"rmsskuid":"89056771"}' ${URL})
  79.  
  80. if [[ $HTTP_STATUS == $HTTP_RESPONSE ]]; then
  81. echo -e "\033[32m APIGateway status $HTTP_STATUS \033[0m"
  82. else
  83. echo -e "\033[31m APIGateway doesn't work \033[0m"
  84. curl -v -s -k -X POST -d '{"rmsskuid":"89056771"}' ${URL}
  85. echo -e "\033[31m expected HTTP_RESPONSE = $HTTP_RESPONSE, got $HTTP_STATUS \033[0m"
  86. echo -e "\033[31m Please see more about this issue on KB in Post deployment issue https://confluence.nordstrom.net/display/ITMF/API+gateway+implementation+details \033[0m"
  87. exit 1
  88. fi
  89.  
  90. # Check CloudWatch events targets
  91. echo -e "\033[33m Check CloudWatch events targets \033[0m"
  92.  
  93. RULES=$(aws events list-rules | jq -r ".Rules[].Name" | egrep "*-${config.ENV}-.\$")
  94. for rule in ${RULES}; do
  95. event_count=$(aws events list-targets-by-rule --rule ${rule} | jq ".Targets[].Id" | wc -l)
  96. if [ ${event_count} -gt 1 ]; then
  97. echo -e "\033[31m CloudWatch targets more than one! Please check CloudWatch Events Rules!!! \033[0m"
  98. exit 1
  99. else
  100. echo -e "\033[32m Cloud watch rule ${rule} has ${event_count} targets \033[0m"
  101. fi
  102. done
  103.  
  104. # Servces health checks
  105. echo -e "\033[33m Check services health checks \033[0m"
  106. LB=$(aws elb describe-load-balancers | jq -r ".LoadBalancerDescriptions[].DNSName" | egrep ".*-${config.ENV}-[0-9].*")
  107. for service in ${LB}; do
  108. res=$(curl --retry 3 --connect-timeout 10 -k -L -so /dev/null -w '%{response_code}' http://${service}:8080/health)
  109. if [ "$res" != "200" ] && [ "$res" != "503" ]; then
  110. echo -e "\033[31m Service ${service} is down!!! HTTP Response = ${res} \033[0m"
  111. exit 1
  112. else
  113. echo -e "\033[32m Service ${service} is UP \033[0m"
  114. fi
  115. done
  116. }
  117. return this
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement