Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def call(Map config) {
- // hc - health check url
- // av - artifact version url
- if [[ "${config.ENV}" == "PROD"]]; then
- hc="https://ui.smrt.vip.nordstrom.com/health"
- av="https://ui.smrt.vip.nordstrom.com/info"
- else
- hc="https://ui-${config.ENV}.nonprod.smrt.vip.nordstrom.com/health"
- av="https://ui-${config.ENV}.nonprod.smrt.vip.nordstrom.com/v.html"
- fi
- // if [ "${new_env}" = "1" ]; then
- // sleep 30
- // cd ${WORKSPACE}
- // echo "Starting liquibase DB migration"
- // // Apply migration
- // echo
- // echo "**** APPLYING MIGRATION smrt-liquibase-${VERSION}.jar TO ${smrt_db_url} ****"
- // echo
- // java -jar -Dsmrt_db_url=${smrt_db_url} -Dsmrt_db_username=${smrt_db_username} -Dsmrt_db_password=${smrt_db_password} ./smrt-liquibase-${VERSION}.jar
- // echo
- // echo "Finished liquibase DB migration"
- // fi
- // Wait for application up
- // hc - health check url
- // av - artifact version url
- echo "Starting check-loop"
- echo "Health check ${hc}"
- echo "Version page ${av}"
- while sleep 20
- do
- echo "Requesting /v.html..."
- curl --max-time 10 -kLsD - ${av}
- echo
- echo "Requesting /health..."
- curl --max-time 10 -kLsD - ${hc}
- echo
- curl -m 10 -kLsD - ${hc} | grep "HTTP/1.1 200" && curl -m 10 -kLsD - ${hc} | grep "UP" && curl -m 10 -kLs ${av} | grep "${VERSION}" && break
- done
- echo "Check-loop finished"
- echo "Final check"
- success=0
- curl -m 10 -kLsD - ${hc} | grep "HTTP/1.1 200" && curl -m 10 -kLsD - ${hc} | grep "UP" && curl -m 10 -kLs ${av} | grep "${VERSION}" && success=1
- echo
- echo "Getting logs for the components"
- for SERVICE in ${config.SERVICES}; do
- TAG="${SERVICE}-${config.ENV}"
- HOST=$(aws ec2 describe-instances --query "Reservations[*].Instances[*].PrivateIpAddress" --output=text --filter Name=tag:Name,Values=${TAG})
- if [ -n "${HOST}" ]; then
- nc -w 3 ${HOST} 22 </dev/null && res="0" || res="1"
- if [ "$res" != "1" ]; then
- NAME="${SERVICE}-${config.ENV}-${HOST}-$(date +%Y-%m-%d).log.txt"
- echo "${TAG} ${HOST} ${NAME}"
- ssh -o StrictHostKeyChecking=no ec2-user@${HOST} -tt "sudo docker ps -a | grep -v ecs-agent | head -2 | tail -1 | awk '{ print \$1 }' | xargs sudo docker logs --tail 500" > ${WORKSPACE}/${NAME}
- if [ "${success}" != "1" ]; then
- ssh -o StrictHostKeyChecking=no ec2-user@${HOST} -t "sudo reboot"
- fi
- fi
- fi
- done
- # Check APIGateway status
- echo -e "\033[33m Checking APIGateway status \033[0m"
- URL="https://api-${config.ENV}.nonprod.smrt.vip.nordstrom.com/smrt/v1/inventory"
- HTTP_RESPONSE='{"Message":"User is not authorized to access this resource with an explicit deny"}'
- HTTP_STATUS=$(curl -s -k -X POST -d '{"rmsskuid":"89056771"}' ${URL})
- if [[ $HTTP_STATUS == $HTTP_RESPONSE ]]; then
- echo -e "\033[32m APIGateway status $HTTP_STATUS \033[0m"
- else
- echo -e "\033[31m APIGateway doesn't work \033[0m"
- curl -v -s -k -X POST -d '{"rmsskuid":"89056771"}' ${URL}
- echo -e "\033[31m expected HTTP_RESPONSE = $HTTP_RESPONSE, got $HTTP_STATUS \033[0m"
- echo -e "\033[31m Please see more about this issue on KB in Post deployment issue https://confluence.nordstrom.net/display/ITMF/API+gateway+implementation+details \033[0m"
- exit 1
- fi
- # Check CloudWatch events targets
- echo -e "\033[33m Check CloudWatch events targets \033[0m"
- RULES=$(aws events list-rules | jq -r ".Rules[].Name" | egrep "*-${config.ENV}-.\$")
- for rule in ${RULES}; do
- event_count=$(aws events list-targets-by-rule --rule ${rule} | jq ".Targets[].Id" | wc -l)
- if [ ${event_count} -gt 1 ]; then
- echo -e "\033[31m CloudWatch targets more than one! Please check CloudWatch Events Rules!!! \033[0m"
- exit 1
- else
- echo -e "\033[32m Cloud watch rule ${rule} has ${event_count} targets \033[0m"
- fi
- done
- # Servces health checks
- echo -e "\033[33m Check services health checks \033[0m"
- LB=$(aws elb describe-load-balancers | jq -r ".LoadBalancerDescriptions[].DNSName" | egrep ".*-${config.ENV}-[0-9].*")
- for service in ${LB}; do
- res=$(curl --retry 3 --connect-timeout 10 -k -L -so /dev/null -w '%{response_code}' http://${service}:8080/health)
- if [ "$res" != "200" ] && [ "$res" != "503" ]; then
- echo -e "\033[31m Service ${service} is down!!! HTTP Response = ${res} \033[0m"
- exit 1
- else
- echo -e "\033[32m Service ${service} is UP \033[0m"
- fi
- done
- }
- return this
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement