API tools faq
paste
Advertisement
funcelot

utils.py

funcelot
Feb 4th, 2020
249
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 13.97 KB | None | 0 0
raw download clone embed print report
  1. from __future__ import division, print_function, unicode_literals
  2.  
  3. import json
  4. import argparse
  5. import yaml
  6. import os
  7. import pprint
  8. import subprocess
  9. import sys
  10. import utils
  11.  
  12. from .arg_parse import Arg_Parse_Yaml
  13.  
  14. class Command:
  15.     def __init__(self):
  16.         self.load_params()
  17.         self.script_path = os.path.dirname(os.path.abspath(__file__))
  18.         self.current_path=os.path.abspath(os.curdir)
  19.  
  20.  
  21.     def sh(self, args, print_output=True, shell=True, cwd=None):
  22.         try:
  23.             try:
  24.                 print(args)
  25.                 if cwd != None:
  26.                     os.chdir(cwd)
  27.                 else:
  28.                     if self.current_path == os.getcwd():
  29.                         os.chdir(os.path.abspath(os.path.curdir))
  30.                     else:
  31.                         os.chdir(os.path.abspath(self.current_path))
  32.                 p = subprocess.Popen(args=args.split(), encoding='utf-8', stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, cwd=cwd)
  33.                 out, err = p.communicate()
  34.                 p.stdout.close()
  35.                 if out:
  36.                     print(out)
  37.                 if err:
  38.                     print(err)
  39.                 return out
  40.             except Exception as e:
  41.                 print(e)
  42.                 return subprocess.check_output(args, shell=shell).decode('utf-8')
  43.         except OSError as ex:
  44.             print(ex)
  45.             sys.exit(32)
  46.    
  47.  
  48.     def sh_inline(self, args, print_output=True, shell=True):
  49.         try:
  50.             return self.sh(args, print_output=print_output, shell=shell).replace('\n','')
  51.         except OSError as ex:
  52.             print(ex)
  53.             sys.exit(32)
  54.  
  55.  
  56.     def access_token(self):
  57.         from oauthlib.oauth2 import BackendApplicationClient
  58.         from requests_oauthlib import OAuth2Session
  59.         APP_TENTANT_ID = os.environ.get("APP_TENTANT_ID",'')
  60.         CLIENT_ID= os.environ.get("CLIENT_ID",'')
  61.         CLIENT_SECRET= os.environ.get("CLIENT_SECRET",'')
  62.         client = BackendApplicationClient(client_id=CLIENT_ID)
  63.         oauth = OAuth2Session(client=client)
  64.         token_url = "https://login.microsoftonline.com/{APP_TENTANT_ID}/oauth2/token"
  65.         print(token_url)
  66.         json_token = oauth.fetch_token(token_url=token_url, client_id=CLIENT_ID, client_secret=CLIENT_SECRET)
  67.         token = dict(json_token)['access_token']
  68.         print(token)
  69.  
  70.     def get_args_command(self):
  71.         return self.params['Command']
  72.        
  73.     def load_params(self):
  74.         arg_Parse_Yaml = Arg_Parse_Yaml()
  75.         script_path = os.path.dirname(os.path.abspath(__file__))
  76.         args_file = "/{0}/../args.yaml".format(script_path)
  77.         # print(args_file)
  78.         params_yaml = arg_Parse_Yaml.get_parameters_yaml(args_file)
  79.         # print(params_yaml)
  80.         # pprint.pprint(params_yaml)
  81.         #NOW READ IN LOCAL ONLY NOT IN SOURCE CONTROL FOR PASSWORDS IF LOCAL ONLY
  82.         #IF CI/CD PASS COME FROM KEY VAULT OR CIRCLECI
  83.         args_file = "/{0}/../../bin/local_only/local_only.yaml".format(script_path)
  84.         # print(args_file)
  85.         if os.path.isfile(args_file):
  86.             secret_params_yaml = arg_Parse_Yaml.get_parameters_yaml(args_file)
  87.             # print(secret_params_yaml)
  88.             # pprint.pprint(secret_params_yaml)
  89.             params_yaml.update(secret_params_yaml)
  90.         self.params = params_yaml
  91.    
  92.     def set_all_vars(self):
  93.         keys={}
  94.         for key, val in self.params.items():
  95.             # print(key, val)
  96.             os.environ[key]=val
  97.             os.putenv(key,val)
  98.             if key != "Command":
  99.                 keys[key]=val
  100.  
  101.         os.environ["VERSION_NUMBER"]=os.environ["DEBUG"]
  102.         os.environ["NAME"]=os.environ['PROJECT_NAME']
  103.  
  104.         os.environ["VERSION"]="{}.{}.{}".format(os.environ['MAJOR'],os.environ['MINOR'],"1213")
  105.         os.environ["ROUTE"]="/{}".format(os.environ['PROJECT_NAME'])
  106.         os.environ["CONTAINER_NAME"]="{}/{}".format(os.environ['DOCKER_REPO'], os.environ['PROJECT_NAME'])
  107.         os.environ["REPO_SERVER"]="{}.azurecr.io".format(os.environ['REPO'])
  108.         os.environ["CONTAINER_FULL_NAME"]="{}/{}:{}".format(os.environ['DOCKER_REPO'], os.environ['PROJECT_NAME'], os.environ['VERSION'])
  109.         os.environ["TAG"]="{}/{}".format(os.environ['REPO_SERVER'], os.environ['CONTAINER_FULL_NAME'])
  110.         os.environ["BASE_IMAGE"]="{}/{}/base_crud-api:{}".format(os.environ['REPO_SERVER'], os.environ['DOCKER_REPO'], os.environ['BASE_IMAGE_VERSION'])
  111.  
  112.         os.putenv("VERSION",os.environ["VERSION"])
  113.         os.putenv("ROUTE",os.environ["ROUTE"])
  114.         os.putenv("CONTAINER_NAME",os.environ["CONTAINER_NAME"])
  115.         os.putenv("REPO_SERVER",os.environ["REPO_SERVER"])
  116.         os.putenv("CONTAINER_FULL_NAME",os.environ["CONTAINER_FULL_NAME"])
  117.         os.putenv("TAG",os.environ["TAG"])
  118.         os.putenv("BASE_IMAGE",os.environ["BASE_IMAGE"])
  119.         os.putenv("HOST_URL",os.environ["HOST_URL"])
  120.  
  121.         keys["VERSION"]=os.environ["VERSION"]
  122.         keys["ROUTE"]=os.environ["ROUTE"]
  123.         keys["CONTAINER_NAME"]=os.environ["CONTAINER_NAME"]
  124.         keys["REPO_SERVER"]=os.environ["REPO_SERVER"]
  125.         keys["CONTAINER_FULL_NAME"]=os.environ["CONTAINER_FULL_NAME"]
  126.         keys["TAG"]=os.environ["TAG"]
  127.         keys["BASE_IMAGE"]=os.environ["BASE_IMAGE"]
  128.         keys["HOST_URL"]=os.environ["HOST_URL"]
  129.        
  130.         env_file = self.current_path + "/all_vars.sh"
  131.         with open(env_file, mode='wb') as f:
  132.             for key, val in keys.items():
  133.                 f.write("export {}='{}'\n".format(key,val).encode("utf-8"))
  134.  
  135.     def set_dirs(self):
  136.         self.set_all_vars()
  137.  
  138.         os.environ["BUILD_SCRIPTS_DIR"] = "{0}/../../bin/".format(self.script_path)
  139.         os.environ["PROJECT_DIR"] = "{0}/../../".format(self.script_path)
  140.         os.environ["BUILD_LIB_DIR"]="{0}/../../bin/build_lib/".format(self.script_path)
  141.         os.environ["ACCEPTANCE_TEST_DIR"]="{0}/../../acceptance/".format(self.script_path)
  142.         os.environ["APPLICATION_DIR"]="{0}/../../app/".format(self.script_path)
  143.         os.environ["AZURE_SCRIPTS_DIR"]="{0}/../../azure/".format(self.script_path)
  144.         os.environ["VENV"]="{}venv/{}".format(os.environ["PROJECT_DIR"], os.environ["PROJECT_NAME"])
  145.  
  146.         os.putenv("BUILD_SCRIPTS_DIR",os.environ["BUILD_SCRIPTS_DIR"])
  147.         os.putenv("PROJECT_DIR",os.environ["PROJECT_DIR"])
  148.         os.putenv("BUILD_LIB_DIR",os.environ["BUILD_LIB_DIR"])
  149.         os.putenv("ACCEPTANCE_TEST_DIR",os.environ["ACCEPTANCE_TEST_DIR"])
  150.         os.putenv("APPLICATION_DIR",os.environ["APPLICATION_DIR"])
  151.         os.putenv("AZURE_SCRIPTS_DIR",os.environ["AZURE_SCRIPTS_DIR"])
  152.  
  153.         keys={}
  154.         keys["BUILD_SCRIPTS_DIR"]=os.environ["BUILD_SCRIPTS_DIR"]
  155.         keys["PROJECT_DIR"]=os.environ["PROJECT_DIR"]
  156.         keys["BUILD_LIB_DIR"]=os.environ["BUILD_LIB_DIR"]
  157.         keys["ACCEPTANCE_TEST_DIR"]=os.environ["ACCEPTANCE_TEST_DIR"]
  158.         keys["APPLICATION_DIR"]=os.environ["APPLICATION_DIR"]
  159.         keys["AZURE_SCRIPTS_DIR"]=os.environ["AZURE_SCRIPTS_DIR"]
  160.  
  161.         env_file = self.current_path + "/dirs.sh"
  162.         with open(env_file, mode='wb') as f:
  163.             for key, val in keys.items():
  164.                 f.write("export {}='{}'\n".format(key,val).encode("utf-8"))
  165.    
  166.     def set_venv(self):
  167.         self.set_dirs()
  168.  
  169.         os.environ["VENV"]="{}venv/{}".format(os.environ["PROJECT_DIR"], os.environ["PROJECT_NAME"])
  170.         os.putenv("VENV",os.environ["VENV"])
  171.  
  172.         keys={}
  173.         keys["VENV"]=os.environ["VENV"]
  174.  
  175.         env_file = self.current_path + "/venv.sh"
  176.         with open(env_file, mode='wb') as f:
  177.             for key, val in keys.items():
  178.                 f.write("export {}='{}'\n".format(key,val).encode("utf-8"))
  179.  
  180.     def with_venv (self):
  181.         self.set_venv()
  182.  
  183.  
  184.     def set_project (self):
  185.         self.set_dirs()
  186.  
  187.         os.environ["PROJECT_NAME"]="{}venv/{}".format(os.environ["PROJECT_DIR"], os.environ["PROJECT_NAME"])
  188.         os.putenv("VENV",os.environ["VENV"])
  189.  
  190.         keys={}
  191.         keys["VENV"]=os.environ["VENV"]
  192.  
  193.         env_file = self.current_path + "/venv.sh"
  194.         with open(env_file, mode='wb') as f:
  195.             for key, val in keys.items():
  196.                 f.write("export {}='{}'\n".format(key,val).encode("utf-8"))
  197.  
  198.     def get_sudo_user(self):
  199.         SUDO = "sudo"
  200.         whoami = self.sh_inline("sudo -S whoami", print_output=False)
  201.         if whoami == "root":
  202.             SUDO = ""
  203.         return SUDO
  204.  
  205.     def az_install(self):
  206.         # Set sudo to work whether logged in as root user or non-root user
  207.         SUDO = self.get_sudo_user()
  208.         which_az = self.sh_inline("which az", print_output=False)
  209.         if len(which_az)==0:
  210.             self.sh(f"{SUDO} apt-get update && {SUDO} apt-get -qqy install apt-transport-https")
  211.    
  212.             command_lsb_release = self.sh_inline("command -v lsb_release", print_output=False)
  213.             if len(command_lsb_release)==0:
  214.                 print("Installing lsb_release")
  215.                 self.sh(f"{SUDO} apt-get -qqy install lsb-release")
  216.  
  217.             AZ_REPO=self.sh_inline("lsb_release -cs", print_output=False)
  218.            
  219.             # Modify your sources list
  220.             self.sh(f'echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ {AZ_REPO} main" | {SUDO} tee /etc/apt/sources.list.d/azure-cli.list')
  221.    
  222.             command_curl = self.sh_inline("command -v curl", print_output=False)
  223.             if len(command_lsb_release)==0:
  224.                 print("Installing curl")
  225.                 self.sh(f"{SUDO}  apt-get -qqy install curl")
  226.  
  227.             # Get the Microsoft signing key
  228.             self.sh(f"curl -L https://packages.microsoft.com/keys/microsoft.asc | {SUDO} apt-key add -")
  229.  
  230.             # Update and install the Azure CLI
  231.            
  232.             self.sh(f"{SUDO} apt-get update")
  233.             self.sh(f"{SUDO} apt-get -qqy install ca-certificates azure-cli")
  234.            
  235.             print("Azure CLI is now installed.")
  236.            
  237.         else:
  238.             print("Azure CLI installed already.")
  239.  
  240.     def az_login(self):
  241.         if len(os.getenv("AZURE_USERNAME","")) > 0:
  242.             print("User credentials detected; logging in with user")
  243.             self.sh("az login -u $AZURE_USERNAME -p $AZURE_PASSWORD")
  244.             self.sh("az acr login -n pmtolk")
  245.  
  246.         elif len(os.getenv("AZURE_SP","")) > 0:
  247.             print("Service Principal credentials detected; logging in with Service Principal")
  248.             self.sh("az login --service-principal --tenant $AZURE_SP_TENANT -u $AZURE_SP -p $AZURE_SP_PASSWORD")
  249.             self.sh("az acr login -n pmtolk")
  250.         else:
  251.             print("Login failed; neither user nor Service Principal credentials were provided")
  252.             sys.exit(1)
  253.  
  254.     def install_odbc(self):
  255.         # Set sudo to work whether logged in as root user or non-root user
  256.         SUDO = self.get_sudo_user()
  257.  
  258.         self.sh(f"{SUDO} curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - ")
  259.        
  260.         #Download appropriate package for the OS version
  261.         #Choose only ONE of the following, corresponding to your OS version
  262.        
  263.         self.sh(f"{SUDO} curl https://packages.microsoft.com/config/ubuntu/18.04/prod.list > /etc/apt/sources.list.d/mssql-release.list")
  264.         self.sh(f"{SUDO} apt-get update")
  265.         self.sh(f"{SUDO} ACCEPT_EULA=Y apt-get -y install msodbcsql17")
  266.  
  267.         # optional: for bcp and sqlcmd
  268.         self.sh(f"{SUDO} ACCEPT_EULA=Y apt-get -y install mssql-tools")
  269.         self.sh("echo 'export PATH=\"\$PATH:/opt/mssql-tools/bin\"' >> ~/.bash_profile")
  270.         self.sh("echo 'export PATH=\"\$PATH:/opt/mssql-tools/bin\"' >> ~/.bashrc")
  271.        
  272.         self.sh("source ~/.bashrc")
  273.        
  274.         # optional: for unixODBC development headers
  275.         self.sh(f"{SUDO} apt-get install -y unixodbc-dev")
  276.  
  277.     def build(self):
  278.  
  279.         PROJECT_DIR=os.getenv("PROJECT_DIR","")
  280.         # BUILD_SCRIPTS_DIR=os.getenv("BUILD_SCRIPTS_DIR","")
  281.         BASE_IMAGE=os.getenv("BASE_IMAGE", "")
  282.         VERSION=os.getenv("VERSION", "")
  283.         PROJECT_NAME=os.getenv("PROJECT_NAME","")
  284.         ROUTE=os.getenv("ROUTE","")
  285.         CONTAINER_NAME=os.getenv("CONTAINER_NAME","")
  286.  
  287.         os.chdir(PROJECT_DIR)
  288.  
  289.         print('build_sh: before calling analyze_and_test')
  290.         # self.sh(f"{BUILD_SCRIPTS_DIR}/analyze_and_test.sh")
  291.        
  292.         print("build_sh: before docker build")
  293.         self.sh(f"docker build --build-arg base_image={BASE_IMAGE} --build-arg VERSION={VERSION} --build-arg NAME={PROJECT_NAME} --build-arg ROUTE={ROUTE} -t {CONTAINER_NAME} .", shell=False, cwd=f"{PROJECT_DIR}bin/base_image")
  294.        
  295.         # print("build_sh: beforeexport image")
  296.  
  297.         # # Capture image_id so we can leverage it for execution and tagging purposes.
  298.         # IMAGE_ID=self.sh_inline(f'docker build --build-arg base_image={BASE_IMAGE} --build-arg VERSION={VERSION} --build-arg NAME={PROJECT_NAME} --build-arg ROUTE={ROUTE} -t {CONTAINER_NAME} . 2>/dev/null')
  299.         # print(f"Successfully built {IMAGE_ID}")
  300.        
  301.         # for word in $IMAGE_ID
  302.         # do
  303.         #   export IMAGE_ID=$word
  304.         # done
  305.        
  306.         # print(f"echo \"Captured [{IMAGE_ID}] as IMAGE_ID.\"")
  307.        
  308.         # print("Acceptance test is currently disabled in build.sh")
  309.         # #"$BUILD_SCRIPTS_DIR/run_acceptance_tests.sh"
  310.         # #echo "Acceptance test ran successfully"
  311.        
  312.         # echo "Running unit test"
  313.         # "$BUILD_SCRIPTS_DIR/run_unit_tests.sh"
  314.  
  315.  
  316.     def build_docker (self):
  317.         self.sh("chmod +x ./bin/*")
  318.         # self.sh("chmod +x ./bin/azure/az_install.sh && ./bin/azure/az_install.sh")
  319.         self.az_install()
  320.        
  321.         # self.sh("chmod +x ./bin/azure/az_login.sh && ./bin/azure/az_login.sh")
  322.         self.az_login()
  323.        
  324.         # self.sh("chmod +x ./bin/base_image/install_odbc.sh && ./bin/base_image/install_odbc.sh")
  325.         self.install_odbc()
  326.        
  327.         # self.sh("chmod +x ./bin/build.sh && ./bin/build.sh")
  328.         self.build()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!