API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Sep 19th, 2018
118
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 14.53 KB | None | 0 0
raw download clone embed print report
  1. /**
  2. * This is a Jenkins pipeline script created to orchestrate xAgent performance workloads
  3. * from the instructions/parameters specified in the canned scenario JSON files.
  4. *
  5. * @author Yaroslav Styranivskyy
  6. * @since 2017-06-22
  7. */
  8.  
  9. import groovy.json.JsonSlurperClassic
  10. import groovy.json.JsonOutput
  11.  
  12. def scenario_config_dir = ""
  13. if (env.HOME)
  14. {
  15. scenario_config_dir = "${env.HOME}/userContent/perf_params"+
  16. '/agent/jobs/pipe_master_abstract/scenario_configs'
  17. }
  18. else if (env.JENKINS_HOME)
  19. {
  20. scenario_config_dir = "${env.JENKINS_HOME}/userContent/perf_params"+
  21. '/agent/jobs/pipe_master_abstract/scenario_configs'
  22. }
  23. def config_json = new JsonSlurperClassic().parseText(params.CONFIG)
  24. def utils = null
  25. def agentFlows = [:]
  26. def allFlowsPassed = true
  27. def allFlowsFailed = true
  28.  
  29. node(config_json.execution_node) {
  30. cleanWs patterns: [[pattern: 'virtualenvs/**', type: 'EXCLUDE']], deleteDirs: true
  31. stage('Prepare') {
  32. utils = load "${env.JENKINS_HOME}/userContent/perf_params"+
  33. '/common/pipeline/utils.groovy'
  34. echo """
  35. **********************************
  36. * JOB CONFIG *
  37. **********************************
  38. ${JsonOutput.prettyPrint(params.CONFIG)}
  39. """
  40. // Read configuration for the specified scenario.
  41. def scenario_config = readFile("${scenario_config_dir}/${config_json.workload_scenario}.json")
  42. def scenario_config_json = new JsonSlurperClassic().parseText(scenario_config)
  43. echo """
  44. **********************************
  45. * CANNED SCENARIO CONFIG *
  46. **********************************
  47. ${JsonOutput.prettyPrint(scenario_config)}
  48. """
  49. /* Merge "agent" element from canned scenario config with all agents,
  50. * while ensuring merged items match the agent platform. */
  51. config_json.hx.agents = config_json.hx.agents.collect {
  52. def agent = scenario_config_json.hx.agent.collectEntries { k, v ->
  53. if(! k.contains('install_config')) {
  54. [(k): (v)]
  55. }
  56. else if(k == 'pre_install_config'
  57. && ! scenario_config_json.hx.agent.containsKey('pre_install_config_'+ it.platform)) {
  58. [(k): (v)]
  59. }
  60. else if(k == 'pre_install_config_'+ it.platform) {
  61. ['pre_install_config': (v)]
  62. }
  63. else if(k == 'post_install_config'
  64. && ! scenario_config_json.hx.agent.containsKey('post_install_config_'+ it.platform)) {
  65. [(k): (v)]
  66. }
  67. else if(k == 'post_install_config_'+ it.platform) {
  68. ['post_install_config': (v)]
  69. }
  70. else {
  71. [:]
  72. }
  73. }
  74. it << agent
  75. }
  76. scenario_config_json.hx.remove('agent')
  77. // Merge supplied config and scenario config.
  78. config_json = utils.mapPlus(scenario_config_json, config_json)
  79. config_json = new JsonSlurperClassic().parseText(utils.evalExpressions(JsonOutput.toJson(config_json)))
  80. writeFile(
  81. file: 'config.json',
  82. text: JsonOutput.prettyPrint(JsonOutput.toJson(config_json))
  83. )
  84. echo """
  85. **********************************
  86. * COMBINED CONFIG *
  87. **********************************
  88. ${JsonOutput.prettyPrint(JsonOutput.toJson(config_json))}
  89. """
  90.  
  91. /* TODO - hx_reset.py should take parameters in the format of config_json,
  92. * so this conversion can be avoided. */
  93. def hx_reset_param = [
  94. hx_host: config_json.hx.hostname,
  95. hx_admin_pass: config_json.hx.password
  96. ]
  97. writeFile(
  98. file: "hx_reset_config.json",
  99. text: JsonOutput.toJson(hx_reset_param)
  100. )
  101. // TODO - temporarily transfer JSON to older HX format.
  102. def hx_config_param = [
  103. hx_host: config_json.hx.hostname,
  104. hx_admin_pass: config_json.hx.password,
  105. hx_api_user: config_json.hx.api_user,
  106. hx_api_pass: config_json.hx.api_pass,
  107. hx_config: config_json.hx.config
  108. ]
  109. writeFile(
  110. file: "hx_configure_config.json",
  111. text: JsonOutput.toJson(hx_config_param)
  112. )
  113. // TODO - end.
  114.  
  115. currentBuild.displayName = "${params.BUILD_TAG}_${config_json.environment}_#${currentBuild.number}"
  116.  
  117. // Checkout projects based on HX release.
  118. def perf_setup_scm = utils.getProjectSCMInfoByHXRelease('perf_setup', config_json.hx.release,GIT_ORG)
  119. utils.checkoutAndSetupProject('perf_setup', perf_setup_scm)
  120.  
  121. def pyhx_workload_scm = utils.getProjectSCMInfoByHXRelease('pyhx_workload', config_json.hx.release,GIT_ORG)
  122. utils.checkoutAndSetupProject('pyhx_workload', pyhx_workload_scm)
  123.  
  124. // Checkout projects based on xAgent version
  125. def agent_veresions = config_json.hx.agents.collect { it.version }.unique()
  126. agent_veresions.each {
  127. // Checkout projects.
  128. perf_setup_scm = utils.getProjectSCMInfoByAgentVersion('perf_setup', it, GIT_ORG)
  129. utils.checkoutAndSetupProject("perf_setup_${it}", perf_setup_scm)
  130.  
  131. pyhx_workload_scm = utils.getProjectSCMInfoByAgentVersion('pyhx_workload', it, GIT_ORG)
  132. utils.checkoutAndSetupProject("pyhx_workload_${it}", pyhx_workload_scm)
  133.  
  134. def pwrg_scm = utils.getProjectSCMInfoByAgentVersion('pwrg', it, GIT_ORG)
  135. utils.checkoutAndSetupProject("pwrg_${it}", pwrg_scm, "pip install -r xAgent/requirements.txt")
  136. }
  137.  
  138. // Setup HX actions if needed.
  139. if(config_json.workload.containsKey('hx_actions')) {
  140. def hx_workload = [:] << config_json.workload
  141. hx_workload['actions'] = hx_workload.remove('hx_actions')
  142. def hx_workload_config_param = [
  143. hx: config_json.hx,
  144. agent: [
  145. platform: 'hx'
  146. ],
  147. workload: hx_workload
  148. ]
  149. writeFile(
  150. file: "hx_actions_config.json",
  151. text: JsonOutput.toJson(hx_workload_config_param)
  152. )
  153.  
  154. agentFlows['HX Actions'] = {
  155. stage('HX Actions') {
  156. utils.runProject('pyhx_workload',
  157. "agent_workload --config-file ${env.WORKSPACE}/hx_actions_config.json"
  158. )
  159. }
  160. }
  161. }
  162.  
  163. // Setup agent flows.
  164. config_json.hx.agents.each { agent_conf ->
  165. def pfm_spec_items = [
  166. 'scenario_description': 'scenario_description_'+ agent_conf.platform,
  167. 'actions': 'actions_'+ agent_conf.platform
  168. ]
  169. def agent_config_param = [
  170. hx: config_json.hx,
  171. agent: agent_conf,
  172. workload:
  173. // Ensure workload actions are for the correct platform.
  174. config_json.workload.collectEntries { k, v ->
  175. if(pfm_spec_items.containsKey(k) && ! config_json.workload.get(pfm_spec_items.get(k))) {
  176. [(k): (v)]
  177. }
  178. else if(! pfm_spec_items.find{ k.startsWith(it.key) } ) {
  179. [(k): (v)]
  180. }
  181. else if(pfm_spec_items.containsValue(k)) {
  182. [(pfm_spec_items.find{ it.value == k }.key): (v)]
  183. }
  184. else {
  185. [:]
  186. }
  187. }
  188. ]
  189. writeFile(
  190. file: "agent_${agent_conf.hostname}_config.json",
  191. text: JsonOutput.toJson(agent_config_param)
  192. )
  193.  
  194. agentFlows["${agent_conf.hostname} Flow"] = {
  195. try {
  196. stage("Configure Agent") {
  197. utils.runProject("perf_setup_${agent_conf.version}",
  198. "agent_configure --config ${env.WORKSPACE}/agent_${agent_conf.hostname}_config.json"
  199. )
  200. }
  201.  
  202. try {
  203. stage('Start Monitoring') {
  204. utils.runProject("perf_setup_${agent_conf.version}",
  205. "agent_data_collection --config ${env.WORKSPACE}/agent_${agent_conf.hostname}_config.json --action start"
  206. )
  207. }
  208.  
  209. stage('Execute Workload') {
  210. utils.runProject("pyhx_workload_${agent_conf.version}",
  211. "agent_workload --config-file ${env.WORKSPACE}/agent_${agent_conf.hostname}_config.json"
  212. )
  213. }
  214. }
  215. catch(exc) {
  216. throw exc
  217. }
  218. finally {
  219. stage('Stop Monitoring') {
  220. def collection_type = (agent_conf.deployment.type == 'uninstall' ||
  221. agent_conf.deployment.type == 'none') ? 'basic' : 'xagent'
  222. utils.runProject("perf_setup_${agent_conf.version}", """
  223. agent_data_collection --config ${env.WORKSPACE}/agent_${agent_conf.hostname}_config.json \
  224. --action stop --collection_type ${collection_type}
  225. """
  226. )
  227. dir("perf_setup_${agent_conf.version}") {
  228. archive (includes: "performance_${agent_conf.hostname}.zip")
  229. }
  230. }
  231. }
  232.  
  233. stage('Generate Report') {
  234. dir("perf_setup_${agent_conf.version}") {
  235. def zipRootEntry = sh(
  236. script: "zipinfo -1 performance_${agent_conf.hostname}.zip | head -1",
  237. returnStdout: true
  238. ).trim()
  239. if(zipRootEntry.equalsIgnoreCase('performance/')) {
  240. unzip(
  241. zipFile: "performance_${agent_conf.hostname}.zip",
  242. dir: "performance_${agent_conf.hostname}"
  243. )
  244. sh "rm -rf performance_${agent_conf.hostname}.zip"
  245. dir("performance_${agent_conf.hostname}") {
  246. sh "mv ${zipRootEntry} performance_${agent_conf.hostname}"
  247. sh "zip -r ../performance_${agent_conf.hostname}.zip ."
  248. }
  249. sh "rm -rf performance_${agent_conf.hostname}"
  250. }
  251. }
  252.  
  253. def reportGenCmd = """cd xAgent
  254. python generator_xagent.py \\
  255. --perf ${env.WORKSPACE}/perf_setup_${agent_conf.version}/performance_${agent_conf.hostname}.zip \\
  256. --output output_${agent_conf.hostname}"""
  257. if(config_json.containsKey("report") && config_json.report.containsKey("dashboard")
  258. && config_json.report.dashboard.type == 'couchdb') {
  259. reportGenCmd += """ \\
  260. --couchdb_ip ${config_json.report.dashboard.couchdb_ip} \\
  261. --couchdb_db ${config_json.report.dashboard.couchdb_db}"""
  262. }
  263. utils.runProject("pwrg_${agent_conf.version}", reportGenCmd)
  264. dir("pwrg_${agent_conf.version}") {
  265. archive (includes: "output_${agent_conf.hostname}/**")
  266. }
  267. }
  268.  
  269. allFlowsFailed = false
  270. }
  271. catch(ex) {
  272. allFlowsPassed = false
  273. throw ex
  274. }
  275. }
  276. }
  277. }
  278.  
  279. if(config_json.hx.deployment.type == "reset") {
  280. stage('Reset HX') {
  281. utils.runProject('perf_setup',
  282. "hx_reset --config_file ${env.WORKSPACE}/hx_reset_config.json --regen_pki"
  283. )
  284. sleep 60
  285. }
  286. }
  287.  
  288. //Configure HX appliance if required.
  289. if(config_json.hx.config.type != 'none') {
  290. stage('Configure HX') {
  291. utils.runProject('perf_setup',
  292. "hx_configure --config_file ${env.WORKSPACE}/hx_configure_config.json"
  293. )
  294. def set_addr = "/bin/bash ${env.JENKINS_HOME}/userContent/policy_scripts/set_default_addr " + config_json.hx.hostname
  295. def sout = new StringBuilder(), serr = new StringBuilder()
  296. def proc = set_addr.execute()
  297. proc.consumeProcessOutput(sout, serr)
  298. proc.waitForOrKill(20000)
  299. def rc = proc.exitValue()
  300. println "out> $sout err> $serr rc> $rc"
  301. if (rc != 0)
  302. {
  303. return(rc)
  304. }
  305. }
  306. }
  307.  
  308. stage('Deploy Agents') {
  309. def agent_deploy = build(job: 'agent_deploy', parameters: [
  310. [$class: 'StringParameterValue', name: 'BUILD_TAG', value: params.BUILD_TAG],
  311. [$class: 'StringParameterValue', name: 'HX_RELEASE', value: config_json.hx.release],
  312. [$class: 'StringParameterValue', name: 'CONFIG', value: JsonOutput.toJson(config_json)],
  313. new org.jvnet.jenkins.plugins.nodelabelparameter.NodeParameterValue(
  314. 'EXECUTION_NODE', 'description', config_json.get('execution_node', 'master')
  315. )
  316. ], wait: true, propagate: false)
  317. def deployOut = agent_deploy.getRawBuild().getLogReader().getText()
  318. echo deployOut
  319. def numFailed = deployOut.findAll(".*failed=1.*").size() + deployOut.findAll(".*unreachable=1.*").size()
  320. if(numFailed > 0 && numFailed < config_json.hx.agents.size()) {
  321. currentBuild.result = 'UNSTABLE'
  322. }
  323. else if(agent_deploy.result == 'FAILURE') {
  324. throw new Exception('Deployment failed for all agents!!!')
  325. }
  326. }
  327.  
  328. parallel(agentFlows)
  329.  
  330. if(!allFlowsPassed && !allFlowsFailed) {
  331. currentBuild.result = 'UNSTABLE'
  332. }
  333. else if(allFlowsFailed) {
  334. currentBuild.result = 'FAILURE'
  335. }
  336. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!