API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Jul 4th, 2017
62
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 11.41 KB | None | 0 0
raw download clone embed print report
  1. diff --git a/buildfarm/release/release-runner.py b/buildfarm/release/release-runner.py
  2. --- a/buildfarm/release/release-runner.py
  3. +++ b/buildfarm/release/release-runner.py
  4. @@ -10,17 +10,17 @@ import subprocess
  5. import hashlib
  6. import functools
  7. import shutil
  8. import tempfile
  9. import requests
  10. from os import path
  11. from optparse import OptionParser
  12. from twisted.python.lockfile import FilesystemLock
  13. -from taskcluster import Scheduler, Index, Queue
  14. +from taskcluster import Index, Queue
  15. from taskcluster.utils import slugId
  16. import yaml
  17.  
  18. site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
  19.  
  20. from kickoff import (get_partials, ReleaseRunner,
  21. make_task_graph_strict_kwargs, long_revision,
  22. get_l10n_config, get_en_US_config, email_release_drivers,
  23. @@ -84,16 +84,25 @@ def assign_and_check_partial_updates(rel
  24. # So people can't run arbitrary functions
  25. CHECKS_MAPPING = {
  26. 'long_revision': check_and_assign_long_revision,
  27. 'l10n_changesets': assign_and_check_l10n_changesets,
  28. 'partial_updates': assign_and_check_partial_updates,
  29. }
  30.  
  31.  
  32. +def resolve_task(task_id, queue):
  33. + curr_status = queue.status(task_id)
  34. + run_id = curr_status['status']['runs'][-1]['runId']
  35. + payload = {"workerGroup": curr_status['status']['workerType'],
  36. + "workerId": "TCTalker"}
  37. + queue.claimTask(task_id, run_id, payload)
  38. + queue.reportCompleted(task_id, run_id)
  39. +
  40. +
  41. def run_prebuild_sanity_checks(release_runner, releases_config):
  42. new_valid_releases = []
  43.  
  44. # results in:
  45. # { 'firefox': ['long_revision', 'l10n_changesets', 'partial_updates']}
  46. checks = {r['product'].lower(): r['checks'] for r in releases_config}
  47.  
  48. for release in release_runner.new_releases:
  49. @@ -338,19 +347,18 @@ def main(options):
  50. beetmover_aws_access_key_id = config["beetmover"].get("aws_access_key_id")
  51. beetmover_aws_secret_access_key = config["beetmover"].get("aws_secret_access_key")
  52. gpg_key_path = config["signing"].get("gpg_key_path")
  53.  
  54. # TODO: replace release sanity with direct checks of en-US and l10n
  55. # revisions (and other things if needed)
  56.  
  57. rr = ReleaseRunner(api_root=api_root, username=username, password=password)
  58. - scheduler = Scheduler(retrying_tc_config)
  59. index = Index(tc_config)
  60. - queue = Queue(tc_config)
  61. + queue = Queue(retrying_tc_config)
  62.  
  63. # Main loop waits for new releases, processes them and exits.
  64. while True:
  65. try:
  66. log.debug('Fetching release requests')
  67. rr.get_release_requests([r['pattern'] for r in config['releases']])
  68. if rr.new_releases:
  69. new_releases = run_prebuild_sanity_checks(
  70. @@ -406,17 +414,17 @@ def main(options):
  71. final_verify_channels = release_channels
  72. publish_to_balrog_channels = release_channels
  73. push_to_releases_enabled = True
  74.  
  75. # XXX: Doesn't work with neither Fennec nor Thunderbird
  76. platforms = branchConfig['release_platforms']
  77.  
  78. try:
  79. - graph_id = slugId()
  80. + task_group_id = None
  81. done = are_en_us_builds_completed(
  82. index=index, release_name=release['name'],
  83. submitted_at=release['submittedAt'],
  84. revision=release['mozillaRevision'],
  85. platforms=platforms, queue=queue,
  86. tc_task_indexes=branchConfig['tc_indexes'][release['product']])
  87. if not done:
  88. log.info(
  89. @@ -504,41 +512,43 @@ def main(options):
  90. "extra_balrog_submitter_params": extra_balrog_submitter_params + " --product " + release["product"].capitalize(),
  91. "publish_to_balrog_channels": publish_to_balrog_channels,
  92. "snap_enabled": branchConfig.get("snap_enabled", {}).get(release["product"], False),
  93. "update_verify_channel": branchConfig.get("update_verify_channel", {}).get(release["product"]),
  94. "update_verify_requires_cdn_push": branchConfig.get("update_verify_requires_cdn_push", False),
  95. }
  96.  
  97. validate_graph_kwargs(queue, gpg_key_path, **kwargs)
  98. - graph = make_task_graph_strict_kwargs(**kwargs)
  99. - rr.update_status(release, "Submitting task graph")
  100. - log.info("Task graph generated!")
  101. + task_group_id, toplevel_task_id, tasks = make_task_graph_strict_kwargs(**kwargs)
  102. + rr.update_status(release, "Submitting tasks")
  103. + log.info("Tasks generated!")
  104. import pprint
  105. - log.debug(pprint.pformat(graph, indent=4, width=160))
  106. - print(scheduler.createTaskGraph(graph_id, graph))
  107. -
  108. + log.debug(pprint.pformat(tasks, indent=4, width=160))
  109. + for task_id, task in tasks.items():
  110. + log.info("Submitting task: %s", task_id)
  111. + queue.createTask(task_id, task)
  112. + resolve_task(toplevel_task_id, queue)
  113. rr.mark_as_completed(release)
  114. l10n_url = rr.release_l10n_api.getL10nFullUrl(release['name'])
  115. email_release_drivers(smtp_server=smtp_server, from_=notify_from,
  116. to=notify_to, release=release,
  117. - task_group_id=graph_id, l10n_url=l10n_url)
  118. + task_group_id=task_group_id, l10n_url=l10n_url)
  119. except Exception as exception:
  120. # We explicitly do not raise an error here because there's no
  121. # reason not to start other releases if creating the Task Graph
  122. # fails for another one. We _do_ need to set this in order to exit
  123. # with the right code, though.
  124. rc = 2
  125. rr.mark_as_failed(
  126. release,
  127. 'Failed to start release promotion (graph ID: %s). Error(s): %s' % (
  128. - graph_id, exception)
  129. + task_group_id, exception)
  130. )
  131. log.exception('Failed to start release "%s" promotion for graph %s. Error(s): %s',
  132. - release['name'], graph_id, exception)
  133. + release['name'], task_group_id, exception)
  134. log.debug('Release failed: %s', release)
  135.  
  136. if rc != 0:
  137. sys.exit(rc)
  138.  
  139. log.debug('Sleeping for %s seconds before polling again', sleeptime)
  140. time.sleep(sleeptime)
  141.  
  142. diff --git a/buildfarm/release/releasetasks_graph_gen.py b/buildfarm/release/releasetasks_graph_gen.py
  143. --- a/buildfarm/release/releasetasks_graph_gen.py
  144. +++ b/buildfarm/release/releasetasks_graph_gen.py
  145. @@ -10,33 +10,32 @@ site.addsitedir(os.path.join(os.path.dir
  146.  
  147. from kickoff import get_partials, ReleaseRunner, make_task_graph_strict_kwargs
  148. from kickoff import get_l10n_config, get_en_US_config
  149. from kickoff import bump_version
  150.  
  151. from release.versions import getAppVersion
  152. from util.file import load_config, get_config
  153.  
  154. -from taskcluster import Scheduler, Index, Queue
  155. +from taskcluster import Index, Queue
  156. from taskcluster.utils import slugId
  157.  
  158. log = logging.getLogger(__name__)
  159.  
  160.  
  161. def main(release_runner_config, release_config, tc_config):
  162.  
  163. api_root = release_runner_config.get('api', 'api_root')
  164. username = release_runner_config.get('api', 'username')
  165. password = release_runner_config.get('api', 'password')
  166.  
  167. - scheduler = Scheduler(tc_config)
  168. index = Index(tc_config)
  169. + queue = Queue(tc_config)
  170.  
  171. rr = ReleaseRunner(api_root=api_root, username=username, password=password)
  172. - graph_id = slugId()
  173. log.info('Generating task graph')
  174. kwargs = {
  175. # release-runner.ini
  176. "signing_pvt_key": release_config['signing_pvt_key'],
  177. "public_key": release_config['docker_worker_key'],
  178. "balrog_username": release_config['balrog_username'],
  179. "balrog_password": release_config['balrog_password'],
  180. "beetmover_aws_access_key_id": release_config['beetmover_aws_access_key_id'],
  181. @@ -115,22 +114,33 @@ def main(release_runner_config, release_
  182. "postrelease_mark_as_shipped_enabled": release_config["postrelease_mark_as_shipped_enabled"],
  183. # TODO: use [] when snaps_enabled is landed
  184. "snap_enabled": release_config.get("snap_enabled", False),
  185. "update_verify_channel": release_config["update_verify_channel"],
  186. "update_verify_requires_cdn_push": release_config["update_verify_requires_cdn_push"],
  187. "release_eta": release_config.get("release_eta"),
  188. }
  189.  
  190. - graph = make_task_graph_strict_kwargs(**kwargs)
  191. - log.info("Submitting task graph")
  192. + task_group_id, toplevel_task_id, tasks = make_task_graph_strict_kwargs(**kwargs)
  193. + log.info("Submitting task group: %s", task_group_id)
  194. import pprint
  195. - log.info(pprint.pformat(graph, indent=4, width=160))
  196. + log.info(pprint.pformat(tasks, indent=4, width=160))
  197. if not options.dry_run:
  198. - print scheduler.createTaskGraph(graph_id, graph)
  199. + for task_id, task in tasks.items():
  200. + log.info("Submitting task: %s", task_id)
  201. + print queue.createTask(task_id, task)
  202. +
  203. +
  204. +def resolve_task(task_id, queue):
  205. + curr_status = queue.status(task_id)
  206. + run_id = curr_status['status']['runs'][-1]['runId']
  207. + payload = {"workerGroup": curr_status['status']['workerType'],
  208. + "workerId": "TCTalker"}
  209. + queue.claimTask(task_id, run_id, payload)
  210. + queue.reportCompleted(task_id, run_id)
  211.  
  212.  
  213. def get_items_from_common_tc_task(common_task_id, tc_config):
  214. tc_task_items = {}
  215. queue = Queue(tc_config)
  216. task = queue.task(common_task_id)
  217. tc_task_items["version"] = task["extra"]["build_props"]["version"]
  218. tc_task_items["build_number"] = task["extra"]["build_props"]["build_number"]
  219. @@ -203,17 +213,18 @@ if __name__ == '__main__':
  220. parser.error('Need to pass a branch and product config')
  221.  
  222. # load config files
  223. release_runner_config = load_config(options.release_runner_ini)
  224. tc_config = {
  225. "credentials": {
  226. "clientId": get_config(release_runner_config, "taskcluster", "client_id", None),
  227. "accessToken": get_config(release_runner_config, "taskcluster", "access_token", None),
  228. - }
  229. + },
  230. + "maxRetries": 12,
  231. }
  232. branch_product_config = load_branch_and_product_config(options.branch_and_product_config)
  233.  
  234. if release_runner_config.getboolean('release-runner', 'verbose'):
  235. log_level = logging.DEBUG
  236. else:
  237. log_level = logging.INFO
  238. logging.basicConfig(filename='releasetasks_graph_gen.log',
  239. diff --git a/lib/python/kickoff/__init__.py b/lib/python/kickoff/__init__.py
  240. --- a/lib/python/kickoff/__init__.py
  241. +++ b/lib/python/kickoff/__init__.py
  242. @@ -317,18 +317,18 @@ def make_task_graph_strict_kwargs(appVer
  243. update_verify_channel=update_verify_channel,
  244. update_verify_requires_cdn_push=update_verify_requires_cdn_push,
  245. funsize_product=funsize_product,
  246. release_eta=release_eta,
  247. )
  248. if extra_balrog_submitter_params:
  249. kwargs["extra_balrog_submitter_params"] = extra_balrog_submitter_params
  250.  
  251. - # don't import releasetasks until required within function impl to avoid global failures
  252. - # during nosetests
  253. - from releasetasks import make_task_graph
  254. - return make_task_graph(**kwargs)
  255. + # don't import releasetasks until required within function impl to avoid
  256. + # global failures during nosetests
  257. + from releasetasks import make_tasks
  258. + return make_tasks(**kwargs)
  259.  
  260.  
  261. def get_funsize_product(product_name):
  262. if product_name == 'devedition': # See bug 1366075
  263. return 'firefox'
  264. return product_name
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!