Advertisement
Guest User

Untitled

a guest
Aug 19th, 2019
91
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.73 KB | None | 0 0
  1. #!/usr/bin/env python3
  2. """
  3. Update Dask configuration based on the configuration
  4. of the running Pod.
  5.  
  6. To be run at startup.
  7. """
  8.  
  9. import os
  10.  
  11. import dask
  12. import yaml
  13.  
  14. from kubernetes import client, config
  15.  
  16. user = os.environ.get('USER')
  17. jhubuser = os.environ.get('JUPYTERHUB_USER')
  18.  
  19.  
  20. def get_volumes():
  21. """Get volume mounts and volumnes defined in the
  22. running Pod.
  23. """
  24. config.load_incluster_config()
  25. v1 = client.CoreV1Api()
  26. namespace = open('/var/run/secrets/kubernetes.io/serviceaccount/namespace').read()
  27. pod_list = v1.list_namespaced_pod(namespace)
  28. this_pod = [pod for pod in pod_list.items if jhubuser in pod.metadata.name]
  29. this_pod = this_pod[0]
  30. volume_mounts = this_pod.spec.containers[0].volume_mounts
  31. volumes = this_pod.spec.volumes
  32. return volumes, volume_mounts
  33.  
  34.  
  35. def save_config(config):
  36. """Save K8s Dask configuration to file.
  37. """
  38. res = yaml.safe_dump({'kubernetes': config})
  39. open(config['worker-template-path'], 'w').write(res)
  40.  
  41.  
  42. volumes, volume_mounts = get_volumes()
  43.  
  44. # Only using hostPath here
  45. dc = dask.config.config['kubernetes']
  46. dc['worker-template']['spec']['volumes'] = [
  47. {'name': vol.name, 'hostPath': vol.host_path.to_dict()}
  48. for vol in volumes
  49. if vol.host_path
  50. ]
  51.  
  52. # Mounted in /data directory
  53. dc['worker-template']['spec']['containers'][0]['volumeMounts'] = [
  54. {'name': vol.name, 'mountPath': vol.mount_path}
  55. for vol in volume_mounts
  56. if 'data' in vol.mount_path
  57. ]
  58.  
  59. # Propagate as well env variables
  60. dc['worker-template']['spec']['containers'][0]['env'] = [
  61. {'name': 'EXTRA_PIP_PACKAGES', 'value': '${EXTRA_PIP_PACKAGES}'},
  62. {'name': 'OMP_NUM_THREADS', 'value': 1}
  63. ]
  64.  
  65. dc['worker-template-path'] = f'/home/{user}/.config/dask/kubernetes.yaml'
  66.  
  67. save_config(dc)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement