Advertisement
Guest User

Untitled

a guest
Jul 25th, 2017
110
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.26 KB | None | 0 0
  1. import time
  2. import math
  3. import pandas as pd
  4. import random
  5. import numpy as np
  6. from bokeh.io import show, output_notebook, push_notebook
  7. from bokeh.resources import CDN
  8. from bokeh.models import (
  9. ColumnDataSource,
  10. HoverTool,
  11. LinearColorMapper,
  12. BasicTicker,
  13. PrintfTickFormatter,
  14. ColorBar,
  15. )
  16. from bokeh.plotting import figure, helpers
  17. output_notebook(resources=CDN)
  18.  
  19. # Function to create the cluster usage "heat map"
  20. def cluster_usage():
  21.  
  22. # Initial values
  23. source = ColumnDataSource(data={"node_ip_address":['127.0.0.1'], "time":[0], "num_tasks":[0]})
  24.  
  25. # Define the color schema
  26. colors = ["#75968f", "#a5bab7", "#c9d9d3", "#e2e2e2", "#dfccce", "#ddb7b1", "#cc7878", "#933b41", "#550b1d"]
  27. mapper = LinearColorMapper(palette=colors, low=0, high=2)
  28.  
  29. TOOLS = "hover,save,xpan,box_zoom,reset,xwheel_zoom"
  30.  
  31. # Create the plot
  32. p = figure(title="Cluster Usage", y_range=list(set(source.data['node_ip_address'])),
  33. x_axis_location="above", plot_width=900, plot_height=500,
  34. tools=TOOLS, toolbar_location='below')
  35.  
  36. # Format the plot axes
  37. p.grid.grid_line_color = None
  38. p.axis.axis_line_color = None
  39. p.axis.major_tick_line_color = None
  40. p.axis.major_label_text_font_size = "10pt"
  41. p.axis.major_label_standoff = 0
  42. p.xaxis.major_label_orientation = math.pi / 3
  43.  
  44. # Plot rectangles
  45. p.rect(x="time", y="node_ip_address", width=1, height=1,
  46. source=source,
  47. fill_color={"field": "num_tasks", "transform": mapper},
  48. line_color=None)
  49.  
  50. # Add legend to the side of the plot
  51. color_bar = ColorBar(color_mapper=mapper, major_label_text_font_size="8pt",
  52. ticker=BasicTicker(desired_num_ticks=len(colors)),
  53. label_standoff=6, border_line_color=None, location=(0, 0))
  54. p.add_layout(color_bar, "right")
  55.  
  56. # Define hover tool
  57. p.select_one(HoverTool).tooltips = [
  58. ("Node IP Address", "@node_ip_address"),
  59. ("Number of tasks running", "@num_tasks"),
  60. ("Time", "@time")
  61. ]
  62.  
  63. # Define the axis labels
  64. p.xaxis.axis_label = "Time in seconds"
  65. p.yaxis.axis_label = "Node IP Address"
  66.  
  67. handle = show(p, notebook_handle=True)
  68.  
  69. # Function to update the heat map
  70. def heat_map_update(abs_earliest, abs_latest, abs_num_tasks, tasks):
  71. granularity = 1
  72. print("hi")
  73. earliest = time.time()
  74. latest = 0
  75.  
  76. for task_id, data in tasks.items():
  77. if data["score"] > latest:
  78. latest = data["score"]
  79. if data["score"] < earliest:
  80. earliest = data["score"]
  81.  
  82. num_buckets = math.ceil((latest - earliest) / granularity)
  83. buckets = [0] * num_buckets
  84.  
  85. worker_info = ray.global_state.workers()
  86. num_tasks = []
  87. nodes = []
  88. times = []
  89. start_point = earliest
  90. end_point = len(buckets) * granularity + earliest
  91.  
  92. # Determine the distribution "buckets" over the timeline
  93. for i in range(0, len(buckets), granularity):
  94. start = i * granularity + earliest
  95. end = (i + 1) * granularity + earliest
  96. t = ray.global_state.task_profiles(start=math.floor(start), end=math.ceil(end))
  97.  
  98. node_to_num = dict()
  99. for task_id, data in t.items():
  100. worker = data["worker_id"]
  101. node = worker_info[worker]["node_ip_address"]
  102. if node not in node_to_num:
  103. node_to_num[node] = 0
  104. node_to_num[node] += 1
  105.  
  106. for node_ip, counter in node_to_num.items():
  107. num_tasks.append(node_to_num[node_ip])
  108. nodes.append(node_ip)
  109. times.append(earliest - abs_earliest + i * granularity)
  110.  
  111. p.y_range = helpers._get_range(list(set(nodes)))
  112.  
  113. if len(num_tasks) == 0:
  114. return
  115. else:
  116. mapper.low = min(min(num_tasks), 0)
  117. mapper.high = max(max(num_tasks), 1)
  118.  
  119. # Update notebook
  120. source.data = {"node_ip_address": nodes, "time": times, "num_tasks": num_tasks}
  121. push_notebook(handle=handle)
  122.  
  123. get_sliders(heat_map_update)
  124.  
  125. cluster_usage()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement