Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- r".*?([0-9]+.[0-9]+ .B).*?[0-9]+.[0-9]+ .?B.*"
- Operator #Hosts Avg Time Max Time #Rows Est. #Rows Peak Mem Est. Peak Mem Detail
- ------------------------------------------------------------------------------------------------------------------------------------
- 09:MERGING-EXCHANGE 1 1s384ms 1s384ms 50 50 0 0 UNPARTITIONED
- 04:TOP-N 20 791.287us 1.209ms 1.00K 50 16.00 KB 2.10 KB
- 08:AGGREGATE 20 6.771ms 11.812ms 47.84K 2.18B 34.11 MB 96.12 GB FINALIZE
- 03:AGGREGATE 20 30.051ms 40.019ms 47.84K 2.18B 34.14 MB 96.12 GB STREAMING
- 02:HASH JOIN 20 3.518ms 5.739ms 1.28M 2.18B 115.16 GB 1.94 MB INNER JOIN, PARTITIONED
- |--06:EXCHANGE 20 363.121ms 411.503ms 36.73M 4.05M 0 0 HASH(ed_edw.cla_lslon.lon_bor_ssn)
- | 01:SCAN HDFS 20 30.293ms 43.324ms 36.73M 4.05M 7.08 MB 88.00 MB ed_edw.cla_lslon
- 05:EXCHANGE 20 1s122ms 1s171ms 47.84K 854.42K 0 0 HASH(ed_edw.cla_lslon.lon_bor_ssn)
- 00:SCAN HDFS 20 1s146ms 1s237ms 47.84K 854.42K 38.71 MB 880.00 MB ed_edw.cla_lslon
- Operator #Hosts Avg Time Max Time #Rows Est. #Rows Peak Mem Est. Peak Mem Detail
- ---------------------------------------------------------------------------------------------------------------------------------
- 08:EXCHANGE 1 911.294ms 911.294ms 100 100 0 0 UNPARTITIONED
- 05:HASH JOIN 28 33.155ms 58.395ms 885 100 265.55 MB 1.94 MB LEFT OUTER JOIN, BROADCAST
- |--07:EXCHANGE 28 4.102ms 23.656ms 52.44K 490 0 0 BROADCAST
- | 04:SELECT 4 601.466us 677.938us 52.44K 490 272.00 KB 0
- | 03:ANALYTIC 4 29.278ms 69.397ms 52.44K 490 4.28 MB 4.00 MB
- | 02:SORT 4 20.544ms 21.396ms 52.44K 490 12.14 MB 12.00 MB
- | 06:EXCHANGE 4 245.005ms 473.348ms 52.44K 490 0 0 HASH(empe_id)
- | 01:SCAN HDFS 4 46.336ms 56.823ms 52.44K 490 16.63 MB 240.00 MB e.empe_hr_dly
- 00:SCAN HDFS 28 333.952ms 1s443ms 1.01K 9.34K 615.47 MB 512.00 MB hadoop.audit_events
- #!/usr/bin/env python
- import urllib2
- import json
- import requests
- from requests.auth import HTTPDigestAuth
- import sys
- import re
- api_user = "admin"
- api_password = "admin"
- datanodes = ["https://datanode10:5000/",
- "https://datanode20:5000/",
- "https://datanode30:5000/"]
- for i, datanode in enumerate(datanodes):
- try:
- response = requests.get(datanode + "queries?json",auth=HTTPDigestAuth(api_user, api_password),verify='/data/certs.pem')
- data = response.json()
- for in_flight_query in data["in_flight_queries"]:
- query_id = in_flight_query["query_id"]
- summary = requests.get(datanode + "query_summary?query_id=" + query_id + "&json" ,auth=HTTPDigestAuth(api_user, api_password),verify='/data/certs.pem')
- sum_data = summary.json()
- result = sum_data['summary']
- #this will only print data from one of the datanodes instead of 2 sets of data
- print result
- except IOError as ioe:
- print("Skipping {0}: {1}".format(i, datanode))
- print ioe
- except Exception as e:
- print(e)
Add Comment
Please, Sign In to add comment