import time
from concurrent.futures import ThreadPoolExecutor
from threading import Thread
import elasticsearch as es


queries = [
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'count(agent)': {'value_count': {'field': 'agent.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'agent': {'cardinality': {'field': 'agent.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'beat.hostname': {'cardinality': {'field': 'beat.hostname.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'beat.name': {'cardinality': {'field': 'beat.name.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'beat.version': {'cardinality': {'field': 'beat.version.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'bytes': {'cardinality': {'field': 'bytes.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.city_name': {'cardinality': {'field': 'geoip.city_name.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.continent_code': {'cardinality': {'field': 'geoip.continent_code.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.country_code2': {'cardinality': {'field': 'geoip.country_code2.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.country_code3': {'cardinality': {'field': 'geoip.country_code3.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.country_name': {'cardinality': {'field': 'geoip.country_name.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.dma_code': {'cardinality': {'field': 'geoip.dma_code'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.ip': {'cardinality': {'field': 'geoip.ip.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.latitude': {'cardinality': {'field': 'geoip.latitude'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.location.lat': {'cardinality': {'field': 'geoip.location.lat'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.location.lon': {'cardinality': {'field': 'geoip.location.lon'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.longitude': {'cardinality': {'field': 'geoip.longitude'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.postal_code': {'cardinality': {'field': 'geoip.postal_code.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.region_code': {'cardinality': {'field': 'geoip.region_code.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.region_name': {'cardinality': {'field': 'geoip.region_name.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'geoip.timezone': {'cardinality': {'field': 'geoip.timezone.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'host.name': {'cardinality': {'field': 'host.name.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'http_version': {'cardinality': {'field': 'http_version.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'input.type': {'cardinality': {'field': 'input.type.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'message': {'cardinality': {'field': 'message.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'offset': {'cardinality': {'field': 'offset'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'prospector.type': {'cardinality': {'field': 'prospector.type.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'referrer': {'cardinality': {'field': 'referrer.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'remote_ip': {'cardinality': {'field': 'remote_ip.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'request': {'cardinality': {'field': 'request.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'request_action': {'cardinality': {'field': 'request_action.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'response': {'cardinality': {'field': 'response.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'source': {'cardinality': {'field': 'source.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'tags': {'cardinality': {'field': 'tags.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'time': {'cardinality': {'field': 'time.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_agent.build': {'cardinality': {'field': 'user_agent.build.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_agent.device': {'cardinality': {'field': 'user_agent.device.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_agent.major': {'cardinality': {'field': 'user_agent.major.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_agent.minor': {'cardinality': {'field': 'user_agent.minor.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_agent.name': {'cardinality': {'field': 'user_agent.name.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_agent.os': {'cardinality': {'field': 'user_agent.os.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_agent.os_major': {'cardinality': {'field': 'user_agent.os_major.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_agent.os_minor': {'cardinality': {'field': 'user_agent.os_minor.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_agent.os_name': {'cardinality': {'field': 'user_agent.os_name.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_agent.patch': {'cardinality': {'field': 'user_agent.patch.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'user_name': {'cardinality': {'field': 'user_name.keyword'}}}},
    {'query': {'match_all': {}}, 'size': 0, 'aggs': {'timeline': {'date_histogram': {'interval': '1d', 'field': '@timestamp'}}}}
]

# for search body
main_query = {'aggs': {}}
for q in queries:
    main_query['aggs'].update(q.get('aggs'))
else:
    q.pop('aggs')
    main_query.update(q)

# for multi search body
msearch_query = [
    {"index": "nginx*", "ignore_unavailable": "true", "timeout": 30000},
    main_query
]

iterable = ((['nginx*'], None, q) for q in queries)

client = es.Elasticsearch(hosts=["ip:9200"], http_auth=None, maxsize=50, timeout=50,
                          connection_class=es.connection.http_requests.RequestsHttpConnection)

start = time.time()
client.search(index=['nginx*'], doc_type=None, body=main_query)
print('Standard Search Query Execution Time: ', time.time() - start)

Output: Standard Search Query Execution Time:  19.81995415687561s

start = time.time()
client.msearch(body=msearch_query)
print('Standard Multi Search Query Execution Time: ', time.time() - start)

Output: Standard Multi Search Query Execution Time:  19.14345407485962s

start = time.time()
with ThreadPoolExecutor(50) as ex:
    ex.map(lambda q: client.search(*q), iterable)
print('Search Query By Multi Threading Execution Time: ', time.time() - start)

Output: Search Query By Multi Threading Execution Time:  20.80817937850952s

start = time.time()
jobs = [Thread(target=client.search, args=arg) for arg in iterable]
# start threads
for job in jobs:
    job.start()
# join threads
for job in jobs:
    job.join()
print('Search Query By Standard Multi Threading Execution Time: ', time.time() - start)

Output: Search Query By Standard Multi Threading Execution Time:  21.506370544433594s