API tools faq
paste
Guest User

Untitled

a guest
Feb 19th, 2018
95
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 23.77 KB | None | 0 0
raw download clone embed print report
  1. Total time: 26.8113 s
  2. File: /home/amanda-simbiose/Documents/API_s1search/slicing_dice-api_python3/api/tests/insertion_performance/wrk_scripts/insert_for_profile.py
  3. Function: index_many_times at line 63
  4. /usr/lib/python3.6/asyncio/events.py:127: RuntimeWarning: coroutine 'KafkaIndexer.run' was never awaited
  5. self._callback(*self._args)
  6.  
  7. Line # Hits Time Per Hit % Time Line Contents
  8. ==============================================================
  9. 63 async def index_many_times():
  10. 64 10001 34022.0 3.4 0.1 for i in range(10000):
  11. 65 10000 126467.0 12.6 0.5 client = await util.get_s1_client(test=False)
  12. 66 10000 239914.0 24.0 0.9 param_list = {'data': {'entity100': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity101': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity102': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity103': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity104': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity105': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity106': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity107': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity108': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity109': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1010': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1011': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1012': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1013': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1014': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1015': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1016': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1017': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1018': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1019': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1020': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1021': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1022': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1023': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1024': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1025': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1026': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1027': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1028': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1029': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1030': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1031': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1032': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1033': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1034': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1035': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1036': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1037': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1038': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1039': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1040': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1041': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1042': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1043': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1044': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1045': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1046': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1047': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1048': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}, 'entity1049': {'string-test-field': 'valueqqr', 'boolean-test-field': 'false', 'numeric-test-field': 151515115, 'date-test-field': '2010-04-12', 'decimal-test-field': 1515.8}}, 'meta': {'project_id': '37251', 'test': False, 'group': '933_37251', 'entities': {'entity100': 2011850, 'entity101': 2006696, 'entity102': 3001210, 'entity103': 2001214, 'entity104': 2001204, 'entity105': 1681, 'entity106': 1671, 'entity107': 1001179, 'entity108': 1001169, 'entity109': 1001159, 'entity1010': 3001195, 'entity1011': 3001185, 'entity1012': 2001189, 'entity1013': 2001179, 'entity1014': 1001152, 'entity1015': 1001142, 'entity1016': 3001178, 'entity1017': 3001168, 'entity1018': 1660, 'entity1019': 1650, 'entity1020': 1640, 'entity1021': 2001164, 'entity1022': 2001154, 'entity1023': 1001127, 'entity1024': 1001117, 'entity1025': 3001153, 'entity1026': 3001143, 'entity1027': 1635, 'entity1028': 1625, 'entity1029': 2001149, 'entity1030': 2001139, 'entity1031': 2001129, 'entity1032': 2001118, 'entity1033': 2001108, 'entity1034': 1001104, 'entity1035': 1001094, 'entity1036': 1610, 'entity1037': 1600, 'entity1038': 2001101, 'entity1039': 2001091, 'entity1040': 1591, 'entity1041': 1581, 'entity1042': 1571, 'entity1043': 3001103, 'entity1044': 3001093, 'entity1045': 1001058, 'entity1046': 1001048, 'entity1047': 1564, 'entity1048': 1554, 'entity1049': 3001132}}}
  13. 67
  14. 68 """
  15. 69 This function validates and sends the param_list to the specific queue
  16. 70 for each users. The client is token aware
  17. 71 :param param_list: The list of parameters to index. Each item must have
  18. 72 a property 'id'.
  19. 73 :type param_list: array of objects.
  20. 74 :returns: a list of all generated recids
  21. 75 """
  22. 76 10000 49475.0 4.9 0.2 if not isinstance(param_list, list):
  23. 77 10000 35636.0 3.6 0.1 param_list = [param_list]
  24. 78
  25. 79 10000 55108.0 5.5 0.2 indexation_per_shard = defaultdict(list)
  26. 80
  27. 81 10000 31377.0 3.1 0.1 is_new_format = True
  28. 82 20000 87649.0 4.4 0.3 for item in param_list:
  29. 83 10000 31872.0 3.2 0.1 if is_new_format:
  30. 84
  31. 85 510000 1644522.0 3.2 6.1 for entity, shard_for_entity in item["meta"]["entities"].items():
  32. 86 500000 5819984.0 11.6 21.7 shard = client.get_shard_id(shard_for_entity)
  33. 87
  34. 88 500000 2250925.0 4.5 8.4 if shard in indexation_per_shard:
  35. 89 460000 1914855.0 4.2 7.1 indexation_item_list = indexation_per_shard[shard]
  36. 90
  37. 91 indexation_item = \
  38. 92 460000 2022991.0 4.4 7.5 indexation_per_shard[shard][len(indexation_item_list) - 1]
  39. 93
  40. 94 460000 1638716.0 3.6 6.1 indexation_item["data"][entity] = item["data"][entity]
  41. 95 460000 1586820.0 3.4 5.9 indexation_item["meta"]["entities"][entity] = shard_for_entity
  42. 96 else:
  43. 97 40000 131905.0 3.3 0.5 indexation_item = {}
  44. 98 40000 135220.0 3.4 0.5 indexation_item["data"] = {}
  45. 99
  46. 100 40000 141414.0 3.5 0.5 indexation_item["data"][entity] = item["data"][entity]
  47. 101 40000 130909.0 3.3 0.5 indexation_item["meta"] = {}
  48. 102 40000 136659.0 3.4 0.5 indexation_item["meta"]["group"] = item["meta"]["group"]
  49. 103 40000 133627.0 3.3 0.5 indexation_item["meta"]["test"] = item["meta"]["test"]
  50. 104 40000 133051.0 3.3 0.5 indexation_item["meta"]["project_id"] = item["meta"]["project_id"]
  51. 105
  52. 106 40000 130311.0 3.3 0.5 if "reindexing" in item["meta"]:
  53. 107 indexation_item["meta"]["reindexing"] = \
  54. 108 item["meta"]["reindexing"]
  55. 109
  56. 110 40000 135906.0 3.4 0.5 indexation_item["meta"]["entities"] = {}
  57. 111 40000 133017.0 3.3 0.5 indexation_item["meta"]["entities"][entity] =shard_for_entity
  58. 112 40000 271638.0 6.8 1.0 indexation_per_shard[shard].append(indexation_item)
  59. 113
  60. 114 else:
  61. 115 shard = client.get_shard_id(item['id'])
  62. 116 indexation_per_shard[shard].append(item)
  63. 117
  64. 118 # if client.record_ids_generation_mode == "server_side":
  65. 119 # await index.index_record_id_server_side(client, indexation_per_shard)
  66. 120 # elif client.record_ids_generation_mode == "client_side":
  67. 121 # await index_record_id_client_side(client, indexation_per_shard, is_new_format)
  68. 122 # else:
  69. 123 # logging.error("Invalid recordIds generation mode!")
  70. 124 #
  71. 125 #
  72. 126 # async def index_record_id_client_side(client, indexation_per_shard, is_new_format):
  73. 127 10000 44973.0 4.5 0.2 record_ids_by_shard = {}
  74. 128
  75. 129 10000 76850.0 7.7 0.3 rec_timestamp = int(time.time() * 1000)
  76. 130
  77. 131 # Create indexing tasks
  78. 132 10000 283751.0 28.4 1.1 tasks = []
  79. 133
  80. 134 50000 195057.0 3.9 0.7 for shard, items in indexation_per_shard.items():
  81. 135 40000 127669.0 3.2 0.5 try:
  82. 136 40000 260969.0 6.5 1.0 record_ids_by_shard[shard] = index.record_id_generator.get_record_ids(shard,len(items))
  83. 137 40000 127731.0 3.2 0.5 except:
  84. 138 40000 234874.0 5.9 0.9 record_ids_by_shard[shard] = []
  85. 139
  86. 140 40000 178036.0 4.5 0.7 record_ids = record_ids_by_shard[shard]
  87. 141
  88. 142 80000 286975.0 3.6 1.1 for indexation in items:
  89. 143 40000 126642.0 3.2 0.5 if is_new_format:
  90. 144 40000 129812.0 3.2 0.5 indexation_meta_data = indexation["meta"]
  91. 145
  92. 146 40000 133071.0 3.3 0.5 if "record_id" not in indexation_meta_data:
  93. 147 40000 126353.0 3.2 0.5 try:
  94. 148 40000 145861.0 3.6 0.5 if len(record_ids)>=1:
  95. 149 record_id = record_ids.pop(0)
  96. 150 indexation_meta_data["record_id"] = record_id
  97. 151 else:
  98. 152 40000 320071.0 8.0 1.2 date_ = time.strftime("%d%m%Y")
  99. 153 40000 486608.0 12.2 1.8 if not client.record_ids_dict:
  100. 154 1 18.0 18.0 0.0 create_record_ids_directory(client)
  101. 155 1 3437.0 3437.0 0.0 client.record_ids_dict = shelve.open(get_record_ids_file_name(client))
  102. 156
  103. 157 1 4.0 4.0 0.0 recid_dict = client.record_ids_dict
  104. 158
  105. 159 40000 185002.0 4.6 0.7 if len(recid_dict) == 0:
  106. 160 1 143.0 143.0 0.0 recid_dict = get_record_id_dict(client)
  107. 161 40000 913537.0 22.8 3.4 if date_ != recid_dict.get("date", None):
  108. 162 1 21.0 21.0 0.0 recid_dict.clear()
  109. 163 1 41.0 41.0 0.0 recid_dict['date'] = date_
  110. 164
  111. 165 40000 191487.0 4.8 0.7 key = str(shard.id)
  112. 166 40000 665426.0 16.6 2.5 record_id_value = recid_dict.get(key, None)
  113. 167
  114. 168 40000 136373.0 3.4 0.5 if record_id_value is None:
  115. 169 4 22.0 5.5 0.0 recid_dict = get_record_id_dict(client)
  116. 170 4 16.0 4.0 0.0 record_id_value = recid_dict.get(key, 0)
  117. 171 4 42.0 10.5 0.0 recid_dict[key] = record_id_value + 1
  118. 172 else:
  119. 173 39996 981933.0 24.6 3.7 recid_dict[key] += 1
  120. 174
  121. 175 40000 156581.0 3.9 0.6 if client not in ALREADY_REGISTERED_CLIENT:
  122. 176 1 3.0 3.0 0.0 atexit.register(save_record_ids_dict, client)
  123. 177 1 3.0 3.0 0.0 ALREADY_REGISTERED_CLIENT.add(client)
  124. 178
  125. 179 40000 234293.0 5.9 0.9 result = "{}#{}#{}".format(record_id_value, date_, client.origin)
  126. 180 40000 143631.0 3.6 0.5 indexation_meta_data["record_id"] =result
  127. 181 #await client.get_auto_generated_recid(shard)
  128. 182 except Exception as e:
  129. 183 logging.error(
  130. 184 "Error trying to generate recordId: %s" % e,
  131. 185 exc_info=True)
  132. 186
  133. 187 40000 136838.0 3.4 0.5 if "record_timestamp" not in indexation_meta_data:
  134. 188 40000 153771.0 3.8 0.6 indexation_meta_data["record_timestamp"] = rec_timestamp
  135. 189 else:
  136. 190 records = indexation['records']
  137. 191
  138. 192 for record in records:
  139. 193 if 'rec_id' not in record:
  140. 194 try:
  141. 195 record['rec_id'] = record_ids.pop(0)
  142. 196 except IndexError:
  143. 197 record['rec_id'] = \
  144. 198 await client.get_auto_generated_recid(shard)
  145. 199 except Exception as e:
  146. 200 logging.error(
  147. 201 "Error trying to generate recordId: %s" % e,
  148. 202 exc_info=True)
  149. 203
  150. 204 if 'rec_timestamp' not in record:
  151. 205 record['rec_timestamp'] = rec_timestamp
  152. 206
  153. 207 40000 289539.0 7.2 1.1 task = client.indexer.run(topic_id=shard, items=items)
  154. 208 40000 149844.0 3.7 0.6 tasks.append(task)
  155.  
  156.  
  157. Process finished with exit code 0
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!