Guest User

2014-05-07, TSergey, sae error log

a guest
May 6th, 2014
282
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 8.54 KB | None | 0 0
  1. 2014-05-06 12:15:46,201 UNHANDLED EXCEPTION (2014-05-06 12:15:46.135909)
  2. Working directory: /opt/noc
  3. <class 'noc.sa.models.reducetask.DoesNotExist'>
  4. ReduceTask matching query does not exist.
  5. START OF TRACEBACK
  6. ------------------------------------------------------------------------
  7. File: /opt/noc/lib/python2.7/site-packages/django/db/models/query.py (Line: 366)
  8. Function: get
  9. 359 if self.query.can_filter():
  10. 360 clone = clone.order_by()
  11. 361 num = len(clone)
  12. 362 if num == 1:
  13. 363 return clone._result_cache[0]
  14. 364 if not num:
  15. 365 raise self.model.DoesNotExist("%s matching query does not exist."
  16. 366 ==> % self.model._meta.object_name)
  17. 367 raise self.model.MultipleObjectsReturned("get() returned more than one %s -- it returned %s! Lookup parameters were %s"
  18. 368 % (self.model._meta.object_name, num, kwargs))
  19. 369
  20. 370 def create(self, **kwargs):
  21. 371 """
  22. 372 Creates a new object with the given kwargs, saving it to the database
  23. Variables:
  24. clone = []
  25. num = 0
  26. self =
  27. [<ReduceTask: 1755430>, <ReduceTask: 1755431>, <ReduceTask: 1755432>, <ReduceTask: 1755434>, <ReduceTask: 1755436>, <ReduceTask: 1755437>, <ReduceTask: 1755438>, <ReduceTask: 1751669>, <ReduceTask: 1751672>, <ReduceTask: 1751475>, <ReduceTask: 1752054>, <ReduceTask: 1752236>, <ReduceTask: 1752727>, <ReduceTask: 1753948>, <ReduceTask: 1754030>, <ReduceTask: 1753892>, <ReduceTask: 1752009>, <ReduceTask: 1753958>, <ReduceTask: 1754031>, <ReduceTask: 1752015>, '...(remaining elements truncated)...']
  28. args = ()
  29. kwargs = {'id__exact': 1755429}
  30. ------------------------------------------------------------------------
  31. File: /opt/noc/lib/python2.7/site-packages/django/db/models/fields/related.py (Line: 350)
  32. Function: __get__
  33. 343 raise self.field.rel.to.DoesNotExist
  34. 344 other_field = self.field.rel.get_related_field()
  35. 345 if other_field.rel:
  36. 346 params = {'%s__pk' % self.field.rel.field_name: val}
  37. 347 else:
  38. 348 params = {'%s__exact' % self.field.rel.field_name: val}
  39. 349 qs = self.get_query_set(instance=instance)
  40. 350 ==> rel_obj = qs.get(**params)
  41. 351 setattr(instance, self.cache_name, rel_obj)
  42. 352 return rel_obj
  43. 353
  44. 354 def __set__(self, instance, value):
  45. 355 if instance is None:
  46. 356 raise AttributeError("%s must be accessed via instance" % self._field.name)
  47. Variables:
  48. qs =
  49. [<ReduceTask: 1755430>, <ReduceTask: 1755431>, <ReduceTask: 1755432>, <ReduceTask: 1755434>, <ReduceTask: 1755436>, <ReduceTask: 1755437>, <ReduceTask: 1755438>, <ReduceTask: 1751669>, <ReduceTask: 1751672>, <ReduceTask: 1751475>, <ReduceTask: 1752054>, <ReduceTask: 1752236>, <ReduceTask: 1752727>, <ReduceTask: 1753948>, <ReduceTask: 1754030>, <ReduceTask: 1753892>, <ReduceTask: 1752009>, <ReduceTask: 1753958>, <ReduceTask: 1754031>, <ReduceTask: 1752015>, '...(remaining elements truncated)...']
  50. val = 1755429
  51. instance = <MapTask: 1772085: s130-1-1.intt DLink.DES21xx.commands>
  52. other_field = <django.db.models.fields.AutoField: id>
  53. self =
  54. <django.db.models.fields.related.ReverseSingleRelatedObjectDescriptor object at 0x3ed1490>
  55. instance_type = <class 'noc.sa.models.maptask.MapTask'>
  56. params = {'id__exact': 1755429}
  57. ------------------------------------------------------------------------
  58. File: /opt/noc/sa/sae/sae.py (Line: 490)
  59. Function: process_mrtasks
  60. 483 # Check reduce task still valid
  61. 484 is_valid_reduce = True
  62. 485 try:
  63. 486 mt.task
  64. 487 except ReduceTask.DoesNotExist:
  65. 488 is_valid_reduce = False
  66. 489 # Check for task timeouts
  67. 490 ==> if mt.task.stop_time < t or not is_valid_reduce:
  68. 491 mt.status = "F"
  69. 492 mt.script_result = dict(code=ERR_TIMEOUT, text="Timed out")
  70. 493 try:
  71. 494 mt.save()
  72. 495 except Exception:
  73. 496 pass # Can raise integrity error if MRT is gone
  74. Variables:
  75. map_callback = <function map_callback at 0x46a79b0>
  76. exec_script = <function exec_script at 0x5209578>
  77. self = <noc.sa.sae.sae.SAE object at 0x7fc066509610>
  78. is_valid_reduce = False
  79. throttled_shards = set([])
  80. mt = <MapTask: 1772085: s130-1-1.intt DLink.DES21xx.commands>
  81. t = datetime.datetime(2014, 5, 6, 12, 15, 41, 198116)
  82. sae_mrt_rate = 0
  83. shard_mrt_rate = {}
  84. ------------------------------------------------------------------------
  85. File: /opt/noc/sa/sae/sae.py (Line: 198)
  86. Function: tick
  87. 191 Called every second. Performs periodic maintainance
  88. 192 and runs pending Map/Reduce tasks
  89. 193 """
  90. 194 t = time.time()
  91. 195 reset_queries() # Clear debug SQL log
  92. 196 if t - self.last_mrtask_check >= self.mrt_schedule_interval:
  93. 197 # Check Map/Reduce task status
  94. 198 ==> self.process_mrtasks()
  95. 199 self.last_mrtask_check = t
  96. 200 if t - self.last_status_refresh >= self.activator_status_interval:
  97. 201 self.refresh_activator_status()
  98. 202
  99. 203 def write_event(self, data, timestamp=None, managed_object=None):
  100. 204 """
  101. Variables:
  102. self = <noc.sa.sae.sae.SAE object at 0x7fc066509610>
  103. t = 1399364141.198082
  104. ------------------------------------------------------------------------
  105. File: /opt/noc/lib/nbsocket/socketfactory.py (Line: 246)
  106. Function: run
  107. 239 time.sleep(1)
  108. 240 last_tick = last_stale = time.time()
  109. 241 while cond() and not self.to_shutdown:
  110. 242 self.loop(1)
  111. 243 t = time.time()
  112. 244 if self.tick_callback and t - last_tick >= 1:
  113. 245 try:
  114. 246 ==> self.tick_callback()
  115. 247 except Exception:
  116. 248 error_report()
  117. 249 logging.info("Restoring from tick() failure")
  118. 250 last_tick = t
  119. 251 if t - last_stale >= 1:
  120. 252 self.close_stale()
  121. Variables:
  122. self = <noc.lib.nbsocket.socketfactory.SocketFactory object at 0x2e66b50>
  123. cond = <function <lambda> at 0x46a7848>
  124. t = 1399364141.198074
  125. last_stale = 1399364139.838194
  126. run_forever = True
  127. last_tick = 1399364139.838194
  128. ------------------------------------------------------------------------
  129. END OF TRACEBACK
  130. 2014-05-06 12:15:46,752 Restoring from tick() failure
  131. 2014-05-06 12:15:46,760 Late answer for map task 1772120 is ignored
  132. 2014-05-06 12:15:46,764 Late answer for map task 1772638 is ignored
  133. 2014-05-06 12:15:46,769 Late answer for map task 1772339 is ignored
  134. 2014-05-06 12:15:46,772 Late answer for map task 1772227 is ignored
  135. 2014-05-06 12:15:46,774 Late answer for map task 1772367 is ignored
  136. 2014-05-06 12:15:46,776 Late answer for map task 1772338 is ignored
  137. 2014-05-06 12:15:46,777 Late answer for map task 1772238 is ignored
  138. 2014-05-06 12:15:46,779 Late answer for map task 1772235 is ignored
  139. 2014-05-06 12:15:46,780 Late answer for map task 1772233 is ignored
  140. 2014-05-06 12:15:46,781 Late answer for map task 1772809 is ignored
  141. 2014-05-06 12:15:46,782 Late answer for map task 1772112 is ignored
  142. 2014-05-06 12:15:46,783 Late answer for map task 1772234 is ignored
  143. 2014-05-06 12:15:46,785 Late answer for map task 1772206 is ignored
  144. 2014-05-06 12:15:46,786 Late answer for map task 1772843 is ignored
  145. 2014-05-06 12:15:46,787 Late answer for map task 1772192 is ignored
  146. 2014-05-06 12:15:46,788 Late answer for map task 1772453 is ignored
  147. 2014-05-06 12:15:46,789 Late answer for map task 1772317 is ignored
  148. 2014-05-06 12:15:46,790 Late answer for map task 1772326 is ignored
  149. 2014-05-06 12:15:47,123 MRT task=1755430/1772970 object=s306-2-1.intt(192.168.250.108) script=DLink.DxS.get_inventory status=completed
Advertisement
Add Comment
Please, Sign In to add comment