Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- UNHANDLED EXCEPTION (2014-10-09 05:30:23.064442)
- Working directory: /opt/noc
- <class 'noc.sa.models.reducetask.DoesNotExist'>
- ReduceTask matching query does not exist.
- START OF TRACEBACK
- ------------------------------------------------------------------------
- File: /opt/noc/lib/python2.7/site-packages/django/db/models/query.py (Line: 366)
- Function: get
- 359 if self.query.can_filter():
- 360 clone = clone.order_by()
- 361 num = len(clone)
- 362 if num == 1:
- 363 return clone._result_cache[0]
- 364 if not num:
- 365 raise self.model.DoesNotExist("%s matching query does not exist."
- 366 ==> % self.model._meta.object_name)
- 367 raise self.model.MultipleObjectsReturned("get() returned more than one %s -- it returned %s! Lookup parameters were %s"
- 368 % (self.model._meta.object_name, num, kwargs))
- 369
- 370 def create(self, **kwargs):
- 371 """
- 372 Creates a new object with the given kwargs, saving it to the database
- Variables:
- clone = []
- num = 0
- self =
- [<ReduceTask: 4182064>, <ReduceTask: 4181442>, <ReduceTask: 4181443>, <ReduceTask: 4182065>, <ReduceTask: 4181445>, <ReduceTask: 4140482>, <ReduceTask: 4182066>, <ReduceTask: 4181751>, <ReduceTask: 4182067>, <ReduceTask: 4182068>, <ReduceTask: 4182069>, <ReduceTask: 4182070>, <ReduceTask: 4181450>, <ReduceTask: 4182071>, <ReduceTask: 4182072>, <ReduceTask: 4182073>, <ReduceTask: 4182074>, <ReduceTask: 4182075>, <ReduceTask: 4141483>, <ReduceTask: 4142025>, '...(remaining elements truncated)...']
- args = ()
- kwargs = {'id__exact': 4181408}
- ------------------------------------------------------------------------
- File: /opt/noc/lib/python2.7/site-packages/django/db/models/fields/related.py (Line: 350)
- Function: __get__
- 343 raise self.field.rel.to.DoesNotExist
- 344 other_field = self.field.rel.get_related_field()
- 345 if other_field.rel:
- 346 params = {'%s__pk' % self.field.rel.field_name: val}
- 347 else:
- 348 params = {'%s__exact' % self.field.rel.field_name: val}
- 349 qs = self.get_query_set(instance=instance)
- 350 ==> rel_obj = qs.get(**params)
- 351 setattr(instance, self.cache_name, rel_obj)
- 352 return rel_obj
- 353
- 354 def __set__(self, instance, value):
- 355 if instance is None:
- 356 raise AttributeError("%s must be accessed via instance" % self._field.name)
- Variables:
- qs =
- [<ReduceTask: 4182064>, <ReduceTask: 4181442>, <ReduceTask: 4181443>, <ReduceTask: 4182065>, <ReduceTask: 4181445>, <ReduceTask: 4140482>, <ReduceTask: 4182066>, <ReduceTask: 4181751>, <ReduceTask: 4182067>, <ReduceTask: 4182068>, <ReduceTask: 4182069>, <ReduceTask: 4182070>, <ReduceTask: 4181450>, <ReduceTask: 4182071>, <ReduceTask: 4182072>, <ReduceTask: 4182073>, <ReduceTask: 4182074>, <ReduceTask: 4182075>, <ReduceTask: 4141483>, <ReduceTask: 4142025>, '...(remaining elements truncated)...']
- val = 4181408
- instance = <MapTask: 4235569: s684-1-1.intt DLink.DxS.get_interfaces>
- other_field = <django.db.models.fields.AutoField: id>
- self =
- <django.db.models.fields.related.ReverseSingleRelatedObjectDescriptor object at 0x43c1610>
- instance_type = <class 'noc.sa.models.maptask.MapTask'>
- params = {'id__exact': 4181408}
- ------------------------------------------------------------------------
- File: /opt/noc/sa/sae/sae.py (Line: 428)
- Function: log_mrt
- 421
- 422 def log_mrt(self, level, task, status, args=None, **kwargs):
- 423 """
- 424 Map/Reduce task logging
- 425 """
- 426 # Log into logfile
- 427 rt = u"-"
- 428 ==> if task.task:
- 429 try:
- 430 rt = u"%s" % task.task.id
- 431 except ReduceTask.DoesNotExist:
- 432 rt = u"?"
- 433 r = [u"MRT task=%s/%d object=%s(%s) script=%s status=%s" % (
- 434 rt, task.id, task.managed_object.name,
- Variables:
- rt = u'-'
- status = 'failed'
- task = <MapTask: 4235569: s684-1-1.intt DLink.DxS.get_interfaces>
- level = 20
- self = <noc.sa.sae.sae.SAE object at 0x7f6f5561c610>
- args = None
- kwargs = {'code': 17, 'error': 'Timed out'}
- ------------------------------------------------------------------------
- File: /opt/noc/sa/sae/sae.py (Line: 541)
- Function: fail_task
- 534 mt.status = "F"
- 535 mt.script_result = dict(code=code, text=text)
- 536 try:
- 537 mt.save()
- 538 except Exception:
- 539 pass # Can raise integrity error if MRT is gone
- 540 self.log_mrt(logging.INFO, task=mt, status="failed",
- 541 ==> code=code, error=text)
- 542
- 543 t = datetime.datetime.now()
- 544 # logging.debug("Processing MRT schedules")
- 545 # Reset rates
- 546 sae_mrt_rate = 0
- 547 shard_mrt_rate = {} # shard_id -> count
- Variables:
- text = 'Timed out'
- self = <noc.sa.sae.sae.SAE object at 0x7f6f5561c610>
- code = 17
- mt = <MapTask: 4235569: s684-1-1.intt DLink.DxS.get_interfaces>
- ------------------------------------------------------------------------
- File: /opt/noc/sa/sae/sae.py (Line: 569)
- Function: process_mrtasks
- 562 is_valid_reduce = True
- 563 try:
- 564 mt.task
- 565 except ReduceTask.DoesNotExist:
- 566 is_valid_reduce = False
- 567 # Check for task timeouts
- 568 if not is_valid_reduce or (mt.task and mt.task.stop_time < t):
- 569 ==> fail_task(mt, ERR_TIMEOUT, text="Timed out")
- 570 continue
- 571 # Check blocked pools
- 572 if mt.managed_object.activator.name in self.blocked_pools:
- 573 # Silently skip task until next round
- 574 continue
- 575 # Check for global rate limit
- Variables:
- map_callback = <function map_callback at 0x7b53cf8>
- exec_script = <function exec_script at 0x7b530c8>
- self = <noc.sa.sae.sae.SAE object at 0x7f6f5561c610>
- is_valid_reduce = False
- throttled_shards = set([])
- mt = <MapTask: 4235569: s684-1-1.intt DLink.DxS.get_interfaces>
- t = datetime.datetime(2014, 10, 9, 5, 30, 22, 489473)
- fail_task = <function fail_task at 0x7b532a8>
- sae_mrt_rate = 0
- shard_mrt_rate = {}
- ------------------------------------------------------------------------
- File: /opt/noc/sa/sae/sae.py (Line: 248)
- Function: tick
- 241 Called every second. Performs periodic maintainance
- 242 and runs pending Map/Reduce tasks
- 243 """
- 244 t = time.time()
- 245 reset_queries() # Clear debug SQL log
- 246 if t - self.last_mrtask_check >= self.mrt_schedule_interval:
- 247 # Check Map/Reduce task status
- 248 ==> self.process_mrtasks()
- 249 self.last_mrtask_check = t
- 250 if t - self.last_status_refresh >= self.activator_status_interval:
- 251 self.refresh_activator_status()
- 252
- 253 def write_event(self, data, timestamp=None, managed_object=None):
- 254 """
- Variables:
- self = <noc.sa.sae.sae.SAE object at 0x7f6f5561c610>
- t = 1412818222.489437
- ------------------------------------------------------------------------
- File: /opt/noc/lib/nbsocket/socketfactory.py (Line: 246)
- Function: run
- 239 time.sleep(1)
- 240 last_tick = last_stale = time.time()
- 241 while cond() and not self.to_shutdown:
- 242 self.loop(1)
- 243 t = time.time()
- 244 if self.tick_callback and t - last_tick >= 1:
- 245 try:
- 246 ==> self.tick_callback()
- 247 except Exception:
- 248 error_report()
- 249 logging.info("Restoring from tick() failure")
- 250 last_tick = t
- 251 if t - last_stale >= 1:
- 252 self.close_stale()
- Variables:
- self = <noc.lib.nbsocket.socketfactory.SocketFactory object at 0x3284450>
- cond = <function <lambda> at 0x4b77e60>
- t = 1412818222.489429
- last_stale = 1412818218.890035
- run_forever = True
- last_tick = 1412818218.890035
- ------------------------------------------------------------------------
- END OF TRACEBACK
Advertisement
Add Comment
Please, Sign In to add comment
