Unhandled exception in noc-sae
Unhandled exception in noc-sae
File: /opt/noc/lib/python2.7/site-packages/django/db/models/query.py, Line: 366
Traceback:
File: /opt/noc/lib/python2.7/site-packages/django/db/models/query.py (Line: 366)
Function: get
  359             if self.query.can_filter():
  360                 clone = clone.order_by()
  361             num = len(clone)
  362             if num == 1:
  363                 return clone._result_cache[0]
  364             if not num:
  365                 raise self.model.DoesNotExist("%s matching query does not exist."
  366 ==>                     % self.model._meta.object_name)
  367             raise self.model.MultipleObjectsReturned("get() returned more than one %s -- it returned %s! Lookup parameters were %s"
  368                     % (self.model._meta.object_name, num, kwargs))
  369
  370         def create(self, **kwargs):
  371             """
  372             Creates a new object with the given kwargs, saving it to the database
Variables:
               clone = []
                 num = 0
                self =
[<ReduceTask: 23803608>, <ReduceTask: 23803609>, <ReduceTask: 23803610>, <ReduceTask: 23803611>, <ReduceTask: 23803612>, <ReduceTask: 23803613>, <ReduceTask: 23803614>, <ReduceTask: 23803615>, <ReduceTask: 23803616>, <ReduceTask: 23803619>, <ReduceTask: 23803620>, <ReduceTask: 23803621>, <ReduceTask: 23803625>, <ReduceTask: 23803627>, <ReduceTask: 23803628>, <ReduceTask: 23803599>, <ReduceTask: 23803606>, <ReduceTask: 23803607>, <ReduceTask: 23803630>, <ReduceTask: 23803631>, '...(remaining elements truncated)...']
                args = ()
              kwargs = {'id__exact': 23803598}
------------------------------------------------------------------------
File: /opt/noc/lib/python2.7/site-packages/django/db/models/fields/related.py (Line: 350)
Function: __get__
  343                     raise self.field.rel.to.DoesNotExist
  344                 other_field = self.field.rel.get_related_field()
  345                 if other_field.rel:
  346                     params = {'%s__pk' % self.field.rel.field_name: val}
  347                 else:
  348                     params = {'%s__exact' % self.field.rel.field_name: val}
  349                 qs = self.get_query_set(instance=instance)
  350 ==>             rel_obj = qs.get(**params)
  351                 setattr(instance, self.cache_name, rel_obj)
  352                 return rel_obj
  353
  354         def __set__(self, instance, value):
  355             if instance is None:
  356                 raise AttributeError("%s must be accessed via instance" % self._field.name)
Variables:
                  qs =
[<ReduceTask: 23803608>, <ReduceTask: 23803609>, <ReduceTask: 23803610>, <ReduceTask: 23803611>, <ReduceTask: 23803612>, <ReduceTask: 23803613>, <ReduceTask: 23803614>, <ReduceTask: 23803615>, <ReduceTask: 23803616>, <ReduceTask: 23803619>, <ReduceTask: 23803620>, <ReduceTask: 23803621>, <ReduceTask: 23803625>, <ReduceTask: 23803627>, <ReduceTask: 23803628>, <ReduceTask: 23803599>, <ReduceTask: 23803606>, <ReduceTask: 23803607>, <ReduceTask: 23803630>, <ReduceTask: 23803631>, '...(remaining elements truncated)...']
                 val = 23803598
            instance = <MapTask: 23804068: nnov-a3 Juniper.JUNOS.get_config>
         other_field = <django.db.models.fields.AutoField: id>
                self =
<django.db.models.fields.related.ReverseSingleRelatedObjectDescriptor object at 0x4597550>
       instance_type = <class 'noc.sa.models.maptask.MapTask'>
              params = {'id__exact': 23803598}
------------------------------------------------------------------------
File: /opt/noc/sa/sae/sae.py (Line: 428)
Function: log_mrt
  421
  422         def log_mrt(self, level, task, status, args=None, **kwargs):
  423             """
  424             Map/Reduce task logging
  425             """
  426             # Log into logfile
  427             rt = u"-"
  428 ==>         if task.task:
  429                 try:
  430                     rt = u"%s" % task.task.id
  431                 except ReduceTask.DoesNotExist:
  432                     rt = u"?"
  433             r = [u"MRT task=%s/%d object=%s(%s) script=%s status=%s" % (
  434                     rt, task.id, task.managed_object.name,
Variables:
                  rt = u'-'
              status = 'failed'
                task = <MapTask: 23804068: nnov-a3 Juniper.JUNOS.get_config>
               level = 20
                self = <noc.sa.sae.sae.SAE object at 0x2430550>
                args = None
              kwargs = {'code': 17, 'error': 'Timed out'}
------------------------------------------------------------------------
File: /opt/noc/sa/sae/sae.py (Line: 541)
Function: fail_task
  534                 mt.status = "F"
  535                 mt.script_result = dict(code=code, text=text)
  536                 try:
  537                     mt.save()
  538                 except Exception:
  539                     pass  # Can raise integrity error if MRT is gone
  540                 self.log_mrt(logging.INFO, task=mt, status="failed",
  541 ==>                 code=code, error=text)
  542
  543             t = datetime.datetime.now()
  544             # logging.debug("Processing MRT schedules")
  545             # Reset rates
  546             sae_mrt_rate = 0
  547             shard_mrt_rate = {}  # shard_id -> count
Variables:
                text = 'Timed out'
                self = <noc.sa.sae.sae.SAE object at 0x2430550>
                code = 17
                  mt = <MapTask: 23804068: nnov-a3 Juniper.JUNOS.get_config>
------------------------------------------------------------------------
File: /opt/noc/sa/sae/sae.py (Line: 569)
Function: process_mrtasks
  562                 is_valid_reduce = True
  563                 try:
  564                     mt.task
  565                 except ReduceTask.DoesNotExist:
  566                     is_valid_reduce = False
  567                 # Check for task timeouts
  568                 if not is_valid_reduce or (mt.task and mt.task.stop_time < t):
  569 ==>                 fail_task(mt, ERR_TIMEOUT, text="Timed out")
  570                     continue
  571                 # Check blocked pools
  572                 if mt.managed_object.activator.name in self.blocked_pools:
  573                     # Silently skip task until next round
  574                     continue
  575                 # Check for global rate limit
Variables:
        map_callback = <function map_callback at 0x7384320>
         exec_script = <function exec_script at 0x7384050>
                self = <noc.sa.sae.sae.SAE object at 0x2430550>
     is_valid_reduce = False
    throttled_shards = set([])
                  mt = <MapTask: 23804068: nnov-a3 Juniper.JUNOS.get_config>
                   t = datetime.datetime(2014, 8, 22, 14, 11, 54, 306313)
           fail_task = <function fail_task at 0x7384758>
        sae_mrt_rate = 0
      shard_mrt_rate = {}
------------------------------------------------------------------------
File: /opt/noc/sa/sae/sae.py (Line: 248)
Function: tick
  241             Called every second. Performs periodic maintainance
  242             and runs pending Map/Reduce tasks
  243             """
  244             t = time.time()
  245             reset_queries()  # Clear debug SQL log
  246             if t - self.last_mrtask_check >= self.mrt_schedule_interval:
  247                 # Check Map/Reduce task status
  248 ==>             self.process_mrtasks()
  249                 self.last_mrtask_check = t
  250             if t - self.last_status_refresh >= self.activator_status_interval:
  251                 self.refresh_activator_status()
  252
  253         def write_event(self, data, timestamp=None, managed_object=None):
  254             """
Variables:
                self = <noc.sa.sae.sae.SAE object at 0x2430550>
                   t = 1408702314.30627
------------------------------------------------------------------------
File: /opt/noc/lib/nbsocket/socketfactory.py (Line: 246)
Function: run
  239                     time.sleep(1)
  240             last_tick = last_stale = time.time()
  241             while cond() and not self.to_shutdown:
  242                 self.loop(1)
  243                 t = time.time()
  244                 if self.tick_callback and t - last_tick >= 1:
  245                     try:
  246 ==>                     self.tick_callback()
  247                     except Exception:
  248                         error_report()
  249                         logging.info("Restoring from tick() failure")
  250                     last_tick = t
  251                 if t - last_stale >= 1:
  252                     self.close_stale()
Variables:
                self = <noc.lib.nbsocket.socketfactory.SocketFactory object at 0x3473290>
                cond = <function <lambda> at 0x4d28410>
                   t = 1408702314.306261
          last_stale = 1408702313.230948
         run_forever = True
           last_tick = 1408702313.230948
------------------------------------------------------------------------