2014-11-26 12:35:07,832 [noc.lib.debug] UNHANDLED EXCEPTION (2014-11-26 12:35:07.823240)
Working directory: /opt/noc
<type 'exceptions.ValueError'>
year is out of range
START OF TRACEBACK
------------------------------------------------------------------------
File: /opt/noc/lib/scheduler/intervaljob.py (Line: 78)
Function: get_next_aligned
   71             """
   72             t = int(time.time())
   73             ts = (t // interval) * interval
   74             if next or ts < t:
   75                 ts += interval
   76             if offset:
   77                 ts += offset * interval
   78 ==>         return datetime.datetime.fromtimestamp(ts)
   79
   80         def get_interval(self):
   81             return self.schedule["interval"]
   82
   83         def get_failed_interval(self):
   84             return self.schedule.get("failed_interval",
Variables:
            interval = 31536000
                  ts = 452461309344000
                next = True
                   t = 1416994507
              offset = 14347409
                 cls = <class 'noc.inv.discovery.jobs.id_discovery.IDDiscoveryJob'>
------------------------------------------------------------------------
File: /opt/noc/lib/scheduler/intervaljob.py (Line: 93)
Function: get_schedule
   86
   87         def get_schedule(self, status):
   88             if status == self.S_SUCCESS:
   89                 i = self.get_interval()
   90                 if not i:
   91                     return None  # Zero interval means disabled job
   92                 offset = self.schedule["offset"] % i
   93 ==>             return self.get_next_aligned(i, next=True, offset=offset)
   94             elif status == self.S_DEFERRED:
   95                 return None  # Left until next initial submit
   96             elif status == self.S_LATE and self.delay_interval:
   97                 return (datetime.datetime.now() +
   98                         datetime.timedelta(
   99                             seconds=random.random() * self.delay_interval))
Variables:
                   i = 31536000
              status = 'S'
                self =
<noc.inv.discovery.jobs.id_discovery.IDDiscoveryJob object at 0x80ec80c10>
              offset = 14347409
------------------------------------------------------------------------
File: /opt/noc/lib/scheduler/scheduler.py (Line: 345)
Function: _complete_job
  338             group = job.get_group()
  339             if group is not None:
  340                 with self.running_lock:
  341                     self.running_count[group] -= 1
  342                     if not self.running_count[group]:
  343                         del self.running_count[group]
  344             on_complete = job.on_complete
  345 ==>         t = job.get_schedule(status)
  346             if t is None:
  347                 # Unschedule job
  348                 self.remove_job(job.name, job.key)
  349             else:
  350                 # Reschedule job
  351                 t1 = time.time()
Variables:
              status = 'S'
               group = 'discovery-144'
                self = <noc.inv.discovery.scheduler.DiscoveryScheduler object at 0x80e839810>
                 job =
<noc.inv.discovery.jobs.id_discovery.IDDiscoveryJob object at 0x80ec80c10>
         on_complete = []
                path = '/tmp/id_discovery/144'
                  tb = None
------------------------------------------------------------------------
File: /opt/noc/lib/scheduler/scheduler.py (Line: 330)
Function: _job_wrapper
  323                     s = job.S_SUCCESS
  324                 else:
  325                     job.logger.info("Job failed (%fsec)",
  326                         time.time() - t0
  327                     )
  328                     job.on_failure()
  329                     s = job.S_FAILED
  330 ==>         self._complete_job(job, s, tb)
  331
  332         def _complete_job(self, job, status, tb):
  333             self.metrics.jobs_time.timer(self.name, job.name, job.key).log(
  334                 job.started, time.time(), status)
  335             if self.to_log_jobs:
  336                 path = os.path.join(self.log_jobs, job.name, str(job.key))
Variables:
                self = <noc.inv.discovery.scheduler.DiscoveryScheduler object at 0x80e839810>
                 job =
<noc.inv.discovery.jobs.id_discovery.IDDiscoveryJob object at 0x80ec80c10>
                  t0 = 1416994507.811981
                   s = 'S'
                   r = True
              kwargs =
{'object': <ManagedObject: cat-balakhna>,
 'result': {'chassis_mac': [{'first_chassis_mac': 'EC:C8:82:E9:BF:80',
                             'last_chassis_mac': 'EC:C8:82:E9:BF:80'}],
            'hostname': 'cat-balakhna.kis.ru'}}
                  tb = None
------------------------------------------------------------------------
File: /opt/noc/lib/scheduler/scheduler.py (Line: 304)
Function: _run_job_handler
  297             if job.threaded:
  298                 t = threading.Thread(target=self._job_wrapper,
  299                     args=(job,), kwargs=kwargs
  300                 )
  301                 t.daemon = True
  302                 t.start()
  303             else:
  304 ==>             return self._job_wrapper(job, **kwargs)
  305
  306         def _job_wrapper(self, job, **kwargs):
  307             tb = None
  308             t0 = time.time()
  309             job.logger.info("Running job handler")
  310             try:
Variables:
                 job =
<noc.inv.discovery.jobs.id_discovery.IDDiscoveryJob object at 0x80ec80c10>
                self = <noc.inv.discovery.scheduler.DiscoveryScheduler object at 0x80e839810>
              kwargs =
{'object': <ManagedObject: cat-balakhna>,
 'result': {'chassis_mac': [{'first_chassis_mac': 'EC:C8:82:E9:BF:80',
                             'last_chassis_mac': 'EC:C8:82:E9:BF:80'}],
            'hostname': 'cat-balakhna.kis.ru'}}
------------------------------------------------------------------------
File: /opt/noc/lib/scheduler/scheduler.py (Line: 390)
Function: complete_mrt_job
  383                 self.reschedule_job(job_name, key, ts, skip_running=True)
  384
  385         def complete_mrt_job(self, t):
  386             job = self.active_mrt.pop(t)
  387             for m in t.maptask_set.all():
  388                 if m.status == "C":
  389                     self._run_job_handler(job, object=m.managed_object,
  390 ==>                     result=m.script_result)
  391                 else:
  392                     self.logger.info("Job %s(%s) is failed",
  393                         job.name, job.get_display_key())
  394                     self._complete_job(job, job.S_FAILED, m.script_result)
  395             t.delete()
  396
Variables:
                 job =
<noc.inv.discovery.jobs.id_discovery.IDDiscoveryJob object at 0x80ec80c10>
                   m = <MapTask: 888776: cat-balakhna Cisco.IOS.get_discovery_id>
                   t = <ReduceTask: 843069>
                self = <noc.inv.discovery.scheduler.DiscoveryScheduler object at 0x80e839810>
------------------------------------------------------------------------
File: /opt/noc/lib/scheduler/scheduler.py (Line: 424)
Function: run_pending
  417                         # Reschedule initial submit
  418                         self.initial_submit_next_check[jcls] = (
  419                             t0 + jcls.initial_submit_interval)
  420             # Check for complete MRT
  421             if self.active_mrt:
  422                 complete = [t for t in self.active_mrt if t.complete]
  423                 for t in complete:
  424 ==>                 self.complete_mrt_job(t)
  425                 self.active_mrt = dict(
  426                     (t, self.active_mrt[t])
  427                         for t in self.active_mrt if t not in complete)
  428             # Check for pending persistent tasks
  429             q = {
  430                 self.ATTR_TS: {"$lte": datetime.datetime.now()},
Variables:
                self = <noc.inv.discovery.scheduler.DiscoveryScheduler object at 0x80e839810>
                   t = <ReduceTask: 843069>
            complete = [<ReduceTask: 843069>]
                   n = 0
------------------------------------------------------------------------
File: /opt/noc/lib/scheduler/scheduler.py (Line: 482)
Function: run
  475                     self.ATTR_STATUS: self.S_RUN
  476                 }, {
  477                     "$set": {self.ATTR_STATUS: self.S_WAIT}
  478                 }, multi=True, safe=True)
  479             self.ensure_indexes()
  480             self.logger.info("Running scheduler")
  481             while True:
  482 ==>             if not self.run_pending():
  483                     time.sleep(1)
  484                 else:
  485                     self.cleanup()
  486
  487         def get_faults(self, job_name, key=None):
  488             """
Variables:
                self = <noc.inv.discovery.scheduler.DiscoveryScheduler object at 0x80e839810>
------------------------------------------------------------------------
File: /opt/noc/inv/discovery/daemon.py (Line: 37)
Function: run
   30
   31         def run(self):
   32             try:
   33                 PerformanceReportJob.submit(self.scheduler,
   34                     key="performance_report", interval=60)
   35             except self.scheduler.JobExists:
   36                 pass
   37 ==>         self.scheduler.run()
   38
   39         def load_config(self):
   40             super(DiscoveryDaemon, self).load_config()
   41             self.load_beef_map()
   42             log_jobs = self.config.get("main", "log_jobs") or None
   43             self.scheduler.set_job_log(log_jobs)
Variables:
                self = <noc.inv.discovery.daemon.DiscoveryDaemon object at 0x80073a6d0>
------------------------------------------------------------------------
File: /opt/noc/lib/daemon/base.py (Line: 413)
Function: guarded_run
  406
  407         def guarded_run(self):
  408             """
  409             Run daemon and catch common exceptions
  410             :return:
  411             """
  412             try:
  413 ==>             self.run()
  414             except KeyboardInterrupt:
  415                 pass
  416             except MemoryError:
  417                 self.logger.error("Out of memory. Exiting.")
  418             except SystemExit:
  419                 self.logger.info("Exiting")
Variables:
                self = <noc.inv.discovery.daemon.DiscoveryDaemon object at 0x80073a6d0>
------------------------------------------------------------------------
END OF TRACEBACK