2012-09-03 12:21:10,283 UNHANDLED EXCEPTION (2012-09-03 12:21:10.281462)
Working directory: /opt/noc
<type 'exceptions.KeyError'>
'schedule'
START OF TRACEBACK
------------------------------------------------------------------------
File: /opt/noc/lib/scheduler/scheduler.py (Line: 263)
Function: run_pending
  256                     # Invalid job class. Park job to FAIL state
  257                     self.error("Invalid job class: %s" % jcls)
  258                     self.set_job_status(job_data[self.ATTR_CLASS],
  259                         job_data[self.ATTR_KEY], self.S_FAIL)
  260                     continue
  261                 job = jcls(self,
  262                     job_data[self.ATTR_KEY], job_data[self.ATTR_DATA],
  263 ==>                 job_data[self.ATTR_SCHEDULE]
  264                 )
  265                 self.run_job(job)
  266                 n += 1
  267             return n
  268
  269         def run(self):
Variables:
                self = <noc.fm.correlator.scheduler.CorrelatorScheduler object at 0x807e7e990>
            job_data = {u'runs': 27408, u'last': datetime.datetime(2012, 9, 3, 12, 5, 0, 232000), u'log': [], u'jcls': u'performance_report', u'ldur': 0.00045299530029296875, u'ts': datetime.datetime(2012, 9, 3, 12, 6), u'tb': None, u's': u'W', u'ls': u'S', u'key': u'report', u'_id': ObjectId('5026b5bef3b682136f000000'), u'data': {u'_noc': {u'interval': 60, u'offset': 0}}}
                jcls = <class 'noc.fm.correlator.jobs.performance_report.PerformanceReportJob'>
                   n = 0
------------------------------------------------------------------------
File: /opt/noc/lib/scheduler/scheduler.py (Line: 280)
Function: run
  273                 self.collection.update({
  274                     self.ATTR_STATUS: self.S_RUN
  275                 }, {
  276                     "$set": {self.ATTR_STATUS: self.S_WAIT}
  277                 }, multi=True, safe=True)
  278             self.info("Running scheduler")
  279             while True:
  280 ==>             if not self.run_pending():
  281                     time.sleep(1)
  282                 else:
  283                     self.cleanup()
  284
  285         def wipe(self):
  286             """
Variables:
                self = <noc.fm.correlator.scheduler.CorrelatorScheduler object at 0x807e7e990>
------------------------------------------------------------------------
File: /opt/noc/fm/correlator/daemon.py (Line: 384)
Function: run
  377                 self.stat_success_count += 1
  378
  379         def run(self):
  380             """
  381             Main daemon loop
  382             """
  383             self.reset_stats()
  384 ==>         self.scheduler.run()

Variables:
                self = <noc.fm.correlator.daemon.Correlator object at 0x806f32a50>
------------------------------------------------------------------------
File: /opt/noc/lib/daemon.py (Line: 364)
Function: guarded_run
  357
  358         def guarded_run(self):
  359             """
  360             Run daemon and catch common exceptions
  361             :return:
  362             """
  363             try:
  364 ==>             self.run()
  365             except KeyboardInterrupt:
  366                 pass
  367             except MemoryError:
  368                 logging.error("Out of memory. Exiting.")
  369             except SystemExit:
  370                 logging.info("Exiting")
Variables:
                self = <noc.fm.correlator.daemon.Correlator object at 0x806f32a50>
------------------------------------------------------------------------
END OF TRACEBACK