2012-09-10 15:12:33,447 [fm.correlator] Running scheduler
2012-09-10 15:12:33,454 UNHANDLED EXCEPTION (2012-09-10 15:12:33.450673)
Working directory: /usr/local/noc
<type 'exceptions.KeyError'>
'schedule'
START OF TRACEBACK
------------------------------------------------------------------------
File: /usr/local/noc/lib/scheduler/scheduler.py (Line: 315)
Function: run_pending
  308                     # Invalid job class. Park job to FAIL state
  309                     self.error("Invalid job class: %s" % jcls)
  310                     self.set_job_status(job_data[self.ATTR_CLASS],
  311                         job_data[self.ATTR_KEY], self.S_FAIL)
  312                     continue
  313                 job = jcls(self,
  314                     job_data[self.ATTR_KEY], job_data[self.ATTR_DATA],
  315 ==>                 job_data[self.ATTR_SCHEDULE]
  316                 )
  317                 self.run_job(job)
  318                 n += 1
  319             return n
  320
  321         def run(self):
Variables:
                jcls = <class 'noc.fm.correlator.jobs.performance_report.PerformanceReportJob'>
           throttled = set([])
                self = <noc.fm.correlator.scheduler.CorrelatorScheduler object at 0x80854dc10>
            job_data = {u'runs': 24490, u'last': datetime.datetime(2012, 9, 6, 11, 32, 0, 619000), u'log': [], u'jcls': u'performance_report', u'ldur': 0.0005838871002197266, u'ts': datetime.datetime(2012, 9, 6, 11, 33), u'tb': None, u's': u'W', u'ls': u'S', u'key': u'report', u'_id': ObjectId('5031e4e1bcf4b182a1000000'), u'data': {u'_noc': {u'interval': 60, u'offset': 0}}}
                   n = 0
                   q = {'s': 'W', 'ts': {'$lte': datetime.datetime(2012, 9, 10, 15, 12, 33, 447784)}}
------------------------------------------------------------------------
File: /usr/local/noc/lib/scheduler/scheduler.py (Line: 332)
Function: run
  325                 self.collection.update({
  326                     self.ATTR_STATUS: self.S_RUN
  327                 }, {
  328                     "$set": {self.ATTR_STATUS: self.S_WAIT}
  329                 }, multi=True, safe=True)
  330             self.info("Running scheduler")
  331             while True:
  332 ==>             if not self.run_pending():
  333                     time.sleep(1)
  334                 else:
  335                     self.cleanup()
  336
  337         def wipe(self):
  338             """
Variables:
                self = <noc.fm.correlator.scheduler.CorrelatorScheduler object at 0x80854dc10>
------------------------------------------------------------------------
File: /usr/local/noc/fm/correlator/daemon.py (Line: 384)
Function: run
  377                 self.stat_success_count += 1
  378
  379         def run(self):
  380             """
  381             Main daemon loop
  382             """
  383             self.reset_stats()
  384 ==>         self.scheduler.run()

Variables:
                self = <noc.fm.correlator.daemon.Correlator object at 0x807020a50>
------------------------------------------------------------------------
File: /usr/local/noc/lib/daemon.py (Line: 364)
Function: guarded_run
  357
  358         def guarded_run(self):
  359             """
  360             Run daemon and catch common exceptions
  361             :return:
  362             """
  363             try:
  364 ==>             self.run()
  365             except KeyboardInterrupt:
  366                 pass
  367             except MemoryError:
  368                 logging.error("Out of memory. Exiting.")
  369             except SystemExit:
  370                 logging.info("Exiting")
Variables:
                self = <noc.fm.correlator.daemon.Correlator object at 0x807020a50>
------------------------------------------------------------------------
END OF TRACEBACK
2012-09-10 15:12:33,455 Removing pidfile: /var/run/noc/noc-correlator.pid
2012-09-10 15:12:33,455 STOP