START OF TRACEBACK
------------------------------------------------------------------------
File: core/dcs/base.py (Line: 272)
Function: resolve
  265                     t = self.dcs.DEFAULT_SERVICE_RESOLUTION_TIMEOUT
  266                 try:
  267                     yield self.ready_event.wait(timeout=t)
  268                 except tornado.gen.TimeoutError:
  269                     metrics["errors", ("type", "dcs_resolver_timeout")] += 1
  270                     if self.critical:
  271                         self.dcs.set_faulty_status("Failed to resolve %s: Timeout" % self.name)
  272 ==>                 raise ResolutionError()
  273             if not wait and not self.ready_event.is_set():
  274                 if self.critical:
  275                     self.dcs.set_faulty_status("Failed to resolve %s: No active services" % self.name)
  276                 raise ResolutionError()
  277             with self.lock:
  278                 if hint and hint in self.service_addresses:
Variables:
         full_result = False
                hint = None
                self = <noc.core.dcs.consuldcs.ConsulResolver object at 0x7f95a6cc3850>
                   t = datetime.timedelta(0, 300)
             timeout = None
                wait = True
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1063)
Function: run
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
 1062                             try:
 1063 ==>                             yielded = self.gen.throw(*exc_info)
 1064                             finally:
 1065                                 # Break up a reference to itself
 1066                                 # for faster GC on CPython.
 1067                                 exc_info = None
 1068                         else:
 1069                             yielded = self.gen.send(value)
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95a6cabe10>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/concurrent.py (Line: 238)
Function: result
  231             before the `Future` is done, so the ``timeout`` is never used.
  232             """
  233             self._clear_tb_log()
  234             if self._result is not None:
  235                 return self._result
  236             if self._exc_info is not None:
  237                 try:
  238 ==>                 raise_exc_info(self._exc_info)
  239                 finally:
  240                     self = None
  241             self._check_done()
  242             return self._result
  243
  244         def exception(self, timeout=None):
Variables:
                self = None
             timeout = None
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1055)
Function: run
 1048                         return
 1049                     self.future = None
 1050                     try:
 1051                         orig_stack_contexts = stack_context._state.contexts
 1052                         exc_info = None
 1053
 1054                         try:
 1055 ==>                         value = future.result()
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95a6c26f10>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: core/dcs/base.py (Line: 124)
Function: resolve
  117
  118         @tornado.gen.coroutine
  119         def resolve(self, name, hint=None, wait=True, timeout=None,
  120                     full_result=False, critical=False):
  121             resolver = yield self.get_resolver(name, critical)
  122             r = yield resolver.resolve(
  123                 hint=hint, wait=wait, timeout=timeout,
  124 ==>             full_result=full_result
  125             )
  126             raise tornado.gen.Return(r)
  127
  128         @tornado.gen.coroutine
  129         def expire_resolvers(self):
  130             with self.resolvers_lock:
Variables:
         full_result = False
                hint = None
                self = <noc.core.dcs.consuldcs.ConsulDCS object at 0x7f95b3dc86d0>
            resolver = <noc.core.dcs.consuldcs.ConsulResolver object at 0x7f95a6cc3850>
                name = 'datastream'
            critical = False
             timeout = None
                wait = True
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1063)
Function: run
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
 1062                             try:
 1063 ==>                             yielded = self.gen.throw(*exc_info)
 1064                             finally:
 1065                                 # Break up a reference to itself
 1066                                 # for faster GC on CPython.
 1067                                 exc_info = None
 1068                         else:
 1069                             yielded = self.gen.send(value)
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95a6c26f10>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/concurrent.py (Line: 238)
Function: result
  231             before the `Future` is done, so the ``timeout`` is never used.
  232             """
  233             self._clear_tb_log()
  234             if self._result is not None:
  235                 return self._result
  236             if self._exc_info is not None:
  237                 try:
  238 ==>                 raise_exc_info(self._exc_info)
  239                 finally:
  240                     self = None
  241             self._check_done()
  242             return self._result
  243
  244         def exception(self, timeout=None):
Variables:
                self = None
             timeout = None
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1055)
Function: run
 1048                         return
 1049                     self.future = None
 1050                     try:
 1051                         orig_stack_contexts = stack_context._state.contexts
 1052                         exc_info = None
 1053
 1054                         try:
 1055 ==>                         value = future.result()
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95b36388d0>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: core/datastream/client.py (Line: 100)
Function: resolve
   93                 if "X-NOC-DataStream-Last-Change" in headers:
   94                     change_id = headers["X-NOC-DataStream-Last-Change"]
   95                 elif not block:
   96                     break  # Empty batch, stop if non-blocking mode
   97
   98         @tornado.gen.coroutine
   99         def resolve(self, host):
  100 ==>         svc = yield self.service.dcs.resolve(host)
  101             host, port = svc.split(":")
  102             raise tornado.gen.Return((host, int(port)))
Variables:
                host = 'datastream'
                self =
<noc.services.ping.datastream.PingDataStreamClient object at 0x7f95a6cabfd0>
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1063)
Function: run
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
 1062                             try:
 1063 ==>                             yielded = self.gen.throw(*exc_info)
 1064                             finally:
 1065                                 # Break up a reference to itself
 1066                                 # for faster GC on CPython.
 1067                                 exc_info = None
 1068                         else:
 1069                             yielded = self.gen.send(value)
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95b36388d0>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/concurrent.py (Line: 238)
Function: result
  231             before the `Future` is done, so the ``timeout`` is never used.
  232             """
  233             self._clear_tb_log()
  234             if self._result is not None:
  235                 return self._result
  236             if self._exc_info is not None:
  237                 try:
  238 ==>                 raise_exc_info(self._exc_info)
  239                 finally:
  240                     self = None
  241             self._check_done()
  242             return self._result
  243
  244         def exception(self, timeout=None):
Variables:
                self = None
             timeout = None
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1055)
Function: run
 1048                         return
 1049                     self.future = None
 1050                     try:
 1051                         orig_stack_contexts = stack_context._state.contexts
 1052                         exc_info = None
 1053
 1054                         try:
 1055 ==>                         value = future.result()
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95a6c35390>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: core/http/client.py (Line: 150)
Function: fetch
  143             host = u.netloc
  144             port = DEFAULT_PORTS.get(u.scheme)
  145             if not port:
  146                 raise tornado.gen.Return((ERR_TIMEOUT, {}, "Cannot resolve port for scheme: %s" % u.scheme))
  147         if is_ipv4(host):
  148             addr = host
  149         else:
  150 ==>         addr = yield resolver(host)
  151         if not addr:
  152             raise tornado.gen.Return((ERR_TIMEOUT, {}, "Cannot resolve host: %s" % host))
  153         # Detect proxy server
  154         if allow_proxy:
  155             proxy = (proxies or SYSTEM_PROXIES).get(u.scheme)
  156         else:
Variables:
    follow_redirects = False
     max_buffer_size = 131072
             io_loop = <tornado.platform.epoll.EPollIOLoop object at 0x7f95a7888f90>
                port = 80
     request_timeout = 3600
            password = None
         allow_proxy = False
            eof_mark = None
             use_tls = False
              method = 'GET'
    content_encoding = None
     connect_timeout = 10
                body = None
     get_ssl_options = <function get_ssl_options at 0x7f95a6cb38c0>
                host = 'datastream'
                user = None
            resolver =
<bound method PingDataStreamClient.resolve of <noc.services.ping.datastream.PingDataStreamClient object at 0x7f95a6cabfd0>>
             proxies = None
       validate_cert = False
                 url =
'http://datastream/api/datastream/cfgping?filter=pool(default)&filter=shard(1,4)&block=1&limit=1000&from=5c4ac921b19531fdd9855d6a'
       max_redirects = 5
             headers = {'X-NOC-API-Access': 'datastream:cfgping'}
                   u =
ParseResult(scheme='http', netloc='datastream', path='/api/datastream/cfgping', params='', query='filter=pool(default)&filter=shard(1,4)&block=1&limit=1000&from=5c4ac921b19531fdd9855d6a', fragment='')
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1063)
Function: run
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
 1062                             try:
 1063 ==>                             yielded = self.gen.throw(*exc_info)
 1064                             finally:
 1065                                 # Break up a reference to itself
 1066                                 # for faster GC on CPython.
 1067                                 exc_info = None
 1068                         else:
 1069                             yielded = self.gen.send(value)
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95a6c35390>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/concurrent.py (Line: 238)
Function: result
  231             before the `Future` is done, so the ``timeout`` is never used.
  232             """
  233             self._clear_tb_log()
  234             if self._result is not None:
  235                 return self._result
  236             if self._exc_info is not None:
  237                 try:
  238 ==>                 raise_exc_info(self._exc_info)
  239                 finally:
  240                     self = None
  241             self._check_done()
  242             return self._result
  243
  244         def exception(self, timeout=None):
Variables:
                self = None
             timeout = None
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1055)
Function: run
 1048                         return
 1049                     self.future = None
 1050                     try:
 1051                         orig_stack_contexts = stack_context._state.contexts
 1052                         exc_info = None
 1053
 1054                         try:
 1055 ==>                         value = future.result()
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95a6c26e10>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: core/datastream/client.py (Line: 73)
Function: query
   66                 if qs:
   67                     url = "%s?%s" % (base_url, "&".join(qs))
   68                 else:
   69                     url = base_url
   70                 # Get data
   71                 logger.debug("Request: %s", url)
   72                 code, headers, data = yield fetch(url, io_loop=io_loop,
   73 ==>                                               resolver=self.resolve, headers=req_headers)
   74                 logger.debug("Response: %s %s", code, headers)
   75                 if code == ERR_TIMEOUT or code == ERR_READ_TIMEOUT:
   76                     continue  # Retry on timeout
   77                 elif code != 200:
   78                     logger.info("Invalid response code: %s", code)
   79                     raise NOCError(code=ERR_DS_BAD_CODE, msg="Invalid response code %s" % code)
Variables:
                  qs =
['filter=pool(default)',
 'filter=shard(1,4)',
 'block=1',
 'limit=1000',
 'from=5c4ac921b19531fdd9855d6a']
                code = 598
             headers = {}
                 url =
'http://datastream/api/datastream/cfgping?filter=pool(default)&filter=shard(1,4)&block=1&limit=1000&from=5c4ac921b19531fdd9855d6a'
             base_qs = ['filter=pool(default)', 'filter=shard(1,4)', 'block=1', 'limit=1000']
                self =
<noc.services.ping.datastream.PingDataStreamClient object at 0x7f95a6cabfd0>
            base_url = 'http://datastream/api/datastream/cfgping'
                item =
{u'address': u'188.68.184.75',
 u'bi_id': 6416174231453432276L,
 u'change_id': u'5c4ac921b19531fdd9855d6a',
 u'count': 3,
 u'id': u'29',
 u'interval': 60,
 u'name': u'MNpon',
 u'policy': u'f',
 u'pool': u'default',
 u'report_attempts': True,
 u'report_rtt': True,
 u'size': 64,
 u'status': None,
 u'timeout': 1000}
               limit = 1000
             filters = ['pool(default)', 'shard(1,4)']
           change_id = '5c4ac921b19531fdd9855d6a'
         req_headers = {'X-NOC-API-Access': 'datastream:cfgping'}
                   x = 'shard(1,4)'
             io_loop = None
                data = 'Request timed out'
               block = 1
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1063)
Function: run
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
 1062                             try:
 1063 ==>                             yielded = self.gen.throw(*exc_info)
 1064                             finally:
 1065                                 # Break up a reference to itself
 1066                                 # for faster GC on CPython.
 1067                                 exc_info = None
 1068                         else:
 1069                             yielded = self.gen.send(value)
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95a6c26e10>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/concurrent.py (Line: 238)
Function: result
  231             before the `Future` is done, so the ``timeout`` is never used.
  232             """
  233             self._clear_tb_log()
  234             if self._result is not None:
  235                 return self._result
  236             if self._exc_info is not None:
  237                 try:
  238 ==>                 raise_exc_info(self._exc_info)
  239                 finally:
  240                     self = None
  241             self._check_done()
  242             return self._result
  243
  244         def exception(self, timeout=None):
Variables:
                self = None
             timeout = None
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1055)
Function: run
 1048                         return
 1049                     self.future = None
 1050                     try:
 1051                         orig_stack_contexts = stack_context._state.contexts
 1052                         exc_info = None
 1053
 1054                         try:
 1055 ==>                         value = future.result()
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95a6c26e50>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: services/ping/service.py (Line: 128)
Function: get_object_mappings
  121                 try:
  122                     yield client.query(
  123                         limit=config.ping.ds_limit,
  124                         filters=[
  125                             "pool(%s)" % config.pool,
  126                             "shard(%d,%d)" % (self.slot_number, self.total_slots)
  127                         ],
  128 ==>                     block=1
  129                     )
  130                 except NOCError as e:
  131                     self.logger.info("Failed to get object mappings: %s", e)
  132                     yield tornado.gen.sleep(1)
  133
  134         def update_probe(self, data):
Variables:
                self = <__main__.PingService object at 0x7f95a7878cd0>
              client =
<noc.services.ping.datastream.PingDataStreamClient object at 0x7f95a6cabfd0>
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1063)
Function: run
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
 1062                             try:
 1063 ==>                             yielded = self.gen.throw(*exc_info)
 1064                             finally:
 1065                                 # Break up a reference to itself
 1066                                 # for faster GC on CPython.
 1067                                 exc_info = None
 1068                         else:
 1069                             yielded = self.gen.send(value)
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7f95a6c26e50>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/concurrent.py (Line: 238)
Function: result
  231             before the `Future` is done, so the ``timeout`` is never used.
  232             """
  233             self._clear_tb_log()
  234             if self._result is not None:
  235                 return self._result
  236             if self._exc_info is not None:
  237                 try:
  238 ==>                 raise_exc_info(self._exc_info)
  239                 finally:
  240                     self = None
  241             self._check_done()
  242             return self._result
  243
  244         def exception(self, timeout=None):
Variables:
                self = None
             timeout = None
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/ioloop.py (Line: 626)
Function: _discard_future_result
  619                     else:
  620                         self.add_future(ret, self._discard_future_result)
  621             except Exception:
  622                 self.handle_callback_exception(callback)
  623
  624         def _discard_future_result(self, future):
  625             """Avoid unhandled-exception warnings from spawned coroutines."""
  626 ==>         future.result()
  627
  628         def handle_callback_exception(self, callback):
  629             """This method is called whenever a callback run by the `IOLoop`
  630             throws an exception.
  631
  632             By default simply logs the exception as an error.  Subclasses
Variables:
                self = <tornado.platform.epoll.EPollIOLoop object at 0x7f95a7888f90>
              future = <tornado.concurrent.Future object at 0x7f95a6cabf10>
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/stack_context.py (Line: 277)
Function: null_wrapper
  270
  271         if not cap_contexts[0][0] and not cap_contexts[0][1]:
  272             # Fast path when there are no active contexts.
  273             def null_wrapper(*args, **kwargs):
  274                 try:
  275                     current_state = _state.contexts
  276                     _state.contexts = cap_contexts[0]
  277 ==>                 return fn(*args, **kwargs)
  278                 finally:
  279                     _state.contexts = current_state
  280             null_wrapper._wrapped = True
  281             return null_wrapper
  282
  283         def wrapped(*args, **kwargs):
Variables:
        cap_contexts = [((), None)]
                args = (<tornado.concurrent.Future object at 0x7f95a6cabf10>,)
       current_state = ((), None)
                  fn =
<bound method EPollIOLoop._discard_future_result of <tornado.platform.epoll.EPollIOLoop object at 0x7f95a7888f90>>
              kwargs = {}
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/ioloop.py (Line: 605)
Function: _run_callback
  598
  599         def _run_callback(self, callback):
  600             """Runs a callback with error handling.
  601
  602             For use in subclasses.
  603             """
  604             try:
  605 ==>             ret = callback()
  606                 if ret is not None:
  607                     from tornado import gen
  608                     # Functions that return Futures typically swallow all
  609                     # exceptions and store them in the Future.  If a Future
  610                     # makes it out to the IOLoop, ensure its exception (if any)
  611                     # gets logged too.
Variables:
            callback = <functools.partial object at 0x7f95a67e0100>
                self = <tornado.platform.epoll.EPollIOLoop object at 0x7f95a7888f90>
------------------------------------------------------------------------
END OF TRACEBACK