debug

root@Node1:/opt/noc# ./noc discovery --debug run box "CPS-TMP"
2018-03-29 04:57:27,506 [noc.core.dcs.base] Resolve near service postgres
2018-03-29 04:57:27,509 [noc.core.dcs.base] Resolved near service postgres to ['10.220.21.122:5432']
2018-03-29 02:57:28,426 [noc.core.dcs.base] Running resolver for service mongo
2018-03-29 02:57:28,427 [noc.core.dcs.base] [mongo] Starting resolver
2018-03-29 02:57:28,428 [noc.core.dcs.base] [mongo] Set active services to: mongo: 10.220.21.122:27017
2018-03-29 02:57:28,430 [noc.lib.nosql] Connecting to MongoDB {'username': 'noc', 'authentication_source': 'noc', 'readPreference': 'secondaryPreferred', 'replicaSet': 'noc', 'db': 'noc', 'socketKeepAlive': True, 'host': 'mongodb://noc:********@10.220.21.122:27017/noc', 'password': '********'}
2018-03-29 02:57:28,467 [noc.core.cache.base] Using cache backend: noc.core.cache.mongo.MongoCache
2018-03-29 02:57:28,564 [noc.main.models.doccategory] Registering Capability
2018-03-29 02:57:28,565 [noc.main.models.doccategory] Registering MetricType
2018-03-29 02:57:28,615 [noc.main.models.doccategory] Registering ConnectionType
2018-03-29 02:57:28,618 [noc.main.models.doccategory] Registering ObjectModel
2018-03-29 02:57:28,628 [noc.main.models.textindex] Adding FTS index for sa.managedobject
2018-03-29 02:57:28,645 [noc.main.models.textindex] Adding FTS index for ip.vrf
2018-03-29 02:57:28,648 [noc.main.models.textindex] Adding FTS index for vc.vc
2018-03-29 02:57:28,651 [noc.main.models.textindex] Adding FTS index for ip.prefix
2018-03-29 02:57:28,656 [noc.main.models.textindex] Adding FTS index for ip.address
2018-03-29 02:57:28,699 [scheduler.discovery] Open collection noc.schedules.discovery.default
Job ID: 5aaa24ca93033a56789f533b
2018-03-29 02:57:28,712 [noc.core.mib] Loading compiled MIBs
2018-03-29 02:57:28,712 [noc.core.mib] Loading compiled MIBs from 'cmibs'
2018-03-29 02:57:28,713 [noc.core.mib] Loading MIB: ENTITY-STATE-MIB
2018-03-29 02:57:28,713 [noc.core.mib] Loading MIB: SNMPv2-MIB
2018-03-29 02:57:28,713 [noc.core.mib] Loading MIB: CISCO-PPPOE-MIB
2018-03-29 02:57:28,714 [noc.core.mib] Loading MIB: RMON2-MIB
2018-03-29 02:57:28,714 [noc.core.mib] Loading MIB: IF-MIB
2018-03-29 02:57:28,714 [noc.core.mib] Loading MIB: RFC1213-MIB
2018-03-29 02:57:28,714 [noc.core.mib] Loading MIB: BRIDGE-MIB
2018-03-29 02:57:28,714 [noc.core.mib] Loading MIB: EtherLike-MIB
2018-03-29 02:57:28,715 [noc.core.mib] Loading MIB: ENTITY-MIB
2018-03-29 02:57:28,715 [noc.core.mib] Loading MIB: IP-MIB
2018-03-29 02:57:28,715 [noc.core.mib] Loading MIB: ENTITY-SENSOR-MIB
2018-03-29 02:57:28,715 [noc.core.mib] Loading MIB: CISCO-VPDN-MGMT-MIB
2018-03-29 02:57:28,716 [noc.core.mib] Loading MIB: LLDP-MIB
2018-03-29 02:57:28,717 [noc.main.models.doccategory] Registering ProfileCheckRule
Loading job context from jobctx-discovery-default-5aaa24ca93033a56789f533b
2018-03-29 02:57:28,757 [scheduler.discovery] [37] [discovery|box|CPS-TMP|profile] Checking profile accordance
2018-03-29 02:57:28,757 [scheduler.discovery] [37] [discovery|box|CPS-TMP|profile] Compiling "Profile Check rules"
2018-03-29 02:57:28,933 [scheduler.discovery] [37] [discovery|box|CPS-TMP|profile] do_check(snmp_v2c_get, 1.3.6.1.2.1.1.1.0)
2018-03-29 02:57:28,944 [scheduler.discovery] [37] [discovery|box|CPS-TMP|profile] Use snmp_v2c_get for request
2018-03-29 02:57:28,945 [stub] Activating stub service
2018-03-29 02:57:28,945 [stub] Starting IOLoop
2018-03-29 02:57:28,945 [noc.core.service.rpc] [activator-default] [SYNC CALL>] activator-default.snmp_v2c_get((u'10.20.52.1', u'kondaro', u'1.3.6.1.2.1.1.1.0'), {})
2018-03-29 02:57:28,945 [noc.core.dcs.base] Running resolver for service activator-default
2018-03-29 02:57:28,946 [noc.core.dcs.base] [activator-default] Starting resolver
2018-03-29 02:57:28,946 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/activator-default?passing=1
2018-03-29 02:57:28,947 [noc.core.http.client] HTTP Response 200
2018-03-29 02:57:28,948 [noc.core.dcs.base] [activator-default] No active services
2018-03-29 02:57:28,949 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/activator-default?index=713254&passing=1
2018-03-29 03:02:29,047 [scheduler.discovery] [37] [discovery|box|CPS-TMP|profile] Set path: None
2018-03-29 03:02:29,288 [scheduler.discovery] [37] [discovery|box|CPS-TMP|profile] UNHANDLED EXCEPTION (2018-03-29 03:02:29.063773)
PROCESS: ./commands/discovery.py
VERSION: 15.05.1+microservices.9255.694ddd84
BRANCH: microservices CHANGESET: 694ddd84
ERROR FINGERPRINT: 0502f500-51c5-58b5-bd65-f54ff7fda8f1
WORKING DIRECTORY: /opt/noc
EXCEPTION: <class 'noc.core.dcs.error.ResolutionError'>
START OF TRACEBACK
------------------------------------------------------------------------
File: core/dcs/base.py (Line: 272)
Function: resolve
  265                     t = self.dcs.DEFAULT_SERVICE_RESOLUTION_TIMEOUT
  266                 try:
  267                     yield self.ready_event.wait(timeout=t)
  268                 except tornado.gen.TimeoutError:
  269                     metrics["errors", ("type", "dcs_resolver_timeout")] += 1
  270                     if self.critical:
  271                         self.dcs.set_faulty_status("Failed to resolve %s: Timeout" % self.name)
  272 ==>                 raise ResolutionError()
  273             if not wait and not self.ready_event.is_set():
  274                 if self.critical:
  275                     self.dcs.set_faulty_status("Failed to resolve %s: No active services" % self.name)
  276                 raise ResolutionError()
  277             with self.lock:
  278                 if hint and hint in self.service_addresses:
Variables:
         full_result = False
                hint = None
                self = <noc.core.dcs.consuldcs.ConsulResolver object at 0x7fcce0fb2c90>
                   t = datetime.timedelta(0, 300)
             timeout = None
                wait = True
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1063)
Function: run
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
 1062                             try:
 1063 ==>                             yielded = self.gen.throw(*exc_info)
 1064                             finally:
 1065                                 # Break up a reference to itself
 1066                                 # for faster GC on CPython.
 1067                                 exc_info = None
 1068                         else:
 1069                             yielded = self.gen.send(value)
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7fcce0fb2e10>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/concurrent.py (Line: 238)
Function: result
  231             before the `Future` is done, so the ``timeout`` is never used.
  232             """
  233             self._clear_tb_log()
  234             if self._result is not None:
  235                 return self._result
  236             if self._exc_info is not None:
  237                 try:
  238 ==>                 raise_exc_info(self._exc_info)
  239                 finally:
  240                     self = None
  241             self._check_done()
  242             return self._result
  243
  244         def exception(self, timeout=None):
Variables:
                self = None
             timeout = None
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1055)
Function: run
 1048                         return
 1049                     self.future = None
 1050                     try:
 1051                         orig_stack_contexts = stack_context._state.contexts
 1052                         exc_info = None
 1053
 1054                         try:
 1055 ==>                         value = future.result()
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7fcce0fb2d50>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: core/dcs/base.py (Line: 124)
Function: resolve
  117
  118         @tornado.gen.coroutine
  119         def resolve(self, name, hint=None, wait=True, timeout=None,
  120                     full_result=False, critical=False):
  121             resolver = yield self.get_resolver(name, critical)
  122             r = yield resolver.resolve(
  123                 hint=hint, wait=wait, timeout=timeout,
  124 ==>             full_result=full_result
  125             )
  126             raise tornado.gen.Return(r)
  127
  128         @tornado.gen.coroutine
  129         def expire_resolvers(self):
  130             with self.resolvers_lock:
Variables:
         full_result = False
                hint = None
                self = <noc.core.dcs.consuldcs.ConsulDCS object at 0x7fcce0fb2490>
            resolver = <noc.core.dcs.consuldcs.ConsulResolver object at 0x7fcce0fb2c90>
                name = u'activator-default'
            critical = False
             timeout = None
                wait = True
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1063)
Function: run
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
 1062                             try:
 1063 ==>                             yielded = self.gen.throw(*exc_info)
 1064                             finally:
 1065                                 # Break up a reference to itself
 1066                                 # for faster GC on CPython.
 1067                                 exc_info = None
 1068                         else:
 1069                             yielded = self.gen.send(value)
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7fcce0fb2d50>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/concurrent.py (Line: 238)
Function: result
  231             before the `Future` is done, so the ``timeout`` is never used.
  232             """
  233             self._clear_tb_log()
  234             if self._result is not None:
  235                 return self._result
  236             if self._exc_info is not None:
  237                 try:
  238 ==>                 raise_exc_info(self._exc_info)
  239                 finally:
  240                     self = None
  241             self._check_done()
  242             return self._result
  243
  244         def exception(self, timeout=None):
Variables:
                self = None
             timeout = None
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1055)
Function: run
 1048                         return
 1049                     self.future = None
 1050                     try:
 1051                         orig_stack_contexts = stack_context._state.contexts
 1052                         exc_info = None
 1053
 1054                         try:
 1055 ==>                         value = future.result()
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7fcce0fb2c10>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: core/service/rpc.py (Line: 125)
Function: _call
  118                 # Get services
  119                 response = None
  120                 for t in self._service.iter_rpc_retry_timeout():
  121                     # Resolve service against service catalog
  122                     if self._hints:
  123                         svc = random.choice(self._hints)
  124                     else:
  125 ==>                     svc = yield self._service.dcs.resolve(self._service_name)
  126                     response = yield make_call(
  127                         "http://%s/api/%s/" % (svc, self._api),
  128                         body
  129                     )
  130                     if response:
  131                         break
Variables:
                body =
'{"params":["10.20.52.1","kondaro","1.3.6.1.2.1.1.1.0"],"method":"snmp_v2c_get","id":0}'
            span_ctx = None
                self = <noc.core.service.rpc.RPCProxy object at 0x7fcce0fb2990>
                args = (u'10.20.52.1', u'kondaro', u'1.3.6.1.2.1.1.1.0')
                  t0 = 1522281448.945723
            response = None
                 tid = 0
                   t = 0.1
              kwargs = {}
                 msg =
{'id': 0,
 'method': 'snmp_v2c_get',
 'params': [u'10.20.52.1', u'kondaro', u'1.3.6.1.2.1.1.1.0']}
              method = 'snmp_v2c_get'
           make_call = <function make_call at 0x7fcce0fba578>
           is_notify = False
             span_id = None
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1063)
Function: run
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
 1062                             try:
 1063 ==>                             yielded = self.gen.throw(*exc_info)
 1064                             finally:
 1065                                 # Break up a reference to itself
 1066                                 # for faster GC on CPython.
 1067                                 exc_info = None
 1068                         else:
 1069                             yielded = self.gen.send(value)
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7fcce0fb2c10>
              future = None
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/concurrent.py (Line: 238)
Function: result
  231             before the `Future` is done, so the ``timeout`` is never used.
  232             """
  233             self._clear_tb_log()
  234             if self._result is not None:
  235                 return self._result
  236             if self._exc_info is not None:
  237                 try:
  238 ==>                 raise_exc_info(self._exc_info)
  239                 finally:
  240                     self = None
  241             self._check_done()
  242             return self._result
  243
  244         def exception(self, timeout=None):
Variables:
                self = None
             timeout = None
------------------------------------------------------------------------
File: local/lib/python2.7/site-packages/tornado/gen.py (Line: 1055)
Function: run
 1048                         return
 1049                     self.future = None
 1050                     try:
 1051                         orig_stack_contexts = stack_context._state.contexts
 1052                         exc_info = None
 1053
 1054                         try:
 1055 ==>                         value = future.result()
 1056                         except Exception:
 1057                             self.had_exception = True
 1058                             exc_info = sys.exc_info()
 1059                         future = None
 1060
 1061                         if exc_info is not None:
Variables:
            exc_info = None
                self = <tornado.gen.Runner object at 0x7fcce0fb2e50>
              future = None
                   e = StopIteration()
 orig_stack_contexts = ((), None)
------------------------------------------------------------------------
File: core/service/rpc.py (Line: 173)
Function: _sync_call
  166                 result = yield _call(item, *args, **kwargs)
  167                 raise tornado.gen.Return(result)
  168
  169             def sync_wrapper(*args, **kwargs):
  170                 @tornado.gen.coroutine
  171                 def _sync_call():
  172                     try:
  173 ==>                     r = yield _call(item, *args, **kwargs)
  174                         result.append(r)
  175                     except tornado.gen.Return as e:
  176                         result.append(e.value)
  177                     except Exception:
  178                         error.append(sys.exc_info())
  179                     finally:
Variables:
              kwargs = {}
                args = (u'10.20.52.1', u'kondaro', u'1.3.6.1.2.1.1.1.0')
                item = 'snmp_v2c_get'
              result = []
               error =
[(<class 'noc.core.dcs.error.ResolutionError'>,
  ResolutionError(),
  <traceback object at 0x7fcce0f6be18>)]
                  ev = <threading._Event object at 0x7fcce0fb2a50>
               _call = <function _call at 0x7fcce0fba2a8>
------------------------------------------------------------------------
File: core/service/rpc.py (Line: 188)
Function: sync_wrapper
  181
  182                 ev = threading.Event()
  183                 result = []
  184                 error = []
  185                 self._service.ioloop.add_callback(_sync_call)
  186                 ev.wait()
  187                 if error:
  188 ==>                 six.reraise(*error[0])
  189                 else:
  190                     return result[0]
  191
  192             if item.startswith("_"):
  193                 return self.__dict__[item]
  194             span_ctx, span_id = get_current_span()
Variables:
                args = (u'10.20.52.1', u'kondaro', u'1.3.6.1.2.1.1.1.0')
          _sync_call = <function _sync_call at 0x7fcce0fba320>
                self = <noc.core.service.rpc.RPCProxy object at 0x7fcce0fb2990>
              kwargs = {}
                item = 'snmp_v2c_get'
              result = []
               error =
[(<class 'noc.core.dcs.error.ResolutionError'>,
  ResolutionError(),
  <traceback object at 0x7fcce0f6be18>)]
                  ev = <threading._Event object at 0x7fcce0fb2a50>
               _call = <function _call at 0x7fcce0fba2a8>
------------------------------------------------------------------------
File: services/discovery/jobs/box/profile.py (Line: 181)
Function: check_snmp_v2c_get
  174                 return open_sync_rpc(
  175                     "activator",
  176                     pool=self.object.pool.name,
  177                     calling_service="discovery"
  178                 ).__getattr__(self.snmp_version_def)(
  179                     self.object.address,
  180                     snmp_ro,
  181 ==>                 param
  182                 )
  183             except RPCError as e:
  184                 self.logger.error("RPC Error: %s", e)
  185                 return None
  186
  187         def check_http_get(self, param):
Variables:
                self =
<noc.services.discovery.jobs.box.profile.ProfileCheck object at 0x7fcce10f1590>
             snmp_ro = u'kondaro'
               param = u'1.3.6.1.2.1.1.1.0'
                caps =
{u'DB | Interfaces': 19,
 u'Network | CDP': True,
 u'SNMP': True,
 u'SNMP | Bulk': True,
 u'SNMP | IF-MIB': True,
 u'SNMP | IF-MIB | HC': True,
 u'SNMP | v1': False,
 u'SNMP | v2c': True,
 u'SNMP | v3': False}
------------------------------------------------------------------------
File: services/discovery/jobs/box/profile.py (Line: 141)
Function: do_check
  134                 self.logger.debug("Using cached value")
  135                 return self.result_cache[method, param]
  136             h = getattr(self, "check_%s" % method, None)
  137             if not h:
  138                 self.logger.error("Invalid check method '%s'. Ignoring",
  139                                   method)
  140                 return None
  141 ==>         result = h(param)
  142             self.result_cache[method, param] = result
  143             return result
  144
  145         def check_snmp_v2c_get(self, param):
  146             """
  147             Perform SNMP v2c GET. Param is OID or symbolic name
Variables:
                   h =
<bound method ProfileCheck.check_snmp_v2c_get of <noc.services.discovery.jobs.box.profile.ProfileCheck object at 0x7fcce10f1590>>
                self =
<noc.services.discovery.jobs.box.profile.ProfileCheck object at 0x7fcce10f1590>
              method = u'snmp_v2c_get'
               param = u'1.3.6.1.2.1.1.1.0'
------------------------------------------------------------------------
File: services/discovery/jobs/box/profile.py (Line: 64)
Function: get_profile
   57             snmp_result = ""
   58             http_result = ""
   59             message = "Cannot detect profile"
   60             fatal = False
   61             for ruleset in self.get_rules():
   62                 for (method, param), actions in ruleset:
   63                     try:
   64 ==>                     result = self.do_check(method, param)
   65                         if not result:
   66                             continue
   67                         if "snmp" in method:
   68                             snmp_result = result
   69                         if "http" in method:
   70                             http_result = result
Variables:
                self =
<noc.services.discovery.jobs.box.profile.ProfileCheck object at 0x7fcce10f1590>
               param = u'1.3.6.1.2.1.1.1.0'
         http_result = ''
             ruleset =
[((u'snmp_v2c_get', u'1.3.6.1.2.1.1.1.0'),
  [(u're',
    u'^Huawei AR.*',
    u'match',
    <Profile: Huawei.VRP>,
    u'Huawei | VRP | AR sysDescr'),
   (u're',
    u'^S5700.*',
    u'match',
    <Profile: Huawei.VRP>,
    u'Huawei | VRP | S5700 sysDescr')]),
 ((u'snmp_v2c_get', u'SNMPv2-MIB::sysObjectID.0'),
  [(u'eq',
    u'1.3.6.1.4.1.27514.1.3.13.0',
    u'match',
    <Profile: Qtech.QSW>,
    u'Qtech | QSW | QSW-2900 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.11.13',
    u'match',
    <Profile: Zyxel.MSAN>,
    u'Zyxel | MSAN | IES1248-51 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.10.1.232',
    u'match',
    <Profile: Huawei.VRP>,
    u'Huawei | VRP | S5628F-HI sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.8.30',
    u'match',
    <Profile: Zyxel.ZyNOS>,
    u'Zyxel | ZyNOS | ES-2226 sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.35419',
    u'match',
    <Profile: ElectronR.KO01M>,
    u'ElectronR | KO01M sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.61.2',
    u'match',
    <Profile: DLink.DES21xx>,
    u'DLink | DES21xx | DES-2108 sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.76.12',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DGS-1210-10P sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.76.15',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DGS-1210-28 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.25506.1.33',
    u'match',
    <Profile: H3C.VRP>,
    u'H3C | VRP | S3600-52P-SI sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.6486.800.1.1.2.1.7.1.10',
    u'match',
    <Profile: Alcatel.AOS>,
    u'Alcatel | AOS | OS6850-U24X'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.123',
    u'match',
    <Profile: Huawei.MA5600T>,
    u'Huawei | MA | MA5603T sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.3320.1.228.0',
    u'match',
    <Profile: BDCOM.xPON>,
    u'BDCOM | xPON | P3310B sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.27514.1.1.1.203',
    u'match',
    <Profile: Qtech.QSW>,
    u'Qtech | QSW | QSW-3400-28T-AC sysObjectID 2'),
   (u'eq',
    u'1.3.6.1.4.1.1795.1.14.9.14.5.3',
    u'match',
    <Profile: Zhone.Bitstorm>,
    u'Zhone | Bitstorm | 8820-A2-xxx sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.6339.1.1.2.59',
    u'match',
    <Profile: Qtech.QSW2800>,
    u'Qtech | QSW | QSW-2800 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.40248.2',
    u'match',
    <Profile: Vitesse.VSC>,
    u'Vitesse | VSC | VSC7460v2 sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.75.17.1',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DES-1210-08P-cx sysObjectID'),
   (u're',
    u'1.3.6.1.4.1.207.1.4.12[56]',
    u'match',
    <Profile: AlliedTelesis.AT8000S>,
    u'AlliedTelesis | ATI | AT-8000S sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.3955.6.9.224.1',
    u'match',
    <Profile: Linksys.SPS2xx>,
    u'Lynksis | SPS2xx | SPS224G4 sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.75.13',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DES-1210-08P sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.10.1.43',
    u'match',
    <Profile: Huawei.VRP>,
    u'Huawei | VRP | S5624P sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.25506.1.37',
    u'match',
    <Profile: H3C.VRP>,
    u'H3C | VRP | S3600-28P-EI sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.259.10.1.22.101',
    u'match',
    <Profile: EdgeCore.ES>,
    u'EdgeCore | ES | ES3528MV2 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.135',
    u'match',
    <Profile: Huawei.MA5600T>,
    u'Huawei | MA | MA5626G sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.2011.2.80.1',
    u'match',
    <Profile: Huawei.VRP3>,
    u'Huawei | MA | MA5605 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.5.2',
    u'match',
    <Profile: Zyxel.MSAN>,
    u'Zyxel | MSAN | IES-1000 sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.76.14',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DGS-1210-20 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.835457.3.4',
    u'match',
    <Profile: Alcatel.TIMOS>,
    u'Alcatel | TIMOS | 7750 SR-7'),
   (u'eq',
    u'1.3.6.1.4.1.43.1.8.41',
    u'match',
    <Profile: 3Com.SuperStack3>,
    u'3Com | SuperStack3 | 3226 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.23.96',
    u'match',
    <Profile: Huawei.VRP>,
    u'Huawei | VRP | S5328C-EI-24S sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.3955.6.1.2016.1',
    u'match',
    <Profile: Linksys.SRW>,
    u'Linksys | SRW | SRW2016 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.8.16',
    u'match',
    <Profile: Zyxel.ZyNOS>,
    u'Zyxel | ZyNOS | ES-2024A sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.31926',
    u'match',
    <Profile: Siklu.EH>,
    u'Siklu | EH | EH-1200L.v700 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.35265.1.46',
    u'match',
    <Profile: Eltex.RG>,
    u'Eltex | RG| RG-1404GF-W sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.78',
    u'match',
    <Profile: Huawei.MA5600T>,
    u'Huawei | HONET | UA5000 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.40248.1',
    u'match',
    <Profile: Vitesse.VSC>,
    u'Vitesse | VSC | VSC7460 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.23.92',
    u'match',
    <Profile: Huawei.VRP>,
    u'Huawei | VRP | S2326TP-EI sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.35265.1.28',
    u'match',
    <Profile: Eltex.DSLAM>,
    u'Eltex | DSLAM | MXA32 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.3320.1.347.0',
    u'match',
    <Profile: BDCOM.IOS>,
    u'BDCOM | IOS | 2208P sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.3320.1.347.0',
    u'match',
    <Profile: BDCOM.IOS>,
    u'BDCOM | IOS | S2210PB sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.259.10.1.27.102',
    u'match',
    <Profile: EdgeCore.ES>,
    u'EdgeCore | ES | ECS3510-52T sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.6527.1.15.1',
    u'match',
    <Profile: Alcatel.TIMOS>,
    u'Alcatel | TIMOS | 7950 XRS'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.23.102',
    u'match',
    <Profile: Huawei.VRP>,
    u'Huawei | VRP | S5300 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.14988.1',
    u'match',
    <Profile: MikroTik.RouterOS>,
    u'MikroTik | RouterOS | sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.231.7.1.2.2.1.101.1.1',
    u'match',
    <Profile: NSN.hiX56xx>,
    u'NAG | NSN | hiX5622G2002GE sysObjectID.0'),
   (u'eq',
    u'1.3.6.1.4.1.3320.1.294.0',
    u'match',
    <Profile: BDCOM.xPON>,
    u'BDCOM | xPON | P3310C sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.8.26',
    u'match',
    <Profile: Zyxel.ZyNOS>,
    u'Zyxel | ZyNOS | ES-3124-4F sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.75.5',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DES-1210-28 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.4874.1.1.1.6.1',
    u'match',
    <Profile: Juniper.JUNOSe>,
    u'Juniper | JUNOSe | E320 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.193.81.1.1.3',
    u'match',
    <Profile: Ericsson.SEOS>,
    u'Ericsson | MINI-LINK | CN510 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.40418.7.2',
    u'match',
    <Profile: NAG.SNR>,
    u'NAG | SNR | SNR-S2940 sysObjectID.0'),
   (u'eq',
    u'1.3.6.1.4.1.25506.1.34',
    u'match',
    <Profile: H3C.VRP>,
    u'H3C | VRP | S3600-28TP-SI sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.75.19.1',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DES-1210-28P V4 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.6.5.21',
    u'match',
    <Profile: Huawei.VRP3>,
    u'Huawei | MA | MA5105 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.6486.800.1.1.2.1.7.1.48',
    u'match',
    <Profile: Alcatel.AOS>,
    u'Alcatel | AOS | OS6850E-U24X'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.11.9',
    u'match',
    <Profile: Zyxel.MSAN>,
    u'Zyxel | ZyNOS | IES1248-71 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.10.1.12',
    u'match',
    <Profile: Huawei.VRP>,
    u'Huawei | VRP | S3928TP-SI sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.61.1',
    u'match',
    <Profile: DLink.DES21xx>,
    u'DLink | DES21xx | DES-2110 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.4249.1.114.1',
    u'match',
    <Profile: Nateks.FlexGain>,
    u'Nateks | FlexGain | FG-ACE120 sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.75.6',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DES-1210-28P sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.75.7',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DES-1210-52 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.248',
    u'match',
    <Profile: Huawei.MA5600T>,
    u'Huawei | MA | MA5608T sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.6.6.1',
    u'match',
    <Profile: Huawei.MA5300>,
    u'Huawei | MA | MA5300 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.27514.1.1.1.48',
    u'match',
    <Profile: Qtech.QSW2800>,
    u'Qtech | QSW | QSW-2800-10T-AC sysObjectID'),
   (u'eq',
    u'1.3.6.1.3.55.1',
    u'match',
    <Profile: Qtech.BFC_PBIC_S>,
    u'Qtech | BFC-PBIC-S sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.169',
    u'match',
    <Profile: Huawei.MA5600T>,
    u'Huawei | MA | MA5616 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.30982.1.1',
    u'match',
    <Profile: TFortis.PSW>,
    u'TFortis | PSW | PSW-2G6F+'),
   (u'eq',
    u'1.3.6.1.4.1.27514.6.212',
    u'match',
    <Profile: Raisecom.ROS>,
    u'Qtech | QSW | QSW-2850-28T sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.35265.1.22',
    u'match',
    <Profile: Eltex.LTP>,
    u'Eltex | LTP | LTP sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.76.10',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DGS-1210-24 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.8886.6.200',
    u'match',
    <Profile: Qtech.QSW8200>,
    u'Qtech | QSW | QSW-8200-28F sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.76.16',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DGS-1210-28P sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.1795.1.14.17.1.3',
    u'match',
    <Profile: Zhone.Bitstorm>,
    u'Zhone | Bitstorm | 4229-A1-520 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.637.61.1',
    u'match',
    <Profile: Alcatel.7302>,
    u'Alcatel | 7302 | 7302 sysObjectID.0'),
   (u'eq',
    u'1.3.6.1.4.1.27514.1.1.1.49',
    u'match',
    <Profile: Qtech.QSW2800>,
    u'Qtech | QSW | QSW-2800-28T sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.21696',
    u'match',
    <Profile: Generic.Host>,
    u'KeyMile | MileGate | 2510 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.27514.1.1.1.221',
    u'match',
    <Profile: Qtech.QSW>,
    u'Qtech | QSW | QSW-3400-28T-AC sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.97.1.9',
    u'match',
    <Profile: DLink.DxS_Cisco_CLI>,
    u'DLink | DGS-3610-26G sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.655.100.10.1',
    u'match',
    <Profile: Carelink.SWG>,
    u'Carelink | SWG | SWG1002 sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.61.3',
    u'match',
    <Profile: DLink.DES21xx>,
    u'DLink | DES21xx | DES-2108-B1 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.27514.1.1.2.59',
    u'match',
    <Profile: Qtech.QSW2800>,
    u'Qtech | QSW | QSW-8200-28F-AC-DC sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.3955.7.2.2000.1',
    u'match',
    <Profile: Alstec.ALS>,
    u'Alstec | ALS | ALS-62000'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.76.11',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DGS-1210-48 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.1332.1.1.1',
    u'match',
    <Profile: Iskratel.VOIP>,
    u'Iskratel | VOIP | ISKRATEL VOIP-1 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.32761',
    u'match',
    <Profile: Rotek.RTBS>,
    u'Rotek | RTBS | RT-BS24-WFN2E sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.76.9',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DGS-1210-16 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.4874.1.1.1.6.2',
    u'match',
    <Profile: Juniper.JUNOSe>,
    u'Juniper | JUNOSe | E120 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.34300.1.6',
    u'match',
    <Profile: Eltex.DSLAM>,
    u'Eltex | DSLAM | MXA24 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.8886.6.140',
    u'match',
    <Profile: Raisecom.ROS>,
    u'Raisecom | ROS | ISCOM2128EA-MA-AC sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.8.12',
    u'match',
    <Profile: Zyxel.ZyNOS>,
    u'Zyxel | ZyNOS | ES-3124 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.171.10.65.1',
    u'match',
    <Profile: DLink.DAS>,
    u'DLink | DAS | DAS-322X sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.27514.6.178',
    u'match',
    <Profile: Qtech.QOS>,
    u'Qtech | QOS | QSW-8200-28F-AC sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.3320.1.263.0',
    u'match',
    <Profile: BDCOM.xPON>,
    u'BDCOM | xPON | P3608 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.11.11',
    u'match',
    <Profile: Zyxel.MSAN>,
    u'Zyxel | ZyNOS | IES-612 sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.94.5',
    u'match',
    <Profile: DLink.DGS3100>,
    u'DLink | DGS3100 | DGS-3100-24TG sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.13.8',
    u'match',
    <Profile: Zyxel.MSAN>,
    u'Zyxel | ZyNOS | IES-6000 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.13.7',
    u'match',
    <Profile: Zyxel.MSAN>,
    u'Zyxel | ZyNOS | IES-5005 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.27514.1.3.26.7',
    u'match',
    <Profile: Qtech.QSW>,
    u'Qtech | QSW | QSW-2910-28F sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.1332.1.24.3',
    u'match',
    <Profile: Iskratel.ESCOM>,
    u'Iskratel | ESCOM | SI3000 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.10002.1',
    u'match',
    <Profile: Rotek.RTBSv1>,
    u'Rotek | RTBSv1 | RT-BS24-WFN2E.v1 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.30803',
    u'match',
    <Profile: Vyatta.Vyatta>,
    u'Vyatta | Vyatta | sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.27514.1.1.1.228',
    u'match',
    <Profile: Qtech.QSW2800>,
    u'Qtech | QSW | QSW-3450-28T-AC sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.11.2.3.7.11.63',
    u'match',
    <Profile: HP.ProCurve>,
    u'HP | ProCurve | 2810-24G sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.35265.1.21.202',
    u'match',
    <Profile: Eltex.LTE>,
    u'Eltex | LTE | LTE sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.890.1.5.11.6',
    u'match',
    <Profile: Zyxel.MSAN>,
    u'Zyxel | ZyNOS | IES-1000 AAM1008-61 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.10.1.200',
    u'match',
    <Profile: Huawei.VRP>,
    u'Huawei | VRP | S2403TP-EA sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.43.1.8.42',
    u'match',
    <Profile: 3Com.SuperStack3>,
    u'3Com | SuperStack3 | 3250 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.27514.1.1.1.281',
    u'match',
    <Profile: Qtech.QSW>,
    u'Qtech | QSW | QSW-3470-28T-AC sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.835457.6.3',
    u'match',
    <Profile: Alcatel.TIMOS>,
    u'Alcatel | TIMOS | 7450 ESS-7'),
   (u'eq',
    u'1.3.6.1.4.1.259.6.10.94',
    u'match',
    <Profile: EdgeCore.ES>,
    u'EdgeCore | ES | ES3528M sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.2011.2.115',
    u'match',
    <Profile: Huawei.MA5600T>,
    u'Huawei | MA | MA5680T sysObjectID'),
   (u'contains',
    u'1.3.6.1.4.1.171.10.76.17',
    u'match',
    <Profile: DLink.DxS_Smart>,
    u'DLink | DxS_Smart | DGS-1210-52 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.4413',
    u'match',
    <Profile: Alstec.24xx>,
    u'Alstec | ALS24300 sysObjectID'),
   (u'eq',
    u'1.3.6.1.4.1.38838.1.10',
    u'match',
    <Profile: Angtel.Topaz>,
    u'Angtel | Topaz | Topaz-2O-16E'),
   (u'eq',
    u'1.3.6.1.4.1.27514.1.1.1.51',
    u'match',
    <Profile: Qtech.QSW2800>,
    u'Qtech | QSW | QSW-2800-10T-DC sysObjectID')]),
 ((u'snmp_v2c_get', u'SNMPv2-MIB::sysDescr.0'),
  [(u'contains',
    u'ALS24100LVT',
    u'match',
    <Profile: Alstec.24xx>,
    u'Alstec | ALS24100 sysDescr.0')])]
         snmp_result = ''
             message = 'Cannot detect profile'
               fatal = False
              method = u'snmp_v2c_get'
             actions =
[(u're',
  u'^Huawei AR.*',
  u'match',
  <Profile: Huawei.VRP>,
  u'Huawei | VRP | AR sysDescr'),
 (u're',
  u'^S5700.*',
  u'match',
  <Profile: Huawei.VRP>,
  u'Huawei | VRP | S5700 sysDescr')]
------------------------------------------------------------------------
File: services/discovery/jobs/box/profile.py (Line: 35)
Function: handler
   28         name = "profile"
   29
   30         _rules_cache = cachetools.TTLCache(10, ttl=60)
   31         snmp_version_def = None
   32
   33         def handler(self):
   34             self.logger.info("Checking profile accordance")
   35 ==>         profile = self.get_profile()
   36             if not profile:
   37                 return  # Cannot detect
   38             if profile.id == self.object.profile.id:
   39                 self.logger.info("Profile is correct: %s", profile)
   40             else:
   41                 self.logger.info(
Variables:
                self =
<noc.services.discovery.jobs.box.profile.ProfileCheck object at 0x7fcce10f1590>
------------------------------------------------------------------------
File: services/discovery/jobs/base.py (Line: 449)
Function: run
  442                                      self.required_script)
  443                     return
  444                 # Check required capabilities
  445                 if not self.has_required_capabilities():
  446                     return
  447                 # Run check
  448                 try:
  449 ==>                 self.handler()
  450                 except RPCRemoteError as e:
  451                     self.logger.error(
  452                         "RPC Remote error (%s): %s",
  453                         e.remote_code, e)
  454                     if e.remote_code:
  455                         message = "Remote error code %s" % e.remote_code
Variables:
                span = <noc.core.span.Span object at 0x7fcce10f1490>
                self =
<noc.services.discovery.jobs.box.profile.ProfileCheck object at 0x7fcce10f1590>
                   e = ResolutionError()
------------------------------------------------------------------------
END OF TRACEBACK
2018-03-29 03:02:29,293 [scheduler.discovery] [37] [discovery|box|CPS-TMP|profile] Writing CP report to /var/lib/noc/cp/crashinfo/new/0502f500-51c5-58b5-bd65-f54ff7fda8f1.json
2018-03-29 03:02:29,296 [noc.core.profile.loader] Loading profile MikroTik.RouterOS
2018-03-29 03:02:29,297 [scheduler.discovery] [37] [discovery|box|CPS-TMP] Using CLI sessions
2018-03-29 03:02:29,312 [scheduler.discovery] [37] [discovery|box|CPS-TMP|version] Checking version
2018-03-29 03:02:29,313 [noc.core.service.rpc] [sae] [SYNC CALL>] sae.get_credentials((37,), {})
2018-03-29 03:02:29,313 [noc.core.dcs.base] Running resolver for service sae
2018-03-29 03:02:29,313 [noc.core.dcs.base] [sae] Starting resolver
2018-03-29 03:02:29,314 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?passing=1
2018-03-29 03:02:29,315 [noc.core.http.client] HTTP Response 200
2018-03-29 03:02:29,315 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:02:29,315 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713425&passing=1
2018-03-29 03:02:29,316 [noc.core.http.client] HTTP POST http://10.220.21.122:58249/api/sae/
2018-03-29 03:02:29,321 [noc.core.http.client] HTTP Response 200
2018-03-29 03:02:29,321 [noc.core.service.rpc] [sae] [CALL<] sae.get_credentials (8.41ms)
2018-03-29 03:02:44,004 [noc.core.http.client] HTTP Response 200
2018-03-29 03:02:44,005 [noc.core.dcs.base] Set faulty status to: Timed out
2018-03-29 03:02:44,005 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/activator-default?index=713254&passing=1
2018-03-29 03:02:44,006 [noc.core.http.client] HTTP Response 200
2018-03-29 03:02:44,006 [noc.core.dcs.base] [activator-default] No active services
2018-03-29 03:02:44,006 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/activator-default?index=713425&passing=1
2018-03-29 03:02:46,810 [noc.core.http.client] HTTP Response 200
2018-03-29 03:02:46,810 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:02:46,810 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713433&passing=1
2018-03-29 03:02:47,192 [noc.core.http.client] HTTP Response 200
2018-03-29 03:02:47,193 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:02:47,193 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713448&passing=1
2018-03-29 03:04:07,108 [noc.core.http.client] HTTP Response 200
2018-03-29 03:04:07,108 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:04:07,108 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713498&passing=1
2018-03-29 03:04:07,421 [noc.core.http.client] HTTP Response 200
2018-03-29 03:04:07,422 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:04:07,422 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713513&passing=1
2018-03-29 03:05:10,294 [noc.core.http.client] HTTP Response 200
2018-03-29 03:05:10,294 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:05:10,294 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713547&passing=1
2018-03-29 03:05:10,780 [noc.core.http.client] HTTP Response 200
2018-03-29 03:05:10,781 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:05:10,781 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713562&passing=1
2018-03-29 03:06:18,098 [noc.core.http.client] HTTP Response 200
2018-03-29 03:06:18,098 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:06:18,099 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713601&passing=1
2018-03-29 03:06:19,112 [noc.core.http.client] HTTP Response 200
2018-03-29 03:06:19,113 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:06:19,113 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713637&passing=1
2018-03-29 03:07:29,381 [scheduler.discovery] [37] [discovery|box|CPS-TMP|version] Set path: None
2018-03-29 03:07:29,381 [scheduler.discovery] [37] [discovery|box|CPS-TMP|version] Terminated due RPC error: activator-default
2018-03-29 03:07:29,381 [scheduler.discovery] [37] [discovery|box|CPS-TMP|caps] Checking capabilities
2018-03-29 03:07:29,382 [noc.core.service.rpc] [sae] [SYNC CALL>] sae.get_credentials((37,), {})
2018-03-29 03:07:29,382 [noc.core.http.client] HTTP POST http://10.220.21.122:60319/api/sae/
2018-03-29 03:07:29,387 [noc.core.http.client] HTTP Response 200
2018-03-29 03:07:29,388 [noc.core.service.rpc] [sae] [CALL<] sae.get_credentials (5.66ms)
2018-03-29 03:07:50,615 [noc.core.http.client] HTTP Response 200
2018-03-29 03:07:50,616 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:07:50,616 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713659&passing=1
2018-03-29 03:07:52,623 [noc.core.http.client] HTTP Response 200
2018-03-29 03:07:52,624 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:07:52,624 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713695&passing=1
2018-03-29 03:08:01,077 [noc.core.http.client] HTTP Response 200
2018-03-29 03:08:01,078 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/activator-default?index=713425&passing=1
2018-03-29 03:08:01,079 [noc.core.http.client] HTTP Response 200
2018-03-29 03:08:01,079 [noc.core.dcs.base] [activator-default] No active services
2018-03-29 03:08:01,079 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/activator-default?index=713704&passing=1
2018-03-29 03:09:15,330 [noc.core.http.client] HTTP Response 200
2018-03-29 03:09:15,331 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:09:15,331 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713712&passing=1
2018-03-29 03:09:16,999 [noc.core.http.client] HTTP Response 200
2018-03-29 03:09:16,999 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:09:17,000 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713748&passing=1
2018-03-29 03:11:12,512 [noc.core.http.client] HTTP Response 200
2018-03-29 03:11:12,513 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:11:12,513 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713791&passing=1
2018-03-29 03:11:13,405 [noc.core.http.client] HTTP Response 200
2018-03-29 03:11:13,405 [noc.core.dcs.base] [sae] Set active services to: 3d6c4b6c-532f-4b8c-b758-b6557dc85f7f: 10.220.21.122:60319, 750911d9-a0e9-4ba2-b725-6a8b3cccdc96: 10.220.21.122:58249
2018-03-29 03:11:13,406 [noc.core.http.client] HTTP GET http://consul:8500/v1/health/service/sae?index=713806&passing=1