API tools faq
paste
Advertisement
mtrower

Binder crashes

mtrower
Sep 10th, 2016
152
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 132.38 KB | None | 0 0
raw download clone embed print report
  1. [ Mar 17 15:06:18 Leaving maintenance because clear requested. ]
  2. [ Mar 17 15:06:18 Enabled. ]
  3. [ Mar 17 15:06:18 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  4. [ Mar 17 15:06:18 Method "start" exited with status 0. ]
  5. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93883,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:06:18.529Z","v":0}
  6. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93883,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:06:18.575Z","v":0}
  7. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93883,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:06:18.576Z","v":0}
  8. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93883,"level":30,"msg":"done with binder init","time":"2016-03-17T15:06:18.576Z","v":0}
  9. { cache:
  10. { _max: 10000,
  11. _lengthCalculator: [Function: naiveLength],
  12. _allowStale: false,
  13. _maxAge: 60000,
  14. _dispose: undefined,
  15. _cache: {},
  16. _lruList: {},
  17. _mru: 0,
  18. _lru: 0,
  19. _length: 0,
  20. _itemCount: 0 },
  21. path: '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta',
  22. zk:
  23. { domain: null,
  24. _events:
  25. { state: [Function: defaultStateListener],
  26. error: [Function],
  27. disconnected: [Object] },
  28. _maxListeners: undefined,
  29. connectionManager:
  30. { domain: null,
  31. _events: [Object],
  32. _maxListeners: undefined,
  33. watcherManager: [Object],
  34. connectionStringParser: [Object],
  35. servers: [Object],
  36. chrootPath: undefined,
  37. nextServerIndex: 1,
  38. serverAttempts: 0,
  39. state: 2,
  40. options: [Object],
  41. spinDelay: 1000,
  42. sessionTimeout: 30000,
  43. connectTimeout: 30000,
  44. pingTimeout: 10000,
  45. connectTimeoutHandler: [Object],
  46. xid: 0,
  47. sessionId: <Buffer 01 53 82 40 72 3f 00 38>,
  48. sessionPassword: <Buffer 3d c1 4e 8e 83 e6 7f 85 07 06 78 a7 44 89 63 c2>,
  49. credentials: [],
  50. zxid: <Buffer 00 00 00 00 00 00 00 00>,
  51. pendingBuffer: null,
  52. packetQueue: [Object],
  53. pendingQueue: [],
  54. socket: [Object] },
  55. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  56. state: { name: 'SYNC_CONNECTED', code: 3 },
  57. closeCalled: false } }
  58. { path: '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta',
  59. err:
  60. { [NO_NODE: Exception: NO_NODE[-101]]
  61. code: -101,
  62. name: 'NO_NODE',
  63. path: undefined,
  64. message: 'Exception: NO_NODE[-101]' },
  65. data: undefined }
  66. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93883,"req_id":1909,"client":"10.43.1.15","port":"48638/udp","query":{"name":"manta.sol.sol.blackshard.net.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":15,"timers":{"zk.getNameRecord":12,"pre-resp":1,"log-after":2},"msg":"DNS query","time":"2016-03-17T15:06:18.908Z","v":0}
  67. { cache:
  68. { _max: 10000,
  69. _lengthCalculator: [Function: naiveLength],
  70. _allowStale: false,
  71. _maxAge: 60000,
  72. _dispose: undefined,
  73. _cache: { '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta': [Object] },
  74. _lruList: { '0': [Object] },
  75. _mru: 1,
  76. _lru: 0,
  77. _length: 1,
  78. _itemCount: 1 },
  79. path: '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta',
  80. zk:
  81. { domain: null,
  82. _events:
  83. { state: [Function: defaultStateListener],
  84. error: [Function],
  85. disconnected: [Object] },
  86. _maxListeners: undefined,
  87. connectionManager:
  88. { domain: null,
  89. _events: [Object],
  90. _maxListeners: undefined,
  91. watcherManager: [Object],
  92. connectionStringParser: [Object],
  93. servers: [Object],
  94. chrootPath: undefined,
  95. nextServerIndex: 1,
  96. serverAttempts: 0,
  97. state: 2,
  98. options: [Object],
  99. spinDelay: 1000,
  100. sessionTimeout: 30000,
  101. connectTimeout: 30000,
  102. pingTimeout: 10000,
  103. connectTimeoutHandler: [Object],
  104. xid: 1,
  105. sessionId: <Buffer 01 53 82 40 72 3f 00 38>,
  106. sessionPassword: <Buffer 3d c1 4e 8e 83 e6 7f 85 07 06 78 a7 44 89 63 c2>,
  107. credentials: [],
  108. zxid: <Buffer 00 00 00 00 00 00 03 9a>,
  109. pendingBuffer: null,
  110. packetQueue: [Object],
  111. pendingQueue: [],
  112. socket: [Object] },
  113. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  114. state: { name: 'SYNC_CONNECTED', code: 3 },
  115. closeCalled: false } }
  116. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93883,"req_id":35664,"client":"10.43.1.5","port":"59707/udp","query":{"name":"manta.sol.sol.blackshard.net.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":3,"timers":{"zk.getNameRecord":3,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:19.147Z","v":0}
  117. { cache:
  118. { _max: 10000,
  119. _lengthCalculator: [Function: naiveLength],
  120. _allowStale: false,
  121. _maxAge: 60000,
  122. _dispose: undefined,
  123. _cache: { '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta': [Object] },
  124. _lruList: { '1': [Object] },
  125. _mru: 2,
  126. _lru: 1,
  127. _length: 1,
  128. _itemCount: 1 },
  129. path: '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta',
  130. zk:
  131. { domain: null,
  132. _events:
  133. { state: [Function: defaultStateListener],
  134. error: [Function],
  135. disconnected: [Object] },
  136. _maxListeners: undefined,
  137. connectionManager:
  138. { domain: null,
  139. _events: [Object],
  140. _maxListeners: undefined,
  141. watcherManager: [Object],
  142. connectionStringParser: [Object],
  143. servers: [Object],
  144. chrootPath: undefined,
  145. nextServerIndex: 1,
  146. serverAttempts: 0,
  147. state: 2,
  148. options: [Object],
  149. spinDelay: 1000,
  150. sessionTimeout: 30000,
  151. connectTimeout: 30000,
  152. pingTimeout: 10000,
  153. connectTimeoutHandler: [Object],
  154. xid: 1,
  155. sessionId: <Buffer 01 53 82 40 72 3f 00 38>,
  156. sessionPassword: <Buffer 3d c1 4e 8e 83 e6 7f 85 07 06 78 a7 44 89 63 c2>,
  157. credentials: [],
  158. zxid: <Buffer 00 00 00 00 00 00 03 9a>,
  159. pendingBuffer: null,
  160. packetQueue: [Object],
  161. pendingQueue: [],
  162. socket: [Object] },
  163. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  164. state: { name: 'SYNC_CONNECTED', code: 3 },
  165. closeCalled: false } }
  166. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93883,"req_id":28426,"client":"10.43.1.9","port":"52864/udp","query":{"name":"manta.sol.sol.blackshard.net.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":3,"timers":{"zk.getNameRecord":3,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:19.164Z","v":0}
  167. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93883,"req_id":40855,"client":"10.43.1.13","port":"50117/udp","query":{"name":"manta.sol.sol.blackshard.net.blackshard.net","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:19.255Z","v":0}
  168. { cache:
  169. { _max: 10000,
  170. _lengthCalculator: [Function: naiveLength],
  171. _allowStale: false,
  172. _maxAge: 60000,
  173. _dispose: undefined,
  174. _cache: { '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta': [Object] },
  175. _lruList: { '2': [Object] },
  176. _mru: 3,
  177. _lru: 2,
  178. _length: 1,
  179. _itemCount: 1 },
  180. path: '/net/blackshard/sol/sol/manta',
  181. zk:
  182. { domain: null,
  183. _events:
  184. { state: [Function: defaultStateListener],
  185. error: [Function],
  186. disconnected: [Object] },
  187. _maxListeners: undefined,
  188. connectionManager:
  189. { domain: null,
  190. _events: [Object],
  191. _maxListeners: undefined,
  192. watcherManager: [Object],
  193. connectionStringParser: [Object],
  194. servers: [Object],
  195. chrootPath: undefined,
  196. nextServerIndex: 1,
  197. serverAttempts: 0,
  198. state: 2,
  199. options: [Object],
  200. spinDelay: 1000,
  201. sessionTimeout: 30000,
  202. connectTimeout: 30000,
  203. pingTimeout: 10000,
  204. connectTimeoutHandler: [Object],
  205. xid: 1,
  206. sessionId: <Buffer 01 53 82 40 72 3f 00 38>,
  207. sessionPassword: <Buffer 3d c1 4e 8e 83 e6 7f 85 07 06 78 a7 44 89 63 c2>,
  208. credentials: [],
  209. zxid: <Buffer 00 00 00 00 00 00 03 9a>,
  210. pendingBuffer: null,
  211. packetQueue: [Object],
  212. pendingQueue: [],
  213. socket: [Object] },
  214. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  215. state: { name: 'SYNC_CONNECTED', code: 3 },
  216. closeCalled: false } }
  217. { path: '/net/blackshard/sol/sol/manta',
  218. err: null,
  219. data: undefined }
  220. Uncaught TypeError: Cannot read property 'toString' of undefined
  221.  
  222. FROM
  223. /opt/smartdc/binder/lib/zk.js:135:42
  224. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  225. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  226. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  227. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  228. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  229. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  230. Socket.emit (events.js:107:17)
  231. readableAddChunk (_stream_readable.js:163:16)
  232. Socket.Readable.push (_stream_readable.js:126:10)
  233. TCP.onread (net.js:540:20)
  234. [ Mar 17 15:06:19 Stopping because process dumped core. ]
  235. [ Mar 17 15:06:19 Executing stop method (:kill). ]
  236. [ Mar 17 15:06:19 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  237. [ Mar 17 15:06:19 Method "start" exited with status 0. ]
  238. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93900,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:06:19.649Z","v":0}
  239. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93900,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:06:19.688Z","v":0}
  240. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93900,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:06:19.689Z","v":0}
  241. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93900,"level":30,"msg":"done with binder init","time":"2016-03-17T15:06:19.689Z","v":0}
  242. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93900,"req_id":19382,"client":"10.43.1.13","port":"34749/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":3,"timers":{"log-after":3},"msg":"DNS query","time":"2016-03-17T15:06:20.114Z","v":0}
  243. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93900,"req_id":19382,"client":"10.43.1.13","port":"48267/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:20.124Z","v":0}
  244. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93900,"req_id":18805,"client":"10.43.1.13","port":"37324/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:20.129Z","v":0}
  245. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93900,"req_id":18805,"client":"10.43.1.13","port":"57254/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:20.134Z","v":0}
  246. { cache:
  247. { _max: 10000,
  248. _lengthCalculator: [Function: naiveLength],
  249. _allowStale: false,
  250. _maxAge: 60000,
  251. _dispose: undefined,
  252. _cache: {},
  253. _lruList: {},
  254. _mru: 0,
  255. _lru: 0,
  256. _length: 0,
  257. _itemCount: 0 },
  258. path: '/net/blackshard/sol/sol/sapi',
  259. zk:
  260. { domain: null,
  261. _events:
  262. { state: [Function: defaultStateListener],
  263. error: [Function],
  264. disconnected: [Object] },
  265. _maxListeners: undefined,
  266. connectionManager:
  267. { domain: null,
  268. _events: [Object],
  269. _maxListeners: undefined,
  270. watcherManager: [Object],
  271. connectionStringParser: [Object],
  272. servers: [Object],
  273. chrootPath: undefined,
  274. nextServerIndex: 1,
  275. serverAttempts: 0,
  276. state: 2,
  277. options: [Object],
  278. spinDelay: 1000,
  279. sessionTimeout: 30000,
  280. connectTimeout: 30000,
  281. pingTimeout: 10000,
  282. connectTimeoutHandler: [Object],
  283. xid: 0,
  284. sessionId: <Buffer 01 53 82 40 72 3f 00 39>,
  285. sessionPassword: <Buffer 50 e2 48 8e 23 32 5f ca fc 48 df d2 18 12 78 8f>,
  286. credentials: [],
  287. zxid: <Buffer 00 00 00 00 00 00 00 00>,
  288. pendingBuffer: null,
  289. packetQueue: [Object],
  290. pendingQueue: [],
  291. socket: [Object] },
  292. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  293. state: { name: 'SYNC_CONNECTED', code: 3 },
  294. closeCalled: false } }
  295. { path: '/net/blackshard/sol/sol/sapi',
  296. err:
  297. { [NO_NODE: Exception: NO_NODE[-101]]
  298. code: -101,
  299. name: 'NO_NODE',
  300. path: undefined,
  301. message: 'Exception: NO_NODE[-101]' },
  302. data: undefined }
  303. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93900,"req_id":19717,"client":"10.43.1.13","port":"55995/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":10,"timers":{"zk.getNameRecord":9,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:20.147Z","v":0}
  304. { cache:
  305. { _max: 10000,
  306. _lengthCalculator: [Function: naiveLength],
  307. _allowStale: false,
  308. _maxAge: 60000,
  309. _dispose: undefined,
  310. _cache: { '/net/blackshard/sol/sol/sapi': [Object] },
  311. _lruList: { '0': [Object] },
  312. _mru: 1,
  313. _lru: 0,
  314. _length: 1,
  315. _itemCount: 1 },
  316. path: '/net/blackshard/sol/sol/manta',
  317. zk:
  318. { domain: null,
  319. _events:
  320. { state: [Function: defaultStateListener],
  321. error: [Function],
  322. disconnected: [Object] },
  323. _maxListeners: undefined,
  324. connectionManager:
  325. { domain: null,
  326. _events: [Object],
  327. _maxListeners: undefined,
  328. watcherManager: [Object],
  329. connectionStringParser: [Object],
  330. servers: [Object],
  331. chrootPath: undefined,
  332. nextServerIndex: 1,
  333. serverAttempts: 0,
  334. state: 2,
  335. options: [Object],
  336. spinDelay: 1000,
  337. sessionTimeout: 30000,
  338. connectTimeout: 30000,
  339. pingTimeout: 10000,
  340. connectTimeoutHandler: [Object],
  341. xid: 1,
  342. sessionId: <Buffer 01 53 82 40 72 3f 00 39>,
  343. sessionPassword: <Buffer 50 e2 48 8e 23 32 5f ca fc 48 df d2 18 12 78 8f>,
  344. credentials: [],
  345. zxid: <Buffer 00 00 00 00 00 00 03 9b>,
  346. pendingBuffer: null,
  347. packetQueue: [Object],
  348. pendingQueue: [],
  349. socket: [Object] },
  350. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  351. state: { name: 'SYNC_CONNECTED', code: 3 },
  352. closeCalled: false } }
  353. { path: '/net/blackshard/sol/sol/manta',
  354. err: null,
  355. data: undefined }
  356. Uncaught TypeError: Cannot read property 'toString' of undefined
  357.  
  358. FROM
  359. /opt/smartdc/binder/lib/zk.js:135:42
  360. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  361. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  362. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  363. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  364. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  365. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  366. Socket.emit (events.js:107:17)
  367. readableAddChunk (_stream_readable.js:163:16)
  368. Socket.Readable.push (_stream_readable.js:126:10)
  369. TCP.onread (net.js:540:20)
  370. [ Mar 17 15:06:20 Stopping because process dumped core. ]
  371. [ Mar 17 15:06:20 Executing stop method (:kill). ]
  372. [ Mar 17 15:06:20 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  373. [ Mar 17 15:06:20 Method "start" exited with status 0. ]
  374. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93908,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:06:20.796Z","v":0}
  375. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93908,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:06:20.852Z","v":0}
  376. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93908,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:06:20.853Z","v":0}
  377. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93908,"level":30,"msg":"done with binder init","time":"2016-03-17T15:06:20.853Z","v":0}
  378. { cache:
  379. { _max: 10000,
  380. _lengthCalculator: [Function: naiveLength],
  381. _allowStale: false,
  382. _maxAge: 60000,
  383. _dispose: undefined,
  384. _cache: {},
  385. _lruList: {},
  386. _mru: 0,
  387. _lru: 0,
  388. _length: 0,
  389. _itemCount: 0 },
  390. path: '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta',
  391. zk:
  392. { domain: null,
  393. _events:
  394. { state: [Function: defaultStateListener],
  395. error: [Function],
  396. disconnected: [Object] },
  397. _maxListeners: undefined,
  398. connectionManager:
  399. { domain: null,
  400. _events: [Object],
  401. _maxListeners: undefined,
  402. watcherManager: [Object],
  403. connectionStringParser: [Object],
  404. servers: [Object],
  405. chrootPath: undefined,
  406. nextServerIndex: 1,
  407. serverAttempts: 0,
  408. state: 2,
  409. options: [Object],
  410. spinDelay: 1000,
  411. sessionTimeout: 30000,
  412. connectTimeout: 30000,
  413. pingTimeout: 10000,
  414. connectTimeoutHandler: [Object],
  415. xid: 0,
  416. sessionId: <Buffer 01 53 82 40 72 3f 00 3a>,
  417. sessionPassword: <Buffer 17 7f 5a 8e 43 4f c1 fb 1c 80 a9 50 9d 77 3a 28>,
  418. credentials: [],
  419. zxid: <Buffer 00 00 00 00 00 00 00 00>,
  420. pendingBuffer: null,
  421. packetQueue: [Object],
  422. pendingQueue: [],
  423. socket: [Object] },
  424. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  425. state: { name: 'SYNC_CONNECTED', code: 3 },
  426. closeCalled: false } }
  427. { path: '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta',
  428. err:
  429. { [NO_NODE: Exception: NO_NODE[-101]]
  430. code: -101,
  431. name: 'NO_NODE',
  432. path: undefined,
  433. message: 'Exception: NO_NODE[-101]' },
  434. data: undefined }
  435. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93908,"req_id":35664,"client":"10.43.1.5","port":"61104/udp","query":{"name":"manta.sol.sol.blackshard.net.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":12,"timers":{"zk.getNameRecord":9,"pre-resp":1,"log-after":2},"msg":"DNS query","time":"2016-03-17T15:06:21.163Z","v":0}
  436. { cache:
  437. { _max: 10000,
  438. _lengthCalculator: [Function: naiveLength],
  439. _allowStale: false,
  440. _maxAge: 60000,
  441. _dispose: undefined,
  442. _cache: { '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta': [Object] },
  443. _lruList: { '0': [Object] },
  444. _mru: 1,
  445. _lru: 0,
  446. _length: 1,
  447. _itemCount: 1 },
  448. path: '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta',
  449. zk:
  450. { domain: null,
  451. _events:
  452. { state: [Function: defaultStateListener],
  453. error: [Function],
  454. disconnected: [Object] },
  455. _maxListeners: undefined,
  456. connectionManager:
  457. { domain: null,
  458. _events: [Object],
  459. _maxListeners: undefined,
  460. watcherManager: [Object],
  461. connectionStringParser: [Object],
  462. servers: [Object],
  463. chrootPath: undefined,
  464. nextServerIndex: 1,
  465. serverAttempts: 0,
  466. state: 2,
  467. options: [Object],
  468. spinDelay: 1000,
  469. sessionTimeout: 30000,
  470. connectTimeout: 30000,
  471. pingTimeout: 10000,
  472. connectTimeoutHandler: [Object],
  473. xid: 1,
  474. sessionId: <Buffer 01 53 82 40 72 3f 00 3a>,
  475. sessionPassword: <Buffer 17 7f 5a 8e 43 4f c1 fb 1c 80 a9 50 9d 77 3a 28>,
  476. credentials: [],
  477. zxid: <Buffer 00 00 00 00 00 00 03 9c>,
  478. pendingBuffer: null,
  479. packetQueue: [Object],
  480. pendingQueue: [],
  481. socket: [Object] },
  482. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  483. state: { name: 'SYNC_CONNECTED', code: 3 },
  484. closeCalled: false } }
  485. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93908,"req_id":28426,"client":"10.43.1.9","port":"36503/udp","query":{"name":"manta.sol.sol.blackshard.net.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":4,"timers":{"zk.getNameRecord":3,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:21.170Z","v":0}
  486. { cache:
  487. { _max: 10000,
  488. _lengthCalculator: [Function: naiveLength],
  489. _allowStale: false,
  490. _maxAge: 60000,
  491. _dispose: undefined,
  492. _cache: { '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta': [Object] },
  493. _lruList: { '1': [Object] },
  494. _mru: 2,
  495. _lru: 1,
  496. _length: 1,
  497. _itemCount: 1 },
  498. path: '/net/blackshard/sol/sol/moray/1',
  499. zk:
  500. { domain: null,
  501. _events:
  502. { state: [Function: defaultStateListener],
  503. error: [Function],
  504. disconnected: [Object] },
  505. _maxListeners: undefined,
  506. connectionManager:
  507. { domain: null,
  508. _events: [Object],
  509. _maxListeners: undefined,
  510. watcherManager: [Object],
  511. connectionStringParser: [Object],
  512. servers: [Object],
  513. chrootPath: undefined,
  514. nextServerIndex: 1,
  515. serverAttempts: 0,
  516. state: 2,
  517. options: [Object],
  518. spinDelay: 1000,
  519. sessionTimeout: 30000,
  520. connectTimeout: 30000,
  521. pingTimeout: 10000,
  522. connectTimeoutHandler: [Object],
  523. xid: 1,
  524. sessionId: <Buffer 01 53 82 40 72 3f 00 3a>,
  525. sessionPassword: <Buffer 17 7f 5a 8e 43 4f c1 fb 1c 80 a9 50 9d 77 3a 28>,
  526. credentials: [],
  527. zxid: <Buffer 00 00 00 00 00 00 03 9c>,
  528. pendingBuffer: null,
  529. packetQueue: [Object],
  530. pendingQueue: [],
  531. socket: [Object] },
  532. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  533. state: { name: 'SYNC_CONNECTED', code: 3 },
  534. closeCalled: false } }
  535. { path: '/net/blackshard/sol/sol/moray/1',
  536. err: null,
  537. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  538. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93908,"req_id":16855,"client":"10.43.1.2","port":"65277/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":9,"timers":{"zk.getNameRecord":4,"zk.loadService":3,"pre-resp":0,"log-after":2},"msg":"DNS query","time":"2016-03-17T15:06:21.192Z","v":0}
  539. { cache:
  540. { _max: 10000,
  541. _lengthCalculator: [Function: naiveLength],
  542. _allowStale: false,
  543. _maxAge: 60000,
  544. _dispose: undefined,
  545. _cache:
  546. { '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta': [Object],
  547. '/net/blackshard/sol/sol/moray/1': [Object],
  548. '/net/blackshard/sol/sol/moray/1__service': [Object] },
  549. _lruList: { '1': [Object], '2': [Object], '3': [Object] },
  550. _mru: 4,
  551. _lru: 1,
  552. _length: 3,
  553. _itemCount: 3 },
  554. path: '/net/blackshard/sol/sol/electric-moray',
  555. zk:
  556. { domain: null,
  557. _events:
  558. { state: [Function: defaultStateListener],
  559. error: [Function],
  560. disconnected: [Object] },
  561. _maxListeners: undefined,
  562. connectionManager:
  563. { domain: null,
  564. _events: [Object],
  565. _maxListeners: undefined,
  566. watcherManager: [Object],
  567. connectionStringParser: [Object],
  568. servers: [Object],
  569. chrootPath: undefined,
  570. nextServerIndex: 1,
  571. serverAttempts: 0,
  572. state: 2,
  573. options: [Object],
  574. spinDelay: 1000,
  575. sessionTimeout: 30000,
  576. connectTimeout: 30000,
  577. pingTimeout: 10000,
  578. connectTimeoutHandler: [Object],
  579. xid: 5,
  580. sessionId: <Buffer 01 53 82 40 72 3f 00 3a>,
  581. sessionPassword: <Buffer 17 7f 5a 8e 43 4f c1 fb 1c 80 a9 50 9d 77 3a 28>,
  582. credentials: [],
  583. zxid: <Buffer 00 00 00 00 00 00 03 9c>,
  584. pendingBuffer: null,
  585. packetQueue: [Object],
  586. pendingQueue: [],
  587. socket: [Object] },
  588. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  589. state: { name: 'SYNC_CONNECTED', code: 3 },
  590. closeCalled: false } }
  591. { cache:
  592. { _max: 10000,
  593. _lengthCalculator: [Function: naiveLength],
  594. _allowStale: false,
  595. _maxAge: 60000,
  596. _dispose: undefined,
  597. _cache:
  598. { '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta': [Object],
  599. '/net/blackshard/sol/sol/moray/1': [Object],
  600. '/net/blackshard/sol/sol/moray/1__service': [Object] },
  601. _lruList: { '1': [Object], '2': [Object], '3': [Object] },
  602. _mru: 4,
  603. _lru: 1,
  604. _length: 3,
  605. _itemCount: 3 },
  606. path: '/net/blackshard/sol/sol/moray/1',
  607. zk:
  608. { domain: null,
  609. _events:
  610. { state: [Function: defaultStateListener],
  611. error: [Function],
  612. disconnected: [Object] },
  613. _maxListeners: undefined,
  614. connectionManager:
  615. { domain: null,
  616. _events: [Object],
  617. _maxListeners: undefined,
  618. watcherManager: [Object],
  619. connectionStringParser: [Object],
  620. servers: [Object],
  621. chrootPath: undefined,
  622. nextServerIndex: 1,
  623. serverAttempts: 0,
  624. state: 2,
  625. options: [Object],
  626. spinDelay: 1000,
  627. sessionTimeout: 30000,
  628. connectTimeout: 30000,
  629. pingTimeout: 10000,
  630. connectTimeoutHandler: [Object],
  631. xid: 6,
  632. sessionId: <Buffer 01 53 82 40 72 3f 00 3a>,
  633. sessionPassword: <Buffer 17 7f 5a 8e 43 4f c1 fb 1c 80 a9 50 9d 77 3a 28>,
  634. credentials: [],
  635. zxid: <Buffer 00 00 00 00 00 00 03 9c>,
  636. pendingBuffer: null,
  637. packetQueue: [Object],
  638. pendingQueue: [Object],
  639. socket: [Object] },
  640. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  641. state: { name: 'SYNC_CONNECTED', code: 3 },
  642. closeCalled: false } }
  643. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93908,"req_id":20183,"client":"10.43.1.11","port":"46228/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":2,"timers":{"zk.getNameRecord":1,"zk.loadService":1,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:21.216Z","v":0}
  644. { path: '/net/blackshard/sol/sol/electric-moray',
  645. err: null,
  646. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  647. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93908,"req_id":40391,"client":"10.43.1.11","port":"46228/udp","query":{"name":"electric-moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.7"],"additional":[],"latency":9,"timers":{"zk.getNameRecord":7,"zk.loadService":2,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:21.218Z","v":0}
  648. { cache:
  649. { _max: 10000,
  650. _lengthCalculator: [Function: naiveLength],
  651. _allowStale: false,
  652. _maxAge: 60000,
  653. _dispose: undefined,
  654. _cache:
  655. { '/net/blackshard/sol/sol/net/blackshard/sol/sol/manta': [Object],
  656. '/net/blackshard/sol/sol/moray/1': [Object],
  657. '/net/blackshard/sol/sol/moray/1__service': [Object],
  658. '/net/blackshard/sol/sol/electric-moray': [Object],
  659. '/net/blackshard/sol/sol/electric-moray__service': [Object] },
  660. _lruList:
  661. { '1': [Object],
  662. '4': [Object],
  663. '5': [Object],
  664. '6': [Object],
  665. '7': [Object] },
  666. _mru: 8,
  667. _lru: 1,
  668. _length: 5,
  669. _itemCount: 5 },
  670. path: '/net/blackshard/sol/sol/manta',
  671. zk:
  672. { domain: null,
  673. _events:
  674. { state: [Function: defaultStateListener],
  675. error: [Function],
  676. disconnected: [Object] },
  677. _maxListeners: undefined,
  678. connectionManager:
  679. { domain: null,
  680. _events: [Object],
  681. _maxListeners: undefined,
  682. watcherManager: [Object],
  683. connectionStringParser: [Object],
  684. servers: [Object],
  685. chrootPath: undefined,
  686. nextServerIndex: 1,
  687. serverAttempts: 0,
  688. state: 2,
  689. options: [Object],
  690. spinDelay: 1000,
  691. sessionTimeout: 30000,
  692. connectTimeout: 30000,
  693. pingTimeout: 10000,
  694. connectTimeoutHandler: [Object],
  695. xid: 8,
  696. sessionId: <Buffer 01 53 82 40 72 3f 00 3a>,
  697. sessionPassword: <Buffer 17 7f 5a 8e 43 4f c1 fb 1c 80 a9 50 9d 77 3a 28>,
  698. credentials: [],
  699. zxid: <Buffer 00 00 00 00 00 00 03 9c>,
  700. pendingBuffer: null,
  701. packetQueue: [Object],
  702. pendingQueue: [],
  703. socket: [Object] },
  704. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  705. state: { name: 'SYNC_CONNECTED', code: 3 },
  706. closeCalled: false } }
  707. { path: '/net/blackshard/sol/sol/manta',
  708. err: null,
  709. data: undefined }
  710. Uncaught TypeError: Cannot read property 'toString' of undefined
  711.  
  712. FROM
  713. /opt/smartdc/binder/lib/zk.js:135:42
  714. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  715. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  716. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  717. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  718. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  719. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  720. Socket.emit (events.js:107:17)
  721. readableAddChunk (_stream_readable.js:163:16)
  722. Socket.Readable.push (_stream_readable.js:126:10)
  723. TCP.onread (net.js:540:20)
  724. [ Mar 17 15:06:21 Stopping because process dumped core. ]
  725. [ Mar 17 15:06:21 Executing stop method (:kill). ]
  726. [ Mar 17 15:06:21 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  727. [ Mar 17 15:06:21 Method "start" exited with status 0. ]
  728. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93924,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:06:21.695Z","v":0}
  729. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93924,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:06:21.753Z","v":0}
  730. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93924,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:06:21.753Z","v":0}
  731. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93924,"level":30,"msg":"done with binder init","time":"2016-03-17T15:06:21.754Z","v":0}
  732. { cache:
  733. { _max: 10000,
  734. _lengthCalculator: [Function: naiveLength],
  735. _allowStale: false,
  736. _maxAge: 60000,
  737. _dispose: undefined,
  738. _cache: {},
  739. _lruList: {},
  740. _mru: 0,
  741. _lru: 0,
  742. _length: 0,
  743. _itemCount: 0 },
  744. path: '/net/blackshard/sol/sol/moray/1',
  745. zk:
  746. { domain: null,
  747. _events:
  748. { state: [Function: defaultStateListener],
  749. error: [Function],
  750. disconnected: [Object] },
  751. _maxListeners: undefined,
  752. connectionManager:
  753. { domain: null,
  754. _events: [Object],
  755. _maxListeners: undefined,
  756. watcherManager: [Object],
  757. connectionStringParser: [Object],
  758. servers: [Object],
  759. chrootPath: undefined,
  760. nextServerIndex: 1,
  761. serverAttempts: 0,
  762. state: 2,
  763. options: [Object],
  764. spinDelay: 1000,
  765. sessionTimeout: 30000,
  766. connectTimeout: 30000,
  767. pingTimeout: 10000,
  768. connectTimeoutHandler: [Object],
  769. xid: 0,
  770. sessionId: <Buffer 01 53 82 40 72 3f 00 3b>,
  771. sessionPassword: <Buffer 2a a0 54 8e e3 9a a0 40 11 c3 10 7c 71 00 4f f5>,
  772. credentials: [],
  773. zxid: <Buffer 00 00 00 00 00 00 00 00>,
  774. pendingBuffer: null,
  775. packetQueue: [Object],
  776. pendingQueue: [],
  777. socket: [Object] },
  778. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  779. state: { name: 'SYNC_CONNECTED', code: 3 },
  780. closeCalled: false } }
  781. { cache:
  782. { _max: 10000,
  783. _lengthCalculator: [Function: naiveLength],
  784. _allowStale: false,
  785. _maxAge: 60000,
  786. _dispose: undefined,
  787. _cache: {},
  788. _lruList: {},
  789. _mru: 0,
  790. _lru: 0,
  791. _length: 0,
  792. _itemCount: 0 },
  793. path: '/net/blackshard/sol/sol/electric-moray',
  794. zk:
  795. { domain: null,
  796. _events:
  797. { state: [Function: defaultStateListener],
  798. error: [Function],
  799. disconnected: [Object] },
  800. _maxListeners: undefined,
  801. connectionManager:
  802. { domain: null,
  803. _events: [Object],
  804. _maxListeners: undefined,
  805. watcherManager: [Object],
  806. connectionStringParser: [Object],
  807. servers: [Object],
  808. chrootPath: undefined,
  809. nextServerIndex: 1,
  810. serverAttempts: 0,
  811. state: 2,
  812. options: [Object],
  813. spinDelay: 1000,
  814. sessionTimeout: 30000,
  815. connectTimeout: 30000,
  816. pingTimeout: 10000,
  817. connectTimeoutHandler: [Object],
  818. xid: 1,
  819. sessionId: <Buffer 01 53 82 40 72 3f 00 3b>,
  820. sessionPassword: <Buffer 2a a0 54 8e e3 9a a0 40 11 c3 10 7c 71 00 4f f5>,
  821. credentials: [],
  822. zxid: <Buffer 00 00 00 00 00 00 00 00>,
  823. pendingBuffer: null,
  824. packetQueue: [Object],
  825. pendingQueue: [Object],
  826. socket: [Object] },
  827. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  828. state: { name: 'SYNC_CONNECTED', code: 3 },
  829. closeCalled: false } }
  830. { path: '/net/blackshard/sol/sol/moray/1',
  831. err: null,
  832. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  833. { path: '/net/blackshard/sol/sol/electric-moray',
  834. err: null,
  835. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  836. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93924,"req_id":17894,"client":"10.43.1.12","port":"55372/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":20,"timers":{"zk.getNameRecord":12,"zk.loadService":5,"pre-resp":0,"log-after":3},"msg":"DNS query","time":"2016-03-17T15:06:21.854Z","v":0}
  837. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93924,"req_id":55185,"client":"10.43.1.12","port":"55372/udp","query":{"name":"electric-moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.7"],"additional":[],"latency":13,"timers":{"zk.getNameRecord":5,"zk.loadService":7,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:21.856Z","v":0}
  838. { cache:
  839. { _max: 10000,
  840. _lengthCalculator: [Function: naiveLength],
  841. _allowStale: false,
  842. _maxAge: 60000,
  843. _dispose: undefined,
  844. _cache:
  845. { '/net/blackshard/sol/sol/moray/1': [Object],
  846. '/net/blackshard/sol/sol/electric-moray': [Object],
  847. '/net/blackshard/sol/sol/moray/1__service': [Object],
  848. '/net/blackshard/sol/sol/electric-moray__service': [Object] },
  849. _lruList: { '0': [Object], '1': [Object], '2': [Object], '3': [Object] },
  850. _mru: 4,
  851. _lru: 0,
  852. _length: 4,
  853. _itemCount: 4 },
  854. path: '/net/blackshard/sol/net/blackshard/sol/sol/manta',
  855. zk:
  856. { domain: null,
  857. _events:
  858. { state: [Function: defaultStateListener],
  859. error: [Function],
  860. disconnected: [Object] },
  861. _maxListeners: undefined,
  862. connectionManager:
  863. { domain: null,
  864. _events: [Object],
  865. _maxListeners: undefined,
  866. watcherManager: [Object],
  867. connectionStringParser: [Object],
  868. servers: [Object],
  869. chrootPath: undefined,
  870. nextServerIndex: 1,
  871. serverAttempts: 0,
  872. state: 2,
  873. options: [Object],
  874. spinDelay: 1000,
  875. sessionTimeout: 30000,
  876. connectTimeout: 30000,
  877. pingTimeout: 10000,
  878. connectTimeoutHandler: [Object],
  879. xid: 7,
  880. sessionId: <Buffer 01 53 82 40 72 3f 00 3b>,
  881. sessionPassword: <Buffer 2a a0 54 8e e3 9a a0 40 11 c3 10 7c 71 00 4f f5>,
  882. credentials: [],
  883. zxid: <Buffer 00 00 00 00 00 00 03 9d>,
  884. pendingBuffer: null,
  885. packetQueue: [Object],
  886. pendingQueue: [],
  887. socket: [Object] },
  888. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  889. state: { name: 'SYNC_CONNECTED', code: 3 },
  890. closeCalled: false } }
  891. { path: '/net/blackshard/sol/net/blackshard/sol/sol/manta',
  892. err:
  893. { [NO_NODE: Exception: NO_NODE[-101]]
  894. code: -101,
  895. name: 'NO_NODE',
  896. path: undefined,
  897. message: 'Exception: NO_NODE[-101]' },
  898. data: undefined }
  899. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93924,"req_id":46919,"client":"10.43.1.15","port":"33171/udp","query":{"name":"manta.sol.sol.blackshard.net...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":4,"timers":{"zk.getNameRecord":3,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:21.930Z","v":0}
  900. { cache:
  901. { _max: 10000,
  902. _lengthCalculator: [Function: naiveLength],
  903. _allowStale: false,
  904. _maxAge: 60000,
  905. _dispose: undefined,
  906. _cache:
  907. { '/net/blackshard/sol/sol/moray/1': [Object],
  908. '/net/blackshard/sol/sol/electric-moray': [Object],
  909. '/net/blackshard/sol/sol/moray/1__service': [Object],
  910. '/net/blackshard/sol/sol/electric-moray__service': [Object],
  911. '/net/blackshard/sol/net/blackshard/sol/sol/manta': [Object] },
  912. _lruList:
  913. { '0': [Object],
  914. '1': [Object],
  915. '2': [Object],
  916. '3': [Object],
  917. '4': [Object] },
  918. _mru: 5,
  919. _lru: 0,
  920. _length: 5,
  921. _itemCount: 5 },
  922. path: '/net/blackshard/sol/net/blackshard/sol/sol/manta',
  923. zk:
  924. { domain: null,
  925. _events:
  926. { state: [Function: defaultStateListener],
  927. error: [Function],
  928. disconnected: [Object] },
  929. _maxListeners: undefined,
  930. connectionManager:
  931. { domain: null,
  932. _events: [Object],
  933. _maxListeners: undefined,
  934. watcherManager: [Object],
  935. connectionStringParser: [Object],
  936. servers: [Object],
  937. chrootPath: undefined,
  938. nextServerIndex: 1,
  939. serverAttempts: 0,
  940. state: 2,
  941. options: [Object],
  942. spinDelay: 1000,
  943. sessionTimeout: 30000,
  944. connectTimeout: 30000,
  945. pingTimeout: 10000,
  946. connectTimeoutHandler: [Object],
  947. xid: 8,
  948. sessionId: <Buffer 01 53 82 40 72 3f 00 3b>,
  949. sessionPassword: <Buffer 2a a0 54 8e e3 9a a0 40 11 c3 10 7c 71 00 4f f5>,
  950. credentials: [],
  951. zxid: <Buffer 00 00 00 00 00 00 03 9d>,
  952. pendingBuffer: null,
  953. packetQueue: [Object],
  954. pendingQueue: [],
  955. socket: [Object] },
  956. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  957. state: { name: 'SYNC_CONNECTED', code: 3 },
  958. closeCalled: false } }
  959. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93924,"req_id":46919,"client":"10.43.1.15","port":"37854/udp","query":{"name":"manta.sol.sol.blackshard.net...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":2,"timers":{"zk.getNameRecord":2,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:21.938Z","v":0}
  960. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93924,"req_id":16415,"client":"10.43.1.15","port":"41522/udp","query":{"name":"manta.sol.sol.blackshard.net.blackshard.net","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:21.942Z","v":0}
  961. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93924,"req_id":16415,"client":"10.43.1.15","port":"58952/udp","query":{"name":"manta.sol.sol.blackshard.net.blackshard.net","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:21.950Z","v":0}
  962. { cache:
  963. { _max: 10000,
  964. _lengthCalculator: [Function: naiveLength],
  965. _allowStale: false,
  966. _maxAge: 60000,
  967. _dispose: undefined,
  968. _cache:
  969. { '/net/blackshard/sol/sol/moray/1': [Object],
  970. '/net/blackshard/sol/sol/electric-moray': [Object],
  971. '/net/blackshard/sol/sol/moray/1__service': [Object],
  972. '/net/blackshard/sol/sol/electric-moray__service': [Object],
  973. '/net/blackshard/sol/net/blackshard/sol/sol/manta': [Object] },
  974. _lruList:
  975. { '0': [Object],
  976. '1': [Object],
  977. '2': [Object],
  978. '3': [Object],
  979. '5': [Object] },
  980. _mru: 6,
  981. _lru: 0,
  982. _length: 5,
  983. _itemCount: 5 },
  984. path: '/net/blackshard/sol/sol/manta',
  985. zk:
  986. { domain: null,
  987. _events:
  988. { state: [Function: defaultStateListener],
  989. error: [Function],
  990. disconnected: [Object] },
  991. _maxListeners: undefined,
  992. connectionManager:
  993. { domain: null,
  994. _events: [Object],
  995. _maxListeners: undefined,
  996. watcherManager: [Object],
  997. connectionStringParser: [Object],
  998. servers: [Object],
  999. chrootPath: undefined,
  1000. nextServerIndex: 1,
  1001. serverAttempts: 0,
  1002. state: 2,
  1003. options: [Object],
  1004. spinDelay: 1000,
  1005. sessionTimeout: 30000,
  1006. connectTimeout: 30000,
  1007. pingTimeout: 10000,
  1008. connectTimeoutHandler: [Object],
  1009. xid: 8,
  1010. sessionId: <Buffer 01 53 82 40 72 3f 00 3b>,
  1011. sessionPassword: <Buffer 2a a0 54 8e e3 9a a0 40 11 c3 10 7c 71 00 4f f5>,
  1012. credentials: [],
  1013. zxid: <Buffer 00 00 00 00 00 00 03 9d>,
  1014. pendingBuffer: null,
  1015. packetQueue: [Object],
  1016. pendingQueue: [],
  1017. socket: [Object] },
  1018. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1019. state: { name: 'SYNC_CONNECTED', code: 3 },
  1020. closeCalled: false } }
  1021. { path: '/net/blackshard/sol/sol/manta',
  1022. err: null,
  1023. data: undefined }
  1024. Uncaught TypeError: Cannot read property 'toString' of undefined
  1025.  
  1026. FROM
  1027. /opt/smartdc/binder/lib/zk.js:135:42
  1028. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  1029. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  1030. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  1031. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  1032. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  1033. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  1034. Socket.emit (events.js:107:17)
  1035. readableAddChunk (_stream_readable.js:163:16)
  1036. Socket.Readable.push (_stream_readable.js:126:10)
  1037. TCP.onread (net.js:540:20)
  1038. [ Mar 17 15:06:22 Stopping because process dumped core. ]
  1039. [ Mar 17 15:06:22 Executing stop method (:kill). ]
  1040. [ Mar 17 15:06:22 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  1041. [ Mar 17 15:06:22 Method "start" exited with status 0. ]
  1042. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:06:22.628Z","v":0}
  1043. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:06:22.663Z","v":0}
  1044. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:06:22.664Z","v":0}
  1045. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"level":30,"msg":"done with binder init","time":"2016-03-17T15:06:22.664Z","v":0}
  1046. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"req_id":56620,"client":"10.43.1.3","port":"56250/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":3,"timers":{"log-after":3},"msg":"DNS query","time":"2016-03-17T15:06:22.797Z","v":0}
  1047. { cache:
  1048. { _max: 10000,
  1049. _lengthCalculator: [Function: naiveLength],
  1050. _allowStale: false,
  1051. _maxAge: 60000,
  1052. _dispose: undefined,
  1053. _cache: {},
  1054. _lruList: {},
  1055. _mru: 0,
  1056. _lru: 0,
  1057. _length: 0,
  1058. _itemCount: 0 },
  1059. path: '/net/blackshard/sol/sol/sapi',
  1060. zk:
  1061. { domain: null,
  1062. _events:
  1063. { state: [Function: defaultStateListener],
  1064. error: [Function],
  1065. disconnected: [Object] },
  1066. _maxListeners: undefined,
  1067. connectionManager:
  1068. { domain: null,
  1069. _events: [Object],
  1070. _maxListeners: undefined,
  1071. watcherManager: [Object],
  1072. connectionStringParser: [Object],
  1073. servers: [Object],
  1074. chrootPath: undefined,
  1075. nextServerIndex: 1,
  1076. serverAttempts: 0,
  1077. state: 2,
  1078. options: [Object],
  1079. spinDelay: 1000,
  1080. sessionTimeout: 30000,
  1081. connectTimeout: 30000,
  1082. pingTimeout: 10000,
  1083. connectTimeoutHandler: [Object],
  1084. xid: 0,
  1085. sessionId: <Buffer 01 53 82 40 72 3f 00 3c>,
  1086. sessionPassword: <Buffer 8a 45 37 8e 03 15 fd 98 dc 11 15 55 93 ac b5 f6>,
  1087. credentials: [],
  1088. zxid: <Buffer 00 00 00 00 00 00 00 00>,
  1089. pendingBuffer: null,
  1090. packetQueue: [Object],
  1091. pendingQueue: [],
  1092. socket: [Object] },
  1093. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1094. state: { name: 'SYNC_CONNECTED', code: 3 },
  1095. closeCalled: false } }
  1096. { path: '/net/blackshard/sol/sol/sapi',
  1097. err:
  1098. { [NO_NODE: Exception: NO_NODE[-101]]
  1099. code: -101,
  1100. name: 'NO_NODE',
  1101. path: undefined,
  1102. message: 'Exception: NO_NODE[-101]' },
  1103. data: undefined }
  1104. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"req_id":10099,"client":"10.43.1.3","port":"62945/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":11,"timers":{"zk.getNameRecord":10,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:22.813Z","v":0}
  1105. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"req_id":43210,"client":"10.43.1.9","port":"59054/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:22.926Z","v":0}
  1106. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"req_id":43210,"client":"10.43.1.9","port":"47255/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:22.933Z","v":0}
  1107. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"req_id":36989,"client":"10.43.1.9","port":"51918/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:22.941Z","v":0}
  1108. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"req_id":36989,"client":"10.43.1.9","port":"46822/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:22.944Z","v":0}
  1109. { cache:
  1110. { _max: 10000,
  1111. _lengthCalculator: [Function: naiveLength],
  1112. _allowStale: false,
  1113. _maxAge: 60000,
  1114. _dispose: undefined,
  1115. _cache: { '/net/blackshard/sol/sol/sapi': [Object] },
  1116. _lruList: { '0': [Object] },
  1117. _mru: 1,
  1118. _lru: 0,
  1119. _length: 1,
  1120. _itemCount: 1 },
  1121. path: '/net/blackshard/sol/sol/sapi',
  1122. zk:
  1123. { domain: null,
  1124. _events:
  1125. { state: [Function: defaultStateListener],
  1126. error: [Function],
  1127. disconnected: [Object] },
  1128. _maxListeners: undefined,
  1129. connectionManager:
  1130. { domain: null,
  1131. _events: [Object],
  1132. _maxListeners: undefined,
  1133. watcherManager: [Object],
  1134. connectionStringParser: [Object],
  1135. servers: [Object],
  1136. chrootPath: undefined,
  1137. nextServerIndex: 1,
  1138. serverAttempts: 0,
  1139. state: 2,
  1140. options: [Object],
  1141. spinDelay: 1000,
  1142. sessionTimeout: 30000,
  1143. connectTimeout: 30000,
  1144. pingTimeout: 10000,
  1145. connectTimeoutHandler: [Object],
  1146. xid: 1,
  1147. sessionId: <Buffer 01 53 82 40 72 3f 00 3c>,
  1148. sessionPassword: <Buffer 8a 45 37 8e 03 15 fd 98 dc 11 15 55 93 ac b5 f6>,
  1149. credentials: [],
  1150. zxid: <Buffer 00 00 00 00 00 00 03 9e>,
  1151. pendingBuffer: null,
  1152. packetQueue: [Object],
  1153. pendingQueue: [],
  1154. socket: [Object] },
  1155. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1156. state: { name: 'SYNC_CONNECTED', code: 3 },
  1157. closeCalled: false } }
  1158. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93930,"req_id":28150,"client":"10.43.1.9","port":"54393/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":3,"timers":{"zk.getNameRecord":3,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:22.950Z","v":0}
  1159. { cache:
  1160. { _max: 10000,
  1161. _lengthCalculator: [Function: naiveLength],
  1162. _allowStale: false,
  1163. _maxAge: 60000,
  1164. _dispose: undefined,
  1165. _cache: { '/net/blackshard/sol/sol/sapi': [Object] },
  1166. _lruList: { '1': [Object] },
  1167. _mru: 2,
  1168. _lru: 1,
  1169. _length: 1,
  1170. _itemCount: 1 },
  1171. path: '/net/blackshard/sol/sol/manta',
  1172. zk:
  1173. { domain: null,
  1174. _events:
  1175. { state: [Function: defaultStateListener],
  1176. error: [Function],
  1177. disconnected: [Object] },
  1178. _maxListeners: undefined,
  1179. connectionManager:
  1180. { domain: null,
  1181. _events: [Object],
  1182. _maxListeners: undefined,
  1183. watcherManager: [Object],
  1184. connectionStringParser: [Object],
  1185. servers: [Object],
  1186. chrootPath: undefined,
  1187. nextServerIndex: 1,
  1188. serverAttempts: 0,
  1189. state: 2,
  1190. options: [Object],
  1191. spinDelay: 1000,
  1192. sessionTimeout: 30000,
  1193. connectTimeout: 30000,
  1194. pingTimeout: 10000,
  1195. connectTimeoutHandler: [Object],
  1196. xid: 1,
  1197. sessionId: <Buffer 01 53 82 40 72 3f 00 3c>,
  1198. sessionPassword: <Buffer 8a 45 37 8e 03 15 fd 98 dc 11 15 55 93 ac b5 f6>,
  1199. credentials: [],
  1200. zxid: <Buffer 00 00 00 00 00 00 03 9e>,
  1201. pendingBuffer: null,
  1202. packetQueue: [Object],
  1203. pendingQueue: [],
  1204. socket: [Object] },
  1205. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1206. state: { name: 'SYNC_CONNECTED', code: 3 },
  1207. closeCalled: false } }
  1208. { path: '/net/blackshard/sol/sol/manta',
  1209. err: null,
  1210. data: undefined }
  1211. Uncaught TypeError: Cannot read property 'toString' of undefined
  1212.  
  1213. FROM
  1214. /opt/smartdc/binder/lib/zk.js:135:42
  1215. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  1216. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  1217. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  1218. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  1219. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  1220. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  1221. Socket.emit (events.js:107:17)
  1222. readableAddChunk (_stream_readable.js:163:16)
  1223. Socket.Readable.push (_stream_readable.js:126:10)
  1224. TCP.onread (net.js:540:20)
  1225. [ Mar 17 15:06:23 Stopping because process dumped core. ]
  1226. [ Mar 17 15:06:23 Executing stop method (:kill). ]
  1227. [ Mar 17 15:06:23 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  1228. [ Mar 17 15:06:23 Method "start" exited with status 0. ]
  1229. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:06:23.778Z","v":0}
  1230. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:06:23.821Z","v":0}
  1231. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:06:23.822Z","v":0}
  1232. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"level":30,"msg":"done with binder init","time":"2016-03-17T15:06:23.822Z","v":0}
  1233. { cache:
  1234. { _max: 10000,
  1235. _lengthCalculator: [Function: naiveLength],
  1236. _allowStale: false,
  1237. _maxAge: 60000,
  1238. _dispose: undefined,
  1239. _cache: {},
  1240. _lruList: {},
  1241. _mru: 0,
  1242. _lru: 0,
  1243. _length: 0,
  1244. _itemCount: 0 },
  1245. path: '/net/blackshard/sol/sol/moray/1',
  1246. zk:
  1247. { domain: null,
  1248. _events:
  1249. { state: [Function: defaultStateListener],
  1250. error: [Function],
  1251. disconnected: [Object] },
  1252. _maxListeners: undefined,
  1253. connectionManager:
  1254. { domain: null,
  1255. _events: [Object],
  1256. _maxListeners: undefined,
  1257. watcherManager: [Object],
  1258. connectionStringParser: [Object],
  1259. servers: [Object],
  1260. chrootPath: undefined,
  1261. nextServerIndex: 1,
  1262. serverAttempts: 0,
  1263. state: 2,
  1264. options: [Object],
  1265. spinDelay: 1000,
  1266. sessionTimeout: 30000,
  1267. connectTimeout: 30000,
  1268. pingTimeout: 10000,
  1269. connectTimeoutHandler: [Object],
  1270. xid: 0,
  1271. sessionId: <Buffer 01 53 82 40 72 3f 00 3d>,
  1272. sessionPassword: <Buffer 9d 66 31 8e a3 60 dc dd d1 54 7c 80 66 35 ca c3>,
  1273. credentials: [],
  1274. zxid: <Buffer 00 00 00 00 00 00 00 00>,
  1275. pendingBuffer: null,
  1276. packetQueue: [Object],
  1277. pendingQueue: [],
  1278. socket: [Object] },
  1279. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1280. state: { name: 'SYNC_CONNECTED', code: 3 },
  1281. closeCalled: false } }
  1282. { path: '/net/blackshard/sol/sol/moray/1',
  1283. err: null,
  1284. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  1285. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"req_id":40150,"client":"10.43.1.11","port":"59110/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":22,"timers":{"zk.getNameRecord":12,"zk.loadService":5,"pre-resp":0,"log-after":5},"msg":"DNS query","time":"2016-03-17T15:06:24.183Z","v":0}
  1286. { cache:
  1287. { _max: 10000,
  1288. _lengthCalculator: [Function: naiveLength],
  1289. _allowStale: false,
  1290. _maxAge: 60000,
  1291. _dispose: undefined,
  1292. _cache:
  1293. { '/net/blackshard/sol/sol/moray/1': [Object],
  1294. '/net/blackshard/sol/sol/moray/1__service': [Object] },
  1295. _lruList: { '0': [Object], '1': [Object] },
  1296. _mru: 2,
  1297. _lru: 0,
  1298. _length: 2,
  1299. _itemCount: 2 },
  1300. path: '/net/blackshard/sol/net/blackshard/sol/sol/manta',
  1301. zk:
  1302. { domain: null,
  1303. _events:
  1304. { state: [Function: defaultStateListener],
  1305. error: [Function],
  1306. disconnected: [Object] },
  1307. _maxListeners: undefined,
  1308. connectionManager:
  1309. { domain: null,
  1310. _events: [Object],
  1311. _maxListeners: undefined,
  1312. watcherManager: [Object],
  1313. connectionStringParser: [Object],
  1314. servers: [Object],
  1315. chrootPath: undefined,
  1316. nextServerIndex: 1,
  1317. serverAttempts: 0,
  1318. state: 2,
  1319. options: [Object],
  1320. spinDelay: 1000,
  1321. sessionTimeout: 30000,
  1322. connectTimeout: 30000,
  1323. pingTimeout: 10000,
  1324. connectTimeoutHandler: [Object],
  1325. xid: 4,
  1326. sessionId: <Buffer 01 53 82 40 72 3f 00 3d>,
  1327. sessionPassword: <Buffer 9d 66 31 8e a3 60 dc dd d1 54 7c 80 66 35 ca c3>,
  1328. credentials: [],
  1329. zxid: <Buffer 00 00 00 00 00 00 03 9f>,
  1330. pendingBuffer: null,
  1331. packetQueue: [Object],
  1332. pendingQueue: [],
  1333. socket: [Object] },
  1334. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1335. state: { name: 'SYNC_CONNECTED', code: 3 },
  1336. closeCalled: false } }
  1337. { path: '/net/blackshard/sol/net/blackshard/sol/sol/manta',
  1338. err:
  1339. { [NO_NODE: Exception: NO_NODE[-101]]
  1340. code: -101,
  1341. name: 'NO_NODE',
  1342. path: undefined,
  1343. message: 'Exception: NO_NODE[-101]' },
  1344. data: undefined }
  1345. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"req_id":61385,"client":"10.43.1.5","port":"36329/udp","query":{"name":"manta.sol.sol.blackshard.net...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":6,"timers":{"zk.getNameRecord":5,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:24.196Z","v":0}
  1346. { cache:
  1347. { _max: 10000,
  1348. _lengthCalculator: [Function: naiveLength],
  1349. _allowStale: false,
  1350. _maxAge: 60000,
  1351. _dispose: undefined,
  1352. _cache:
  1353. { '/net/blackshard/sol/sol/moray/1': [Object],
  1354. '/net/blackshard/sol/sol/moray/1__service': [Object],
  1355. '/net/blackshard/sol/net/blackshard/sol/sol/manta': [Object] },
  1356. _lruList: { '0': [Object], '1': [Object], '2': [Object] },
  1357. _mru: 3,
  1358. _lru: 0,
  1359. _length: 3,
  1360. _itemCount: 3 },
  1361. path: '/net/blackshard/sol/net/blackshard/sol/sol/manta',
  1362. zk:
  1363. { domain: null,
  1364. _events:
  1365. { state: [Function: defaultStateListener],
  1366. error: [Function],
  1367. disconnected: [Object] },
  1368. _maxListeners: undefined,
  1369. connectionManager:
  1370. { domain: null,
  1371. _events: [Object],
  1372. _maxListeners: undefined,
  1373. watcherManager: [Object],
  1374. connectionStringParser: [Object],
  1375. servers: [Object],
  1376. chrootPath: undefined,
  1377. nextServerIndex: 1,
  1378. serverAttempts: 0,
  1379. state: 2,
  1380. options: [Object],
  1381. spinDelay: 1000,
  1382. sessionTimeout: 30000,
  1383. connectTimeout: 30000,
  1384. pingTimeout: 10000,
  1385. connectTimeoutHandler: [Object],
  1386. xid: 5,
  1387. sessionId: <Buffer 01 53 82 40 72 3f 00 3d>,
  1388. sessionPassword: <Buffer 9d 66 31 8e a3 60 dc dd d1 54 7c 80 66 35 ca c3>,
  1389. credentials: [],
  1390. zxid: <Buffer 00 00 00 00 00 00 03 9f>,
  1391. pendingBuffer: null,
  1392. packetQueue: [Object],
  1393. pendingQueue: [],
  1394. socket: [Object] },
  1395. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1396. state: { name: 'SYNC_CONNECTED', code: 3 },
  1397. closeCalled: false } }
  1398. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"req_id":61385,"client":"10.43.1.5","port":"40296/udp","query":{"name":"manta.sol.sol.blackshard.net...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":3,"timers":{"zk.getNameRecord":3,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:24.209Z","v":0}
  1399. { cache:
  1400. { _max: 10000,
  1401. _lengthCalculator: [Function: naiveLength],
  1402. _allowStale: false,
  1403. _maxAge: 60000,
  1404. _dispose: undefined,
  1405. _cache:
  1406. { '/net/blackshard/sol/sol/moray/1': [Object],
  1407. '/net/blackshard/sol/sol/moray/1__service': [Object],
  1408. '/net/blackshard/sol/net/blackshard/sol/sol/manta': [Object] },
  1409. _lruList: { '0': [Object], '1': [Object], '3': [Object] },
  1410. _mru: 4,
  1411. _lru: 0,
  1412. _length: 3,
  1413. _itemCount: 3 },
  1414. path: '/net/blackshard/sol/net/blackshard/sol/sol/manta',
  1415. zk:
  1416. { domain: null,
  1417. _events:
  1418. { state: [Function: defaultStateListener],
  1419. error: [Function],
  1420. disconnected: [Object] },
  1421. _maxListeners: undefined,
  1422. connectionManager:
  1423. { domain: null,
  1424. _events: [Object],
  1425. _maxListeners: undefined,
  1426. watcherManager: [Object],
  1427. connectionStringParser: [Object],
  1428. servers: [Object],
  1429. chrootPath: undefined,
  1430. nextServerIndex: 1,
  1431. serverAttempts: 0,
  1432. state: 2,
  1433. options: [Object],
  1434. spinDelay: 1000,
  1435. sessionTimeout: 30000,
  1436. connectTimeout: 30000,
  1437. pingTimeout: 10000,
  1438. connectTimeoutHandler: [Object],
  1439. xid: 5,
  1440. sessionId: <Buffer 01 53 82 40 72 3f 00 3d>,
  1441. sessionPassword: <Buffer 9d 66 31 8e a3 60 dc dd d1 54 7c 80 66 35 ca c3>,
  1442. credentials: [],
  1443. zxid: <Buffer 00 00 00 00 00 00 03 9f>,
  1444. pendingBuffer: null,
  1445. packetQueue: [Object],
  1446. pendingQueue: [],
  1447. socket: [Object] },
  1448. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1449. state: { name: 'SYNC_CONNECTED', code: 3 },
  1450. closeCalled: false } }
  1451. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"req_id":52362,"client":"10.43.1.9","port":"62489/udp","query":{"name":"manta.sol.sol.blackshard.net...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":2,"timers":{"zk.getNameRecord":2,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:24.218Z","v":0}
  1452. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"req_id":38534,"client":"10.43.1.5","port":"51017/udp","query":{"name":"manta.sol.sol.blackshard.net.blackshard.net","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:24.223Z","v":0}
  1453. { cache:
  1454. { _max: 10000,
  1455. _lengthCalculator: [Function: naiveLength],
  1456. _allowStale: false,
  1457. _maxAge: 60000,
  1458. _dispose: undefined,
  1459. _cache:
  1460. { '/net/blackshard/sol/sol/moray/1': [Object],
  1461. '/net/blackshard/sol/sol/moray/1__service': [Object],
  1462. '/net/blackshard/sol/net/blackshard/sol/sol/manta': [Object] },
  1463. _lruList: { '0': [Object], '1': [Object], '4': [Object] },
  1464. _mru: 5,
  1465. _lru: 0,
  1466. _length: 3,
  1467. _itemCount: 3 },
  1468. path: '/net/blackshard/sol/net/blackshard/sol/sol/manta',
  1469. zk:
  1470. { domain: null,
  1471. _events:
  1472. { state: [Function: defaultStateListener],
  1473. error: [Function],
  1474. disconnected: [Object] },
  1475. _maxListeners: undefined,
  1476. connectionManager:
  1477. { domain: null,
  1478. _events: [Object],
  1479. _maxListeners: undefined,
  1480. watcherManager: [Object],
  1481. connectionStringParser: [Object],
  1482. servers: [Object],
  1483. chrootPath: undefined,
  1484. nextServerIndex: 1,
  1485. serverAttempts: 0,
  1486. state: 2,
  1487. options: [Object],
  1488. spinDelay: 1000,
  1489. sessionTimeout: 30000,
  1490. connectTimeout: 30000,
  1491. pingTimeout: 10000,
  1492. connectTimeoutHandler: [Object],
  1493. xid: 5,
  1494. sessionId: <Buffer 01 53 82 40 72 3f 00 3d>,
  1495. sessionPassword: <Buffer 9d 66 31 8e a3 60 dc dd d1 54 7c 80 66 35 ca c3>,
  1496. credentials: [],
  1497. zxid: <Buffer 00 00 00 00 00 00 03 9f>,
  1498. pendingBuffer: null,
  1499. packetQueue: [Object],
  1500. pendingQueue: [],
  1501. socket: [Object] },
  1502. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1503. state: { name: 'SYNC_CONNECTED', code: 3 },
  1504. closeCalled: false } }
  1505. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"req_id":52362,"client":"10.43.1.9","port":"62309/udp","query":{"name":"manta.sol.sol.blackshard.net...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":3,"timers":{"zk.getNameRecord":3,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:24.251Z","v":0}
  1506. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"req_id":38534,"client":"10.43.1.5","port":"47108/udp","query":{"name":"manta.sol.sol.blackshard.net.blackshard.net","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:24.252Z","v":0}
  1507. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"req_id":55630,"client":"10.43.1.9","port":"43033/udp","query":{"name":"manta.sol.sol.blackshard.net.blackshard.net","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:24.257Z","v":0}
  1508. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93945,"req_id":55630,"client":"10.43.1.9","port":"58577/udp","query":{"name":"manta.sol.sol.blackshard.net.blackshard.net","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:24.265Z","v":0}
  1509. { cache:
  1510. { _max: 10000,
  1511. _lengthCalculator: [Function: naiveLength],
  1512. _allowStale: false,
  1513. _maxAge: 60000,
  1514. _dispose: undefined,
  1515. _cache:
  1516. { '/net/blackshard/sol/sol/moray/1': [Object],
  1517. '/net/blackshard/sol/sol/moray/1__service': [Object],
  1518. '/net/blackshard/sol/net/blackshard/sol/sol/manta': [Object] },
  1519. _lruList: { '0': [Object], '1': [Object], '5': [Object] },
  1520. _mru: 6,
  1521. _lru: 0,
  1522. _length: 3,
  1523. _itemCount: 3 },
  1524. path: '/net/blackshard/sol/sol/manta',
  1525. zk:
  1526. { domain: null,
  1527. _events:
  1528. { state: [Function: defaultStateListener],
  1529. error: [Function],
  1530. disconnected: [Object] },
  1531. _maxListeners: undefined,
  1532. connectionManager:
  1533. { domain: null,
  1534. _events: [Object],
  1535. _maxListeners: undefined,
  1536. watcherManager: [Object],
  1537. connectionStringParser: [Object],
  1538. servers: [Object],
  1539. chrootPath: undefined,
  1540. nextServerIndex: 1,
  1541. serverAttempts: 0,
  1542. state: 2,
  1543. options: [Object],
  1544. spinDelay: 1000,
  1545. sessionTimeout: 30000,
  1546. connectTimeout: 30000,
  1547. pingTimeout: 10000,
  1548. connectTimeoutHandler: [Object],
  1549. xid: 5,
  1550. sessionId: <Buffer 01 53 82 40 72 3f 00 3d>,
  1551. sessionPassword: <Buffer 9d 66 31 8e a3 60 dc dd d1 54 7c 80 66 35 ca c3>,
  1552. credentials: [],
  1553. zxid: <Buffer 00 00 00 00 00 00 03 9f>,
  1554. pendingBuffer: null,
  1555. packetQueue: [Object],
  1556. pendingQueue: [],
  1557. socket: [Object] },
  1558. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1559. state: { name: 'SYNC_CONNECTED', code: 3 },
  1560. closeCalled: false } }
  1561. { path: '/net/blackshard/sol/sol/manta',
  1562. err: null,
  1563. data: undefined }
  1564. Uncaught TypeError: Cannot read property 'toString' of undefined
  1565.  
  1566. FROM
  1567. /opt/smartdc/binder/lib/zk.js:135:42
  1568. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  1569. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  1570. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  1571. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  1572. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  1573. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  1574. Socket.emit (events.js:107:17)
  1575. readableAddChunk (_stream_readable.js:163:16)
  1576. Socket.Readable.push (_stream_readable.js:126:10)
  1577. TCP.onread (net.js:540:20)
  1578. [ Mar 17 15:06:24 Stopping because process dumped core. ]
  1579. [ Mar 17 15:06:24 Executing stop method (:kill). ]
  1580. [ Mar 17 15:06:24 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  1581. [ Mar 17 15:06:24 Method "start" exited with status 0. ]
  1582. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93960,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:06:24.846Z","v":0}
  1583. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93960,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:06:24.883Z","v":0}
  1584. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93960,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:06:24.883Z","v":0}
  1585. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93960,"level":30,"msg":"done with binder init","time":"2016-03-17T15:06:24.883Z","v":0}
  1586. { cache:
  1587. { _max: 10000,
  1588. _lengthCalculator: [Function: naiveLength],
  1589. _allowStale: false,
  1590. _maxAge: 60000,
  1591. _dispose: undefined,
  1592. _cache: {},
  1593. _lruList: {},
  1594. _mru: 0,
  1595. _lru: 0,
  1596. _length: 0,
  1597. _itemCount: 0 },
  1598. path: '/net/blackshard/sol/sol/sapi',
  1599. zk:
  1600. { domain: null,
  1601. _events:
  1602. { state: [Function: defaultStateListener],
  1603. error: [Function],
  1604. disconnected: [Object] },
  1605. _maxListeners: undefined,
  1606. connectionManager:
  1607. { domain: null,
  1608. _events: [Object],
  1609. _maxListeners: undefined,
  1610. watcherManager: [Object],
  1611. connectionStringParser: [Object],
  1612. servers: [Object],
  1613. chrootPath: undefined,
  1614. nextServerIndex: 1,
  1615. serverAttempts: 0,
  1616. state: 2,
  1617. options: [Object],
  1618. spinDelay: 1000,
  1619. sessionTimeout: 30000,
  1620. connectTimeout: 30000,
  1621. pingTimeout: 10000,
  1622. connectTimeoutHandler: [Object],
  1623. xid: 0,
  1624. sessionId: <Buffer 01 53 82 40 72 3f 00 3e>,
  1625. sessionPassword: <Buffer 63 03 43 8e c3 7d 3e 0f f1 8b 46 fe eb 9a 8c 5c>,
  1626. credentials: [],
  1627. zxid: <Buffer 00 00 00 00 00 00 00 00>,
  1628. pendingBuffer: null,
  1629. packetQueue: [Object],
  1630. pendingQueue: [],
  1631. socket: [Object] },
  1632. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1633. state: { name: 'SYNC_CONNECTED', code: 3 },
  1634. closeCalled: false } }
  1635. { path: '/net/blackshard/sol/sol/sapi',
  1636. err:
  1637. { [NO_NODE: Exception: NO_NODE[-101]]
  1638. code: -101,
  1639. name: 'NO_NODE',
  1640. path: undefined,
  1641. message: 'Exception: NO_NODE[-101]' },
  1642. data: undefined }
  1643. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93960,"req_id":63256,"client":"10.43.1.11","port":"51753/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":13,"timers":{"zk.getNameRecord":10,"pre-resp":1,"log-after":2},"msg":"DNS query","time":"2016-03-17T15:06:24.965Z","v":0}
  1644. { cache:
  1645. { _max: 10000,
  1646. _lengthCalculator: [Function: naiveLength],
  1647. _allowStale: false,
  1648. _maxAge: 60000,
  1649. _dispose: undefined,
  1650. _cache: { '/net/blackshard/sol/sol/sapi': [Object] },
  1651. _lruList: { '0': [Object] },
  1652. _mru: 1,
  1653. _lru: 0,
  1654. _length: 1,
  1655. _itemCount: 1 },
  1656. path: '/net/blackshard/sol/sol/electric-moray',
  1657. zk:
  1658. { domain: null,
  1659. _events:
  1660. { state: [Function: defaultStateListener],
  1661. error: [Function],
  1662. disconnected: [Object] },
  1663. _maxListeners: undefined,
  1664. connectionManager:
  1665. { domain: null,
  1666. _events: [Object],
  1667. _maxListeners: undefined,
  1668. watcherManager: [Object],
  1669. connectionStringParser: [Object],
  1670. servers: [Object],
  1671. chrootPath: undefined,
  1672. nextServerIndex: 1,
  1673. serverAttempts: 0,
  1674. state: 2,
  1675. options: [Object],
  1676. spinDelay: 1000,
  1677. sessionTimeout: 30000,
  1678. connectTimeout: 30000,
  1679. pingTimeout: 10000,
  1680. connectTimeoutHandler: [Object],
  1681. xid: 1,
  1682. sessionId: <Buffer 01 53 82 40 72 3f 00 3e>,
  1683. sessionPassword: <Buffer 63 03 43 8e c3 7d 3e 0f f1 8b 46 fe eb 9a 8c 5c>,
  1684. credentials: [],
  1685. zxid: <Buffer 00 00 00 00 00 00 03 a0>,
  1686. pendingBuffer: null,
  1687. packetQueue: [Object],
  1688. pendingQueue: [],
  1689. socket: [Object] },
  1690. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1691. state: { name: 'SYNC_CONNECTED', code: 3 },
  1692. closeCalled: false } }
  1693. { path: '/net/blackshard/sol/sol/electric-moray',
  1694. err: null,
  1695. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  1696. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93960,"req_id":9632,"client":"10.43.1.14","port":"33040/udp","query":{"name":"electric-moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.7"],"additional":[],"latency":10,"timers":{"zk.getNameRecord":6,"zk.loadService":2,"pre-resp":1,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:24.991Z","v":0}
  1697. { cache:
  1698. { _max: 10000,
  1699. _lengthCalculator: [Function: naiveLength],
  1700. _allowStale: false,
  1701. _maxAge: 60000,
  1702. _dispose: undefined,
  1703. _cache:
  1704. { '/net/blackshard/sol/sol/sapi': [Object],
  1705. '/net/blackshard/sol/sol/electric-moray': [Object],
  1706. '/net/blackshard/sol/sol/electric-moray__service': [Object] },
  1707. _lruList: { '0': [Object], '1': [Object], '2': [Object] },
  1708. _mru: 3,
  1709. _lru: 0,
  1710. _length: 3,
  1711. _itemCount: 3 },
  1712. path: '/net/blackshard/sol/sol/moray/1',
  1713. zk:
  1714. { domain: null,
  1715. _events:
  1716. { state: [Function: defaultStateListener],
  1717. error: [Function],
  1718. disconnected: [Object] },
  1719. _maxListeners: undefined,
  1720. connectionManager:
  1721. { domain: null,
  1722. _events: [Object],
  1723. _maxListeners: undefined,
  1724. watcherManager: [Object],
  1725. connectionStringParser: [Object],
  1726. servers: [Object],
  1727. chrootPath: undefined,
  1728. nextServerIndex: 1,
  1729. serverAttempts: 0,
  1730. state: 2,
  1731. options: [Object],
  1732. spinDelay: 1000,
  1733. sessionTimeout: 30000,
  1734. connectTimeout: 30000,
  1735. pingTimeout: 10000,
  1736. connectTimeoutHandler: [Object],
  1737. xid: 4,
  1738. sessionId: <Buffer 01 53 82 40 72 3f 00 3e>,
  1739. sessionPassword: <Buffer 63 03 43 8e c3 7d 3e 0f f1 8b 46 fe eb 9a 8c 5c>,
  1740. credentials: [],
  1741. zxid: <Buffer 00 00 00 00 00 00 03 a0>,
  1742. pendingBuffer: null,
  1743. packetQueue: [Object],
  1744. pendingQueue: [],
  1745. socket: [Object] },
  1746. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1747. state: { name: 'SYNC_CONNECTED', code: 3 },
  1748. closeCalled: false } }
  1749. { cache:
  1750. { _max: 10000,
  1751. _lengthCalculator: [Function: naiveLength],
  1752. _allowStale: false,
  1753. _maxAge: 60000,
  1754. _dispose: undefined,
  1755. _cache:
  1756. { '/net/blackshard/sol/sol/sapi': [Object],
  1757. '/net/blackshard/sol/sol/electric-moray': [Object],
  1758. '/net/blackshard/sol/sol/electric-moray__service': [Object] },
  1759. _lruList: { '0': [Object], '1': [Object], '2': [Object] },
  1760. _mru: 3,
  1761. _lru: 0,
  1762. _length: 3,
  1763. _itemCount: 3 },
  1764. path: '/net/blackshard/sol/sol/electric-moray',
  1765. zk:
  1766. { domain: null,
  1767. _events:
  1768. { state: [Function: defaultStateListener],
  1769. error: [Function],
  1770. disconnected: [Object] },
  1771. _maxListeners: undefined,
  1772. connectionManager:
  1773. { domain: null,
  1774. _events: [Object],
  1775. _maxListeners: undefined,
  1776. watcherManager: [Object],
  1777. connectionStringParser: [Object],
  1778. servers: [Object],
  1779. chrootPath: undefined,
  1780. nextServerIndex: 1,
  1781. serverAttempts: 0,
  1782. state: 2,
  1783. options: [Object],
  1784. spinDelay: 1000,
  1785. sessionTimeout: 30000,
  1786. connectTimeout: 30000,
  1787. pingTimeout: 10000,
  1788. connectTimeoutHandler: [Object],
  1789. xid: 5,
  1790. sessionId: <Buffer 01 53 82 40 72 3f 00 3e>,
  1791. sessionPassword: <Buffer 63 03 43 8e c3 7d 3e 0f f1 8b 46 fe eb 9a 8c 5c>,
  1792. credentials: [],
  1793. zxid: <Buffer 00 00 00 00 00 00 03 a0>,
  1794. pendingBuffer: null,
  1795. packetQueue: [Object],
  1796. pendingQueue: [Object],
  1797. socket: [Object] },
  1798. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1799. state: { name: 'SYNC_CONNECTED', code: 3 },
  1800. closeCalled: false } }
  1801. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93960,"req_id":18569,"client":"10.43.1.11","port":"49375/udp","query":{"name":"electric-moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.7"],"additional":[],"latency":2,"timers":{"zk.getNameRecord":1,"zk.loadService":0,"pre-resp":1,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:25.198Z","v":0}
  1802. { path: '/net/blackshard/sol/sol/moray/1',
  1803. err: null,
  1804. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  1805. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93960,"req_id":19468,"client":"10.43.1.11","port":"49375/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":6,"timers":{"zk.getNameRecord":5,"zk.loadService":1,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:25.201Z","v":0}
  1806. { cache:
  1807. { _max: 10000,
  1808. _lengthCalculator: [Function: naiveLength],
  1809. _allowStale: false,
  1810. _maxAge: 60000,
  1811. _dispose: undefined,
  1812. _cache:
  1813. { '/net/blackshard/sol/sol/sapi': [Object],
  1814. '/net/blackshard/sol/sol/electric-moray': [Object],
  1815. '/net/blackshard/sol/sol/electric-moray__service': [Object],
  1816. '/net/blackshard/sol/sol/moray/1': [Object],
  1817. '/net/blackshard/sol/sol/moray/1__service': [Object] },
  1818. _lruList:
  1819. { '0': [Object],
  1820. '3': [Object],
  1821. '4': [Object],
  1822. '5': [Object],
  1823. '6': [Object] },
  1824. _mru: 7,
  1825. _lru: 0,
  1826. _length: 5,
  1827. _itemCount: 5 },
  1828. path: '/net/blackshard/sol/sol/manta',
  1829. zk:
  1830. { domain: null,
  1831. _events:
  1832. { state: [Function: defaultStateListener],
  1833. error: [Function],
  1834. disconnected: [Object] },
  1835. _maxListeners: undefined,
  1836. connectionManager:
  1837. { domain: null,
  1838. _events: [Object],
  1839. _maxListeners: undefined,
  1840. watcherManager: [Object],
  1841. connectionStringParser: [Object],
  1842. servers: [Object],
  1843. chrootPath: undefined,
  1844. nextServerIndex: 1,
  1845. serverAttempts: 0,
  1846. state: 2,
  1847. options: [Object],
  1848. spinDelay: 1000,
  1849. sessionTimeout: 30000,
  1850. connectTimeout: 30000,
  1851. pingTimeout: 10000,
  1852. connectTimeoutHandler: [Object],
  1853. xid: 8,
  1854. sessionId: <Buffer 01 53 82 40 72 3f 00 3e>,
  1855. sessionPassword: <Buffer 63 03 43 8e c3 7d 3e 0f f1 8b 46 fe eb 9a 8c 5c>,
  1856. credentials: [],
  1857. zxid: <Buffer 00 00 00 00 00 00 03 a0>,
  1858. pendingBuffer: null,
  1859. packetQueue: [Object],
  1860. pendingQueue: [],
  1861. socket: [Object] },
  1862. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1863. state: { name: 'SYNC_CONNECTED', code: 3 },
  1864. closeCalled: false } }
  1865. { path: '/net/blackshard/sol/sol/manta',
  1866. err: null,
  1867. data: undefined }
  1868. Uncaught TypeError: Cannot read property 'toString' of undefined
  1869.  
  1870. FROM
  1871. /opt/smartdc/binder/lib/zk.js:135:42
  1872. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  1873. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  1874. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  1875. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  1876. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  1877. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  1878. Socket.emit (events.js:107:17)
  1879. readableAddChunk (_stream_readable.js:163:16)
  1880. Socket.Readable.push (_stream_readable.js:126:10)
  1881. TCP.onread (net.js:540:20)
  1882. [ Mar 17 15:06:25 Stopping because process dumped core. ]
  1883. [ Mar 17 15:06:25 Executing stop method (:kill). ]
  1884. [ Mar 17 15:06:25 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  1885. [ Mar 17 15:06:25 Method "start" exited with status 0. ]
  1886. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:06:25.816Z","v":0}
  1887. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:06:25.876Z","v":0}
  1888. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:06:25.877Z","v":0}
  1889. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"level":30,"msg":"done with binder init","time":"2016-03-17T15:06:25.877Z","v":0}
  1890. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"req_id":54208,"client":"10.43.1.10","port":"52010/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":3,"timers":{"log-after":3},"msg":"DNS query","time":"2016-03-17T15:06:25.883Z","v":0}
  1891. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"req_id":54208,"client":"10.43.1.10","port":"61070/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:25.891Z","v":0}
  1892. { cache:
  1893. { _max: 10000,
  1894. _lengthCalculator: [Function: naiveLength],
  1895. _allowStale: false,
  1896. _maxAge: 60000,
  1897. _dispose: undefined,
  1898. _cache: {},
  1899. _lruList: {},
  1900. _mru: 0,
  1901. _lru: 0,
  1902. _length: 0,
  1903. _itemCount: 0 },
  1904. path: '/net/blackshard/sol/sol/sapi',
  1905. zk:
  1906. { domain: null,
  1907. _events:
  1908. { state: [Function: defaultStateListener],
  1909. error: [Function],
  1910. disconnected: [Object] },
  1911. _maxListeners: undefined,
  1912. connectionManager:
  1913. { domain: null,
  1914. _events: [Object],
  1915. _maxListeners: undefined,
  1916. watcherManager: [Object],
  1917. connectionStringParser: [Object],
  1918. servers: [Object],
  1919. chrootPath: undefined,
  1920. nextServerIndex: 1,
  1921. serverAttempts: 0,
  1922. state: 2,
  1923. options: [Object],
  1924. spinDelay: 1000,
  1925. sessionTimeout: 30000,
  1926. connectTimeout: 30000,
  1927. pingTimeout: 10000,
  1928. connectTimeoutHandler: [Object],
  1929. xid: 0,
  1930. sessionId: <Buffer 01 53 82 40 72 3f 00 3f>,
  1931. sessionPassword: <Buffer 77 24 3d 8e 63 c9 1d 54 e6 ce ad 29 bf 23 a1 29>,
  1932. credentials: [],
  1933. zxid: <Buffer 00 00 00 00 00 00 00 00>,
  1934. pendingBuffer: null,
  1935. packetQueue: [Object],
  1936. pendingQueue: [],
  1937. socket: [Object] },
  1938. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  1939. state: { name: 'SYNC_CONNECTED', code: 3 },
  1940. closeCalled: false } }
  1941. { path: '/net/blackshard/sol/sol/sapi',
  1942. err:
  1943. { [NO_NODE: Exception: NO_NODE[-101]]
  1944. code: -101,
  1945. name: 'NO_NODE',
  1946. path: undefined,
  1947. message: 'Exception: NO_NODE[-101]' },
  1948. data: undefined }
  1949. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"req_id":11053,"client":"10.43.1.10","port":"50893/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":9,"timers":{"zk.getNameRecord":9,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:25.904Z","v":0}
  1950. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"req_id":43593,"client":"10.43.1.7","port":"44671/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:06:26.233Z","v":0}
  1951. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"req_id":43593,"client":"10.43.1.7","port":"53389/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:26.236Z","v":0}
  1952. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"req_id":2650,"client":"10.43.1.7","port":"63127/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:26.239Z","v":0}
  1953. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"req_id":2650,"client":"10.43.1.7","port":"38075/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:26.243Z","v":0}
  1954. { cache:
  1955. { _max: 10000,
  1956. _lengthCalculator: [Function: naiveLength],
  1957. _allowStale: false,
  1958. _maxAge: 60000,
  1959. _dispose: undefined,
  1960. _cache: { '/net/blackshard/sol/sol/sapi': [Object] },
  1961. _lruList: { '0': [Object] },
  1962. _mru: 1,
  1963. _lru: 0,
  1964. _length: 1,
  1965. _itemCount: 1 },
  1966. path: '/net/blackshard/sol/sol/sapi',
  1967. zk:
  1968. { domain: null,
  1969. _events:
  1970. { state: [Function: defaultStateListener],
  1971. error: [Function],
  1972. disconnected: [Object] },
  1973. _maxListeners: undefined,
  1974. connectionManager:
  1975. { domain: null,
  1976. _events: [Object],
  1977. _maxListeners: undefined,
  1978. watcherManager: [Object],
  1979. connectionStringParser: [Object],
  1980. servers: [Object],
  1981. chrootPath: undefined,
  1982. nextServerIndex: 1,
  1983. serverAttempts: 0,
  1984. state: 2,
  1985. options: [Object],
  1986. spinDelay: 1000,
  1987. sessionTimeout: 30000,
  1988. connectTimeout: 30000,
  1989. pingTimeout: 10000,
  1990. connectTimeoutHandler: [Object],
  1991. xid: 1,
  1992. sessionId: <Buffer 01 53 82 40 72 3f 00 3f>,
  1993. sessionPassword: <Buffer 77 24 3d 8e 63 c9 1d 54 e6 ce ad 29 bf 23 a1 29>,
  1994. credentials: [],
  1995. zxid: <Buffer 00 00 00 00 00 00 03 a1>,
  1996. pendingBuffer: null,
  1997. packetQueue: [Object],
  1998. pendingQueue: [],
  1999. socket: [Object] },
  2000. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  2001. state: { name: 'SYNC_CONNECTED', code: 3 },
  2002. closeCalled: false } }
  2003. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93974,"req_id":22240,"client":"10.43.1.7","port":"45764/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":3,"timers":{"zk.getNameRecord":3,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:06:26.249Z","v":0}
  2004. { cache:
  2005. { _max: 10000,
  2006. _lengthCalculator: [Function: naiveLength],
  2007. _allowStale: false,
  2008. _maxAge: 60000,
  2009. _dispose: undefined,
  2010. _cache: { '/net/blackshard/sol/sol/sapi': [Object] },
  2011. _lruList: { '1': [Object] },
  2012. _mru: 2,
  2013. _lru: 1,
  2014. _length: 1,
  2015. _itemCount: 1 },
  2016. path: '/net/blackshard/sol/sol/manta',
  2017. zk:
  2018. { domain: null,
  2019. _events:
  2020. { state: [Function: defaultStateListener],
  2021. error: [Function],
  2022. disconnected: [Object] },
  2023. _maxListeners: undefined,
  2024. connectionManager:
  2025. { domain: null,
  2026. _events: [Object],
  2027. _maxListeners: undefined,
  2028. watcherManager: [Object],
  2029. connectionStringParser: [Object],
  2030. servers: [Object],
  2031. chrootPath: undefined,
  2032. nextServerIndex: 1,
  2033. serverAttempts: 0,
  2034. state: 2,
  2035. options: [Object],
  2036. spinDelay: 1000,
  2037. sessionTimeout: 30000,
  2038. connectTimeout: 30000,
  2039. pingTimeout: 10000,
  2040. connectTimeoutHandler: [Object],
  2041. xid: 1,
  2042. sessionId: <Buffer 01 53 82 40 72 3f 00 3f>,
  2043. sessionPassword: <Buffer 77 24 3d 8e 63 c9 1d 54 e6 ce ad 29 bf 23 a1 29>,
  2044. credentials: [],
  2045. zxid: <Buffer 00 00 00 00 00 00 03 a1>,
  2046. pendingBuffer: null,
  2047. packetQueue: [Object],
  2048. pendingQueue: [],
  2049. socket: [Object] },
  2050. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  2051. state: { name: 'SYNC_CONNECTED', code: 3 },
  2052. closeCalled: false } }
  2053. { path: '/net/blackshard/sol/sol/manta',
  2054. err: null,
  2055. data: undefined }
  2056. Uncaught TypeError: Cannot read property 'toString' of undefined
  2057.  
  2058. FROM
  2059. /opt/smartdc/binder/lib/zk.js:135:42
  2060. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2061. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2062. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2063. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2064. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2065. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2066. Socket.emit (events.js:107:17)
  2067. readableAddChunk (_stream_readable.js:163:16)
  2068. Socket.Readable.push (_stream_readable.js:126:10)
  2069. TCP.onread (net.js:540:20)
  2070. [ Mar 17 15:06:26 Stopping because process dumped core. ]
  2071. [ Mar 17 15:06:26 Executing stop method (:kill). ]
  2072. [ Mar 17 15:06:26 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  2073. [ Mar 17 15:06:26 Method "start" exited with status 0. ]
  2074. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93988,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:06:27.149Z","v":0}
  2075. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93988,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:06:27.187Z","v":0}
  2076. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93988,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:06:27.188Z","v":0}
  2077. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":93988,"level":30,"msg":"done with binder init","time":"2016-03-17T15:06:27.188Z","v":0}
  2078. { cache:
  2079. { _max: 10000,
  2080. _lengthCalculator: [Function: naiveLength],
  2081. _allowStale: false,
  2082. _maxAge: 60000,
  2083. _dispose: undefined,
  2084. _cache: {},
  2085. _lruList: {},
  2086. _mru: 0,
  2087. _lru: 0,
  2088. _length: 0,
  2089. _itemCount: 0 },
  2090. path: '/net/blackshard/sol/sol/manta',
  2091. zk:
  2092. { domain: null,
  2093. _events:
  2094. { state: [Function: defaultStateListener],
  2095. error: [Function],
  2096. disconnected: [Object] },
  2097. _maxListeners: undefined,
  2098. connectionManager:
  2099. { domain: null,
  2100. _events: [Object],
  2101. _maxListeners: undefined,
  2102. watcherManager: [Object],
  2103. connectionStringParser: [Object],
  2104. servers: [Object],
  2105. chrootPath: undefined,
  2106. nextServerIndex: 1,
  2107. serverAttempts: 0,
  2108. state: 2,
  2109. options: [Object],
  2110. spinDelay: 1000,
  2111. sessionTimeout: 30000,
  2112. connectTimeout: 30000,
  2113. pingTimeout: 10000,
  2114. connectTimeoutHandler: [Object],
  2115. xid: 0,
  2116. sessionId: <Buffer 01 53 82 40 72 3f 00 40>,
  2117. sessionPassword: <Buffer 39 ec 63 8d 82 b7 63 48 58 7c 9a 6f 55 ea 98 cd>,
  2118. credentials: [],
  2119. zxid: <Buffer 00 00 00 00 00 00 00 00>,
  2120. pendingBuffer: null,
  2121. packetQueue: [Object],
  2122. pendingQueue: [],
  2123. socket: [Object] },
  2124. options: { spinDelay: 1000, sessionTimeout: 30000, retries: 0 },
  2125. state: { name: 'SYNC_CONNECTED', code: 3 },
  2126. closeCalled: false } }
  2127. { path: '/net/blackshard/sol/sol/manta',
  2128. err: null,
  2129. data: undefined }
  2130. Uncaught TypeError: Cannot read property 'toString' of undefined
  2131.  
  2132. FROM
  2133. /opt/smartdc/binder/lib/zk.js:135:42
  2134. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2135. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2136. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2137. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2138. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2139. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2140. Socket.emit (events.js:107:17)
  2141. readableAddChunk (_stream_readable.js:163:16)
  2142. Socket.Readable.push (_stream_readable.js:126:10)
  2143. TCP.onread (net.js:540:20)
  2144. [ Mar 17 15:06:27 Stopping because process dumped core. ]
  2145. [ Mar 17 15:06:27 Executing stop method (:kill). ]
  2146. [ Mar 17 15:06:27 Restarting too quickly, changing state to maintenance. ]
  2147. [ Mar 17 15:09:20 Leaving maintenance because clear requested. ]
  2148. [ Mar 17 15:09:20 Enabled. ]
  2149. [ Mar 17 15:09:20 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  2150. [ Mar 17 15:09:20 Method "start" exited with status 0. ]
  2151. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95710,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:09:21.053Z","v":0}
  2152. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95710,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:09:21.100Z","v":0}
  2153. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95710,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:09:21.100Z","v":0}
  2154. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95710,"level":30,"msg":"done with binder init","time":"2016-03-17T15:09:21.101Z","v":0}
  2155. { path: '/net/blackshard/sol/sol/moray/1',
  2156. err: null,
  2157. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2158. { path: '/net/blackshard/sol/sol/electric-moray',
  2159. err: null,
  2160. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2161. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95710,"req_id":15271,"client":"10.43.1.11","port":"44128/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":28,"timers":{"zk.getNameRecord":12,"zk.loadService":8,"pre-resp":0,"log-after":8},"msg":"DNS query","time":"2016-03-17T15:09:21.521Z","v":0}
  2162. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95710,"req_id":18851,"client":"10.43.1.11","port":"44128/udp","query":{"name":"electric-moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.7"],"additional":[],"latency":27,"timers":{"zk.getNameRecord":10,"zk.loadService":16,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:21.524Z","v":0}
  2163. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95710,"req_id":14033,"client":"10.43.1.11","port":"44128/udp","query":{"name":"electric-moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.7"],"additional":[],"latency":2,"timers":{"zk.getNameRecord":1,"zk.loadService":0,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:21.527Z","v":0}
  2164. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95710,"req_id":16878,"client":"10.43.1.11","port":"44128/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":0,"timers":{"zk.getNameRecord":0,"zk.loadService":0,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:21.528Z","v":0}
  2165. { path: '/net/blackshard/sol/sol/manta',
  2166. err: null,
  2167. data: undefined }
  2168. Uncaught TypeError: Cannot read property 'toString' of undefined
  2169.  
  2170. FROM
  2171. /opt/smartdc/binder/lib/zk.js:133:42
  2172. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2173. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2174. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2175. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2176. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2177. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2178. Socket.emit (events.js:107:17)
  2179. readableAddChunk (_stream_readable.js:163:16)
  2180. Socket.Readable.push (_stream_readable.js:126:10)
  2181. TCP.onread (net.js:540:20)
  2182. [ Mar 17 15:09:21 Stopping because process dumped core. ]
  2183. [ Mar 17 15:09:21 Executing stop method (:kill). ]
  2184. [ Mar 17 15:09:21 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  2185. [ Mar 17 15:09:21 Method "start" exited with status 0. ]
  2186. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95728,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:09:22.206Z","v":0}
  2187. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95728,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:09:22.261Z","v":0}
  2188. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95728,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:09:22.262Z","v":0}
  2189. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95728,"level":30,"msg":"done with binder init","time":"2016-03-17T15:09:22.262Z","v":0}
  2190. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95728,"req_id":43136,"client":"10.43.1.15","port":"64156/udp","query":{"name":"manta.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":3,"timers":{"log-after":3},"msg":"DNS query","time":"2016-03-17T15:09:22.421Z","v":0}
  2191. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95728,"req_id":43136,"client":"10.43.1.15","port":"63948/udp","query":{"name":"manta.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:22.427Z","v":0}
  2192. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95728,"req_id":40272,"client":"10.43.1.15","port":"59511/udp","query":{"name":"manta.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:22.431Z","v":0}
  2193. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95728,"req_id":40272,"client":"10.43.1.15","port":"45221/udp","query":{"name":"manta.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:22.438Z","v":0}
  2194. { path: '/net/blackshard/sol/sol/manta',
  2195. err: null,
  2196. data: undefined }
  2197. Uncaught TypeError: Cannot read property 'toString' of undefined
  2198.  
  2199. FROM
  2200. /opt/smartdc/binder/lib/zk.js:133:42
  2201. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2202. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2203. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2204. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2205. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2206. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2207. Socket.emit (events.js:107:17)
  2208. readableAddChunk (_stream_readable.js:163:16)
  2209. Socket.Readable.push (_stream_readable.js:126:10)
  2210. TCP.onread (net.js:540:20)
  2211. [ Mar 17 15:09:22 Stopping because process dumped core. ]
  2212. [ Mar 17 15:09:22 Executing stop method (:kill). ]
  2213. [ Mar 17 15:09:22 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  2214. [ Mar 17 15:09:22 Method "start" exited with status 0. ]
  2215. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:09:22.947Z","v":0}
  2216. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:09:23.002Z","v":0}
  2217. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:09:23.003Z","v":0}
  2218. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"level":30,"msg":"done with binder init","time":"2016-03-17T15:09:23.003Z","v":0}
  2219. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"req_id":44678,"client":"10.43.1.12","port":"37644/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":3,"timers":{"log-after":2},"msg":"DNS query","time":"2016-03-17T15:09:23.579Z","v":0}
  2220. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"req_id":44678,"client":"10.43.1.12","port":"42947/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:23.585Z","v":0}
  2221. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"req_id":64029,"client":"10.43.1.12","port":"48328/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:23.588Z","v":0}
  2222. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"req_id":64029,"client":"10.43.1.12","port":"41639/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:23.592Z","v":0}
  2223. { path: '/net/blackshard/sol/sol/sapi',
  2224. err:
  2225. { [NO_NODE: Exception: NO_NODE[-101]]
  2226. code: -101,
  2227. name: 'NO_NODE',
  2228. path: undefined,
  2229. message: 'Exception: NO_NODE[-101]' },
  2230. data: undefined }
  2231. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"req_id":62198,"client":"10.43.1.12","port":"46124/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":8,"timers":{"zk.getNameRecord":7,"pre-resp":1,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:23.603Z","v":0}
  2232. { path: '/net/blackshard/sol/sol/moray/1',
  2233. err: null,
  2234. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2235. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"req_id":8998,"client":"10.43.1.11","port":"46123/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":6,"timers":{"zk.getNameRecord":2,"zk.loadService":3,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:23.716Z","v":0}
  2236. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95732,"req_id":55510,"client":"10.43.1.7","port":"36912/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":1,"timers":{"zk.getNameRecord":0,"zk.loadService":0,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:23.790Z","v":0}
  2237. { path: '/net/blackshard/sol/sol/manta',
  2238. err: null,
  2239. data: undefined }
  2240. Uncaught TypeError: Cannot read property 'toString' of undefined
  2241.  
  2242. FROM
  2243. /opt/smartdc/binder/lib/zk.js:133:42
  2244. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2245. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2246. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2247. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2248. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2249. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2250. Socket.emit (events.js:107:17)
  2251. readableAddChunk (_stream_readable.js:163:16)
  2252. Socket.Readable.push (_stream_readable.js:126:10)
  2253. TCP.onread (net.js:540:20)
  2254. [ Mar 17 15:09:24 Stopping because process dumped core. ]
  2255. [ Mar 17 15:09:24 Executing stop method (:kill). ]
  2256. [ Mar 17 15:09:24 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  2257. [ Mar 17 15:09:24 Method "start" exited with status 0. ]
  2258. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95750,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:09:24.368Z","v":0}
  2259. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95750,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:09:24.427Z","v":0}
  2260. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95750,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:09:24.427Z","v":0}
  2261. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95750,"level":30,"msg":"done with binder init","time":"2016-03-17T15:09:24.427Z","v":0}
  2262. { path: '/net/blackshard/sol/sol/moray/1',
  2263. err: null,
  2264. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2265. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95750,"req_id":20332,"client":"10.43.1.11","port":"44128/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":16,"timers":{"zk.getNameRecord":8,"zk.loadService":4,"pre-resp":1,"log-after":3},"msg":"DNS query","time":"2016-03-17T15:09:24.524Z","v":0}
  2266. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95750,"req_id":3520,"client":"10.43.1.5","port":"42063/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:24.530Z","v":0}
  2267. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95750,"req_id":3520,"client":"10.43.1.5","port":"33182/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:24.535Z","v":0}
  2268. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95750,"req_id":23700,"client":"10.43.1.5","port":"46928/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:24.538Z","v":0}
  2269. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95750,"req_id":23700,"client":"10.43.1.5","port":"38490/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:24.542Z","v":0}
  2270. { path: '/net/blackshard/sol/sol/sapi',
  2271. err:
  2272. { [NO_NODE: Exception: NO_NODE[-101]]
  2273. code: -101,
  2274. name: 'NO_NODE',
  2275. path: undefined,
  2276. message: 'Exception: NO_NODE[-101]' },
  2277. data: undefined }
  2278. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95750,"req_id":34451,"client":"10.43.1.5","port":"57852/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":3,"timers":{"zk.getNameRecord":3,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:24.549Z","v":0}
  2279. { path: '/net/blackshard/sol/sol/manta',
  2280. err: null,
  2281. data: undefined }
  2282. Uncaught TypeError: Cannot read property 'toString' of undefined
  2283.  
  2284. FROM
  2285. /opt/smartdc/binder/lib/zk.js:133:42
  2286. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2287. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2288. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2289. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2290. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2291. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2292. Socket.emit (events.js:107:17)
  2293. readableAddChunk (_stream_readable.js:163:16)
  2294. Socket.Readable.push (_stream_readable.js:126:10)
  2295. TCP.onread (net.js:540:20)
  2296. [ Mar 17 15:09:24 Stopping because process dumped core. ]
  2297. [ Mar 17 15:09:24 Executing stop method (:kill). ]
  2298. [ Mar 17 15:09:25 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  2299. [ Mar 17 15:09:25 Method "start" exited with status 0. ]
  2300. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95766,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:09:25.311Z","v":0}
  2301. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95766,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:09:25.350Z","v":0}
  2302. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95766,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:09:25.350Z","v":0}
  2303. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95766,"level":30,"msg":"done with binder init","time":"2016-03-17T15:09:25.351Z","v":0}
  2304. { path: '/net/blackshard/sol/sol/electric-moray',
  2305. err: null,
  2306. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2307. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95766,"req_id":49616,"client":"10.43.1.11","port":"62434/udp","query":{"name":"electric-moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.7"],"additional":[],"latency":14,"timers":{"zk.getNameRecord":7,"zk.loadService":3,"pre-resp":1,"log-after":3},"msg":"DNS query","time":"2016-03-17T15:09:25.596Z","v":0}
  2308. { path: '/net/blackshard/sol/sol/moray/1',
  2309. err: null,
  2310. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2311. { path: '/net/blackshard/sol/sol/moray/1',
  2312. err: null,
  2313. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2314. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95766,"req_id":64002,"client":"10.43.1.11","port":"59833/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":6,"timers":{"zk.getNameRecord":2,"zk.loadService":3,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:25.877Z","v":0}
  2315. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95766,"req_id":34419,"client":"10.43.1.11","port":"59833/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":6,"timers":{"zk.getNameRecord":2,"zk.loadService":4,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:25.878Z","v":0}
  2316. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95766,"req_id":63355,"client":"10.43.1.10","port":"51280/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:25.889Z","v":0}
  2317. { path: '/net/blackshard/sol/sol/manta',
  2318. err: null,
  2319. data: undefined }
  2320. Uncaught TypeError: Cannot read property 'toString' of undefined
  2321.  
  2322. FROM
  2323. /opt/smartdc/binder/lib/zk.js:133:42
  2324. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2325. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2326. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2327. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2328. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2329. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2330. Socket.emit (events.js:107:17)
  2331. readableAddChunk (_stream_readable.js:163:16)
  2332. Socket.Readable.push (_stream_readable.js:126:10)
  2333. TCP.onread (net.js:540:20)
  2334. [ Mar 17 15:09:26 Stopping because process dumped core. ]
  2335. [ Mar 17 15:09:26 Executing stop method (:kill). ]
  2336. [ Mar 17 15:09:26 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  2337. [ Mar 17 15:09:26 Method "start" exited with status 0. ]
  2338. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:09:26.401Z","v":0}
  2339. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:09:26.442Z","v":0}
  2340. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:09:26.443Z","v":0}
  2341. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"level":30,"msg":"done with binder init","time":"2016-03-17T15:09:26.443Z","v":0}
  2342. { path: '/net/blackshard/sol/sol/moray/1',
  2343. err: null,
  2344. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2345. { path: '/net/blackshard/sol/sol/electric-moray',
  2346. err: null,
  2347. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2348. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"req_id":47815,"client":"10.43.1.12","port":"56617/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":18,"timers":{"zk.getNameRecord":9,"zk.loadService":5,"pre-resp":1,"log-after":3},"msg":"DNS query","time":"2016-03-17T15:09:26.552Z","v":0}
  2349. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"req_id":45659,"client":"10.43.1.12","port":"56617/udp","query":{"name":"electric-moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.7"],"additional":[],"latency":16,"timers":{"zk.getNameRecord":7,"zk.loadService":8,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:26.554Z","v":0}
  2350. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"req_id":38625,"client":"10.43.1.15","port":"43469/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:26.647Z","v":0}
  2351. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"req_id":38625,"client":"10.43.1.15","port":"51443/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:26.651Z","v":0}
  2352. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"req_id":23124,"client":"10.43.1.15","port":"40024/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:26.655Z","v":0}
  2353. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"req_id":23124,"client":"10.43.1.15","port":"38032/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:26.658Z","v":0}
  2354. { path: '/net/blackshard/sol/sol/sapi',
  2355. err:
  2356. { [NO_NODE: Exception: NO_NODE[-101]]
  2357. code: -101,
  2358. name: 'NO_NODE',
  2359. path: undefined,
  2360. message: 'Exception: NO_NODE[-101]' },
  2361. data: undefined }
  2362. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"req_id":31321,"client":"10.43.1.15","port":"36003/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":3,"timers":{"zk.getNameRecord":3,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:26.665Z","v":0}
  2363. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95780,"req_id":30913,"client":"10.43.1.11","port":"44128/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":1,"timers":{"zk.getNameRecord":1,"zk.loadService":0,"pre-resp":0,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:26.675Z","v":0}
  2364. { path: '/net/blackshard/sol/sol/manta',
  2365. err: null,
  2366. data: undefined }
  2367. Uncaught TypeError: Cannot read property 'toString' of undefined
  2368.  
  2369. FROM
  2370. /opt/smartdc/binder/lib/zk.js:133:42
  2371. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2372. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2373. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2374. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2375. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2376. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2377. Socket.emit (events.js:107:17)
  2378. readableAddChunk (_stream_readable.js:163:16)
  2379. Socket.Readable.push (_stream_readable.js:126:10)
  2380. TCP.onread (net.js:540:20)
  2381. [ Mar 17 15:09:27 Stopping because process dumped core. ]
  2382. [ Mar 17 15:09:27 Executing stop method (:kill). ]
  2383. [ Mar 17 15:09:27 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  2384. [ Mar 17 15:09:27 Method "start" exited with status 0. ]
  2385. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95799,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:09:27.474Z","v":0}
  2386. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95799,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:09:27.514Z","v":0}
  2387. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95799,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:09:27.515Z","v":0}
  2388. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95799,"level":30,"msg":"done with binder init","time":"2016-03-17T15:09:27.515Z","v":0}
  2389. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95799,"req_id":56912,"client":"10.43.1.3","port":"56143/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":2,"timers":{"log-after":2},"msg":"DNS query","time":"2016-03-17T15:09:27.808Z","v":0}
  2390. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95799,"req_id":56912,"client":"10.43.1.3","port":"36704/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:27.813Z","v":0}
  2391. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95799,"req_id":64898,"client":"10.43.1.3","port":"42805/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:27.817Z","v":0}
  2392. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95799,"req_id":64898,"client":"10.43.1.3","port":"64327/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:27.820Z","v":0}
  2393. { path: '/net/blackshard/sol/sol/sapi',
  2394. err:
  2395. { [NO_NODE: Exception: NO_NODE[-101]]
  2396. code: -101,
  2397. name: 'NO_NODE',
  2398. path: undefined,
  2399. message: 'Exception: NO_NODE[-101]' },
  2400. data: undefined }
  2401. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95799,"req_id":6671,"client":"10.43.1.3","port":"57615/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":8,"timers":{"zk.getNameRecord":7,"pre-resp":1,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:27.830Z","v":0}
  2402. { path: '/net/blackshard/sol/sol/manta',
  2403. err: null,
  2404. data: undefined }
  2405. Uncaught TypeError: Cannot read property 'toString' of undefined
  2406.  
  2407. FROM
  2408. /opt/smartdc/binder/lib/zk.js:133:42
  2409. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2410. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2411. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2412. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2413. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2414. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2415. Socket.emit (events.js:107:17)
  2416. readableAddChunk (_stream_readable.js:163:16)
  2417. Socket.Readable.push (_stream_readable.js:126:10)
  2418. TCP.onread (net.js:540:20)
  2419. [ Mar 17 15:09:28 Stopping because process dumped core. ]
  2420. [ Mar 17 15:09:28 Executing stop method (:kill). ]
  2421. [ Mar 17 15:09:28 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  2422. [ Mar 17 15:09:28 Method "start" exited with status 0. ]
  2423. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95806,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:09:28.293Z","v":0}
  2424. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95806,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:09:28.355Z","v":0}
  2425. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95806,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:09:28.356Z","v":0}
  2426. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95806,"level":30,"msg":"done with binder init","time":"2016-03-17T15:09:28.356Z","v":0}
  2427. { path: '/net/blackshard/sol/sol/electric-moray',
  2428. err: null,
  2429. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2430. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95806,"req_id":49331,"client":"10.43.1.14","port":"33990/udp","query":{"name":"electric-moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.7"],"additional":[],"latency":16,"timers":{"zk.getNameRecord":9,"zk.loadService":3,"pre-resp":1,"log-after":3},"msg":"DNS query","time":"2016-03-17T15:09:28.414Z","v":0}
  2431. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95806,"req_id":8789,"client":"10.43.1.11","port":"39803/udp","query":{"name":"electric-moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.7"],"additional":[],"latency":2,"timers":{"zk.getNameRecord":1,"zk.loadService":0,"pre-resp":0,"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:28.783Z","v":0}
  2432. { path: '/net/blackshard/sol/sol/manta',
  2433. err: null,
  2434. data: undefined }
  2435. Uncaught TypeError: Cannot read property 'toString' of undefined
  2436.  
  2437. FROM
  2438. /opt/smartdc/binder/lib/zk.js:133:42
  2439. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2440. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2441. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2442. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2443. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2444. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2445. Socket.emit (events.js:107:17)
  2446. readableAddChunk (_stream_readable.js:163:16)
  2447. Socket.Readable.push (_stream_readable.js:126:10)
  2448. TCP.onread (net.js:540:20)
  2449. [ Mar 17 15:09:28 Stopping because process dumped core. ]
  2450. [ Mar 17 15:09:29 Executing stop method (:kill). ]
  2451. [ Mar 17 15:09:29 Executing start method ("node --abort_on_uncaught_exception main.js &"). ]
  2452. [ Mar 17 15:09:29 Method "start" exited with status 0. ]
  2453. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"level":30,"expiry":60000,"size":10000,"port":53,"dnsDomain":"sol.blackshard.net","datacenterName":"sol","configFile":"./etc/config.json","msg":"starting with options","time":"2016-03-17T15:09:29.401Z","v":0}
  2454. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"level":30,"port":53,"msg":"TCP DNS service started","time":"2016-03-17T15:09:29.457Z","v":0}
  2455. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"level":30,"port":53,"msg":"UDP DNS service started","time":"2016-03-17T15:09:29.457Z","v":0}
  2456. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"level":30,"msg":"done with binder init","time":"2016-03-17T15:09:29.458Z","v":0}
  2457. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"req_id":54623,"client":"10.43.1.14","port":"48848/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":3,"timers":{"log-after":3},"msg":"DNS query","time":"2016-03-17T15:09:29.605Z","v":0}
  2458. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"req_id":54623,"client":"10.43.1.14","port":"64399/udp","query":{"name":"sapi.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:29.614Z","v":0}
  2459. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"req_id":11938,"client":"10.43.1.14","port":"42081/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":0,"timers":{"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:29.618Z","v":0}
  2460. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"req_id":11938,"client":"10.43.1.14","port":"53658/udp","query":{"name":"sapi.sol.sol.blackshard.net.sol.sol.blackshard.net","type":"AAAA"},"edns":false,"level":30,"rcode":"notimp","answers":[],"additional":[],"latency":1,"timers":{"log-after":1},"msg":"DNS query","time":"2016-03-17T15:09:29.624Z","v":0}
  2461. { path: '/net/blackshard/sol/sol/sapi',
  2462. err:
  2463. { [NO_NODE: Exception: NO_NODE[-101]]
  2464. code: -101,
  2465. name: 'NO_NODE',
  2466. path: undefined,
  2467. message: 'Exception: NO_NODE[-101]' },
  2468. data: undefined }
  2469. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"req_id":11392,"client":"10.43.1.14","port":"50729/udp","query":{"name":"sapi.sol...","type":"A"},"edns":false,"level":30,"rcode":"servfail","answers":[],"additional":[],"latency":10,"timers":{"zk.getNameRecord":9,"pre-resp":1,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:29.639Z","v":0}
  2470. { path: '/net/blackshard/sol/sol/moray/1',
  2471. err: null,
  2472. data: <Buffer 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 76 69 63 65 22 3a 7b 22 74 79 70 65 22 3a 22 73 65 72 76 69 63 65 22 2c 22 73 65 72 ... > }
  2473. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"req_id":22639,"client":"10.43.1.11","port":"53396/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":14,"timers":{"zk.getNameRecord":8,"zk.loadService":4,"pre-resp":0,"log-after":2},"msg":"DNS query","time":"2016-03-17T15:09:29.647Z","v":0}
  2474. {"name":"binder","hostname":"7dd4e6f5-e0ff-4b98-b11f-b5d9207a62c6","pid":95821,"req_id":57818,"client":"10.43.1.11","port":"53396/udp","query":{"name":"1.moray.sol...","type":"A"},"edns":false,"level":30,"rcode":"noerror","answers":["A 10.43.1.6"],"additional":[],"latency":6,"timers":{"zk.getNameRecord":0,"zk.loadService":5,"pre-resp":1,"log-after":0},"msg":"DNS query","time":"2016-03-17T15:09:29.648Z","v":0}
  2475. { path: '/net/blackshard/sol/sol/manta',
  2476. err: null,
  2477. data: undefined }
  2478. Uncaught TypeError: Cannot read property 'toString' of undefined
  2479.  
  2480. FROM
  2481. /opt/smartdc/binder/lib/zk.js:133:42
  2482. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:165:26
  2483. Object.async.whilst (/opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:627:13)
  2484. /opt/smartdc/binder/node_modules/node-zookeeper-client/node_modules/async/lib/async.js:623:23
  2485. /opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:145:21
  2486. Object.callback (/opt/smartdc/binder/node_modules/node-zookeeper-client/index.js:582:17)
  2487. ConnectionManager.onSocketData (/opt/smartdc/binder/node_modules/node-zookeeper-client/lib/ConnectionManager.js:562:35)
  2488. Socket.emit (events.js:107:17)
  2489. readableAddChunk (_stream_readable.js:163:16)
  2490. Socket.Readable.push (_stream_readable.js:126:10)
  2491. TCP.onread (net.js:540:20)
  2492. [ Mar 17 15:09:30 Stopping because process dumped core. ]
  2493. [ Mar 17 15:09:30 Executing stop method (:kill). ]
  2494. [ Mar 17 15:09:30 Restarting too quickly, changing state to maintenance. ]
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!