Apr 18 09:49:21 janus dockerd[1507]: time="2019-04-18T09:49:21.265113686Z" level=warning msg="memberlist: Was able to connect to 3af987f41544 but other probes failed, network may be misconfigured"
Apr 18 09:49:22 janus dockerd[1507]: time="2019-04-18T09:49:22.266022948Z" level=warning msg="memberlist: Was able to connect to 3af987f41544 but other probes failed, network may be misconfigured"
Apr 18 09:49:23 janus dockerd[1507]: time="2019-04-18T09:49:23.266784400Z" level=info msg="memberlist: Suspect 4adb11869318 has failed, no acks received"
Apr 18 09:49:25 janus dockerd[1507]: time="2019-04-18T09:49:25.139673348Z" level=error msg="heartbeat to manager {0ipcceidwwiwvbtt17gzm85qh 157.230.233.54:2377} failed" error="rpc error: code = DeadlineExceeded desc = context deadline exceeded" method="(*session).heartbeat" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw session.id=ujzh9mwhqrsdwvhd59nx98pg7 sessionID=ujzh9mwhqrsdwvhd59nx98pg7
Apr 18 09:49:25 janus dockerd[1507]: time="2019-04-18T09:49:25.140357481Z" level=error msg="agent: session failed" backoff=100ms error="rpc error: code = DeadlineExceeded desc = context deadline exceeded" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:25 janus dockerd[1507]: time="2019-04-18T09:49:25.140808542Z" level=info msg="parsed scheme: \"\"" module=grpc
Apr 18 09:49:25 janus dockerd[1507]: time="2019-04-18T09:49:25.141096434Z" level=info msg="scheme \"\" not registered, fallback to default scheme" module=grpc
Apr 18 09:49:25 janus dockerd[1507]: time="2019-04-18T09:49:25.141603122Z" level=info msg="manager selected by agent for new session: {0ipcceidwwiwvbtt17gzm85qh 157.230.233.54:2377}" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:25 janus dockerd[1507]: time="2019-04-18T09:49:25.141734630Z" level=info msg="ccResolverWrapper: sending new addresses to cc: [{157.230.233.54:2377 0  <nil>}]" module=grpc
Apr 18 09:49:25 janus dockerd[1507]: time="2019-04-18T09:49:25.141992006Z" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Apr 18 09:49:25 janus dockerd[1507]: time="2019-04-18T09:49:25.142073678Z" level=info msg="pickfirstBalancer: HandleSubConnStateChange: 0xc420169ca0, CONNECTING" module=grpc
Apr 18 09:49:25 janus dockerd[1507]: time="2019-04-18T09:49:25.141953965Z" level=info msg="waiting 99.356175ms before registering session" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:25 janus dockerd[1507]: time="2019-04-18T09:49:25.267614963Z" level=warning msg="memberlist: Was able to connect to 4adb11869318 but other probes failed, network may be misconfigured"
Apr 18 09:49:26 janus dockerd[1507]: time="2019-04-18T09:49:26.267901488Z" level=info msg="memberlist: Suspect 3af987f41544 has failed, no acks received"
Apr 18 09:49:26 janus dockerd[1507]: time="2019-04-18T09:49:26.468992358Z" level=warning msg="memberlist: Refuting a suspect message (from: e7b284330420)"
Apr 18 09:49:28 janus dockerd[1507]: time="2019-04-18T09:49:28.268244800Z" level=warning msg="memberlist: Was able to connect to 3af987f41544 but other probes failed, network may be misconfigured"
Apr 18 09:49:30 janus dockerd[1507]: time="2019-04-18T09:49:30.241756224Z" level=error msg="agent: session failed" backoff=300ms error="session initiation timed out" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:30 janus dockerd[1507]: time="2019-04-18T09:49:30.241890984Z" level=info msg="parsed scheme: \"\"" module=grpc
Apr 18 09:49:30 janus dockerd[1507]: time="2019-04-18T09:49:30.241927218Z" level=info msg="scheme \"\" not registered, fallback to default scheme" module=grpc
Apr 18 09:49:30 janus dockerd[1507]: time="2019-04-18T09:49:30.242272705Z" level=info msg="manager selected by agent for new session: {0ipcceidwwiwvbtt17gzm85qh 157.230.233.54:2377}" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:30 janus dockerd[1507]: time="2019-04-18T09:49:30.242317533Z" level=info msg="waiting 244.443604ms before registering session" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:30 janus dockerd[1507]: time="2019-04-18T09:49:30.242383708Z" level=info msg="ccResolverWrapper: sending new addresses to cc: [{157.230.233.54:2377 0  <nil>}]" module=grpc
Apr 18 09:49:30 janus dockerd[1507]: time="2019-04-18T09:49:30.242438990Z" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Apr 18 09:49:30 janus dockerd[1507]: time="2019-04-18T09:49:30.242495184Z" level=info msg="pickfirstBalancer: HandleSubConnStateChange: 0xc420169110, CONNECTING" module=grpc
Apr 18 09:49:30 janus dockerd[1507]: time="2019-04-18T09:49:30.268553787Z" level=info msg="memberlist: Suspect 4adb11869318 has failed, no acks received"
Apr 18 09:49:33 janus dockerd[1507]: time="2019-04-18T09:49:33.268914127Z" level=info msg="memberlist: Suspect 4adb11869318 has failed, no acks received"
Apr 18 09:49:34 janus dockerd[1507]: time="2019-04-18T09:49:34.268903472Z" level=info msg="memberlist: Marking 4adb11869318 as failed, suspect timeout reached (0 peer confirmations)"
Apr 18 09:49:34 janus dockerd[1507]: time="2019-04-18T09:49:34.268993067Z" level=info msg="Node 4adb11869318/157.230.233.54, left gossip cluster"
Apr 18 09:49:34 janus dockerd[1507]: time="2019-04-18T09:49:34.269028860Z" level=info msg="Node 4adb11869318 change state NodeActive --> NodeFailed"
Apr 18 09:49:34 janus dockerd[1507]: time="2019-04-18T09:49:34.269333679Z" level=info msg="Node 4adb11869318/157.230.233.54, added to failed nodes list"
Apr 18 09:49:34 janus kernel: [3504251.394876] IPVS: __ip_vs_del_service: enter
Apr 18 09:49:34 janus kernel: [3504251.495917] IPVS: __ip_vs_del_service: enter
Apr 18 09:49:34 janus kernel: [3504251.575377] IPVS: __ip_vs_del_service: enter
Apr 18 09:49:34 janus kernel: [3504251.659992] IPVS: __ip_vs_del_service: enter
Apr 18 09:49:35 janus dockerd[1507]: time="2019-04-18T09:49:35.487208278Z" level=error msg="agent: session failed" backoff=700ms error="session initiation timed out" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:35 janus dockerd[1507]: time="2019-04-18T09:49:35.487868265Z" level=info msg="parsed scheme: \"\"" module=grpc
Apr 18 09:49:35 janus dockerd[1507]: time="2019-04-18T09:49:35.488186744Z" level=info msg="scheme \"\" not registered, fallback to default scheme" module=grpc
Apr 18 09:49:35 janus dockerd[1507]: time="2019-04-18T09:49:35.488704776Z" level=info msg="manager selected by agent for new session: {0ipcceidwwiwvbtt17gzm85qh 157.230.233.54:2377}" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:35 janus dockerd[1507]: time="2019-04-18T09:49:35.489027214Z" level=info msg="waiting 365.409385ms before registering session" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:35 janus dockerd[1507]: time="2019-04-18T09:49:35.488844574Z" level=info msg="ccResolverWrapper: sending new addresses to cc: [{157.230.233.54:2377 0  <nil>}]" module=grpc
Apr 18 09:49:35 janus dockerd[1507]: time="2019-04-18T09:49:35.489582889Z" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Apr 18 09:49:35 janus dockerd[1507]: time="2019-04-18T09:49:35.489939617Z" level=info msg="pickfirstBalancer: HandleSubConnStateChange: 0xc421378040, CONNECTING" module=grpc
Apr 18 09:49:37 janus dockerd[1507]: time="2019-04-18T09:49:37.269649164Z" level=info msg="memberlist: Suspect 3af987f41544 has failed, no acks received"
Apr 18 09:49:38 janus do-agent[1404]: 2019/04/18 09:49:38 Sending metrics to DigitalOcean: Post https://nyc1.sonar.digitalocean.com/v1/metrics/droplet_id/135362746: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
Apr 18 09:49:39 janus dockerd[1507]: time="2019-04-18T09:49:39.438069338Z" level=warning msg="memberlist: Push/Pull with 4adb11869318 failed: dial tcp 157.230.233.54:7946: i/o timeout"
Apr 18 09:49:40 janus dockerd[1507]: time="2019-04-18T09:49:40.854892764Z" level=error msg="agent: session failed" backoff=1.5s error="session initiation timed out" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:40 janus dockerd[1507]: time="2019-04-18T09:49:40.855429738Z" level=info msg="parsed scheme: \"\"" module=grpc
Apr 18 09:49:40 janus dockerd[1507]: time="2019-04-18T09:49:40.855681824Z" level=info msg="scheme \"\" not registered, fallback to default scheme" module=grpc
Apr 18 09:49:40 janus dockerd[1507]: time="2019-04-18T09:49:40.856119073Z" level=info msg="manager selected by agent for new session: {0ipcceidwwiwvbtt17gzm85qh 157.230.233.54:2377}" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:40 janus dockerd[1507]: time="2019-04-18T09:49:40.856362886Z" level=info msg="waiting 432.166258ms before registering session" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:40 janus dockerd[1507]: time="2019-04-18T09:49:40.856255184Z" level=info msg="ccResolverWrapper: sending new addresses to cc: [{157.230.233.54:2377 0  <nil>}]" module=grpc
Apr 18 09:49:40 janus dockerd[1507]: time="2019-04-18T09:49:40.856800015Z" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Apr 18 09:49:40 janus dockerd[1507]: time="2019-04-18T09:49:40.857067979Z" level=info msg="pickfirstBalancer: HandleSubConnStateChange: 0xc421378a70, CONNECTING" module=grpc
Apr 18 09:49:41 janus dockerd[1507]: time="2019-04-18T09:49:41.269968071Z" level=info msg="memberlist: Marking 3af987f41544 as failed, suspect timeout reached (0 peer confirmations)"
Apr 18 09:49:41 janus dockerd[1507]: time="2019-04-18T09:49:41.270588789Z" level=info msg="Node 3af987f41544/134.209.118.8, left gossip cluster"
Apr 18 09:49:41 janus dockerd[1507]: time="2019-04-18T09:49:41.270947436Z" level=info msg="Node 3af987f41544 change state NodeActive --> NodeFailed"
Apr 18 09:49:41 janus dockerd[1507]: time="2019-04-18T09:49:41.271527701Z" level=info msg="Node 3af987f41544/134.209.118.8, added to failed nodes list"
Apr 18 09:49:41 janus kernel: [3504258.396729] IPVS: __ip_vs_del_service: enter
Apr 18 09:49:41 janus kernel: [3504258.486638] IPVS: __ip_vs_del_service: enter
Apr 18 09:49:41 janus kernel: [3504258.575787] IPVS: __ip_vs_del_service: enter
Apr 18 09:49:42 janus dockerd[1507]: time="2019-04-18T09:49:42.270019328Z" level=info msg="memberlist: Suspect 3af987f41544 has failed, no acks received"
Apr 18 09:49:45 janus dockerd[1507]: time="2019-04-18T09:49:45.142356141Z" level=warning msg="Failed to dial 157.230.233.54:2377: grpc: the connection is closing; please retry." module=grpc
Apr 18 09:49:46 janus dockerd[1507]: time="2019-04-18T09:49:46.288940894Z" level=error msg="agent: session failed" backoff=3.1s error="session initiation timed out" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:46 janus dockerd[1507]: time="2019-04-18T09:49:46.289051117Z" level=info msg="parsed scheme: \"\"" module=grpc
Apr 18 09:49:46 janus dockerd[1507]: time="2019-04-18T09:49:46.289067940Z" level=info msg="scheme \"\" not registered, fallback to default scheme" module=grpc
Apr 18 09:49:46 janus dockerd[1507]: time="2019-04-18T09:49:46.289289871Z" level=info msg="manager selected by agent for new session: {0ipcceidwwiwvbtt17gzm85qh 157.230.233.54:2377}" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:46 janus dockerd[1507]: time="2019-04-18T09:49:46.289332266Z" level=info msg="waiting 2.506216736s before registering session" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:46 janus dockerd[1507]: time="2019-04-18T09:49:46.289365687Z" level=info msg="ccResolverWrapper: sending new addresses to cc: [{157.230.233.54:2377 0  <nil>}]" module=grpc
Apr 18 09:49:46 janus dockerd[1507]: time="2019-04-18T09:49:46.289382561Z" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Apr 18 09:49:46 janus dockerd[1507]: time="2019-04-18T09:49:46.289446337Z" level=info msg="pickfirstBalancer: HandleSubConnStateChange: 0xc420d15910, CONNECTING" module=grpc
Apr 18 09:49:50 janus dockerd[1507]: time="2019-04-18T09:49:50.242871855Z" level=warning msg="Failed to dial 157.230.233.54:2377: grpc: the connection is closing; please retry." module=grpc
Apr 18 09:49:53 janus dockerd[1507]: time="2019-04-18T09:49:53.796030247Z" level=error msg="agent: session failed" backoff=6.3s error="session initiation timed out" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:53 janus dockerd[1507]: time="2019-04-18T09:49:53.796147710Z" level=info msg="parsed scheme: \"\"" module=grpc
Apr 18 09:49:53 janus dockerd[1507]: time="2019-04-18T09:49:53.796164114Z" level=info msg="scheme \"\" not registered, fallback to default scheme" module=grpc
Apr 18 09:49:53 janus dockerd[1507]: time="2019-04-18T09:49:53.796396647Z" level=info msg="manager selected by agent for new session: {0ipcceidwwiwvbtt17gzm85qh 157.230.233.54:2377}" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:53 janus dockerd[1507]: time="2019-04-18T09:49:53.796442884Z" level=info msg="waiting 5.029222702s before registering session" module=node/agent node.id=gctoo2ifeyh7q36gndsvj8wzw
Apr 18 09:49:53 janus dockerd[1507]: time="2019-04-18T09:49:53.796478551Z" level=info msg="ccResolverWrapper: sending new addresses to cc: [{157.230.233.54:2377 0  <nil>}]" module=grpc
Apr 18 09:49:53 janus dockerd[1507]: time="2019-04-18T09:49:53.796497536Z" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Apr 18 09:49:53 janus dockerd[1507]: time="2019-04-18T09:49:53.796563651Z" level=info msg="pickfirstBalancer: HandleSubConnStateChange: 0xc4214ed5b0, CONNECTING" module=grpc
Apr 18 09:49:55 janus dockerd[1507]: time="2019-04-18T09:49:55.490382209Z" level=warning msg="Failed to dial 157.230.233.54:2377: grpc: the connection is closing; please retry." module=grpc
Apr 18 09:49:59 janus dockerd[1507]: time="2019-04-18T09:49:59.265377050Z" level=error msg="Failed to join memberlist [157.230.233.54] on retry: 1 error(s) occurred:\n\n* Failed to join 157.230.233.54: dial tcp 157.230.233.54:7946: i/o timeout"
Apr 18 09:49:59 janus dockerd[1507]: time="2019-04-18T09:49:59.801429986Z" level=info msg="pickfirstBalancer: HandleSubConnStateChange: 0xc4214ed5b0, READY" module=grpc