LOG NODE1
Feb 3 17:48:03 wsguardian1 kernel: e1000e: eth4 NIC Link is Down
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: PingAck did not arrive in time.
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: peer( Primary -> Unknown ) conn( Connected -> NetworkFailure ) pdsk( UpToDate -> DUnknown ) susp( 0 -> 1 )
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: asender terminated
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: Terminating asender thread
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: Connection closed
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: conn( NetworkFailure -> Unconnected )
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: receiver terminated
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: Restarting receiver thread
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: receiver (re)started
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: conn( Unconnected -> WFConnection )
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_db: helper command: /sbin/drbdadm fence-peer wsg_db
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: PingAck did not arrive in time.
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: peer( Primary -> Unknown ) conn( Connected -> NetworkFailure ) pdsk( UpToDate -> DUnknown ) susp( 0 -> 1 )
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: asender terminated
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: Terminating asender thread
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: Connection closed
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: conn( NetworkFailure -> Unconnected )
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: receiver terminated
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: Restarting receiver thread
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: receiver (re)started
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: conn( Unconnected -> WFConnection )
Feb 3 17:48:13 wsguardian1 kernel: d-con wsg_config: helper command: /sbin/drbdadm fence-peer wsg_config
Feb 3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [uname] is at: [/bin/uname]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [fence_node] is at: [/usr/sbin/fence_node]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [cman_tool] is at: [/usr/sbin/cman_tool]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian1 rhcs_fence: 74; Attempting to fence peer using RHCS from DRBD...
Feb 3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_RESOURCE] == [wsg_config]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_MINOR] == [1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_PEERS] == [wsguardian2]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 454; DEBUG: shell call: [/usr/sbin/cman_tool status]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [uname] is at: [/bin/uname]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [fence_node] is at: [/usr/sbin/fence_node]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [cman_tool] is at: [/usr/sbin/cman_tool]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian1 rhcs_fence: 74; Attempting to fence peer using RHCS from DRBD...
Feb 3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_RESOURCE] == [wsg_db]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_MINOR] == [0]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_PEERS] == [wsguardian2]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 454; DEBUG: shell call: [/usr/sbin/cman_tool status]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Version: 6.2.0
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Config Version: 2
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Name: WSGClust
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Id: 30485
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Member: Yes
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Generation: 28
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Membership state: Cluster-Member
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Nodes: 2
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Expected votes: 1
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Total votes: 2
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Version: 6.2.0
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Node votes: 1
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Config Version: 2
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Quorum: 1
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Active subsystems: 7
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Name: WSGClust
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Flags: 2node
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Id: 30485
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Ports Bound: 0
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Member: Yes
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Node name: wsguardian1
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Generation: 28
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Membership state: Cluster-Member
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Nodes: 2
Feb 3 17:48:13 wsguardian1 rhcs_fence: 469; DEBUG: Attempt to get local node name via 'cman_tool status' exited with: [0]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Expected votes: 1
Feb 3 17:48:13 wsguardian1 rhcs_fence: 471; DEBUG: I am: [wsguardian1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Total votes: 2
Feb 3 17:48:13 wsguardian1 rhcs_fence: 403; DEBUG: Checking the state of resource with minor number: [1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Node votes: 1
Feb 3 17:48:13 wsguardian1 rhcs_fence: 409; DEBUG: minor: [1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Quorum: 1
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Active subsystems: 7
Feb 3 17:48:13 wsguardian1 rhcs_fence: 412; DEBUG: shell call: [</proc/drbd]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Flags: 2node
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Ports Bound: 0
Feb 3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Node name: wsguardian1
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: version: 8.4.1 (api:1/proto:86-100)
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: GIT-hash: 91b4c048c1a0e06777b5f65d312b38d47abaea80 build by dag@Build64R6, 2011-12-21 06:08:50
Feb 3 17:48:13 wsguardian1 rhcs_fence: 469; DEBUG: Attempt to get local node name via 'cman_tool status' exited with: [0]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: 0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb 3 17:48:13 wsguardian1 rhcs_fence: 471; DEBUG: I am: [wsguardian1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb 3 17:48:13 wsguardian1 rhcs_fence: 403; DEBUG: Checking the state of resource with minor number: [0]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: 1: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb 3 17:48:13 wsguardian1 rhcs_fence: 409; DEBUG: minor: [0]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 423; DEBUG: read state of minor: [1] as: [UpToDate]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 412; DEBUG: shell call: [</proc/drbd]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 425; DEBUG: sys::local_res_uptodate: [1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: version: 8.4.1 (api:1/proto:86-100)
Feb 3 17:48:13 wsguardian1 rhcs_fence: 430; DEBUG: Attempt to collect UpToDate information device with minor: [1] exited with: [0]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: GIT-hash: 91b4c048c1a0e06777b5f65d312b38d47abaea80 build by dag@Build64R6, 2011-12-21 06:08:50
Feb 3 17:48:13 wsguardian1 rhcs_fence: 432; DEBUG: UpToDate: [1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: 0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb 3 17:48:13 wsguardian1 rhcs_fence: 367; DEBUG: shell call: [/usr/sbin/cman_tool -a -F id,name,type,addr nodes]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 423; DEBUG: read state of minor: [0] as: [UpToDate]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 425; DEBUG: sys::local_res_uptodate: [1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: 1: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb 3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb 3 17:48:13 wsguardian1 rhcs_fence: 430; DEBUG: Attempt to collect UpToDate information device with minor: [0] exited with: [0]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 432; DEBUG: UpToDate: [1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 367; DEBUG: shell call: [/usr/sbin/cman_tool -a -F id,name,type,addr nodes]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 374; DEBUG: id: [1], name: [wsguardian1], member: [M], address: [192.168.201.43]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 380; DEBUG: output: 1 wsguardian1 M 192.168.201.43
Feb 3 17:48:13 wsguardian1 rhcs_fence: 374; DEBUG: id: [2], name: [wsguardian2], member: [M], address: [192.168.201.180]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 380; DEBUG: output: 2 wsguardian2 M 192.168.201.180
Feb 3 17:48:13 wsguardian1 rhcs_fence: 384; DEBUG: Attempt to gather cluster member information exited with: [0]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 374; DEBUG: id: [1], name: [wsguardian1], member: [M], address: [192.168.201.43]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 343; I have identified my target: [wsguardian2]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 380; DEBUG: output: 1 wsguardian1 M 192.168.201.43
Feb 3 17:48:13 wsguardian1 rhcs_fence: 191; DEBUG: I am: [wsguardian1] and my id is: [1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 374; DEBUG: id: [2], name: [wsguardian2], member: [M], address: [192.168.201.180]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 380; DEBUG: output: 2 wsguardian2 M 192.168.201.180
Feb 3 17:48:13 wsguardian1 rhcs_fence: 223; I am the first node, so I won't delay.
Feb 3 17:48:13 wsguardian1 rhcs_fence: 301; Target node: [wsguardian2] is a cluster member, attempting to eject.
Feb 3 17:48:13 wsguardian1 rhcs_fence: 384; DEBUG: Attempt to gather cluster member information exited with: [0]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 304; DEBUG: shell call: [/usr/sbin/cman_tool kill -f wsguardian2]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 343; I have identified my target: [wsguardian2]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 191; DEBUG: I am: [wsguardian1] and my id is: [1]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 223; I am the first node, so I won't delay.
Feb 3 17:48:13 wsguardian1 rhcs_fence: 301; Target node: [wsguardian2] is a cluster member, attempting to eject.
Feb 3 17:48:13 wsguardian1 rhcs_fence: 304; DEBUG: shell call: [/usr/sbin/cman_tool kill -f wsguardian2]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 313; DEBUG: Attempt to force-remove node: [wsguardian2] exited with: [256]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 243; Fencing target: [wsguardian2]...
Feb 3 17:48:13 wsguardian1 rhcs_fence: 247; DEBUG: shell call: [/usr/sbin/fence_node -v wsguardian2]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 313; DEBUG: Attempt to force-remove node: [wsguardian2] exited with: [256]
Feb 3 17:48:13 wsguardian1 rhcs_fence: 243; Fencing target: [wsguardian2]...
Feb 3 17:48:13 wsguardian1 rhcs_fence: 247; DEBUG: shell call: [/usr/sbin/fence_node -v wsguardian2]
LOG NODE2
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: PingAck did not arrive in time.
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: peer( Primary -> Unknown ) conn( Connected -> NetworkFailure ) pdsk( UpToDate -> DUnknown ) susp( 0 -> 1 )
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: asender terminated
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: Terminating asender thread
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: Connection closed
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: conn( NetworkFailure -> Unconnected )
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: receiver terminated
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: Restarting receiver thread
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: receiver (re)started
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: conn( Unconnected -> WFConnection )
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_db: helper command: /sbin/drbdadm fence-peer wsg_db
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: PingAck did not arrive in time.
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: peer( Primary -> Unknown ) conn( Connected -> NetworkFailure ) pdsk( UpToDate -> DUnknown ) susp( 0 -> 1 )
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: asender terminated
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: Terminating asender thread
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: Connection closed
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: conn( NetworkFailure -> Unconnected )
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: receiver terminated
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: Restarting receiver thread
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: receiver (re)started
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: conn( Unconnected -> WFConnection )
Feb 3 17:48:13 wsguardian2 kernel: d-con wsg_config: helper command: /sbin/drbdadm fence-peer wsg_config
Feb 3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [uname] is at: [/bin/uname]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [fence_node] is at: [/usr/sbin/fence_node]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [cman_tool] is at: [/usr/sbin/cman_tool]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian2 rhcs_fence: 74; Attempting to fence peer using RHCS from DRBD...
Feb 3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_RESOURCE] == [wsg_config]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_MINOR] == [1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_PEERS] == [wsguardian1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 454; DEBUG: shell call: [/usr/sbin/cman_tool status]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [uname] is at: [/bin/uname]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [fence_node] is at: [/usr/sbin/fence_node]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [cman_tool] is at: [/usr/sbin/cman_tool]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb 3 17:48:13 wsguardian2 rhcs_fence: 74; Attempting to fence peer using RHCS from DRBD...
Feb 3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_RESOURCE] == [wsg_db]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_MINOR] == [0]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_PEERS] == [wsguardian1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 454; DEBUG: shell call: [/usr/sbin/cman_tool status]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Version: 6.2.0
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Config Version: 2
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Name: WSGClust
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Id: 30485
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Member: Yes
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Generation: 28
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Membership state: Cluster-Member
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Nodes: 2
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Version: 6.2.0
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Expected votes: 1
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Total votes: 2
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Config Version: 2
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Node votes: 1
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Name: WSGClust
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Quorum: 1
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Id: 30485
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Active subsystems: 7
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Member: Yes
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Flags: 2node
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Generation: 28
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Membership state: Cluster-Member
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Ports Bound: 0
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Nodes: 2
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Node name: wsguardian2
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Expected votes: 1
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Total votes: 2
Feb 3 17:48:13 wsguardian2 rhcs_fence: 469; DEBUG: Attempt to get local node name via 'cman_tool status' exited with: [0]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Node votes: 1
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Quorum: 1
Feb 3 17:48:13 wsguardian2 rhcs_fence: 471; DEBUG: I am: [wsguardian2]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Active subsystems: 7
Feb 3 17:48:13 wsguardian2 rhcs_fence: 403; DEBUG: Checking the state of resource with minor number: [1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Flags: 2node
Feb 3 17:48:13 wsguardian2 rhcs_fence: 409; DEBUG: minor: [1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Ports Bound: 0
Feb 3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Node name: wsguardian2
Feb 3 17:48:13 wsguardian2 rhcs_fence: 469; DEBUG: Attempt to get local node name via 'cman_tool status' exited with: [0]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 412; DEBUG: shell call: [</proc/drbd]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 471; DEBUG: I am: [wsguardian2]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 403; DEBUG: Checking the state of resource with minor number: [0]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 409; DEBUG: minor: [0]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: version: 8.4.1 (api:1/proto:86-100)
Feb 3 17:48:13 wsguardian2 rhcs_fence: 412; DEBUG: shell call: [</proc/drbd]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: GIT-hash: 91b4c048c1a0e06777b5f65d312b38d47abaea80 build by dag@Build64R6, 2011-12-21 06:08:50
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: version: 8.4.1 (api:1/proto:86-100)
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: 0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: GIT-hash: 91b4c048c1a0e06777b5f65d312b38d47abaea80 build by dag@Build64R6, 2011-12-21 06:08:50
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: 1: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: 0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb 3 17:48:13 wsguardian2 rhcs_fence: 423; DEBUG: read state of minor: [1] as: [UpToDate]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 425; DEBUG: sys::local_res_uptodate: [1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 423; DEBUG: read state of minor: [0] as: [UpToDate]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb 3 17:48:13 wsguardian2 rhcs_fence: 425; DEBUG: sys::local_res_uptodate: [1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 430; DEBUG: Attempt to collect UpToDate information device with minor: [1] exited with: [0]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb 3 17:48:13 wsguardian2 rhcs_fence: 432; DEBUG: UpToDate: [1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: 1: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb 3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb 3 17:48:13 wsguardian2 rhcs_fence: 367; DEBUG: shell call: [/usr/sbin/cman_tool -a -F id,name,type,addr nodes]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 430; DEBUG: Attempt to collect UpToDate information device with minor: [0] exited with: [0]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 432; DEBUG: UpToDate: [1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 367; DEBUG: shell call: [/usr/sbin/cman_tool -a -F id,name,type,addr nodes]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 374; DEBUG: id: [1], name: [wsguardian1], member: [M], address: [192.168.201.43]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 380; DEBUG: output: 1 wsguardian1 M 192.168.201.43
Feb 3 17:48:13 wsguardian2 rhcs_fence: 374; DEBUG: id: [2], name: [wsguardian2], member: [M], address: [192.168.201.180]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 380; DEBUG: output: 2 wsguardian2 M 192.168.201.180
Feb 3 17:48:13 wsguardian2 rhcs_fence: 384; DEBUG: Attempt to gather cluster member information exited with: [0]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 343; I have identified my target: [wsguardian1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 374; DEBUG: id: [1], name: [wsguardian1], member: [M], address: [192.168.201.43]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 191; DEBUG: I am: [wsguardian2] and my id is: [2]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 380; DEBUG: output: 1 wsguardian1 M 192.168.201.43
Feb 3 17:48:13 wsguardian2 rhcs_fence: 217; Delaying for: [9] seconds to avoid dual-fencing...
Feb 3 17:48:13 wsguardian2 rhcs_fence: 374; DEBUG: id: [2], name: [wsguardian2], member: [M], address: [192.168.201.180]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 380; DEBUG: output: 2 wsguardian2 M 192.168.201.180
Feb 3 17:48:13 wsguardian2 rhcs_fence: 384; DEBUG: Attempt to gather cluster member information exited with: [0]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 343; I have identified my target: [wsguardian1]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 191; DEBUG: I am: [wsguardian2] and my id is: [2]
Feb 3 17:48:13 wsguardian2 rhcs_fence: 217; Delaying for: [9] seconds to avoid dual-fencing...
Feb 3 17:48:22 wsguardian2 rhcs_fence: 219; DEBUG: Right then, break over.
Feb 3 17:48:22 wsguardian2 rhcs_fence: 301; Target node: [wsguardian1] is a cluster member, attempting to eject.
Feb 3 17:48:22 wsguardian2 rhcs_fence: 219; DEBUG: Right then, break over.
Feb 3 17:48:22 wsguardian2 rhcs_fence: 304; DEBUG: shell call: [/usr/sbin/cman_tool kill -f wsguardian1]
Feb 3 17:48:22 wsguardian2 rhcs_fence: 301; Target node: [wsguardian1] is a cluster member, attempting to eject.
Feb 3 17:48:22 wsguardian2 rhcs_fence: 304; DEBUG: shell call: [/usr/sbin/cman_tool kill -f wsguardian1]
Feb 3 17:48:22 wsguardian2 rhcs_fence: 313; DEBUG: Attempt to force-remove node: [wsguardian1] exited with: [256]
Feb 3 17:48:22 wsguardian2 rhcs_fence: 243; Fencing target: [wsguardian1]...
Feb 3 17:48:22 wsguardian2 rhcs_fence: 313; DEBUG: Attempt to force-remove node: [wsguardian1] exited with: [256]
Feb 3 17:48:22 wsguardian2 rhcs_fence: 247; DEBUG: shell call: [/usr/sbin/fence_node -v wsguardian1]
Feb 3 17:48:22 wsguardian2 rhcs_fence: 243; Fencing target: [wsguardian1]...
Feb 3 17:48:22 wsguardian2 rhcs_fence: 247; DEBUG: shell call: [/usr/sbin/fence_node -v wsguardian1]
cluster.conf
<?xml version="1.0"?>
<cluster config_version="2" name="WSGClust">
<cman expected_votes="1" two_node="1"/>
<clusternodes>
<clusternode name="wsguardian1" nodeid="1">
<fence>
<method name="wsguardian1_ipmi">
<device name="ipmi1" action="reboot"/>
</method>
</fence>
</clusternode>
<clusternode name="wsguardian2" nodeid="2">
<fence>
<method name="wsguardian2_ipmi">
<device name="ipmi2" action="reboot"/>
</method>
</fence>
</clusternode>
</clusternodes>
<fencedevices>
<fencedevice agent="fence_ipmilan" ipaddr="192.168.201.220" lanplus="1" login="ADMIN" name="ipmi1" passwd="easy"/>
<fencedevice agent="fence_ipmilan" ipaddr="192.168.201.186" lanplus="1" login="ADMIN" name="ipmi2" passwd="easy"/>
</fencedevices>
<fence_daemon clean_start="0" post_fail_delay="0" post_join_delay="30"/>
</cluster>
drbd.conf
global {
usage-count no;
}
common {
protocol C;
syncer {
rate 100M;
al-extents 3389;
}
# This tells DRBD what policy to use when a fence is required.
disk {
fencing resource-and-stonith;
}
handlers {
#outdate-peer "/sbin/obliterate-peer.sh";
outdate-peer "/usr/sbin/rhcs_fence";
split-brain "/usr/lib/drbd/notify-split-brain.sh kortux@gmail.com";
}
net {
sndbuf-size 1024k;
# Tell DRBD to allow dual-primary.
allow-two-primaries;
# Set the recover policy for split-brain recover when no device
# in the resource was primary.
after-sb-0pri discard-zero-changes;
# Now if one device was primary.
after-sb-1pri discard-secondary;
# Finally, set the policy when both nodes were Primary. The
# only viable option is 'disconnect', which tells DRBD to
# simply tear-down the DRBD resource right away and wait for
# the administrator to manually invalidate one side of the
# resource.
after-sb-2pri disconnect;
}
startup {
become-primary-on both;
}
}
resource wsg_db {
device /dev/drbd0;
meta-disk internal;
on wsguardian1 {
address 192.168.253.1:7788;
disk /dev/rootvg/wsg_data_lv;
}
on wsguardian2 {
address 192.168.253.2:7788;
disk /dev/rootvg/wsg_data_lv;
}
}
resource wsg_config {
device /dev/drbd1;
meta-disk internal;
on wsguardian1 {
address 192.168.253.1:7789;
disk /dev/rootvg/wsg_config_lv;
}
on wsguardian2 {
address 192.168.253.2:7789;
disk /dev/rootvg/wsg_config_lv;
}
}