LOG NODE1
Feb  3 17:48:03 wsguardian1 kernel: e1000e: eth4 NIC Link is Down
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: PingAck did not arrive in time.
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: peer( Primary -> Unknown ) conn( Connected -> NetworkFailure ) pdsk( UpToDate -> DUnknown ) susp( 0 -> 1 ) 
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: asender terminated
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: Terminating asender thread
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: Connection closed
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: conn( NetworkFailure -> Unconnected ) 
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: receiver terminated
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: Restarting receiver thread
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: receiver (re)started
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: conn( Unconnected -> WFConnection ) 
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_db: helper command: /sbin/drbdadm fence-peer wsg_db
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: PingAck did not arrive in time.
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: peer( Primary -> Unknown ) conn( Connected -> NetworkFailure ) pdsk( UpToDate -> DUnknown ) susp( 0 -> 1 ) 
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: asender terminated
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: Terminating asender thread
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: Connection closed
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: conn( NetworkFailure -> Unconnected ) 
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: receiver terminated
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: Restarting receiver thread
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: receiver (re)started
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: conn( Unconnected -> WFConnection ) 
Feb  3 17:48:13 wsguardian1 kernel: d-con wsg_config: helper command: /sbin/drbdadm fence-peer wsg_config
Feb  3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [uname] is at: [/bin/uname]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [fence_node] is at: [/usr/sbin/fence_node]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [cman_tool] is at: [/usr/sbin/cman_tool]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian1 rhcs_fence: 74; Attempting to fence peer using RHCS from DRBD...
Feb  3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_RESOURCE] == [wsg_config]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_MINOR] == [1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_PEERS] == [wsguardian2]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 454; DEBUG: shell call: [/usr/sbin/cman_tool status]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [uname] is at: [/bin/uname]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [fence_node] is at: [/usr/sbin/fence_node]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian1 rhcs_fence: 125; DEBUG: Checking if: [cman_tool] is at: [/usr/sbin/cman_tool]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian1 rhcs_fence: 74; Attempting to fence peer using RHCS from DRBD...
Feb  3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_RESOURCE] == [wsg_db]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_MINOR] == [0]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_PEERS] == [wsguardian2]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 454; DEBUG: shell call: [/usr/sbin/cman_tool status]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Version: 6.2.0
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Config Version: 2
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Name: WSGClust
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Id: 30485
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Member: Yes
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Generation: 28
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Membership state: Cluster-Member
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Nodes: 2
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Expected votes: 1
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Total votes: 2
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Version: 6.2.0
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Node votes: 1
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Config Version: 2
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Quorum: 1  
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Active subsystems: 7
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Name: WSGClust
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Flags: 2node 
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Id: 30485
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Ports Bound: 0  
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Member: Yes
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Node name: wsguardian1
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Cluster Generation: 28
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Membership state: Cluster-Member
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Nodes: 2
Feb  3 17:48:13 wsguardian1 rhcs_fence: 469; DEBUG: Attempt to get local node name via 'cman_tool status' exited with: [0]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Expected votes: 1
Feb  3 17:48:13 wsguardian1 rhcs_fence: 471; DEBUG: I am: [wsguardian1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Total votes: 2
Feb  3 17:48:13 wsguardian1 rhcs_fence: 403; DEBUG: Checking the state of resource with minor number: [1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Node votes: 1
Feb  3 17:48:13 wsguardian1 rhcs_fence: 409; DEBUG: minor: [1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Quorum: 1  
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Active subsystems: 7
Feb  3 17:48:13 wsguardian1 rhcs_fence: 412; DEBUG: shell call: [</proc/drbd]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Flags: 2node 
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Ports Bound: 0  
Feb  3 17:48:13 wsguardian1 rhcs_fence: 460; DEBUG: output: Node name: wsguardian1
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: version: 8.4.1 (api:1/proto:86-100)
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: GIT-hash: 91b4c048c1a0e06777b5f65d312b38d47abaea80 build by dag@Build64R6, 2011-12-21 06:08:50
Feb  3 17:48:13 wsguardian1 rhcs_fence: 469; DEBUG: Attempt to get local node name via 'cman_tool status' exited with: [0]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output:  0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb  3 17:48:13 wsguardian1 rhcs_fence: 471; DEBUG: I am: [wsguardian1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output:     ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb  3 17:48:13 wsguardian1 rhcs_fence: 403; DEBUG: Checking the state of resource with minor number: [0]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output:  1: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb  3 17:48:13 wsguardian1 rhcs_fence: 409; DEBUG: minor: [0]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 423; DEBUG: read state of minor: [1] as: [UpToDate]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 412; DEBUG: shell call: [</proc/drbd]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 425; DEBUG: sys::local_res_uptodate: [1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output:     ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: version: 8.4.1 (api:1/proto:86-100)
Feb  3 17:48:13 wsguardian1 rhcs_fence: 430; DEBUG: Attempt to collect UpToDate information device with minor: [1] exited with: [0]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output: GIT-hash: 91b4c048c1a0e06777b5f65d312b38d47abaea80 build by dag@Build64R6, 2011-12-21 06:08:50
Feb  3 17:48:13 wsguardian1 rhcs_fence: 432; DEBUG: UpToDate: [1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output:  0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb  3 17:48:13 wsguardian1 rhcs_fence: 367; DEBUG: shell call: [/usr/sbin/cman_tool -a -F id,name,type,addr nodes]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 423; DEBUG: read state of minor: [0] as: [UpToDate]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 425; DEBUG: sys::local_res_uptodate: [1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output:     ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output:  1: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb  3 17:48:13 wsguardian1 rhcs_fence: 418; DEBUG: output:     ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb  3 17:48:13 wsguardian1 rhcs_fence: 430; DEBUG: Attempt to collect UpToDate information device with minor: [0] exited with: [0]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 432; DEBUG: UpToDate: [1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 367; DEBUG: shell call: [/usr/sbin/cman_tool -a -F id,name,type,addr nodes]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 374; DEBUG: id: [1], name: [wsguardian1], member: [M], address: [192.168.201.43]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 380; DEBUG: output: 1 wsguardian1 M 192.168.201.43 
Feb  3 17:48:13 wsguardian1 rhcs_fence: 374; DEBUG: id: [2], name: [wsguardian2], member: [M], address: [192.168.201.180]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 380; DEBUG: output: 2 wsguardian2 M 192.168.201.180 
Feb  3 17:48:13 wsguardian1 rhcs_fence: 384; DEBUG: Attempt to gather cluster member information exited with: [0]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 374; DEBUG: id: [1], name: [wsguardian1], member: [M], address: [192.168.201.43]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 343; I have identified my target: [wsguardian2]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 380; DEBUG: output: 1 wsguardian1 M 192.168.201.43 
Feb  3 17:48:13 wsguardian1 rhcs_fence: 191; DEBUG: I am: [wsguardian1] and my id is: [1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 374; DEBUG: id: [2], name: [wsguardian2], member: [M], address: [192.168.201.180]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 380; DEBUG: output: 2 wsguardian2 M 192.168.201.180 
Feb  3 17:48:13 wsguardian1 rhcs_fence: 223; I am the first node, so I won't delay.
Feb  3 17:48:13 wsguardian1 rhcs_fence: 301; Target node: [wsguardian2] is a cluster member, attempting to eject.
Feb  3 17:48:13 wsguardian1 rhcs_fence: 384; DEBUG: Attempt to gather cluster member information exited with: [0]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 304; DEBUG: shell call: [/usr/sbin/cman_tool kill -f wsguardian2]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 343; I have identified my target: [wsguardian2]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 191; DEBUG: I am: [wsguardian1] and my id is: [1]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 223; I am the first node, so I won't delay.
Feb  3 17:48:13 wsguardian1 rhcs_fence: 301; Target node: [wsguardian2] is a cluster member, attempting to eject.
Feb  3 17:48:13 wsguardian1 rhcs_fence: 304; DEBUG: shell call: [/usr/sbin/cman_tool kill -f wsguardian2]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 313; DEBUG: Attempt to force-remove node: [wsguardian2] exited with: [256]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 243; Fencing target: [wsguardian2]...
Feb  3 17:48:13 wsguardian1 rhcs_fence: 247; DEBUG: shell call: [/usr/sbin/fence_node -v wsguardian2]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 313; DEBUG: Attempt to force-remove node: [wsguardian2] exited with: [256]
Feb  3 17:48:13 wsguardian1 rhcs_fence: 243; Fencing target: [wsguardian2]...
Feb  3 17:48:13 wsguardian1 rhcs_fence: 247; DEBUG: shell call: [/usr/sbin/fence_node -v wsguardian2]

LOG NODE2

Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: PingAck did not arrive in time.
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: peer( Primary -> Unknown ) conn( Connected -> NetworkFailure ) pdsk( UpToDate -> DUnknown ) susp( 0 -> 1 ) 
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: asender terminated
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: Terminating asender thread
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: Connection closed
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: conn( NetworkFailure -> Unconnected ) 
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: receiver terminated
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: Restarting receiver thread
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: receiver (re)started
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: conn( Unconnected -> WFConnection ) 
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_db: helper command: /sbin/drbdadm fence-peer wsg_db
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: PingAck did not arrive in time.
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: peer( Primary -> Unknown ) conn( Connected -> NetworkFailure ) pdsk( UpToDate -> DUnknown ) susp( 0 -> 1 ) 
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: asender terminated
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: Terminating asender thread
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: Connection closed
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: conn( NetworkFailure -> Unconnected ) 
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: receiver terminated
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: Restarting receiver thread
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: receiver (re)started
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: conn( Unconnected -> WFConnection ) 
Feb  3 17:48:13 wsguardian2 kernel: d-con wsg_config: helper command: /sbin/drbdadm fence-peer wsg_config
Feb  3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [uname] is at: [/bin/uname]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [fence_node] is at: [/usr/sbin/fence_node]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [cman_tool] is at: [/usr/sbin/cman_tool]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian2 rhcs_fence: 74; Attempting to fence peer using RHCS from DRBD...
Feb  3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_RESOURCE] == [wsg_config]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_MINOR] == [1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_PEERS] == [wsguardian1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 454; DEBUG: shell call: [/usr/sbin/cman_tool status]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [uname] is at: [/bin/uname]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [fence_node] is at: [/usr/sbin/fence_node]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian2 rhcs_fence: 125; DEBUG: Checking if: [cman_tool] is at: [/usr/sbin/cman_tool]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 156; DEBUG: Found!
Feb  3 17:48:13 wsguardian2 rhcs_fence: 74; Attempting to fence peer using RHCS from DRBD...
Feb  3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_RESOURCE] == [wsg_db]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_MINOR] == [0]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 80; DEBUG: Environment variable: [DRBD_PEERS] == [wsguardian1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 454; DEBUG: shell call: [/usr/sbin/cman_tool status]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Version: 6.2.0
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Config Version: 2
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Name: WSGClust
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Id: 30485
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Member: Yes
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Generation: 28
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Membership state: Cluster-Member
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Nodes: 2
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Version: 6.2.0
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Expected votes: 1
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Total votes: 2
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Config Version: 2
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Node votes: 1
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Name: WSGClust
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Quorum: 1  
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Id: 30485
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Active subsystems: 7
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Member: Yes
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Flags: 2node 
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Cluster Generation: 28
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Membership state: Cluster-Member
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Ports Bound: 0  
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Nodes: 2
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Node name: wsguardian2
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Expected votes: 1
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Total votes: 2
Feb  3 17:48:13 wsguardian2 rhcs_fence: 469; DEBUG: Attempt to get local node name via 'cman_tool status' exited with: [0]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Node votes: 1
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Quorum: 1  
Feb  3 17:48:13 wsguardian2 rhcs_fence: 471; DEBUG: I am: [wsguardian2]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Active subsystems: 7
Feb  3 17:48:13 wsguardian2 rhcs_fence: 403; DEBUG: Checking the state of resource with minor number: [1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Flags: 2node 
Feb  3 17:48:13 wsguardian2 rhcs_fence: 409; DEBUG: minor: [1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Ports Bound: 0  
Feb  3 17:48:13 wsguardian2 rhcs_fence: 460; DEBUG: output: Node name: wsguardian2
Feb  3 17:48:13 wsguardian2 rhcs_fence: 469; DEBUG: Attempt to get local node name via 'cman_tool status' exited with: [0]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 412; DEBUG: shell call: [</proc/drbd]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 471; DEBUG: I am: [wsguardian2]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 403; DEBUG: Checking the state of resource with minor number: [0]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 409; DEBUG: minor: [0]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: version: 8.4.1 (api:1/proto:86-100)
Feb  3 17:48:13 wsguardian2 rhcs_fence: 412; DEBUG: shell call: [</proc/drbd]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: GIT-hash: 91b4c048c1a0e06777b5f65d312b38d47abaea80 build by dag@Build64R6, 2011-12-21 06:08:50
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: version: 8.4.1 (api:1/proto:86-100)
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output:  0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output:     ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output: GIT-hash: 91b4c048c1a0e06777b5f65d312b38d47abaea80 build by dag@Build64R6, 2011-12-21 06:08:50
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output:  1: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output:  0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb  3 17:48:13 wsguardian2 rhcs_fence: 423; DEBUG: read state of minor: [1] as: [UpToDate]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 425; DEBUG: sys::local_res_uptodate: [1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 423; DEBUG: read state of minor: [0] as: [UpToDate]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output:     ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb  3 17:48:13 wsguardian2 rhcs_fence: 425; DEBUG: sys::local_res_uptodate: [1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 430; DEBUG: Attempt to collect UpToDate information device with minor: [1] exited with: [0]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output:     ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb  3 17:48:13 wsguardian2 rhcs_fence: 432; DEBUG: UpToDate: [1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output:  1: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C s-----
Feb  3 17:48:13 wsguardian2 rhcs_fence: 418; DEBUG: output:     ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
Feb  3 17:48:13 wsguardian2 rhcs_fence: 367; DEBUG: shell call: [/usr/sbin/cman_tool -a -F id,name,type,addr nodes]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 430; DEBUG: Attempt to collect UpToDate information device with minor: [0] exited with: [0]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 432; DEBUG: UpToDate: [1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 367; DEBUG: shell call: [/usr/sbin/cman_tool -a -F id,name,type,addr nodes]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 374; DEBUG: id: [1], name: [wsguardian1], member: [M], address: [192.168.201.43]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 380; DEBUG: output: 1 wsguardian1 M 192.168.201.43 
Feb  3 17:48:13 wsguardian2 rhcs_fence: 374; DEBUG: id: [2], name: [wsguardian2], member: [M], address: [192.168.201.180]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 380; DEBUG: output: 2 wsguardian2 M 192.168.201.180 
Feb  3 17:48:13 wsguardian2 rhcs_fence: 384; DEBUG: Attempt to gather cluster member information exited with: [0]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 343; I have identified my target: [wsguardian1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 374; DEBUG: id: [1], name: [wsguardian1], member: [M], address: [192.168.201.43]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 191; DEBUG: I am: [wsguardian2] and my id is: [2]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 380; DEBUG: output: 1 wsguardian1 M 192.168.201.43 
Feb  3 17:48:13 wsguardian2 rhcs_fence: 217; Delaying for: [9] seconds to avoid dual-fencing...
Feb  3 17:48:13 wsguardian2 rhcs_fence: 374; DEBUG: id: [2], name: [wsguardian2], member: [M], address: [192.168.201.180]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 380; DEBUG: output: 2 wsguardian2 M 192.168.201.180 
Feb  3 17:48:13 wsguardian2 rhcs_fence: 384; DEBUG: Attempt to gather cluster member information exited with: [0]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 343; I have identified my target: [wsguardian1]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 191; DEBUG: I am: [wsguardian2] and my id is: [2]
Feb  3 17:48:13 wsguardian2 rhcs_fence: 217; Delaying for: [9] seconds to avoid dual-fencing...
Feb  3 17:48:22 wsguardian2 rhcs_fence: 219; DEBUG: Right then, break over.
Feb  3 17:48:22 wsguardian2 rhcs_fence: 301; Target node: [wsguardian1] is a cluster member, attempting to eject.
Feb  3 17:48:22 wsguardian2 rhcs_fence: 219; DEBUG: Right then, break over.
Feb  3 17:48:22 wsguardian2 rhcs_fence: 304; DEBUG: shell call: [/usr/sbin/cman_tool kill -f wsguardian1]
Feb  3 17:48:22 wsguardian2 rhcs_fence: 301; Target node: [wsguardian1] is a cluster member, attempting to eject.
Feb  3 17:48:22 wsguardian2 rhcs_fence: 304; DEBUG: shell call: [/usr/sbin/cman_tool kill -f wsguardian1]
Feb  3 17:48:22 wsguardian2 rhcs_fence: 313; DEBUG: Attempt to force-remove node: [wsguardian1] exited with: [256]
Feb  3 17:48:22 wsguardian2 rhcs_fence: 243; Fencing target: [wsguardian1]...
Feb  3 17:48:22 wsguardian2 rhcs_fence: 313; DEBUG: Attempt to force-remove node: [wsguardian1] exited with: [256]
Feb  3 17:48:22 wsguardian2 rhcs_fence: 247; DEBUG: shell call: [/usr/sbin/fence_node -v wsguardian1]
Feb  3 17:48:22 wsguardian2 rhcs_fence: 243; Fencing target: [wsguardian1]...
Feb  3 17:48:22 wsguardian2 rhcs_fence: 247; DEBUG: shell call: [/usr/sbin/fence_node -v wsguardian1]



cluster.conf
<?xml version="1.0"?>
<cluster config_version="2" name="WSGClust">
       	<cman expected_votes="1" two_node="1"/>
       	<clusternodes>
               	<clusternode name="wsguardian1" nodeid="1">
                       	<fence>
                               	<method name="wsguardian1_ipmi">
                                       	<device name="ipmi1" action="reboot"/>
                               	</method>
                       	</fence>
               	</clusternode>
               	<clusternode name="wsguardian2" nodeid="2">
                       	<fence>
                               	<method name="wsguardian2_ipmi">
                                       	<device name="ipmi2" action="reboot"/>
                               	</method>
                       	</fence>
               	</clusternode>
       	</clusternodes>
       	<fencedevices>
               	<fencedevice agent="fence_ipmilan" ipaddr="192.168.201.220" lanplus="1" login="ADMIN" name="ipmi1" passwd="easy"/>
               	<fencedevice agent="fence_ipmilan" ipaddr="192.168.201.186" lanplus="1" login="ADMIN" name="ipmi2" passwd="easy"/>
       	</fencedevices>
	<fence_daemon clean_start="0" post_fail_delay="0" post_join_delay="30"/>
</cluster>

drbd.conf
global {
    usage-count no;
}
common {
    protocol C;

    syncer {
        rate 100M;
        al-extents 3389;
    }
 
    # This tells DRBD what policy to use when a fence is required.
    disk {

	    fencing		resource-and-stonith;
    }

    handlers {

        #outdate-peer	"/sbin/obliterate-peer.sh";
        outdate-peer	"/usr/sbin/rhcs_fence";
	
        split-brain "/usr/lib/drbd/notify-split-brain.sh kortux@gmail.com";

    }
  
    net {
        sndbuf-size 1024k;
	# Tell DRBD to allow dual-primary.
        allow-two-primaries;

	# Set the recover policy for split-brain recover when no device
	# in the resource was primary.
	after-sb-0pri	discard-zero-changes;

	# Now if one device was primary.
	after-sb-1pri	discard-secondary;

	# Finally, set the policy when both nodes were Primary. The
	# only viable option is 'disconnect', which tells DRBD to
	# simply tear-down the DRBD resource right away and wait for
	# the administrator to manually invalidate one side of the
	# resource.
	after-sb-2pri	disconnect;
    }
    startup {
        become-primary-on both;
    }
  
}
resource wsg_db {
    device /dev/drbd0;
    meta-disk internal;
    
 
    on wsguardian1 {
        address 192.168.253.1:7788;
	disk /dev/rootvg/wsg_data_lv;
    }
 
    on wsguardian2 {
        address 192.168.253.2:7788;
	disk /dev/rootvg/wsg_data_lv;
    }
}
 
resource wsg_config {
    device /dev/drbd1;
    meta-disk internal;

    on wsguardian1 {
        address 192.168.253.1:7789;
	disk /dev/rootvg/wsg_config_lv;
    }
 
    on wsguardian2 {
        address 192.168.253.2:7789;
	disk /dev/rootvg/wsg_config_lv;
    }
}