Given volfile:
+------------------------------------------------------------------------------+
  1: volume vms01-client-0
  2:     type protocol/client
  3:     option remote-host 192.168.111.3
  4:     option remote-subvolume /mnt/vms01
  5:     option transport-type tcp
  6: end-volume
  7:
  8: volume vms01-client-1
  9:     type protocol/client
 10:     option remote-host 192.168.111.5
 11:     option remote-subvolume /vms01
 12:     option transport-type tcp
 13: end-volume
 14:
 15: volume vms01-replicate-0
 16:     type cluster/replicate
 17:     subvolumes vms01-client-0 vms01-client-1
 18: end-volume
 19:
 20: volume vms01-write-behind
 21:     type performance/write-behind
 22:     subvolumes vms01-replicate-0
 23: end-volume
 24:
 25: volume vms01-read-ahead
 26:     type performance/read-ahead
 27:     subvolumes vms01-write-behind
 28: end-volume
 29:
 30: volume vms01-io-cache
 31:     type performance/io-cache
 32:     subvolumes vms01-read-ahead
 33: end-volume
 34:
 35: volume vms01-quick-read
 36:     type performance/quick-read
 37:     subvolumes vms01-io-cache
 38: end-volume
 39:
 40: volume vms01-stat-prefetch
 41:     type performance/stat-prefetch
 42:     subvolumes vms01-quick-read
 43: end-volume
 44:
 45: volume vms01
 46:     type debug/io-stats
 47:     subvolumes vms01-stat-prefetch
 48: end-volume

+------------------------------------------------------------------------------+
[2010-12-09 16:47:35.997501] I [client-handshake.c:1005:select_server_supported_programs] vms01-client-1: Using Program GlusterFS-3.1.0, Num (1298437), Version (310)
[2010-12-09 16:47:35.998017] I [client-handshake.c:841:client_setvolume_cbk] vms01-client-1: Connected to 192.168.111.5:24010, attached to remote volume '/vms01'.
[2010-12-09 16:47:35.998088] I [afr-common.c:2571:afr_notify] vms01-replicate-0: Subvolume 'vms01-client-1' came back up; going online.
[2010-12-09 16:47:36.7868] I [fuse-bridge.c:2817:fuse_init] glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.13 kernel 7.13
[2010-12-09 16:47:36.7932] I [client-handshake.c:1005:select_server_supported_programs] vms01-client-0: Using Program GlusterFS-3.1.0, Num (1298437), Version (310)
[2010-12-09 16:47:36.8644] I [client-handshake.c:841:client_setvolume_cbk] vms01-client-0: Connected to 192.168.111.3:24009, attached to remote volume '/mnt/vms01'.
[2010-12-09 16:47:36.8799] I [afr-common.c:819:afr_fresh_lookup_cbk] vms01-replicate-0: added root inode
[2010-12-09 16:49:44.805392] I [afr-common.c:716:afr_lookup_done] vms01-replicate-0: background  data self-heal triggered. path: /vz/private/107/var/cache/apt/pkgcache.bin
[2010-12-09 16:49:45.20681] I [afr-self-heal-algorithm.c:976:sh_diff_loop_driver] vms01-replicate-0: diff  data self-heal on /vz/private/107/var/cache/apt/pkgcache.bin: 3 blocks of 192 were different (1.56%)
[2010-12-09 16:49:45.23655] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] vms01-replicate-0: background  data self-heal completed on /vz/private/107/var/cache/apt/pkgcache.bin
[2010-12-09 16:50:58.310014] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(WRITE(13)) called at 2010-12-09 16:50:58.123334
[2010-12-09 16:50:58.313403] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(WRITE(13)) called at 2010-12-09 16:50:58.123605
[2010-12-09 16:50:58.314237] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(FINODELK(30)) called at 2010-12-09 16:50:58.124435
[2010-12-09 16:50:58.314391] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(WRITE(13)) called at 2010-12-09 16:50:58.124860
[2010-12-09 16:50:58.315230] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(WRITE(13)) called at 2010-12-09 16:50:58.125078
[2010-12-09 16:50:58.316001] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(WRITE(13)) called at 2010-12-09 16:50:58.125324
[2010-12-09 16:50:58.318623] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(FINODELK(30)) called at 2010-12-09 16:50:58.131183
[2010-12-09 16:50:58.318736] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(FINODELK(30)) called at 2010-12-09 16:50:58.142805
[2010-12-09 16:50:58.318815] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(FINODELK(30)) called at 2010-12-09 16:50:58.153990
[2010-12-09 16:50:58.318890] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(FINODELK(30)) called at 2010-12-09 16:50:58.165103
[2010-12-09 16:50:58.319034] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(FINODELK(30)) called at 2010-12-09 16:50:58.176276
[2010-12-09 16:50:58.319108] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(FINODELK(30)) called at 2010-12-09 16:50:58.187359
[2010-12-09 16:50:58.319182] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(FINODELK(30)) called at 2010-12-09 16:50:58.198406
[2010-12-09 16:50:58.319291] E [rpc-clnt.c:338:saved_frames_unwind] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x77) [0x7f80fd5a6d17] (-->/usr/local/lib/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x7e) [0x7f80fd5a64ae] (-->/usr/local/lib/libgfrpc.so.0(saved_frames_destroy+0xe) [0x7f80fd5a640e]))) rpc-clnt: forced unwinding frame type(GlusterFS 3.1) op(FINODELK(30)) called at 2010-12-09 16:50:58.210484
[2010-12-09 16:50:58.319368] I [client.c:1587:client_rpc_notify] vms01-client-0: disconnected
[2010-12-09 16:50:58.319836] I [client-handshake.c:1005:select_server_supported_programs] vms01-client-0: Using Program GlusterFS-3.1.0, Num (1298437), Version (310)
[2010-12-09 16:50:58.320431] I [client-handshake.c:841:client_setvolume_cbk] vms01-client-0: Connected to 192.168.111.3:24009, attached to remote volume '/mnt/vms01'.
[2010-12-09 16:50:58.320492] I [client-handshake.c:710:client_post_handshake] vms01-client-0: 195 fds open - Delaying child_up until they are re-opened
[2010-12-09 16:50:58.328844] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/sbin/init succeeded (0)
[2010-12-09 16:50:58.329051] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/var/log/init.log succeeded (0)
[2010-12-09 16:50:58.329204] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/lib/libsepol.so.1 succeeded (0)
[2010-12-09 16:50:58.329352] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/lib/libc-2.7.so succeeded (0)
[2010-12-09 16:50:58.329496] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/lib/ld-2.7.so succeeded (0)
[2010-12-09 16:50:58.329640] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/lib/libselinux.so.1 succeeded (0)
[2010-12-09 16:50:58.329782] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/sbin/init-logger succeeded (0)
[2010-12-09 16:50:58.329928] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/lib/libdl-2.7.so succeeded (0)
[2010-12-09 16:50:58.330077] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/lib/ld-2.7.so succeeded (0)
[2010-12-09 16:50:58.330221] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/sbin/portmap succeeded (0)
[2010-12-09 16:50:58.330367] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/lib/libc-2.7.so succeeded (0)
[2010-12-09 16:50:58.330514] I [client-handshake.c:407:client3_1_reopen_cbk] vms01-client-0: reopen on /vz/private/107/lib/libnsl-2.7.so succeeded (0)
pending frames:
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)
frame : type(1) op(WRITE)

patchset: v3.1.1-9-g147b20c
signal received: 11
time of crash: 2010-12-09 16:50:58
configuration details:
argp 1
backtrace 1
dlfcn 1
fdatasync 1
libpthread 1
llistxattr 1
setfsid 1
spinlock 1
epoll.h 1
xattr.h 1
st_atim.tv_nsec 1
package-string: glusterfs 3.1.2qa1
/lib/libc.so.6[0x7f80fcc41f60]
/lib/libc.so.6(strlen+0x10)[0x7f80fcc8b090]
/lib/libc.so.6(_IO_vfprintf+0x38f0)[0x7f80fcc57ae0]
/lib/libc.so.6(vasprintf+0xbf)[0x7f80fcc7cf4f]
/usr/local/lib/libglusterfs.so.0(_gf_log+0x337)[0x7f80fd7c9117]
/usr/local/lib/glusterfs/3.1.2qa1/xlator/protocol/client.so(client3_1_reopen_cbk+0x347)[0x7f80fb432d97]
/usr/local/lib/libgfrpc.so.0(rpc_clnt_handle_reply+0xa4)[0x7f80fd5a6b24]
/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0xcd)[0x7f80fd5a6d6d]
/usr/local/lib/libgfrpc.so.0(rpc_transport_notify+0x2e)[0x7f80fd5a1dce]
/usr/local/lib/glusterfs/3.1.2qa1/rpc-transport/socket.so(socket_event_poll_in+0x3f)[0x7f80fb854e4f]
/usr/local/lib/glusterfs/3.1.2qa1/rpc-transport/socket.so(socket_event_handler+0x11c)[0x7f80fb854f8c]
/usr/local/lib/libglusterfs.so.0[0x7f80fd7e71f4]
/usr/local/sbin/glusterfs(main+0x242)[0x4054a2]
/lib/libc.so.6(__libc_start_main+0xe6)[0x7f80fcc2e1a6]
/usr/local/sbin/glusterfs[0x403159]
---------