[2018-12-12 16:32:58.690803] I [MSGID: 100030] [glusterfsd.c:2454:main] 0-/usr/sbin/glusterfs: Started running /usr/sbin/glusterfs version 3.8.8 (args: /usr/sbin/glusterfs --log-file=gluster.log --xlator-option=str01-cl-vol.master-key=/data/glusterfs/private/str01-cl.vol-encryption.master-key --volfile-server=localhost --volfile-id=/str01-cl-vol /mnt)
[2018-12-12 16:32:58.698053] I [MSGID: 101190] [event-epoll.c:628:event_dispatch_epoll_worker] 0-epoll: Started thread with index 1
[2018-12-12 16:32:58.700436] I [MSGID: 101173] [graph.c:269:gf_add_cmdline_options] 0-str01-cl-vol: adding option 'master-key' for volume 'str01-cl-vol' with value '/data/glusterfs/private/str01-cl.vol-encryption.master-key'
[2018-12-12 16:32:58.700986] I [crypt.c:4459:init] 0-crypt: crypt xlator loaded
[2018-12-12 16:32:58.717656] I [MSGID: 101190] [event-epoll.c:628:event_dispatch_epoll_worker] 0-epoll: Started thread with index 2
[2018-12-12 16:32:58.720121] W [MSGID: 101174] [graph.c:360:_log_if_unknown_option] 0-str01-cl-vol: option 'master-key' is not recognized
[2018-12-12 16:32:58.720284] I [MSGID: 114020] [client.c:2356:notify] 0-str01-cl-vol-client-0: parent translators are ready, attempting connect on transport
[2018-12-12 16:32:58.720680] I [MSGID: 114020] [client.c:2356:notify] 0-str01-cl-vol-client-1: parent translators are ready, attempting connect on transport
[2018-12-12 16:32:58.720905] I [MSGID: 114020] [client.c:2356:notify] 0-str01-cl-vol-client-2: parent translators are ready, attempting connect on transport
[2018-12-12 16:32:58.721007] I [rpc-clnt.c:1965:rpc_clnt_reconfig] 0-str01-cl-vol-client-0: changing port to 49154 (from 0)
[2018-12-12 16:32:58.721128] I [MSGID: 114020] [client.c:2356:notify] 0-str01-cl-vol-client-3: parent translators are ready, attempting connect on transport
[2018-12-12 16:32:58.721290] I [MSGID: 114020] [client.c:2356:notify] 0-str01-cl-vol-client-4: parent translators are ready, attempting connect on transport
[2018-12-12 16:32:58.721437] I [MSGID: 114020] [client.c:2356:notify] 0-str01-cl-vol-client-5: parent translators are ready, attempting connect on transport
Final graph:
+------------------------------------------------------------------------------+
  1: volume str01-cl-vol-client-0
  2:     type protocol/client
  3:     option ping-timeout 42
  4:     option remote-host str01
  5:     option remote-subvolume /data/glusterfs/brick1/str01-cl-vol
  6:     option transport-type socket
  7:     option transport.address-family inet
  8:     option username 93644120-cb29-4e20-95d8-e12ae91d69a4
  9:     option password 5550c3dc-1dbd-4748-8639-364189453be8
 10:     option send-gids true
 11: end-volume
 12:
 13: volume str01-cl-vol-client-1
 14:     type protocol/client
 15:     option ping-timeout 42
 16:     option remote-host str02
 17:     option remote-subvolume /data/glusterfs/brick1/str01-cl-vol
 18:     option transport-type socket
 19:     option transport.address-family inet
 20:     option username 93644120-cb29-4e20-95d8-e12ae91d69a4
 21:     option password 5550c3dc-1dbd-4748-8639-364189453be8
 22:     option send-gids true
 23: end-volume
 24:
 25: volume str01-cl-vol-client-2
 26:     type protocol/client
[2018-12-12 16:32:58.721644] I [MSGID: 114057] [client-handshake.c:1447:select_server_supported_programs] 0-str01-cl-vol-client-0: Using Program GlusterFS 3.3, Num (1298437), Version (330)
 27:     option ping-timeout 42
 28:     option remote-host str03
 29:     option remote-subvolume /data/glusterfs/brick1/str01-cl-vol
 30:     option transport-type socket
 31:     option transport.address-family inet
 32:     option username 93644120-cb29-4e20-95d8-e12ae91d69a4
 33:     option password 5550c3dc-1dbd-4748-8639-364189453be8
 34:     option send-gids true
 35: end-volume
 36:
 37: volume str01-cl-vol-disperse-0
 38:     type cluster/disperse
 39:     option redundancy 1
 40:     subvolumes str01-cl-vol-client-0 str01-cl-vol-client-1 str01-cl-vol-client-2
 41: end-volume
 42:
 43: volume str01-cl-vol-client-3
 44:     type protocol/client
 45:     option ping-timeout 42
 46:     option remote-host str04
 47:     option remote-subvolume /data/glusterfs/brick1/str01-cl-vol
 48:     option transport-type socket
 49:     option transport.address-family inet
 50:     option username 93644120-cb29-4e20-95d8-e12ae91d69a4
 51:     option password 5550c3dc-1dbd-4748-8639-364189453be8
 52:     option send-gids true
 53: end-volume
 54:
 55: volume str01-cl-vol-client-4
 56:     type protocol/client
 57:     option ping-timeout 42
 58:     option remote-host str05
 59:     option remote-subvolume /data/glusterfs/brick1/str01-cl-vol
 60:     option transport-type socket
 61:     option transport.address-family inet
 62:     option username 93644120-cb29-4e20-95d8-e12ae91d69a4
 63:     option password 5550c3dc-1dbd-4748-8639-364189453be8
 64:     option send-gids true
 65: end-volume
 66:
 67: volume str01-cl-vol-client-5
 68:     type protocol/client
 69:     option ping-timeout 42
 70:     option remote-host str06
 71:     option remote-subvolume /data/glusterfs/brick1/str01-cl-vol
 72:     option transport-type socket
 73:     option transport.address-family inet
 74:     option username 93644120-cb29-4e20-95d8-e12ae91d69a4
 75:     option password 5550c3dc-1dbd-4748-8639-364189453be8
 76:     option send-gids true
 77: end-volume
 78:
 79: volume str01-cl-vol-disperse-1
 80:     type cluster/disperse
 81:     option redundancy 1
 82:     subvolumes str01-cl-vol-client-3 str01-cl-vol-client-4 str01-cl-vol-client-5
 83: end-volume
 84:
 85: volume str01-cl-vol-dht
 86:     type cluster/distribute
 87:     option lock-migration off
 88:     subvolumes str01-cl-vol-disperse-0 str01-cl-vol-disperse-1
 89: end-volume
 90:
 91: volume str01-cl-vol-crypt
 92:     type encryption/crypt
 93:     option master-key /data/glusterfs/private/str01-cl.vol-encryption.master-key
 94:     option data-key-size 512
 95:     subvolumes str01-cl-vol-dht
 96: end-volume
 97:
 98: volume str01-cl-vol-read-ahead
 99:     type performance/read-ahead
100:     subvolumes str01-cl-vol-crypt
101: end-volume
102:
103: volume str01-cl-vol-readdir-ahead
104:     type performance/readdir-ahead
105:     subvolumes str01-cl-vol-read-ahead
106: end-volume
107:
108: volume str01-cl-vol-io-cache
109:     type performance/io-cache
110:     subvolumes str01-cl-vol-readdir-ahead
111: end-volume
112:
113: volume str01-cl-vol-md-cache
114:     type performance/md-cache
115:     subvolumes str01-cl-vol-io-cache
116: end-volume
117:
118: volume str01-cl-vol
119:     type debug/io-stats
120:     option master-key /data/glusterfs/private/str01-cl.vol-encryption.master-key
121:     option log-level INFO
122:     option latency-measurement off
123:     option count-fop-hits off
124:     subvolumes str01-cl-vol-md-cache
125: end-volume
126:
127: volume meta-autoload
128:     type meta
129:     subvolumes str01-cl-vol
130: end-volume
131:
+------------------------------------------------------------------------------+
[2018-12-12 16:32:58.722027] I [MSGID: 114046] [client-handshake.c:1223:client_setvolume_cbk] 0-str01-cl-vol-client-0: Connected to str01-cl-vol-client-0, attached to remote volume '/data/glusterfs/brick1/str01-cl-vol'.
[2018-12-12 16:32:58.722042] I [MSGID: 114047] [client-handshake.c:1234:client_setvolume_cbk] 0-str01-cl-vol-client-0: Server and Client lk-version numbers are not same, reopening the fds
[2018-12-12 16:32:58.722078] I [rpc-clnt.c:1965:rpc_clnt_reconfig] 0-str01-cl-vol-client-1: changing port to 49153 (from 0)
[2018-12-12 16:32:58.722144] I [MSGID: 114035] [client-handshake.c:202:client_set_lk_version_cbk] 0-str01-cl-vol-client-0: Server lk version = 1
[2018-12-12 16:32:58.722327] I [rpc-clnt.c:1965:rpc_clnt_reconfig] 0-str01-cl-vol-client-2: changing port to 49153 (from 0)
[2018-12-12 16:32:58.722354] I [rpc-clnt.c:1965:rpc_clnt_reconfig] 0-str01-cl-vol-client-3: changing port to 49153 (from 0)
[2018-12-12 16:32:58.722553] I [rpc-clnt.c:1965:rpc_clnt_reconfig] 0-str01-cl-vol-client-4: changing port to 49152 (from 0)
[2018-12-12 16:32:58.722731] I [rpc-clnt.c:1965:rpc_clnt_reconfig] 0-str01-cl-vol-client-5: changing port to 49153 (from 0)
[2018-12-12 16:32:58.722841] I [MSGID: 114057] [client-handshake.c:1447:select_server_supported_programs] 0-str01-cl-vol-client-1: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2018-12-12 16:32:58.723113] I [MSGID: 114057] [client-handshake.c:1447:select_server_supported_programs] 0-str01-cl-vol-client-3: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2018-12-12 16:32:58.723161] I [MSGID: 114057] [client-handshake.c:1447:select_server_supported_programs] 0-str01-cl-vol-client-2: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2018-12-12 16:32:58.723363] I [MSGID: 114046] [client-handshake.c:1223:client_setvolume_cbk] 0-str01-cl-vol-client-1: Connected to str01-cl-vol-client-1, attached to remote volume '/data/glusterfs/brick1/str01-cl-vol'.
[2018-12-12 16:32:58.723376] I [MSGID: 114047] [client-handshake.c:1234:client_setvolume_cbk] 0-str01-cl-vol-client-1: Server and Client lk-version numbers are not same, reopening the fds
[2018-12-12 16:32:58.723434] I [MSGID: 114057] [client-handshake.c:1447:select_server_supported_programs] 0-str01-cl-vol-client-4: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2018-12-12 16:32:58.723665] I [MSGID: 114046] [client-handshake.c:1223:client_setvolume_cbk] 0-str01-cl-vol-client-3: Connected to str01-cl-vol-client-3, attached to remote volume '/data/glusterfs/brick1/str01-cl-vol'.
[2018-12-12 16:32:58.723680] I [MSGID: 114047] [client-handshake.c:1234:client_setvolume_cbk] 0-str01-cl-vol-client-3: Server and Client lk-version numbers are not same, reopening the fds
[2018-12-12 16:32:58.723733] I [MSGID: 114035] [client-handshake.c:202:client_set_lk_version_cbk] 0-str01-cl-vol-client-1: Server lk version = 1
[2018-12-12 16:32:58.723739] I [MSGID: 114046] [client-handshake.c:1223:client_setvolume_cbk] 0-str01-cl-vol-client-2: Connected to str01-cl-vol-client-2, attached to remote volume '/data/glusterfs/brick1/str01-cl-vol'.
[2018-12-12 16:32:58.723754] I [MSGID: 114047] [client-handshake.c:1234:client_setvolume_cbk] 0-str01-cl-vol-client-2: Server and Client lk-version numbers are not same, reopening the fds
[2018-12-12 16:32:58.723788] I [MSGID: 122061] [ec.c:315:ec_up] 0-str01-cl-vol-disperse-0: Going UP
[2018-12-12 16:32:58.724344] I [MSGID: 114057] [client-handshake.c:1447:select_server_supported_programs] 0-str01-cl-vol-client-5: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2018-12-12 16:32:58.724364] I [MSGID: 114035] [client-handshake.c:202:client_set_lk_version_cbk] 0-str01-cl-vol-client-3: Server lk version = 1
[2018-12-12 16:32:58.724393] I [MSGID: 114035] [client-handshake.c:202:client_set_lk_version_cbk] 0-str01-cl-vol-client-2: Server lk version = 1
[2018-12-12 16:32:58.724398] I [MSGID: 114046] [client-handshake.c:1223:client_setvolume_cbk] 0-str01-cl-vol-client-4: Connected to str01-cl-vol-client-4, attached to remote volume '/data/glusterfs/brick1/str01-cl-vol'.
[2018-12-12 16:32:58.724413] I [MSGID: 114047] [client-handshake.c:1234:client_setvolume_cbk] 0-str01-cl-vol-client-4: Server and Client lk-version numbers are not same, reopening the fds
[2018-12-12 16:32:58.724915] I [MSGID: 114046] [client-handshake.c:1223:client_setvolume_cbk] 0-str01-cl-vol-client-5: Connected to str01-cl-vol-client-5, attached to remote volume '/data/glusterfs/brick1/str01-cl-vol'.
, reopening the fds
[2018-12-12 16:32:58.724979] I [MSGID: 122061] [ec.c:315:ec_up] 0-str01-cl-vol-disperse-1: Going UP
[2018-12-12 16:32:58.725201] I [MSGID: 114035] [client-handshake.c:202:client_set_lk_version_cbk] 0-str01-cl-vol-client-4: Server lk version = 1
[2018-12-12 16:32:58.730467] I [MSGID: 114035] [client-handshake.c:202:client_set_lk_version_cbk] 0-str01-cl-vol-client-5: Server lk version = 1
[2018-12-12 16:32:58.730564] I [fuse-bridge.c:4153:fuse_init] 0-glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.24 kernel 7.26
[2018-12-12 16:32:58.730592] I [fuse-bridge.c:4838:fuse_graph_sync] 0-fuse: switched to graph 0
[2018-12-12 16:32:58.732929] E [mem-pool.c:579:mem_put] (-->/usr/lib/x86_64-linux-gnu/glusterfs/3.8.8/xlator/mount/fuse.so(+0x1bed3) [0x7f133c7e4ed3] -->/usr/lib/x86_64-linux-gnu/glusterfs/3.8.8/xlator/mount/fuse.so(+0x1a70a) [0x7f133c7e370a] -->/usr/lib/x86_64-linux-gnu/libglusterfs.so.0(mem_put+0x190) [0x7f133fbe59c0] ) 0-mem-pool: mem-pool ptr is NULL
[2018-12-12 16:32:58.733780] E [mem-pool.c:579:mem_put] (-->/usr/lib/x86_64-linux-gnu/glusterfs/3.8.8/xlator/mount/fuse.so(+0xa112) [0x7f133c7d3112] -->/usr/lib/x86_64-linux-gnu/glusterfs/3.8.8/xlator/mount/fuse.so(+0x9f9a) [0x7f133c7d2f9a] -->/usr/lib/x86_64-linux-gnu/libglusterfs.so.0(mem_put+0x190) [0x7f133fbe59c0] ) 0-mem-pool: mem-pool ptr is NULL