root@gluster9a:/var/log/glusterfs# cat glfsheal-gw-runqueues.log
[2022-05-26 08:51:50.566228 +0000] I [io-stats.c:3708:ios_sample_buf_size_configure] 0-gw-runqueues: Configure ios_sample_buf  size is 1024 because ios_sample_interval is 0
[2022-05-26 08:51:50.569475 +0000] I [MSGID: 104045] [glfs-primary.c:81:notify] 0-gfapi: New graph coming up [{graph-uuid=676c7573-7465-7239-612d-32343039312d}, {id=0}]
[2022-05-26 08:51:50.569877 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-0: parent translators are ready, attempting connect on transport []
[2022-05-26 08:51:50.574606 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-0: changing port to 49155 (from 0)
[2022-05-26 08:51:50.574641 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-0: intentional socket shutdown(9)
[2022-05-26 08:51:50.575223 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-0: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:51:50.575700 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-1: parent translators are ready, attempting connect on transport []
[2022-05-26 08:51:50.575942 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-0: Connected, attached to remote volume [{conn-name=gw-runqueues-client-0}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:51:50.575968 +0000] I [MSGID: 108005] [afr-common.c:6065:__afr_handle_child_up_event] 0-gw-runqueues-replicate-0: Subvolume 'gw-runqueues-client-0' came back up; going online.
[2022-05-26 08:51:50.576359 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-2: parent translators are ready, attempting connect on transport []
Final graph:
+------------------------------------------------------------------------------+
  1: volume gw-runqueues-client-0
  2:     type protocol/client
  3:     option opversion 90000
  4:     option clnt-lk-version 1
  5:     option volfile-checksum 0
  6:     option volfile-key gw-runqueues
  7:     option client-version 9.1
  8:     option process-name gfapi.glfsheal
  9:     option process-uuid CTX_ID:9d19d9d7-0162-49df-80d8-666ddde69076-GRAPH_ID:0-PID:24091-HOST:gluster9a-PC_NAME:gw-runqueues-client-0-RECON_NO:-0
 10:     option fops-version 1298437
 11:     option ping-timeout 42
 12:     option remote-host gluster9a
 13:     option remote-subvolume /data/brick/gw-runqueues
 14:     option transport-type socket
 15:     option transport.address-family inet
 16:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 17:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 18:     option transport.socket.ssl-enabled off
 19:     option transport.tcp-user-timeout 0
 20:     option transport.socket.keepalive-time 20
 21:     option transport.socket.keepalive-interval 2
 22:     option transport.socket.keepalive-count 9
 23:     option strict-locks off
 24:     option send-gids true
 25: end-volume
 26:
 27: volume gw-runqueues-client-1
 28:     type protocol/client
 29:     option ping-timeout 42
 30:     option remote-host gluster9b
 31:     option remote-subvolume /data/brick/gw-runqueues
 32:     option transport-type socket
 33:     option transport.address-family inet
 34:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 35:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 36:     option transport.socket.ssl-enabled off
 37:     option transport.tcp-user-timeout 0
 38:     option transport.socket.keepalive-time 20
 39:     option transport.socket.keepalive-interval 2
 40:     option transport.socket.keepalive-count 9
 41:     option strict-locks off
 42:     option send-gids true
 43: end-volume
 44:
 45: volume gw-runqueues-client-2
 46:     type protocol/client
 47:     option ping-timeout 42
 48:     option remote-host gluster9c
 49:     option remote-subvolume /data/brick/gw-runqueues
 50:     option transport-type socket
 51:     option transport.address-family inet
 52:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 53:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 54:     option transport.socket.ssl-enabled off
 55:     option transport.tcp-user-timeout 0
 56:     option transport.socket.keepalive-time 20
 57:     option transport.socket.keepalive-interval 2
 58:     option transport.socket.keepalive-count 9
 59:     option strict-locks off
 60:     option send-gids true
 61: end-volume
 62:
 63: volume gw-runqueues-replicate-0
 64:     type cluster/replicate
 65:     option background-self-heal-count 0
 66:     option halo-enabled off
 67:     option afr-pending-xattr gw-runqueues-client-0,gw-runqueues-client-1,gw-runqueues-client-2
 68:     option volume-id fd06b0b6-3a87-44ed-bfbe-c243a7ff4e22
 69:     option granular-entry-heal on
 70:     option use-compound-fops off
 71:     option use-anonymous-inode yes
 72:     subvolumes gw-runqueues-client-0 gw-runqueues-client-1 gw-runqueues-client-2
 73: end-volume
 74:
 75: volume gw-runqueues-dht
 76:     type cluster/distribute
 77:     option lock-migration off
 78:     option force-migration off
 79:     subvolumes gw-runqueues-replicate-0
 80: end-volume
 81:
 82: volume gw-runqueues-utime
 83:     type features/utime
 84:     option noatime on
 85:     subvolumes gw-runqueues-dht
 86: end-volume
 87:
 88: volume gw-runqueues-write-behind
 89:     type performance/write-behind
 90:     subvolumes gw-runqueues-utime
 91: end-volume
 92:
 93: volume gw-runqueues-open-behind
 94:     type performance/open-behind
 95:     subvolumes gw-runqueues-write-behind
 96: end-volume
 97:
 98: volume gw-runqueues-quick-read
 99:     type performance/quick-read
100:     subvolumes gw-runqueues-open-behind
101: end-volume
102:
103: volume gw-runqueues-md-cache
104:     type performance/md-cache
105:     subvolumes gw-runqueues-quick-read
106: end-volume
107:
108: volume gw-runqueues
109:     type debug/io-stats
110:     option log-level INFO
111:     option threads 16
112:     option latency-measurement off
113:     option count-fop-hits off
114:     option global-threading off
115:     subvolumes gw-runqueues-md-cache
116: end-volume
117:
118: volume meta-autoload
119:     type meta
120:     subvolumes gw-runqueues
121: end-volume
122:
+------------------------------------------------------------------------------+
[2022-05-26 08:51:50.579384 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-2: changing port to 49155 (from 0)
[2022-05-26 08:51:50.579458 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-2: intentional socket shutdown(11)
[2022-05-26 08:51:50.579759 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-1: changing port to 49155 (from 0)
[2022-05-26 08:51:50.579802 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-1: intentional socket shutdown(9)
[2022-05-26 08:51:50.581562 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-2: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:51:50.582573 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-1: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:51:50.583099 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-2: Connected, attached to remote volume [{conn-name=gw-runqueues-client-2}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:51:50.583189 +0000] I [MSGID: 108002] [afr-common.c:6435:afr_notify] 0-gw-runqueues-replicate-0: Client-quorum is met
[2022-05-26 08:51:50.584292 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-1: Connected, attached to remote volume [{conn-name=gw-runqueues-client-1}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:51:50.587896 +0000] I [MSGID: 108031] [afr-common.c:3203:afr_local_discovery_cbk] 0-gw-runqueues-replicate-0: selecting local read_child gw-runqueues-client-0
[2022-05-26 08:51:50.588964 +0000] I [MSGID: 104041] [glfs-resolve.c:974:__glfs_active_subvol] 0-gw-runqueues: switched to graph [{subvol=676c7573-7465-7239-612d-32343039312d}, {id=0}]
[2022-05-26 08:52:01.356913 +0000] I [io-stats.c:3708:ios_sample_buf_size_configure] 0-gw-runqueues: Configure ios_sample_buf  size is 1024 because ios_sample_interval is 0
[2022-05-26 08:52:01.359157 +0000] I [MSGID: 104045] [glfs-primary.c:81:notify] 0-gfapi: New graph coming up [{graph-uuid=676c7573-7465-7239-612d-32343132312d}, {id=0}]
[2022-05-26 08:52:01.359197 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-0: parent translators are ready, attempting connect on transport []
[2022-05-26 08:52:01.360336 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-1: parent translators are ready, attempting connect on transport []
[2022-05-26 08:52:01.360515 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-0: changing port to 49155 (from 0)
[2022-05-26 08:52:01.360549 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-0: intentional socket shutdown(9)
[2022-05-26 08:52:01.361011 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-0: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:01.361689 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-0: Connected, attached to remote volume [{conn-name=gw-runqueues-client-0}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:01.361714 +0000] I [MSGID: 108005] [afr-common.c:6065:__afr_handle_child_up_event] 0-gw-runqueues-replicate-0: Subvolume 'gw-runqueues-client-0' came back up; going online.
[2022-05-26 08:52:01.361759 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-2: parent translators are ready, attempting connect on transport []
Final graph:
+------------------------------------------------------------------------------+
  1: volume gw-runqueues-client-0
  2:     type protocol/client
  3:     option opversion 90000
  4:     option clnt-lk-version 1
  5:     option volfile-checksum 0
  6:     option volfile-key gw-runqueues
  7:     option client-version 9.1
  8:     option process-name gfapi.glfsheal
  9:     option process-uuid CTX_ID:285063b9-af49-46ff-8f37-3877001866d5-GRAPH_ID:0-PID:24121-HOST:gluster9a-PC_NAME:gw-runqueues-client-0-RECON_NO:-0
 10:     option fops-version 1298437
 11:     option ping-timeout 42
 12:     option remote-host gluster9a
 13:     option remote-subvolume /data/brick/gw-runqueues
 14:     option transport-type socket
 15:     option transport.address-family inet
 16:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 17:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 18:     option transport.socket.ssl-enabled off
 19:     option transport.tcp-user-timeout 0
 20:     option transport.socket.keepalive-time 20
 21:     option transport.socket.keepalive-interval 2
 22:     option transport.socket.keepalive-count 9
 23:     option strict-locks off
 24:     option send-gids true
 25: end-volume
 26:
 27: volume gw-runqueues-client-1
 28:     type protocol/client
 29:     option ping-timeout 42
 30:     option remote-host gluster9b
 31:     option remote-subvolume /data/brick/gw-runqueues
 32:     option transport-type socket
 33:     option transport.address-family inet
 34:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 35:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 36:     option transport.socket.ssl-enabled off
 37:     option transport.tcp-user-timeout 0
 38:     option transport.socket.keepalive-time 20
 39:     option transport.socket.keepalive-interval 2
 40:     option transport.socket.keepalive-count 9
 41:     option strict-locks off
 42:     option send-gids true
 43: end-volume
 44:
 45: volume gw-runqueues-client-2
 46:     type protocol/client
 47:     option ping-timeout 42
 48:     option remote-host gluster9c
 49:     option remote-subvolume /data/brick/gw-runqueues
 50:     option transport-type socket
 51:     option transport.address-family inet
 52:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 53:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 54:     option transport.socket.ssl-enabled off
 55:     option transport.tcp-user-timeout 0
 56:     option transport.socket.keepalive-time 20
 57:     option transport.socket.keepalive-interval 2
 58:     option transport.socket.keepalive-count 9
 59:     option strict-locks off
 60:     option send-gids true
 61: end-volume
 62:
 63: volume gw-runqueues-replicate-0
 64:     type cluster/replicate
 65:     option background-self-heal-count 0
 66:     option halo-enabled off
 67:     option afr-pending-xattr gw-runqueues-client-0,gw-runqueues-client-1,gw-runqueues-client-2
 68:     option volume-id fd06b0b6-3a87-44ed-bfbe-c243a7ff4e22
 69:     option granular-entry-heal on
 70:     option use-compound-fops off
 71:     option use-anonymous-inode yes
 72:     subvolumes gw-runqueues-client-0 gw-runqueues-client-1 gw-runqueues-client-2
 73: end-volume
 74:
 75: volume gw-runqueues-dht
 76:     type cluster/distribute
 77:     option lock-migration off
 78:     option force-migration off
 79:     subvolumes gw-runqueues-replicate-0
 80: end-volume
 81:
 82: volume gw-runqueues-utime
 83:     type features/utime
 84:     option noatime on
 85:     subvolumes gw-runqueues-dht
 86: end-volume
 87:
 88: volume gw-runqueues-write-behind
 89:     type performance/write-behind
 90:     subvolumes gw-runqueues-utime
 91: end-volume
 92:
 93: volume gw-runqueues-open-behind
 94:     type performance/open-behind
 95:     subvolumes gw-runqueues-write-behind
 96: end-volume
 97:
 98: volume gw-runqueues-quick-read
 99:     type performance/quick-read
100:     subvolumes gw-runqueues-open-behind
101: end-volume
102:
103: volume gw-runqueues-md-cache
104:     type performance/md-cache
105:     subvolumes gw-runqueues-quick-read
106: end-volume
107:
108: volume gw-runqueues
109:     type debug/io-stats
110:     option log-level INFO
111:     option threads 16
112:     option latency-measurement off
113:     option count-fop-hits off
114:     option global-threading off
115:     subvolumes gw-runqueues-md-cache
116: end-volume
117:
118: volume meta-autoload
119:     type meta
120:     subvolumes gw-runqueues
121: end-volume
122:
+------------------------------------------------------------------------------+
[2022-05-26 08:52:01.364525 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-1: changing port to 49155 (from 0)
[2022-05-26 08:52:01.364581 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-1: intentional socket shutdown(9)
[2022-05-26 08:52:01.364911 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-2: changing port to 49155 (from 0)
[2022-05-26 08:52:01.365022 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-2: intentional socket shutdown(12)
[2022-05-26 08:52:01.367120 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-1: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:01.367294 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-2: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:01.369022 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-1: Connected, attached to remote volume [{conn-name=gw-runqueues-client-1}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:01.369126 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-2: Connected, attached to remote volume [{conn-name=gw-runqueues-client-2}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:01.369238 +0000] I [MSGID: 108002] [afr-common.c:6435:afr_notify] 0-gw-runqueues-replicate-0: Client-quorum is met
[2022-05-26 08:52:01.371365 +0000] I [MSGID: 108031] [afr-common.c:3203:afr_local_discovery_cbk] 0-gw-runqueues-replicate-0: selecting local read_child gw-runqueues-client-0
[2022-05-26 08:52:01.372902 +0000] I [MSGID: 104041] [glfs-resolve.c:974:__glfs_active_subvol] 0-gw-runqueues: switched to graph [{subvol=676c7573-7465-7239-612d-32343132312d}, {id=0}]
[2022-05-26 08:52:06.713177 +0000] I [io-stats.c:3708:ios_sample_buf_size_configure] 0-gw-runqueues: Configure ios_sample_buf  size is 1024 because ios_sample_interval is 0
[2022-05-26 08:52:06.716022 +0000] I [MSGID: 104045] [glfs-primary.c:81:notify] 0-gfapi: New graph coming up [{graph-uuid=676c7573-7465-7239-612d-32343134322d}, {id=0}]
[2022-05-26 08:52:06.716065 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-0: parent translators are ready, attempting connect on transport []
[2022-05-26 08:52:06.717659 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-0: changing port to 49155 (from 0)
[2022-05-26 08:52:06.717693 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-0: intentional socket shutdown(9)
[2022-05-26 08:52:06.718269 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-1: parent translators are ready, attempting connect on transport []
[2022-05-26 08:52:06.718365 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-0: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:06.719170 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-0: Connected, attached to remote volume [{conn-name=gw-runqueues-client-0}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:06.719200 +0000] I [MSGID: 108005] [afr-common.c:6065:__afr_handle_child_up_event] 0-gw-runqueues-replicate-0: Subvolume 'gw-runqueues-client-0' came back up; going online.
[2022-05-26 08:52:06.719667 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-2: parent translators are ready, attempting connect on transport []
Final graph:
+------------------------------------------------------------------------------+
  1: volume gw-runqueues-client-0
  2:     type protocol/client
  3:     option opversion 90000
  4:     option clnt-lk-version 1
  5:     option volfile-checksum 0
  6:     option volfile-key gw-runqueues
  7:     option client-version 9.1
  8:     option process-name gfapi.glfsheal
  9:     option process-uuid CTX_ID:c577a909-32ca-4603-b976-a1bf06a0fd94-GRAPH_ID:0-PID:24142-HOST:gluster9a-PC_NAME:gw-runqueues-client-0-RECON_NO:-0
 10:     option fops-version 1298437
 11:     option ping-timeout 42
 12:     option remote-host gluster9a
 13:     option remote-subvolume /data/brick/gw-runqueues
 14:     option transport-type socket
 15:     option transport.address-family inet
 16:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 17:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 18:     option transport.socket.ssl-enabled off
 19:     option transport.tcp-user-timeout 0
 20:     option transport.socket.keepalive-time 20
 21:     option transport.socket.keepalive-interval 2
 22:     option transport.socket.keepalive-count 9
 23:     option strict-locks off
 24:     option send-gids true
 25: end-volume
 26:
 27: volume gw-runqueues-client-1
 28:     type protocol/client
 29:     option ping-timeout 42
 30:     option remote-host gluster9b
 31:     option remote-subvolume /data/brick/gw-runqueues
 32:     option transport-type socket
 33:     option transport.address-family inet
 34:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 35:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 36:     option transport.socket.ssl-enabled off
 37:     option transport.tcp-user-timeout 0
 38:     option transport.socket.keepalive-time 20
 39:     option transport.socket.keepalive-interval 2
 40:     option transport.socket.keepalive-count 9
 41:     option strict-locks off
 42:     option send-gids true
 43: end-volume
 44:
 45: volume gw-runqueues-client-2
 46:     type protocol/client
 47:     option ping-timeout 42
 48:     option remote-host gluster9c
 49:     option remote-subvolume /data/brick/gw-runqueues
 50:     option transport-type socket
 51:     option transport.address-family inet
 52:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 53:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 54:     option transport.socket.ssl-enabled off
 55:     option transport.tcp-user-timeout 0
 56:     option transport.socket.keepalive-time 20
 57:     option transport.socket.keepalive-interval 2
 58:     option transport.socket.keepalive-count 9
 59:     option strict-locks off
 60:     option send-gids true
 61: end-volume
 62:
 63: volume gw-runqueues-replicate-0
 64:     type cluster/replicate
 65:     option background-self-heal-count 0
 66:     option halo-enabled off
 67:     option afr-pending-xattr gw-runqueues-client-0,gw-runqueues-client-1,gw-runqueues-client-2
 68:     option volume-id fd06b0b6-3a87-44ed-bfbe-c243a7ff4e22
 69:     option granular-entry-heal on
 70:     option use-compound-fops off
 71:     option use-anonymous-inode yes
 72:     subvolumes gw-runqueues-client-0 gw-runqueues-client-1 gw-runqueues-client-2
 73: end-volume
 74:
 75: volume gw-runqueues-dht
 76:     type cluster/distribute
 77:     option lock-migration off
 78:     option force-migration off
 79:     subvolumes gw-runqueues-replicate-0
 80: end-volume
 81:
 82: volume gw-runqueues-utime
 83:     type features/utime
 84:     option noatime on
 85:     subvolumes gw-runqueues-dht
 86: end-volume
 87:
 88: volume gw-runqueues-write-behind
 89:     type performance/write-behind
 90:     subvolumes gw-runqueues-utime
 91: end-volume
 92:
 93: volume gw-runqueues-open-behind
 94:     type performance/open-behind
 95:     subvolumes gw-runqueues-write-behind
 96: end-volume
 97:
 98: volume gw-runqueues-quick-read
 99:     type performance/quick-read
100:     subvolumes gw-runqueues-open-behind
101: end-volume
102:
103: volume gw-runqueues-md-cache
104:     type performance/md-cache
105:     subvolumes gw-runqueues-quick-read
106: end-volume
107:
108: volume gw-runqueues
109:     type debug/io-stats
110:     option log-level INFO
111:     option threads 16
112:     option latency-measurement off
113:     option count-fop-hits off
114:     option global-threading off
115:     subvolumes gw-runqueues-md-cache
116: end-volume
117:
118: volume meta-autoload
119:     type meta
120:     subvolumes gw-runqueues
121: end-volume
122:
+------------------------------------------------------------------------------+
[2022-05-26 08:52:06.722622 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-2: changing port to 49155 (from 0)
[2022-05-26 08:52:06.722802 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-2: intentional socket shutdown(11)
[2022-05-26 08:52:06.722848 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-1: changing port to 49155 (from 0)
[2022-05-26 08:52:06.722928 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-1: intentional socket shutdown(9)
[2022-05-26 08:52:06.725347 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-1: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:06.725546 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-2: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:06.727014 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-2: Connected, attached to remote volume [{conn-name=gw-runqueues-client-2}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:06.727068 +0000] I [MSGID: 108002] [afr-common.c:6435:afr_notify] 0-gw-runqueues-replicate-0: Client-quorum is met
[2022-05-26 08:52:06.727308 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-1: Connected, attached to remote volume [{conn-name=gw-runqueues-client-1}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:06.730123 +0000] I [MSGID: 108031] [afr-common.c:3203:afr_local_discovery_cbk] 0-gw-runqueues-replicate-0: selecting local read_child gw-runqueues-client-0
[2022-05-26 08:52:06.730342 +0000] I [MSGID: 104041] [glfs-resolve.c:974:__glfs_active_subvol] 0-gw-runqueues: switched to graph [{subvol=676c7573-7465-7239-612d-32343134322d}, {id=0}]
[2022-05-26 08:52:47.955085 +0000] I [io-stats.c:3708:ios_sample_buf_size_configure] 0-gw-runqueues: Configure ios_sample_buf  size is 1024 because ios_sample_interval is 0
[2022-05-26 08:52:47.957291 +0000] I [MSGID: 104045] [glfs-primary.c:81:notify] 0-gfapi: New graph coming up [{graph-uuid=676c7573-7465-7239-612d-32343137352d}, {id=0}]
[2022-05-26 08:52:47.957329 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-0: parent translators are ready, attempting connect on transport []
[2022-05-26 08:52:47.958459 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-1: parent translators are ready, attempting connect on transport []
[2022-05-26 08:52:47.958807 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-0: changing port to 49155 (from 0)
[2022-05-26 08:52:47.958864 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-0: intentional socket shutdown(9)
[2022-05-26 08:52:47.959050 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-2: parent translators are ready, attempting connect on transport []
Final graph:
+------------------------------------------------------------------------------+
  1: volume gw-runqueues-client-0
  2:     type protocol/client
  3:     option ping-timeout 42
  4:     option remote-host gluster9a
  5:     option remote-subvolume /data/brick/gw-runqueues
  6:     option transport-type socket
  7:     option transport.address-family inet
  8:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
  9:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 10:     option transport.socket.ssl-enabled off
 11:     option transport.tcp-user-timeout 0
 12:     option transport.socket.keepalive-time 20
 13:     option transport.socket.keepalive-interval 2
 14:     option transport.socket.keepalive-count 9
 15:     option strict-locks off
 16:     option send-gids true
 17: end-volume
 18:
 19: volume gw-runqueues-client-1
 20:     type protocol/client
 21:     option ping-timeout 42
 22:     option remote-host gluster9b
 23:     option remote-subvolume /data/brick/gw-runqueues
 24:     option transport-type socket
 25:     option transport.address-family inet
 26:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 27:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 28:     option transport.socket.ssl-enabled off
 29:     option transport.tcp-user-timeout 0
 30:     option transport.socket.keepalive-time 20
 31:     option transport.socket.keepalive-interval 2
 32:     option transport.socket.keepalive-count 9
 33:     option strict-locks off
 34:     option send-gids true
 35: end-volume
 36:
 37: volume gw-runqueues-client-2
 38:     type protocol/client
 39:     option ping-timeout 42
 40:     option remote-host gluster9c
 41:     option remote-subvolume /data/brick/gw-runqueues
 42:     option transport-type socket
 43:     option transport.address-family inet
 44:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 45:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 46:     option transport.socket.ssl-enabled off
 47:     option transport.tcp-user-timeout 0
 48:     option transport.socket.keepalive-time 20
 49:     option transport.socket.keepalive-interval 2
 50:     option transport.socket.keepalive-count 9
 51:     option strict-locks off
 52:     option send-gids true
 53: end-volume
 54:
 55: volume gw-runqueues-replicate-0
 56:     type cluster/replicate
 57:     option background-self-heal-count 0
 58:     option halo-enabled off
 59:     option afr-pending-xattr gw-runqueues-client-0,gw-runqueues-client-1,gw-runqueues-client-2
 60:     option volume-id fd06b0b6-3a87-44ed-bfbe-c243a7ff4e22
 61:     option granular-entry-heal on
 62:     option use-compound-fops off
 63:     option use-anonymous-inode yes
 64:     subvolumes gw-runqueues-client-0 gw-runqueues-client-1 gw-runqueues-client-2
 65: end-volume
 66:
 67: volume gw-runqueues-dht
 68:     type cluster/distribute
 69:     option lock-migration off
 70:     option force-migration off
 71:     subvolumes gw-runqueues-replicate-0
 72: end-volume
 73:
 74: volume gw-runqueues-utime
 75:     type features/utime
 76:     option noatime on
 77:     subvolumes gw-runqueues-dht
 78: end-volume
 79:
 80: volume gw-runqueues-write-behind
 81:     type performance/write-behind
 82:     subvolumes gw-runqueues-utime
 83: end-volume
 84:
 85: volume gw-runqueues-open-behind
 86:     type performance/open-behind
 87:     subvolumes gw-runqueues-write-behind
 88: end-volume
 89:
 90: volume gw-runqueues-quick-read
 91:     type performance/quick-read
 92:     subvolumes gw-runqueues-open-behind
 93: end-volume
 94:
 95: volume gw-runqueues-md-cache
 96:     type performance/md-cache
 97:     subvolumes gw-runqueues-quick-read
 98: end-volume
 99:
100: volume gw-runqueues
101:     type debug/io-stats
102:     option log-level INFO
103:     option threads 16
104:     option latency-measurement off
105:     option count-fop-hits off
106:     option global-threading off
107:     subvolumes gw-runqueues-md-cache
108: end-volume
109:
110: volume meta-autoload
111:     type meta
112:     subvolumes gw-runqueues
113: end-volume
114:
+------------------------------------------------------------------------------+
[2022-05-26 08:52:47.959642 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-0: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:47.962060 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-0: Connected, attached to remote volume [{conn-name=gw-runqueues-client-0}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:47.962087 +0000] I [MSGID: 108005] [afr-common.c:6065:__afr_handle_child_up_event] 0-gw-runqueues-replicate-0: Subvolume 'gw-runqueues-client-0' came back up; going online.
[2022-05-26 08:52:47.962120 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-1: changing port to 49155 (from 0)
[2022-05-26 08:52:47.962221 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-1: intentional socket shutdown(10)
[2022-05-26 08:52:47.962245 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-2: changing port to 49155 (from 0)
[2022-05-26 08:52:47.962337 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-2: intentional socket shutdown(12)
[2022-05-26 08:52:47.964429 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-2: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:47.964783 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-1: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:47.966089 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-2: Connected, attached to remote volume [{conn-name=gw-runqueues-client-2}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:47.966190 +0000] I [MSGID: 108002] [afr-common.c:6435:afr_notify] 0-gw-runqueues-replicate-0: Client-quorum is met
[2022-05-26 08:52:47.966576 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-1: Connected, attached to remote volume [{conn-name=gw-runqueues-client-1}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:47.969507 +0000] I [MSGID: 108031] [afr-common.c:3203:afr_local_discovery_cbk] 0-gw-runqueues-replicate-0: selecting local read_child gw-runqueues-client-0
[2022-05-26 08:52:47.970383 +0000] I [MSGID: 104041] [glfs-resolve.c:974:__glfs_active_subvol] 0-gw-runqueues: switched to graph [{subvol=676c7573-7465-7239-612d-32343137352d}, {id=0}]
[2022-05-26 08:52:56.643452 +0000] I [io-stats.c:3708:ios_sample_buf_size_configure] 0-gw-runqueues: Configure ios_sample_buf  size is 1024 because ios_sample_interval is 0
[2022-05-26 08:52:56.645697 +0000] I [MSGID: 104045] [glfs-primary.c:81:notify] 0-gfapi: New graph coming up [{graph-uuid=676c7573-7465-7239-612d-32343139362d}, {id=0}]
[2022-05-26 08:52:56.645735 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-0: parent translators are ready, attempting connect on transport []
[2022-05-26 08:52:56.646771 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-1: parent translators are ready, attempting connect on transport []
[2022-05-26 08:52:56.647153 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-0: changing port to 49155 (from 0)
[2022-05-26 08:52:56.647200 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-0: intentional socket shutdown(9)
[2022-05-26 08:52:56.647333 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-2: parent translators are ready, attempting connect on transport []
Final graph:
+------------------------------------------------------------------------------+
  1: volume gw-runqueues-client-0
  2:     type protocol/client
  3:     option ping-timeout 42
  4:     option remote-host gluster9a
  5:     option remote-subvolume /data/brick/gw-runqueues
  6:     option transport-type socket
  7:     option transport.address-family inet
  8:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
  9:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 10:     option transport.socket.ssl-enabled off
 11:     option transport.tcp-user-timeout 0
 12:     option transport.socket.keepalive-time 20
 13:     option transport.socket.keepalive-interval 2
 14:     option transport.socket.keepalive-count 9
 15:     option strict-locks off
 16:     option send-gids true
 17: end-volume
 18:
 19: volume gw-runqueues-client-1
 20:     type protocol/client
 21:     option ping-timeout 42
 22:     option remote-host gluster9b
 23:     option remote-subvolume /data/brick/gw-runqueues
 24:     option transport-type socket
 25:     option transport.address-family inet
 26:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 27:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 28:     option transport.socket.ssl-enabled off
 29:     option transport.tcp-user-timeout 0
 30:     option transport.socket.keepalive-time 20
 31:     option transport.socket.keepalive-interval 2
 32:     option transport.socket.keepalive-count 9
 33:     option strict-locks off
 34:     option send-gids true
 35: end-volume
 36:
 37: volume gw-runqueues-client-2
 38:     type protocol/client
 39:     option ping-timeout 42
 40:     option remote-host gluster9c
 41:     option remote-subvolume /data/brick/gw-runqueues
 42:     option transport-type socket
 43:     option transport.address-family inet
 44:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 45:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 46:     option transport.socket.ssl-enabled off
 47:     option transport.tcp-user-timeout 0
 48:     option transport.socket.keepalive-time 20
 49:     option transport.socket.keepalive-interval 2
 50:     option transport.socket.keepalive-count 9
 51:     option strict-locks off
 52:     option send-gids true
 53: end-volume
 54:
 55: volume gw-runqueues-replicate-0
 56:     type cluster/replicate
 57:     option background-self-heal-count 0
 58:     option halo-enabled off
 59:     option afr-pending-xattr gw-runqueues-client-0,gw-runqueues-client-1,gw-runqueues-client-2
 60:     option volume-id fd06b0b6-3a87-44ed-bfbe-c243a7ff4e22
 61:     option granular-entry-heal on
 62:     option use-compound-fops off
 63:     option use-anonymous-inode yes
 64:     subvolumes gw-runqueues-client-0 gw-runqueues-client-1 gw-runqueues-client-2
 65: end-volume
 66:
 67: volume gw-runqueues-dht
 68:     type cluster/distribute
 69:     option lock-migration off
 70:     option force-migration off
 71:     subvolumes gw-runqueues-replicate-0
 72: end-volume
 73:
 74: volume gw-runqueues-utime
 75:     type features/utime
 76:     option noatime on
 77:     subvolumes gw-runqueues-dht
 78: end-volume
 79:
 80: volume gw-runqueues-write-behind
 81:     type performance/write-behind
 82:     subvolumes gw-runqueues-utime
 83: end-volume
 84:
 85: volume gw-runqueues-open-behind
 86:     type performance/open-behind
 87:     subvolumes gw-runqueues-write-behind
 88: end-volume
 89:
 90: volume gw-runqueues-quick-read
 91:     type performance/quick-read
 92:     subvolumes gw-runqueues-open-behind
 93: end-volume
 94:
 95: volume gw-runqueues-md-cache
 96:     type performance/md-cache
 97:     subvolumes gw-runqueues-quick-read
 98: end-volume
 99:
100: volume gw-runqueues
101:     type debug/io-stats
102:     option log-level INFO
103:     option threads 16
104:     option latency-measurement off
105:     option count-fop-hits off
106:     option global-threading off
107:     subvolumes gw-runqueues-md-cache
108: end-volume
109:
110: volume meta-autoload
111:     type meta
112:     subvolumes gw-runqueues
113: end-volume
114:
+------------------------------------------------------------------------------+
[2022-05-26 08:52:56.648693 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-0: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:56.650226 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-0: Connected, attached to remote volume [{conn-name=gw-runqueues-client-0}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:56.650250 +0000] I [MSGID: 108005] [afr-common.c:6065:__afr_handle_child_up_event] 0-gw-runqueues-replicate-0: Subvolume 'gw-runqueues-client-0' came back up; going online.
[2022-05-26 08:52:56.650429 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-2: changing port to 49155 (from 0)
[2022-05-26 08:52:56.650540 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-2: intentional socket shutdown(12)
[2022-05-26 08:52:56.650705 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-1: changing port to 49155 (from 0)
[2022-05-26 08:52:56.650765 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-1: intentional socket shutdown(10)
[2022-05-26 08:52:56.652929 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-2: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:56.653215 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-1: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:52:56.654955 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-2: Connected, attached to remote volume [{conn-name=gw-runqueues-client-2}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:56.655047 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-1: Connected, attached to remote volume [{conn-name=gw-runqueues-client-1}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:52:56.655140 +0000] I [MSGID: 108002] [afr-common.c:6435:afr_notify] 0-gw-runqueues-replicate-0: Client-quorum is met
[2022-05-26 08:52:56.657816 +0000] I [MSGID: 108031] [afr-common.c:3203:afr_local_discovery_cbk] 0-gw-runqueues-replicate-0: selecting local read_child gw-runqueues-client-0
[2022-05-26 08:52:56.658523 +0000] I [MSGID: 104041] [glfs-resolve.c:974:__glfs_active_subvol] 0-gw-runqueues: switched to graph [{subvol=676c7573-7465-7239-612d-32343139362d}, {id=0}]
[2022-05-26 08:53:00.905890 +0000] I [io-stats.c:3708:ios_sample_buf_size_configure] 0-gw-runqueues: Configure ios_sample_buf  size is 1024 because ios_sample_interval is 0
[2022-05-26 08:53:00.908136 +0000] I [MSGID: 104045] [glfs-primary.c:81:notify] 0-gfapi: New graph coming up [{graph-uuid=676c7573-7465-7239-612d-32343231372d}, {id=0}]
[2022-05-26 08:53:00.908186 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-0: parent translators are ready, attempting connect on transport []
[2022-05-26 08:53:00.909233 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-1: parent translators are ready, attempting connect on transport []
[2022-05-26 08:53:00.909579 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-0: changing port to 49155 (from 0)
[2022-05-26 08:53:00.909625 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-0: intentional socket shutdown(9)
[2022-05-26 08:53:00.909775 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-2: parent translators are ready, attempting connect on transport []
Final graph:
+------------------------------------------------------------------------------+
  1: volume gw-runqueues-client-0
  2:     type protocol/client
  3:     option ping-timeout 42
  4:     option remote-host gluster9a
  5:     option remote-subvolume /data/brick/gw-runqueues
  6:     option transport-type socket
  7:     option transport.address-family inet
  8:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
  9:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 10:     option transport.socket.ssl-enabled off
 11:     option transport.tcp-user-timeout 0
 12:     option transport.socket.keepalive-time 20
 13:     option transport.socket.keepalive-interval 2
 14:     option transport.socket.keepalive-count 9
 15:     option strict-locks off
 16:     option send-gids true
 17: end-volume
 18:
 19: volume gw-runqueues-client-1
 20:     type protocol/client
 21:     option ping-timeout 42
 22:     option remote-host gluster9b
 23:     option remote-subvolume /data/brick/gw-runqueues
 24:     option transport-type socket
 25:     option transport.address-family inet
 26:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 27:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 28:     option transport.socket.ssl-enabled off
 29:     option transport.tcp-user-timeout 0
 30:     option transport.socket.keepalive-time 20
 31:     option transport.socket.keepalive-interval 2
 32:     option transport.socket.keepalive-count 9
 33:     option strict-locks off
 34:     option send-gids true
 35: end-volume
 36:
 37: volume gw-runqueues-client-2
 38:     type protocol/client
 39:     option ping-timeout 42
 40:     option remote-host gluster9c
 41:     option remote-subvolume /data/brick/gw-runqueues
 42:     option transport-type socket
 43:     option transport.address-family inet
 44:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 45:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 46:     option transport.socket.ssl-enabled off
 47:     option transport.tcp-user-timeout 0
 48:     option transport.socket.keepalive-time 20
 49:     option transport.socket.keepalive-interval 2
 50:     option transport.socket.keepalive-count 9
 51:     option strict-locks off
 52:     option send-gids true
 53: end-volume
 54:
 55: volume gw-runqueues-replicate-0
 56:     type cluster/replicate
 57:     option background-self-heal-count 0
 58:     option halo-enabled off
 59:     option afr-pending-xattr gw-runqueues-client-0,gw-runqueues-client-1,gw-runqueues-client-2
 60:     option volume-id fd06b0b6-3a87-44ed-bfbe-c243a7ff4e22
 61:     option granular-entry-heal on
 62:     option use-compound-fops off
 63:     option use-anonymous-inode yes
 64:     subvolumes gw-runqueues-client-0 gw-runqueues-client-1 gw-runqueues-client-2
 65: end-volume
 66:
 67: volume gw-runqueues-dht
 68:     type cluster/distribute
 69:     option lock-migration off
 70:     option force-migration off
 71:     subvolumes gw-runqueues-replicate-0
 72: end-volume
 73:
 74: volume gw-runqueues-utime
 75:     type features/utime
 76:     option noatime on
 77:     subvolumes gw-runqueues-dht
 78: end-volume
 79:
 80: volume gw-runqueues-write-behind
 81:     type performance/write-behind
 82:     subvolumes gw-runqueues-utime
 83: end-volume
 84:
 85: volume gw-runqueues-open-behind
 86:     type performance/open-behind
 87:     subvolumes gw-runqueues-write-behind
 88: end-volume
 89:
 90: volume gw-runqueues-quick-read
 91:     type performance/quick-read
 92:     subvolumes gw-runqueues-open-behind
 93: end-volume
 94:
 95: volume gw-runqueues-md-cache
 96:     type performance/md-cache
 97:     subvolumes gw-runqueues-quick-read
 98: end-volume
 99:
100: volume gw-runqueues
101:     type debug/io-stats
102:     option log-level INFO
103:     option threads 16
104:     option latency-measurement off
105:     option count-fop-hits off
106:     option global-threading off
107:     subvolumes gw-runqueues-md-cache
108: end-volume
109:
110: volume meta-autoload
111:     type meta
112:     subvolumes gw-runqueues
113: end-volume
114:
+------------------------------------------------------------------------------+
[2022-05-26 08:53:00.910791 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-0: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:53:00.911688 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-0: Connected, attached to remote volume [{conn-name=gw-runqueues-client-0}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:53:00.911716 +0000] I [MSGID: 108005] [afr-common.c:6065:__afr_handle_child_up_event] 0-gw-runqueues-replicate-0: Subvolume 'gw-runqueues-client-0' came back up; going online.
[2022-05-26 08:53:00.913437 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-2: changing port to 49155 (from 0)
[2022-05-26 08:53:00.913499 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-2: intentional socket shutdown(12)
[2022-05-26 08:53:00.913688 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-1: changing port to 49155 (from 0)
[2022-05-26 08:53:00.913875 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-1: intentional socket shutdown(10)
[2022-05-26 08:53:00.916044 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-2: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:53:00.916373 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-1: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 08:53:00.917873 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-2: Connected, attached to remote volume [{conn-name=gw-runqueues-client-2}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:53:00.917985 +0000] I [MSGID: 108002] [afr-common.c:6435:afr_notify] 0-gw-runqueues-replicate-0: Client-quorum is met
[2022-05-26 08:53:00.918232 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-1: Connected, attached to remote volume [{conn-name=gw-runqueues-client-1}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 08:53:00.920484 +0000] I [MSGID: 108031] [afr-common.c:3203:afr_local_discovery_cbk] 0-gw-runqueues-replicate-0: selecting local read_child gw-runqueues-client-0
[2022-05-26 08:53:00.921283 +0000] I [MSGID: 104041] [glfs-resolve.c:974:__glfs_active_subvol] 0-gw-runqueues: switched to graph [{subvol=676c7573-7465-7239-612d-32343231372d}, {id=0}]
[2022-05-26 12:37:10.773020 +0000] I [io-stats.c:3708:ios_sample_buf_size_configure] 0-gw-runqueues: Configure ios_sample_buf  size is 1024 because ios_sample_interval is 0
[2022-05-26 12:37:10.775214 +0000] I [MSGID: 104045] [glfs-primary.c:81:notify] 0-gfapi: New graph coming up [{graph-uuid=676c7573-7465-7239-612d-32393933382d}, {id=0}]
[2022-05-26 12:37:10.775255 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-0: parent translators are ready, attempting connect on transport []
[2022-05-26 12:37:10.776392 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-1: parent translators are ready, attempting connect on transport []
[2022-05-26 12:37:10.776580 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-0: changing port to 49155 (from 0)
[2022-05-26 12:37:10.776615 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-0: intentional socket shutdown(9)
[2022-05-26 12:37:10.777074 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-0: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 12:37:10.777461 +0000] I [MSGID: 114020] [client.c:2319:notify] 0-gw-runqueues-client-2: parent translators are ready, attempting connect on transport []
[2022-05-26 12:37:10.777807 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-0: Connected, attached to remote volume [{conn-name=gw-runqueues-client-0}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 12:37:10.777832 +0000] I [MSGID: 108005] [afr-common.c:6065:__afr_handle_child_up_event] 0-gw-runqueues-replicate-0: Subvolume 'gw-runqueues-client-0' came back up; going online.
Final graph:
+------------------------------------------------------------------------------+
  1: volume gw-runqueues-client-0
  2:     type protocol/client
  3:     option opversion 90000
  4:     option clnt-lk-version 1
  5:     option volfile-checksum 0
  6:     option volfile-key gw-runqueues
  7:     option client-version 9.1
  8:     option process-name gfapi.glfsheal
  9:     option process-uuid CTX_ID:15598a0f-50d7-4a9f-94c5-2b558a429f77-GRAPH_ID:0-PID:29938-HOST:gluster9a-PC_NAME:gw-runqueues-client-0-RECON_NO:-0
 10:     option fops-version 1298437
 11:     option ping-timeout 42
 12:     option remote-host gluster9a
 13:     option remote-subvolume /data/brick/gw-runqueues
 14:     option transport-type socket
 15:     option transport.address-family inet
 16:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 17:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 18:     option transport.socket.ssl-enabled off
 19:     option transport.tcp-user-timeout 0
 20:     option transport.socket.keepalive-time 20
 21:     option transport.socket.keepalive-interval 2
 22:     option transport.socket.keepalive-count 9
 23:     option strict-locks off
 24:     option send-gids true
 25: end-volume
 26:
 27: volume gw-runqueues-client-1
 28:     type protocol/client
 29:     option ping-timeout 42
 30:     option remote-host gluster9b
 31:     option remote-subvolume /data/brick/gw-runqueues
 32:     option transport-type socket
 33:     option transport.address-family inet
 34:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 35:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 36:     option transport.socket.ssl-enabled off
 37:     option transport.tcp-user-timeout 0
 38:     option transport.socket.keepalive-time 20
 39:     option transport.socket.keepalive-interval 2
 40:     option transport.socket.keepalive-count 9
 41:     option strict-locks off
 42:     option send-gids true
 43: end-volume
 44:
 45: volume gw-runqueues-client-2
 46:     type protocol/client
 47:     option ping-timeout 42
 48:     option remote-host gluster9c
 49:     option remote-subvolume /data/brick/gw-runqueues
 50:     option transport-type socket
 51:     option transport.address-family inet
 52:     option username 45d2f5df-9faf-434e-b7cc-a6deec48f2aa
 53:     option password a5c84cc1-1be2-4001-b899-4777c1a7de53
 54:     option transport.socket.ssl-enabled off
 55:     option transport.tcp-user-timeout 0
 56:     option transport.socket.keepalive-time 20
 57:     option transport.socket.keepalive-interval 2
 58:     option transport.socket.keepalive-count 9
 59:     option strict-locks off
 60:     option send-gids true
 61: end-volume
 62:
 63: volume gw-runqueues-replicate-0
 64:     type cluster/replicate
 65:     option background-self-heal-count 0
 66:     option halo-enabled off
 67:     option afr-pending-xattr gw-runqueues-client-0,gw-runqueues-client-1,gw-runqueues-client-2
 68:     option volume-id fd06b0b6-3a87-44ed-bfbe-c243a7ff4e22
 69:     option granular-entry-heal on
 70:     option use-compound-fops off
 71:     option use-anonymous-inode yes
 72:     subvolumes gw-runqueues-client-0 gw-runqueues-client-1 gw-runqueues-client-2
 73: end-volume
 74:
 75: volume gw-runqueues-dht
 76:     type cluster/distribute
 77:     option lock-migration off
 78:     option force-migration off
 79:     subvolumes gw-runqueues-replicate-0
 80: end-volume
 81:
 82: volume gw-runqueues-utime
 83:     type features/utime
 84:     option noatime on
 85:     subvolumes gw-runqueues-dht
 86: end-volume
 87:
 88: volume gw-runqueues-write-behind
 89:     type performance/write-behind
 90:     subvolumes gw-runqueues-utime
 91: end-volume
 92:
 93: volume gw-runqueues-open-behind
 94:     type performance/open-behind
 95:     subvolumes gw-runqueues-write-behind
 96: end-volume
 97:
 98: volume gw-runqueues-quick-read
 99:     type performance/quick-read
100:     subvolumes gw-runqueues-open-behind
101: end-volume
102:
103: volume gw-runqueues-md-cache
104:     type performance/md-cache
105:     subvolumes gw-runqueues-quick-read
106: end-volume
107:
108: volume gw-runqueues
109:     type debug/io-stats
110:     option log-level INFO
111:     option threads 16
112:     option latency-measurement off
113:     option count-fop-hits off
114:     option global-threading off
115:     subvolumes gw-runqueues-md-cache
116: end-volume
117:
118: volume meta-autoload
119:     type meta
120:     subvolumes gw-runqueues
121: end-volume
122:
+------------------------------------------------------------------------------+
[2022-05-26 12:37:10.780624 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-2: changing port to 49155 (from 0)
[2022-05-26 12:37:10.780732 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-2: intentional socket shutdown(10)
[2022-05-26 12:37:10.781234 +0000] I [rpc-clnt.c:1968:rpc_clnt_reconfig] 0-gw-runqueues-client-1: changing port to 49155 (from 0)
[2022-05-26 12:37:10.781493 +0000] I [socket.c:849:__socket_shutdown] 0-gw-runqueues-client-1: intentional socket shutdown(9)
[2022-05-26 12:37:10.783414 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-2: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 12:37:10.784177 +0000] I [MSGID: 114057] [client-handshake.c:1128:select_server_supported_programs] 0-gw-runqueues-client-1: Using Program [{Program-name=GlusterFS 4.x v1}, {Num=1298437}, {Version=400}]
[2022-05-26 12:37:10.785105 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-2: Connected, attached to remote volume [{conn-name=gw-runqueues-client-2}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 12:37:10.785188 +0000] I [MSGID: 108002] [afr-common.c:6435:afr_notify] 0-gw-runqueues-replicate-0: Client-quorum is met
[2022-05-26 12:37:10.786151 +0000] I [MSGID: 114046] [client-handshake.c:857:client_setvolume_cbk] 0-gw-runqueues-client-1: Connected, attached to remote volume [{conn-name=gw-runqueues-client-1}, {remote_subvol=/data/brick/gw-runqueues}]
[2022-05-26 12:37:10.788910 +0000] I [MSGID: 108031] [afr-common.c:3203:afr_local_discovery_cbk] 0-gw-runqueues-replicate-0: selecting local read_child gw-runqueues-client-0
[2022-05-26 12:37:10.790075 +0000] I [MSGID: 104041] [glfs-resolve.c:974:__glfs_active_subvol] 0-gw-runqueues: switched to graph [{subvol=676c7573-7465-7239-612d-32393933382d}, {id=0}]