API tools faq
paste
Guest User

Untitled

a guest
Nov 7th, 2026
16
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 15.55 KB | None | 0 0
raw download clone embed print report
  1. #!/bin/bash
  2. # Sample: rebuild a four-node CX7 ring and the NFS model shares.
  3. # All site-specific values are placeholders; fill in the CONFIG block and the
  4. # interface names before running.
  5. #
  6. # Addresses persist via netplan. NFS persists via fstab automounts and nfs-server.
  7. # Cross-ring forwarding persists via sysctl and cx7-ring-forward.service.
  8. #
  9. # Ring, walked from node1:
  10. # node1 right 10.100.128.2 -- node2 left 10.100.128.1 twin .129
  11. # node2 right 10.100.134.1 -- node3 left 10.100.134.2 twin .135
  12. # node3 right 10.100.136.1 -- node4 left 10.100.136.2 twin .137
  13. # node4 right 10.100.132.2 -- node1 left 10.100.132.1 twin .133
  14. #
  15. # The 10.100.128.0/20 plan and the enp*/enP* interface names are examples.
  16. # Take the real names from `ip -br link`.
  17. #
  18. # Run from node1, in a terminal so sudo can ask for a password on each machine:
  19. # sudo bash ./cx7-ring-apply.sh --all
  20. #
  21. # --all configures node2 and node4, reaches node3 over its management IP,
  22. # configures node3, then configures node1. If node3 was not reachable, fix
  23. # that and rerun with --finish to configure node3 and reload node1's mounts.
  24. set -euo pipefail
  25.  
  26. # ---- edit for your cluster ----
  27. USER_NAME="<username>" # login name on every node
  28. USER_HOME="/home/$USER_NAME"
  29. MODELS_DIR="<models folder/hf cache>"
  30.  
  31. # Host short names are assumed to be node1..node4; change the case labels
  32. # below if yours differ. node3 is reached once over its management/Wi-Fi IP,
  33. # because its ring address does not exist until this script has run there.
  34. NODE3_WIFI="192.0.2.170" # RFC 5737 TEST-NET-1 placeholder
  35.  
  36. # One public key per node. Replace the key material and the comments.
  37. CLUSTER_KEYS=(
  38. "ssh-ed25519 <NODE1_PUBKEY> $USER_NAME@node1"
  39. "ssh-ed25519 <NODE2_PUBKEY> $USER_NAME@node2"
  40. "ssh-ed25519 <NODE4_PUBKEY> $USER_NAME@node4"
  41. )
  42. # -------------------------------
  43.  
  44. MARK_BEGIN="# BEGIN cx7-models"
  45. MARK_END="# END cx7-models"
  46.  
  47. die() { echo "error: $*" >&2; exit 1; }
  48.  
  49. host_short() { hostname -s; }
  50.  
  51. write_netplan() {
  52. local body="$1"
  53. local dest=/etc/netplan/99-cx7-ring.yaml
  54. if [[ -f /etc/netplan/99-nvidia-sync-cluster.yaml && ! -f /etc/netplan/99-nvidia-sync-cluster.yaml.bak-ring ]]; then
  55. cp -a /etc/netplan/99-nvidia-sync-cluster.yaml /etc/netplan/99-nvidia-sync-cluster.yaml.bak-ring
  56. fi
  57. # The old file still holds the triangle addresses. Removing it stops boot from putting them back.
  58. rm -f /etc/netplan/99-nvidia-sync-cluster.yaml
  59. printf '%s\n' "$body" > "$dest"
  60. chmod 600 "$dest"
  61. if ! netplan generate; then
  62. rm -f "$dest"
  63. if [[ -f /etc/netplan/99-nvidia-sync-cluster.yaml.bak-ring ]]; then
  64. cp -a /etc/netplan/99-nvidia-sync-cluster.yaml.bak-ring /etc/netplan/99-nvidia-sync-cluster.yaml
  65. fi
  66. die "netplan generate failed, old cluster file restored"
  67. fi
  68. netplan apply
  69. }
  70.  
  71. del_addr() {
  72. local iface="$1" cidr="$2"
  73. if ip -4 addr show dev "$iface" | grep -q " ${cidr} "; then
  74. ip addr del "$cidr" dev "$iface" || true
  75. fi
  76. }
  77.  
  78. write_exports() {
  79. local line="$1"
  80. local tmp
  81. tmp=$(mktemp)
  82. if [[ -f /etc/exports ]]; then
  83. awk -v dir="$MODELS_DIR" '$1 != dir' /etc/exports > "$tmp" || true
  84. fi
  85. printf '%s\n' "$line" >> "$tmp"
  86. cat "$tmp" > /etc/exports
  87. rm -f "$tmp"
  88. exportfs -ra
  89. }
  90.  
  91. write_fstab() {
  92. local block="$1"
  93. local tmp
  94. tmp=$(mktemp)
  95. awk -v b="$MARK_BEGIN" -v e="$MARK_END" -v home="$USER_HOME" '
  96. $0 == b {skip=1; next}
  97. $0 == e {skip=0; next}
  98. skip {next}
  99. index($2, home "/models-") == 1 {next}
  100. {print}
  101. ' /etc/fstab > "$tmp"
  102. printf '\n%s\n%s\n%s\n' "$MARK_BEGIN" "$block" "$MARK_END" >> "$tmp"
  103. cat "$tmp" > /etc/fstab
  104. rm -f "$tmp"
  105. }
  106.  
  107. ensure_mountpoint() {
  108. local dir="$1"
  109. mkdir -p "$dir"
  110. chown "$USER_NAME:$USER_NAME" "$dir"
  111. }
  112.  
  113. start_automounts() {
  114. local name unit
  115. systemctl daemon-reload
  116. for name in "$@"; do
  117. ensure_mountpoint "$USER_HOME/$name"
  118. unit=$(systemd-escape --path "$USER_HOME/$name")
  119. systemctl enable "$unit.automount" >/dev/null 2>&1 || true
  120. systemctl restart "$unit.automount" || echo "warning: $name automount did not start"
  121. done
  122. }
  123.  
  124. install_forward_unit() {
  125. cat > /etc/systemd/system/cx7-ring-forward.service << 'EOF'
  126. [Unit]
  127. Description=Accept CX7 ring forwards between 10.100.128.0/20
  128. After=docker.service network-online.target
  129. Wants=network-online.target
  130.  
  131. [Service]
  132. Type=oneshot
  133. RemainAfterExit=yes
  134. ExecStart=/bin/bash -c 'iptables -C FORWARD -s 10.100.128.0/20 -d 10.100.128.0/20 -j ACCEPT 2>/dev/null || iptables -I FORWARD 1 -s 10.100.128.0/20 -d 10.100.128.0/20 -j ACCEPT'
  135.  
  136. [Install]
  137. WantedBy=multi-user.target
  138. EOF
  139. systemctl daemon-reload
  140. systemctl enable --now cx7-ring-forward.service
  141. }
  142.  
  143. install_sysctl() {
  144. cat > /etc/sysctl.d/99-cx7-ring.conf << 'EOF'
  145. net.ipv4.ip_forward=1
  146. net.ipv4.conf.all.rp_filter=2
  147. net.ipv4.conf.default.rp_filter=2
  148. EOF
  149. sysctl --system >/dev/null
  150. }
  151.  
  152. allow_ufw() {
  153. if command -v ufw >/dev/null && ufw status | grep -q 'Status: active'; then
  154. ufw allow from 10.100.128.0/20 to any port 2049 proto tcp >/dev/null || true
  155. ufw route allow from 10.100.128.0/20 to 10.100.128.0/20 >/dev/null || true
  156. fi
  157. }
  158.  
  159. install_cluster_keys() {
  160. local home="$USER_HOME"
  161. local auth="$home/.ssh/authorized_keys"
  162. mkdir -p "$home/.ssh"
  163. touch "$auth"
  164. chown "$USER_NAME:$USER_NAME" "$home/.ssh" "$auth"
  165. chmod 700 "$home/.ssh"
  166. chmod 600 "$auth"
  167. local pub
  168. for pub in "${CLUSTER_KEYS[@]}"; do
  169. grep -qxF "$pub" "$auth" || echo "$pub" >> "$auth"
  170. done
  171. if [[ ! -f "$home/.ssh/id_ed25519_cluster" ]]; then
  172. sudo -u "$USER_NAME" ssh-keygen -t ed25519 -N "" -q -f "$home/.ssh/id_ed25519_cluster" -C "$USER_NAME@$(host_short)"
  173. fi
  174. chown "$USER_NAME:$USER_NAME" "$home/.ssh/id_ed25519_cluster" "$home/.ssh/id_ed25519_cluster.pub"
  175. chmod 600 "$home/.ssh/id_ed25519_cluster"
  176. }
  177.  
  178. apply_local() {
  179. [[ $(id -u) -eq 0 ]] || die "run as root: sudo bash $0 --local"
  180. local name
  181. name=$(host_short)
  182. install_sysctl
  183. install_forward_unit
  184. allow_ufw
  185. if ! systemctl cat nfs-server.service >/dev/null 2>&1; then
  186. apt-get update -qq
  187. DEBIAN_FRONTEND=noninteractive apt-get install -y nfs-kernel-server
  188. fi
  189. systemctl enable --now nfs-server
  190. install_cluster_keys
  191. mkdir -p "$MODELS_DIR"
  192. chown "$USER_NAME:$USER_NAME" "$MODELS_DIR"
  193.  
  194. case "$name" in
  195. node1)
  196. write_netplan "$(cat << 'EOF'
  197. network:
  198. version: 2
  199. ethernets:
  200. enp1s0f1np1:
  201. dhcp4: false
  202. dhcp6: false
  203. optional: true
  204. mtu: 9000
  205. addresses: [10.100.128.2/24]
  206. routes:
  207. - to: 10.100.134.0/24
  208. via: 10.100.128.1
  209. enP2p1s0f1np1:
  210. dhcp4: false
  211. dhcp6: false
  212. optional: true
  213. mtu: 9000
  214. addresses: [10.100.129.2/24]
  215. routes:
  216. - to: 10.100.135.0/24
  217. via: 10.100.129.1
  218. enp1s0f0np0:
  219. dhcp4: false
  220. dhcp6: false
  221. optional: true
  222. mtu: 9000
  223. addresses: [10.100.132.1/24]
  224. routes:
  225. - to: 10.100.136.0/24
  226. via: 10.100.132.2
  227. enP2p1s0f0np0:
  228. dhcp4: false
  229. dhcp6: false
  230. optional: true
  231. mtu: 9000
  232. addresses: [10.100.133.1/24]
  233. routes:
  234. - to: 10.100.137.0/24
  235. via: 10.100.133.2
  236. EOF
  237. )"
  238. write_exports "$MODELS_DIR 10.100.128.1(ro,async,no_subtree_check,no_root_squash) 10.100.132.2(ro,async,no_subtree_check,no_root_squash) 10.100.134.2(ro,async,no_subtree_check,no_root_squash)"
  239. write_fstab "$(cat << EOF
  240. 10.100.128.1:$MODELS_DIR $USER_HOME/models-node2 nfs4 nofail,x-systemd.automount,_netdev,noatime 0 0
  241. 10.100.132.2:$MODELS_DIR $USER_HOME/models-node4 nfs4 nofail,x-systemd.automount,_netdev,noatime 0 0
  242. 10.100.134.2:$MODELS_DIR $USER_HOME/models-node3 nfs4 nofail,x-systemd.automount,_netdev,noatime 0 0
  243. EOF
  244. )"
  245. start_automounts models-node2 models-node4 models-node3
  246. ;;
  247. node2)
  248. del_addr enp1s0f1np1 10.100.130.1/24
  249. del_addr enP2p1s0f1np1 10.100.131.1/24
  250. write_netplan "$(cat << 'EOF'
  251. network:
  252. version: 2
  253. ethernets:
  254. enp1s0f0np0:
  255. dhcp4: false
  256. dhcp6: false
  257. optional: true
  258. mtu: 9000
  259. addresses: [10.100.128.1/24]
  260. routes:
  261. - to: 10.100.132.0/24
  262. via: 10.100.128.2
  263. enP2p1s0f0np0:
  264. dhcp4: false
  265. dhcp6: false
  266. optional: true
  267. mtu: 9000
  268. addresses: [10.100.129.1/24]
  269. routes:
  270. - to: 10.100.133.0/24
  271. via: 10.100.129.2
  272. enp1s0f1np1:
  273. dhcp4: false
  274. dhcp6: false
  275. optional: true
  276. mtu: 9000
  277. addresses: [10.100.134.1/24]
  278. routes:
  279. - to: 10.100.136.0/24
  280. via: 10.100.134.2
  281. enP2p1s0f1np1:
  282. dhcp4: false
  283. dhcp6: false
  284. optional: true
  285. mtu: 9000
  286. addresses: [10.100.135.1/24]
  287. routes:
  288. - to: 10.100.137.0/24
  289. via: 10.100.135.2
  290. EOF
  291. )"
  292. write_exports "$MODELS_DIR 10.100.128.2(rw,async,no_subtree_check,no_root_squash) 10.100.134.2(rw,async,no_subtree_check,no_root_squash) 10.100.132.2(rw,async,no_subtree_check,no_root_squash)"
  293. write_fstab "$(cat << EOF
  294. 10.100.128.2:$MODELS_DIR $USER_HOME/models-node1 nfs4 nofail,x-systemd.automount,_netdev,noatime,ro 0 0
  295. 10.100.134.2:$MODELS_DIR $USER_HOME/models-node3 nfs4 nofail,x-systemd.automount,_netdev,noatime 0 0
  296. 10.100.132.2:$MODELS_DIR $USER_HOME/models-node4 nfs4 nofail,x-systemd.automount,_netdev,noatime 0 0
  297. EOF
  298. )"
  299. start_automounts models-node1 models-node3 models-node4
  300. ;;
  301. node4)
  302. del_addr enp1s0f0np0 10.100.130.2/24
  303. del_addr enP2p1s0f0np0 10.100.131.2/24
  304. write_netplan "$(cat << 'EOF'
  305. network:
  306. version: 2
  307. ethernets:
  308. enp1s0f1np1:
  309. dhcp4: false
  310. dhcp6: false
  311. optional: true
  312. mtu: 9000
  313. addresses: [10.100.132.2/24]
  314. routes:
  315. - to: 10.100.128.0/24
  316. via: 10.100.132.1
  317. enP2p1s0f1np1:
  318. dhcp4: false
  319. dhcp6: false
  320. optional: true
  321. mtu: 9000
  322. addresses: [10.100.133.2/24]
  323. routes:
  324. - to: 10.100.129.0/24
  325. via: 10.100.133.1
  326. enp1s0f0np0:
  327. dhcp4: false
  328. dhcp6: false
  329. optional: true
  330. mtu: 9000
  331. addresses: [10.100.136.2/24]
  332. routes:
  333. - to: 10.100.134.0/24
  334. via: 10.100.136.1
  335. enP2p1s0f0np0:
  336. dhcp4: false
  337. dhcp6: false
  338. optional: true
  339. mtu: 9000
  340. addresses: [10.100.137.2/24]
  341. routes:
  342. - to: 10.100.135.0/24
  343. via: 10.100.137.1
  344. EOF
  345. )"
  346. write_exports "$MODELS_DIR 10.100.132.1(rw,async,no_subtree_check,no_root_squash) 10.100.136.1(rw,async,no_subtree_check,no_root_squash) 10.100.128.1(rw,async,no_subtree_check,no_root_squash)"
  347. write_fstab "$(cat << EOF
  348. 10.100.132.1:$MODELS_DIR $USER_HOME/models-node1 nfs4 nofail,x-systemd.automount,_netdev,noatime,ro 0 0
  349. 10.100.136.1:$MODELS_DIR $USER_HOME/models-node3 nfs4 nofail,x-systemd.automount,_netdev,noatime 0 0
  350. 10.100.128.1:$MODELS_DIR $USER_HOME/models-node2 nfs4 nofail,x-systemd.automount,_netdev,noatime 0 0
  351. EOF
  352. )"
  353. start_automounts models-node1 models-node3 models-node2
  354. ;;
  355. node3)
  356. write_netplan "$(cat << 'EOF'
  357. network:
  358. version: 2
  359. ethernets:
  360. enp1s0f0np0:
  361. dhcp4: false
  362. dhcp6: false
  363. optional: true
  364. mtu: 9000
  365. addresses: [10.100.134.2/24]
  366. routes:
  367. - to: 10.100.128.0/24
  368. via: 10.100.134.1
  369. enP2p1s0f0np0:
  370. dhcp4: false
  371. dhcp6: false
  372. optional: true
  373. mtu: 9000
  374. addresses: [10.100.135.2/24]
  375. routes:
  376. - to: 10.100.129.0/24
  377. via: 10.100.135.1
  378. enp1s0f1np1:
  379. dhcp4: false
  380. dhcp6: false
  381. optional: true
  382. mtu: 9000
  383. addresses: [10.100.136.1/24]
  384. routes:
  385. - to: 10.100.132.0/24
  386. via: 10.100.136.2
  387. enP2p1s0f1np1:
  388. dhcp4: false
  389. dhcp6: false
  390. optional: true
  391. mtu: 9000
  392. addresses: [10.100.137.1/24]
  393. routes:
  394. - to: 10.100.133.0/24
  395. via: 10.100.137.2
  396. EOF
  397. )"
  398. write_exports "$MODELS_DIR 10.100.134.1(rw,async,no_subtree_check,no_root_squash) 10.100.136.2(rw,async,no_subtree_check,no_root_squash) 10.100.128.2(rw,async,no_subtree_check,no_root_squash)"
  399. write_fstab "$(cat << EOF
  400. 10.100.134.1:$MODELS_DIR $USER_HOME/models-node2 nfs4 nofail,x-systemd.automount,_netdev,noatime 0 0
  401. 10.100.136.2:$MODELS_DIR $USER_HOME/models-node4 nfs4 nofail,x-systemd.automount,_netdev,noatime 0 0
  402. 10.100.128.2:$MODELS_DIR $USER_HOME/models-node1 nfs4 nofail,x-systemd.automount,_netdev,noatime,ro 0 0
  403. EOF
  404. )"
  405. start_automounts models-node2 models-node4 models-node1
  406. ;;
  407. *)
  408. die "this host is '$name', expected node1, node2, node3, or node4"
  409. ;;
  410. esac
  411. echo "applied on $name"
  412. ip -4 -br addr show | awk '/enp1s0|enP2p1s0/'
  413. }
  414.  
  415. # ssh options that ignore ~/.ssh/config. The host aliases point at Wi-Fi, and
  416. # netplan apply drops that route for a few seconds.
  417. cluster_ssh() {
  418. sudo -u "$USER_NAME" ssh -F /dev/null \
  419. -o IdentitiesOnly=yes \
  420. -o IdentityFile="$USER_HOME/.ssh/id_ed25519_cluster" \
  421. -o StrictHostKeyChecking=accept-new \
  422. -o User="$USER_NAME" \
  423. -o ConnectTimeout=8 \
  424. "$@"
  425. }
  426.  
  427. wait_ping() {
  428. local ip="$1" i
  429. for i in $(seq 1 20); do
  430. ping -c 1 -W 1 "$ip" >/dev/null 2>&1 && return 0
  431. sleep 1
  432. done
  433. return 1
  434. }
  435.  
  436. run_remote() {
  437. local name="$1" dest="$2" err
  438. echo "---- $name ($dest) ----"
  439. if ! err=$(cluster_ssh -o BatchMode=yes "$dest" true 2>&1); then
  440. echo "cannot ssh to $name at $dest: $err" >&2
  441. if [[ "$name" == node3 && "$err" == *Permission\ denied* ]]; then
  442. echo
  443. echo "node3 rejected the cluster key. On node3, as $USER_NAME, run:"
  444. echo
  445. echo " mkdir -p ~/.ssh && chmod 700 ~/.ssh"
  446. echo " touch ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys"
  447. echo " cat >> ~/.ssh/authorized_keys << 'KEYS'"
  448. printf '%s\n' "${CLUSTER_KEYS[@]}"
  449. echo " KEYS"
  450. fi
  451. return 1
  452. fi
  453. sudo -u "$USER_NAME" scp -F /dev/null \
  454. -o IdentitiesOnly=yes \
  455. -o IdentityFile="$USER_HOME/.ssh/id_ed25519_cluster" \
  456. -o StrictHostKeyChecking=accept-new \
  457. "$0" "$USER_NAME@${dest}:/tmp/cx7-ring-apply.sh"
  458. cluster_ssh -t "$dest" "sudo bash /tmp/cx7-ring-apply.sh --local"
  459. }
  460.  
  461. apply_all() {
  462. [[ $(id -u) -eq 0 ]] || die "run as root: sudo bash $0 --all"
  463. [[ $(host_short) == node1 ]] || die "--all is run from node1"
  464. # netplan apply on node1 drops Wi-Fi and kills an SSH session. Configure the
  465. # other three first, and ignore SIGHUP so the last step still finishes.
  466. trap '' HUP
  467. local failed=0
  468. run_remote node2 10.100.128.1 || failed=1
  469. run_remote node4 10.100.132.2 || failed=1
  470. echo "waiting for the management route to node3"
  471. if wait_ping "$NODE3_WIFI"; then
  472. run_remote node3 "$NODE3_WIFI" || failed=1
  473. else
  474. echo "node3 did not answer at $NODE3_WIFI" >&2
  475. failed=1
  476. fi
  477. [[ "$failed" -eq 0 ]] || exit 1
  478. apply_local
  479. }
  480.  
  481. finish_ring() {
  482. [[ $(id -u) -eq 0 ]] || die "run as root: sudo bash $0 --finish"
  483. [[ $(host_short) == node1 ]] || die "--finish is run from node1"
  484. run_remote node3 "$NODE3_WIFI"
  485. # node1's fstab is already correct. Reload the automounts now that node3 exists.
  486. systemctl daemon-reload
  487. # systemd escapes dashes in unit names; this assumes USER_NAME has none.
  488. systemctl reset-failed "home-${USER_NAME}-models*" || true
  489. start_automounts models-node2 models-node4 models-node3
  490. }
  491.  
  492. case "${1:---local}" in
  493. --local) apply_local ;;
  494. --all) apply_all ;;
  495. --finish) finish_ring ;;
  496. *) die "use --local, --all, or --finish" ;;
  497. esac
  498.  
Tags: ai DGX SPARK
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!