1
0

runtime: publish SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1

This commit is contained in:
homelab-runtime-publisher
2026-09-27 17:38:07 +00:00
parent 616ad1714f
commit 845f239b29
4 changed files with 1414 additions and 26 deletions
File diff suppressed because one or more lines are too long
@@ -0,0 +1,698 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=netbird-peer-health-postclose-rca-fix
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=b20b0b61da3227c774ea059fc1736a643fd915f309c46be3b20d12393af4ee8b
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=39727a42b270ac80c2cc47f04631bae8554a678a939e5c3b74ed7593c7574f72
SANITIZED_OUTPUT_SHA256=ad60d3ada9c55ba6ad6dba757229bff607de7c72ac7813502fecbc1895149e1b
OUTPUT_BEGIN
NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_BEGIN=true
COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1
MODE=read-only
COMPONENT=netbird-peer-health-postclose-rca-fix
MUTATION_BOUNDARY=NONE_READ_ONLY_POSTCLOSE_PEER_HEALTH_RCA
=== AUTHORITY ===
HOST=pve01
CLUSTER_QUORATE=Yes
REFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED
REFERENCE_SHA256_EXPECTED=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED
ERROR_REGISTER_SHA256_EXPECTED=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
SOURCE_73331_PRESENT=true
SOURCE_73331_STATUS=FAIL
SOURCE_73331_RC=73
SOURCE_73331_CHANGES_MADE=false
SOURCE_73331_MUTATION_OUTCOME=NO_MUTATION
SOURCE_73331_DECISION=FAIL_NETBIRD_PEER_HEALTH_RCA_PRECONDITION
SOURCE_73331_NEXT_GATE=FIX_ONLY_FAILED_RCA_PRECONDITION
CLOSEOUT_73330_PRESENT=true
CLOSEOUT_73330_STATUS=OK
CLOSEOUT_73330_RC=0
CLOSEOUT_73330_FINAL_ACCEPTANCE=PASS
CLOSEOUT_73330_TASK_RESULT=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT
CLOSEOUT_73330_DECISION=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT
CLOSEOUT_73330_NEXT_GATE=NONE
=== CURRENT_NETBIRD ===
Peers detail:
pve02.netbird.selfhosted:
NetBird IP: 100.100.7.2
NetBird IPv6: 2001:db8:ffff::
Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
home-pc.netbird.selfhosted:
NetBird IP: 100.100.17.159
NetBird IPv6: 2001:db8:ffff::1
Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
relay.netbird.selfhosted:
NetBird IP: 100.100.19.1
NetBird IPv6: 2001:db8:ffff::2
Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE=
Status: Connected
-- detail --
Connection type: P2P
ICE candidate (Local/Remote): host/srflx
ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822
Relay server address: rels://nb.anon-N2556.domain:443
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: 51 seconds ago
Transfer status (received/sent) 6.1 KiB/16.5 KiB
Quantum resistance: false
Networks: -
Latency: 57.51222ms
pve03.netbird.selfhosted:
NetBird IP: 100.100.34.141
NetBird IPv6: 2001:db8:ffff::3
Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
edge-vm.netbird.selfhosted:
NetBird IP: 100.100.60.182
NetBird IPv6: 2001:db8:ffff::4
Public key: qggHuS6pj1odPRoMwZ5FjiXfmNuWiYvtyrZvCfunEBE=
Status: Connected
-- detail --
Connection type: P2P
ICE candidate (Local/Remote): host/host
ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820
Relay server address: rels://nb.anon-N2556.domain:443
Last connection update: 1 hour, 42 minutes ago
Last WireGuard handshake: 1 minute, 48 seconds ago
Transfer status (received/sent) 3.8 MiB/1.8 MiB
Quantum resistance: false
Networks: -
Latency: 892.297µs
desktop-moy.netbird.selfhosted:
NetBird IP: 100.100.76.159
NetBird IPv6: 2001:db8:ffff::5
Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go=
Status: Connected
-- detail --
Connection type: Relayed
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address: rels://nb.anon-N2556.domain:443
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: 33 seconds ago
Transfer status (received/sent) 10.9 KiB/7.5 KiB
Quantum resistance: false
Networks: -
Latency: 0s
comp01.netbird.selfhosted:
NetBird IP: 100.100.123.213
NetBird IPv6: 2001:db8:ffff::6
Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
e3qxxx.netbird.selfhosted:
NetBird IP: 100.100.125.70
NetBird IPv6: 2001:db8:ffff::7
Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
mail.netbird.selfhosted:
NetBird IP: 100.100.147.204
NetBird IPv6: 2001:db8:ffff::8
Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
desktop-server.netbird.selfhosted:
NetBird IP: 100.100.178.43
NetBird IPv6: 2001:db8:ffff::9
Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
e3qxxx-183-106.netbird.selfhosted:
NetBird IP: 100.100.183.106
NetBird IPv6: 2001:db8:ffff::a
Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
Events:
[INFO] SYSTEM (efca2c93-00b0-4c32-9caf-ffe0f983cdd3)
Message: daemon config changed (source=startup)
Time: 1 hour, 43 minutes ago
Metadata: source: startup, type: config_changed
[INFO] SYSTEM (f5af64fc-2fdd-4a3b-830c-626b8a7eea5c)
Message: Network map updated
Time: 1 hour, 43 minutes ago
OS: linux/amd64
Daemon version: 0.73.2
CLI version: 0.73.2
Profile: default
Management: Connected to https://nb.anon-6E0bs.domain:443
Signal: Connected to https://nb.anon-N2556.domain:443
Relays:
[stun:nb.anon-N2556.domain:3478] is Checking...
[rels://nb.anon-N2556.domain:443] is Available via ws
Nameservers:
FQDN: pve01.netbird.selfhosted
NetBird IP: 100.100.131.41/16
NetBird IPv6: 2001:db8:ffff::b/64
Interface type: Kernel
Wireguard port: 51820
Quantum resistance: false
Lazy connection: false
SSH Server: Disabled
Networks: -
Peers count: 3/11 Connected
CURRENT_MANAGEMENT_CONNECTED=true
CURRENT_SIGNAL_CONNECTED=true
CURRENT_PEER_LINE=Peers count: 3/11 Connected
CURRENT_RELAY_LINE=Relays:
=== HEALTH_UNIT ===
Result=exit-code
ExecMainCode=1
ExecMainStatus=1
ExecStart={ path=/root/netbird-peers-health.sh ; argv[]=/root/netbird-peers-health.sh ; ignore_errors=no ; start_time=[Sun 2026-09-27 20:37:01 MSK] ; stop_time=[Sun 2026-09-27 20:37:24 MSK] ; pid=4112376 ; code=exited ; status=1 }
LoadState=loaded
ActiveState=failed
SubState=failed
FragmentPath=/etc/systemd/system/netbird-peers-health.service
HEALTH_UNIT_SHA256=168f6ced7110530ca37908dff300a6ae86de5149ad6347035560cc0e64339816
--- UNIT_CAT ---
# /etc/systemd/system/netbird-peers-health.service
[Unit]
Description=NetBird peers health check
After=network-online.target
[Service]
Type=oneshot
ExecStart=/root/netbird-peers-health.sh
--- TIMER_CAT ---
# /etc/systemd/system/netbird-peers-health.timer
[Unit]
Description=Run NetBird peers health check
[Timer]
OnBootSec=2m
OnUnitActiveSec=5m
Persistent=true
[Install]
WantedBy=timers.target
HEALTH_EXECSTART=/root/netbird-peers-health.sh
HEALTH_SCRIPT_PATH=/root/netbird-peers-health.sh
HEALTH_SCRIPT_SHA256=c531216ceec9962ea0536f0a929676f6bfe70339efee6f001a0cd2b07e4f35d1
--- HEALTH_SCRIPT_REDACTED_BEGIN ---
#!/usr/bin/env bash
set -u
HEALTH="/var/lib/homelab-health/netbird-peers.txt"
TMP="$(mktemp)"
TMP2="$(mktemp)"
TS="$(date -u +%Y%m%dT%H%M%SZ)"
ATTEMPT_USED="1"
OVERALL="OK"
collect_pve01() {
echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)"
echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)"
echo "NBSTATUS=$(netbird status 2>/dev/null | tr '\n' ';' | cut -c1-3000 || true)"
}
collect_root_host() {
local host="$1"
ssh -o BatchMode=yes -o ConnectTimeout=8 root@"$host" 'echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)"; echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)"; echo "NBSTATUS=$(netbird status 2>/dev/null | tr "\n" ";" | cut -c1-3000 || true)"'
}
collect_edge() {
ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)"; echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)"; echo "NBSTATUS=$(sudo netbird status 2>/dev/null | tr "\n" ";" | cut -c1-3000 || true)"'
}
record_node() {
local out="$1"
local name="$2"
local data="$3"
local service wt0 nbstatus ip mgmt signal peers
service="$(printf "%s\n" "$data" | sed -n 's/^SERVICE=//p' | tail -n1)"
wt0="$(printf "%s\n" "$data" | sed -n 's/^WT0=//p' | tail -n1)"
nbstatus="$(printf "%s\n" "$data" | sed -n 's/^NBSTATUS=//p' | tail -n1)"
ip="$(printf "%s" "$wt0" | sed -n 's/^wt0[[:space:]]\+UNKNOWN[[:space:]]\+\([^ ]*\).*/\1/p')"
printf "%s" "$nbstatus" | grep -q "Management: Connected" && mgmt="yes" || mgmt="no"
printf "%s" "$nbstatus" | grep -q "Signal: Connected" && signal="yes" || signal="no"
peers="$(printf "%s" "$nbstatus" | sed -n 's/.*Peers count: \([^;]*\).*/\1/p' | tail -n1)"
[ -n "$peers" ] || peers="unknown"
if [ "$service" != "active/enabled" ] || [ -z "$ip" ] || [ "$mgmt" != "yes" ] || [ "$signal" != "yes" ]; then
OVERALL="ERROR"
fi
echo "$name service=${service:-missing} wt0_ip=${ip:-missing} management=$mgmt signal=$signal peers=$peers" >>"$out"
}
run_node() {
local out="$1"
local name="$2"
shift 2
local data
if data="$("$@" 2>/tmp/netbird-peers-health-node.err)"; then
record_node "$out" "$name" "$data"
else
OVERALL="ERROR"
echo "$name service=ERROR wt0_ip=missing management=no signal=no peers=ssh_or_command_failed" >>"$out"
fi
}
run_all() {
local out="$1"
OVERALL="OK"
: > "$out"
run_node "$out" "pve01" collect_pve01
run_node "$out" "pve02" collect_root_host [PRIVATE_IP]
run_node "$out" "pve03" collect_root_host [PRIVATE_IP]
run_node "$out" "edge-vm" collect_edge
}
mkdir -p "$(dirname "$HEALTH")"
run_all "$TMP"
if [ "$OVERALL" != "OK" ]; then
sleep 15
ATTEMPT_USED="2"
run_all "$TMP2"
cp "$TMP2" "$TMP"
fi
OUT="$(mktemp)"
{
echo "STATUS=$OVERALL"
echo "SERVICE=netbird-peers"
echo "TIMESTAMP_UTC=$TS"
echo "ATTEMPT_USED=$ATTEMPT_USED"
cat "$TMP"
} > "$OUT"
chmod 0644 "$OUT"
mv "$OUT" "$HEALTH"
rm -f "$TMP" "$TMP2"
[ "$OVERALL" = "OK" ]
--- HEALTH_SCRIPT_REDACTED_END ---
=== HEALTH_FILE ===
HEALTH_FILE_META=480:root:root:2026-09-27 20:37:24.472057535 +0300
HEALTH_FILE_SHA256=2512770acb6314656393870b3cd19351109672d2c7c26e2170bbf2c8d2eff400
STATUS=ERROR
SERVICE=netbird-peers
TIMESTAMP_UTC=20260927T173701Z
ATTEMPT_USED=2
pve01 service=active/enabled wt0_ip=100.100.131.41/16 management=yes signal=yes peers=3/11 Connected
pve02 service=active/enabled wt0_ip=100.100.7.2/16 management=no signal=no peers=0/11 Connected
pve03 service=active/enabled wt0_ip=100.100.34.141/16 management=no signal=no peers=0/11 Connected
edge-vm service=active/enabled wt0_ip=100.100.60.182/16 management=yes signal=yes peers=3/11 Connected
=== HEALTH_JOURNAL_RECENT ===
2026-09-27T19:56:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T19:56:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:01:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:01:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:06:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:06:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:11:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:11:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:16:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:16:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:21:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:21:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:26:32+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:26:55+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:31:38+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:32:01+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:37:01+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:37:24+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
=== EDGE_COMPARISON ===
EDGE_LIVE_HOSTKEY_FP=SHA256:YQsA2nif7qAaa5mSUb9U0h8Q5DGy8SRCV5f+FyKUpOo
EDGE_SSH_AUTH=true
Peers detail:
pve02.netbird.selfhosted:
NetBird IP: 100.100.7.2
NetBird IPv6: 2001:db8:ffff::
Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
home-pc.netbird.selfhosted:
NetBird IP: 100.100.17.159
NetBird IPv6: 2001:db8:ffff::1
Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 14 hours, 49 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
relay.netbird.selfhosted:
NetBird IP: 100.100.19.1
NetBird IPv6: 2001:db8:ffff::2
Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE=
Status: Connected
-- detail --
Connection type: P2P
ICE candidate (Local/Remote): host/srflx
ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822
Relay server address: rels://nb.anon-MQCTn.domain:443
Last connection update: 4 hours, 28 minutes ago
Last WireGuard handshake: 2 minutes, 6 seconds ago
Transfer status (received/sent) 142.7 KiB/38.7 KiB
Quantum resistance: false
Networks: -
Latency: 57.639982ms
pve03.netbird.selfhosted:
NetBird IP: 100.100.34.141
NetBird IPv6: 2001:db8:ffff::3
Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
desktop-moy.netbird.selfhosted:
NetBird IP: 100.100.76.159
NetBird IPv6: 2001:db8:ffff::4
Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go=
Status: Connected
-- detail --
Connection type: Relayed
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address: rels://nb.anon-MQCTn.domain:443
Last connection update: 3 hours, 34 minutes ago
Last WireGuard handshake: 1 minute, 58 seconds ago
Transfer status (received/sent) 34.1 KiB/9.5 KiB
Quantum resistance: false
Networks: -
Latency: 0s
comp01.netbird.selfhosted:
NetBird IP: 100.100.123.213
NetBird IPv6: 2001:db8:ffff::5
Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
e3qxxx.netbird.selfhosted:
NetBird IP: 100.100.125.70
NetBird IPv6: 2001:db8:ffff::6
Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
pve01.netbird.selfhosted:
NetBird IP: 100.100.131.41
NetBird IPv6: 2001:db8:ffff::7
Public key: rt0YF+0ieQPRRQrY6v+Hd2hTP8wBjK3a3zzU5kPGlS8=
Status: Connected
-- detail --
Connection type: P2P
ICE candidate (Local/Remote): host/prflx
ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820
Relay server address: rels://nb.anon-MQCTn.domain:443
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: 1 minute, 50 seconds ago
Transfer status (received/sent) 1.8 MiB/3.8 MiB
Quantum resistance: false
Networks: -
Latency: 910.974µs
mail.netbird.selfhosted:
NetBird IP: 100.100.147.204
NetBird IPv6: 2001:db8:ffff::8
Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
desktop-server.netbird.selfhosted:
NetBird IP: 100.100.178.43
NetBird IPv6: 2001:db8:ffff::9
Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
e3qxxx-183-106.netbird.selfhosted:
NetBird IP: 100.100.183.106
NetBird IPv6: 2001:db8:ffff::a
Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
Events:
[INFO] SYSTEM (50e1e0e8-d706-4288-a5f0-7d2f76b20209)
Message: daemon config changed (source=startup)
Time: 19 hours, 23 minutes ago
Metadata: source: startup, type: config_changed
[INFO] SYSTEM (dc5f191e-9af0-4b4b-bd71-5d4bef18f46c)
Message: Network map updated
Time: 15 hours, 54 minutes ago
[INFO] SYSTEM (a3f5af7c-c1f2-48b5-991b-2ae6d05008ab)
Message: Network map updated
Time: 13 hours, 38 minutes ago
[INFO] SYSTEM (3c6e7858-3c31-46a6-a553-cb69db89b0e3)
Message: Network map updated
Time: 6 hours, 54 minutes ago
[INFO] SYSTEM (aeacd9ec-e926-4148-96d5-c3d384cfced1)
Message: Network map updated
Time: 4 hours, 31 minutes ago
[INFO] SYSTEM (f5eb9e93-22d2-49f9-bded-cd8985a49c93)
Message: Network map updated
Time: 4 hours, 28 minutes ago
OS: linux/amd64
Daemon version: 0.73.2
CLI version: 0.73.2
Profile: default
Management: Connected to https://nb.anon-UmaXu.domain:443
Signal: Connected to https://nb.anon-MQCTn.domain:443
Relays:
[stun:nb.anon-MQCTn.domain:3478] is Checking...
[rels://nb.anon-MQCTn.domain:443] is Available via ws
Nameservers:
FQDN: edge-vm.netbird.selfhosted
NetBird IP: 100.100.60.182/16
NetBird IPv6: 2001:db8:ffff::b/64
Interface type: Kernel
Wireguard port: 51820
Quantum resistance: false
Lazy connection: false
SSH Server: Disabled
Networks: -
Peers count: 3/11 Connected
=== CLASSIFICATION ===
CONTROL_PLANE_CONNECTED=true
HEALTH_UNIT_RESULT=exit-code
DECISION=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_COLLECTED
NEXT_GATE=READ_EXACT_73332_AND_FIX_ONLY_PROVEN_PEER_HEALTH_CAUSE
TASK_RESULT=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END
+11 -11
View File
File diff suppressed because one or more lines are too long
+667 -15
View File
@@ -1,22 +1,22 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-73331R1
STATUS=FAIL
RC=73
COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=netbird-peer-health-postclose-rca
COMPONENT=netbird-peer-health-postclose-rca-fix
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=1b59ca704ae2d255adca6ff919aac3c5e37c2a667fa821f8c3c9a1a65c8846a4
COMMAND_SHA256=b20b0b61da3227c774ea059fc1736a643fd915f309c46be3b20d12393af4ee8b
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=73
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
@@ -24,23 +24,675 @@ MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=bb11cc30283a0a267ef627800c4107f52d9a85f537ac0228b9b54dd719ae8aee
SANITIZED_OUTPUT_SHA256=bb11cc30283a0a267ef627800c4107f52d9a85f537ac0228b9b54dd719ae8aee
RAW_EVIDENCE_SHA256=39727a42b270ac80c2cc47f04631bae8554a678a939e5c3b74ed7593c7574f72
SANITIZED_OUTPUT_SHA256=ad60d3ada9c55ba6ad6dba757229bff607de7c72ac7813502fecbc1895149e1b
OUTPUT_BEGIN
NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_BEGIN=true
COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-73331R1
COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1
MODE=read-only
COMPONENT=netbird-peer-health-postclose-rca
COMPONENT=netbird-peer-health-postclose-rca-fix
MUTATION_BOUNDARY=NONE_READ_ONLY_POSTCLOSE_PEER_HEALTH_RCA
=== AUTHORITY ===
HOST=pve01
CLUSTER_QUORATE=Yes
REFERENCE_SHA256=
ERROR_REGISTER_SHA256=
REFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED
REFERENCE_SHA256_EXPECTED=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED
ERROR_REGISTER_SHA256_EXPECTED=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
DECISION=FAIL_NETBIRD_PEER_HEALTH_RCA_PRECONDITION
NEXT_GATE=FIX_ONLY_FAILED_RCA_PRECONDITION
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-73331R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
SOURCE_73331_PRESENT=true
SOURCE_73331_STATUS=FAIL
SOURCE_73331_RC=73
SOURCE_73331_CHANGES_MADE=false
SOURCE_73331_MUTATION_OUTCOME=NO_MUTATION
SOURCE_73331_DECISION=FAIL_NETBIRD_PEER_HEALTH_RCA_PRECONDITION
SOURCE_73331_NEXT_GATE=FIX_ONLY_FAILED_RCA_PRECONDITION
CLOSEOUT_73330_PRESENT=true
CLOSEOUT_73330_STATUS=OK
CLOSEOUT_73330_RC=0
CLOSEOUT_73330_FINAL_ACCEPTANCE=PASS
CLOSEOUT_73330_TASK_RESULT=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT
CLOSEOUT_73330_DECISION=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT
CLOSEOUT_73330_NEXT_GATE=NONE
=== CURRENT_NETBIRD ===
Peers detail:
pve02.netbird.selfhosted:
NetBird IP: 100.100.7.2
NetBird IPv6: 2001:db8:ffff::
Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
home-pc.netbird.selfhosted:
NetBird IP: 100.100.17.159
NetBird IPv6: 2001:db8:ffff::1
Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
relay.netbird.selfhosted:
NetBird IP: 100.100.19.1
NetBird IPv6: 2001:db8:ffff::2
Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE=
Status: Connected
-- detail --
Connection type: P2P
ICE candidate (Local/Remote): host/srflx
ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822
Relay server address: rels://nb.anon-N2556.domain:443
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: 51 seconds ago
Transfer status (received/sent) 6.1 KiB/16.5 KiB
Quantum resistance: false
Networks: -
Latency: 57.51222ms
pve03.netbird.selfhosted:
NetBird IP: 100.100.34.141
NetBird IPv6: 2001:db8:ffff::3
Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
edge-vm.netbird.selfhosted:
NetBird IP: 100.100.60.182
NetBird IPv6: 2001:db8:ffff::4
Public key: qggHuS6pj1odPRoMwZ5FjiXfmNuWiYvtyrZvCfunEBE=
Status: Connected
-- detail --
Connection type: P2P
ICE candidate (Local/Remote): host/host
ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820
Relay server address: rels://nb.anon-N2556.domain:443
Last connection update: 1 hour, 42 minutes ago
Last WireGuard handshake: 1 minute, 48 seconds ago
Transfer status (received/sent) 3.8 MiB/1.8 MiB
Quantum resistance: false
Networks: -
Latency: 892.297µs
desktop-moy.netbird.selfhosted:
NetBird IP: 100.100.76.159
NetBird IPv6: 2001:db8:ffff::5
Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go=
Status: Connected
-- detail --
Connection type: Relayed
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address: rels://nb.anon-N2556.domain:443
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: 33 seconds ago
Transfer status (received/sent) 10.9 KiB/7.5 KiB
Quantum resistance: false
Networks: -
Latency: 0s
comp01.netbird.selfhosted:
NetBird IP: 100.100.123.213
NetBird IPv6: 2001:db8:ffff::6
Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
e3qxxx.netbird.selfhosted:
NetBird IP: 100.100.125.70
NetBird IPv6: 2001:db8:ffff::7
Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
mail.netbird.selfhosted:
NetBird IP: 100.100.147.204
NetBird IPv6: 2001:db8:ffff::8
Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
desktop-server.netbird.selfhosted:
NetBird IP: 100.100.178.43
NetBird IPv6: 2001:db8:ffff::9
Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
e3qxxx-183-106.netbird.selfhosted:
NetBird IP: 100.100.183.106
NetBird IPv6: 2001:db8:ffff::a
Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
Events:
[INFO] SYSTEM (efca2c93-00b0-4c32-9caf-ffe0f983cdd3)
Message: daemon config changed (source=startup)
Time: 1 hour, 43 minutes ago
Metadata: source: startup, type: config_changed
[INFO] SYSTEM (f5af64fc-2fdd-4a3b-830c-626b8a7eea5c)
Message: Network map updated
Time: 1 hour, 43 minutes ago
OS: linux/amd64
Daemon version: 0.73.2
CLI version: 0.73.2
Profile: default
Management: Connected to https://nb.anon-6E0bs.domain:443
Signal: Connected to https://nb.anon-N2556.domain:443
Relays:
[stun:nb.anon-N2556.domain:3478] is Checking...
[rels://nb.anon-N2556.domain:443] is Available via ws
Nameservers:
FQDN: pve01.netbird.selfhosted
NetBird IP: 100.100.131.41/16
NetBird IPv6: 2001:db8:ffff::b/64
Interface type: Kernel
Wireguard port: 51820
Quantum resistance: false
Lazy connection: false
SSH Server: Disabled
Networks: -
Peers count: 3/11 Connected
CURRENT_MANAGEMENT_CONNECTED=true
CURRENT_SIGNAL_CONNECTED=true
CURRENT_PEER_LINE=Peers count: 3/11 Connected
CURRENT_RELAY_LINE=Relays:
=== HEALTH_UNIT ===
Result=exit-code
ExecMainCode=1
ExecMainStatus=1
ExecStart={ path=/root/netbird-peers-health.sh ; argv[]=/root/netbird-peers-health.sh ; ignore_errors=no ; start_time=[Sun 2026-09-27 20:37:01 MSK] ; stop_time=[Sun 2026-09-27 20:37:24 MSK] ; pid=4112376 ; code=exited ; status=1 }
LoadState=loaded
ActiveState=failed
SubState=failed
FragmentPath=/etc/systemd/system/netbird-peers-health.service
HEALTH_UNIT_SHA256=168f6ced7110530ca37908dff300a6ae86de5149ad6347035560cc0e64339816
--- UNIT_CAT ---
# /etc/systemd/system/netbird-peers-health.service
[Unit]
Description=NetBird peers health check
After=network-online.target
[Service]
Type=oneshot
ExecStart=/root/netbird-peers-health.sh
--- TIMER_CAT ---
# /etc/systemd/system/netbird-peers-health.timer
[Unit]
Description=Run NetBird peers health check
[Timer]
OnBootSec=2m
OnUnitActiveSec=5m
Persistent=true
[Install]
WantedBy=timers.target
HEALTH_EXECSTART=/root/netbird-peers-health.sh
HEALTH_SCRIPT_PATH=/root/netbird-peers-health.sh
HEALTH_SCRIPT_SHA256=c531216ceec9962ea0536f0a929676f6bfe70339efee6f001a0cd2b07e4f35d1
--- HEALTH_SCRIPT_REDACTED_BEGIN ---
#!/usr/bin/env bash
set -u
HEALTH="/var/lib/homelab-health/netbird-peers.txt"
TMP="$(mktemp)"
TMP2="$(mktemp)"
TS="$(date -u +%Y%m%dT%H%M%SZ)"
ATTEMPT_USED="1"
OVERALL="OK"
collect_pve01() {
echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)"
echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)"
echo "NBSTATUS=$(netbird status 2>/dev/null | tr '\n' ';' | cut -c1-3000 || true)"
}
collect_root_host() {
local host="$1"
ssh -o BatchMode=yes -o ConnectTimeout=8 root@"$host" 'echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)"; echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)"; echo "NBSTATUS=$(netbird status 2>/dev/null | tr "\n" ";" | cut -c1-3000 || true)"'
}
collect_edge() {
ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)"; echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)"; echo "NBSTATUS=$(sudo netbird status 2>/dev/null | tr "\n" ";" | cut -c1-3000 || true)"'
}
record_node() {
local out="$1"
local name="$2"
local data="$3"
local service wt0 nbstatus ip mgmt signal peers
service="$(printf "%s\n" "$data" | sed -n 's/^SERVICE=//p' | tail -n1)"
wt0="$(printf "%s\n" "$data" | sed -n 's/^WT0=//p' | tail -n1)"
nbstatus="$(printf "%s\n" "$data" | sed -n 's/^NBSTATUS=//p' | tail -n1)"
ip="$(printf "%s" "$wt0" | sed -n 's/^wt0[[:space:]]\+UNKNOWN[[:space:]]\+\([^ ]*\).*/\1/p')"
printf "%s" "$nbstatus" | grep -q "Management: Connected" && mgmt="yes" || mgmt="no"
printf "%s" "$nbstatus" | grep -q "Signal: Connected" && signal="yes" || signal="no"
peers="$(printf "%s" "$nbstatus" | sed -n 's/.*Peers count: \([^;]*\).*/\1/p' | tail -n1)"
[ -n "$peers" ] || peers="unknown"
if [ "$service" != "active/enabled" ] || [ -z "$ip" ] || [ "$mgmt" != "yes" ] || [ "$signal" != "yes" ]; then
OVERALL="ERROR"
fi
echo "$name service=${service:-missing} wt0_ip=${ip:-missing} management=$mgmt signal=$signal peers=$peers" >>"$out"
}
run_node() {
local out="$1"
local name="$2"
shift 2
local data
if data="$("$@" 2>/tmp/netbird-peers-health-node.err)"; then
record_node "$out" "$name" "$data"
else
OVERALL="ERROR"
echo "$name service=ERROR wt0_ip=missing management=no signal=no peers=ssh_or_command_failed" >>"$out"
fi
}
run_all() {
local out="$1"
OVERALL="OK"
: > "$out"
run_node "$out" "pve01" collect_pve01
run_node "$out" "pve02" collect_root_host [PRIVATE_IP]
run_node "$out" "pve03" collect_root_host [PRIVATE_IP]
run_node "$out" "edge-vm" collect_edge
}
mkdir -p "$(dirname "$HEALTH")"
run_all "$TMP"
if [ "$OVERALL" != "OK" ]; then
sleep 15
ATTEMPT_USED="2"
run_all "$TMP2"
cp "$TMP2" "$TMP"
fi
OUT="$(mktemp)"
{
echo "STATUS=$OVERALL"
echo "SERVICE=netbird-peers"
echo "TIMESTAMP_UTC=$TS"
echo "ATTEMPT_USED=$ATTEMPT_USED"
cat "$TMP"
} > "$OUT"
chmod 0644 "$OUT"
mv "$OUT" "$HEALTH"
rm -f "$TMP" "$TMP2"
[ "$OVERALL" = "OK" ]
--- HEALTH_SCRIPT_REDACTED_END ---
=== HEALTH_FILE ===
HEALTH_FILE_META=480:root:root:2026-09-27 20:37:24.472057535 +0300
HEALTH_FILE_SHA256=2512770acb6314656393870b3cd19351109672d2c7c26e2170bbf2c8d2eff400
STATUS=ERROR
SERVICE=netbird-peers
TIMESTAMP_UTC=20260927T173701Z
ATTEMPT_USED=2
pve01 service=active/enabled wt0_ip=100.100.131.41/16 management=yes signal=yes peers=3/11 Connected
pve02 service=active/enabled wt0_ip=100.100.7.2/16 management=no signal=no peers=0/11 Connected
pve03 service=active/enabled wt0_ip=100.100.34.141/16 management=no signal=no peers=0/11 Connected
edge-vm service=active/enabled wt0_ip=100.100.60.182/16 management=yes signal=yes peers=3/11 Connected
=== HEALTH_JOURNAL_RECENT ===
2026-09-27T19:56:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T19:56:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:01:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:01:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:06:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:06:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:11:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:11:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:16:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:16:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:21:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:21:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:26:32+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:26:55+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:31:38+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:32:01+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
2026-09-27T20:37:01+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...
2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE
2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.
2026-09-27T20:37:24+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.
=== EDGE_COMPARISON ===
EDGE_LIVE_HOSTKEY_FP=SHA256:YQsA2nif7qAaa5mSUb9U0h8Q5DGy8SRCV5f+FyKUpOo
EDGE_SSH_AUTH=true
Peers detail:
pve02.netbird.selfhosted:
NetBird IP: 100.100.7.2
NetBird IPv6: 2001:db8:ffff::
Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
home-pc.netbird.selfhosted:
NetBird IP: 100.100.17.159
NetBird IPv6: 2001:db8:ffff::1
Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 14 hours, 49 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
relay.netbird.selfhosted:
NetBird IP: 100.100.19.1
NetBird IPv6: 2001:db8:ffff::2
Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE=
Status: Connected
-- detail --
Connection type: P2P
ICE candidate (Local/Remote): host/srflx
ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822
Relay server address: rels://nb.anon-MQCTn.domain:443
Last connection update: 4 hours, 28 minutes ago
Last WireGuard handshake: 2 minutes, 6 seconds ago
Transfer status (received/sent) 142.7 KiB/38.7 KiB
Quantum resistance: false
Networks: -
Latency: 57.639982ms
pve03.netbird.selfhosted:
NetBird IP: 100.100.34.141
NetBird IPv6: 2001:db8:ffff::3
Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
desktop-moy.netbird.selfhosted:
NetBird IP: 100.100.76.159
NetBird IPv6: 2001:db8:ffff::4
Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go=
Status: Connected
-- detail --
Connection type: Relayed
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address: rels://nb.anon-MQCTn.domain:443
Last connection update: 3 hours, 34 minutes ago
Last WireGuard handshake: 1 minute, 58 seconds ago
Transfer status (received/sent) 34.1 KiB/9.5 KiB
Quantum resistance: false
Networks: -
Latency: 0s
comp01.netbird.selfhosted:
NetBird IP: 100.100.123.213
NetBird IPv6: 2001:db8:ffff::5
Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
e3qxxx.netbird.selfhosted:
NetBird IP: 100.100.125.70
NetBird IPv6: 2001:db8:ffff::6
Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
pve01.netbird.selfhosted:
NetBird IP: 100.100.131.41
NetBird IPv6: 2001:db8:ffff::7
Public key: rt0YF+0ieQPRRQrY6v+Hd2hTP8wBjK3a3zzU5kPGlS8=
Status: Connected
-- detail --
Connection type: P2P
ICE candidate (Local/Remote): host/prflx
ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820
Relay server address: rels://nb.anon-MQCTn.domain:443
Last connection update: 1 hour, 43 minutes ago
Last WireGuard handshake: 1 minute, 50 seconds ago
Transfer status (received/sent) 1.8 MiB/3.8 MiB
Quantum resistance: false
Networks: -
Latency: 910.974µs
mail.netbird.selfhosted:
NetBird IP: 100.100.147.204
NetBird IPv6: 2001:db8:ffff::8
Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
desktop-server.netbird.selfhosted:
NetBird IP: 100.100.178.43
NetBird IPv6: 2001:db8:ffff::9
Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
e3qxxx-183-106.netbird.selfhosted:
NetBird IP: 100.100.183.106
NetBird IPv6: 2001:db8:ffff::a
Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o=
Status: Connecting
-- detail --
Connection type: -
ICE candidate (Local/Remote): -/-
ICE candidate endpoints (Local/Remote): -/-
Relay server address:
Last connection update: 15 hours, 54 minutes ago
Last WireGuard handshake: -
Transfer status (received/sent) 0 B/0 B
Quantum resistance: false
Networks: -
Latency: 0s
Events:
[INFO] SYSTEM (50e1e0e8-d706-4288-a5f0-7d2f76b20209)
Message: daemon config changed (source=startup)
Time: 19 hours, 23 minutes ago
Metadata: source: startup, type: config_changed
[INFO] SYSTEM (dc5f191e-9af0-4b4b-bd71-5d4bef18f46c)
Message: Network map updated
Time: 15 hours, 54 minutes ago
[INFO] SYSTEM (a3f5af7c-c1f2-48b5-991b-2ae6d05008ab)
Message: Network map updated
Time: 13 hours, 38 minutes ago
[INFO] SYSTEM (3c6e7858-3c31-46a6-a553-cb69db89b0e3)
Message: Network map updated
Time: 6 hours, 54 minutes ago
[INFO] SYSTEM (aeacd9ec-e926-4148-96d5-c3d384cfced1)
Message: Network map updated
Time: 4 hours, 31 minutes ago
[INFO] SYSTEM (f5eb9e93-22d2-49f9-bded-cd8985a49c93)
Message: Network map updated
Time: 4 hours, 28 minutes ago
OS: linux/amd64
Daemon version: 0.73.2
CLI version: 0.73.2
Profile: default
Management: Connected to https://nb.anon-UmaXu.domain:443
Signal: Connected to https://nb.anon-MQCTn.domain:443
Relays:
[stun:nb.anon-MQCTn.domain:3478] is Checking...
[rels://nb.anon-MQCTn.domain:443] is Available via ws
Nameservers:
FQDN: edge-vm.netbird.selfhosted
NetBird IP: 100.100.60.182/16
NetBird IPv6: 2001:db8:ffff::b/64
Interface type: Kernel
Wireguard port: 51820
Quantum resistance: false
Lazy connection: false
SSH Server: Disabled
Networks: -
Peers count: 3/11 Connected
=== CLASSIFICATION ===
CONTROL_PLANE_CONNECTED=true
HEALTH_UNIT_RESULT=exit-code
DECISION=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_COLLECTED
NEXT_GATE=READ_EXACT_73332_AND_FIX_ONLY_PROVEN_PEER_HEALTH_CAUSE
TASK_RESULT=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END