diff --git a/runtime/history/SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1.json b/runtime/history/SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1.json new file mode 100644 index 00000000..250bf8ff --- /dev/null +++ b/runtime/history/SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "netbird-peer-health-postclose-rca-fix", + "started_at_utc": "2026-09-27T17:38:02Z", + "finished_at_utc": "2026-09-27T17:38:06Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "b20b0b61da3227c774ea059fc1736a643fd915f309c46be3b20d12393af4ee8b", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "39727a42b270ac80c2cc47f04631bae8554a678a939e5c3b74ed7593c7574f72", + "sanitized_output_sha256": "ad60d3ada9c55ba6ad6dba757229bff607de7c72ac7813502fecbc1895149e1b", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1\nMODE=read-only\nCOMPONENT=netbird-peer-health-postclose-rca-fix\nMUTATION_BOUNDARY=NONE_READ_ONLY_POSTCLOSE_PEER_HEALTH_RCA\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED\nREFERENCE_SHA256_EXPECTED=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED\nERROR_REGISTER_SHA256_EXPECTED=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_73331_PRESENT=true\nSOURCE_73331_STATUS=FAIL\nSOURCE_73331_RC=73\nSOURCE_73331_CHANGES_MADE=false\nSOURCE_73331_MUTATION_OUTCOME=NO_MUTATION\nSOURCE_73331_DECISION=FAIL_NETBIRD_PEER_HEALTH_RCA_PRECONDITION\nSOURCE_73331_NEXT_GATE=FIX_ONLY_FAILED_RCA_PRECONDITION\nCLOSEOUT_73330_PRESENT=true\nCLOSEOUT_73330_STATUS=OK\nCLOSEOUT_73330_RC=0\nCLOSEOUT_73330_FINAL_ACCEPTANCE=PASS\nCLOSEOUT_73330_TASK_RESULT=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT\nCLOSEOUT_73330_DECISION=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT\nCLOSEOUT_73330_NEXT_GATE=NONE\n=== CURRENT_NETBIRD ===\nPeers detail:\n pve02.netbird.selfhosted:\n NetBird IP: 100.100.7.2\n NetBird IPv6: 2001:db8:ffff::\n Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n home-pc.netbird.selfhosted:\n NetBird IP: 100.100.17.159\n NetBird IPv6: 2001:db8:ffff::1\n Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n relay.netbird.selfhosted:\n NetBird IP: 100.100.19.1\n NetBird IPv6: 2001:db8:ffff::2\n Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE=\n Status: Connected\n -- detail --\n Connection type: P2P\n ICE candidate (Local/Remote): host/srflx\n ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822\n Relay server address: rels://nb.anon-N2556.domain:443\n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: 51 seconds ago\n Transfer status (received/sent) 6.1 KiB/16.5 KiB\n Quantum resistance: false\n Networks: -\n Latency: 57.51222ms\n\n pve03.netbird.selfhosted:\n NetBird IP: 100.100.34.141\n NetBird IPv6: 2001:db8:ffff::3\n Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n edge-vm.netbird.selfhosted:\n NetBird IP: 100.100.60.182\n NetBird IPv6: 2001:db8:ffff::4\n Public key: qggHuS6pj1odPRoMwZ5FjiXfmNuWiYvtyrZvCfunEBE=\n Status: Connected\n -- detail --\n Connection type: P2P\n ICE candidate (Local/Remote): host/host\n ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820\n Relay server address: rels://nb.anon-N2556.domain:443\n Last connection update: 1 hour, 42 minutes ago\n Last WireGuard handshake: 1 minute, 48 seconds ago\n Transfer status (received/sent) 3.8 MiB/1.8 MiB\n Quantum resistance: false\n Networks: -\n Latency: 892.297µs\n\n desktop-moy.netbird.selfhosted:\n NetBird IP: 100.100.76.159\n NetBird IPv6: 2001:db8:ffff::5\n Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go=\n Status: Connected\n -- detail --\n Connection type: Relayed\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: rels://nb.anon-N2556.domain:443\n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: 33 seconds ago\n Transfer status (received/sent) 10.9 KiB/7.5 KiB\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n comp01.netbird.selfhosted:\n NetBird IP: 100.100.123.213\n NetBird IPv6: 2001:db8:ffff::6\n Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n e3qxxx.netbird.selfhosted:\n NetBird IP: 100.100.125.70\n NetBird IPv6: 2001:db8:ffff::7\n Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n mail.netbird.selfhosted:\n NetBird IP: 100.100.147.204\n NetBird IPv6: 2001:db8:ffff::8\n Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n desktop-server.netbird.selfhosted:\n NetBird IP: 100.100.178.43\n NetBird IPv6: 2001:db8:ffff::9\n Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n e3qxxx-183-106.netbird.selfhosted:\n NetBird IP: 100.100.183.106\n NetBird IPv6: 2001:db8:ffff::a\n Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\nEvents:\n [INFO] SYSTEM (efca2c93-00b0-4c32-9caf-ffe0f983cdd3)\n Message: daemon config changed (source=startup)\n Time: 1 hour, 43 minutes ago\n Metadata: source: startup, type: config_changed\n [INFO] SYSTEM (f5af64fc-2fdd-4a3b-830c-626b8a7eea5c)\n Message: Network map updated\n Time: 1 hour, 43 minutes ago\nOS: linux/amd64\nDaemon version: 0.73.2\nCLI version: 0.73.2\nProfile: default\nManagement: Connected to https://nb.anon-6E0bs.domain:443\nSignal: Connected to https://nb.anon-N2556.domain:443\nRelays: \n [stun:nb.anon-N2556.domain:3478] is Checking...\n [rels://nb.anon-N2556.domain:443] is Available via ws\nNameservers: \nFQDN: pve01.netbird.selfhosted\nNetBird IP: 100.100.131.41/16\nNetBird IPv6: 2001:db8:ffff::b/64\nInterface type: Kernel\nWireguard port: 51820\nQuantum resistance: false\nLazy connection: false\nSSH Server: Disabled\nNetworks: -\nPeers count: 3/11 Connected\nCURRENT_MANAGEMENT_CONNECTED=true\nCURRENT_SIGNAL_CONNECTED=true\nCURRENT_PEER_LINE=Peers count: 3/11 Connected\nCURRENT_RELAY_LINE=Relays: \n=== HEALTH_UNIT ===\nResult=exit-code\nExecMainCode=1\nExecMainStatus=1\nExecStart={ path=/root/netbird-peers-health.sh ; argv[]=/root/netbird-peers-health.sh ; ignore_errors=no ; start_time=[Sun 2026-09-27 20:37:01 MSK] ; stop_time=[Sun 2026-09-27 20:37:24 MSK] ; pid=4112376 ; code=exited ; status=1 }\nLoadState=loaded\nActiveState=failed\nSubState=failed\nFragmentPath=/etc/systemd/system/netbird-peers-health.service\nHEALTH_UNIT_SHA256=168f6ced7110530ca37908dff300a6ae86de5149ad6347035560cc0e64339816\n--- UNIT_CAT ---\n# /etc/systemd/system/netbird-peers-health.service\n[Unit]\nDescription=NetBird peers health check\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/root/netbird-peers-health.sh\n--- TIMER_CAT ---\n# /etc/systemd/system/netbird-peers-health.timer\n[Unit]\nDescription=Run NetBird peers health check\n\n[Timer]\nOnBootSec=2m\nOnUnitActiveSec=5m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\nHEALTH_EXECSTART=/root/netbird-peers-health.sh\nHEALTH_SCRIPT_PATH=/root/netbird-peers-health.sh\nHEALTH_SCRIPT_SHA256=c531216ceec9962ea0536f0a929676f6bfe70339efee6f001a0cd2b07e4f35d1\n--- HEALTH_SCRIPT_REDACTED_BEGIN ---\n#!/usr/bin/env bash\nset -u\n\nHEALTH=\"/var/lib/homelab-health/netbird-peers.txt\"\nTMP=\"$(mktemp)\"\nTMP2=\"$(mktemp)\"\nTS=\"$(date -u +%Y%m%dT%H%M%SZ)\"\nATTEMPT_USED=\"1\"\nOVERALL=\"OK\"\n\ncollect_pve01() {\n echo \"SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)\"\n echo \"WT0=$(ip -br addr show wt0 2>/dev/null || true)\"\n echo \"NBSTATUS=$(netbird status 2>/dev/null | tr '\\n' ';' | cut -c1-3000 || true)\"\n}\n\ncollect_root_host() {\n local host=\"$1\"\n ssh -o BatchMode=yes -o ConnectTimeout=8 root@\"$host\" 'echo \"SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)\"; echo \"WT0=$(ip -br addr show wt0 2>/dev/null || true)\"; echo \"NBSTATUS=$(netbird status 2>/dev/null | tr \"\\n\" \";\" | cut -c1-3000 || true)\"'\n}\n\ncollect_edge() {\n ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'echo \"SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)\"; echo \"WT0=$(ip -br addr show wt0 2>/dev/null || true)\"; echo \"NBSTATUS=$(sudo netbird status 2>/dev/null | tr \"\\n\" \";\" | cut -c1-3000 || true)\"'\n}\n\nrecord_node() {\n local out=\"$1\"\n local name=\"$2\"\n local data=\"$3\"\n\n local service wt0 nbstatus ip mgmt signal peers\n service=\"$(printf \"%s\\n\" \"$data\" | sed -n 's/^SERVICE=//p' | tail -n1)\"\n wt0=\"$(printf \"%s\\n\" \"$data\" | sed -n 's/^WT0=//p' | tail -n1)\"\n nbstatus=\"$(printf \"%s\\n\" \"$data\" | sed -n 's/^NBSTATUS=//p' | tail -n1)\"\n ip=\"$(printf \"%s\" \"$wt0\" | sed -n 's/^wt0[[:space:]]\\+UNKNOWN[[:space:]]\\+\\([^ ]*\\).*/\\1/p')\"\n\n printf \"%s\" \"$nbstatus\" | grep -q \"Management: Connected\" && mgmt=\"yes\" || mgmt=\"no\"\n printf \"%s\" \"$nbstatus\" | grep -q \"Signal: Connected\" && signal=\"yes\" || signal=\"no\"\n peers=\"$(printf \"%s\" \"$nbstatus\" | sed -n 's/.*Peers count: \\([^;]*\\).*/\\1/p' | tail -n1)\"\n [ -n \"$peers\" ] || peers=\"unknown\"\n\n if [ \"$service\" != \"active/enabled\" ] || [ -z \"$ip\" ] || [ \"$mgmt\" != \"yes\" ] || [ \"$signal\" != \"yes\" ]; then\n OVERALL=\"ERROR\"\n fi\n\n echo \"$name service=${service:-missing} wt0_ip=${ip:-missing} management=$mgmt signal=$signal peers=$peers\" >>\"$out\"\n}\n\nrun_node() {\n local out=\"$1\"\n local name=\"$2\"\n shift 2\n local data\n if data=\"$(\"$@\" 2>/tmp/netbird-peers-health-node.err)\"; then\n record_node \"$out\" \"$name\" \"$data\"\n else\n OVERALL=\"ERROR\"\n echo \"$name service=ERROR wt0_ip=missing management=no signal=no peers=ssh_or_command_failed\" >>\"$out\"\n fi\n}\n\nrun_all() {\n local out=\"$1\"\n OVERALL=\"OK\"\n : > \"$out\"\n run_node \"$out\" \"pve01\" collect_pve01\n run_node \"$out\" \"pve02\" collect_root_host [PRIVATE_IP]\n run_node \"$out\" \"pve03\" collect_root_host [PRIVATE_IP]\n run_node \"$out\" \"edge-vm\" collect_edge\n}\n\nmkdir -p \"$(dirname \"$HEALTH\")\"\n\nrun_all \"$TMP\"\n\nif [ \"$OVERALL\" != \"OK\" ]; then\n sleep 15\n ATTEMPT_USED=\"2\"\n run_all \"$TMP2\"\n cp \"$TMP2\" \"$TMP\"\nfi\n\nOUT=\"$(mktemp)\"\n{\n echo \"STATUS=$OVERALL\"\n echo \"SERVICE=netbird-peers\"\n echo \"TIMESTAMP_UTC=$TS\"\n echo \"ATTEMPT_USED=$ATTEMPT_USED\"\n cat \"$TMP\"\n} > \"$OUT\"\n\nchmod 0644 \"$OUT\"\nmv \"$OUT\" \"$HEALTH\"\nrm -f \"$TMP\" \"$TMP2\"\n\n[ \"$OVERALL\" = \"OK\" ]\n--- HEALTH_SCRIPT_REDACTED_END ---\n=== HEALTH_FILE ===\nHEALTH_FILE_META=480:root:root:2026-09-27 20:37:24.472057535 +0300\nHEALTH_FILE_SHA256=2512770acb6314656393870b3cd19351109672d2c7c26e2170bbf2c8d2eff400\nSTATUS=ERROR\nSERVICE=netbird-peers\nTIMESTAMP_UTC=20260927T173701Z\nATTEMPT_USED=2\npve01 service=active/enabled wt0_ip=100.100.131.41/16 management=yes signal=yes peers=3/11 Connected\npve02 service=active/enabled wt0_ip=100.100.7.2/16 management=no signal=no peers=0/11 Connected\npve03 service=active/enabled wt0_ip=100.100.34.141/16 management=no signal=no peers=0/11 Connected\nedge-vm service=active/enabled wt0_ip=100.100.60.182/16 management=yes signal=yes peers=3/11 Connected\n=== HEALTH_JOURNAL_RECENT ===\n2026-09-27T19:56:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T19:56:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:01:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:01:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:06:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:06:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:11:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:11:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:16:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:16:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:21:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:21:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:26:32+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:26:55+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:31:38+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:32:01+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:37:01+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:37:24+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n=== EDGE_COMPARISON ===\nEDGE_LIVE_HOSTKEY_FP=SHA256:YQsA2nif7qAaa5mSUb9U0h8Q5DGy8SRCV5f+FyKUpOo\nEDGE_SSH_AUTH=true\nPeers detail:\n pve02.netbird.selfhosted:\n NetBird IP: 100.100.7.2\n NetBird IPv6: 2001:db8:ffff::\n Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n home-pc.netbird.selfhosted:\n NetBird IP: 100.100.17.159\n NetBird IPv6: 2001:db8:ffff::1\n Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 14 hours, 49 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n relay.netbird.selfhosted:\n NetBird IP: 100.100.19.1\n NetBird IPv6: 2001:db8:ffff::2\n Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE=\n Status: Connected\n -- detail --\n Connection type: P2P\n ICE candidate (Local/Remote): host/srflx\n ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822\n Relay server address: rels://nb.anon-MQCTn.domain:443\n Last connection update: 4 hours, 28 minutes ago\n Last WireGuard handshake: 2 minutes, 6 seconds ago\n Transfer status (received/sent) 142.7 KiB/38.7 KiB\n Quantum resistance: false\n Networks: -\n Latency: 57.639982ms\n\n pve03.netbird.selfhosted:\n NetBird IP: 100.100.34.141\n NetBird IPv6: 2001:db8:ffff::3\n Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n desktop-moy.netbird.selfhosted:\n NetBird IP: 100.100.76.159\n NetBird IPv6: 2001:db8:ffff::4\n Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go=\n Status: Connected\n -- detail --\n Connection type: Relayed\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: rels://nb.anon-MQCTn.domain:443\n Last connection update: 3 hours, 34 minutes ago\n Last WireGuard handshake: 1 minute, 58 seconds ago\n Transfer status (received/sent) 34.1 KiB/9.5 KiB\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n comp01.netbird.selfhosted:\n NetBird IP: 100.100.123.213\n NetBird IPv6: 2001:db8:ffff::5\n Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n e3qxxx.netbird.selfhosted:\n NetBird IP: 100.100.125.70\n NetBird IPv6: 2001:db8:ffff::6\n Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n pve01.netbird.selfhosted:\n NetBird IP: 100.100.131.41\n NetBird IPv6: 2001:db8:ffff::7\n Public key: rt0YF+0ieQPRRQrY6v+Hd2hTP8wBjK3a3zzU5kPGlS8=\n Status: Connected\n -- detail --\n Connection type: P2P\n ICE candidate (Local/Remote): host/prflx\n ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820\n Relay server address: rels://nb.anon-MQCTn.domain:443\n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: 1 minute, 50 seconds ago\n Transfer status (received/sent) 1.8 MiB/3.8 MiB\n Quantum resistance: false\n Networks: -\n Latency: 910.974µs\n\n mail.netbird.selfhosted:\n NetBird IP: 100.100.147.204\n NetBird IPv6: 2001:db8:ffff::8\n Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n desktop-server.netbird.selfhosted:\n NetBird IP: 100.100.178.43\n NetBird IPv6: 2001:db8:ffff::9\n Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n e3qxxx-183-106.netbird.selfhosted:\n NetBird IP: 100.100.183.106\n NetBird IPv6: 2001:db8:ffff::a\n Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\nEvents:\n [INFO] SYSTEM (50e1e0e8-d706-4288-a5f0-7d2f76b20209)\n Message: daemon config changed (source=startup)\n Time: 19 hours, 23 minutes ago\n Metadata: source: startup, type: config_changed\n [INFO] SYSTEM (dc5f191e-9af0-4b4b-bd71-5d4bef18f46c)\n Message: Network map updated\n Time: 15 hours, 54 minutes ago\n [INFO] SYSTEM (a3f5af7c-c1f2-48b5-991b-2ae6d05008ab)\n Message: Network map updated\n Time: 13 hours, 38 minutes ago\n [INFO] SYSTEM (3c6e7858-3c31-46a6-a553-cb69db89b0e3)\n Message: Network map updated\n Time: 6 hours, 54 minutes ago\n [INFO] SYSTEM (aeacd9ec-e926-4148-96d5-c3d384cfced1)\n Message: Network map updated\n Time: 4 hours, 31 minutes ago\n [INFO] SYSTEM (f5eb9e93-22d2-49f9-bded-cd8985a49c93)\n Message: Network map updated\n Time: 4 hours, 28 minutes ago\nOS: linux/amd64\nDaemon version: 0.73.2\nCLI version: 0.73.2\nProfile: default\nManagement: Connected to https://nb.anon-UmaXu.domain:443\nSignal: Connected to https://nb.anon-MQCTn.domain:443\nRelays: \n [stun:nb.anon-MQCTn.domain:3478] is Checking...\n [rels://nb.anon-MQCTn.domain:443] is Available via ws\nNameservers: \nFQDN: edge-vm.netbird.selfhosted\nNetBird IP: 100.100.60.182/16\nNetBird IPv6: 2001:db8:ffff::b/64\nInterface type: Kernel\nWireguard port: 51820\nQuantum resistance: false\nLazy connection: false\nSSH Server: Disabled\nNetworks: -\nPeers count: 3/11 Connected\n=== CLASSIFICATION ===\nCONTROL_PLANE_CONNECTED=true\nHEALTH_UNIT_RESULT=exit-code\nDECISION=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_COLLECTED\nNEXT_GATE=READ_EXACT_73332_AND_FIX_ONLY_PROVEN_PEER_HEALTH_CAUSE\nTASK_RESULT=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1.txt b/runtime/history/SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1.txt new file mode 100644 index 00000000..944b4516 --- /dev/null +++ b/runtime/history/SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1.txt @@ -0,0 +1,698 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=netbird-peer-health-postclose-rca-fix +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=b20b0b61da3227c774ea059fc1736a643fd915f309c46be3b20d12393af4ee8b +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=39727a42b270ac80c2cc47f04631bae8554a678a939e5c3b74ed7593c7574f72 +SANITIZED_OUTPUT_SHA256=ad60d3ada9c55ba6ad6dba757229bff607de7c72ac7813502fecbc1895149e1b +OUTPUT_BEGIN +NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_BEGIN=true +COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1 +MODE=read-only +COMPONENT=netbird-peer-health-postclose-rca-fix +MUTATION_BOUNDARY=NONE_READ_ONLY_POSTCLOSE_PEER_HEALTH_RCA +=== AUTHORITY === +HOST=pve01 +CLUSTER_QUORATE=Yes +REFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED +REFERENCE_SHA256_EXPECTED=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED +ERROR_REGISTER_SHA256_EXPECTED=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +SOURCE_73331_PRESENT=true +SOURCE_73331_STATUS=FAIL +SOURCE_73331_RC=73 +SOURCE_73331_CHANGES_MADE=false +SOURCE_73331_MUTATION_OUTCOME=NO_MUTATION +SOURCE_73331_DECISION=FAIL_NETBIRD_PEER_HEALTH_RCA_PRECONDITION +SOURCE_73331_NEXT_GATE=FIX_ONLY_FAILED_RCA_PRECONDITION +CLOSEOUT_73330_PRESENT=true +CLOSEOUT_73330_STATUS=OK +CLOSEOUT_73330_RC=0 +CLOSEOUT_73330_FINAL_ACCEPTANCE=PASS +CLOSEOUT_73330_TASK_RESULT=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT +CLOSEOUT_73330_DECISION=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT +CLOSEOUT_73330_NEXT_GATE=NONE +=== CURRENT_NETBIRD === +Peers detail: + pve02.netbird.selfhosted: + NetBird IP: 100.100.7.2 + NetBird IPv6: 2001:db8:ffff:: + Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + home-pc.netbird.selfhosted: + NetBird IP: 100.100.17.159 + NetBird IPv6: 2001:db8:ffff::1 + Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + relay.netbird.selfhosted: + NetBird IP: 100.100.19.1 + NetBird IPv6: 2001:db8:ffff::2 + Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE= + Status: Connected + -- detail -- + Connection type: P2P + ICE candidate (Local/Remote): host/srflx + ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822 + Relay server address: rels://nb.anon-N2556.domain:443 + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: 51 seconds ago + Transfer status (received/sent) 6.1 KiB/16.5 KiB + Quantum resistance: false + Networks: - + Latency: 57.51222ms + + pve03.netbird.selfhosted: + NetBird IP: 100.100.34.141 + NetBird IPv6: 2001:db8:ffff::3 + Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + edge-vm.netbird.selfhosted: + NetBird IP: 100.100.60.182 + NetBird IPv6: 2001:db8:ffff::4 + Public key: qggHuS6pj1odPRoMwZ5FjiXfmNuWiYvtyrZvCfunEBE= + Status: Connected + -- detail -- + Connection type: P2P + ICE candidate (Local/Remote): host/host + ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820 + Relay server address: rels://nb.anon-N2556.domain:443 + Last connection update: 1 hour, 42 minutes ago + Last WireGuard handshake: 1 minute, 48 seconds ago + Transfer status (received/sent) 3.8 MiB/1.8 MiB + Quantum resistance: false + Networks: - + Latency: 892.297µs + + desktop-moy.netbird.selfhosted: + NetBird IP: 100.100.76.159 + NetBird IPv6: 2001:db8:ffff::5 + Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go= + Status: Connected + -- detail -- + Connection type: Relayed + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: rels://nb.anon-N2556.domain:443 + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: 33 seconds ago + Transfer status (received/sent) 10.9 KiB/7.5 KiB + Quantum resistance: false + Networks: - + Latency: 0s + + comp01.netbird.selfhosted: + NetBird IP: 100.100.123.213 + NetBird IPv6: 2001:db8:ffff::6 + Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + e3qxxx.netbird.selfhosted: + NetBird IP: 100.100.125.70 + NetBird IPv6: 2001:db8:ffff::7 + Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + mail.netbird.selfhosted: + NetBird IP: 100.100.147.204 + NetBird IPv6: 2001:db8:ffff::8 + Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + desktop-server.netbird.selfhosted: + NetBird IP: 100.100.178.43 + NetBird IPv6: 2001:db8:ffff::9 + Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + e3qxxx-183-106.netbird.selfhosted: + NetBird IP: 100.100.183.106 + NetBird IPv6: 2001:db8:ffff::a + Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + +Events: + [INFO] SYSTEM (efca2c93-00b0-4c32-9caf-ffe0f983cdd3) + Message: daemon config changed (source=startup) + Time: 1 hour, 43 minutes ago + Metadata: source: startup, type: config_changed + [INFO] SYSTEM (f5af64fc-2fdd-4a3b-830c-626b8a7eea5c) + Message: Network map updated + Time: 1 hour, 43 minutes ago +OS: linux/amd64 +Daemon version: 0.73.2 +CLI version: 0.73.2 +Profile: default +Management: Connected to https://nb.anon-6E0bs.domain:443 +Signal: Connected to https://nb.anon-N2556.domain:443 +Relays: + [stun:nb.anon-N2556.domain:3478] is Checking... + [rels://nb.anon-N2556.domain:443] is Available via ws +Nameservers: +FQDN: pve01.netbird.selfhosted +NetBird IP: 100.100.131.41/16 +NetBird IPv6: 2001:db8:ffff::b/64 +Interface type: Kernel +Wireguard port: 51820 +Quantum resistance: false +Lazy connection: false +SSH Server: Disabled +Networks: - +Peers count: 3/11 Connected +CURRENT_MANAGEMENT_CONNECTED=true +CURRENT_SIGNAL_CONNECTED=true +CURRENT_PEER_LINE=Peers count: 3/11 Connected +CURRENT_RELAY_LINE=Relays: +=== HEALTH_UNIT === +Result=exit-code +ExecMainCode=1 +ExecMainStatus=1 +ExecStart={ path=/root/netbird-peers-health.sh ; argv[]=/root/netbird-peers-health.sh ; ignore_errors=no ; start_time=[Sun 2026-09-27 20:37:01 MSK] ; stop_time=[Sun 2026-09-27 20:37:24 MSK] ; pid=4112376 ; code=exited ; status=1 } +LoadState=loaded +ActiveState=failed +SubState=failed +FragmentPath=/etc/systemd/system/netbird-peers-health.service +HEALTH_UNIT_SHA256=168f6ced7110530ca37908dff300a6ae86de5149ad6347035560cc0e64339816 +--- UNIT_CAT --- +# /etc/systemd/system/netbird-peers-health.service +[Unit] +Description=NetBird peers health check +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/root/netbird-peers-health.sh +--- TIMER_CAT --- +# /etc/systemd/system/netbird-peers-health.timer +[Unit] +Description=Run NetBird peers health check + +[Timer] +OnBootSec=2m +OnUnitActiveSec=5m +Persistent=true + +[Install] +WantedBy=timers.target +HEALTH_EXECSTART=/root/netbird-peers-health.sh +HEALTH_SCRIPT_PATH=/root/netbird-peers-health.sh +HEALTH_SCRIPT_SHA256=c531216ceec9962ea0536f0a929676f6bfe70339efee6f001a0cd2b07e4f35d1 +--- HEALTH_SCRIPT_REDACTED_BEGIN --- +#!/usr/bin/env bash +set -u + +HEALTH="/var/lib/homelab-health/netbird-peers.txt" +TMP="$(mktemp)" +TMP2="$(mktemp)" +TS="$(date -u +%Y%m%dT%H%M%SZ)" +ATTEMPT_USED="1" +OVERALL="OK" + +collect_pve01() { + echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)" + echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)" + echo "NBSTATUS=$(netbird status 2>/dev/null | tr '\n' ';' | cut -c1-3000 || true)" +} + +collect_root_host() { + local host="$1" + ssh -o BatchMode=yes -o ConnectTimeout=8 root@"$host" 'echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)"; echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)"; echo "NBSTATUS=$(netbird status 2>/dev/null | tr "\n" ";" | cut -c1-3000 || true)"' +} + +collect_edge() { + ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)"; echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)"; echo "NBSTATUS=$(sudo netbird status 2>/dev/null | tr "\n" ";" | cut -c1-3000 || true)"' +} + +record_node() { + local out="$1" + local name="$2" + local data="$3" + + local service wt0 nbstatus ip mgmt signal peers + service="$(printf "%s\n" "$data" | sed -n 's/^SERVICE=//p' | tail -n1)" + wt0="$(printf "%s\n" "$data" | sed -n 's/^WT0=//p' | tail -n1)" + nbstatus="$(printf "%s\n" "$data" | sed -n 's/^NBSTATUS=//p' | tail -n1)" + ip="$(printf "%s" "$wt0" | sed -n 's/^wt0[[:space:]]\+UNKNOWN[[:space:]]\+\([^ ]*\).*/\1/p')" + + printf "%s" "$nbstatus" | grep -q "Management: Connected" && mgmt="yes" || mgmt="no" + printf "%s" "$nbstatus" | grep -q "Signal: Connected" && signal="yes" || signal="no" + peers="$(printf "%s" "$nbstatus" | sed -n 's/.*Peers count: \([^;]*\).*/\1/p' | tail -n1)" + [ -n "$peers" ] || peers="unknown" + + if [ "$service" != "active/enabled" ] || [ -z "$ip" ] || [ "$mgmt" != "yes" ] || [ "$signal" != "yes" ]; then + OVERALL="ERROR" + fi + + echo "$name service=${service:-missing} wt0_ip=${ip:-missing} management=$mgmt signal=$signal peers=$peers" >>"$out" +} + +run_node() { + local out="$1" + local name="$2" + shift 2 + local data + if data="$("$@" 2>/tmp/netbird-peers-health-node.err)"; then + record_node "$out" "$name" "$data" + else + OVERALL="ERROR" + echo "$name service=ERROR wt0_ip=missing management=no signal=no peers=ssh_or_command_failed" >>"$out" + fi +} + +run_all() { + local out="$1" + OVERALL="OK" + : > "$out" + run_node "$out" "pve01" collect_pve01 + run_node "$out" "pve02" collect_root_host [PRIVATE_IP] + run_node "$out" "pve03" collect_root_host [PRIVATE_IP] + run_node "$out" "edge-vm" collect_edge +} + +mkdir -p "$(dirname "$HEALTH")" + +run_all "$TMP" + +if [ "$OVERALL" != "OK" ]; then + sleep 15 + ATTEMPT_USED="2" + run_all "$TMP2" + cp "$TMP2" "$TMP" +fi + +OUT="$(mktemp)" +{ + echo "STATUS=$OVERALL" + echo "SERVICE=netbird-peers" + echo "TIMESTAMP_UTC=$TS" + echo "ATTEMPT_USED=$ATTEMPT_USED" + cat "$TMP" +} > "$OUT" + +chmod 0644 "$OUT" +mv "$OUT" "$HEALTH" +rm -f "$TMP" "$TMP2" + +[ "$OVERALL" = "OK" ] +--- HEALTH_SCRIPT_REDACTED_END --- +=== HEALTH_FILE === +HEALTH_FILE_META=480:root:root:2026-09-27 20:37:24.472057535 +0300 +HEALTH_FILE_SHA256=2512770acb6314656393870b3cd19351109672d2c7c26e2170bbf2c8d2eff400 +STATUS=ERROR +SERVICE=netbird-peers +TIMESTAMP_UTC=20260927T173701Z +ATTEMPT_USED=2 +pve01 service=active/enabled wt0_ip=100.100.131.41/16 management=yes signal=yes peers=3/11 Connected +pve02 service=active/enabled wt0_ip=100.100.7.2/16 management=no signal=no peers=0/11 Connected +pve03 service=active/enabled wt0_ip=100.100.34.141/16 management=no signal=no peers=0/11 Connected +edge-vm service=active/enabled wt0_ip=100.100.60.182/16 management=yes signal=yes peers=3/11 Connected +=== HEALTH_JOURNAL_RECENT === +2026-09-27T19:56:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T19:56:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:01:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:01:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:06:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:06:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:11:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:11:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:16:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:16:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:21:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:21:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:26:32+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:26:55+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:31:38+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:32:01+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:37:01+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:37:24+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +=== EDGE_COMPARISON === +EDGE_LIVE_HOSTKEY_FP=SHA256:YQsA2nif7qAaa5mSUb9U0h8Q5DGy8SRCV5f+FyKUpOo +EDGE_SSH_AUTH=true +Peers detail: + pve02.netbird.selfhosted: + NetBird IP: 100.100.7.2 + NetBird IPv6: 2001:db8:ffff:: + Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + home-pc.netbird.selfhosted: + NetBird IP: 100.100.17.159 + NetBird IPv6: 2001:db8:ffff::1 + Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 14 hours, 49 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + relay.netbird.selfhosted: + NetBird IP: 100.100.19.1 + NetBird IPv6: 2001:db8:ffff::2 + Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE= + Status: Connected + -- detail -- + Connection type: P2P + ICE candidate (Local/Remote): host/srflx + ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822 + Relay server address: rels://nb.anon-MQCTn.domain:443 + Last connection update: 4 hours, 28 minutes ago + Last WireGuard handshake: 2 minutes, 6 seconds ago + Transfer status (received/sent) 142.7 KiB/38.7 KiB + Quantum resistance: false + Networks: - + Latency: 57.639982ms + + pve03.netbird.selfhosted: + NetBird IP: 100.100.34.141 + NetBird IPv6: 2001:db8:ffff::3 + Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + desktop-moy.netbird.selfhosted: + NetBird IP: 100.100.76.159 + NetBird IPv6: 2001:db8:ffff::4 + Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go= + Status: Connected + -- detail -- + Connection type: Relayed + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: rels://nb.anon-MQCTn.domain:443 + Last connection update: 3 hours, 34 minutes ago + Last WireGuard handshake: 1 minute, 58 seconds ago + Transfer status (received/sent) 34.1 KiB/9.5 KiB + Quantum resistance: false + Networks: - + Latency: 0s + + comp01.netbird.selfhosted: + NetBird IP: 100.100.123.213 + NetBird IPv6: 2001:db8:ffff::5 + Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + e3qxxx.netbird.selfhosted: + NetBird IP: 100.100.125.70 + NetBird IPv6: 2001:db8:ffff::6 + Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + pve01.netbird.selfhosted: + NetBird IP: 100.100.131.41 + NetBird IPv6: 2001:db8:ffff::7 + Public key: rt0YF+0ieQPRRQrY6v+Hd2hTP8wBjK3a3zzU5kPGlS8= + Status: Connected + -- detail -- + Connection type: P2P + ICE candidate (Local/Remote): host/prflx + ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820 + Relay server address: rels://nb.anon-MQCTn.domain:443 + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: 1 minute, 50 seconds ago + Transfer status (received/sent) 1.8 MiB/3.8 MiB + Quantum resistance: false + Networks: - + Latency: 910.974µs + + mail.netbird.selfhosted: + NetBird IP: 100.100.147.204 + NetBird IPv6: 2001:db8:ffff::8 + Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + desktop-server.netbird.selfhosted: + NetBird IP: 100.100.178.43 + NetBird IPv6: 2001:db8:ffff::9 + Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + e3qxxx-183-106.netbird.selfhosted: + NetBird IP: 100.100.183.106 + NetBird IPv6: 2001:db8:ffff::a + Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + +Events: + [INFO] SYSTEM (50e1e0e8-d706-4288-a5f0-7d2f76b20209) + Message: daemon config changed (source=startup) + Time: 19 hours, 23 minutes ago + Metadata: source: startup, type: config_changed + [INFO] SYSTEM (dc5f191e-9af0-4b4b-bd71-5d4bef18f46c) + Message: Network map updated + Time: 15 hours, 54 minutes ago + [INFO] SYSTEM (a3f5af7c-c1f2-48b5-991b-2ae6d05008ab) + Message: Network map updated + Time: 13 hours, 38 minutes ago + [INFO] SYSTEM (3c6e7858-3c31-46a6-a553-cb69db89b0e3) + Message: Network map updated + Time: 6 hours, 54 minutes ago + [INFO] SYSTEM (aeacd9ec-e926-4148-96d5-c3d384cfced1) + Message: Network map updated + Time: 4 hours, 31 minutes ago + [INFO] SYSTEM (f5eb9e93-22d2-49f9-bded-cd8985a49c93) + Message: Network map updated + Time: 4 hours, 28 minutes ago +OS: linux/amd64 +Daemon version: 0.73.2 +CLI version: 0.73.2 +Profile: default +Management: Connected to https://nb.anon-UmaXu.domain:443 +Signal: Connected to https://nb.anon-MQCTn.domain:443 +Relays: + [stun:nb.anon-MQCTn.domain:3478] is Checking... + [rels://nb.anon-MQCTn.domain:443] is Available via ws +Nameservers: +FQDN: edge-vm.netbird.selfhosted +NetBird IP: 100.100.60.182/16 +NetBird IPv6: 2001:db8:ffff::b/64 +Interface type: Kernel +Wireguard port: 51820 +Quantum resistance: false +Lazy connection: false +SSH Server: Disabled +Networks: - +Peers count: 3/11 Connected +=== CLASSIFICATION === +CONTROL_PLANE_CONNECTED=true +HEALTH_UNIT_RESULT=exit-code +DECISION=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_COLLECTED +NEXT_GATE=READ_EXACT_73332_AND_FIX_ONLY_PROVEN_PEER_HEALTH_CAUSE +TASK_RESULT=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 47a84e2b..250bf8ff 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-73331R1", - "status": "FAIL", - "rc": 73, + "command_id": "SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1", + "status": "OK", + "rc": 0, "host": "pve01", "mode": "read-only", - "component": "netbird-peer-health-postclose-rca", - "started_at_utc": "2026-09-27T17:33:54Z", - "finished_at_utc": "2026-09-27T17:33:55Z", + "component": "netbird-peer-health-postclose-rca-fix", + "started_at_utc": "2026-09-27T17:38:02Z", + "finished_at_utc": "2026-09-27T17:38:06Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "1b59ca704ae2d255adca6ff919aac3c5e37c2a667fa821f8c3c9a1a65c8846a4", + "command_sha256": "b20b0b61da3227c774ea059fc1736a643fd915f309c46be3b20d12393af4ee8b", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -21,7 +21,7 @@ "result_contract_valid": true, "result_contract_status": null, "result_contract_error": null, - "command_rc": 73, + "command_rc": 0, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "bb11cc30283a0a267ef627800c4107f52d9a85f537ac0228b9b54dd719ae8aee", - "sanitized_output_sha256": "bb11cc30283a0a267ef627800c4107f52d9a85f537ac0228b9b54dd719ae8aee", + "raw_evidence_sha256": "39727a42b270ac80c2cc47f04631bae8554a678a939e5c3b74ed7593c7574f72", + "sanitized_output_sha256": "ad60d3ada9c55ba6ad6dba757229bff607de7c72ac7813502fecbc1895149e1b", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-73331R1\nMODE=read-only\nCOMPONENT=netbird-peer-health-postclose-rca\nMUTATION_BOUNDARY=NONE_READ_ONLY_POSTCLOSE_PEER_HEALTH_RCA\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_SHA256=\nERROR_REGISTER_SHA256=\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nDECISION=FAIL_NETBIRD_PEER_HEALTH_RCA_PRECONDITION\nNEXT_GATE=FIX_ONLY_FAILED_RCA_PRECONDITION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-73331R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1\nMODE=read-only\nCOMPONENT=netbird-peer-health-postclose-rca-fix\nMUTATION_BOUNDARY=NONE_READ_ONLY_POSTCLOSE_PEER_HEALTH_RCA\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED\nREFERENCE_SHA256_EXPECTED=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED\nERROR_REGISTER_SHA256_EXPECTED=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_73331_PRESENT=true\nSOURCE_73331_STATUS=FAIL\nSOURCE_73331_RC=73\nSOURCE_73331_CHANGES_MADE=false\nSOURCE_73331_MUTATION_OUTCOME=NO_MUTATION\nSOURCE_73331_DECISION=FAIL_NETBIRD_PEER_HEALTH_RCA_PRECONDITION\nSOURCE_73331_NEXT_GATE=FIX_ONLY_FAILED_RCA_PRECONDITION\nCLOSEOUT_73330_PRESENT=true\nCLOSEOUT_73330_STATUS=OK\nCLOSEOUT_73330_RC=0\nCLOSEOUT_73330_FINAL_ACCEPTANCE=PASS\nCLOSEOUT_73330_TASK_RESULT=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT\nCLOSEOUT_73330_DECISION=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT\nCLOSEOUT_73330_NEXT_GATE=NONE\n=== CURRENT_NETBIRD ===\nPeers detail:\n pve02.netbird.selfhosted:\n NetBird IP: 100.100.7.2\n NetBird IPv6: 2001:db8:ffff::\n Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n home-pc.netbird.selfhosted:\n NetBird IP: 100.100.17.159\n NetBird IPv6: 2001:db8:ffff::1\n Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n relay.netbird.selfhosted:\n NetBird IP: 100.100.19.1\n NetBird IPv6: 2001:db8:ffff::2\n Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE=\n Status: Connected\n -- detail --\n Connection type: P2P\n ICE candidate (Local/Remote): host/srflx\n ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822\n Relay server address: rels://nb.anon-N2556.domain:443\n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: 51 seconds ago\n Transfer status (received/sent) 6.1 KiB/16.5 KiB\n Quantum resistance: false\n Networks: -\n Latency: 57.51222ms\n\n pve03.netbird.selfhosted:\n NetBird IP: 100.100.34.141\n NetBird IPv6: 2001:db8:ffff::3\n Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n edge-vm.netbird.selfhosted:\n NetBird IP: 100.100.60.182\n NetBird IPv6: 2001:db8:ffff::4\n Public key: qggHuS6pj1odPRoMwZ5FjiXfmNuWiYvtyrZvCfunEBE=\n Status: Connected\n -- detail --\n Connection type: P2P\n ICE candidate (Local/Remote): host/host\n ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820\n Relay server address: rels://nb.anon-N2556.domain:443\n Last connection update: 1 hour, 42 minutes ago\n Last WireGuard handshake: 1 minute, 48 seconds ago\n Transfer status (received/sent) 3.8 MiB/1.8 MiB\n Quantum resistance: false\n Networks: -\n Latency: 892.297µs\n\n desktop-moy.netbird.selfhosted:\n NetBird IP: 100.100.76.159\n NetBird IPv6: 2001:db8:ffff::5\n Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go=\n Status: Connected\n -- detail --\n Connection type: Relayed\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: rels://nb.anon-N2556.domain:443\n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: 33 seconds ago\n Transfer status (received/sent) 10.9 KiB/7.5 KiB\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n comp01.netbird.selfhosted:\n NetBird IP: 100.100.123.213\n NetBird IPv6: 2001:db8:ffff::6\n Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n e3qxxx.netbird.selfhosted:\n NetBird IP: 100.100.125.70\n NetBird IPv6: 2001:db8:ffff::7\n Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n mail.netbird.selfhosted:\n NetBird IP: 100.100.147.204\n NetBird IPv6: 2001:db8:ffff::8\n Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n desktop-server.netbird.selfhosted:\n NetBird IP: 100.100.178.43\n NetBird IPv6: 2001:db8:ffff::9\n Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n e3qxxx-183-106.netbird.selfhosted:\n NetBird IP: 100.100.183.106\n NetBird IPv6: 2001:db8:ffff::a\n Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\nEvents:\n [INFO] SYSTEM (efca2c93-00b0-4c32-9caf-ffe0f983cdd3)\n Message: daemon config changed (source=startup)\n Time: 1 hour, 43 minutes ago\n Metadata: source: startup, type: config_changed\n [INFO] SYSTEM (f5af64fc-2fdd-4a3b-830c-626b8a7eea5c)\n Message: Network map updated\n Time: 1 hour, 43 minutes ago\nOS: linux/amd64\nDaemon version: 0.73.2\nCLI version: 0.73.2\nProfile: default\nManagement: Connected to https://nb.anon-6E0bs.domain:443\nSignal: Connected to https://nb.anon-N2556.domain:443\nRelays: \n [stun:nb.anon-N2556.domain:3478] is Checking...\n [rels://nb.anon-N2556.domain:443] is Available via ws\nNameservers: \nFQDN: pve01.netbird.selfhosted\nNetBird IP: 100.100.131.41/16\nNetBird IPv6: 2001:db8:ffff::b/64\nInterface type: Kernel\nWireguard port: 51820\nQuantum resistance: false\nLazy connection: false\nSSH Server: Disabled\nNetworks: -\nPeers count: 3/11 Connected\nCURRENT_MANAGEMENT_CONNECTED=true\nCURRENT_SIGNAL_CONNECTED=true\nCURRENT_PEER_LINE=Peers count: 3/11 Connected\nCURRENT_RELAY_LINE=Relays: \n=== HEALTH_UNIT ===\nResult=exit-code\nExecMainCode=1\nExecMainStatus=1\nExecStart={ path=/root/netbird-peers-health.sh ; argv[]=/root/netbird-peers-health.sh ; ignore_errors=no ; start_time=[Sun 2026-09-27 20:37:01 MSK] ; stop_time=[Sun 2026-09-27 20:37:24 MSK] ; pid=4112376 ; code=exited ; status=1 }\nLoadState=loaded\nActiveState=failed\nSubState=failed\nFragmentPath=/etc/systemd/system/netbird-peers-health.service\nHEALTH_UNIT_SHA256=168f6ced7110530ca37908dff300a6ae86de5149ad6347035560cc0e64339816\n--- UNIT_CAT ---\n# /etc/systemd/system/netbird-peers-health.service\n[Unit]\nDescription=NetBird peers health check\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/root/netbird-peers-health.sh\n--- TIMER_CAT ---\n# /etc/systemd/system/netbird-peers-health.timer\n[Unit]\nDescription=Run NetBird peers health check\n\n[Timer]\nOnBootSec=2m\nOnUnitActiveSec=5m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\nHEALTH_EXECSTART=/root/netbird-peers-health.sh\nHEALTH_SCRIPT_PATH=/root/netbird-peers-health.sh\nHEALTH_SCRIPT_SHA256=c531216ceec9962ea0536f0a929676f6bfe70339efee6f001a0cd2b07e4f35d1\n--- HEALTH_SCRIPT_REDACTED_BEGIN ---\n#!/usr/bin/env bash\nset -u\n\nHEALTH=\"/var/lib/homelab-health/netbird-peers.txt\"\nTMP=\"$(mktemp)\"\nTMP2=\"$(mktemp)\"\nTS=\"$(date -u +%Y%m%dT%H%M%SZ)\"\nATTEMPT_USED=\"1\"\nOVERALL=\"OK\"\n\ncollect_pve01() {\n echo \"SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)\"\n echo \"WT0=$(ip -br addr show wt0 2>/dev/null || true)\"\n echo \"NBSTATUS=$(netbird status 2>/dev/null | tr '\\n' ';' | cut -c1-3000 || true)\"\n}\n\ncollect_root_host() {\n local host=\"$1\"\n ssh -o BatchMode=yes -o ConnectTimeout=8 root@\"$host\" 'echo \"SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)\"; echo \"WT0=$(ip -br addr show wt0 2>/dev/null || true)\"; echo \"NBSTATUS=$(netbird status 2>/dev/null | tr \"\\n\" \";\" | cut -c1-3000 || true)\"'\n}\n\ncollect_edge() {\n ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'echo \"SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)\"; echo \"WT0=$(ip -br addr show wt0 2>/dev/null || true)\"; echo \"NBSTATUS=$(sudo netbird status 2>/dev/null | tr \"\\n\" \";\" | cut -c1-3000 || true)\"'\n}\n\nrecord_node() {\n local out=\"$1\"\n local name=\"$2\"\n local data=\"$3\"\n\n local service wt0 nbstatus ip mgmt signal peers\n service=\"$(printf \"%s\\n\" \"$data\" | sed -n 's/^SERVICE=//p' | tail -n1)\"\n wt0=\"$(printf \"%s\\n\" \"$data\" | sed -n 's/^WT0=//p' | tail -n1)\"\n nbstatus=\"$(printf \"%s\\n\" \"$data\" | sed -n 's/^NBSTATUS=//p' | tail -n1)\"\n ip=\"$(printf \"%s\" \"$wt0\" | sed -n 's/^wt0[[:space:]]\\+UNKNOWN[[:space:]]\\+\\([^ ]*\\).*/\\1/p')\"\n\n printf \"%s\" \"$nbstatus\" | grep -q \"Management: Connected\" && mgmt=\"yes\" || mgmt=\"no\"\n printf \"%s\" \"$nbstatus\" | grep -q \"Signal: Connected\" && signal=\"yes\" || signal=\"no\"\n peers=\"$(printf \"%s\" \"$nbstatus\" | sed -n 's/.*Peers count: \\([^;]*\\).*/\\1/p' | tail -n1)\"\n [ -n \"$peers\" ] || peers=\"unknown\"\n\n if [ \"$service\" != \"active/enabled\" ] || [ -z \"$ip\" ] || [ \"$mgmt\" != \"yes\" ] || [ \"$signal\" != \"yes\" ]; then\n OVERALL=\"ERROR\"\n fi\n\n echo \"$name service=${service:-missing} wt0_ip=${ip:-missing} management=$mgmt signal=$signal peers=$peers\" >>\"$out\"\n}\n\nrun_node() {\n local out=\"$1\"\n local name=\"$2\"\n shift 2\n local data\n if data=\"$(\"$@\" 2>/tmp/netbird-peers-health-node.err)\"; then\n record_node \"$out\" \"$name\" \"$data\"\n else\n OVERALL=\"ERROR\"\n echo \"$name service=ERROR wt0_ip=missing management=no signal=no peers=ssh_or_command_failed\" >>\"$out\"\n fi\n}\n\nrun_all() {\n local out=\"$1\"\n OVERALL=\"OK\"\n : > \"$out\"\n run_node \"$out\" \"pve01\" collect_pve01\n run_node \"$out\" \"pve02\" collect_root_host [PRIVATE_IP]\n run_node \"$out\" \"pve03\" collect_root_host [PRIVATE_IP]\n run_node \"$out\" \"edge-vm\" collect_edge\n}\n\nmkdir -p \"$(dirname \"$HEALTH\")\"\n\nrun_all \"$TMP\"\n\nif [ \"$OVERALL\" != \"OK\" ]; then\n sleep 15\n ATTEMPT_USED=\"2\"\n run_all \"$TMP2\"\n cp \"$TMP2\" \"$TMP\"\nfi\n\nOUT=\"$(mktemp)\"\n{\n echo \"STATUS=$OVERALL\"\n echo \"SERVICE=netbird-peers\"\n echo \"TIMESTAMP_UTC=$TS\"\n echo \"ATTEMPT_USED=$ATTEMPT_USED\"\n cat \"$TMP\"\n} > \"$OUT\"\n\nchmod 0644 \"$OUT\"\nmv \"$OUT\" \"$HEALTH\"\nrm -f \"$TMP\" \"$TMP2\"\n\n[ \"$OVERALL\" = \"OK\" ]\n--- HEALTH_SCRIPT_REDACTED_END ---\n=== HEALTH_FILE ===\nHEALTH_FILE_META=480:root:root:2026-09-27 20:37:24.472057535 +0300\nHEALTH_FILE_SHA256=2512770acb6314656393870b3cd19351109672d2c7c26e2170bbf2c8d2eff400\nSTATUS=ERROR\nSERVICE=netbird-peers\nTIMESTAMP_UTC=20260927T173701Z\nATTEMPT_USED=2\npve01 service=active/enabled wt0_ip=100.100.131.41/16 management=yes signal=yes peers=3/11 Connected\npve02 service=active/enabled wt0_ip=100.100.7.2/16 management=no signal=no peers=0/11 Connected\npve03 service=active/enabled wt0_ip=100.100.34.141/16 management=no signal=no peers=0/11 Connected\nedge-vm service=active/enabled wt0_ip=100.100.60.182/16 management=yes signal=yes peers=3/11 Connected\n=== HEALTH_JOURNAL_RECENT ===\n2026-09-27T19:56:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T19:56:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:01:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:01:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:06:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:06:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:11:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:11:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:16:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:16:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:21:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:21:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:26:32+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:26:55+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:31:38+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:32:01+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n2026-09-27T20:37:01+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check...\n2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'.\n2026-09-27T20:37:24+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check.\n=== EDGE_COMPARISON ===\nEDGE_LIVE_HOSTKEY_FP=SHA256:YQsA2nif7qAaa5mSUb9U0h8Q5DGy8SRCV5f+FyKUpOo\nEDGE_SSH_AUTH=true\nPeers detail:\n pve02.netbird.selfhosted:\n NetBird IP: 100.100.7.2\n NetBird IPv6: 2001:db8:ffff::\n Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n home-pc.netbird.selfhosted:\n NetBird IP: 100.100.17.159\n NetBird IPv6: 2001:db8:ffff::1\n Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 14 hours, 49 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n relay.netbird.selfhosted:\n NetBird IP: 100.100.19.1\n NetBird IPv6: 2001:db8:ffff::2\n Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE=\n Status: Connected\n -- detail --\n Connection type: P2P\n ICE candidate (Local/Remote): host/srflx\n ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822\n Relay server address: rels://nb.anon-MQCTn.domain:443\n Last connection update: 4 hours, 28 minutes ago\n Last WireGuard handshake: 2 minutes, 6 seconds ago\n Transfer status (received/sent) 142.7 KiB/38.7 KiB\n Quantum resistance: false\n Networks: -\n Latency: 57.639982ms\n\n pve03.netbird.selfhosted:\n NetBird IP: 100.100.34.141\n NetBird IPv6: 2001:db8:ffff::3\n Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n desktop-moy.netbird.selfhosted:\n NetBird IP: 100.100.76.159\n NetBird IPv6: 2001:db8:ffff::4\n Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go=\n Status: Connected\n -- detail --\n Connection type: Relayed\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: rels://nb.anon-MQCTn.domain:443\n Last connection update: 3 hours, 34 minutes ago\n Last WireGuard handshake: 1 minute, 58 seconds ago\n Transfer status (received/sent) 34.1 KiB/9.5 KiB\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n comp01.netbird.selfhosted:\n NetBird IP: 100.100.123.213\n NetBird IPv6: 2001:db8:ffff::5\n Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n e3qxxx.netbird.selfhosted:\n NetBird IP: 100.100.125.70\n NetBird IPv6: 2001:db8:ffff::6\n Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n pve01.netbird.selfhosted:\n NetBird IP: 100.100.131.41\n NetBird IPv6: 2001:db8:ffff::7\n Public key: rt0YF+0ieQPRRQrY6v+Hd2hTP8wBjK3a3zzU5kPGlS8=\n Status: Connected\n -- detail --\n Connection type: P2P\n ICE candidate (Local/Remote): host/prflx\n ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820\n Relay server address: rels://nb.anon-MQCTn.domain:443\n Last connection update: 1 hour, 43 minutes ago\n Last WireGuard handshake: 1 minute, 50 seconds ago\n Transfer status (received/sent) 1.8 MiB/3.8 MiB\n Quantum resistance: false\n Networks: -\n Latency: 910.974µs\n\n mail.netbird.selfhosted:\n NetBird IP: 100.100.147.204\n NetBird IPv6: 2001:db8:ffff::8\n Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n desktop-server.netbird.selfhosted:\n NetBird IP: 100.100.178.43\n NetBird IPv6: 2001:db8:ffff::9\n Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\n e3qxxx-183-106.netbird.selfhosted:\n NetBird IP: 100.100.183.106\n NetBird IPv6: 2001:db8:ffff::a\n Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o=\n Status: Connecting\n -- detail --\n Connection type: -\n ICE candidate (Local/Remote): -/-\n ICE candidate endpoints (Local/Remote): -/-\n Relay server address: \n Last connection update: 15 hours, 54 minutes ago\n Last WireGuard handshake: -\n Transfer status (received/sent) 0 B/0 B\n Quantum resistance: false\n Networks: -\n Latency: 0s\n\nEvents:\n [INFO] SYSTEM (50e1e0e8-d706-4288-a5f0-7d2f76b20209)\n Message: daemon config changed (source=startup)\n Time: 19 hours, 23 minutes ago\n Metadata: source: startup, type: config_changed\n [INFO] SYSTEM (dc5f191e-9af0-4b4b-bd71-5d4bef18f46c)\n Message: Network map updated\n Time: 15 hours, 54 minutes ago\n [INFO] SYSTEM (a3f5af7c-c1f2-48b5-991b-2ae6d05008ab)\n Message: Network map updated\n Time: 13 hours, 38 minutes ago\n [INFO] SYSTEM (3c6e7858-3c31-46a6-a553-cb69db89b0e3)\n Message: Network map updated\n Time: 6 hours, 54 minutes ago\n [INFO] SYSTEM (aeacd9ec-e926-4148-96d5-c3d384cfced1)\n Message: Network map updated\n Time: 4 hours, 31 minutes ago\n [INFO] SYSTEM (f5eb9e93-22d2-49f9-bded-cd8985a49c93)\n Message: Network map updated\n Time: 4 hours, 28 minutes ago\nOS: linux/amd64\nDaemon version: 0.73.2\nCLI version: 0.73.2\nProfile: default\nManagement: Connected to https://nb.anon-UmaXu.domain:443\nSignal: Connected to https://nb.anon-MQCTn.domain:443\nRelays: \n [stun:nb.anon-MQCTn.domain:3478] is Checking...\n [rels://nb.anon-MQCTn.domain:443] is Available via ws\nNameservers: \nFQDN: edge-vm.netbird.selfhosted\nNetBird IP: 100.100.60.182/16\nNetBird IPv6: 2001:db8:ffff::b/64\nInterface type: Kernel\nWireguard port: 51820\nQuantum resistance: false\nLazy connection: false\nSSH Server: Disabled\nNetworks: -\nPeers count: 3/11 Connected\n=== CLASSIFICATION ===\nCONTROL_PLANE_CONNECTED=true\nHEALTH_UNIT_RESULT=exit-code\nDECISION=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_COLLECTED\nNEXT_GATE=READ_EXACT_73332_AND_FIX_ONLY_PROVEN_PEER_HEALTH_CAUSE\nTASK_RESULT=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index c46e6dbf..944b4516 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-73331R1 -STATUS=FAIL -RC=73 +COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1 +STATUS=OK +RC=0 HOST=pve01 MODE=read-only -COMPONENT=netbird-peer-health-postclose-rca +COMPONENT=netbird-peer-health-postclose-rca-fix REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=1b59ca704ae2d255adca6ff919aac3c5e37c2a667fa821f8c3c9a1a65c8846a4 +COMMAND_SHA256=b20b0b61da3227c774ea059fc1736a643fd915f309c46be3b20d12393af4ee8b DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=73 +COMMAND_RC=0 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,23 +24,675 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=bb11cc30283a0a267ef627800c4107f52d9a85f537ac0228b9b54dd719ae8aee -SANITIZED_OUTPUT_SHA256=bb11cc30283a0a267ef627800c4107f52d9a85f537ac0228b9b54dd719ae8aee +RAW_EVIDENCE_SHA256=39727a42b270ac80c2cc47f04631bae8554a678a939e5c3b74ed7593c7574f72 +SANITIZED_OUTPUT_SHA256=ad60d3ada9c55ba6ad6dba757229bff607de7c72ac7813502fecbc1895149e1b OUTPUT_BEGIN NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_BEGIN=true -COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-73331R1 +COMMAND_ID=SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1 MODE=read-only -COMPONENT=netbird-peer-health-postclose-rca +COMPONENT=netbird-peer-health-postclose-rca-fix MUTATION_BOUNDARY=NONE_READ_ONLY_POSTCLOSE_PEER_HEALTH_RCA === AUTHORITY === HOST=pve01 CLUSTER_QUORATE=Yes -REFERENCE_SHA256= -ERROR_REGISTER_SHA256= +REFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED +REFERENCE_SHA256_EXPECTED=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED +ERROR_REGISTER_SHA256_EXPECTED=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -DECISION=FAIL_NETBIRD_PEER_HEALTH_RCA_PRECONDITION -NEXT_GATE=FIX_ONLY_FAILED_RCA_PRECONDITION -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-73331R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +SOURCE_73331_PRESENT=true +SOURCE_73331_STATUS=FAIL +SOURCE_73331_RC=73 +SOURCE_73331_CHANGES_MADE=false +SOURCE_73331_MUTATION_OUTCOME=NO_MUTATION +SOURCE_73331_DECISION=FAIL_NETBIRD_PEER_HEALTH_RCA_PRECONDITION +SOURCE_73331_NEXT_GATE=FIX_ONLY_FAILED_RCA_PRECONDITION +CLOSEOUT_73330_PRESENT=true +CLOSEOUT_73330_STATUS=OK +CLOSEOUT_73330_RC=0 +CLOSEOUT_73330_FINAL_ACCEPTANCE=PASS +CLOSEOUT_73330_TASK_RESULT=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT +CLOSEOUT_73330_DECISION=PASS_NETBIRD_SKLADCHIK_FULL_CLOSEOUT +CLOSEOUT_73330_NEXT_GATE=NONE +=== CURRENT_NETBIRD === +Peers detail: + pve02.netbird.selfhosted: + NetBird IP: 100.100.7.2 + NetBird IPv6: 2001:db8:ffff:: + Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + home-pc.netbird.selfhosted: + NetBird IP: 100.100.17.159 + NetBird IPv6: 2001:db8:ffff::1 + Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + relay.netbird.selfhosted: + NetBird IP: 100.100.19.1 + NetBird IPv6: 2001:db8:ffff::2 + Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE= + Status: Connected + -- detail -- + Connection type: P2P + ICE candidate (Local/Remote): host/srflx + ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822 + Relay server address: rels://nb.anon-N2556.domain:443 + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: 51 seconds ago + Transfer status (received/sent) 6.1 KiB/16.5 KiB + Quantum resistance: false + Networks: - + Latency: 57.51222ms + + pve03.netbird.selfhosted: + NetBird IP: 100.100.34.141 + NetBird IPv6: 2001:db8:ffff::3 + Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + edge-vm.netbird.selfhosted: + NetBird IP: 100.100.60.182 + NetBird IPv6: 2001:db8:ffff::4 + Public key: qggHuS6pj1odPRoMwZ5FjiXfmNuWiYvtyrZvCfunEBE= + Status: Connected + -- detail -- + Connection type: P2P + ICE candidate (Local/Remote): host/host + ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820 + Relay server address: rels://nb.anon-N2556.domain:443 + Last connection update: 1 hour, 42 minutes ago + Last WireGuard handshake: 1 minute, 48 seconds ago + Transfer status (received/sent) 3.8 MiB/1.8 MiB + Quantum resistance: false + Networks: - + Latency: 892.297µs + + desktop-moy.netbird.selfhosted: + NetBird IP: 100.100.76.159 + NetBird IPv6: 2001:db8:ffff::5 + Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go= + Status: Connected + -- detail -- + Connection type: Relayed + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: rels://nb.anon-N2556.domain:443 + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: 33 seconds ago + Transfer status (received/sent) 10.9 KiB/7.5 KiB + Quantum resistance: false + Networks: - + Latency: 0s + + comp01.netbird.selfhosted: + NetBird IP: 100.100.123.213 + NetBird IPv6: 2001:db8:ffff::6 + Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + e3qxxx.netbird.selfhosted: + NetBird IP: 100.100.125.70 + NetBird IPv6: 2001:db8:ffff::7 + Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + mail.netbird.selfhosted: + NetBird IP: 100.100.147.204 + NetBird IPv6: 2001:db8:ffff::8 + Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + desktop-server.netbird.selfhosted: + NetBird IP: 100.100.178.43 + NetBird IPv6: 2001:db8:ffff::9 + Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + e3qxxx-183-106.netbird.selfhosted: + NetBird IP: 100.100.183.106 + NetBird IPv6: 2001:db8:ffff::a + Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + +Events: + [INFO] SYSTEM (efca2c93-00b0-4c32-9caf-ffe0f983cdd3) + Message: daemon config changed (source=startup) + Time: 1 hour, 43 minutes ago + Metadata: source: startup, type: config_changed + [INFO] SYSTEM (f5af64fc-2fdd-4a3b-830c-626b8a7eea5c) + Message: Network map updated + Time: 1 hour, 43 minutes ago +OS: linux/amd64 +Daemon version: 0.73.2 +CLI version: 0.73.2 +Profile: default +Management: Connected to https://nb.anon-6E0bs.domain:443 +Signal: Connected to https://nb.anon-N2556.domain:443 +Relays: + [stun:nb.anon-N2556.domain:3478] is Checking... + [rels://nb.anon-N2556.domain:443] is Available via ws +Nameservers: +FQDN: pve01.netbird.selfhosted +NetBird IP: 100.100.131.41/16 +NetBird IPv6: 2001:db8:ffff::b/64 +Interface type: Kernel +Wireguard port: 51820 +Quantum resistance: false +Lazy connection: false +SSH Server: Disabled +Networks: - +Peers count: 3/11 Connected +CURRENT_MANAGEMENT_CONNECTED=true +CURRENT_SIGNAL_CONNECTED=true +CURRENT_PEER_LINE=Peers count: 3/11 Connected +CURRENT_RELAY_LINE=Relays: +=== HEALTH_UNIT === +Result=exit-code +ExecMainCode=1 +ExecMainStatus=1 +ExecStart={ path=/root/netbird-peers-health.sh ; argv[]=/root/netbird-peers-health.sh ; ignore_errors=no ; start_time=[Sun 2026-09-27 20:37:01 MSK] ; stop_time=[Sun 2026-09-27 20:37:24 MSK] ; pid=4112376 ; code=exited ; status=1 } +LoadState=loaded +ActiveState=failed +SubState=failed +FragmentPath=/etc/systemd/system/netbird-peers-health.service +HEALTH_UNIT_SHA256=168f6ced7110530ca37908dff300a6ae86de5149ad6347035560cc0e64339816 +--- UNIT_CAT --- +# /etc/systemd/system/netbird-peers-health.service +[Unit] +Description=NetBird peers health check +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/root/netbird-peers-health.sh +--- TIMER_CAT --- +# /etc/systemd/system/netbird-peers-health.timer +[Unit] +Description=Run NetBird peers health check + +[Timer] +OnBootSec=2m +OnUnitActiveSec=5m +Persistent=true + +[Install] +WantedBy=timers.target +HEALTH_EXECSTART=/root/netbird-peers-health.sh +HEALTH_SCRIPT_PATH=/root/netbird-peers-health.sh +HEALTH_SCRIPT_SHA256=c531216ceec9962ea0536f0a929676f6bfe70339efee6f001a0cd2b07e4f35d1 +--- HEALTH_SCRIPT_REDACTED_BEGIN --- +#!/usr/bin/env bash +set -u + +HEALTH="/var/lib/homelab-health/netbird-peers.txt" +TMP="$(mktemp)" +TMP2="$(mktemp)" +TS="$(date -u +%Y%m%dT%H%M%SZ)" +ATTEMPT_USED="1" +OVERALL="OK" + +collect_pve01() { + echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)" + echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)" + echo "NBSTATUS=$(netbird status 2>/dev/null | tr '\n' ';' | cut -c1-3000 || true)" +} + +collect_root_host() { + local host="$1" + ssh -o BatchMode=yes -o ConnectTimeout=8 root@"$host" 'echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)"; echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)"; echo "NBSTATUS=$(netbird status 2>/dev/null | tr "\n" ";" | cut -c1-3000 || true)"' +} + +collect_edge() { + ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'echo "SERVICE=$(systemctl is-active netbird 2>/dev/null || true)/$(systemctl is-enabled netbird 2>/dev/null || true)"; echo "WT0=$(ip -br addr show wt0 2>/dev/null || true)"; echo "NBSTATUS=$(sudo netbird status 2>/dev/null | tr "\n" ";" | cut -c1-3000 || true)"' +} + +record_node() { + local out="$1" + local name="$2" + local data="$3" + + local service wt0 nbstatus ip mgmt signal peers + service="$(printf "%s\n" "$data" | sed -n 's/^SERVICE=//p' | tail -n1)" + wt0="$(printf "%s\n" "$data" | sed -n 's/^WT0=//p' | tail -n1)" + nbstatus="$(printf "%s\n" "$data" | sed -n 's/^NBSTATUS=//p' | tail -n1)" + ip="$(printf "%s" "$wt0" | sed -n 's/^wt0[[:space:]]\+UNKNOWN[[:space:]]\+\([^ ]*\).*/\1/p')" + + printf "%s" "$nbstatus" | grep -q "Management: Connected" && mgmt="yes" || mgmt="no" + printf "%s" "$nbstatus" | grep -q "Signal: Connected" && signal="yes" || signal="no" + peers="$(printf "%s" "$nbstatus" | sed -n 's/.*Peers count: \([^;]*\).*/\1/p' | tail -n1)" + [ -n "$peers" ] || peers="unknown" + + if [ "$service" != "active/enabled" ] || [ -z "$ip" ] || [ "$mgmt" != "yes" ] || [ "$signal" != "yes" ]; then + OVERALL="ERROR" + fi + + echo "$name service=${service:-missing} wt0_ip=${ip:-missing} management=$mgmt signal=$signal peers=$peers" >>"$out" +} + +run_node() { + local out="$1" + local name="$2" + shift 2 + local data + if data="$("$@" 2>/tmp/netbird-peers-health-node.err)"; then + record_node "$out" "$name" "$data" + else + OVERALL="ERROR" + echo "$name service=ERROR wt0_ip=missing management=no signal=no peers=ssh_or_command_failed" >>"$out" + fi +} + +run_all() { + local out="$1" + OVERALL="OK" + : > "$out" + run_node "$out" "pve01" collect_pve01 + run_node "$out" "pve02" collect_root_host [PRIVATE_IP] + run_node "$out" "pve03" collect_root_host [PRIVATE_IP] + run_node "$out" "edge-vm" collect_edge +} + +mkdir -p "$(dirname "$HEALTH")" + +run_all "$TMP" + +if [ "$OVERALL" != "OK" ]; then + sleep 15 + ATTEMPT_USED="2" + run_all "$TMP2" + cp "$TMP2" "$TMP" +fi + +OUT="$(mktemp)" +{ + echo "STATUS=$OVERALL" + echo "SERVICE=netbird-peers" + echo "TIMESTAMP_UTC=$TS" + echo "ATTEMPT_USED=$ATTEMPT_USED" + cat "$TMP" +} > "$OUT" + +chmod 0644 "$OUT" +mv "$OUT" "$HEALTH" +rm -f "$TMP" "$TMP2" + +[ "$OVERALL" = "OK" ] +--- HEALTH_SCRIPT_REDACTED_END --- +=== HEALTH_FILE === +HEALTH_FILE_META=480:root:root:2026-09-27 20:37:24.472057535 +0300 +HEALTH_FILE_SHA256=2512770acb6314656393870b3cd19351109672d2c7c26e2170bbf2c8d2eff400 +STATUS=ERROR +SERVICE=netbird-peers +TIMESTAMP_UTC=20260927T173701Z +ATTEMPT_USED=2 +pve01 service=active/enabled wt0_ip=100.100.131.41/16 management=yes signal=yes peers=3/11 Connected +pve02 service=active/enabled wt0_ip=100.100.7.2/16 management=no signal=no peers=0/11 Connected +pve03 service=active/enabled wt0_ip=100.100.34.141/16 management=no signal=no peers=0/11 Connected +edge-vm service=active/enabled wt0_ip=100.100.60.182/16 management=yes signal=yes peers=3/11 Connected +=== HEALTH_JOURNAL_RECENT === +2026-09-27T19:56:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T19:56:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T19:56:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:01:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:01:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:01:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:06:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:06:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:06:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:11:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:11:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:11:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:16:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:16:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:16:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:21:16+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:21:39+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:21:39+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:26:32+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:26:55+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:26:55+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:31:38+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:32:01+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:32:01+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +2026-09-27T20:37:01+03:00 pve01 systemd[1]: Starting netbird-peers-health.service - NetBird peers health check... +2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Main process exited, code=exited, status=1/FAILURE +2026-09-27T20:37:24+03:00 pve01 systemd[1]: netbird-peers-health.service: Failed with result 'exit-code'. +2026-09-27T20:37:24+03:00 pve01 systemd[1]: Failed to start netbird-peers-health.service - NetBird peers health check. +=== EDGE_COMPARISON === +EDGE_LIVE_HOSTKEY_FP=SHA256:YQsA2nif7qAaa5mSUb9U0h8Q5DGy8SRCV5f+FyKUpOo +EDGE_SSH_AUTH=true +Peers detail: + pve02.netbird.selfhosted: + NetBird IP: 100.100.7.2 + NetBird IPv6: 2001:db8:ffff:: + Public key: ioHRHMCrw1revg7p7drr2MZxudHpqfhn9GIHvuDNHBs= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + home-pc.netbird.selfhosted: + NetBird IP: 100.100.17.159 + NetBird IPv6: 2001:db8:ffff::1 + Public key: CoTpTOUD2TU924swK54vv1yhdCKiMwc5sM8PJVRyWj8= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 14 hours, 49 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + relay.netbird.selfhosted: + NetBird IP: 100.100.19.1 + NetBird IPv6: 2001:db8:ffff::2 + Public key: fj5TW6oRrm5KdlrEQ/Zh/JFOO4DBNIIumt6TmR3FZyE= + Status: Connected + -- detail -- + Connection type: P2P + ICE candidate (Local/Remote): host/srflx + ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/198.51.100.0:51822 + Relay server address: rels://nb.anon-MQCTn.domain:443 + Last connection update: 4 hours, 28 minutes ago + Last WireGuard handshake: 2 minutes, 6 seconds ago + Transfer status (received/sent) 142.7 KiB/38.7 KiB + Quantum resistance: false + Networks: - + Latency: 57.639982ms + + pve03.netbird.selfhosted: + NetBird IP: 100.100.34.141 + NetBird IPv6: 2001:db8:ffff::3 + Public key: oXaA1VeKq1d/v5F8w0v/7G1DFL0yJMmxdSk/4X9j2j4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + desktop-moy.netbird.selfhosted: + NetBird IP: 100.100.76.159 + NetBird IPv6: 2001:db8:ffff::4 + Public key: E6+z+dlShOTEO+JOFoHKGcVHqnAcLZGz/Q8i9VAh5Go= + Status: Connected + -- detail -- + Connection type: Relayed + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: rels://nb.anon-MQCTn.domain:443 + Last connection update: 3 hours, 34 minutes ago + Last WireGuard handshake: 1 minute, 58 seconds ago + Transfer status (received/sent) 34.1 KiB/9.5 KiB + Quantum resistance: false + Networks: - + Latency: 0s + + comp01.netbird.selfhosted: + NetBird IP: 100.100.123.213 + NetBird IPv6: 2001:db8:ffff::5 + Public key: FJOIhkkFfX7aDMw9wDCYbcH4LPV1FDi2AgVnbMYCcn4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + e3qxxx.netbird.selfhosted: + NetBird IP: 100.100.125.70 + NetBird IPv6: 2001:db8:ffff::6 + Public key: 1KFF8QAhtRRdXk6+UbG685CyIXKfUWB/dh9CkgDR0U4= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + pve01.netbird.selfhosted: + NetBird IP: 100.100.131.41 + NetBird IPv6: 2001:db8:ffff::7 + Public key: rt0YF+0ieQPRRQrY6v+Hd2hTP8wBjK3a3zzU5kPGlS8= + Status: Connected + -- detail -- + Connection type: P2P + ICE candidate (Local/Remote): host/prflx + ICE candidate endpoints (Local/Remote): [PRIVATE_IP]:51820/[PRIVATE_IP]:51820 + Relay server address: rels://nb.anon-MQCTn.domain:443 + Last connection update: 1 hour, 43 minutes ago + Last WireGuard handshake: 1 minute, 50 seconds ago + Transfer status (received/sent) 1.8 MiB/3.8 MiB + Quantum resistance: false + Networks: - + Latency: 910.974µs + + mail.netbird.selfhosted: + NetBird IP: 100.100.147.204 + NetBird IPv6: 2001:db8:ffff::8 + Public key: bxGZ10vUCYEymrmSG1y79MW3vLFhZMeBQpo3MMeaiAA= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + desktop-server.netbird.selfhosted: + NetBird IP: 100.100.178.43 + NetBird IPv6: 2001:db8:ffff::9 + Public key: NKkIZLkKyWCA94ApPBGEji7naiy9GGYCoLJ6YRwVdSM= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + + e3qxxx-183-106.netbird.selfhosted: + NetBird IP: 100.100.183.106 + NetBird IPv6: 2001:db8:ffff::a + Public key: Vr8YnY6JHXD6aBKcSGtS+oodjQ+eizauZCOFnZfEb0o= + Status: Connecting + -- detail -- + Connection type: - + ICE candidate (Local/Remote): -/- + ICE candidate endpoints (Local/Remote): -/- + Relay server address: + Last connection update: 15 hours, 54 minutes ago + Last WireGuard handshake: - + Transfer status (received/sent) 0 B/0 B + Quantum resistance: false + Networks: - + Latency: 0s + +Events: + [INFO] SYSTEM (50e1e0e8-d706-4288-a5f0-7d2f76b20209) + Message: daemon config changed (source=startup) + Time: 19 hours, 23 minutes ago + Metadata: source: startup, type: config_changed + [INFO] SYSTEM (dc5f191e-9af0-4b4b-bd71-5d4bef18f46c) + Message: Network map updated + Time: 15 hours, 54 minutes ago + [INFO] SYSTEM (a3f5af7c-c1f2-48b5-991b-2ae6d05008ab) + Message: Network map updated + Time: 13 hours, 38 minutes ago + [INFO] SYSTEM (3c6e7858-3c31-46a6-a553-cb69db89b0e3) + Message: Network map updated + Time: 6 hours, 54 minutes ago + [INFO] SYSTEM (aeacd9ec-e926-4148-96d5-c3d384cfced1) + Message: Network map updated + Time: 4 hours, 31 minutes ago + [INFO] SYSTEM (f5eb9e93-22d2-49f9-bded-cd8985a49c93) + Message: Network map updated + Time: 4 hours, 28 minutes ago +OS: linux/amd64 +Daemon version: 0.73.2 +CLI version: 0.73.2 +Profile: default +Management: Connected to https://nb.anon-UmaXu.domain:443 +Signal: Connected to https://nb.anon-MQCTn.domain:443 +Relays: + [stun:nb.anon-MQCTn.domain:3478] is Checking... + [rels://nb.anon-MQCTn.domain:443] is Available via ws +Nameservers: +FQDN: edge-vm.netbird.selfhosted +NetBird IP: 100.100.60.182/16 +NetBird IPv6: 2001:db8:ffff::b/64 +Interface type: Kernel +Wireguard port: 51820 +Quantum resistance: false +Lazy connection: false +SSH Server: Disabled +Networks: - +Peers count: 3/11 Connected +=== CLASSIFICATION === +CONTROL_PLANE_CONNECTED=true +HEALTH_UNIT_RESULT=exit-code +DECISION=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_COLLECTED +NEXT_GATE=READ_EXACT_73332_AND_FIX_ONLY_PROVEN_PEER_HEALTH_CAUSE +TASK_RESULT=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PEER-HEALTH-POSTCLOSE-RCA-FIX-73332R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END