diff --git a/runtime/history/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1.json b/runtime/history/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1.json new file mode 100644 index 00000000..e30fe550 --- /dev/null +++ b/runtime/history/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1", + "status": "FAIL", + "rc": 31, + "host": "pve01", + "mode": "change", + "component": "netbird-pve23-relay-override-repair", + "started_at_utc": "2026-09-27T17:50:57Z", + "finished_at_utc": "2026-09-27T17:51:19Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "858ca707cc2639542b347eecec8562ffb0404891bdd2dc52facf0711cf78a8d8", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": true, + "result_contract_valid": true, + "result_contract_status": "FAIL", + "result_contract_error": null, + "command_rc": 31, + "changes_made": true, + "rollback_started": true, + "rollback_restored": false, + "mutation_outcome": "ROLLBACK_FAILED", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "55e81f5f3fb26257de530bd3965969f28254c1423e2fa67b5e9a202de585e57a", + "sanitized_output_sha256": "6f86a937e51d2a3d550ef83a579bdce2fcbf152399f53cb91bca7d699f34d631", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "NETBIRD_PVE23_RELAY_OVERRIDE_REPAIR_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\nMODE=change\nCOMPONENT=netbird-pve23-relay-override-repair\nMUTATION_BOUNDARY=PVE02_PVE03_ONLY_SCOPED_ETC_HOSTS_NETBIRD_RELAY_OVERRIDE_PLUS_NETBIRD_RESTART_PLUS_EXISTING_PEER_HEALTH_ONESHOT;NO_PVE01_HOSTS_OR_NETBIRD_CHANGE;NO_EDGE_CHANGE;NO_ROUTE_FIREWALL_DNS_PROVIDER_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED\nERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_73333_STATUS=OK\nSOURCE_73333_RC=0\nSOURCE_73333_DECISION=PASS_PVE23_MISSING_RELAY_OVERRIDE_ROOT_CAUSE\nPVE01_RELAY_OVERRIDE_COUNT=1\nPVE01_FORCED85_DASH=200\nPVE01_FORCED85_OIDC=200\nPVE02_SSH_HOSTNAME=pve02\nPVE03_SSH_HOSTNAME=pve03\n=== PVE02_PRE ===\nHOSTNAME=pve02\nSERVICE=active/enabled\nWT0=wt0 UNKNOWN 100.100.7.2/16 [PRIVATE_IPV6]/64 \nManagement: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nSignal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout\nRelays: 0/2 Available\nFQDN: pve02.netbird.selfhosted\nNetBird IP: 100.100.7.2/16\nPeers count: 0/11 Connected\nHOSTS_BEGIN\nHOSTS_END\nHOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nDNS_nb_pvepro_ru=46.16.34.129\nDNS_nb_gram1_ru=46.16.34.129\nHTTPS46_DASH=28|000\nHTTPS46_OIDC=28|000\nHTTPS85_DASH=0|200\nHTTPS85_OIDC=0|200\nPVE02_MUTATION_REQUIRED=true\n=== PVE03_PRE ===\nHOSTNAME=pve03\nSERVICE=active/enabled\nWT0=wt0 UNKNOWN 100.100.34.141/16 [PRIVATE_IPV6]/64 \nManagement: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nSignal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->46.16.34.129:443: read: connection timed out\nRelays: 0/2 Available\nFQDN: pve03.netbird.selfhosted\nNetBird IP: 100.100.34.141/16\nPeers count: 0/11 Connected\nHOSTS_BEGIN\nHOSTS_END\nHOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nDNS_nb_pvepro_ru=46.16.34.129\nDNS_nb_gram1_ru=46.16.34.129\nHTTPS46_DASH=28|000\nHTTPS46_OIDC=28|000\nHTTPS85_DASH=0|200\nHTTPS85_OIDC=0|200\nPVE03_MUTATION_REQUIRED=true\nACTION=APPLY_PVE02_RELAY_OVERRIDE\ncp: cannot create regular file '\"\"/var/lib/homelab-change-rollback/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\"/hosts.before\"': No such file or directory\nROLLBACK_PVE02_BEGIN=true\nROLLBACK_PVE02_RESTORE_COMMAND=false\nDECISION=FAIL_PVE23_RELAY_OVERRIDE_ROLLED_BACK\nFAILURE=PVE02_RELAY_OVERRIDE_OR_NETBIRD_ACCEPTANCE_FAILED\nNEXT_GATE=READ_EXACT_73334_AND_FIX_ONLY_FAILED_NODE_OR_ROLLBACK\nROLLBACK_RESTORED=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":true,\"rollback_restored\":false}\n" +} diff --git a/runtime/history/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1.txt b/runtime/history/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1.txt new file mode 100644 index 00000000..80762a5a --- /dev/null +++ b/runtime/history/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1.txt @@ -0,0 +1,100 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1 +STATUS=FAIL +RC=31 +HOST=pve01 +MODE=change +COMPONENT=netbird-pve23-relay-override-repair +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=858ca707cc2639542b347eecec8562ffb0404891bdd2dc52facf0711cf78a8d8 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=true +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=31 +CHANGES_MADE=true +ROLLBACK_STARTED=true +ROLLBACK_RESTORED=false +MUTATION_OUTCOME=ROLLBACK_FAILED +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=55e81f5f3fb26257de530bd3965969f28254c1423e2fa67b5e9a202de585e57a +SANITIZED_OUTPUT_SHA256=6f86a937e51d2a3d550ef83a579bdce2fcbf152399f53cb91bca7d699f34d631 +OUTPUT_BEGIN +NETBIRD_PVE23_RELAY_OVERRIDE_REPAIR_BEGIN=true +COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1 +MODE=change +COMPONENT=netbird-pve23-relay-override-repair +MUTATION_BOUNDARY=PVE02_PVE03_ONLY_SCOPED_ETC_HOSTS_NETBIRD_RELAY_OVERRIDE_PLUS_NETBIRD_RESTART_PLUS_EXISTING_PEER_HEALTH_ONESHOT;NO_PVE01_HOSTS_OR_NETBIRD_CHANGE;NO_EDGE_CHANGE;NO_ROUTE_FIREWALL_DNS_PROVIDER_CHANGE +=== AUTHORITY === +HOST=pve01 +CLUSTER_QUORATE=Yes +REFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED +ERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +SOURCE_73333_STATUS=OK +SOURCE_73333_RC=0 +SOURCE_73333_DECISION=PASS_PVE23_MISSING_RELAY_OVERRIDE_ROOT_CAUSE +PVE01_RELAY_OVERRIDE_COUNT=1 +PVE01_FORCED85_DASH=200 +PVE01_FORCED85_OIDC=200 +PVE02_SSH_HOSTNAME=pve02 +PVE03_SSH_HOSTNAME=pve03 +=== PVE02_PRE === +HOSTNAME=pve02 +SERVICE=active/enabled +WT0=wt0 UNKNOWN 100.100.7.2/16 [PRIVATE_IPV6]/64 +Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready +Signal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout +Relays: 0/2 Available +FQDN: pve02.netbird.selfhosted +NetBird IP: 100.100.7.2/16 +Peers count: 0/11 Connected +HOSTS_BEGIN +HOSTS_END +HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec +DNS_nb_pvepro_ru=46.16.34.129 +DNS_nb_gram1_ru=46.16.34.129 +HTTPS46_DASH=28|000 +HTTPS46_OIDC=28|000 +HTTPS85_DASH=0|200 +HTTPS85_OIDC=0|200 +PVE02_MUTATION_REQUIRED=true +=== PVE03_PRE === +HOSTNAME=pve03 +SERVICE=active/enabled +WT0=wt0 UNKNOWN 100.100.34.141/16 [PRIVATE_IPV6]/64 +Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready +Signal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->46.16.34.129:443: read: connection timed out +Relays: 0/2 Available +FQDN: pve03.netbird.selfhosted +NetBird IP: 100.100.34.141/16 +Peers count: 0/11 Connected +HOSTS_BEGIN +HOSTS_END +HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec +DNS_nb_pvepro_ru=46.16.34.129 +DNS_nb_gram1_ru=46.16.34.129 +HTTPS46_DASH=28|000 +HTTPS46_OIDC=28|000 +HTTPS85_DASH=0|200 +HTTPS85_OIDC=0|200 +PVE03_MUTATION_REQUIRED=true +ACTION=APPLY_PVE02_RELAY_OVERRIDE +cp: cannot create regular file '""/var/lib/homelab-change-rollback/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1"/hosts.before"': No such file or directory +ROLLBACK_PVE02_BEGIN=true +ROLLBACK_PVE02_RESTORE_COMMAND=false +DECISION=FAIL_PVE23_RELAY_OVERRIDE_ROLLED_BACK +FAILURE=PVE02_RELAY_OVERRIDE_OR_NETBIRD_ACCEPTANCE_FAILED +NEXT_GATE=READ_EXACT_73334_AND_FIX_ONLY_FAILED_NODE_OR_ROLLBACK +ROLLBACK_RESTORED=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1","status":"FAIL","changes_made":true,"rollback_started":true,"rollback_restored":false} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index db373ada..e30fe550 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260927-NETBIRD-PVE23-ENDPOINT-RCA-73333R1", - "status": "OK", - "rc": 0, + "command_id": "SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1", + "status": "FAIL", + "rc": 31, "host": "pve01", - "mode": "read-only", - "component": "netbird-pve23-endpoint-rca", - "started_at_utc": "2026-09-27T17:44:40Z", - "finished_at_utc": "2026-09-27T17:45:10Z", + "mode": "change", + "component": "netbird-pve23-relay-override-repair", + "started_at_utc": "2026-09-27T17:50:57Z", + "finished_at_utc": "2026-09-27T17:51:19Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "57892c95d8f7cb4f0d7f70f2e3346b0928dc48203f8f7f6c070b10e89009d115", + "command_sha256": "858ca707cc2639542b347eecec8562ffb0404891bdd2dc52facf0711cf78a8d8", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": false, + "change_declared": true, "result_contract_valid": true, - "result_contract_status": null, + "result_contract_status": "FAIL", "result_contract_error": null, - "command_rc": 0, - "changes_made": false, - "rollback_started": false, - "rollback_restored": null, - "mutation_outcome": "NO_MUTATION", + "command_rc": 31, + "changes_made": true, + "rollback_started": true, + "rollback_restored": false, + "mutation_outcome": "ROLLBACK_FAILED", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "bd3c4b7dd9c769d3076b3f01e1db57945267af69c4e07b0dd059278951cbaa4f", - "sanitized_output_sha256": "bdae30bb50399fd8610244b327588258e4822333ce7722ab7addda76a5c52c25", + "raw_evidence_sha256": "55e81f5f3fb26257de530bd3965969f28254c1423e2fa67b5e9a202de585e57a", + "sanitized_output_sha256": "6f86a937e51d2a3d550ef83a579bdce2fcbf152399f53cb91bca7d699f34d631", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "NETBIRD_PVE23_ENDPOINT_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-ENDPOINT-RCA-73333R1\nMODE=read-only\nCOMPONENT=netbird-pve23-endpoint-rca\nMUTATION_BOUNDARY=NONE_READ_ONLY_PVE23_ENDPOINT_RCA\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED\nREFERENCE_SHA256_EXPECTED=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED\nERROR_REGISTER_SHA256_EXPECTED=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_73332_PRESENT=true\nSOURCE_73332_STATUS=OK\nSOURCE_73332_RC=0\nSOURCE_73332_DECISION=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_COLLECTED\n=== PVE01 ===\nPVE01_COLLECT_RC=0\nNODE_HOSTNAME=pve01\nNODE_SERVICE=active/enabled\nNODE_WT0=wt0 UNKNOWN 100.100.131.41/16 [PRIVATE_IPV6]/64 \nNODE_STATUS_Management: Connected to https://nb.anon-D6PDi.domain:443\nNODE_STATUS_Signal: Connected to https://nb.anon-dBX1c.domain:443\nNODE_STATUS_Relays: \nNODE_STATUS_ [stun:nb.anon-dBX1c.domain:3478] is Checking...\nNODE_STATUS_ [rels://nb.anon-dBX1c.domain:443] is Available via ws\nNODE_STATUS_FQDN: pve01.netbird.selfhosted\nNODE_STATUS_NetBird IP: 100.100.131.41/16\nNODE_STATUS_Peers count: 3/11 Connected\nNODE_CONFIG_PATH=missing\nNODE_HOSTS_NETBIRD_BEGIN\n85.121.4.192 nb.pvepro.ru nb.gram1.ru # WORKERS2-73318-NETBIRD-RELAY\nNODE_HOSTS_NETBIRD_END\nNODE_DNS_nb_pvepro_ru=85.121.4.192\nNODE_DNS_nb_gram1_ru=85.121.4.192\nNODE_ROUTE_USA=46.16.34.129 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 \nNODE_ROUTE_RELAY=85.121.4.192 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 \nNODE_TCP46_443=OPEN\nNODE_TCP85_443=OPEN\nNODE_HTTPS46_DASH=28|000\nNODE_HTTPS46_OIDC=28|000\nNODE_HTTPS85_DASH=0|200\nNODE_HTTPS85_OIDC=0|200\nNODE_NETBIRD_JOURNAL_BEGIN\nNODE_NETBIRD_JOURNAL_END\n=== PVE02 ===\nPVE02_COLLECT_RC=0\nNODE_HOSTNAME=pve02\nNODE_SERVICE=active/enabled\nNODE_WT0=wt0 UNKNOWN 100.100.7.2/16 [PRIVATE_IPV6]/64 \nNODE_STATUS_Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nNODE_STATUS_Signal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout\nNODE_STATUS_Relays: \nNODE_STATUS_ [stun:nb.anon-nlb6N.domain:3478] is Checking...\nNODE_STATUS_ [rels://nb.anon-nlb6N.domain:443] is Unavailable, reason: connect to relay server: context deadline exceeded\nNODE_STATUS_FQDN: pve02.netbird.selfhosted\nNODE_STATUS_NetBird IP: 100.100.7.2/16\nNODE_STATUS_Peers count: 0/11 Connected\nNODE_CONFIG_PATH=missing\nNODE_HOSTS_NETBIRD_BEGIN\nNODE_HOSTS_NETBIRD_END\nNODE_DNS_nb_pvepro_ru=46.16.34.129\nNODE_DNS_nb_gram1_ru=46.16.34.129\nNODE_ROUTE_USA=46.16.34.129 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 \nNODE_ROUTE_RELAY=85.121.4.192 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 \nNODE_TCP46_443=OPEN\nNODE_TCP85_443=OPEN\nNODE_HTTPS46_DASH=28|000\nNODE_HTTPS46_OIDC=28|000\nNODE_HTTPS85_DASH=0|200\nNODE_HTTPS85_OIDC=0|200\nNODE_NETBIRD_JOURNAL_BEGIN\nNODE_NETBIRD_JOURNAL_END\n=== PVE03 ===\nPVE03_COLLECT_RC=0\nNODE_HOSTNAME=pve03\nNODE_SERVICE=active/enabled\nNODE_WT0=wt0 UNKNOWN 100.100.34.141/16 [PRIVATE_IPV6]/64 \nNODE_STATUS_Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nNODE_STATUS_Signal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->198.51.100.0:443: read: connection timed out\nNODE_STATUS_Relays: \nNODE_STATUS_ [stun:nb.anon-pdcjY.domain:3478] is Checking...\nNODE_STATUS_ [rels://nb.anon-pdcjY.domain:443] is Unavailable, reason: connect to relay server: context deadline exceeded\nNODE_STATUS_FQDN: pve03.netbird.selfhosted\nNODE_STATUS_NetBird IP: 100.100.34.141/16\nNODE_STATUS_Peers count: 0/11 Connected\nNODE_CONFIG_PATH=missing\nNODE_HOSTS_NETBIRD_BEGIN\nNODE_HOSTS_NETBIRD_END\nNODE_DNS_nb_pvepro_ru=46.16.34.129\nNODE_DNS_nb_gram1_ru=46.16.34.129\nNODE_ROUTE_USA=46.16.34.129 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 \nNODE_ROUTE_RELAY=85.121.4.192 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 \nNODE_TCP46_443=OPEN\nNODE_TCP85_443=OPEN\nNODE_HTTPS46_DASH=28|000\nNODE_HTTPS46_OIDC=28|000\nNODE_HTTPS85_DASH=0|200\nNODE_HTTPS85_OIDC=0|200\nNODE_NETBIRD_JOURNAL_BEGIN\nNODE_NETBIRD_JOURNAL_END\n=== CLASSIFICATION ===\nPVE01_CONTROL_CONNECTED=true\nPVE02_CONTROL_CONNECTED=false\nPVE03_CONTROL_CONNECTED=false\nPVE01_RELAY_HOSTS_OVERRIDE=true\nPVE02_RELAY_HOSTS_OVERRIDE=false\nPVE03_RELAY_HOSTS_OVERRIDE=false\nPVE02_DNS_HAS_46=true\nPVE03_DNS_HAS_46=true\nPVE02_FORCED_85_READY=true\nPVE03_FORCED_85_READY=true\nPVE02_FORCED_46_READY=false\nPVE03_FORCED_46_READY=false\nDECISION=PASS_PVE23_MISSING_RELAY_OVERRIDE_ROOT_CAUSE\nNEXT_GATE=APPLY_SCOPED_PVE23_RELAY_OVERRIDE_AND_RESTART_NETBIRD\nTASK_RESULT=PASS_NETBIRD_PVE23_ENDPOINT_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-PVE23-ENDPOINT-RCA-73333R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "NETBIRD_PVE23_RELAY_OVERRIDE_REPAIR_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\nMODE=change\nCOMPONENT=netbird-pve23-relay-override-repair\nMUTATION_BOUNDARY=PVE02_PVE03_ONLY_SCOPED_ETC_HOSTS_NETBIRD_RELAY_OVERRIDE_PLUS_NETBIRD_RESTART_PLUS_EXISTING_PEER_HEALTH_ONESHOT;NO_PVE01_HOSTS_OR_NETBIRD_CHANGE;NO_EDGE_CHANGE;NO_ROUTE_FIREWALL_DNS_PROVIDER_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED\nERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_73333_STATUS=OK\nSOURCE_73333_RC=0\nSOURCE_73333_DECISION=PASS_PVE23_MISSING_RELAY_OVERRIDE_ROOT_CAUSE\nPVE01_RELAY_OVERRIDE_COUNT=1\nPVE01_FORCED85_DASH=200\nPVE01_FORCED85_OIDC=200\nPVE02_SSH_HOSTNAME=pve02\nPVE03_SSH_HOSTNAME=pve03\n=== PVE02_PRE ===\nHOSTNAME=pve02\nSERVICE=active/enabled\nWT0=wt0 UNKNOWN 100.100.7.2/16 [PRIVATE_IPV6]/64 \nManagement: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nSignal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout\nRelays: 0/2 Available\nFQDN: pve02.netbird.selfhosted\nNetBird IP: 100.100.7.2/16\nPeers count: 0/11 Connected\nHOSTS_BEGIN\nHOSTS_END\nHOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nDNS_nb_pvepro_ru=46.16.34.129\nDNS_nb_gram1_ru=46.16.34.129\nHTTPS46_DASH=28|000\nHTTPS46_OIDC=28|000\nHTTPS85_DASH=0|200\nHTTPS85_OIDC=0|200\nPVE02_MUTATION_REQUIRED=true\n=== PVE03_PRE ===\nHOSTNAME=pve03\nSERVICE=active/enabled\nWT0=wt0 UNKNOWN 100.100.34.141/16 [PRIVATE_IPV6]/64 \nManagement: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nSignal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->46.16.34.129:443: read: connection timed out\nRelays: 0/2 Available\nFQDN: pve03.netbird.selfhosted\nNetBird IP: 100.100.34.141/16\nPeers count: 0/11 Connected\nHOSTS_BEGIN\nHOSTS_END\nHOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nDNS_nb_pvepro_ru=46.16.34.129\nDNS_nb_gram1_ru=46.16.34.129\nHTTPS46_DASH=28|000\nHTTPS46_OIDC=28|000\nHTTPS85_DASH=0|200\nHTTPS85_OIDC=0|200\nPVE03_MUTATION_REQUIRED=true\nACTION=APPLY_PVE02_RELAY_OVERRIDE\ncp: cannot create regular file '\"\"/var/lib/homelab-change-rollback/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\"/hosts.before\"': No such file or directory\nROLLBACK_PVE02_BEGIN=true\nROLLBACK_PVE02_RESTORE_COMMAND=false\nDECISION=FAIL_PVE23_RELAY_OVERRIDE_ROLLED_BACK\nFAILURE=PVE02_RELAY_OVERRIDE_OR_NETBIRD_ACCEPTANCE_FAILED\nNEXT_GATE=READ_EXACT_73334_AND_FIX_ONLY_FAILED_NODE_OR_ROLLBACK\nROLLBACK_RESTORED=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":true,\"rollback_restored\":false}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index c159c311..80762a5a 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,151 +1,100 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-ENDPOINT-RCA-73333R1 -STATUS=OK -RC=0 +COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1 +STATUS=FAIL +RC=31 HOST=pve01 -MODE=read-only -COMPONENT=netbird-pve23-endpoint-rca +MODE=change +COMPONENT=netbird-pve23-relay-override-repair REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=57892c95d8f7cb4f0d7f70f2e3346b0928dc48203f8f7f6c070b10e89009d115 +COMMAND_SHA256=858ca707cc2639542b347eecec8562ffb0404891bdd2dc52facf0711cf78a8d8 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=false +CHANGE_DECLARED=true RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_STATUS=FAIL RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=0 -CHANGES_MADE=false -ROLLBACK_STARTED=false -ROLLBACK_RESTORED=null -MUTATION_OUTCOME=NO_MUTATION +COMMAND_RC=31 +CHANGES_MADE=true +ROLLBACK_STARTED=true +ROLLBACK_RESTORED=false +MUTATION_OUTCOME=ROLLBACK_FAILED SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=bd3c4b7dd9c769d3076b3f01e1db57945267af69c4e07b0dd059278951cbaa4f -SANITIZED_OUTPUT_SHA256=bdae30bb50399fd8610244b327588258e4822333ce7722ab7addda76a5c52c25 +RAW_EVIDENCE_SHA256=55e81f5f3fb26257de530bd3965969f28254c1423e2fa67b5e9a202de585e57a +SANITIZED_OUTPUT_SHA256=6f86a937e51d2a3d550ef83a579bdce2fcbf152399f53cb91bca7d699f34d631 OUTPUT_BEGIN -NETBIRD_PVE23_ENDPOINT_RCA_BEGIN=true -COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-ENDPOINT-RCA-73333R1 -MODE=read-only -COMPONENT=netbird-pve23-endpoint-rca -MUTATION_BOUNDARY=NONE_READ_ONLY_PVE23_ENDPOINT_RCA +NETBIRD_PVE23_RELAY_OVERRIDE_REPAIR_BEGIN=true +COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1 +MODE=change +COMPONENT=netbird-pve23-relay-override-repair +MUTATION_BOUNDARY=PVE02_PVE03_ONLY_SCOPED_ETC_HOSTS_NETBIRD_RELAY_OVERRIDE_PLUS_NETBIRD_RESTART_PLUS_EXISTING_PEER_HEALTH_ONESHOT;NO_PVE01_HOSTS_OR_NETBIRD_CHANGE;NO_EDGE_CHANGE;NO_ROUTE_FIREWALL_DNS_PROVIDER_CHANGE === AUTHORITY === HOST=pve01 CLUSTER_QUORATE=Yes REFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED -REFERENCE_SHA256_EXPECTED=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED -ERROR_REGISTER_SHA256_EXPECTED=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -SOURCE_73332_PRESENT=true -SOURCE_73332_STATUS=OK -SOURCE_73332_RC=0 -SOURCE_73332_DECISION=PASS_NETBIRD_PEER_HEALTH_POSTCLOSE_RCA_COLLECTED -=== PVE01 === -PVE01_COLLECT_RC=0 -NODE_HOSTNAME=pve01 -NODE_SERVICE=active/enabled -NODE_WT0=wt0 UNKNOWN 100.100.131.41/16 [PRIVATE_IPV6]/64 -NODE_STATUS_Management: Connected to https://nb.anon-D6PDi.domain:443 -NODE_STATUS_Signal: Connected to https://nb.anon-dBX1c.domain:443 -NODE_STATUS_Relays: -NODE_STATUS_ [stun:nb.anon-dBX1c.domain:3478] is Checking... -NODE_STATUS_ [rels://nb.anon-dBX1c.domain:443] is Available via ws -NODE_STATUS_FQDN: pve01.netbird.selfhosted -NODE_STATUS_NetBird IP: 100.100.131.41/16 -NODE_STATUS_Peers count: 3/11 Connected -NODE_CONFIG_PATH=missing -NODE_HOSTS_NETBIRD_BEGIN -85.121.4.192 nb.pvepro.ru nb.gram1.ru # WORKERS2-73318-NETBIRD-RELAY -NODE_HOSTS_NETBIRD_END -NODE_DNS_nb_pvepro_ru=85.121.4.192 -NODE_DNS_nb_gram1_ru=85.121.4.192 -NODE_ROUTE_USA=46.16.34.129 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 -NODE_ROUTE_RELAY=85.121.4.192 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 -NODE_TCP46_443=OPEN -NODE_TCP85_443=OPEN -NODE_HTTPS46_DASH=28|000 -NODE_HTTPS46_OIDC=28|000 -NODE_HTTPS85_DASH=0|200 -NODE_HTTPS85_OIDC=0|200 -NODE_NETBIRD_JOURNAL_BEGIN -NODE_NETBIRD_JOURNAL_END -=== PVE02 === -PVE02_COLLECT_RC=0 -NODE_HOSTNAME=pve02 -NODE_SERVICE=active/enabled -NODE_WT0=wt0 UNKNOWN 100.100.7.2/16 [PRIVATE_IPV6]/64 -NODE_STATUS_Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready -NODE_STATUS_Signal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout -NODE_STATUS_Relays: -NODE_STATUS_ [stun:nb.anon-nlb6N.domain:3478] is Checking... -NODE_STATUS_ [rels://nb.anon-nlb6N.domain:443] is Unavailable, reason: connect to relay server: context deadline exceeded -NODE_STATUS_FQDN: pve02.netbird.selfhosted -NODE_STATUS_NetBird IP: 100.100.7.2/16 -NODE_STATUS_Peers count: 0/11 Connected -NODE_CONFIG_PATH=missing -NODE_HOSTS_NETBIRD_BEGIN -NODE_HOSTS_NETBIRD_END -NODE_DNS_nb_pvepro_ru=46.16.34.129 -NODE_DNS_nb_gram1_ru=46.16.34.129 -NODE_ROUTE_USA=46.16.34.129 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 -NODE_ROUTE_RELAY=85.121.4.192 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 -NODE_TCP46_443=OPEN -NODE_TCP85_443=OPEN -NODE_HTTPS46_DASH=28|000 -NODE_HTTPS46_OIDC=28|000 -NODE_HTTPS85_DASH=0|200 -NODE_HTTPS85_OIDC=0|200 -NODE_NETBIRD_JOURNAL_BEGIN -NODE_NETBIRD_JOURNAL_END -=== PVE03 === -PVE03_COLLECT_RC=0 -NODE_HOSTNAME=pve03 -NODE_SERVICE=active/enabled -NODE_WT0=wt0 UNKNOWN 100.100.34.141/16 [PRIVATE_IPV6]/64 -NODE_STATUS_Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready -NODE_STATUS_Signal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->198.51.100.0:443: read: connection timed out -NODE_STATUS_Relays: -NODE_STATUS_ [stun:nb.anon-pdcjY.domain:3478] is Checking... -NODE_STATUS_ [rels://nb.anon-pdcjY.domain:443] is Unavailable, reason: connect to relay server: context deadline exceeded -NODE_STATUS_FQDN: pve03.netbird.selfhosted -NODE_STATUS_NetBird IP: 100.100.34.141/16 -NODE_STATUS_Peers count: 0/11 Connected -NODE_CONFIG_PATH=missing -NODE_HOSTS_NETBIRD_BEGIN -NODE_HOSTS_NETBIRD_END -NODE_DNS_nb_pvepro_ru=46.16.34.129 -NODE_DNS_nb_gram1_ru=46.16.34.129 -NODE_ROUTE_USA=46.16.34.129 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 -NODE_ROUTE_RELAY=85.121.4.192 via [PRIVATE_IP] dev vmbr0 src [PRIVATE_IP] uid 0 -NODE_TCP46_443=OPEN -NODE_TCP85_443=OPEN -NODE_HTTPS46_DASH=28|000 -NODE_HTTPS46_OIDC=28|000 -NODE_HTTPS85_DASH=0|200 -NODE_HTTPS85_OIDC=0|200 -NODE_NETBIRD_JOURNAL_BEGIN -NODE_NETBIRD_JOURNAL_END -=== CLASSIFICATION === -PVE01_CONTROL_CONNECTED=true -PVE02_CONTROL_CONNECTED=false -PVE03_CONTROL_CONNECTED=false -PVE01_RELAY_HOSTS_OVERRIDE=true -PVE02_RELAY_HOSTS_OVERRIDE=false -PVE03_RELAY_HOSTS_OVERRIDE=false -PVE02_DNS_HAS_46=true -PVE03_DNS_HAS_46=true -PVE02_FORCED_85_READY=true -PVE03_FORCED_85_READY=true -PVE02_FORCED_46_READY=false -PVE03_FORCED_46_READY=false -DECISION=PASS_PVE23_MISSING_RELAY_OVERRIDE_ROOT_CAUSE -NEXT_GATE=APPLY_SCOPED_PVE23_RELAY_OVERRIDE_AND_RESTART_NETBIRD -TASK_RESULT=PASS_NETBIRD_PVE23_ENDPOINT_RCA -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PVE23-ENDPOINT-RCA-73333R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +SOURCE_73333_STATUS=OK +SOURCE_73333_RC=0 +SOURCE_73333_DECISION=PASS_PVE23_MISSING_RELAY_OVERRIDE_ROOT_CAUSE +PVE01_RELAY_OVERRIDE_COUNT=1 +PVE01_FORCED85_DASH=200 +PVE01_FORCED85_OIDC=200 +PVE02_SSH_HOSTNAME=pve02 +PVE03_SSH_HOSTNAME=pve03 +=== PVE02_PRE === +HOSTNAME=pve02 +SERVICE=active/enabled +WT0=wt0 UNKNOWN 100.100.7.2/16 [PRIVATE_IPV6]/64 +Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready +Signal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout +Relays: 0/2 Available +FQDN: pve02.netbird.selfhosted +NetBird IP: 100.100.7.2/16 +Peers count: 0/11 Connected +HOSTS_BEGIN +HOSTS_END +HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec +DNS_nb_pvepro_ru=46.16.34.129 +DNS_nb_gram1_ru=46.16.34.129 +HTTPS46_DASH=28|000 +HTTPS46_OIDC=28|000 +HTTPS85_DASH=0|200 +HTTPS85_OIDC=0|200 +PVE02_MUTATION_REQUIRED=true +=== PVE03_PRE === +HOSTNAME=pve03 +SERVICE=active/enabled +WT0=wt0 UNKNOWN 100.100.34.141/16 [PRIVATE_IPV6]/64 +Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready +Signal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->46.16.34.129:443: read: connection timed out +Relays: 0/2 Available +FQDN: pve03.netbird.selfhosted +NetBird IP: 100.100.34.141/16 +Peers count: 0/11 Connected +HOSTS_BEGIN +HOSTS_END +HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec +DNS_nb_pvepro_ru=46.16.34.129 +DNS_nb_gram1_ru=46.16.34.129 +HTTPS46_DASH=28|000 +HTTPS46_OIDC=28|000 +HTTPS85_DASH=0|200 +HTTPS85_OIDC=0|200 +PVE03_MUTATION_REQUIRED=true +ACTION=APPLY_PVE02_RELAY_OVERRIDE +cp: cannot create regular file '""/var/lib/homelab-change-rollback/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1"/hosts.before"': No such file or directory +ROLLBACK_PVE02_BEGIN=true +ROLLBACK_PVE02_RESTORE_COMMAND=false +DECISION=FAIL_PVE23_RELAY_OVERRIDE_ROLLED_BACK +FAILURE=PVE02_RELAY_OVERRIDE_OR_NETBIRD_ACCEPTANCE_FAILED +NEXT_GATE=READ_EXACT_73334_AND_FIX_ONLY_FAILED_NODE_OR_ROLLBACK +ROLLBACK_RESTORED=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1","status":"FAIL","changes_made":true,"rollback_started":true,"rollback_restored":false} OUTPUT_END CHAT_OUTPUT_END