From 594c768449737a890ea62efd612f1a280b0321c5 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Sun, 27 Sep 2026 15:57:53 +0000 Subject: [PATCH] runtime: publish SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1 --- ...LAY-INGRESS-PVE-EDGE-RECOVERY-73318R1.json | 38 ++++++++ ...ELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1.txt | 88 +++++++++++++++++++ runtime/latest.json | 24 ++--- runtime/latest.txt | 81 ++++++++++++----- 4 files changed, 198 insertions(+), 33 deletions(-) create mode 100644 runtime/history/SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1.json create mode 100644 runtime/history/SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1.txt diff --git a/runtime/history/SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1.json b/runtime/history/SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1.json new file mode 100644 index 00000000..98568772 --- /dev/null +++ b/runtime/history/SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1", + "status": "FAIL", + "rc": 31, + "host": "pve01", + "mode": "change", + "component": "netbird-relay-ingress-pve-edge-recovery", + "started_at_utc": "2026-09-27T15:54:35Z", + "finished_at_utc": "2026-09-27T15:57:52Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "b5d155d3138d5d9aff741f1f8ee1dc6d91f389bcb6d150b76c5d67047e1841bd", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": true, + "result_contract_valid": true, + "result_contract_status": "FAIL", + "result_contract_error": null, + "command_rc": 31, + "changes_made": true, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "MUTATED", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "704c3dfd4261a98dde693184f8d8997876aeeb2e9c7c377ed93c33df51512ff8", + "sanitized_output_sha256": "704c3dfd4261a98dde693184f8d8997876aeeb2e9c7c377ed93c33df51512ff8", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "NETBIRD_RELAY_RECOVERY_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1\nMODE=change\nCOMPONENT=netbird-relay-ingress-pve-edge-recovery\nMUTATION_BOUNDARY=RELAY_NETBIRD_INGRESS_SOCKET_ENABLE_IF_NEEDED_PLUS_PVE01_HOSTS_OVERRIDE_FOR_NB_NAMES_PLUS_NETBIRD_RESTART_PLUS_HEALTH_SUPERVISOR_ONESHOTS;NO_DNS_PROVIDER_EDIT;NO_ROUTE_EDIT;NO_FIREWALL_EDIT;NO_NETBIRD_CONFIG_EDIT;NO_COOKIE_EDIT\nCLUSTER_QUORATE=true\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\n=== PRE_NETBIRD ===\nError: status failed: failed connecting to Management Service : create connection: dial context: context deadline exceeded\nPVE_TO_RELAY_DASHBOARD_PRE_HTTP=200\nPVE_TO_RELAY_OIDC_PRE_HTTP=200\nRELAY_INGRESS_READY=true\nPVE_HOSTS_OVERRIDE=APPLIED\nPVE_HOSTS_SHA256=a496860cfb488920de67686aa7d555627acc902891c8cc27ba5b1b399c248830\nPVE_RESOLVE_NB_PVEPRO=85.121.4.192\nPVE_RESOLVE_NB_GRAM1=85.121.4.192\nACTION=PVE01_NETBIRD_RESTART_VIA_RELAY\nFINAL_NETBIRD_MANAGEMENT=true\nFINAL_NETBIRD_SIGNAL=true\nFINAL_NETBIRD_Management: Connected\nFINAL_NETBIRD_Signal: Connected\nFINAL_NETBIRD_Relays: 1/1 Available\nFINAL_NETBIRD_NetBird IP: 100.100.131.41/16\nFINAL_NETBIRD_Networks: -\nFINAL_NETBIRD_Peers count: 0/0 Connected\nPOST_WAIT_NETBIRD_Relays: 1/2 Available\nPOST_WAIT_NETBIRD_Networks: -\nPOST_WAIT_NETBIRD_Peers count: 3/11 Connected\n=== HEALTH_AND_SKLADCHIK_RECONCILE ===\nNETBIRD_PEERS_HEALTH_RC=1\nPRIVATE_VPN_HOSTS_HEALTH_RC=0\nSKLADCHIK_SUPERVISOR_ATTEMPT_1_RC=1\nSKLADCHIK_SUPERVISOR_ATTEMPT_2_RC=1\nSKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: SUPERVISOR_STATUS=FAIL\nSKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: SUPERVISOR_PROBLEMS=monitor_status_not_ok\nSKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: MONITOR=ERROR reports= open= resolved=\nSKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nSKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: SUPERVISOR_STATUS=FAIL\nSKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: SUPERVISOR_PROBLEMS=monitor_status_not_ok\nSKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: MONITOR=ERROR reports= open= resolved=\nSKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nSKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: SUPERVISOR_STATUS=FAIL\nSKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: SUPERVISOR_PROBLEMS=monitor_status_not_ok\nSKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: MONITOR=ERROR reports= open= resolved=\nSKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nSKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: SUPERVISOR_STATUS=FAIL\nSKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: SUPERVISOR_PROBLEMS=monitor_status_not_ok\nSKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: MONITOR=ERROR reports= open= resolved=\nSKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nSKLADCHIK_SUPERVISOR_OK=false\nTASK_RESULT=PASS_NETBIRD_RESTORED_BUT_SKLADCHIK_PENDING\nDECISION=HOLD_SKLADCHIK_AFTER_NETBIRD\nNEXT_GATE=RECONCILE_SKLADCHIK_ONLY\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1.txt b/runtime/history/SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1.txt new file mode 100644 index 00000000..2db22371 --- /dev/null +++ b/runtime/history/SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1.txt @@ -0,0 +1,88 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1 +STATUS=FAIL +RC=31 +HOST=pve01 +MODE=change +COMPONENT=netbird-relay-ingress-pve-edge-recovery +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=b5d155d3138d5d9aff741f1f8ee1dc6d91f389bcb6d150b76c5d67047e1841bd +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=true +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=31 +CHANGES_MADE=true +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=MUTATED +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=704c3dfd4261a98dde693184f8d8997876aeeb2e9c7c377ed93c33df51512ff8 +SANITIZED_OUTPUT_SHA256=704c3dfd4261a98dde693184f8d8997876aeeb2e9c7c377ed93c33df51512ff8 +OUTPUT_BEGIN +NETBIRD_RELAY_RECOVERY_BEGIN=true +COMMAND_ID=SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1 +MODE=change +COMPONENT=netbird-relay-ingress-pve-edge-recovery +MUTATION_BOUNDARY=RELAY_NETBIRD_INGRESS_SOCKET_ENABLE_IF_NEEDED_PLUS_PVE01_HOSTS_OVERRIDE_FOR_NB_NAMES_PLUS_NETBIRD_RESTART_PLUS_HEALTH_SUPERVISOR_ONESHOTS;NO_DNS_PROVIDER_EDIT;NO_ROUTE_EDIT;NO_FIREWALL_EDIT;NO_NETBIRD_CONFIG_EDIT;NO_COOKIE_EDIT +CLUSTER_QUORATE=true +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +=== PRE_NETBIRD === +Error: status failed: failed connecting to Management Service : create connection: dial context: context deadline exceeded +PVE_TO_RELAY_DASHBOARD_PRE_HTTP=200 +PVE_TO_RELAY_OIDC_PRE_HTTP=200 +RELAY_INGRESS_READY=true +PVE_HOSTS_OVERRIDE=APPLIED +PVE_HOSTS_SHA256=a496860cfb488920de67686aa7d555627acc902891c8cc27ba5b1b399c248830 +PVE_RESOLVE_NB_PVEPRO=85.121.4.192 +PVE_RESOLVE_NB_GRAM1=85.121.4.192 +ACTION=PVE01_NETBIRD_RESTART_VIA_RELAY +FINAL_NETBIRD_MANAGEMENT=true +FINAL_NETBIRD_SIGNAL=true +FINAL_NETBIRD_Management: Connected +FINAL_NETBIRD_Signal: Connected +FINAL_NETBIRD_Relays: 1/1 Available +FINAL_NETBIRD_NetBird IP: 100.100.131.41/16 +FINAL_NETBIRD_Networks: - +FINAL_NETBIRD_Peers count: 0/0 Connected +POST_WAIT_NETBIRD_Relays: 1/2 Available +POST_WAIT_NETBIRD_Networks: - +POST_WAIT_NETBIRD_Peers count: 3/11 Connected +=== HEALTH_AND_SKLADCHIK_RECONCILE === +NETBIRD_PEERS_HEALTH_RC=1 +PRIVATE_VPN_HOSTS_HEALTH_RC=0 +SKLADCHIK_SUPERVISOR_ATTEMPT_1_RC=1 +SKLADCHIK_SUPERVISOR_ATTEMPT_2_RC=1 +SKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: SUPERVISOR_STATUS=FAIL +SKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: SUPERVISOR_PROBLEMS=monitor_status_not_ok +SKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: MONITOR=ERROR reports= open= resolved= +SKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +SKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: SUPERVISOR_STATUS=FAIL +SKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: SUPERVISOR_PROBLEMS=monitor_status_not_ok +SKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: MONITOR=ERROR reports= open= resolved= +SKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +SKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: SUPERVISOR_STATUS=FAIL +SKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: SUPERVISOR_PROBLEMS=monitor_status_not_ok +SKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: MONITOR=ERROR reports= open= resolved= +SKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +SKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: SUPERVISOR_STATUS=FAIL +SKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: SUPERVISOR_PROBLEMS=monitor_status_not_ok +SKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: MONITOR=ERROR reports= open= resolved= +SKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +SKLADCHIK_SUPERVISOR_OK=false +TASK_RESULT=PASS_NETBIRD_RESTORED_BUT_SKLADCHIK_PENDING +DECISION=HOLD_SKLADCHIK_AFTER_NETBIRD +NEXT_GATE=RECONCILE_SKLADCHIK_ONLY +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 2b890a71..98568772 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260927-USA-OPERATOR-REPAIR-NETBIRD-SKLADCHIK-73317R1", + "command_id": "SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1", "status": "FAIL", - "rc": 19, + "rc": 31, "host": "pve01", "mode": "change", - "component": "usa-operator-repair-netbird-skladchik", - "started_at_utc": "2026-09-27T15:40:54Z", - "finished_at_utc": "2026-09-27T15:41:19Z", + "component": "netbird-relay-ingress-pve-edge-recovery", + "started_at_utc": "2026-09-27T15:54:35Z", + "finished_at_utc": "2026-09-27T15:57:52Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "cf2cea63d66ab598af639003c264aea93e42e80b1eb4319f2b58593639322fab", + "command_sha256": "b5d155d3138d5d9aff741f1f8ee1dc6d91f389bcb6d150b76c5d67047e1841bd", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -21,18 +21,18 @@ "result_contract_valid": true, "result_contract_status": "FAIL", "result_contract_error": null, - "command_rc": 19, - "changes_made": false, + "command_rc": 31, + "changes_made": true, "rollback_started": false, "rollback_restored": null, - "mutation_outcome": "NO_MUTATION", + "mutation_outcome": "MUTATED", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "5b1d98a9c1f799dbe1cd684ee21eeac56dd2fdee5c201131bfd014a1c47401a0", - "sanitized_output_sha256": "5b1d98a9c1f799dbe1cd684ee21eeac56dd2fdee5c201131bfd014a1c47401a0", + "raw_evidence_sha256": "704c3dfd4261a98dde693184f8d8997876aeeb2e9c7c377ed93c33df51512ff8", + "sanitized_output_sha256": "704c3dfd4261a98dde693184f8d8997876aeeb2e9c7c377ed93c33df51512ff8", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "USA_OPERATOR_REPAIR_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-USA-OPERATOR-REPAIR-NETBIRD-SKLADCHIK-73317R1\nMODE=change\nCOMPONENT=usa-operator-repair-netbird-skladchik\nMUTATION_BOUNDARY=USA_AUTHORIZED_KEYS_OPERATOR_REPAIR_IF_NEEDED_PLUS_EXISTING_USA_NETBIRD_STACK_RESTART_IF_NEEDED_PLUS_PVE01_NETBIRD_RESTART_PLUS_HEALTH_SUPERVISOR_ONESHOTS;NO_DNS_EDIT;NO_ROUTE_EDIT;NO_FIREWALL_EDIT;NO_NETBIRD_CONFIG_EDIT;NO_COMPOSE_FILE_EDIT;NO_COOKIE_EDIT\nCLUSTER_QUORATE=true\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nUSA_OPERATOR_KEY_FP=SHA256:wipXiBB3cpAwyWap9LcB185WjEEmDxtRJsI/UuMVAp0\nUSA_LIVE_HOSTKEY_FP=SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE\nUSA_OPERATOR_AUTH_PRE=false\nFORWARDED_AGENT_PRESENT=true\nFORWARDED_PAGEANT_USA_AUTH=false\nDECISION=HOLD_WINDOWS_PAGEANT_NOT_AUTHORIZED_ON_USA\nNEXT_GATE=PROVIDER_CONSOLE_OR_OTHER_USA_AUTHORITY\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-USA-OPERATOR-REPAIR-NETBIRD-SKLADCHIK-73317R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "NETBIRD_RELAY_RECOVERY_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1\nMODE=change\nCOMPONENT=netbird-relay-ingress-pve-edge-recovery\nMUTATION_BOUNDARY=RELAY_NETBIRD_INGRESS_SOCKET_ENABLE_IF_NEEDED_PLUS_PVE01_HOSTS_OVERRIDE_FOR_NB_NAMES_PLUS_NETBIRD_RESTART_PLUS_HEALTH_SUPERVISOR_ONESHOTS;NO_DNS_PROVIDER_EDIT;NO_ROUTE_EDIT;NO_FIREWALL_EDIT;NO_NETBIRD_CONFIG_EDIT;NO_COOKIE_EDIT\nCLUSTER_QUORATE=true\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\n=== PRE_NETBIRD ===\nError: status failed: failed connecting to Management Service : create connection: dial context: context deadline exceeded\nPVE_TO_RELAY_DASHBOARD_PRE_HTTP=200\nPVE_TO_RELAY_OIDC_PRE_HTTP=200\nRELAY_INGRESS_READY=true\nPVE_HOSTS_OVERRIDE=APPLIED\nPVE_HOSTS_SHA256=a496860cfb488920de67686aa7d555627acc902891c8cc27ba5b1b399c248830\nPVE_RESOLVE_NB_PVEPRO=85.121.4.192\nPVE_RESOLVE_NB_GRAM1=85.121.4.192\nACTION=PVE01_NETBIRD_RESTART_VIA_RELAY\nFINAL_NETBIRD_MANAGEMENT=true\nFINAL_NETBIRD_SIGNAL=true\nFINAL_NETBIRD_Management: Connected\nFINAL_NETBIRD_Signal: Connected\nFINAL_NETBIRD_Relays: 1/1 Available\nFINAL_NETBIRD_NetBird IP: 100.100.131.41/16\nFINAL_NETBIRD_Networks: -\nFINAL_NETBIRD_Peers count: 0/0 Connected\nPOST_WAIT_NETBIRD_Relays: 1/2 Available\nPOST_WAIT_NETBIRD_Networks: -\nPOST_WAIT_NETBIRD_Peers count: 3/11 Connected\n=== HEALTH_AND_SKLADCHIK_RECONCILE ===\nNETBIRD_PEERS_HEALTH_RC=1\nPRIVATE_VPN_HOSTS_HEALTH_RC=0\nSKLADCHIK_SUPERVISOR_ATTEMPT_1_RC=1\nSKLADCHIK_SUPERVISOR_ATTEMPT_2_RC=1\nSKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: SUPERVISOR_STATUS=FAIL\nSKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: SUPERVISOR_PROBLEMS=monitor_status_not_ok\nSKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: MONITOR=ERROR reports= open= resolved=\nSKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nSKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: SUPERVISOR_STATUS=FAIL\nSKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: SUPERVISOR_PROBLEMS=monitor_status_not_ok\nSKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: MONITOR=ERROR reports= open= resolved=\nSKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nSKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: SUPERVISOR_STATUS=FAIL\nSKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: SUPERVISOR_PROBLEMS=monitor_status_not_ok\nSKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: MONITOR=ERROR reports= open= resolved=\nSKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nSKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: SUPERVISOR_STATUS=FAIL\nSKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: SUPERVISOR_PROBLEMS=monitor_status_not_ok\nSKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: MONITOR=ERROR reports= open= resolved=\nSKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nSKLADCHIK_SUPERVISOR_OK=false\nTASK_RESULT=PASS_NETBIRD_RESTORED_BUT_SKLADCHIK_PENDING\nDECISION=HOLD_SKLADCHIK_AFTER_NETBIRD\nNEXT_GATE=RECONCILE_SKLADCHIK_ONLY\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index c764a3bc..2db22371 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,49 +1,88 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260927-USA-OPERATOR-REPAIR-NETBIRD-SKLADCHIK-73317R1 +COMMAND_ID=SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1 STATUS=FAIL -RC=19 +RC=31 HOST=pve01 MODE=change -COMPONENT=usa-operator-repair-netbird-skladchik +COMPONENT=netbird-relay-ingress-pve-edge-recovery REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=cf2cea63d66ab598af639003c264aea93e42e80b1eb4319f2b58593639322fab +COMMAND_SHA256=b5d155d3138d5d9aff741f1f8ee1dc6d91f389bcb6d150b76c5d67047e1841bd DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=true RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=FAIL RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=19 -CHANGES_MADE=false +COMMAND_RC=31 +CHANGES_MADE=true ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=NO_MUTATION +MUTATION_OUTCOME=MUTATED SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=5b1d98a9c1f799dbe1cd684ee21eeac56dd2fdee5c201131bfd014a1c47401a0 -SANITIZED_OUTPUT_SHA256=5b1d98a9c1f799dbe1cd684ee21eeac56dd2fdee5c201131bfd014a1c47401a0 +RAW_EVIDENCE_SHA256=704c3dfd4261a98dde693184f8d8997876aeeb2e9c7c377ed93c33df51512ff8 +SANITIZED_OUTPUT_SHA256=704c3dfd4261a98dde693184f8d8997876aeeb2e9c7c377ed93c33df51512ff8 OUTPUT_BEGIN -USA_OPERATOR_REPAIR_BEGIN=true -COMMAND_ID=SUPPORT-260927-USA-OPERATOR-REPAIR-NETBIRD-SKLADCHIK-73317R1 +NETBIRD_RELAY_RECOVERY_BEGIN=true +COMMAND_ID=SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1 MODE=change -COMPONENT=usa-operator-repair-netbird-skladchik -MUTATION_BOUNDARY=USA_AUTHORIZED_KEYS_OPERATOR_REPAIR_IF_NEEDED_PLUS_EXISTING_USA_NETBIRD_STACK_RESTART_IF_NEEDED_PLUS_PVE01_NETBIRD_RESTART_PLUS_HEALTH_SUPERVISOR_ONESHOTS;NO_DNS_EDIT;NO_ROUTE_EDIT;NO_FIREWALL_EDIT;NO_NETBIRD_CONFIG_EDIT;NO_COMPOSE_FILE_EDIT;NO_COOKIE_EDIT +COMPONENT=netbird-relay-ingress-pve-edge-recovery +MUTATION_BOUNDARY=RELAY_NETBIRD_INGRESS_SOCKET_ENABLE_IF_NEEDED_PLUS_PVE01_HOSTS_OVERRIDE_FOR_NB_NAMES_PLUS_NETBIRD_RESTART_PLUS_HEALTH_SUPERVISOR_ONESHOTS;NO_DNS_PROVIDER_EDIT;NO_ROUTE_EDIT;NO_FIREWALL_EDIT;NO_NETBIRD_CONFIG_EDIT;NO_COOKIE_EDIT CLUSTER_QUORATE=true REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -USA_OPERATOR_KEY_FP=SHA256:wipXiBB3cpAwyWap9LcB185WjEEmDxtRJsI/UuMVAp0 -USA_LIVE_HOSTKEY_FP=SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE -USA_OPERATOR_AUTH_PRE=false -FORWARDED_AGENT_PRESENT=true -FORWARDED_PAGEANT_USA_AUTH=false -DECISION=HOLD_WINDOWS_PAGEANT_NOT_AUTHORIZED_ON_USA -NEXT_GATE=PROVIDER_CONSOLE_OR_OTHER_USA_AUTHORITY -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-USA-OPERATOR-REPAIR-NETBIRD-SKLADCHIK-73317R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +=== PRE_NETBIRD === +Error: status failed: failed connecting to Management Service : create connection: dial context: context deadline exceeded +PVE_TO_RELAY_DASHBOARD_PRE_HTTP=200 +PVE_TO_RELAY_OIDC_PRE_HTTP=200 +RELAY_INGRESS_READY=true +PVE_HOSTS_OVERRIDE=APPLIED +PVE_HOSTS_SHA256=a496860cfb488920de67686aa7d555627acc902891c8cc27ba5b1b399c248830 +PVE_RESOLVE_NB_PVEPRO=85.121.4.192 +PVE_RESOLVE_NB_GRAM1=85.121.4.192 +ACTION=PVE01_NETBIRD_RESTART_VIA_RELAY +FINAL_NETBIRD_MANAGEMENT=true +FINAL_NETBIRD_SIGNAL=true +FINAL_NETBIRD_Management: Connected +FINAL_NETBIRD_Signal: Connected +FINAL_NETBIRD_Relays: 1/1 Available +FINAL_NETBIRD_NetBird IP: 100.100.131.41/16 +FINAL_NETBIRD_Networks: - +FINAL_NETBIRD_Peers count: 0/0 Connected +POST_WAIT_NETBIRD_Relays: 1/2 Available +POST_WAIT_NETBIRD_Networks: - +POST_WAIT_NETBIRD_Peers count: 3/11 Connected +=== HEALTH_AND_SKLADCHIK_RECONCILE === +NETBIRD_PEERS_HEALTH_RC=1 +PRIVATE_VPN_HOSTS_HEALTH_RC=0 +SKLADCHIK_SUPERVISOR_ATTEMPT_1_RC=1 +SKLADCHIK_SUPERVISOR_ATTEMPT_2_RC=1 +SKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: SUPERVISOR_STATUS=FAIL +SKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: SUPERVISOR_PROBLEMS=monitor_status_not_ok +SKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: MONITOR=ERROR reports= open= resolved= +SKLADCHIK_SUPERVISOR_Sep 27 18:37:18 pve01 skladchik-reports-monitor-supervisor[3735415]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +SKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: SUPERVISOR_STATUS=FAIL +SKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: SUPERVISOR_PROBLEMS=monitor_status_not_ok +SKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: MONITOR=ERROR reports= open= resolved= +SKLADCHIK_SUPERVISOR_Sep 27 18:47:22 pve01 skladchik-reports-monitor-supervisor[3770780]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +SKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: SUPERVISOR_STATUS=FAIL +SKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: SUPERVISOR_PROBLEMS=monitor_status_not_ok +SKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: MONITOR=ERROR reports= open= resolved= +SKLADCHIK_SUPERVISOR_Sep 27 18:55:51 pve01 skladchik-reports-monitor-supervisor[3797677]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +SKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: SUPERVISOR_STATUS=FAIL +SKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: SUPERVISOR_PROBLEMS=monitor_status_not_ok +SKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: MONITOR=ERROR reports= open= resolved= +SKLADCHIK_SUPERVISOR_Sep 27 18:56:51 pve01 skladchik-reports-monitor-supervisor[3798189]: CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +SKLADCHIK_SUPERVISOR_OK=false +TASK_RESULT=PASS_NETBIRD_RESTORED_BUT_SKLADCHIK_PENDING +DECISION=HOLD_SKLADCHIK_AFTER_NETBIRD +NEXT_GATE=RECONCILE_SKLADCHIK_ONLY +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-RELAY-INGRESS-PVE-EDGE-RECOVERY-73318R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END