From 12c7b786ad4e31a6369fb9b27cd595ff585a4611 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Sun, 27 Sep 2026 17:56:04 +0000 Subject: [PATCH] runtime: publish SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1 --- ...D-PVE23-73334-STATE-RECONCILE-73335R1.json | 38 +++ ...RD-PVE23-73334-STATE-RECONCILE-73335R1.txt | 268 ++++++++++++++++ runtime/latest.json | 36 +-- runtime/latest.txt | 294 ++++++++++++++---- 4 files changed, 555 insertions(+), 81 deletions(-) create mode 100644 runtime/history/SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1.json create mode 100644 runtime/history/SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1.txt diff --git a/runtime/history/SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1.json b/runtime/history/SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1.json new file mode 100644 index 00000000..1d3d3a6f --- /dev/null +++ b/runtime/history/SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "netbird-pve23-73334-state-reconcile", + "started_at_utc": "2026-09-27T17:56:00Z", + "finished_at_utc": "2026-09-27T17:56:04Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "4dead7ca19039a978e9bbe45cbb1a249427dfef54185fbe8a3333b0b0e36930e", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "168378de6b928bbd79b2023d110ef3698fcc0e2d08f13076a8705fc19a841f3b", + "sanitized_output_sha256": "3938d5636827b2b17ab8ee468913bb11059f77ff5979c6054796a5d85a0b61ef", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "NETBIRD_PVE23_73334_STATE_RECONCILE_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1\nMODE=read-only\nCOMPONENT=netbird-pve23-73334-state-reconcile\nMUTATION_BOUNDARY=NONE_READ_ONLY_73334_ROLLBACK_FAILED_STATE_RECONCILE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED\nERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_73334_STATUS=FAIL\nSOURCE_73334_RC=31\nSOURCE_73334_MUTATION_OUTCOME=ROLLBACK_FAILED\nSOURCE_73334_ROLLBACK_RESTORED=false\nSOURCE_73334_DECISION=FAIL_PVE23_RELAY_OVERRIDE_ROLLED_BACK\nPVE02_SNAPSHOT_RC=0\n=== PVE02_CURRENT ===\nHOSTNAME=pve02\nHOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nHOSTS_MARKER_COUNT=19\nHOSTS_NETBIRD_COUNT=0\nHOSTS_NETBIRD_BEGIN\nHOSTS_NETBIRD_END\nCANONICAL_ROLLBACK_EXISTS=false\nMALFORMED_ROOT_DOUBLEQUOTE_DIR=true\nSTRAY_PATHS_BEGIN\n/root|TYPE=d|MODE=700\n/root/dns2-AdGuardHome-diagnostic-20260624T000953Z.txt|TYPE=f|MODE=644\n/root/cluster-audit-20260629T201245|TYPE=d|MODE=755\n/root/cluster-audit-20260629T201245/12_pve02_codevipe_cloud_backup_audit.txt|TYPE=f|MODE=644\n/root/cluster-audit-20260629T201245/19_pve02_host_automation_audit.txt|TYPE=f|MODE=644\n/root/cluster-audit-20260629T201245/02_pve02_audit.txt|TYPE=f|MODE=644\n/root/cluster-audit-20260629T201245/16_pve02_ct_111_audit.txt|TYPE=f|MODE=644\n/root/cluster-audit-20260629T201245/16_pve02_ct_113_audit.txt|TYPE=f|MODE=644\n/root/auto-pve02-20260616T023537Z.log|TYPE=f|MODE=644\n/root/597_enable_forum_persistent_msmtp.sh|TYPE=f|MODE=700\n/root/.profile|TYPE=f|MODE=644\n/root/\"\"|TYPE=d|MODE=755\n/root/\"\"/var|TYPE=d|MODE=755\n/root/\"\"/var/lib|TYPE=d|MODE=755\n/root/\"\"/var/lib/homelab-change-rollback|TYPE=d|MODE=755\n/root/\"\"/var/lib/homelab-change-rollback/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\"\"|TYPE=d|MODE=700\n/root/.bash_history|TYPE=f|MODE=600\n/root/codevipe-upload|TYPE=d|MODE=755\n/root/codevipe-upload/_live.aleisa_cvbd.3266202.sql.gz|TYPE=f|MODE=644\n/root/codevipe-upload/_live.aleisa.codevipe.ru.1files.3266201.tar.gz|TYPE=f|MODE=644\n/root/617_XENFORO_RUN_JOBS_ONCE_PROOF.txt|TYPE=f|MODE=644\n/root/pve01-reboot-test.log|TYPE=f|MODE=644\n/root/vm180-retirement-components-20260904T053954Z|TYPE=d|MODE=700\n/root/vm180-retirement-components-20260904T053954Z/usr|TYPE=d|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/usr/local|TYPE=d|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/usr/local/sbin|TYPE=d|MODE=700\n/root/vm180-retirement-components-20260904T053954Z/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup|TYPE=f|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/etc|TYPE=d|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/etc/systemd|TYPE=d|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system|TYPE=d|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system/homelab-cluster-admin-vm-mail-cloud-backup.timer|TYPE=f|MODE=644\n/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system/homelab-cluster-admin-vm-mail-cloud-backup.service|TYPE=f|MODE=644\n/root/.config|TYPE=d|MODE=755\n/root/.config/netbird|TYPE=d|MODE=755\n/root/.config/netbird/active_profile.txt|TYPE=f|MODE=600\n/root/.config/rclone|TYPE=d|MODE=700\n/root/.config/rclone/[SENSITIVE_FILE]|TYPE=f|MODE=600\n/root/pve01-root.pub|TYPE=f|MODE=644\n/root/auto-pve02-20260719T004725Z.log|TYPE=f|MODE=644\n/root/610_forum_fresh5_cloud_restore_drill.sh|TYPE=f|MODE=700\n/root/pve01-reboot-controller-20260623T0041.sh|TYPE=f|MODE=700\n/root/770_pve02_mailru_upload_restore_20260702T085400Z.sh|TYPE=f|MODE=644\n/root/auto-pve02-20260615T125928Z.log|TYPE=f|MODE=644\n/root/auto-pve02-20260705T004851Z.log|TYPE=f|MODE=644\n/root/pve01-reboot-controller-20260623T0041.log|TYPE=f|MODE=644\n/root/auto-pve02-20260615T095831Z.log|TYPE=f|MODE=644\n[SENSITIVE_PATH]|TYPE=d|MODE=700\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n/root/594_forum_smtp_one_shot_test.sh|TYPE=f|MODE=700\n/root/auto-pve02-20260712T004344Z.log|TYPE=f|MODE=644\n/root/dns2-AdGuardHome-20260624T000709Z.tar.gz|TYPE=f|MODE=644\n/root/homelab-apply-cluster-internal-ip.sh|TYPE=f|MODE=755\n/root/aghome-add-dnsrewrite.py|TYPE=f|MODE=644\n/root/pve01-reboot-test.sh|TYPE=f|MODE=755\n/root/backups|TYPE=d|MODE=755\n/root/backups/forum-prod-codevipe|TYPE=d|MODE=700\n/root/backups/forum-prod-codevipe/20260701T001326+0300|TYPE=d|MODE=700\n/root/backups/forum-prod-codevipe/20260701T001326+0300/SHA256SUMS|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260701T001326+0300/db_counts.txt|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260701T001326+0300/SHA256SUMS.local|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260701T001326+0300/codevipe.sql.zst|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260701T001326+0300/codevipe-public.tar.zst|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260629T191251+0300|TYPE=d|MODE=700\n/root/backups/forum-prod-codevipe/20260629T191251+0300/SHA256SUMS|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260629T191251+0300/db_counts.txt|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260629T191251+0300/SHA256SUMS.local|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260629T191251+0300/codevipe.sql.zst|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260629T191251+0300/codevipe-public.tar.zst|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260630T121120+0300|TYPE=d|MODE=700\n/root/backups/forum-prod-codevipe/20260630T121120+0300/SHA256SUMS|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260630T121120+0300/db_counts.txt|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260630T121120+0300/SHA256SUMS.local|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260630T121120+0300/codevipe.sql.zst|TYPE=f|MODE=644\nSTRAY_PATHS_END\nManagement: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nSignal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout\nRelays: 0/2 Available\nFQDN: pve02.netbird.selfhosted\nNetBird IP: 100.100.7.2/16\nPeers count: 0/11 Connected\nExecMainStartTimestamp=Fri 2026-08-28 16:31:51 MSK\nActiveState=active\nSubState=running\nActiveEnterTimestamp=Fri 2026-08-28 16:31:51 MSK\nPVE03_SNAPSHOT_RC=0\n=== PVE03_CURRENT ===\nHOSTNAME=pve03\nHOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nHOSTS_MARKER_COUNT=19\nHOSTS_NETBIRD_COUNT=0\nHOSTS_NETBIRD_BEGIN\nHOSTS_NETBIRD_END\nCANONICAL_ROLLBACK_EXISTS=false\nMALFORMED_ROOT_DOUBLEQUOTE_DIR=false\nSTRAY_PATHS_BEGIN\n/root|TYPE=d|MODE=700\n/root/.bashrc|TYPE=f|MODE=644\n/root/ct200-final-config-20260904T055123Z|TYPE=d|MODE=700\n/root/ct200-final-config-20260904T055123Z/200.conf.before-destroy|TYPE=f|MODE=400\n/root/auto-pve03-20260615T125951Z.log|TYPE=f|MODE=644\n/root/homelab-notes|TYPE=d|MODE=755\n/root/auto-pve03-20260615T095854Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260616T023600Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260719T004800Z.log|TYPE=f|MODE=644\n/root/baseline-pve03|TYPE=d|MODE=700\n/root/baseline-pve03/network|TYPE=d|MODE=700\n/root/baseline-pve03/network/listeners-tcp-udp.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/resolver.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/ip-link.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/ip-address.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/routes.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/firewall-summary.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/rules.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/listeners-unix.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host|TYPE=d|MODE=700\n/root/baseline-pve03/host/os-release.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/uname.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/df.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/lscpu.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/kernel-cmdline.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/findmnt.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/hostnamectl.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/timedatectl.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/security-frameworks.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/date.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/lsblk.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/uptime.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/memory.txt|TYPE=f|MODE=600\n/root/baseline-pve03/gitea|TYPE=d|MODE=700\n/root/baseline-pve03/gitea/discovery.txt|TYPE=f|MODE=600\n/root/baseline-pve03/gitea/process-metadata.txt|TYPE=f|MODE=600\n/root/baseline-pve03/gitea/systemd.txt|TYPE=f|MODE=600\n/root/baseline-pve03/storage|TYPE=d|MODE=700\n/root/baseline-pve03/storage/gitea-paths.txt|TYPE=f|MODE=600\n/root/baseline-pve03/packages|TYPE=d|MODE=700\n/root/baseline-pve03/packages/installed-relevant.txt|TYPE=f|MODE=600\n/root/baseline-pve03/packages/versions.txt|TYPE=f|MODE=600\n/root/baseline-pve03/SHA256SUMS|TYPE=f|MODE=600\n/root/baseline-pve03/runtime|TYPE=d|MODE=700\n/root/baseline-pve03/runtime/running-services.txt|TYPE=f|MODE=600\n/root/baseline-pve03/runtime/service-files.txt|TYPE=f|MODE=600\n/root/baseline-pve03/runtime/cron-locations.txt|TYPE=f|MODE=600\n/root/baseline-pve03/runtime/processes.txt|TYPE=f|MODE=600\n/root/baseline-pve03/runtime/timers.txt|TYPE=f|MODE=600\n/root/baseline-pve03/README.txt|TYPE=f|MODE=600\n/root/baseline-pve03.tar.gz.sha256|TYPE=f|MODE=600\n/root/newfi-task7-restore78-attempt.json|TYPE=f|MODE=600\n/root/auto-pve03-20260615T184348Z.log|TYPE=f|MODE=644\n/root/newfi-task7-restore76-attempt.json|TYPE=f|MODE=600\n/root/.config|TYPE=d|MODE=755\n/root/.config/rclone|TYPE=d|MODE=700\n/root/.config/rclone/[SENSITIVE_FILE]|TYPE=f|MODE=600\n/root/.config/netbird|TYPE=d|MODE=755\n/root/.config/netbird/active_profile.txt|TYPE=f|MODE=600\n/root/homelab-cluster-internal-hosts.sh|TYPE=f|MODE=755\n/root/.profile|TYPE=f|MODE=644\n/root/cluster-baseline.sh|TYPE=f|MODE=700\n/root/newfi75-worker.log|TYPE=f|MODE=600\n/root/homelab-apply-cluster-internal-ip.sh|TYPE=f|MODE=755\n/root/qm-130-config-before-resource-resize-20260615T192848Z.txt|TYPE=f|MODE=644\n/root/auto-pve03-20260615T121611Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260615T201509Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260615T145447Z.log|TYPE=f|MODE=644\n/root/.forward|TYPE=f|MODE=644\n/root/run-pve03-manual-backups-20260615T092743Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260705T004933Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260628T004914Z.log|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z|TYPE=d|MODE=700\n/root/newfi-pre-vm211-20260904T162806Z/debian-image.SHA256|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z/qm-list.before.txt|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z/pvesm-status.before.txt|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z/debian-image.qemu-info.txt|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z/ip-62.before.txt|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z/debian-image.SHA512|TYPE=f|MODE=644\n/root/pve01-root-id_rsa.pub|TYPE=f|MODE=644\nSTRAY_PATHS_END\nManagement: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nSignal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->46.16.34.129:443: read: connection timed out\nRelays: 0/2 Available\nFQDN: pve03.netbird.selfhosted\nNetBird IP: 100.100.34.141/16\nPeers count: 0/11 Connected\nExecMainStartTimestamp=Fri 2026-08-28 16:34:03 MSK\nActiveState=active\nSubState=running\nActiveEnterTimestamp=Fri 2026-08-28 16:34:03 MSK\n=== CLASSIFICATION ===\nEXPECTED_PRE_HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nPVE02_HOSTS_UNCHANGED=true\nPVE03_HOSTS_UNCHANGED=true\nPVE02_MARKER_COUNT=19\nPVE03_MARKER_COUNT=19\nPVE02_NETBIRD_HOSTS_COUNT=0\nPVE03_NETBIRD_HOSTS_COUNT=0\nPVE02_CANONICAL_ROLLBACK_EXISTS=false\nPVE03_CANONICAL_ROLLBACK_EXISTS=false\nPVE02_STRAY_PATH_COUNT=80\nPVE03_STRAY_PATH_COUNT=80\nDECISION=HOLD_73334_PARTIAL_STATE_PRESENT\nNEXT_GATE=TARGETED_73334_PARTIAL_STATE_ROLLBACK_BEFORE_ANY_RETRY\nTASK_RESULT=PASS_73334_STATE_RECONCILE_PARTIAL_MUTATION_FOUND\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1.txt b/runtime/history/SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1.txt new file mode 100644 index 00000000..40c358fc --- /dev/null +++ b/runtime/history/SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1.txt @@ -0,0 +1,268 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=netbird-pve23-73334-state-reconcile +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=4dead7ca19039a978e9bbe45cbb1a249427dfef54185fbe8a3333b0b0e36930e +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=168378de6b928bbd79b2023d110ef3698fcc0e2d08f13076a8705fc19a841f3b +SANITIZED_OUTPUT_SHA256=3938d5636827b2b17ab8ee468913bb11059f77ff5979c6054796a5d85a0b61ef +OUTPUT_BEGIN +NETBIRD_PVE23_73334_STATE_RECONCILE_BEGIN=true +COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1 +MODE=read-only +COMPONENT=netbird-pve23-73334-state-reconcile +MUTATION_BOUNDARY=NONE_READ_ONLY_73334_ROLLBACK_FAILED_STATE_RECONCILE +=== AUTHORITY === +HOST=pve01 +CLUSTER_QUORATE=Yes +REFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED +ERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +SOURCE_73334_STATUS=FAIL +SOURCE_73334_RC=31 +SOURCE_73334_MUTATION_OUTCOME=ROLLBACK_FAILED +SOURCE_73334_ROLLBACK_RESTORED=false +SOURCE_73334_DECISION=FAIL_PVE23_RELAY_OVERRIDE_ROLLED_BACK +PVE02_SNAPSHOT_RC=0 +=== PVE02_CURRENT === +HOSTNAME=pve02 +HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec +HOSTS_MARKER_COUNT=19 +HOSTS_NETBIRD_COUNT=0 +HOSTS_NETBIRD_BEGIN +HOSTS_NETBIRD_END +CANONICAL_ROLLBACK_EXISTS=false +MALFORMED_ROOT_DOUBLEQUOTE_DIR=true +STRAY_PATHS_BEGIN +/root|TYPE=d|MODE=700 +/root/dns2-AdGuardHome-diagnostic-20260624T000953Z.txt|TYPE=f|MODE=644 +/root/cluster-audit-20260629T201245|TYPE=d|MODE=755 +/root/cluster-audit-20260629T201245/12_pve02_codevipe_cloud_backup_audit.txt|TYPE=f|MODE=644 +/root/cluster-audit-20260629T201245/19_pve02_host_automation_audit.txt|TYPE=f|MODE=644 +/root/cluster-audit-20260629T201245/02_pve02_audit.txt|TYPE=f|MODE=644 +/root/cluster-audit-20260629T201245/16_pve02_ct_111_audit.txt|TYPE=f|MODE=644 +/root/cluster-audit-20260629T201245/16_pve02_ct_113_audit.txt|TYPE=f|MODE=644 +/root/auto-pve02-20260616T023537Z.log|TYPE=f|MODE=644 +/root/597_enable_forum_persistent_msmtp.sh|TYPE=f|MODE=700 +/root/.profile|TYPE=f|MODE=644 +/root/""|TYPE=d|MODE=755 +/root/""/var|TYPE=d|MODE=755 +/root/""/var/lib|TYPE=d|MODE=755 +/root/""/var/lib/homelab-change-rollback|TYPE=d|MODE=755 +/root/""/var/lib/homelab-change-rollback/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1""|TYPE=d|MODE=700 +/root/.bash_history|TYPE=f|MODE=600 +/root/codevipe-upload|TYPE=d|MODE=755 +/root/codevipe-upload/_live.aleisa_cvbd.3266202.sql.gz|TYPE=f|MODE=644 +/root/codevipe-upload/_live.aleisa.codevipe.ru.1files.3266201.tar.gz|TYPE=f|MODE=644 +/root/617_XENFORO_RUN_JOBS_ONCE_PROOF.txt|TYPE=f|MODE=644 +/root/pve01-reboot-test.log|TYPE=f|MODE=644 +/root/vm180-retirement-components-20260904T053954Z|TYPE=d|MODE=700 +/root/vm180-retirement-components-20260904T053954Z/usr|TYPE=d|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/usr/local|TYPE=d|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/usr/local/sbin|TYPE=d|MODE=700 +/root/vm180-retirement-components-20260904T053954Z/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup|TYPE=f|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/etc|TYPE=d|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/etc/systemd|TYPE=d|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system|TYPE=d|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system/homelab-cluster-admin-vm-mail-cloud-backup.timer|TYPE=f|MODE=644 +/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system/homelab-cluster-admin-vm-mail-cloud-backup.service|TYPE=f|MODE=644 +/root/.config|TYPE=d|MODE=755 +/root/.config/netbird|TYPE=d|MODE=755 +/root/.config/netbird/active_profile.txt|TYPE=f|MODE=600 +/root/.config/rclone|TYPE=d|MODE=700 +/root/.config/rclone/[SENSITIVE_FILE]|TYPE=f|MODE=600 +/root/pve01-root.pub|TYPE=f|MODE=644 +/root/auto-pve02-20260719T004725Z.log|TYPE=f|MODE=644 +/root/610_forum_fresh5_cloud_restore_drill.sh|TYPE=f|MODE=700 +/root/pve01-reboot-controller-20260623T0041.sh|TYPE=f|MODE=700 +/root/770_pve02_mailru_upload_restore_20260702T085400Z.sh|TYPE=f|MODE=644 +/root/auto-pve02-20260615T125928Z.log|TYPE=f|MODE=644 +/root/auto-pve02-20260705T004851Z.log|TYPE=f|MODE=644 +/root/pve01-reboot-controller-20260623T0041.log|TYPE=f|MODE=644 +/root/auto-pve02-20260615T095831Z.log|TYPE=f|MODE=644 +[SENSITIVE_PATH]|TYPE=d|MODE=700 +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +/root/594_forum_smtp_one_shot_test.sh|TYPE=f|MODE=700 +/root/auto-pve02-20260712T004344Z.log|TYPE=f|MODE=644 +/root/dns2-AdGuardHome-20260624T000709Z.tar.gz|TYPE=f|MODE=644 +/root/homelab-apply-cluster-internal-ip.sh|TYPE=f|MODE=755 +/root/aghome-add-dnsrewrite.py|TYPE=f|MODE=644 +/root/pve01-reboot-test.sh|TYPE=f|MODE=755 +/root/backups|TYPE=d|MODE=755 +/root/backups/forum-prod-codevipe|TYPE=d|MODE=700 +/root/backups/forum-prod-codevipe/20260701T001326+0300|TYPE=d|MODE=700 +/root/backups/forum-prod-codevipe/20260701T001326+0300/SHA256SUMS|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260701T001326+0300/db_counts.txt|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260701T001326+0300/SHA256SUMS.local|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260701T001326+0300/codevipe.sql.zst|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260701T001326+0300/codevipe-public.tar.zst|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260629T191251+0300|TYPE=d|MODE=700 +/root/backups/forum-prod-codevipe/20260629T191251+0300/SHA256SUMS|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260629T191251+0300/db_counts.txt|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260629T191251+0300/SHA256SUMS.local|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260629T191251+0300/codevipe.sql.zst|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260629T191251+0300/codevipe-public.tar.zst|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260630T121120+0300|TYPE=d|MODE=700 +/root/backups/forum-prod-codevipe/20260630T121120+0300/SHA256SUMS|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260630T121120+0300/db_counts.txt|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260630T121120+0300/SHA256SUMS.local|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260630T121120+0300/codevipe.sql.zst|TYPE=f|MODE=644 +STRAY_PATHS_END +Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready +Signal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout +Relays: 0/2 Available +FQDN: pve02.netbird.selfhosted +NetBird IP: 100.100.7.2/16 +Peers count: 0/11 Connected +ExecMainStartTimestamp=Fri 2026-08-28 16:31:51 MSK +ActiveState=active +SubState=running +ActiveEnterTimestamp=Fri 2026-08-28 16:31:51 MSK +PVE03_SNAPSHOT_RC=0 +=== PVE03_CURRENT === +HOSTNAME=pve03 +HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec +HOSTS_MARKER_COUNT=19 +HOSTS_NETBIRD_COUNT=0 +HOSTS_NETBIRD_BEGIN +HOSTS_NETBIRD_END +CANONICAL_ROLLBACK_EXISTS=false +MALFORMED_ROOT_DOUBLEQUOTE_DIR=false +STRAY_PATHS_BEGIN +/root|TYPE=d|MODE=700 +/root/.bashrc|TYPE=f|MODE=644 +/root/ct200-final-config-20260904T055123Z|TYPE=d|MODE=700 +/root/ct200-final-config-20260904T055123Z/200.conf.before-destroy|TYPE=f|MODE=400 +/root/auto-pve03-20260615T125951Z.log|TYPE=f|MODE=644 +/root/homelab-notes|TYPE=d|MODE=755 +/root/auto-pve03-20260615T095854Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260616T023600Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260719T004800Z.log|TYPE=f|MODE=644 +/root/baseline-pve03|TYPE=d|MODE=700 +/root/baseline-pve03/network|TYPE=d|MODE=700 +/root/baseline-pve03/network/listeners-tcp-udp.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/resolver.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/ip-link.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/ip-address.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/routes.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/firewall-summary.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/rules.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/listeners-unix.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host|TYPE=d|MODE=700 +/root/baseline-pve03/host/os-release.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/uname.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/df.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/lscpu.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/kernel-cmdline.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/findmnt.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/hostnamectl.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/timedatectl.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/security-frameworks.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/date.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/lsblk.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/uptime.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/memory.txt|TYPE=f|MODE=600 +/root/baseline-pve03/gitea|TYPE=d|MODE=700 +/root/baseline-pve03/gitea/discovery.txt|TYPE=f|MODE=600 +/root/baseline-pve03/gitea/process-metadata.txt|TYPE=f|MODE=600 +/root/baseline-pve03/gitea/systemd.txt|TYPE=f|MODE=600 +/root/baseline-pve03/storage|TYPE=d|MODE=700 +/root/baseline-pve03/storage/gitea-paths.txt|TYPE=f|MODE=600 +/root/baseline-pve03/packages|TYPE=d|MODE=700 +/root/baseline-pve03/packages/installed-relevant.txt|TYPE=f|MODE=600 +/root/baseline-pve03/packages/versions.txt|TYPE=f|MODE=600 +/root/baseline-pve03/SHA256SUMS|TYPE=f|MODE=600 +/root/baseline-pve03/runtime|TYPE=d|MODE=700 +/root/baseline-pve03/runtime/running-services.txt|TYPE=f|MODE=600 +/root/baseline-pve03/runtime/service-files.txt|TYPE=f|MODE=600 +/root/baseline-pve03/runtime/cron-locations.txt|TYPE=f|MODE=600 +/root/baseline-pve03/runtime/processes.txt|TYPE=f|MODE=600 +/root/baseline-pve03/runtime/timers.txt|TYPE=f|MODE=600 +/root/baseline-pve03/README.txt|TYPE=f|MODE=600 +/root/baseline-pve03.tar.gz.sha256|TYPE=f|MODE=600 +/root/newfi-task7-restore78-attempt.json|TYPE=f|MODE=600 +/root/auto-pve03-20260615T184348Z.log|TYPE=f|MODE=644 +/root/newfi-task7-restore76-attempt.json|TYPE=f|MODE=600 +/root/.config|TYPE=d|MODE=755 +/root/.config/rclone|TYPE=d|MODE=700 +/root/.config/rclone/[SENSITIVE_FILE]|TYPE=f|MODE=600 +/root/.config/netbird|TYPE=d|MODE=755 +/root/.config/netbird/active_profile.txt|TYPE=f|MODE=600 +/root/homelab-cluster-internal-hosts.sh|TYPE=f|MODE=755 +/root/.profile|TYPE=f|MODE=644 +/root/cluster-baseline.sh|TYPE=f|MODE=700 +/root/newfi75-worker.log|TYPE=f|MODE=600 +/root/homelab-apply-cluster-internal-ip.sh|TYPE=f|MODE=755 +/root/qm-130-config-before-resource-resize-20260615T192848Z.txt|TYPE=f|MODE=644 +/root/auto-pve03-20260615T121611Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260615T201509Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260615T145447Z.log|TYPE=f|MODE=644 +/root/.forward|TYPE=f|MODE=644 +/root/run-pve03-manual-backups-20260615T092743Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260705T004933Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260628T004914Z.log|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z|TYPE=d|MODE=700 +/root/newfi-pre-vm211-20260904T162806Z/debian-image.SHA256|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z/qm-list.before.txt|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z/pvesm-status.before.txt|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z/debian-image.qemu-info.txt|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z/ip-62.before.txt|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z/debian-image.SHA512|TYPE=f|MODE=644 +/root/pve01-root-id_rsa.pub|TYPE=f|MODE=644 +STRAY_PATHS_END +Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready +Signal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->46.16.34.129:443: read: connection timed out +Relays: 0/2 Available +FQDN: pve03.netbird.selfhosted +NetBird IP: 100.100.34.141/16 +Peers count: 0/11 Connected +ExecMainStartTimestamp=Fri 2026-08-28 16:34:03 MSK +ActiveState=active +SubState=running +ActiveEnterTimestamp=Fri 2026-08-28 16:34:03 MSK +=== CLASSIFICATION === +EXPECTED_PRE_HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec +PVE02_HOSTS_UNCHANGED=true +PVE03_HOSTS_UNCHANGED=true +PVE02_MARKER_COUNT=19 +PVE03_MARKER_COUNT=19 +PVE02_NETBIRD_HOSTS_COUNT=0 +PVE03_NETBIRD_HOSTS_COUNT=0 +PVE02_CANONICAL_ROLLBACK_EXISTS=false +PVE03_CANONICAL_ROLLBACK_EXISTS=false +PVE02_STRAY_PATH_COUNT=80 +PVE03_STRAY_PATH_COUNT=80 +DECISION=HOLD_73334_PARTIAL_STATE_PRESENT +NEXT_GATE=TARGETED_73334_PARTIAL_STATE_ROLLBACK_BEFORE_ANY_RETRY +TASK_RESULT=PASS_73334_STATE_RECONCILE_PARTIAL_MUTATION_FOUND +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index e30fe550..1d3d3a6f 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1", - "status": "FAIL", - "rc": 31, + "command_id": "SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "netbird-pve23-relay-override-repair", - "started_at_utc": "2026-09-27T17:50:57Z", - "finished_at_utc": "2026-09-27T17:51:19Z", + "mode": "read-only", + "component": "netbird-pve23-73334-state-reconcile", + "started_at_utc": "2026-09-27T17:56:00Z", + "finished_at_utc": "2026-09-27T17:56:04Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "858ca707cc2639542b347eecec8562ffb0404891bdd2dc52facf0711cf78a8d8", + "command_sha256": "4dead7ca19039a978e9bbe45cbb1a249427dfef54185fbe8a3333b0b0e36930e", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, + "change_declared": false, "result_contract_valid": true, - "result_contract_status": "FAIL", + "result_contract_status": null, "result_contract_error": null, - "command_rc": 31, - "changes_made": true, - "rollback_started": true, - "rollback_restored": false, - "mutation_outcome": "ROLLBACK_FAILED", + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "55e81f5f3fb26257de530bd3965969f28254c1423e2fa67b5e9a202de585e57a", - "sanitized_output_sha256": "6f86a937e51d2a3d550ef83a579bdce2fcbf152399f53cb91bca7d699f34d631", + "raw_evidence_sha256": "168378de6b928bbd79b2023d110ef3698fcc0e2d08f13076a8705fc19a841f3b", + "sanitized_output_sha256": "3938d5636827b2b17ab8ee468913bb11059f77ff5979c6054796a5d85a0b61ef", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "NETBIRD_PVE23_RELAY_OVERRIDE_REPAIR_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\nMODE=change\nCOMPONENT=netbird-pve23-relay-override-repair\nMUTATION_BOUNDARY=PVE02_PVE03_ONLY_SCOPED_ETC_HOSTS_NETBIRD_RELAY_OVERRIDE_PLUS_NETBIRD_RESTART_PLUS_EXISTING_PEER_HEALTH_ONESHOT;NO_PVE01_HOSTS_OR_NETBIRD_CHANGE;NO_EDGE_CHANGE;NO_ROUTE_FIREWALL_DNS_PROVIDER_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED\nERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_73333_STATUS=OK\nSOURCE_73333_RC=0\nSOURCE_73333_DECISION=PASS_PVE23_MISSING_RELAY_OVERRIDE_ROOT_CAUSE\nPVE01_RELAY_OVERRIDE_COUNT=1\nPVE01_FORCED85_DASH=200\nPVE01_FORCED85_OIDC=200\nPVE02_SSH_HOSTNAME=pve02\nPVE03_SSH_HOSTNAME=pve03\n=== PVE02_PRE ===\nHOSTNAME=pve02\nSERVICE=active/enabled\nWT0=wt0 UNKNOWN 100.100.7.2/16 [PRIVATE_IPV6]/64 \nManagement: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nSignal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout\nRelays: 0/2 Available\nFQDN: pve02.netbird.selfhosted\nNetBird IP: 100.100.7.2/16\nPeers count: 0/11 Connected\nHOSTS_BEGIN\nHOSTS_END\nHOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nDNS_nb_pvepro_ru=46.16.34.129\nDNS_nb_gram1_ru=46.16.34.129\nHTTPS46_DASH=28|000\nHTTPS46_OIDC=28|000\nHTTPS85_DASH=0|200\nHTTPS85_OIDC=0|200\nPVE02_MUTATION_REQUIRED=true\n=== PVE03_PRE ===\nHOSTNAME=pve03\nSERVICE=active/enabled\nWT0=wt0 UNKNOWN 100.100.34.141/16 [PRIVATE_IPV6]/64 \nManagement: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nSignal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->46.16.34.129:443: read: connection timed out\nRelays: 0/2 Available\nFQDN: pve03.netbird.selfhosted\nNetBird IP: 100.100.34.141/16\nPeers count: 0/11 Connected\nHOSTS_BEGIN\nHOSTS_END\nHOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nDNS_nb_pvepro_ru=46.16.34.129\nDNS_nb_gram1_ru=46.16.34.129\nHTTPS46_DASH=28|000\nHTTPS46_OIDC=28|000\nHTTPS85_DASH=0|200\nHTTPS85_OIDC=0|200\nPVE03_MUTATION_REQUIRED=true\nACTION=APPLY_PVE02_RELAY_OVERRIDE\ncp: cannot create regular file '\"\"/var/lib/homelab-change-rollback/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\"/hosts.before\"': No such file or directory\nROLLBACK_PVE02_BEGIN=true\nROLLBACK_PVE02_RESTORE_COMMAND=false\nDECISION=FAIL_PVE23_RELAY_OVERRIDE_ROLLED_BACK\nFAILURE=PVE02_RELAY_OVERRIDE_OR_NETBIRD_ACCEPTANCE_FAILED\nNEXT_GATE=READ_EXACT_73334_AND_FIX_ONLY_FAILED_NODE_OR_ROLLBACK\nROLLBACK_RESTORED=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":true,\"rollback_restored\":false}\n" + "output": "NETBIRD_PVE23_73334_STATE_RECONCILE_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1\nMODE=read-only\nCOMPONENT=netbird-pve23-73334-state-reconcile\nMUTATION_BOUNDARY=NONE_READ_ONLY_73334_ROLLBACK_FAILED_STATE_RECONCILE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nREFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED\nERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_73334_STATUS=FAIL\nSOURCE_73334_RC=31\nSOURCE_73334_MUTATION_OUTCOME=ROLLBACK_FAILED\nSOURCE_73334_ROLLBACK_RESTORED=false\nSOURCE_73334_DECISION=FAIL_PVE23_RELAY_OVERRIDE_ROLLED_BACK\nPVE02_SNAPSHOT_RC=0\n=== PVE02_CURRENT ===\nHOSTNAME=pve02\nHOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nHOSTS_MARKER_COUNT=19\nHOSTS_NETBIRD_COUNT=0\nHOSTS_NETBIRD_BEGIN\nHOSTS_NETBIRD_END\nCANONICAL_ROLLBACK_EXISTS=false\nMALFORMED_ROOT_DOUBLEQUOTE_DIR=true\nSTRAY_PATHS_BEGIN\n/root|TYPE=d|MODE=700\n/root/dns2-AdGuardHome-diagnostic-20260624T000953Z.txt|TYPE=f|MODE=644\n/root/cluster-audit-20260629T201245|TYPE=d|MODE=755\n/root/cluster-audit-20260629T201245/12_pve02_codevipe_cloud_backup_audit.txt|TYPE=f|MODE=644\n/root/cluster-audit-20260629T201245/19_pve02_host_automation_audit.txt|TYPE=f|MODE=644\n/root/cluster-audit-20260629T201245/02_pve02_audit.txt|TYPE=f|MODE=644\n/root/cluster-audit-20260629T201245/16_pve02_ct_111_audit.txt|TYPE=f|MODE=644\n/root/cluster-audit-20260629T201245/16_pve02_ct_113_audit.txt|TYPE=f|MODE=644\n/root/auto-pve02-20260616T023537Z.log|TYPE=f|MODE=644\n/root/597_enable_forum_persistent_msmtp.sh|TYPE=f|MODE=700\n/root/.profile|TYPE=f|MODE=644\n/root/\"\"|TYPE=d|MODE=755\n/root/\"\"/var|TYPE=d|MODE=755\n/root/\"\"/var/lib|TYPE=d|MODE=755\n/root/\"\"/var/lib/homelab-change-rollback|TYPE=d|MODE=755\n/root/\"\"/var/lib/homelab-change-rollback/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1\"\"|TYPE=d|MODE=700\n/root/.bash_history|TYPE=f|MODE=600\n/root/codevipe-upload|TYPE=d|MODE=755\n/root/codevipe-upload/_live.aleisa_cvbd.3266202.sql.gz|TYPE=f|MODE=644\n/root/codevipe-upload/_live.aleisa.codevipe.ru.1files.3266201.tar.gz|TYPE=f|MODE=644\n/root/617_XENFORO_RUN_JOBS_ONCE_PROOF.txt|TYPE=f|MODE=644\n/root/pve01-reboot-test.log|TYPE=f|MODE=644\n/root/vm180-retirement-components-20260904T053954Z|TYPE=d|MODE=700\n/root/vm180-retirement-components-20260904T053954Z/usr|TYPE=d|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/usr/local|TYPE=d|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/usr/local/sbin|TYPE=d|MODE=700\n/root/vm180-retirement-components-20260904T053954Z/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup|TYPE=f|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/etc|TYPE=d|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/etc/systemd|TYPE=d|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system|TYPE=d|MODE=755\n/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system/homelab-cluster-admin-vm-mail-cloud-backup.timer|TYPE=f|MODE=644\n/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system/homelab-cluster-admin-vm-mail-cloud-backup.service|TYPE=f|MODE=644\n/root/.config|TYPE=d|MODE=755\n/root/.config/netbird|TYPE=d|MODE=755\n/root/.config/netbird/active_profile.txt|TYPE=f|MODE=600\n/root/.config/rclone|TYPE=d|MODE=700\n/root/.config/rclone/[SENSITIVE_FILE]|TYPE=f|MODE=600\n/root/pve01-root.pub|TYPE=f|MODE=644\n/root/auto-pve02-20260719T004725Z.log|TYPE=f|MODE=644\n/root/610_forum_fresh5_cloud_restore_drill.sh|TYPE=f|MODE=700\n/root/pve01-reboot-controller-20260623T0041.sh|TYPE=f|MODE=700\n/root/770_pve02_mailru_upload_restore_20260702T085400Z.sh|TYPE=f|MODE=644\n/root/auto-pve02-20260615T125928Z.log|TYPE=f|MODE=644\n/root/auto-pve02-20260705T004851Z.log|TYPE=f|MODE=644\n/root/pve01-reboot-controller-20260623T0041.log|TYPE=f|MODE=644\n/root/auto-pve02-20260615T095831Z.log|TYPE=f|MODE=644\n[SENSITIVE_PATH]|TYPE=d|MODE=700\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n[SENSITIVE_PATH]\n/root/594_forum_smtp_one_shot_test.sh|TYPE=f|MODE=700\n/root/auto-pve02-20260712T004344Z.log|TYPE=f|MODE=644\n/root/dns2-AdGuardHome-20260624T000709Z.tar.gz|TYPE=f|MODE=644\n/root/homelab-apply-cluster-internal-ip.sh|TYPE=f|MODE=755\n/root/aghome-add-dnsrewrite.py|TYPE=f|MODE=644\n/root/pve01-reboot-test.sh|TYPE=f|MODE=755\n/root/backups|TYPE=d|MODE=755\n/root/backups/forum-prod-codevipe|TYPE=d|MODE=700\n/root/backups/forum-prod-codevipe/20260701T001326+0300|TYPE=d|MODE=700\n/root/backups/forum-prod-codevipe/20260701T001326+0300/SHA256SUMS|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260701T001326+0300/db_counts.txt|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260701T001326+0300/SHA256SUMS.local|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260701T001326+0300/codevipe.sql.zst|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260701T001326+0300/codevipe-public.tar.zst|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260629T191251+0300|TYPE=d|MODE=700\n/root/backups/forum-prod-codevipe/20260629T191251+0300/SHA256SUMS|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260629T191251+0300/db_counts.txt|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260629T191251+0300/SHA256SUMS.local|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260629T191251+0300/codevipe.sql.zst|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260629T191251+0300/codevipe-public.tar.zst|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260630T121120+0300|TYPE=d|MODE=700\n/root/backups/forum-prod-codevipe/20260630T121120+0300/SHA256SUMS|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260630T121120+0300/db_counts.txt|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260630T121120+0300/SHA256SUMS.local|TYPE=f|MODE=644\n/root/backups/forum-prod-codevipe/20260630T121120+0300/codevipe.sql.zst|TYPE=f|MODE=644\nSTRAY_PATHS_END\nManagement: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nSignal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout\nRelays: 0/2 Available\nFQDN: pve02.netbird.selfhosted\nNetBird IP: 100.100.7.2/16\nPeers count: 0/11 Connected\nExecMainStartTimestamp=Fri 2026-08-28 16:31:51 MSK\nActiveState=active\nSubState=running\nActiveEnterTimestamp=Fri 2026-08-28 16:31:51 MSK\nPVE03_SNAPSHOT_RC=0\n=== PVE03_CURRENT ===\nHOSTNAME=pve03\nHOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nHOSTS_MARKER_COUNT=19\nHOSTS_NETBIRD_COUNT=0\nHOSTS_NETBIRD_BEGIN\nHOSTS_NETBIRD_END\nCANONICAL_ROLLBACK_EXISTS=false\nMALFORMED_ROOT_DOUBLEQUOTE_DIR=false\nSTRAY_PATHS_BEGIN\n/root|TYPE=d|MODE=700\n/root/.bashrc|TYPE=f|MODE=644\n/root/ct200-final-config-20260904T055123Z|TYPE=d|MODE=700\n/root/ct200-final-config-20260904T055123Z/200.conf.before-destroy|TYPE=f|MODE=400\n/root/auto-pve03-20260615T125951Z.log|TYPE=f|MODE=644\n/root/homelab-notes|TYPE=d|MODE=755\n/root/auto-pve03-20260615T095854Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260616T023600Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260719T004800Z.log|TYPE=f|MODE=644\n/root/baseline-pve03|TYPE=d|MODE=700\n/root/baseline-pve03/network|TYPE=d|MODE=700\n/root/baseline-pve03/network/listeners-tcp-udp.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/resolver.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/ip-link.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/ip-address.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/routes.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/firewall-summary.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/rules.txt|TYPE=f|MODE=600\n/root/baseline-pve03/network/listeners-unix.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host|TYPE=d|MODE=700\n/root/baseline-pve03/host/os-release.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/uname.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/df.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/lscpu.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/kernel-cmdline.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/findmnt.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/hostnamectl.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/timedatectl.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/security-frameworks.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/date.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/lsblk.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/uptime.txt|TYPE=f|MODE=600\n/root/baseline-pve03/host/memory.txt|TYPE=f|MODE=600\n/root/baseline-pve03/gitea|TYPE=d|MODE=700\n/root/baseline-pve03/gitea/discovery.txt|TYPE=f|MODE=600\n/root/baseline-pve03/gitea/process-metadata.txt|TYPE=f|MODE=600\n/root/baseline-pve03/gitea/systemd.txt|TYPE=f|MODE=600\n/root/baseline-pve03/storage|TYPE=d|MODE=700\n/root/baseline-pve03/storage/gitea-paths.txt|TYPE=f|MODE=600\n/root/baseline-pve03/packages|TYPE=d|MODE=700\n/root/baseline-pve03/packages/installed-relevant.txt|TYPE=f|MODE=600\n/root/baseline-pve03/packages/versions.txt|TYPE=f|MODE=600\n/root/baseline-pve03/SHA256SUMS|TYPE=f|MODE=600\n/root/baseline-pve03/runtime|TYPE=d|MODE=700\n/root/baseline-pve03/runtime/running-services.txt|TYPE=f|MODE=600\n/root/baseline-pve03/runtime/service-files.txt|TYPE=f|MODE=600\n/root/baseline-pve03/runtime/cron-locations.txt|TYPE=f|MODE=600\n/root/baseline-pve03/runtime/processes.txt|TYPE=f|MODE=600\n/root/baseline-pve03/runtime/timers.txt|TYPE=f|MODE=600\n/root/baseline-pve03/README.txt|TYPE=f|MODE=600\n/root/baseline-pve03.tar.gz.sha256|TYPE=f|MODE=600\n/root/newfi-task7-restore78-attempt.json|TYPE=f|MODE=600\n/root/auto-pve03-20260615T184348Z.log|TYPE=f|MODE=644\n/root/newfi-task7-restore76-attempt.json|TYPE=f|MODE=600\n/root/.config|TYPE=d|MODE=755\n/root/.config/rclone|TYPE=d|MODE=700\n/root/.config/rclone/[SENSITIVE_FILE]|TYPE=f|MODE=600\n/root/.config/netbird|TYPE=d|MODE=755\n/root/.config/netbird/active_profile.txt|TYPE=f|MODE=600\n/root/homelab-cluster-internal-hosts.sh|TYPE=f|MODE=755\n/root/.profile|TYPE=f|MODE=644\n/root/cluster-baseline.sh|TYPE=f|MODE=700\n/root/newfi75-worker.log|TYPE=f|MODE=600\n/root/homelab-apply-cluster-internal-ip.sh|TYPE=f|MODE=755\n/root/qm-130-config-before-resource-resize-20260615T192848Z.txt|TYPE=f|MODE=644\n/root/auto-pve03-20260615T121611Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260615T201509Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260615T145447Z.log|TYPE=f|MODE=644\n/root/.forward|TYPE=f|MODE=644\n/root/run-pve03-manual-backups-20260615T092743Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260705T004933Z.log|TYPE=f|MODE=644\n/root/auto-pve03-20260628T004914Z.log|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z|TYPE=d|MODE=700\n/root/newfi-pre-vm211-20260904T162806Z/debian-image.SHA256|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z/qm-list.before.txt|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z/pvesm-status.before.txt|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z/debian-image.qemu-info.txt|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z/ip-62.before.txt|TYPE=f|MODE=644\n/root/newfi-pre-vm211-20260904T162806Z/debian-image.SHA512|TYPE=f|MODE=644\n/root/pve01-root-id_rsa.pub|TYPE=f|MODE=644\nSTRAY_PATHS_END\nManagement: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready\nSignal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->46.16.34.129:443: read: connection timed out\nRelays: 0/2 Available\nFQDN: pve03.netbird.selfhosted\nNetBird IP: 100.100.34.141/16\nPeers count: 0/11 Connected\nExecMainStartTimestamp=Fri 2026-08-28 16:34:03 MSK\nActiveState=active\nSubState=running\nActiveEnterTimestamp=Fri 2026-08-28 16:34:03 MSK\n=== CLASSIFICATION ===\nEXPECTED_PRE_HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\nPVE02_HOSTS_UNCHANGED=true\nPVE03_HOSTS_UNCHANGED=true\nPVE02_MARKER_COUNT=19\nPVE03_MARKER_COUNT=19\nPVE02_NETBIRD_HOSTS_COUNT=0\nPVE03_NETBIRD_HOSTS_COUNT=0\nPVE02_CANONICAL_ROLLBACK_EXISTS=false\nPVE03_CANONICAL_ROLLBACK_EXISTS=false\nPVE02_STRAY_PATH_COUNT=80\nPVE03_STRAY_PATH_COUNT=80\nDECISION=HOLD_73334_PARTIAL_STATE_PRESENT\nNEXT_GATE=TARGETED_73334_PARTIAL_STATE_ROLLBACK_BEFORE_ANY_RETRY\nTASK_RESULT=PASS_73334_STATE_RECONCILE_PARTIAL_MUTATION_FOUND\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 80762a5a..40c358fc 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,100 +1,268 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1 -STATUS=FAIL -RC=31 +COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=netbird-pve23-relay-override-repair +MODE=read-only +COMPONENT=netbird-pve23-73334-state-reconcile REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=858ca707cc2639542b347eecec8562ffb0404891bdd2dc52facf0711cf78a8d8 +COMMAND_SHA256=4dead7ca19039a978e9bbe45cbb1a249427dfef54185fbe8a3333b0b0e36930e DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true +CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=31 -CHANGES_MADE=true -ROLLBACK_STARTED=true -ROLLBACK_RESTORED=false -MUTATION_OUTCOME=ROLLBACK_FAILED +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=55e81f5f3fb26257de530bd3965969f28254c1423e2fa67b5e9a202de585e57a -SANITIZED_OUTPUT_SHA256=6f86a937e51d2a3d550ef83a579bdce2fcbf152399f53cb91bca7d699f34d631 +RAW_EVIDENCE_SHA256=168378de6b928bbd79b2023d110ef3698fcc0e2d08f13076a8705fc19a841f3b +SANITIZED_OUTPUT_SHA256=3938d5636827b2b17ab8ee468913bb11059f77ff5979c6054796a5d85a0b61ef OUTPUT_BEGIN -NETBIRD_PVE23_RELAY_OVERRIDE_REPAIR_BEGIN=true -COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1 -MODE=change -COMPONENT=netbird-pve23-relay-override-repair -MUTATION_BOUNDARY=PVE02_PVE03_ONLY_SCOPED_ETC_HOSTS_NETBIRD_RELAY_OVERRIDE_PLUS_NETBIRD_RESTART_PLUS_EXISTING_PEER_HEALTH_ONESHOT;NO_PVE01_HOSTS_OR_NETBIRD_CHANGE;NO_EDGE_CHANGE;NO_ROUTE_FIREWALL_DNS_PROVIDER_CHANGE +NETBIRD_PVE23_73334_STATE_RECONCILE_BEGIN=true +COMMAND_ID=SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1 +MODE=read-only +COMPONENT=netbird-pve23-73334-state-reconcile +MUTATION_BOUNDARY=NONE_READ_ONLY_73334_ROLLBACK_FAILED_STATE_RECONCILE === AUTHORITY === HOST=pve01 CLUSTER_QUORATE=Yes REFERENCE_REGISTER_AUTHORITY=RUNNER_ENFORCED ERROR_REGISTER_AUTHORITY=RUNNER_ENFORCED RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -SOURCE_73333_STATUS=OK -SOURCE_73333_RC=0 -SOURCE_73333_DECISION=PASS_PVE23_MISSING_RELAY_OVERRIDE_ROOT_CAUSE -PVE01_RELAY_OVERRIDE_COUNT=1 -PVE01_FORCED85_DASH=200 -PVE01_FORCED85_OIDC=200 -PVE02_SSH_HOSTNAME=pve02 -PVE03_SSH_HOSTNAME=pve03 -=== PVE02_PRE === +SOURCE_73334_STATUS=FAIL +SOURCE_73334_RC=31 +SOURCE_73334_MUTATION_OUTCOME=ROLLBACK_FAILED +SOURCE_73334_ROLLBACK_RESTORED=false +SOURCE_73334_DECISION=FAIL_PVE23_RELAY_OVERRIDE_ROLLED_BACK +PVE02_SNAPSHOT_RC=0 +=== PVE02_CURRENT === HOSTNAME=pve02 -SERVICE=active/enabled -WT0=wt0 UNKNOWN 100.100.7.2/16 [PRIVATE_IPV6]/64 +HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec +HOSTS_MARKER_COUNT=19 +HOSTS_NETBIRD_COUNT=0 +HOSTS_NETBIRD_BEGIN +HOSTS_NETBIRD_END +CANONICAL_ROLLBACK_EXISTS=false +MALFORMED_ROOT_DOUBLEQUOTE_DIR=true +STRAY_PATHS_BEGIN +/root|TYPE=d|MODE=700 +/root/dns2-AdGuardHome-diagnostic-20260624T000953Z.txt|TYPE=f|MODE=644 +/root/cluster-audit-20260629T201245|TYPE=d|MODE=755 +/root/cluster-audit-20260629T201245/12_pve02_codevipe_cloud_backup_audit.txt|TYPE=f|MODE=644 +/root/cluster-audit-20260629T201245/19_pve02_host_automation_audit.txt|TYPE=f|MODE=644 +/root/cluster-audit-20260629T201245/02_pve02_audit.txt|TYPE=f|MODE=644 +/root/cluster-audit-20260629T201245/16_pve02_ct_111_audit.txt|TYPE=f|MODE=644 +/root/cluster-audit-20260629T201245/16_pve02_ct_113_audit.txt|TYPE=f|MODE=644 +/root/auto-pve02-20260616T023537Z.log|TYPE=f|MODE=644 +/root/597_enable_forum_persistent_msmtp.sh|TYPE=f|MODE=700 +/root/.profile|TYPE=f|MODE=644 +/root/""|TYPE=d|MODE=755 +/root/""/var|TYPE=d|MODE=755 +/root/""/var/lib|TYPE=d|MODE=755 +/root/""/var/lib/homelab-change-rollback|TYPE=d|MODE=755 +/root/""/var/lib/homelab-change-rollback/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1""|TYPE=d|MODE=700 +/root/.bash_history|TYPE=f|MODE=600 +/root/codevipe-upload|TYPE=d|MODE=755 +/root/codevipe-upload/_live.aleisa_cvbd.3266202.sql.gz|TYPE=f|MODE=644 +/root/codevipe-upload/_live.aleisa.codevipe.ru.1files.3266201.tar.gz|TYPE=f|MODE=644 +/root/617_XENFORO_RUN_JOBS_ONCE_PROOF.txt|TYPE=f|MODE=644 +/root/pve01-reboot-test.log|TYPE=f|MODE=644 +/root/vm180-retirement-components-20260904T053954Z|TYPE=d|MODE=700 +/root/vm180-retirement-components-20260904T053954Z/usr|TYPE=d|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/usr/local|TYPE=d|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/usr/local/sbin|TYPE=d|MODE=700 +/root/vm180-retirement-components-20260904T053954Z/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup|TYPE=f|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/etc|TYPE=d|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/etc/systemd|TYPE=d|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system|TYPE=d|MODE=755 +/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system/homelab-cluster-admin-vm-mail-cloud-backup.timer|TYPE=f|MODE=644 +/root/vm180-retirement-components-20260904T053954Z/etc/systemd/system/homelab-cluster-admin-vm-mail-cloud-backup.service|TYPE=f|MODE=644 +/root/.config|TYPE=d|MODE=755 +/root/.config/netbird|TYPE=d|MODE=755 +/root/.config/netbird/active_profile.txt|TYPE=f|MODE=600 +/root/.config/rclone|TYPE=d|MODE=700 +/root/.config/rclone/[SENSITIVE_FILE]|TYPE=f|MODE=600 +/root/pve01-root.pub|TYPE=f|MODE=644 +/root/auto-pve02-20260719T004725Z.log|TYPE=f|MODE=644 +/root/610_forum_fresh5_cloud_restore_drill.sh|TYPE=f|MODE=700 +/root/pve01-reboot-controller-20260623T0041.sh|TYPE=f|MODE=700 +/root/770_pve02_mailru_upload_restore_20260702T085400Z.sh|TYPE=f|MODE=644 +/root/auto-pve02-20260615T125928Z.log|TYPE=f|MODE=644 +/root/auto-pve02-20260705T004851Z.log|TYPE=f|MODE=644 +/root/pve01-reboot-controller-20260623T0041.log|TYPE=f|MODE=644 +/root/auto-pve02-20260615T095831Z.log|TYPE=f|MODE=644 +[SENSITIVE_PATH]|TYPE=d|MODE=700 +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +[SENSITIVE_PATH] +/root/594_forum_smtp_one_shot_test.sh|TYPE=f|MODE=700 +/root/auto-pve02-20260712T004344Z.log|TYPE=f|MODE=644 +/root/dns2-AdGuardHome-20260624T000709Z.tar.gz|TYPE=f|MODE=644 +/root/homelab-apply-cluster-internal-ip.sh|TYPE=f|MODE=755 +/root/aghome-add-dnsrewrite.py|TYPE=f|MODE=644 +/root/pve01-reboot-test.sh|TYPE=f|MODE=755 +/root/backups|TYPE=d|MODE=755 +/root/backups/forum-prod-codevipe|TYPE=d|MODE=700 +/root/backups/forum-prod-codevipe/20260701T001326+0300|TYPE=d|MODE=700 +/root/backups/forum-prod-codevipe/20260701T001326+0300/SHA256SUMS|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260701T001326+0300/db_counts.txt|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260701T001326+0300/SHA256SUMS.local|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260701T001326+0300/codevipe.sql.zst|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260701T001326+0300/codevipe-public.tar.zst|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260629T191251+0300|TYPE=d|MODE=700 +/root/backups/forum-prod-codevipe/20260629T191251+0300/SHA256SUMS|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260629T191251+0300/db_counts.txt|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260629T191251+0300/SHA256SUMS.local|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260629T191251+0300/codevipe.sql.zst|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260629T191251+0300/codevipe-public.tar.zst|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260630T121120+0300|TYPE=d|MODE=700 +/root/backups/forum-prod-codevipe/20260630T121120+0300/SHA256SUMS|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260630T121120+0300/db_counts.txt|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260630T121120+0300/SHA256SUMS.local|TYPE=f|MODE=644 +/root/backups/forum-prod-codevipe/20260630T121120+0300/codevipe.sql.zst|TYPE=f|MODE=644 +STRAY_PATHS_END Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready Signal: Disconnected, reason: rpc error: code = Unavailable desc = keepalive ping failed to receive ACK within timeout Relays: 0/2 Available FQDN: pve02.netbird.selfhosted NetBird IP: 100.100.7.2/16 Peers count: 0/11 Connected -HOSTS_BEGIN -HOSTS_END -HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec -DNS_nb_pvepro_ru=46.16.34.129 -DNS_nb_gram1_ru=46.16.34.129 -HTTPS46_DASH=28|000 -HTTPS46_OIDC=28|000 -HTTPS85_DASH=0|200 -HTTPS85_OIDC=0|200 -PVE02_MUTATION_REQUIRED=true -=== PVE03_PRE === +ExecMainStartTimestamp=Fri 2026-08-28 16:31:51 MSK +ActiveState=active +SubState=running +ActiveEnterTimestamp=Fri 2026-08-28 16:31:51 MSK +PVE03_SNAPSHOT_RC=0 +=== PVE03_CURRENT === HOSTNAME=pve03 -SERVICE=active/enabled -WT0=wt0 UNKNOWN 100.100.34.141/16 [PRIVATE_IPV6]/64 +HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec +HOSTS_MARKER_COUNT=19 +HOSTS_NETBIRD_COUNT=0 +HOSTS_NETBIRD_BEGIN +HOSTS_NETBIRD_END +CANONICAL_ROLLBACK_EXISTS=false +MALFORMED_ROOT_DOUBLEQUOTE_DIR=false +STRAY_PATHS_BEGIN +/root|TYPE=d|MODE=700 +/root/.bashrc|TYPE=f|MODE=644 +/root/ct200-final-config-20260904T055123Z|TYPE=d|MODE=700 +/root/ct200-final-config-20260904T055123Z/200.conf.before-destroy|TYPE=f|MODE=400 +/root/auto-pve03-20260615T125951Z.log|TYPE=f|MODE=644 +/root/homelab-notes|TYPE=d|MODE=755 +/root/auto-pve03-20260615T095854Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260616T023600Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260719T004800Z.log|TYPE=f|MODE=644 +/root/baseline-pve03|TYPE=d|MODE=700 +/root/baseline-pve03/network|TYPE=d|MODE=700 +/root/baseline-pve03/network/listeners-tcp-udp.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/resolver.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/ip-link.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/ip-address.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/routes.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/firewall-summary.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/rules.txt|TYPE=f|MODE=600 +/root/baseline-pve03/network/listeners-unix.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host|TYPE=d|MODE=700 +/root/baseline-pve03/host/os-release.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/uname.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/df.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/lscpu.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/kernel-cmdline.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/findmnt.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/hostnamectl.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/timedatectl.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/security-frameworks.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/date.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/lsblk.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/uptime.txt|TYPE=f|MODE=600 +/root/baseline-pve03/host/memory.txt|TYPE=f|MODE=600 +/root/baseline-pve03/gitea|TYPE=d|MODE=700 +/root/baseline-pve03/gitea/discovery.txt|TYPE=f|MODE=600 +/root/baseline-pve03/gitea/process-metadata.txt|TYPE=f|MODE=600 +/root/baseline-pve03/gitea/systemd.txt|TYPE=f|MODE=600 +/root/baseline-pve03/storage|TYPE=d|MODE=700 +/root/baseline-pve03/storage/gitea-paths.txt|TYPE=f|MODE=600 +/root/baseline-pve03/packages|TYPE=d|MODE=700 +/root/baseline-pve03/packages/installed-relevant.txt|TYPE=f|MODE=600 +/root/baseline-pve03/packages/versions.txt|TYPE=f|MODE=600 +/root/baseline-pve03/SHA256SUMS|TYPE=f|MODE=600 +/root/baseline-pve03/runtime|TYPE=d|MODE=700 +/root/baseline-pve03/runtime/running-services.txt|TYPE=f|MODE=600 +/root/baseline-pve03/runtime/service-files.txt|TYPE=f|MODE=600 +/root/baseline-pve03/runtime/cron-locations.txt|TYPE=f|MODE=600 +/root/baseline-pve03/runtime/processes.txt|TYPE=f|MODE=600 +/root/baseline-pve03/runtime/timers.txt|TYPE=f|MODE=600 +/root/baseline-pve03/README.txt|TYPE=f|MODE=600 +/root/baseline-pve03.tar.gz.sha256|TYPE=f|MODE=600 +/root/newfi-task7-restore78-attempt.json|TYPE=f|MODE=600 +/root/auto-pve03-20260615T184348Z.log|TYPE=f|MODE=644 +/root/newfi-task7-restore76-attempt.json|TYPE=f|MODE=600 +/root/.config|TYPE=d|MODE=755 +/root/.config/rclone|TYPE=d|MODE=700 +/root/.config/rclone/[SENSITIVE_FILE]|TYPE=f|MODE=600 +/root/.config/netbird|TYPE=d|MODE=755 +/root/.config/netbird/active_profile.txt|TYPE=f|MODE=600 +/root/homelab-cluster-internal-hosts.sh|TYPE=f|MODE=755 +/root/.profile|TYPE=f|MODE=644 +/root/cluster-baseline.sh|TYPE=f|MODE=700 +/root/newfi75-worker.log|TYPE=f|MODE=600 +/root/homelab-apply-cluster-internal-ip.sh|TYPE=f|MODE=755 +/root/qm-130-config-before-resource-resize-20260615T192848Z.txt|TYPE=f|MODE=644 +/root/auto-pve03-20260615T121611Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260615T201509Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260615T145447Z.log|TYPE=f|MODE=644 +/root/.forward|TYPE=f|MODE=644 +/root/run-pve03-manual-backups-20260615T092743Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260705T004933Z.log|TYPE=f|MODE=644 +/root/auto-pve03-20260628T004914Z.log|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z|TYPE=d|MODE=700 +/root/newfi-pre-vm211-20260904T162806Z/debian-image.SHA256|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z/qm-list.before.txt|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z/pvesm-status.before.txt|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z/debian-image.qemu-info.txt|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z/ip-62.before.txt|TYPE=f|MODE=644 +/root/newfi-pre-vm211-20260904T162806Z/debian-image.SHA512|TYPE=f|MODE=644 +/root/pve01-root-id_rsa.pub|TYPE=f|MODE=644 +STRAY_PATHS_END Management: Disconnected, reason: rpc error: code = DeadlineExceeded desc = context deadline exceeded while waiting for connections to become ready Signal: Disconnected, reason: rpc error: code = Unavailable desc = error reading from server: read tcp [PRIVATE_IP]:50508->46.16.34.129:443: read: connection timed out Relays: 0/2 Available FQDN: pve03.netbird.selfhosted NetBird IP: 100.100.34.141/16 Peers count: 0/11 Connected -HOSTS_BEGIN -HOSTS_END -HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec -DNS_nb_pvepro_ru=46.16.34.129 -DNS_nb_gram1_ru=46.16.34.129 -HTTPS46_DASH=28|000 -HTTPS46_OIDC=28|000 -HTTPS85_DASH=0|200 -HTTPS85_OIDC=0|200 -PVE03_MUTATION_REQUIRED=true -ACTION=APPLY_PVE02_RELAY_OVERRIDE -cp: cannot create regular file '""/var/lib/homelab-change-rollback/SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1"/hosts.before"': No such file or directory -ROLLBACK_PVE02_BEGIN=true -ROLLBACK_PVE02_RESTORE_COMMAND=false -DECISION=FAIL_PVE23_RELAY_OVERRIDE_ROLLED_BACK -FAILURE=PVE02_RELAY_OVERRIDE_OR_NETBIRD_ACCEPTANCE_FAILED -NEXT_GATE=READ_EXACT_73334_AND_FIX_ONLY_FAILED_NODE_OR_ROLLBACK -ROLLBACK_RESTORED=false -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PVE23-RELAY-OVERRIDE-REPAIR-73334R1","status":"FAIL","changes_made":true,"rollback_started":true,"rollback_restored":false} +ExecMainStartTimestamp=Fri 2026-08-28 16:34:03 MSK +ActiveState=active +SubState=running +ActiveEnterTimestamp=Fri 2026-08-28 16:34:03 MSK +=== CLASSIFICATION === +EXPECTED_PRE_HOSTS_SHA=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec +PVE02_HOSTS_UNCHANGED=true +PVE03_HOSTS_UNCHANGED=true +PVE02_MARKER_COUNT=19 +PVE03_MARKER_COUNT=19 +PVE02_NETBIRD_HOSTS_COUNT=0 +PVE03_NETBIRD_HOSTS_COUNT=0 +PVE02_CANONICAL_ROLLBACK_EXISTS=false +PVE03_CANONICAL_ROLLBACK_EXISTS=false +PVE02_STRAY_PATH_COUNT=80 +PVE03_STRAY_PATH_COUNT=80 +DECISION=HOLD_73334_PARTIAL_STATE_PRESENT +NEXT_GATE=TARGETED_73334_PARTIAL_STATE_ROLLBACK_BEFORE_ANY_RETRY +TASK_RESULT=PASS_73334_STATE_RECONCILE_PARTIAL_MUTATION_FOUND +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-NETBIRD-PVE23-73334-STATE-RECONCILE-73335R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END