diff --git a/runtime/history/FORUM-260923-REPAIR-ERROR-ALERT6F2BR1.json b/runtime/history/FORUM-260923-REPAIR-ERROR-ALERT6F2BR1.json new file mode 100644 index 00000000..4f8843ea --- /dev/null +++ b/runtime/history/FORUM-260923-REPAIR-ERROR-ALERT6F2BR1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "FORUM-260923-REPAIR-ERROR-ALERT6F2BR1", + "status": "FAIL", + "rc": 3, + "host": "pve01", + "mode": "change", + "component": "forum-error-alert", + "started_at_utc": "2026-09-23T21:10:56Z", + "finished_at_utc": "2026-09-23T21:11:02Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "75e2d85aade37899d9174d3a1e02df965f4a07157d05e9c6f8b6b75dfdfa9828", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": true, + "result_contract_valid": true, + "result_contract_status": "FAIL", + "result_contract_error": null, + "command_rc": 3, + "changes_made": true, + "rollback_started": true, + "rollback_restored": true, + "mutation_outcome": "ROLLED_BACK", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2", + "sanitized_output_sha256": "a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "CID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1\nMODE=change\nCOMPONENT=forum-error-alert\nMUTATION_BOUNDARY=PVE01_FORUM_HEALTH_ADD_KINGOFWOLK;VM160_NEW_XF_ERROR_ALERT_SCRIPT_SERVICE_TIMER_STATE;ONE_CANARY_EMAIL;NO_XF_DB_WRITE;NO_XF_ERROR_DELETE;NO_MAIL_TRANSPORT_RECONFIG;NO_CLOUD;NO_VM_CONFIG\nEXECUTION_STARTED=true\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nHEALTH_PRE_SHA256=d7cee34b7d0e93c578f9203ba6e3634d304fee3ea5f3629a576dd683f1b50b29\nPVE02_TRANSPORT=PASS\nCHANGES_MADE=true\nPVE01_HEALTH_PATCHED=true\nHEALTH_POST_SHA256=4ee0c432fd906ba2d73198c965d0e57e4e7001781d61161b64f3c51625f034bd\nPVE02_VM160_STATE=running\nPVE02_QGA_TRANSPORT_RC=0\nVM160_GUEST_EXITCODE=90\nGUEST_CHANGE_BEGIN=true\nGUEST_FAILURE=MONITOR_PREFLIGHT\nGUEST_ROLLBACK_STARTED=true\nGUEST_ROLLBACK_RESTORED=true\nVM160_GUEST_STDERR_PRESENT=true\nVM160_GUEST_STDERR_SHA256=fd6627b9e2d9e0eca04003d62a4416ea377f7de34a5afb1800512ed9cf9094bd\nVM160_GUEST_STDERR_BYTES=94\nFAILURE=VM160_ALERT_INSTALL_OR_CANARY\nROLLBACK_STARTED=true\nROLLBACK_RESTORED=true\nDECISION=FAIL_REPAIR_ROLLED_BACK\nNEXT_GATE=REVIEW_FAILURE\nTASK_RESULT=FAIL_FORUM_ERROR_ALERT_REPAIR\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"FORUM-260923-REPAIR-ERROR-ALERT6F2BR1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":true,\"rollback_restored\":true}\n" +} diff --git a/runtime/history/FORUM-260923-REPAIR-ERROR-ALERT6F2BR1.txt b/runtime/history/FORUM-260923-REPAIR-ERROR-ALERT6F2BR1.txt new file mode 100644 index 00000000..816e4804 --- /dev/null +++ b/runtime/history/FORUM-260923-REPAIR-ERROR-ALERT6F2BR1.txt @@ -0,0 +1,62 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1 +STATUS=FAIL +RC=3 +HOST=pve01 +MODE=change +COMPONENT=forum-error-alert +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=75e2d85aade37899d9174d3a1e02df965f4a07157d05e9c6f8b6b75dfdfa9828 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=true +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=3 +CHANGES_MADE=true +ROLLBACK_STARTED=true +ROLLBACK_RESTORED=true +MUTATION_OUTCOME=ROLLED_BACK +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2 +SANITIZED_OUTPUT_SHA256=a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2 +OUTPUT_BEGIN +CID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1 +MODE=change +COMPONENT=forum-error-alert +MUTATION_BOUNDARY=PVE01_FORUM_HEALTH_ADD_KINGOFWOLK;VM160_NEW_XF_ERROR_ALERT_SCRIPT_SERVICE_TIMER_STATE;ONE_CANARY_EMAIL;NO_XF_DB_WRITE;NO_XF_ERROR_DELETE;NO_MAIL_TRANSPORT_RECONFIG;NO_CLOUD;NO_VM_CONFIG +EXECUTION_STARTED=true +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +HEALTH_PRE_SHA256=d7cee34b7d0e93c578f9203ba6e3634d304fee3ea5f3629a576dd683f1b50b29 +PVE02_TRANSPORT=PASS +CHANGES_MADE=true +PVE01_HEALTH_PATCHED=true +HEALTH_POST_SHA256=4ee0c432fd906ba2d73198c965d0e57e4e7001781d61161b64f3c51625f034bd +PVE02_VM160_STATE=running +PVE02_QGA_TRANSPORT_RC=0 +VM160_GUEST_EXITCODE=90 +GUEST_CHANGE_BEGIN=true +GUEST_FAILURE=MONITOR_PREFLIGHT +GUEST_ROLLBACK_STARTED=true +GUEST_ROLLBACK_RESTORED=true +VM160_GUEST_STDERR_PRESENT=true +VM160_GUEST_STDERR_SHA256=fd6627b9e2d9e0eca04003d62a4416ea377f7de34a5afb1800512ed9cf9094bd +VM160_GUEST_STDERR_BYTES=94 +FAILURE=VM160_ALERT_INSTALL_OR_CANARY +ROLLBACK_STARTED=true +ROLLBACK_RESTORED=true +DECISION=FAIL_REPAIR_ROLLED_BACK +NEXT_GATE=REVIEW_FAILURE +TASK_RESULT=FAIL_FORUM_ERROR_ALERT_REPAIR +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"FORUM-260923-REPAIR-ERROR-ALERT6F2BR1","status":"FAIL","changes_made":true,"rollback_started":true,"rollback_restored":true} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index bf528bb9..4f8843ea 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260923-HOMELAB-MOLDOVA-ACTIVE-DEPS-RECONCILE-1132R1", - "status": "OK", - "rc": 0, + "command_id": "FORUM-260923-REPAIR-ERROR-ALERT6F2BR1", + "status": "FAIL", + "rc": 3, "host": "pve01", - "mode": "read-only", - "component": "homelab-moldova-active-deps-reconcile", - "started_at_utc": "2026-09-23T21:07:51Z", - "finished_at_utc": "2026-09-23T21:07:54Z", + "mode": "change", + "component": "forum-error-alert", + "started_at_utc": "2026-09-23T21:10:56Z", + "finished_at_utc": "2026-09-23T21:11:02Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "8d956c461cae8f6370656cb989564264780767a2e0ae02073cc7210302312eca", + "command_sha256": "75e2d85aade37899d9174d3a1e02df965f4a07157d05e9c6f8b6b75dfdfa9828", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": false, + "change_declared": true, "result_contract_valid": true, - "result_contract_status": null, + "result_contract_status": "FAIL", "result_contract_error": null, - "command_rc": 0, - "changes_made": false, - "rollback_started": false, - "rollback_restored": null, - "mutation_outcome": "NO_MUTATION", + "command_rc": 3, + "changes_made": true, + "rollback_started": true, + "rollback_restored": true, + "mutation_outcome": "ROLLED_BACK", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "85e3392765d3a95b9a1be4ad77e1d2d9c767b493fedfff9b4e785d1177c95daf", - "sanitized_output_sha256": "85e3392765d3a95b9a1be4ad77e1d2d9c767b493fedfff9b4e785d1177c95daf", + "raw_evidence_sha256": "a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2", + "sanitized_output_sha256": "a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_MOLDOVA_ACTIVE_DEPS_RECONCILE_BEGIN=1\nCOMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-ACTIVE-DEPS-RECONCILE-1132R1\nMODE=read-only\nCOMPONENT=homelab-moldova-active-deps-reconcile\nMUTATION_BOUNDARY=NONE;READ_ONLY_RECONCILE_ONLY;NO_PROVIDER_CALL;NO_MOLDOVA_SSH;NO_NETWORK_MUTATION;NO_SYSTEMD_ACTION;NO_DNS_CHANGE;NO_BACKUP_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT\nPURPOSE=CLASSIFY_MD_RELAY_ACTIVE_POLICY_MONITORING_AND_BACKUP_AUDIT_BLOCKERS_BEFORE_ANY_DETACH\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nAUTHORITY_GATE=PASS\nFRESH_CONTEXT_BEGIN=1\nNOW_UTC=2026-09-23T21:07:51Z\nHOSTNAME=pve01\nFRESH_CONTEXT_END=1\nMD_RELAY_TARGET_POLICY_BEGIN=1\nTARGETS_SHA256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b\nTARGETS_PARSE=true\nMATCH_COUNT=2\nTARGET_SAFE={\"due_seconds\":86400,\"enabled\":true,\"green1\":true,\"host_present\":true,\"host_sha256\":\"731a154cebb02c619b63295a7316e8484883299059110ff4200455a92aa396fd\",\"hostkey_present\":true,\"hostkey_sha256\":\"21e3a92d98f47c5f79187d8fb6a55d04160b5f0460952d2e332dfeff78bcbcc6\",\"logical_id\":\"md-relay\",\"port\":22,\"scope\":\"vps\",\"type\":\"vps_md\",\"user\":\"ops\"}\nTARGET_SAFE={\"due_seconds\":86400,\"enabled\":false,\"green1\":true,\"host_present\":true,\"host_sha256\":\"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16\",\"hostkey_present\":true,\"hostkey_sha256\":\"b67f1e81ebf030d7fd26ca5e2dc238c158bc4ad6e603cd6cc27588bee1d0628f\",\"logical_id\":\"us-netbird\",\"port\":22,\"scope\":\"vps\",\"type\":\"vps_us\",\"user\":\"root\"}\nMD_RELAY_TARGET_POLICY_END=1\nMD_RELAY_RUNTIME_STATE_BEGIN=1\nSTATE_REF|path=/var/lib/homelab-backup-v2/state/md-relay.json|mtime=2026-09-23 03:38:55.856821284 +0300|sha256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76\n1:{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-md-relay-20260923T003832Z-8051ad720cce\",\"green_verified\":false,\"last_attempt_epoch\":1790123935,\"last_success_epoch\":1790123935,\"logical_id\":\"md-relay\",\"manifest_sha256\":\"b57ca8ee55d8622f6aeb044d3063fd0bb7cb6ada83f50511c87894d083a2a428\",\"payload_sha256\":\"0d65b5ba587ec9f393e7a54af1df2e86b54332509da617d6bfe594b42d5fa3f8\",\"replicas\":[{\"crypt\":\"pve03-mail-01-crypt\",\"physical\":\"pve03-mail-01\"},{\"crypt\":\"pve01-mail-04-crypt\",\"physical\":\"pve01-mail-04\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-23T00:38:55Z\"}\nSTATE_REF|path=/var/lib/homelab-health/crowdsec-capi-netbird-final.txt|mtime=2026-07-02 22:30:48.260143650 +0300|sha256=20b03407e972e39c3a8b7e8ca008bf4a4b6928ed3e0e9a822be886c3cd63fae0\n1:STATUS=OK\n8:PRIMARY_EXIT=relay.netbird.selfhosted\nSTATE_REF|path=/var/lib/homelab-health/homelab-backup-baseline.txt|mtime=2026-06-17 23:02:13.952559036 +0300|sha256=4959757d29bb503315c60468dc0eb080b43cefde23d48e8ab192f6bd93b564d3\n1:STATUS=OK\n10:ENDPOINT_SUMMARY=disk-space.txt:OK;immich-media-backup.txt:OK;immich-media-offhost.txt:OK;memos-backup.txt:OK;memos-offhost.txt:OK;paperless-backup.txt:OK;paperless-offhost.txt:OK;netbird-vps-backup.txt:OK;netbird-vps-offhost.txt:OK;netbird-vps-restore.txt:OK;netbird-peers.txt:OK;filebrowser-backup.txt:OK;filebrowser-offhost.txt:OK;filebrowser-restore.txt:OK;app-stacks-batch-backup.txt:OK;app-stacks-batch-offhost.txt:OK;app-stacks-batch-restore.txt:OK;\n11:KUMA_MONITORS=1|Homelab Backup Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/backup.txt;24|Immich Media Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-backup.txt;25|Immich Media Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-offhost.txt;26|Disk Space Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/disk-space.txt;27|Memos Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-backup.txt;28|Memos Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-offhost.txt;29|Paperless Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-backup.txt;30|Paperless Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-offhost.txt;31|NetBird VPS Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-backup.txt;32|NetBird VPS Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-offhost.txt;33|NetBird VPS Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-restore.txt;34|NetBird Homelab Peers|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-peers.txt;35|File Browser Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-backup.txt;36|File Browser Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-offhost.txt;37|File Browser Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-restore.txt;38|App Stacks Batch Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-backup.txt;39|App Stacks Batch Offhost|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-offhost.txt;40|App Stacks Batch Restore|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-restore.txt;\nSTATE_REF|path=/var/lib/homelab-health/homepage-homelab-router-audit/homepage-homelab-router-audit-core-apps-live-homepage.txt|mtime=2026-07-02 23:09:46.546385105 +0300|sha256=a19529d0a5ce2242ce362bd02bcb0be586dd82964203f7cba38396106a6b886e\n1:REMOTE_STATUS=REVIEW\n47: 7\t/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n119:SCRIPT_CANDIDATE=/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n120:-rw-r--r-- 1 root root 8819 Jun 30 23:35 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n121:76506f002c640c7d6460fd16961df4ee1b884238a8cea7a945f6e8a2c5ec6dd4 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n276:REMOTE_STATUS=OK_772D_DONE\nSTATE_REF|path=/var/lib/homelab-health/homepage-homelab-router-audit/homepage-homelab-router-audit-edge-vm.txt|mtime=2026-07-02 23:09:35.781222820 +0300|sha256=40c927fbe0d53452a205a4cdc97e6a52272e6430c0f4a0217ac188ef9393a399\n1:REMOTE_STATUS=REVIEW\n48: 11\t/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n348:SCRIPT_CANDIDATE=/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n349:-rw-r--r-- 1 root root 8819 Jun 30 23:35 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n350:76506f002c640c7d6460fd16961df4ee1b884238a8cea7a945f6e8a2c5ec6dd4 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n499:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083

\n565:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083

\n631:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083

\n697:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083

\n784:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083

\n1006:homelab-router-moscow-health-http.service enabled enabled\n1007:router-moscow-health.service enabled enabled\n1023:NPMPLUS_ROUTE\tid=8\tdomain_names=[\"home.gram1.ru\"]\tforward_scheme=http\tforward_host=[IP]\tforward_port=3000\tenabled=1\tssl_forced=1\taccess_list_id=1\tadvanced_config=\tlocations=[]\n1024:NPMPLUS_ROUTE\tid=17\tdomain_names=[\"homepage.vpn.gram1.ru\"]\tforward_scheme=http\tforward_host=[IP]\tforward_port=3000\tenabled=1\tssl_forced=1\taccess_list_id=1\tadvanced_config=\tlocations=[]\n1026:REMOTE_STATUS=OK_772D_DONE\nSTATE_REF|path=/var/lib/homelab-health/netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e\n1:STATUS=OK\n2:SERVICE=netbird-vps-backup\n5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z\n6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest\nSTATE_REF|path=/var/lib/homelab-health/netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b\n1:STATUS=OK\n5:DST=/mnt/staging/offhost/netbird-vps-backups-from-pve01\nSTATE_REF|path=/var/lib/homelab-health/netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb\n1:STATUS=OK\n5:ARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz\nSTATE_REF|path=/var/lib/homelab-health/netbird-vps-trust.txt|mtime=2026-07-18 01:54:49.485000918 +0300|sha256=643c858c4b1cfa77a7914349ccb05434a94e27a358320b687658684665d5c1ac\n1:STATUS=OK\n5:JOB_NAME=netbird-vps-backup\n7:TIMER=netbird-vps-backup.timer\n8:SERVICE=netbird-vps-backup.service\n9:TIMER_ENABLED=enabled\n14:SERVICE_EXEC_MAIN_STATUS=0\n16:SOURCE_STATUS=OK\n20:FRESHNESS=FRESH\n23:CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nMD_RELAY_RUNTIME_STATE_END=1\nBACKUP_AUDIT_RECONCILE_BEGIN=1\nUNIT|name=homelab-backup-v2.timer|load=loaded|active=active|sub=waiting|result=success\nUNIT|name=homelab-backup-v2.service|load=loaded|active=inactive|sub=dead|result=success\nUNIT|name=homelab-backup-audit.timer|load=loaded|active=active|sub=waiting|result=success\nUNIT|name=homelab-backup-audit.service|load=loaded|active=failed|sub=failed|result=exit-code\nBACKUP_HEAVY_LOCK_NOW=FREE\nSep 23 [IPV6] pve01 homelab-backup-audit[1770869]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[1770869]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[1874934]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[1874934]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[1968089]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[1968089]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2071373]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2071373]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2164434]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2164434]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2655306]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2655306]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2655306]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 24 [IPV6] pve01 homelab-backup-audit[2759058]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 24 [IPV6] pve01 homelab-backup-audit[2759058]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 24 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 24 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nBACKUP_AUDIT_MD_RELAY_DIRECT_ISSUE=false\nBACKUP_AUDIT_RECONCILE_END=1\nNETBIRD_LOCAL_RECONCILE_BEGIN=1\nNETBIRD_STATUS_JSON_RC=0\nNETBIRD_JSON_PARSE=true\nNETBIRD_DAEMON_STATUS=Connected\nNETBIRD_PEER|fqdn=relay.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=edge-vm.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=mail.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_NETWORKS_LIST_RC=0\nNETBIRD_ROUTES_LIST_RC=0\nNETBIRD_LOCAL_RECONCILE_END=1\nMONITORING_RECONCILE_BEGIN=1\nKUMA_POLICY_SHA256=87e1f14bde3e5eb2d0512c949e1e32d7fe3214f38797559b6e3e1a9d855f059a\nKUMA_LIVE_QUERY_ATTEMPT=true\nKUMA_LIVE_MONITOR|31|NetBird VPS Backup|1|keyword|http://[IP]:9101/netbird-vps-backup.txt\nKUMA_LIVE_MONITOR|32|NetBird VPS Offhost Backup|1|keyword|http://[IP]:9101/netbird-vps-offhost.txt\nKUMA_LIVE_MONITOR|33|NetBird VPS Restore Validation|1|keyword|http://[IP]:9101/netbird-vps-restore.txt\nKUMA_LIVE_MONITOR|34|NetBird Homelab Peers|1|keyword|http://[IP]:9101/netbird-peers.txt\nKUMA_LIVE_MONITOR|77|NetBird Dashboard|1|http|https://nb.pvepro.ru/\nMONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e\n1:STATUS=OK\n2:SERVICE=netbird-vps-backup\n4:VPS_IP=[IP]\n5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z\n6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest\nMONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b\n1:STATUS=OK\n2:SERVICE=netbird-vps-offhost\n5:DST=/mnt/staging/offhost/netbird-vps-backups-from-pve01\nMONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb\n1:STATUS=OK\n2:SERVICE=netbird-vps-restore-validation\n5:ARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz\n9:ROOT_NETBIRD_ENTRIES=7\nMONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-peers.txt|mtime=2026-09-24 00:03:22.325368652 +0300|sha256=dd1843335b572b28ec3b1c081478853841d6d8acf7209ab09522860d2cefd8cc\n1:STATUS=ERROR\n2:SERVICE=netbird-peers\n5:pve01 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected\n6:pve02 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected\n7:pve03 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected\n8:edge-vm service=active/enabled wt0_ip=[IP]/16 management=yes signal=yes peers=2/11 Connected\nMONITOR_HEALTH_REF|path=/var/lib/homelab-health/vps-identity-audit.txt|mtime=2026-09-23 22:36:10.160214214 +0300|sha256=2b969313bfdbe51ced1af4d8a7f78ab62dab7bd5e66fa419b9a47208ae426db6\n1:STATUS=OK TS=20260923T193559Z TYPE=vps-identity-audit VPS_IP=[IP] ALIAS=netbird.vpn.gram1.ru ALIAS_IP=[IP] SSH_TRUST=BLOCKED_NO_BACKED_UP_HOST_KEYS BACKUP_HOSTKEYS=NO KNOWN_HOSTS_UNCHANGED=YES SECRET_PRINTED=REDACTED\nMONITORING_RECONCILE_END=1\nSTATIC_ACTIVE_REFERENCE_RECONCILE_BEGIN=1\nACTIVE_REF|path=/etc/homelab-backup-v2/targets.json|mtime=2026-09-22 12:19:09.941039348 +0300|sha256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b\n169: \"logical_id\": \"md-relay\",\nACTIVE_REF|path=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf|mtime=2026-07-08 08:25:22.050610124 +0300|sha256=4cf3f2825b770b32447624a611bce605d97987bd2c6e8862c84d35b166f9102a\n3:ExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof\nACTIVE_REF|path=/usr/local/sbin/homelab-vps-identity-audit|mtime=2026-06-24 16:00:07.135818676 +0300|sha256=b709691c52ae48432b70e197eb4810b6c374a05911cee3b9fdba4ecc67c76c3a\n23:backup=\"$(find /mnt/staging/netbird-vps-backups/snapshots -maxdepth 2 -type f -name 'netbird-vps-backup.tgz' 2>/dev/null | sort | tail -n1 || true)\"\nSTATIC_ACTIVE_REFERENCE_RECONCILE_END=1\nMD_RELAY_ENABLED_CLASS=true\nDECISION=HOLD_DELETE_MD_RELAY_ACTIVE_BACKUP_POLICY\nNEXT_GATE=DETACH_MD_RELAY_FROM_BACKUPV2_POLICY_WITH_ROLLBACK\nTASK_RESULT=PASS_MOLDOVA_ACTIVE_DEPS_RECONCILE_EVIDENCE_COLLECTED\nWORKERS2_MOLDOVA_ACTIVE_DEPS_RECONCILE_END=1\n" + "output": "CID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1\nMODE=change\nCOMPONENT=forum-error-alert\nMUTATION_BOUNDARY=PVE01_FORUM_HEALTH_ADD_KINGOFWOLK;VM160_NEW_XF_ERROR_ALERT_SCRIPT_SERVICE_TIMER_STATE;ONE_CANARY_EMAIL;NO_XF_DB_WRITE;NO_XF_ERROR_DELETE;NO_MAIL_TRANSPORT_RECONFIG;NO_CLOUD;NO_VM_CONFIG\nEXECUTION_STARTED=true\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nHEALTH_PRE_SHA256=d7cee34b7d0e93c578f9203ba6e3634d304fee3ea5f3629a576dd683f1b50b29\nPVE02_TRANSPORT=PASS\nCHANGES_MADE=true\nPVE01_HEALTH_PATCHED=true\nHEALTH_POST_SHA256=4ee0c432fd906ba2d73198c965d0e57e4e7001781d61161b64f3c51625f034bd\nPVE02_VM160_STATE=running\nPVE02_QGA_TRANSPORT_RC=0\nVM160_GUEST_EXITCODE=90\nGUEST_CHANGE_BEGIN=true\nGUEST_FAILURE=MONITOR_PREFLIGHT\nGUEST_ROLLBACK_STARTED=true\nGUEST_ROLLBACK_RESTORED=true\nVM160_GUEST_STDERR_PRESENT=true\nVM160_GUEST_STDERR_SHA256=fd6627b9e2d9e0eca04003d62a4416ea377f7de34a5afb1800512ed9cf9094bd\nVM160_GUEST_STDERR_BYTES=94\nFAILURE=VM160_ALERT_INSTALL_OR_CANARY\nROLLBACK_STARTED=true\nROLLBACK_RESTORED=true\nDECISION=FAIL_REPAIR_ROLLED_BACK\nNEXT_GATE=REVIEW_FAILURE\nTASK_RESULT=FAIL_FORUM_ERROR_ALERT_REPAIR\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"FORUM-260923-REPAIR-ERROR-ALERT6F2BR1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":true,\"rollback_restored\":true}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 8fa4c8e0..816e4804 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,222 +1,62 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-ACTIVE-DEPS-RECONCILE-1132R1 -STATUS=OK -RC=0 +COMMAND_ID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1 +STATUS=FAIL +RC=3 HOST=pve01 -MODE=read-only -COMPONENT=homelab-moldova-active-deps-reconcile +MODE=change +COMPONENT=forum-error-alert REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=8d956c461cae8f6370656cb989564264780767a2e0ae02073cc7210302312eca +COMMAND_SHA256=75e2d85aade37899d9174d3a1e02df965f4a07157d05e9c6f8b6b75dfdfa9828 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=false +CHANGE_DECLARED=true RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_STATUS=FAIL RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=0 -CHANGES_MADE=false -ROLLBACK_STARTED=false -ROLLBACK_RESTORED=null -MUTATION_OUTCOME=NO_MUTATION +COMMAND_RC=3 +CHANGES_MADE=true +ROLLBACK_STARTED=true +ROLLBACK_RESTORED=true +MUTATION_OUTCOME=ROLLED_BACK SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=85e3392765d3a95b9a1be4ad77e1d2d9c767b493fedfff9b4e785d1177c95daf -SANITIZED_OUTPUT_SHA256=85e3392765d3a95b9a1be4ad77e1d2d9c767b493fedfff9b4e785d1177c95daf +RAW_EVIDENCE_SHA256=a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2 +SANITIZED_OUTPUT_SHA256=a12460c1c7d6801be5267f3d307dfc67a155d1965fb76603039da18b91f1a6e2 OUTPUT_BEGIN -WORKERS2_MOLDOVA_ACTIVE_DEPS_RECONCILE_BEGIN=1 -COMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-ACTIVE-DEPS-RECONCILE-1132R1 -MODE=read-only -COMPONENT=homelab-moldova-active-deps-reconcile -MUTATION_BOUNDARY=NONE;READ_ONLY_RECONCILE_ONLY;NO_PROVIDER_CALL;NO_MOLDOVA_SSH;NO_NETWORK_MUTATION;NO_SYSTEMD_ACTION;NO_DNS_CHANGE;NO_BACKUP_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT -PURPOSE=CLASSIFY_MD_RELAY_ACTIVE_POLICY_MONITORING_AND_BACKUP_AUDIT_BLOCKERS_BEFORE_ANY_DETACH -REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 -ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -AUTHORITY_GATE=PASS -FRESH_CONTEXT_BEGIN=1 -NOW_UTC=2026-09-23T21:07:51Z -HOSTNAME=pve01 -FRESH_CONTEXT_END=1 -MD_RELAY_TARGET_POLICY_BEGIN=1 -TARGETS_SHA256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b -TARGETS_PARSE=true -MATCH_COUNT=2 -TARGET_SAFE={"due_seconds":86400,"enabled":true,"green1":true,"host_present":true,"host_sha256":"731a154cebb02c619b63295a7316e8484883299059110ff4200455a92aa396fd","hostkey_present":true,"hostkey_sha256":"21e3a92d98f47c5f79187d8fb6a55d04160b5f0460952d2e332dfeff78bcbcc6","logical_id":"md-relay","port":22,"scope":"vps","type":"vps_md","user":"ops"} -TARGET_SAFE={"due_seconds":86400,"enabled":false,"green1":true,"host_present":true,"host_sha256":"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16","hostkey_present":true,"hostkey_sha256":"b67f1e81ebf030d7fd26ca5e2dc238c158bc4ad6e603cd6cc27588bee1d0628f","logical_id":"us-netbird","port":22,"scope":"vps","type":"vps_us","user":"root"} -MD_RELAY_TARGET_POLICY_END=1 -MD_RELAY_RUNTIME_STATE_BEGIN=1 -STATE_REF|path=/var/lib/homelab-backup-v2/state/md-relay.json|mtime=2026-09-23 03:38:55.856821284 +0300|sha256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76 -1:{"consecutive_failures":0,"failure_backoff_seconds":0,"generation":"v2-md-relay-20260923T003832Z-8051ad720cce","green_verified":false,"last_attempt_epoch":1790123935,"last_success_epoch":1790123935,"logical_id":"md-relay","manifest_sha256":"b57ca8ee55d8622f6aeb044d3063fd0bb7cb6ada83f50511c87894d083a2a428","payload_sha256":"0d65b5ba587ec9f393e7a54af1df2e86b54332509da617d6bfe594b42d5fa3f8","replicas":[{"crypt":"pve03-mail-01-crypt","physical":"pve03-mail-01"},{"crypt":"pve01-mail-04-crypt","physical":"pve01-mail-04"}],"result":"SUCCESS","retry_after_epoch":0,"schema":1,"updated_at_utc":"2026-09-23T00:38:55Z"} -STATE_REF|path=/var/lib/homelab-health/crowdsec-capi-netbird-final.txt|mtime=2026-07-02 22:30:48.260143650 +0300|sha256=20b03407e972e39c3a8b7e8ca008bf4a4b6928ed3e0e9a822be886c3cd63fae0 -1:STATUS=OK -8:PRIMARY_EXIT=relay.netbird.selfhosted -STATE_REF|path=/var/lib/homelab-health/homelab-backup-baseline.txt|mtime=2026-06-17 23:02:13.952559036 +0300|sha256=4959757d29bb503315c60468dc0eb080b43cefde23d48e8ab192f6bd93b564d3 -1:STATUS=OK -10:ENDPOINT_SUMMARY=disk-space.txt:OK;immich-media-backup.txt:OK;immich-media-offhost.txt:OK;memos-backup.txt:OK;memos-offhost.txt:OK;paperless-backup.txt:OK;paperless-offhost.txt:OK;netbird-vps-backup.txt:OK;netbird-vps-offhost.txt:OK;netbird-vps-restore.txt:OK;netbird-peers.txt:OK;filebrowser-backup.txt:OK;filebrowser-offhost.txt:OK;filebrowser-restore.txt:OK;app-stacks-batch-backup.txt:OK;app-stacks-batch-offhost.txt:OK;app-stacks-batch-restore.txt:OK; -11:KUMA_MONITORS=1|Homelab Backup Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/backup.txt;24|Immich Media Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-backup.txt;25|Immich Media Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-offhost.txt;26|Disk Space Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/disk-space.txt;27|Memos Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-backup.txt;28|Memos Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-offhost.txt;29|Paperless Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-backup.txt;30|Paperless Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-offhost.txt;31|NetBird VPS Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-backup.txt;32|NetBird VPS Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-offhost.txt;33|NetBird VPS Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-restore.txt;34|NetBird Homelab Peers|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-peers.txt;35|File Browser Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-backup.txt;36|File Browser Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-offhost.txt;37|File Browser Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-restore.txt;38|App Stacks Batch Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-backup.txt;39|App Stacks Batch Offhost|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-offhost.txt;40|App Stacks Batch Restore|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-restore.txt; -STATE_REF|path=/var/lib/homelab-health/homepage-homelab-router-audit/homepage-homelab-router-audit-core-apps-live-homepage.txt|mtime=2026-07-02 23:09:46.546385105 +0300|sha256=a19529d0a5ce2242ce362bd02bcb0be586dd82964203f7cba38396106a6b886e -1:REMOTE_STATUS=REVIEW -47: 7 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z -119:SCRIPT_CANDIDATE=/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z -120:-rw-r--r-- 1 root root 8819 Jun 30 23:35 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z -121:76506f002c640c7d6460fd16961df4ee1b884238a8cea7a945f6e8a2c5ec6dd4 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z -276:REMOTE_STATUS=OK_772D_DONE -STATE_REF|path=/var/lib/homelab-health/homepage-homelab-router-audit/homepage-homelab-router-audit-edge-vm.txt|mtime=2026-07-02 23:09:35.781222820 +0300|sha256=40c927fbe0d53452a205a4cdc97e6a52272e6430c0f4a0217ac188ef9393a399 -1:REMOTE_STATUS=REVIEW -48: 11 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z -348:SCRIPT_CANDIDATE=/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z -349:-rw-r--r-- 1 root root 8819 Jun 30 23:35 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z -350:76506f002c640c7d6460fd16961df4ee1b884238a8cea7a945f6e8a2c5ec6dd4 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z -499:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083

-565:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083

-631:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083

-697:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083

-784:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083

-1006:homelab-router-moscow-health-http.service enabled enabled -1007:router-moscow-health.service enabled enabled -1023:NPMPLUS_ROUTE id=8 domain_names=["home.gram1.ru"] forward_scheme=http forward_host=[IP] forward_port=3000 enabled=1 ssl_forced=1 access_list_id=1 advanced_config= locations=[] -1024:NPMPLUS_ROUTE id=17 domain_names=["homepage.vpn.gram1.ru"] forward_scheme=http forward_host=[IP] forward_port=3000 enabled=1 ssl_forced=1 access_list_id=1 advanced_config= locations=[] -1026:REMOTE_STATUS=OK_772D_DONE -STATE_REF|path=/var/lib/homelab-health/netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e -1:STATUS=OK -2:SERVICE=netbird-vps-backup -5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z -6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest -STATE_REF|path=/var/lib/homelab-health/netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b -1:STATUS=OK -5:DST=/mnt/staging/offhost/netbird-vps-backups-from-pve01 -STATE_REF|path=/var/lib/homelab-health/netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb -1:STATUS=OK -5:ARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz -STATE_REF|path=/var/lib/homelab-health/netbird-vps-trust.txt|mtime=2026-07-18 01:54:49.485000918 +0300|sha256=643c858c4b1cfa77a7914349ccb05434a94e27a358320b687658684665d5c1ac -1:STATUS=OK -5:JOB_NAME=netbird-vps-backup -7:TIMER=netbird-vps-backup.timer -8:SERVICE=netbird-vps-backup.service -9:TIMER_ENABLED=enabled -14:SERVICE_EXEC_MAIN_STATUS=0 -16:SOURCE_STATUS=OK -20:FRESHNESS=FRESH -23:CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -MD_RELAY_RUNTIME_STATE_END=1 -BACKUP_AUDIT_RECONCILE_BEGIN=1 -UNIT|name=homelab-backup-v2.timer|load=loaded|active=active|sub=waiting|result=success -UNIT|name=homelab-backup-v2.service|load=loaded|active=inactive|sub=dead|result=success -UNIT|name=homelab-backup-audit.timer|load=loaded|active=active|sub=waiting|result=success -UNIT|name=homelab-backup-audit.service|load=loaded|active=failed|sub=failed|result=exit-code -BACKUP_HEAVY_LOCK_NOW=FREE -Sep 23 [IPV6] pve01 homelab-backup-audit[1770869]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -Sep 23 [IPV6] pve01 homelab-backup-audit[1770869]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 homelab-backup-audit[1874934]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -Sep 23 [IPV6] pve01 homelab-backup-audit[1874934]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 homelab-backup-audit[1968089]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -Sep 23 [IPV6] pve01 homelab-backup-audit[1968089]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 homelab-backup-audit[2071373]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -Sep 23 [IPV6] pve01 homelab-backup-audit[2071373]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 homelab-backup-audit[2164434]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -Sep 23 [IPV6] pve01 homelab-backup-audit[2164434]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 homelab-backup-audit[2655306]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2655306]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2655306]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 24 [IPV6] pve01 homelab-backup-audit[2759058]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 24 [IPV6] pve01 homelab-backup-audit[2759058]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 24 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 24 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -BACKUP_AUDIT_MD_RELAY_DIRECT_ISSUE=false -BACKUP_AUDIT_RECONCILE_END=1 -NETBIRD_LOCAL_RECONCILE_BEGIN=1 -NETBIRD_STATUS_JSON_RC=0 -NETBIRD_JSON_PARSE=true -NETBIRD_DAEMON_STATUS=Connected -NETBIRD_PEER|fqdn=relay.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_PEER|fqdn=edge-vm.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_PEER|fqdn=mail.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_NETWORKS_LIST_RC=0 -NETBIRD_ROUTES_LIST_RC=0 -NETBIRD_LOCAL_RECONCILE_END=1 -MONITORING_RECONCILE_BEGIN=1 -KUMA_POLICY_SHA256=87e1f14bde3e5eb2d0512c949e1e32d7fe3214f38797559b6e3e1a9d855f059a -KUMA_LIVE_QUERY_ATTEMPT=true -KUMA_LIVE_MONITOR|31|NetBird VPS Backup|1|keyword|http://[IP]:9101/netbird-vps-backup.txt -KUMA_LIVE_MONITOR|32|NetBird VPS Offhost Backup|1|keyword|http://[IP]:9101/netbird-vps-offhost.txt -KUMA_LIVE_MONITOR|33|NetBird VPS Restore Validation|1|keyword|http://[IP]:9101/netbird-vps-restore.txt -KUMA_LIVE_MONITOR|34|NetBird Homelab Peers|1|keyword|http://[IP]:9101/netbird-peers.txt -KUMA_LIVE_MONITOR|77|NetBird Dashboard|1|http|https://nb.pvepro.ru/ -MONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e -1:STATUS=OK -2:SERVICE=netbird-vps-backup -4:VPS_IP=[IP] -5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z -6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest -MONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b -1:STATUS=OK -2:SERVICE=netbird-vps-offhost -5:DST=/mnt/staging/offhost/netbird-vps-backups-from-pve01 -MONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb -1:STATUS=OK -2:SERVICE=netbird-vps-restore-validation -5:ARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz -9:ROOT_NETBIRD_ENTRIES=7 -MONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-peers.txt|mtime=2026-09-24 00:03:22.325368652 +0300|sha256=dd1843335b572b28ec3b1c081478853841d6d8acf7209ab09522860d2cefd8cc -1:STATUS=ERROR -2:SERVICE=netbird-peers -5:pve01 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected -6:pve02 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected -7:pve03 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected -8:edge-vm service=active/enabled wt0_ip=[IP]/16 management=yes signal=yes peers=2/11 Connected -MONITOR_HEALTH_REF|path=/var/lib/homelab-health/vps-identity-audit.txt|mtime=2026-09-23 22:36:10.160214214 +0300|sha256=2b969313bfdbe51ced1af4d8a7f78ab62dab7bd5e66fa419b9a47208ae426db6 -1:STATUS=OK TS=20260923T193559Z TYPE=vps-identity-audit VPS_IP=[IP] ALIAS=netbird.vpn.gram1.ru ALIAS_IP=[IP] SSH_TRUST=BLOCKED_NO_BACKED_UP_HOST_KEYS BACKUP_HOSTKEYS=NO KNOWN_HOSTS_UNCHANGED=YES SECRET_PRINTED=REDACTED -MONITORING_RECONCILE_END=1 -STATIC_ACTIVE_REFERENCE_RECONCILE_BEGIN=1 -ACTIVE_REF|path=/etc/homelab-backup-v2/targets.json|mtime=2026-09-22 12:19:09.941039348 +0300|sha256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b -169: "logical_id": "md-relay", -ACTIVE_REF|path=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf|mtime=2026-07-08 08:25:22.050610124 +0300|sha256=4cf3f2825b770b32447624a611bce605d97987bd2c6e8862c84d35b166f9102a -3:ExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof -ACTIVE_REF|path=/usr/local/sbin/homelab-vps-identity-audit|mtime=2026-06-24 16:00:07.135818676 +0300|sha256=b709691c52ae48432b70e197eb4810b6c374a05911cee3b9fdba4ecc67c76c3a -23:backup="$(find /mnt/staging/netbird-vps-backups/snapshots -maxdepth 2 -type f -name 'netbird-vps-backup.tgz' 2>/dev/null | sort | tail -n1 || true)" -STATIC_ACTIVE_REFERENCE_RECONCILE_END=1 -MD_RELAY_ENABLED_CLASS=true -DECISION=HOLD_DELETE_MD_RELAY_ACTIVE_BACKUP_POLICY -NEXT_GATE=DETACH_MD_RELAY_FROM_BACKUPV2_POLICY_WITH_ROLLBACK -TASK_RESULT=PASS_MOLDOVA_ACTIVE_DEPS_RECONCILE_EVIDENCE_COLLECTED -WORKERS2_MOLDOVA_ACTIVE_DEPS_RECONCILE_END=1 +CID=FORUM-260923-REPAIR-ERROR-ALERT6F2BR1 +MODE=change +COMPONENT=forum-error-alert +MUTATION_BOUNDARY=PVE01_FORUM_HEALTH_ADD_KINGOFWOLK;VM160_NEW_XF_ERROR_ALERT_SCRIPT_SERVICE_TIMER_STATE;ONE_CANARY_EMAIL;NO_XF_DB_WRITE;NO_XF_ERROR_DELETE;NO_MAIL_TRANSPORT_RECONFIG;NO_CLOUD;NO_VM_CONFIG +EXECUTION_STARTED=true +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +HEALTH_PRE_SHA256=d7cee34b7d0e93c578f9203ba6e3634d304fee3ea5f3629a576dd683f1b50b29 +PVE02_TRANSPORT=PASS +CHANGES_MADE=true +PVE01_HEALTH_PATCHED=true +HEALTH_POST_SHA256=4ee0c432fd906ba2d73198c965d0e57e4e7001781d61161b64f3c51625f034bd +PVE02_VM160_STATE=running +PVE02_QGA_TRANSPORT_RC=0 +VM160_GUEST_EXITCODE=90 +GUEST_CHANGE_BEGIN=true +GUEST_FAILURE=MONITOR_PREFLIGHT +GUEST_ROLLBACK_STARTED=true +GUEST_ROLLBACK_RESTORED=true +VM160_GUEST_STDERR_PRESENT=true +VM160_GUEST_STDERR_SHA256=fd6627b9e2d9e0eca04003d62a4416ea377f7de34a5afb1800512ed9cf9094bd +VM160_GUEST_STDERR_BYTES=94 +FAILURE=VM160_ALERT_INSTALL_OR_CANARY +ROLLBACK_STARTED=true +ROLLBACK_RESTORED=true +DECISION=FAIL_REPAIR_ROLLED_BACK +NEXT_GATE=REVIEW_FAILURE +TASK_RESULT=FAIL_FORUM_ERROR_ALERT_REPAIR +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"FORUM-260923-REPAIR-ERROR-ALERT6F2BR1","status":"FAIL","changes_made":true,"rollback_started":true,"rollback_restored":true} OUTPUT_END CHAT_OUTPUT_END