homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity
Files
6426cf81b1985b39c12865fbf6248221ede51189
homelab-public-context/runtime/latest.json
T

39 lines
22 KiB
JSON
Raw Blame History

{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "SUPPORT-260923-HOMELAB-MOLDOVA-ACTIVE-DEPS-RECONCILE-1132R1",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "read-only",
"component": "homelab-moldova-active-deps-reconcile",
"started_at_utc": "2026-09-23T21:07:51Z",
"finished_at_utc": "2026-09-23T21:07:54Z",
"reference_register_checked": true,
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
"error_register_checked": true,
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
"command_sha256": "8d956c461cae8f6370656cb989564264780767a2e0ae02073cc7210302312eca",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
"change_declared": false,
"result_contract_valid": true,
"result_contract_status": null,
"result_contract_error": null,
"command_rc": 0,
"changes_made": false,
"rollback_started": false,
"rollback_restored": null,
"mutation_outcome": "NO_MUTATION",
"sanitized": true,
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "85e3392765d3a95b9a1be4ad77e1d2d9c767b493fedfff9b4e785d1177c95daf",
"sanitized_output_sha256": "85e3392765d3a95b9a1be4ad77e1d2d9c767b493fedfff9b4e785d1177c95daf",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "WORKERS2_MOLDOVA_ACTIVE_DEPS_RECONCILE_BEGIN=1\nCOMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-ACTIVE-DEPS-RECONCILE-1132R1\nMODE=read-only\nCOMPONENT=homelab-moldova-active-deps-reconcile\nMUTATION_BOUNDARY=NONE;READ_ONLY_RECONCILE_ONLY;NO_PROVIDER_CALL;NO_MOLDOVA_SSH;NO_NETWORK_MUTATION;NO_SYSTEMD_ACTION;NO_DNS_CHANGE;NO_BACKUP_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT\nPURPOSE=CLASSIFY_MD_RELAY_ACTIVE_POLICY_MONITORING_AND_BACKUP_AUDIT_BLOCKERS_BEFORE_ANY_DETACH\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nAUTHORITY_GATE=PASS\nFRESH_CONTEXT_BEGIN=1\nNOW_UTC=2026-09-23T21:07:51Z\nHOSTNAME=pve01\nFRESH_CONTEXT_END=1\nMD_RELAY_TARGET_POLICY_BEGIN=1\nTARGETS_SHA256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b\nTARGETS_PARSE=true\nMATCH_COUNT=2\nTARGET_SAFE={\"due_seconds\":86400,\"enabled\":true,\"green1\":true,\"host_present\":true,\"host_sha256\":\"731a154cebb02c619b63295a7316e8484883299059110ff4200455a92aa396fd\",\"hostkey_present\":true,\"hostkey_sha256\":\"21e3a92d98f47c5f79187d8fb6a55d04160b5f0460952d2e332dfeff78bcbcc6\",\"logical_id\":\"md-relay\",\"port\":22,\"scope\":\"vps\",\"type\":\"vps_md\",\"user\":\"ops\"}\nTARGET_SAFE={\"due_seconds\":86400,\"enabled\":false,\"green1\":true,\"host_present\":true,\"host_sha256\":\"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16\",\"hostkey_present\":true,\"hostkey_sha256\":\"b67f1e81ebf030d7fd26ca5e2dc238c158bc4ad6e603cd6cc27588bee1d0628f\",\"logical_id\":\"us-netbird\",\"port\":22,\"scope\":\"vps\",\"type\":\"vps_us\",\"user\":\"root\"}\nMD_RELAY_TARGET_POLICY_END=1\nMD_RELAY_RUNTIME_STATE_BEGIN=1\nSTATE_REF|path=/var/lib/homelab-backup-v2/state/md-relay.json|mtime=2026-09-23 03:38:55.856821284 +0300|sha256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76\n1:{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-md-relay-20260923T003832Z-8051ad720cce\",\"green_verified\":false,\"last_attempt_epoch\":1790123935,\"last_success_epoch\":1790123935,\"logical_id\":\"md-relay\",\"manifest_sha256\":\"b57ca8ee55d8622f6aeb044d3063fd0bb7cb6ada83f50511c87894d083a2a428\",\"payload_sha256\":\"0d65b5ba587ec9f393e7a54af1df2e86b54332509da617d6bfe594b42d5fa3f8\",\"replicas\":[{\"crypt\":\"pve03-mail-01-crypt\",\"physical\":\"pve03-mail-01\"},{\"crypt\":\"pve01-mail-04-crypt\",\"physical\":\"pve01-mail-04\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-23T00:38:55Z\"}\nSTATE_REF|path=/var/lib/homelab-health/crowdsec-capi-netbird-final.txt|mtime=2026-07-02 22:30:48.260143650 +0300|sha256=20b03407e972e39c3a8b7e8ca008bf4a4b6928ed3e0e9a822be886c3cd63fae0\n1:STATUS=OK\n8:PRIMARY_EXIT=relay.netbird.selfhosted\nSTATE_REF|path=/var/lib/homelab-health/homelab-backup-baseline.txt|mtime=2026-06-17 23:02:13.952559036 +0300|sha256=4959757d29bb503315c60468dc0eb080b43cefde23d48e8ab192f6bd93b564d3\n1:STATUS=OK\n10:ENDPOINT_SUMMARY=disk-space.txt:OK;immich-media-backup.txt:OK;immich-media-offhost.txt:OK;memos-backup.txt:OK;memos-offhost.txt:OK;paperless-backup.txt:OK;paperless-offhost.txt:OK;netbird-vps-backup.txt:OK;netbird-vps-offhost.txt:OK;netbird-vps-restore.txt:OK;netbird-peers.txt:OK;filebrowser-backup.txt:OK;filebrowser-offhost.txt:OK;filebrowser-restore.txt:OK;app-stacks-batch-backup.txt:OK;app-stacks-batch-offhost.txt:OK;app-stacks-batch-restore.txt:OK;\n11:KUMA_MONITORS=1|Homelab Backup Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/backup.txt;24|Immich Media Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-backup.txt;25|Immich Media Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-offhost.txt;26|Disk Space Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/disk-space.txt;27|Memos Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-backup.txt;28|Memos Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-offhost.txt;29|Paperless Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-backup.txt;30|Paperless Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-offhost.txt;31|NetBird VPS Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-backup.txt;32|NetBird VPS Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-offhost.txt;33|NetBird VPS Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-restore.txt;34|NetBird Homelab Peers|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-peers.txt;35|File Browser Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-backup.txt;36|File Browser Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-offhost.txt;37|File Browser Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-restore.txt;38|App Stacks Batch Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-backup.txt;39|App Stacks Batch Offhost|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-offhost.txt;40|App Stacks Batch Restore|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-restore.txt;\nSTATE_REF|path=/var/lib/homelab-health/homepage-homelab-router-audit/homepage-homelab-router-audit-core-apps-live-homepage.txt|mtime=2026-07-02 23:09:46.546385105 +0300|sha256=a19529d0a5ce2242ce362bd02bcb0be586dd82964203f7cba38396106a6b886e\n1:REMOTE_STATUS=REVIEW\n47: 7\t/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n119:SCRIPT_CANDIDATE=/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n120:-rw-r--r-- 1 root root 8819 Jun 30 23:35 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n121:76506f002c640c7d6460fd16961df4ee1b884238a8cea7a945f6e8a2c5ec6dd4 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n276:REMOTE_STATUS=OK_772D_DONE\nSTATE_REF|path=/var/lib/homelab-health/homepage-homelab-router-audit/homepage-homelab-router-audit-edge-vm.txt|mtime=2026-07-02 23:09:35.781222820 +0300|sha256=40c927fbe0d53452a205a4cdc97e6a52272e6430c0f4a0217ac188ef9393a399\n1:REMOTE_STATUS=REVIEW\n48: 11\t/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n348:SCRIPT_CANDIDATE=/opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n349:-rw-r--r-- 1 root root 8819 Jun 30 23:35 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n350:76506f002c640c7d6460fd16961df4ee1b884238a8cea7a945f6e8a2c5ec6dd4 /opt/stacks/homepage/config/services.yaml.bak-external-moldova-relay-20260630T203500Z\n499:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083</p></div>\n565:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083</p></div>\n631:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083</p></div>\n697:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083</p></div>\n784:200:patch_status=OK_ROUTER_BOOKMARK_HTTPS_5083</p></div>\n1006:homelab-router-moscow-health-http.service enabled enabled\n1007:router-moscow-health.service enabled enabled\n1023:NPMPLUS_ROUTE\tid=8\tdomain_names=[\"home.gram1.ru\"]\tforward_scheme=http\tforward_host=[IP]\tforward_port=3000\tenabled=1\tssl_forced=1\taccess_list_id=1\tadvanced_config=\tlocations=[]\n1024:NPMPLUS_ROUTE\tid=17\tdomain_names=[\"homepage.vpn.gram1.ru\"]\tforward_scheme=http\tforward_host=[IP]\tforward_port=3000\tenabled=1\tssl_forced=1\taccess_list_id=1\tadvanced_config=\tlocations=[]\n1026:REMOTE_STATUS=OK_772D_DONE\nSTATE_REF|path=/var/lib/homelab-health/netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e\n1:STATUS=OK\n2:SERVICE=netbird-vps-backup\n5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z\n6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest\nSTATE_REF|path=/var/lib/homelab-health/netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b\n1:STATUS=OK\n5:DST=/mnt/staging/offhost/netbird-vps-backups-from-pve01\nSTATE_REF|path=/var/lib/homelab-health/netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb\n1:STATUS=OK\n5:ARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz\nSTATE_REF|path=/var/lib/homelab-health/netbird-vps-trust.txt|mtime=2026-07-18 01:54:49.485000918 +0300|sha256=643c858c4b1cfa77a7914349ccb05434a94e27a358320b687658684665d5c1ac\n1:STATUS=OK\n5:JOB_NAME=netbird-vps-backup\n7:TIMER=netbird-vps-backup.timer\n8:SERVICE=netbird-vps-backup.service\n9:TIMER_ENABLED=enabled\n14:SERVICE_EXEC_MAIN_STATUS=0\n16:SOURCE_STATUS=OK\n20:FRESHNESS=FRESH\n23:CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nMD_RELAY_RUNTIME_STATE_END=1\nBACKUP_AUDIT_RECONCILE_BEGIN=1\nUNIT|name=homelab-backup-v2.timer|load=loaded|active=active|sub=waiting|result=success\nUNIT|name=homelab-backup-v2.service|load=loaded|active=inactive|sub=dead|result=success\nUNIT|name=homelab-backup-audit.timer|load=loaded|active=active|sub=waiting|result=success\nUNIT|name=homelab-backup-audit.service|load=loaded|active=failed|sub=failed|result=exit-code\nBACKUP_HEAVY_LOCK_NOW=FREE\nSep 23 [IPV6] pve01 homelab-backup-audit[1770869]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[1770869]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[1874934]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[1874934]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[1968089]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[1968089]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2071373]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2071373]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2164434]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2164434]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 homelab-backup-audit[2655306]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2655306]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2655306]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 24 [IPV6] pve01 homelab-backup-audit[2759058]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 24 [IPV6] pve01 homelab-backup-audit[2759058]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 24 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 24 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nBACKUP_AUDIT_MD_RELAY_DIRECT_ISSUE=false\nBACKUP_AUDIT_RECONCILE_END=1\nNETBIRD_LOCAL_RECONCILE_BEGIN=1\nNETBIRD_STATUS_JSON_RC=0\nNETBIRD_JSON_PARSE=true\nNETBIRD_DAEMON_STATUS=Connected\nNETBIRD_PEER|fqdn=relay.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=edge-vm.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=mail.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_NETWORKS_LIST_RC=0\nNETBIRD_ROUTES_LIST_RC=0\nNETBIRD_LOCAL_RECONCILE_END=1\nMONITORING_RECONCILE_BEGIN=1\nKUMA_POLICY_SHA256=87e1f14bde3e5eb2d0512c949e1e32d7fe3214f38797559b6e3e1a9d855f059a\nKUMA_LIVE_QUERY_ATTEMPT=true\nKUMA_LIVE_MONITOR|31|NetBird VPS Backup|1|keyword|http://[IP]:9101/netbird-vps-backup.txt\nKUMA_LIVE_MONITOR|32|NetBird VPS Offhost Backup|1|keyword|http://[IP]:9101/netbird-vps-offhost.txt\nKUMA_LIVE_MONITOR|33|NetBird VPS Restore Validation|1|keyword|http://[IP]:9101/netbird-vps-restore.txt\nKUMA_LIVE_MONITOR|34|NetBird Homelab Peers|1|keyword|http://[IP]:9101/netbird-peers.txt\nKUMA_LIVE_MONITOR|77|NetBird Dashboard|1|http|https://nb.pvepro.ru/\nMONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e\n1:STATUS=OK\n2:SERVICE=netbird-vps-backup\n4:VPS_IP=[IP]\n5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z\n6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest\nMONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b\n1:STATUS=OK\n2:SERVICE=netbird-vps-offhost\n5:DST=/mnt/staging/offhost/netbird-vps-backups-from-pve01\nMONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb\n1:STATUS=OK\n2:SERVICE=netbird-vps-restore-validation\n5:ARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz\n9:ROOT_NETBIRD_ENTRIES=7\nMONITOR_HEALTH_REF|path=/var/lib/homelab-health/netbird-peers.txt|mtime=2026-09-24 00:03:22.325368652 +0300|sha256=dd1843335b572b28ec3b1c081478853841d6d8acf7209ab09522860d2cefd8cc\n1:STATUS=ERROR\n2:SERVICE=netbird-peers\n5:pve01 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected\n6:pve02 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected\n7:pve03 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected\n8:edge-vm service=active/enabled wt0_ip=[IP]/16 management=yes signal=yes peers=2/11 Connected\nMONITOR_HEALTH_REF|path=/var/lib/homelab-health/vps-identity-audit.txt|mtime=2026-09-23 22:36:10.160214214 +0300|sha256=2b969313bfdbe51ced1af4d8a7f78ab62dab7bd5e66fa419b9a47208ae426db6\n1:STATUS=OK TS=20260923T193559Z TYPE=vps-identity-audit VPS_IP=[IP] ALIAS=netbird.vpn.gram1.ru ALIAS_IP=[IP] SSH_TRUST=BLOCKED_NO_BACKED_UP_HOST_KEYS BACKUP_HOSTKEYS=NO KNOWN_HOSTS_UNCHANGED=YES SECRET_PRINTED=REDACTED\nMONITORING_RECONCILE_END=1\nSTATIC_ACTIVE_REFERENCE_RECONCILE_BEGIN=1\nACTIVE_REF|path=/etc/homelab-backup-v2/targets.json|mtime=2026-09-22 12:19:09.941039348 +0300|sha256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b\n169: \"logical_id\": \"md-relay\",\nACTIVE_REF|path=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf|mtime=2026-07-08 08:25:22.050610124 +0300|sha256=4cf3f2825b770b32447624a611bce605d97987bd2c6e8862c84d35b166f9102a\n3:ExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof\nACTIVE_REF|path=/usr/local/sbin/homelab-vps-identity-audit|mtime=2026-06-24 16:00:07.135818676 +0300|sha256=b709691c52ae48432b70e197eb4810b6c374a05911cee3b9fdba4ecc67c76c3a\n23:backup=\"$(find /mnt/staging/netbird-vps-backups/snapshots -maxdepth 2 -type f -name 'netbird-vps-backup.tgz' 2>/dev/null | sort | tail -n1 || true)\"\nSTATIC_ACTIVE_REFERENCE_RECONCILE_END=1\nMD_RELAY_ENABLED_CLASS=true\nDECISION=HOLD_DELETE_MD_RELAY_ACTIVE_BACKUP_POLICY\nNEXT_GATE=DETACH_MD_RELAY_FROM_BACKUPV2_POLICY_WITH_ROLLBACK\nTASK_RESULT=PASS_MOLDOVA_ACTIVE_DEPS_RECONCILE_EVIDENCE_COLLECTED\nWORKERS2_MOLDOVA_ACTIVE_DEPS_RECONCILE_END=1\n"
}
View Git Blame Copy Permalink