CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-RCA-READONLY-1003R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=03295cba963c2536ee5ce7ad40c21850433236f1e9ec96e224aeb2b1a7e3f434
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=fd17fa51e31c0a565d811d2b81812750b64f56d64b1d1942b6bfb10f95f6ff65
SANITIZED_OUTPUT_SHA256=fd17fa51e31c0a565d811d2b81812750b64f56d64b1d1942b6bfb10f95f6ff65
OUTPUT_BEGIN
WORKERS2_BACKUP_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-RCA-READONLY-1003R1
MODE=read-only
COMPONENT=homelab-backup-rca
HOST=pve01
UID=0
TIME_UTC=2026-09-21T01:07:23.504818+00:00
MUTATION_BOUNDARY=NO_FETCH_NO_RESET_NO_SERVICE_ACTION_NO_BACKUP_NO_RESTORE_NO_DELETE_NO_MAIL
===== CANONICAL_DIVERGENCE_RCA =====
GIT_HEAD={"rc":0,"stderr_sha256":null,"stdout":"d5e2ef5639398a762283e47143c17b494f8ed579"}
GIT_BRANCH={"rc":0,"stderr_sha256":null,"stdout":"main"}
GIT_LOCAL_MAIN={"rc":0,"stderr_sha256":null,"stdout":"d5e2ef5639398a762283e47143c17b494f8ed579"}
GIT_TRACKING_ORIGIN_MAIN={"rc":0,"stderr_sha256":null,"stdout":"d5e2ef5639398a762283e47143c17b494f8ed579"}
GIT_STATUS={"rc":0,"stderr_sha256":null,"stdout":"## main...origin/main"}
GIT_WORKTREES={"rc":0,"stderr_sha256":null,"stdout":"worktree /srv/homelab-ops\nHEAD d5e2ef5639398a762283e47143c17b494f8ed579\nbranch refs/heads/main\n\nworktree /root/homelab-ops-cr-2026-0001\nHEAD d29752e1de327ad588bf3f1689f277f55e33e58c\nbranch refs/heads/cr-2026-0001-handoff-recovery\n\nworktree /root/homelab-ops-cr-2026-0002\nHEAD 320e66a4f0855fa391d8eb862bc997347fb61595\nbranch refs/heads/cr-2026-0002-offhost-retention-remediation\n\nworktree /root/homelab-ops-cr-2026-0003\nHEAD b0f9935dce45a7dee2949aef342cadec90ff38a2\nbranch refs/heads/cr-2026-0003-cluster-storage-retention-remediation\n\nworktree /root/homelab-ops-cr-2026-0004\nHEAD bf80410f592f8eb9eaa80695f609870fe3fb85fd\nbranch refs/heads/cr-2026-0004-backup-tail-closure\n\nworktree /root/homelab-ops-cr-2026-0005\nHEAD 94951a2a329bdeb41b831a26e96f3dcbb4a79bd8\nbranch refs/heads/cr-2026-0005-codex-bootstrap-activation\n\nworktree /root/homelab-ops-cr-2026-0006\nHEAD 16e84298035f5e4ec7e2b7d39967ad5e40ea3d4b\nbranch refs/heads/cr-2026-0006-cluster-cloud-quorum\n\nworktree /root/homelab-ops-cr-2026-0007\nHEAD d399d24440f948b518761ecfd516521993861b43\nbranch refs/heads/cr-2026-0007-skladchik-controlled-reauth\n\nworktree /root/homelab-ops-cr-2026-0008\nHEAD 1f7abe5acf856ad2a2abe10ef2a9452485edb073\nbranch refs/heads/cr-2026-0008-skladchik-snapshot-integrity\n\nworktree /root/homelab-ops-cr-2026-0009\nHEAD 7bd1b050f8d64c7b55dfd51ead63f0adda87f2df\nbranch refs/heads/cr-2026-0009-skladchik-concurrency-guard\n\nworktree /root/homelab-ops-cr-2026-0011\nHEAD bb25a4acfdfa827345963d65e5e2c1fe2a3f8ba9\nbranch refs/heads/cr-2026-0011-skladchik-cookie-jar-reset\n\nworktree /root/homelab-ops-cr-2026-0012\nHEAD f5b22d867ea467987008408e3fb1fb0e17fa7b2e\nbranch refs/heads/cr-2026-0012-skladchik-cookie-refresh-runbook\n\nworktree /root/homelab-ops-cr-2026-0013\nHEAD 25b275f96b54551e26f07516c72651d85f235ad0\nbranch refs/heads/cr-2026-0013-vm160-worker-bootstrap-transition\n\nworktree /root/homelab-ops-cr-2026-0020\nHEAD f556984e410ed819b4cb1e00635dceb43f0ffc5d\nbranch refs/heads/cr-2026-0020-app-cloud-quorum\n\nworktree /root/homelab-ops-cr-2026-0021\nHEAD 3c92036f3000189493fc47557c4cb03e7a412c82\nbranch refs/heads/cr-2026-0021-vm-backup-freshness\n\nworktree /root/homelab-ops-cr-2026-0022\nHEAD a497e0f15f1c8d11c3603a4cf5a6a65654cae6fb\nbranch refs/heads/cr-2026-0022-app-cloud-quorum-freshness\n\nworktree /root/homelab-ops-cr-2026-0024\nHEAD 332e8d3f8d2742a05a8d73637cac6a2defc3a825\nbranch refs/heads/cr-2026-0024-hbarhub-cloud-quorum\n\nworktree /root/homelab-ops-cr-2026-0025\nHEAD 73890efb9e99bf02a09daa0e547ab36b48c83b30\nbranch refs/heads/cr-2026-0025-scrutiny-cloud-quorum\n\nworktree /root/homelab-ops-cr-2026-0026\nHEAD ebc08fe5d6beef3cc927bc7d23f6ab37b35d947d\nbranch refs/heads/cr-2026-0026-backup-scheduler-rpo-hardening\n\nworktree /root/homelab-ops-cr-2026-0027\nHEAD d5a14cbaef47db39f17a94e248e183f0916bdab1\nbranch refs/heads/cr-2026-0027-backup-lock-fd-hardening\n\nworktree /root/homelab-ops-cr-2026-0028\nHEAD 6e3707e90181318be3e8616b14ee553968c485dd\nbranch refs/heads/cr-2026-0028-backup-scheduler-fairness-granularity\n\nworktree /root/homelab-ops-cr-2026-0029\nHEAD dd81175a24ff64048b36d6ec08dcd3addfc681e9\nbranch refs/heads/cr-2026-0029-control-bootstrap-cloud-quorum\n\nworktree /root/homelab-ops-cr-2026-0030\nHEAD a9fdc65084bcd82c1537409df106c18eb474440d\nbranch refs/heads/cr-2026-0030-control-configs-source-closure\n\nworktree /root/homelab-ops-cr-2026-0031\nHEAD f8941f01f41d03b3368a33df868f7ea11311eb6a\nbranch refs/heads/cr-2026-0031-vm-restore-evidence-hardening\n\nworktree /root/homelab-ops-cr-2026-0032\nHEAD 8ad7fcc4537c48de47daac086087d247886c58d9\nbranch refs/heads/cr-2026-0032-backup-unified-monitoring\n\nworktree /root/homelab-ops-cr-2026-0033\nHEAD 42fd895f421ebc177006c5eb4173f7b4f40ad47f\nbranch refs/heads/cr-2026-0033-backup-retention-safe-gc\n\nworktree /root/homelab-ops-cr-2026-0034\nHEAD 233a1aa2b3c7c2c59469412fef8dfe7ffd499505\nbranch refs/heads/cr-2026-0034-control-final-generation\n\nworktree /root/homelab-ops-cr-2026-0035\nHEAD c7a72a516b74e73b07df646374fa938eac055f32\nbranch refs/heads/cr-2026-0035-skladchik-moderator-ct200\n\nworktree /root/homelab-ops-cr-2026-0036\nHEAD 6e25fbcdce85c8599438de8f27de8632edc26592\nbranch refs/heads/cr-2026-0036-skladchik-moderator-v2-application\n\nworktree /root/homelab-ops-cr-2026-0046\nHEAD 6fc45798c21bd66c561c14059e14f09b7d0a8d63\nbranch refs/heads/cr-2026-0046-reports-monitor-canonical-backup-supervision\n\nworktree /root/homelab-ops-cr-2026-0047\nHEAD 67f61be11854a41effd2d6bb116fb0cbfb59692e\nbranch refs/heads/cr-2026-0047-vm170-apps-b-retirement\n\nworktree /root/homelab-ops-cr-2026-0048\nHEAD c9bfa5bd5dceff8bd294178f36d933768c924474\nbranch refs/heads/cr-2026-0048-immutable-offline-verifier-retirement\n\nworktree /root/homelab-ops-cr-2026-0049\nHEAD 801579bd301ad0b28f462f462919660b38041344\nbranch refs/heads/cr-2026-0049-pve03-r8152-lifecycle-repair\n\nworktree /root/homelab-ops-cr-2026-0050\nHEAD 6c1165dccfb6e84b571f0caac6773fd427bf800d\nbranch refs/heads/cr-2026-0050-backup-matrix-residue-retirement\n\nworktree /root/homelab-ops-cr-2026-0051\nHEAD 569bfa463f5cd080c62a07cd1eac2a53aad6cd7c\nbranch refs/heads/cr-2026-0051-cr0044-sealed-latch-clearance\n\nworktree /root/homelab-ops-cr-2026-0052\nHEAD 920f38561cdc9da81656734d3a8ffc8125700ca3\nbranch refs/heads/cr-2026-0052-vm160-superseded-tail-latch-clearance\n\nworktree /root/homelab-ops-cr-2026-0053\nHEAD 677e6d3c4fd8dc6ded8f65dc44a0bfdef327a68b\nbranch refs/heads/cr-2026-0053-moderator-ru-dark-competitor-retirement\n\nworktree /root/homelab-ops-cr-2026-0054\nHEAD 3f02810a16b711700a9da87189c63e38b1fa324b\nbranch refs/heads/cr-2026-0054-backup-control-coverage-hardening\n\nworktree /root/homelab-ops-cr-2026-0055\nHEAD 902126179efd488e20938a091dcee36109829b86\nbranch refs/heads/cr-2026-0055-backup-retention-plan-stability-hotfix\n\nworktree /root/homelab-ops-cr-2026-0056\nHEAD b69b63f531fb71eb95ab147bbfce57591f90e0a4\nbranch refs/heads/cr-2026-0056-backup-retention-transaction-resume-hotfix\n\nworktree /root/homelab-ops-cr-2026-0057\nHEAD 19af420239c92d1c284eb6a114c4eaf63cb36f61\nbranch refs/heads/cr-2026-0057-control-refresh\n\nworktree /root/homelab-ops-cr-2026-0058\nHEAD 4b427f5f0406cab6757cf8889fd479a62d218261\nbranch refs/heads/cr-2026-0058-legacy-backup-target-retirement\n\nworktree /root/homelab-ops-cr-2026-0060\nHEAD 1a0e4ce27b94bbfbf7b60829974a728b5eba3ad2\nbranch refs/heads/cr-2026-0060-backup-retention-remote-list-failclosed-hotfix\n\nworktree /root/homelab-ops-cr-2026-0064\nHEAD d3a11bde0e73ddfd7049693782d44100c0c1130e\nbranch refs/heads/cr-2026-0064-backup-retention-pinned-scheduled-scope-hotfix\n\nworktree /root/homelab-ops-cr-2026-0065\nHEAD 2c2f378aec7dc4ac23c633a31f738a850fface56\nbranch refs/heads/cr-2026-0065-backup-retention-dynamic-pinned-prune-mailru-cleanup\n\nworktree /root/homelab-ops-cr-2026-0066\nHEAD bcf018f263d9fc607eddd1796e5ef306870c6b7b\nbranch refs/heads/cr-2026-0066-legacy-backup-target-retirement\n\nworktree /root/homelab-ops-cr-2026-0067\nHEAD b5b032fd817f56dedb149ee04f20ac50d27d8527\nbranch refs/heads/cr-2026-0067-final-control-refresh\n\nworktree /root/homelab-ops-cr-2026-0068\nHEAD e404b160874d54a2219605685db6ea7a93c0adb1\nbranch refs/heads/cr-2026-0068-gitea-execution-journal\n\nworktree /root/homelab-ops-cr-2026-0069\nHEAD 59c0242eb4ed52569409bca74db4ab1294ee5a7e\nbranch refs/heads/cr-2026-0069-safety-foundation\n\nworktree /root/homelab-ops-cr-2026-0070\nHEAD dbf8266d5f2bc4c87024ea8f0eb5f0e8922c7519\nbranch refs/heads/cr-2026-0070-backup-target-lifecycle-registry\n\nworktree /root/homelab-ops-cr-2026-0071\nHEAD 07cb58a803022b5538e30b201380e215a5490c84\nbranch refs/heads/cr-2026-0071-backup-observability\n\nworktree /root/homelab-ops-cr-2026-0072\nHEAD 34d76b1cfd4071d94290edac50a31a8c0d4c11c8\nbranch refs/heads/cr-2026-0072-gfs-retention\n\nworktree /root/homelab-ops-cr-2026-0073\nHEAD 4ca24ca175ce9b8fa6a2664aebf12f48924897b9\nbranch refs/heads/cr-2026-0073-skladchik-reports-monitor-cookie-persistence-hotfix\n\nworktree /root/homelab-ops-cr-2026-0074\nHEAD 804a2391818f842d768edf434ffeaaf129e576f5\nbranch refs/heads/cr-2026-0074-backup-delete-evidence-hardening\n\nworktree /root/homelab-ops-cr-2026-0075\nHEAD 316ff2eb99ca61d02a6d2374bb0b0631fcc5a32b\nbranch refs/heads/cr-2026-0075-rotating-boot-drill-escrow-freshness\n\nworktree /root/homelab-ops-cr-2026-0076\nHEAD 611c08e8f6914a5912e528e019aa0fe8cd780c42\nbranch refs/heads/cr-2026-0076-snikket-vm150-rebuild\n\nworktree /root/homelab-ops-cr-2026-0077\nHEAD d72d8cf0862107eff9146e434f2ba5b711f67850\nbranch refs/heads/cr-2026-0077-router-moscow-remote-access\n\nworktree /root/homelab-ops-cr-2026-0078\nHEAD eb010cef52173ddba0f24f5486acab471efa05d6\nbranch refs/heads/cr-2026-0078-snikket-turn-durability\n\nworktree /root/homelab-ops-cr-2026-0079\nHEAD 6de47658e38b9cbd0b685ce94650d859581bed33\nbranch refs/heads/cr-2026-0079-mail-dr-observer-contract\n\nworktree /root/homelab-ops-cr-2026-0080\nHEAD aee8cabd123d77f23621f07241bf1600900c5de6\nbranch refs/heads/cr-2026-0080-snikket-edge-cutover\n\nworktree /root/homelab-ops-cr-2026-0081\nHEAD f418f45ae134724c4f23a2712718fb842db9a6ae\nbranch refs/heads/cr-2026-0081-snikket-hardening\n\nworktree /root/homelab-ops-cr-2026-0082\nHEAD 5f375858c3f91b28c870943b59bb2cccee6429da\nbranch refs/heads/cr-2026-0082-retire-vm180-cluster-admin\n\nworktree /root/homelab-ops-cr-2026-0083\nHEAD d8ae1b62d5d09a2a0249541967daf1ffd15eb932\ndetached\n\nworktree /root/homelab-ops-cr-2026-0095\nHEAD d5e2ef5639398a762283e47143c17b494f8ed579\nbranch refs/heads/cr-2026-0095-security-hardening-post-dns\n\nworktree /root/homelab-ops-cr-2026-0095-control\nHEAD 63b41a65ee300f0ec518d183bc84d3e2c7f2a5bd\nbranch refs/heads/cr-2026-0095-control-refresh-current\n\nworktree /root/homelab-ops-cr-2026-0096\nHEAD c38493fce630bf1601101c7e67c6c86f63df5bb9\nbranch refs/heads/cr-2026-0096-gram-crypto-analytics\n\nworktree /srv/homelab-ops-cr-2026-0018\nHEAD b25e80a67fb97ff6c0aa272b25675b9f7c82fb96\nbranch refs/heads/cr-2026-0018-backup-cloud-quorum-foundation\n\nworktree /srv/homelab-ops-worktrees/cr-2026-0015\nHEAD 60f53ea6fe6724196802a46de3de19c7889a51cc\nbranch refs/heads/cr-2026-0015-pve03-r8152-rx-ring-canary\n\nworktree /srv/homelab-ops-worktrees/cr-2026-0017-inventory-foundation\nHEAD 1c20ac860cbdd2ca7a05ef63d7011ac5937a7e17\nbranch refs/heads/cr-2026-0017-inventory-foundation"}
LIVE_GITEA_MAIN={"rc":0,"sha":"326622a4864fa9b5f43b69981733236bb42c4d7b","stderr_sha256":null}
LIVE_GITEA_MAIN_OBJECT_PRESENT_LOCALLY=false
LOCAL_MAIN_RECENT_LOG=["d5e2ef5639398a762283e47143c17b494f8ed579|2026-08-20T03:12:45+03:00|HEAD -> main, origin/main, cr-2026-0095-security-hardening-post-dns|CR-2026-0087: close cluster cleanup source state","3f650992cebd7619809d3abecffa479b7a42df50|2026-08-19T17:54:08Z||Merge pull request 'CR-2026-0086: homelab-safe-run' (#19) from cr-2026-0086-homelab-safe-run into main","1ead97c08aa530ee944fdee591a13e6a7b613e55|2026-08-19T20:41:45+03:00||CR-2026-0086: make SRC test detached-HEAD safe","927de3bbfb13fc01ee432c730c7be5c1fb923aa7|2026-08-19T20:26:04+03:00||CR-2026-0086: fix REG-031 validator self-trigger","55cbe43f3cbe850edbe504c0fb751bacee58e78e|2026-08-19T19:40:00+03:00||CR-2026-0086: add homelab-safe-run","96dc93186959d0b4eee0b828e66464de4d67a64b|2026-08-19T07:20:02+03:00||docs(CR-2026-0086): record safe-run implementation handoff","5dda7d81c2b66d57ea54c359a41b5f8cda31e61b|2026-08-19T04:16:29Z||Merge pull request 'CR-2026-0084: \u041f\u043e\u043b\u043d\u044b\u0439 \u043e\u043f\u0435\u0440\u0430\u0442\u0438\u0432\u043d\u044b\u0439 \u0441\u043f\u0440\u0430\u0432\u043e\u0447\u043d\u0438\u043a \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430' (#18) from cr-2026-0084-cluster-kb into main","772e862ee9b775a2d3eaea10b214688ab75b7434|2026-08-19T07:04:49+03:00||docs(CR-2026-0084): add cluster operational knowledge base","d8ae1b62d5d09a2a0249541967daf1ffd15eb932|2026-08-18T11:24:38Z||Merge pull request 'Cr 2026 0083 gitea edge publisher v3' (#17) from cr-2026-0083-gitea-edge-publisher-v3 into main","18742673ffd918015957897572ff0bb10a247d0c|2026-08-18T12:35:56+03:00||fix(CR-2026-0083): correct nginx map URI keys","866ebb4ab239f517f91a2a3134f7cfa55f0c9e6b|2026-08-18T11:45:02+03:00||fix(CR-2026-0083): enforce V3 root cache policy","064c6f23ab8a7038b0b0f64545ddf1965c5f860e|2026-08-18T08:42:28+03:00||fix(CR-2026-0083): harden EDGE Publisher V3 deployment","0adcbca6564d316beedf9003d23efb2762c9d8f4|2026-08-18T02:57:44+03:00||CR-2026-0083: ignore inactive nginx vhosts","59e55420f0043370558a624379059b8132cb4dc8|2026-08-18T01:52:46+03:00||CR-2026-0083: support live context branches schema","f0d2d006953079a07c9510e2489d01c4969826b4|2026-08-18T00:30:16+03:00||CR-2026-0083: add Gitea EDGE Publisher V3 source","8a2e58e3ace69b76d4508434290cd9600cba3270|2026-08-17T15:11:21Z||Merge pull request 'CR-2026-0081: fix VM150 EDGE egress probe' (#16) from cr-2026-0081-snikket-hardening into main","f418f45ae134724c4f23a2712718fb842db9a6ae|2026-08-17T18:10:53+03:00|origin/cr-2026-0081-snikket-hardening, cr-2026-0081-snikket-hardening|CR-2026-0081: fix VM150 EDGE egress probe parsing","17e094035bd395a8a632bafef2570be5004f91cc|2026-08-17T15:04:12Z||Merge pull request 'CR-2026-0081: fix Netcraze Snikket forward parsing' (#15) from cr-2026-0081-snikket-hardening into main","d971d99fabb587786f7af751809267c5b121b015|2026-08-17T18:03:50+03:00||CR-2026-0081: accept router rule comments safely","58313735ab3507975331922e9e016d0b20e473d0|2026-08-17T14:51:25Z||Merge pull request 'CR-2026-0081: retire obsolete Snikket home forwards' (#14) from cr-2026-0081-snikket-hardening into main","a417b2113c56c07c88b192636e7a9752fd338c5d|2026-08-17T17:51:00+03:00||CR-2026-0081: make home-forward tests CI checkout portable","0bdb4794d3910ec579bc539dcb75f49dd36f86cc|2026-08-17T13:49:14+03:00||CR-2026-0081: retire obsolete Snikket home forwards","6c2f65c3d98a20f6b5581cb019bbc6b371d343fb|2026-08-17T10:10:26Z||Merge pull request 'CR-2026-0081: fix legacy TURN no-consumer scan' (#13) from cr-2026-0081-snikket-hardening into main","6c3c5db756b1c5fa77dee0d14fcb23432662240a|2026-08-17T13:09:58+03:00||CR-2026-0081: harden optional Snikket scan roots"]
===== FAILED_UNITS_RCA =====
UNIT_PROPS={"props":{"ActiveState":"failed","ExecMainExitTimestamp":"Wed 2026-09-16 21:30:17 MSK","ExecMainStartTimestamp":"Wed 2026-09-16 21:30:11 MSK","ExecMainStatus":"65","FragmentPath":"/etc/systemd/system/homelab-app-cloud-quorum-queue.service","Id":"homelab-app-cloud-quorum-queue.service","LoadState":"loaded","OnFailure":"","Result":"exit-code","SubState":"failed","TriggeredBy":"","Triggers":"","UnitFileState":"static"},"rc":0,"unit":"homelab-app-cloud-quorum-queue.service"}
UNIT_FRAGMENT={"bytes":247,"exists":true,"path":"/etc/systemd/system/homelab-app-cloud-quorum-queue.service","selected_lines":["Description=Homelab canonical application cloud quorum backup queue","After=network-online.target","Wants=network-online.target","Type=oneshot","ExecStart=/usr/local/sbin/homelab-app-cloud-quorum-queue all"],"sha256":"70b78ba529c270ec61d26a123d656385a8b2e2e4206ccf623111fa6ed3854602","unit":"homelab-app-cloud-quorum-queue.service"}
LIVE_EXECUTABLE={"bytes":7855,"path":"/usr/local/sbin/homelab-app-cloud-quorum-queue","repo_matches":[],"sha256":"1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650","unit":"homelab-app-cloud-quorum-queue.service"}
UNIT_JOURNAL_FOCUS={"lines":["2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: FRESHNESS_STATUS=OK","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: VMID=130","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: DECISION=SEALED_PROVEN","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: GENERATION=vm130-pve03-20260916T105207Z","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: UPDATED_AT_UTC=2026-09-16T11:25:25Z","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: SEALED_EPOCH=1789557925","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm130/vm130-pve03-20260916T105207Z.json","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=25488 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: FRESHNESS_STATUS=OK","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: VMID=171","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: DECISION=SEALED_PROVEN","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: GENERATION=vm171-pve01-20260916T023018Z","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: UPDATED_AT_UTC=2026-09-16T02:35:03Z","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: SEALED_EPOCH=1789526103","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json","2026-09-16T21:30:16.481055+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=57311 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=57311","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: FRESHNESS_STATUS=OK","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: APP=filebrowser","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: DECISION=SEALED_PROVEN","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: GENERATION=app-filebrowser-pve01-20260916T113558Z","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: UPDATED_AT_UTC=2026-09-16T11:36:13Z","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: SEALED_EPOCH=1789558573","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=24842 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: FRESHNESS_STATUS=OK","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: APP=skladchik-reports-monitor","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: DECISION=SEALED_PROVEN","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: UPDATED_AT_UTC=2026-09-16T11:36:40Z","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: SEALED_EPOCH=1789558600","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=24815 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: FRESHNESS_STATUS=OK","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: APP=immich","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: DECISION=SEALED_PROVEN","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: GENERATION=app-immich-pve01-20260916T113650Z","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: UPDATED_AT_UTC=2026-09-16T11:37:38Z","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: SEALED_EPOCH=1789558658","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260916T113650Z.json","2026-09-16T21:30:16.482046+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=24757 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: FRESHNESS_STATUS=OK","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: APP=hbarhub","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: DECISION=SEALED_PROVEN","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: GENERATION=app-hbarhub-pve01-20260916T113814Z","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: UPDATED_AT_UTC=2026-09-16T11:39:24Z","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: SEALED_EPOCH=1789558764","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=24652 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: CLASS_DISPATCH KIND=APP DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1","2026-09-16T21:30:16.482569+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=57311","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: FRESHNESS_STATUS=OK","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: VMID=171","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: DECISION=SEALED_PROVEN","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: GENERATION=vm171-pve01-20260916T023018Z","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: UPDATED_AT_UTC=2026-09-16T02:35:03Z","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: SEALED_EPOCH=1789526103","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=57313 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: QUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: VM_QUEUE_HEAVY_LOCK_BUSY","2026-09-16T21:30:17.122944+03:00 pve01 homelab-app-cloud-quorum-queue[2923369]: VM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171","2026-09-16T21:30:17.123456+03:00 pve01 systemd[1]: homelab-app-cloud-quorum-queue.service: Main process exited, code=exited, status=65/DATAERR","2026-09-16T21:30:17.123522+03:00 pve01 systemd[1]: homelab-app-cloud-quorum-queue.service: Failed with result 'exit-code'.","2026-09-16T21:30:17.123670+03:00 pve01 systemd[1]: Failed to start homelab-app-cloud-quorum-queue.service - Homelab canonical application cloud quorum backup queue.","2026-09-16T21:30:17.123819+03:00 pve01 systemd[1]: homelab-app-cloud-quorum-queue.service: Consumed 4.607s CPU time, 147.2M memory peak."],"matched_count":220,"rc":0,"stderr_sha256":null,"unit":"homelab-app-cloud-quorum-queue.service"}
UNIT_PROPS={"props":{"ActiveState":"failed","ExecMainExitTimestamp":"Mon 2026-09-21 03:22:42 MSK","ExecMainStartTimestamp":"Mon 2026-09-21 03:22:42 MSK","ExecMainStatus":"66","FragmentPath":"/etc/systemd/system/homelab-backup-retention-gc.service","Id":"homelab-backup-retention-gc.service","LoadState":"loaded","OnFailure":"","Result":"exit-code","SubState":"failed","TriggeredBy":"homelab-backup-retention-gc.timer","Triggers":"","UnitFileState":"static"},"rc":0,"unit":"homelab-backup-retention-gc.service"}
UNIT_FRAGMENT={"bytes":327,"exists":true,"path":"/etc/systemd/system/homelab-backup-retention-gc.service","selected_lines":["Description=Homelab owner-aware cloud retention GC","After=network-online.target","Wants=network-online.target","ConditionPathExists=/usr/local/sbin/homelab-backup-retention-gc","Type=oneshot","ExecStart=/usr/local/sbin/homelab-backup-retention-gc scheduled"],"sha256":"0c49e10bdb225eeca0af5fca80aaf109addec2b18a6b892c0fe17c28b656ad7c","unit":"homelab-backup-retention-gc.service"}
LIVE_EXECUTABLE={"bytes":32975,"path":"/usr/local/sbin/homelab-backup-retention-gc","repo_matches":[],"sha256":"8adb6c10f5911f39c53775b89eac3cd1823f1ed34b4caccade5279c213e998a3","unit":"homelab-backup-retention-gc.service"}
UNIT_JOURNAL_FOCUS={"lines":["2026-09-08T03:20:54.252289+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-08T03:20:54.252408+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-09T03:34:04.598099+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-09T03:34:04.654643+03:00 pve01 homelab-backup-retention-gc[1472354]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-09T03:34:04.656930+03:00 pve01 homelab-backup-retention-gc[1472344]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-09T03:34:04.657170+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-09T03:34:04.657229+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-09T03:34:04.657333+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-10T03:21:52.217281+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-10T03:21:52.273549+03:00 pve01 homelab-backup-retention-gc[1742589]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-10T03:21:52.275762+03:00 pve01 homelab-backup-retention-gc[1742570]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-10T03:21:52.276112+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-10T03:21:52.276178+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-10T03:21:52.276329+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-11T03:33:55.361640+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-11T03:33:55.419880+03:00 pve01 homelab-backup-retention-gc[2092372]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-11T03:33:55.422875+03:00 pve01 homelab-backup-retention-gc[2092363]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-11T03:33:55.423342+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-11T03:33:55.423428+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-11T03:33:55.423616+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-12T03:32:47.966340+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-12T03:32:48.026064+03:00 pve01 homelab-backup-retention-gc[2424656]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-12T03:32:48.028487+03:00 pve01 homelab-backup-retention-gc[2424647]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-12T03:32:48.029073+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-12T03:32:48.029146+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-12T03:32:48.029283+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-13T03:32:16.312886+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-13T03:32:16.359887+03:00 pve01 homelab-backup-retention-gc[2753313]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-13T03:32:16.362194+03:00 pve01 homelab-backup-retention-gc[2753304]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-13T03:32:16.362540+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-13T03:32:16.362664+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-13T03:32:16.362910+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-14T03:26:56.409590+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-14T03:26:56.451369+03:00 pve01 homelab-backup-retention-gc[3055369]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-14T03:26:56.454102+03:00 pve01 homelab-backup-retention-gc[3055360]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-14T03:26:56.454647+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-14T03:26:56.454726+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-14T03:26:56.454922+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-15T03:34:58.227190+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-15T03:34:58.274629+03:00 pve01 homelab-backup-retention-gc[3431512]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-15T03:34:58.276982+03:00 pve01 homelab-backup-retention-gc[3431503]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-15T03:34:58.277238+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-15T03:34:58.277364+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-15T03:34:58.277590+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-16T03:25:32.216558+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-16T03:25:32.254486+03:00 pve01 homelab-backup-retention-gc[3715286]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-16T03:25:32.257142+03:00 pve01 homelab-backup-retention-gc[3715277]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-16T03:25:32.257534+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-16T03:25:32.257665+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-16T03:25:32.257943+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-17T03:21:32.217475+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-17T03:21:32.272451+03:00 pve01 homelab-backup-retention-gc[4021284]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-17T03:21:32.274644+03:00 pve01 homelab-backup-retention-gc[4021275]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-17T03:21:32.274963+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-17T03:21:32.275084+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-17T03:21:32.275307+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-18T03:22:16.321590+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-18T03:22:16.370214+03:00 pve01 homelab-backup-retention-gc[233892]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-18T03:22:16.372770+03:00 pve01 homelab-backup-retention-gc[233882]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-18T03:22:16.373053+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-18T03:22:16.373124+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-18T03:22:16.373267+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-19T03:23:32.216608+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-19T03:23:32.257484+03:00 pve01 homelab-backup-retention-gc[594585]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-19T03:23:32.259711+03:00 pve01 homelab-backup-retention-gc[594576]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-19T03:23:32.259964+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-19T03:23:32.260042+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-19T03:23:32.260190+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-20T03:34:58.227328+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-20T03:34:58.277578+03:00 pve01 homelab-backup-retention-gc[1022100]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-20T03:34:58.280508+03:00 pve01 homelab-backup-retention-gc[1022091]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-20T03:34:58.280925+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-20T03:34:58.281060+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-20T03:34:58.281313+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC.","2026-09-21T03:22:42.217498+03:00 pve01 systemd[1]: Starting homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC...","2026-09-21T03:22:42.266114+03:00 pve01 homelab-backup-retention-gc[1395029]: GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm200","2026-09-21T03:22:42.268544+03:00 pve01 homelab-backup-retention-gc[1395011]: GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID","2026-09-21T03:22:42.269136+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Main process exited, code=exited, status=66/NOINPUT","2026-09-21T03:22:42.269222+03:00 pve01 systemd[1]: homelab-backup-retention-gc.service: Failed with result 'exit-code'.","2026-09-21T03:22:42.269392+03:00 pve01 systemd[1]: Failed to start homelab-backup-retention-gc.service - Homelab owner-aware cloud retention GC."],"matched_count":84,"rc":0,"stderr_sha256":null,"unit":"homelab-backup-retention-gc.service"}
UNIT_PROPS={"props":{"ActiveState":"failed","ExecMainExitTimestamp":"Wed 2026-09-16 20:33:02 MSK","ExecMainStartTimestamp":"Wed 2026-09-16 20:32:57 MSK","ExecMainStatus":"65","FragmentPath":"/etc/systemd/system/homelab-vm-cloud-quorum-queue.service","Id":"homelab-vm-cloud-quorum-queue.service","LoadState":"loaded","OnFailure":"","Result":"exit-code","SubState":"failed","TriggeredBy":"","Triggers":"","UnitFileState":"static"},"rc":0,"unit":"homelab-vm-cloud-quorum-queue.service"}
UNIT_FRAGMENT={"bytes":240,"exists":true,"path":"/etc/systemd/system/homelab-vm-cloud-quorum-queue.service","selected_lines":["Description=Homelab canonical VM/CT cloud quorum backup queue","After=network-online.target","Wants=network-online.target","Type=oneshot","ExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all"],"sha256":"6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759","unit":"homelab-vm-cloud-quorum-queue.service"}
LIVE_EXECUTABLE={"bytes":8681,"path":"/usr/local/sbin/homelab-vm-cloud-quorum-queue","repo_matches":[],"sha256":"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612","unit":"homelab-vm-cloud-quorum-queue.service"}
UNIT_JOURNAL_FOCUS={"lines":["2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: FRESHNESS_STATUS=OK","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: VMID=130","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: DECISION=SEALED_PROVEN","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: GENERATION=vm130-pve03-20260916T105207Z","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: UPDATED_AT_UTC=2026-09-16T11:25:25Z","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: SEALED_EPOCH=1789557925","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm130/vm130-pve03-20260916T105207Z.json","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=22053 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: FRESHNESS_STATUS=OK","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: VMID=171","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: DECISION=SEALED_PROVEN","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: GENERATION=vm171-pve01-20260916T023018Z","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: UPDATED_AT_UTC=2026-09-16T02:35:03Z","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: SEALED_EPOCH=1789526103","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json","2026-09-16T20:33:02.036266+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: FRESHNESS_STATUS=OK","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: APP=filebrowser","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: DECISION=SEALED_PROVEN","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: GENERATION=app-filebrowser-pve01-20260916T113558Z","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: UPDATED_AT_UTC=2026-09-16T11:36:13Z","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: SEALED_EPOCH=1789558573","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: FRESHNESS_STATUS=OK","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: APP=skladchik-reports-monitor","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: DECISION=SEALED_PROVEN","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: UPDATED_AT_UTC=2026-09-16T11:36:40Z","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: SEALED_EPOCH=1789558600","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: FRESHNESS_STATUS=OK","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: APP=immich","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: DECISION=SEALED_PROVEN","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: GENERATION=app-immich-pve01-20260916T113650Z","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: UPDATED_AT_UTC=2026-09-16T11:37:38Z","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: SEALED_EPOCH=1789558658","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260916T113650Z.json","2026-09-16T20:33:02.037301+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: FRESHNESS_STATUS=OK","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: APP=hbarhub","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: DECISION=SEALED_PROVEN","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: GENERATION=app-hbarhub-pve01-20260916T113814Z","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: UPDATED_AT_UTC=2026-09-16T11:39:24Z","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: SEALED_EPOCH=1789558764","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: CLASS_DISPATCH KIND=APP DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1","2026-09-16T20:33:02.037865+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: FRESHNESS_STATUS=OK","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: VMID=171","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: DECISION=SEALED_PROVEN","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: GENERATION=vm171-pve01-20260916T023018Z","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: UPDATED_AT_UTC=2026-09-16T02:35:03Z","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: SEALED_EPOCH=1789526103","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: QUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: VM_QUEUE_HEAVY_LOCK_BUSY","2026-09-16T20:33:02.677727+03:00 pve01 homelab-vm-cloud-quorum-queue[2746943]: VM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171","2026-09-16T20:33:02.678469+03:00 pve01 systemd[1]: homelab-vm-cloud-quorum-queue.service: Main process exited, code=exited, status=65/DATAERR","2026-09-16T20:33:02.678534+03:00 pve01 systemd[1]: homelab-vm-cloud-quorum-queue.service: Failed with result 'exit-code'.","2026-09-16T20:33:02.678682+03:00 pve01 systemd[1]: Failed to start homelab-vm-cloud-quorum-queue.service - Homelab canonical VM/CT cloud quorum backup queue.","2026-09-16T20:33:02.678798+03:00 pve01 systemd[1]: homelab-vm-cloud-quorum-queue.service: Consumed 4.492s CPU time, 147.2M memory peak."],"matched_count":220,"rc":0,"stderr_sha256":null,"unit":"homelab-vm-cloud-quorum-queue.service"}
===== SCHEDULER_AND_REPLACEMENT_PATH =====
SCHEDULER_UNIT={"props":{"ActiveState":"inactive","FragmentPath":"/etc/systemd/system/homelab-app-cloud-quorum-queue.timer","Id":"homelab-app-cloud-quorum-queue.timer","LoadState":"loaded","OnFailure":"","Result":"success","SubState":"dead","TriggeredBy":"","Triggers":"homelab-app-cloud-quorum-queue.service","UnitFileState":"disabled"},"rc":0,"unit":"homelab-app-cloud-quorum-queue.timer"}
SCHEDULER_FRAGMENT={"bytes":200,"exists":true,"path":"/etc/systemd/system/homelab-app-cloud-quorum-queue.timer","selected_lines":["Description=Run homelab canonical application cloud quorum freshness queue every 2 hours"],"sha256":"af2eedce3df53029b591d31715a3f3e6688ddb997b4823f487e70c339ce88078","unit":"homelab-app-cloud-quorum-queue.timer"}
SCHEDULER_UNIT={"props":{"ActiveState":"inactive","FragmentPath":"/etc/systemd/system/homelab-vm-cloud-quorum-queue.timer","Id":"homelab-vm-cloud-quorum-queue.timer","LoadState":"loaded","OnFailure":"","Result":"success","SubState":"dead","TriggeredBy":"","Triggers":"homelab-vm-cloud-quorum-queue.service","UnitFileState":"disabled"},"rc":0,"unit":"homelab-vm-cloud-quorum-queue.timer"}
SCHEDULER_FRAGMENT={"bytes":244,"exists":true,"path":"/etc/systemd/system/homelab-vm-cloud-quorum-queue.timer","selected_lines":["Description=Run canonical Homelab VM/CT cloud quorum freshness queue every 4 hours"],"sha256":"9a78c9ea507116cbb9f5645fe697e34964c905b017cc98b17977cd5006aa5050","unit":"homelab-vm-cloud-quorum-queue.timer"}
SCHEDULER_UNIT={"props":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-backup-retention-gc.timer","Id":"homelab-backup-retention-gc.timer","LoadState":"loaded","OnFailure":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-backup-retention-gc.service","UnitFileState":"enabled"},"rc":0,"unit":"homelab-backup-retention-gc.timer"}
SCHEDULER_FRAGMENT={"bytes":237,"exists":true,"path":"/etc/systemd/system/homelab-backup-retention-gc.timer","selected_lines":["Description=Run homelab owner-aware cloud retention GC daily"],"sha256":"a0e763670745a5ef7c04b54da23114a013f47075cf36cceef0218bae9b82ea2f","unit":"homelab-backup-retention-gc.timer"}
SCHEDULER_UNIT={"props":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-backup-v2.timer","Id":"homelab-backup-v2.timer","LoadState":"loaded","OnFailure":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-backup-v2.service","UnitFileState":"enabled"},"rc":0,"unit":"homelab-backup-v2.timer"}
SCHEDULER_FRAGMENT={"bytes":192,"exists":true,"path":"/etc/systemd/system/homelab-backup-v2.timer","selected_lines":["Description=Homelab Backup V2 scheduler timer"],"sha256":"5469057602452b303a01ccb8057085c1b0e447973e5ed198cebf7dd5daf58e9d","unit":"homelab-backup-v2.timer"}
SCHEDULER_UNIT={"props":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-backup-audit.timer","Id":"homelab-backup-audit.timer","LoadState":"loaded","OnFailure":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-backup-audit.service","UnitFileState":"enabled"},"rc":0,"unit":"homelab-backup-audit.timer"}
SCHEDULER_FRAGMENT={"bytes":221,"exists":true,"path":"/etc/systemd/system/homelab-backup-audit.timer","selected_lines":["Description=Run homelab unified backup and DR audit"],"sha256":"987695f83ee993bd4d716b8365308211110ac4c684f0bd9b17dfdb1e523e466a","unit":"homelab-backup-audit.timer"}
SCHEDULER_UNIT={"props":{"ActiveState":"inactive","ExecMainExitTimestamp":"Mon 2026-09-21 03:51:17 MSK","ExecMainStartTimestamp":"Mon 2026-09-21 03:47:58 MSK","ExecMainStatus":"0","FragmentPath":"/etc/systemd/system/homelab-backup-v2.service","Id":"homelab-backup-v2.service","LoadState":"loaded","OnFailure":"","Result":"success","SubState":"dead","TriggeredBy":"homelab-backup-v2.timer","Triggers":"","UnitFileState":"static"},"rc":0,"unit":"homelab-backup-v2.service"}
SCHEDULER_FRAGMENT={"bytes":309,"exists":true,"path":"/etc/systemd/system/homelab-backup-v2.service","selected_lines":["Description=Homelab Backup V2 manifest-driven scheduler","After=network-online.target","Wants=network-online.target","Type=oneshot","User=root","Group=root","ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled","TimeoutStartSec=8h"],"sha256":"1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565","unit":"homelab-backup-v2.service"}
SCHEDULER_UNIT={"props":{"ActiveState":"inactive","ExecMainExitTimestamp":"","ExecMainStartTimestamp":"","ExecMainStatus":"0","FragmentPath":"","Id":"homelab-backup-dispatcher.service","LoadState":"not-found","OnFailure":"","Result":"success","SubState":"dead","TriggeredBy":"","Triggers":"","UnitFileState":""},"rc":0,"unit":"homelab-backup-dispatcher.service"}
SCHEDULER_UNIT={"props":{"ActiveState":"inactive","ExecMainExitTimestamp":"Mon 2026-09-21 04:06:18 MSK","ExecMainStartTimestamp":"Mon 2026-09-21 04:06:11 MSK","ExecMainStatus":"0","FragmentPath":"/etc/systemd/system/homelab-backup-audit.service","Id":"homelab-backup-audit.service","LoadState":"loaded","OnFailure":"","Result":"success","SubState":"dead","TriggeredBy":"homelab-backup-audit.timer","Triggers":"","UnitFileState":"static"},"rc":0,"unit":"homelab-backup-audit.service"}
SCHEDULER_FRAGMENT={"bytes":285,"exists":true,"path":"/etc/systemd/system/homelab-backup-audit.service","selected_lines":["Description=Homelab unified backup and DR audit","After=network-online.target","Wants=network-online.target","Type=oneshot","ExecStart=/usr/local/sbin/homelab-backup-audit scheduled","User=root","Group=root"],"sha256":"7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa","unit":"homelab-backup-audit.service"}
SCHEDULER_JOURNAL_FOCUS={"lines":["2026-09-20T15:19:59.382893+03:00 pve01 homelab-backup-v2[3306237]: SCHEDULE_RESULT=FAIL","2026-09-20T15:19:59.382893+03:00 pve01 homelab-backup-v2[3306237]: SCHEDULE_REASON=CT112_PIPELINE_FAILED","2026-09-20T15:19:59.392482+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=2/INVALIDARGUMENT","2026-09-20T15:19:59.392549+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T15:19:59.392691+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T15:56:31.790799+03:00 pve01 homelab-backup-v2[3417390]: SCHEDULE_RESULT=FAIL","2026-09-20T15:56:31.790799+03:00 pve01 homelab-backup-v2[3417390]: SCHEDULE_REASON=PIPE_FAILED","2026-09-20T15:56:31.797290+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=2/INVALIDARGUMENT","2026-09-20T15:56:31.797372+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T15:56:31.797541+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T16:31:22.568658+03:00 pve01 homelab-backup-v2[3535131]: SCHEDULE_RESULT=FAIL","2026-09-20T16:31:22.568658+03:00 pve01 homelab-backup-v2[3535131]: SCHEDULE_REASON=CT112_PIPELINE_FAILED","2026-09-20T16:31:22.578275+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=2/INVALIDARGUMENT","2026-09-20T16:31:22.578409+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T16:31:22.578669+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T17:41:42.675536+03:00 pve01 homelab-backup-v2[3751454]: SCHEDULE_RESULT=FAIL","2026-09-20T17:41:42.675536+03:00 pve01 homelab-backup-v2[3751454]: SCHEDULE_REASON=CT112_PIPELINE_FAILED","2026-09-20T17:41:42.681725+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=2/INVALIDARGUMENT","2026-09-20T17:41:42.681881+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T17:41:42.682116+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T20:03:09.246625+03:00 pve01 homelab-backup-v2[5417]: SCHEDULE_RESULT=FAIL","2026-09-20T20:03:09.246625+03:00 pve01 homelab-backup-v2[5417]: SCHEDULE_REASON=CT112_PIPELINE_FAILED","2026-09-20T20:03:09.252597+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=2/INVALIDARGUMENT","2026-09-20T20:03:09.252660+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T20:03:09.252827+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T21:12:11.772892+03:00 pve01 vzdump[223484]: INFO: starting new backup job: vzdump 110 --ionice 7 --stdexcludes 0 --node pve01 --mode snapshot --zstd 1 --compress zstd --tmpdir /var/tmp/homelab-vzdump-v2-v2-ct110-20260920T181158Z-25703b1ef77f --lockwait 0 --prune-backups 'keep-all=1' --bwlimit 32768 --remove 0","2026-09-20T22:29:31.800106+03:00 pve01 homelab-backup-v2[459026]: SCHEDULE_RESULT=FAIL","2026-09-20T22:29:31.800106+03:00 pve01 homelab-backup-v2[459026]: SCHEDULE_REASON=PIPE_FAILED","2026-09-20T22:29:31.805768+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=2/INVALIDARGUMENT","2026-09-20T22:29:31.805836+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T22:29:31.805983+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T23:09:29.556447+03:00 pve01 homelab-backup-v2[593177]: LXC_EXECUTION_DIAG={\"executor\":\"remote-bounded-ssh-cwd-root\",\"node\":\"pve02\",\"producer_rc\":0,\"stderr_bytes_observed\":1035,\"stderr_bytes_retained\":1035,\"stderr_stream_sha256\":\"bce35788cf8d44be1cd8cf226e02ad4d65baf010cb8e8dfc4eb7a5abb32ced63\",\"stderr_truncated\":false}","2026-09-21T00:12:09.285479+03:00 pve01 homelab-backup-v2[782723]: SCHEDULE_RESULT=FAIL","2026-09-21T00:12:09.285479+03:00 pve01 homelab-backup-v2[782723]: SCHEDULE_REASON=CT112_PIPELINE_FAILED","2026-09-21T00:12:09.292330+03:00 pve01 systemd[1]: homelab-backup-v2.service: Main process exited, code=exited, status=2/INVALIDARGUMENT","2026-09-21T00:12:09.292481+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-21T00:12:09.292801+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-21T00:50:11.913977+03:00 pve01 vzdump[915941]: INFO: starting new backup job: vzdump 112 --prune-backups 'keep-all=1' --bwlimit 32768 --node pve01 --tmpdir /var/tmp/homelab-vzdump-v2-v2-ct112-20260920T214958Z-30e3291e1543 --stdexcludes 0 --zstd 1 --remove 0 --mode snapshot --compress zstd --ionice 7 --lockwait 0","2026-09-21T01:26:45.553982+03:00 pve01 homelab-backup-v2[1018631]: LXC_EXECUTION_DIAG={\"executor\":\"remote-bounded-ssh-cwd-root\",\"node\":\"pve02\",\"producer_rc\":0,\"stderr_bytes_observed\":1038,\"stderr_bytes_retained\":1038,\"stderr_stream_sha256\":\"5590f20ce6d625aae5e745852a46ffd5331ef67462164b084208650f60f43eeb\",\"stderr_truncated\":false}"],"matched_count":39,"rc":0,"stderr_sha256":null,"unit":"homelab-backup-v2.service"}
SCHEDULER_JOURNAL_FOCUS={"lines":[],"matched_count":0,"rc":0,"stderr_sha256":null,"unit":"homelab-backup-dispatcher.service"}
SCHEDULER_JOURNAL_FOCUS={"lines":["2026-09-21T04:06:11.171973+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=CONTROL_BACKUP|status=OK|path=/usr/local/sbin/homelab-control-backup|sha256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57","2026-09-21T04:06:11.173643+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=CONTROL_BUILDER|status=OK|path=/usr/local/libexec/homelab-control-bundle-builder|sha256=2e1c3935b5016c09aad21f34a36f7e1f53188bbc34a3f033e59fe964571ec3f0","2026-09-21T04:06:11.175667+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=CONTROL_TIMER|status=OK|path=/etc/systemd/system/homelab-control-backup.timer|sha256=7c4136da20b9cd9162293380f5794d93e2c6099ce7d87ec00e0e7e19b6cd44a3","2026-09-21T04:06:11.177976+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=CONTROL_SERVICE|status=OK|path=/etc/systemd/system/homelab-control-backup.service|sha256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250","2026-09-21T04:06:11.180300+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=CONTROL_RUNTIME_MANIFEST|status=OK|path=/etc/homelab-backup/control-runtime-paths.txt|sha256=6777f95159db0fa213017f286f2cee6cc51e9963a01a0578cee8586e8731b2be","2026-09-21T04:06:11.182286+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=CONTROL_SYSTEMD_MANIFEST|status=OK|path=/etc/homelab-backup/control-systemd-paths.txt|sha256=7f990d6257b43f97c3c8407945ca9dae6113dc9064c6cf1f25e7fcdaf99d5163","2026-09-21T04:06:11.184151+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=VM_TIMER|status=OK|path=/etc/systemd/system/homelab-vm-cloud-quorum-queue.timer|sha256=9a78c9ea507116cbb9f5645fe697e34964c905b017cc98b17977cd5006aa5050","2026-09-21T04:06:11.186295+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=APP_TIMER|status=OK|path=/etc/systemd/system/homelab-app-cloud-quorum-queue.timer|sha256=af2eedce3df53029b591d31715a3f3e6688ddb997b4823f487e70c339ce88078","2026-09-21T04:06:11.188350+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=VM_SERVICE|status=OK|path=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service|sha256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759","2026-09-21T04:06:11.190330+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=APP_SERVICE|status=OK|path=/etc/systemd/system/homelab-app-cloud-quorum-queue.service|sha256=70b78ba529c270ec61d26a123d656385a8b2e2e4206ccf623111fa6ed3854602","2026-09-21T04:06:11.192348+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=CR0031_SEAL|status=OK|path=/var/lib/homelab-admin/seals/backup-vm-restore-evidence-hardening-v1.json|sha256=5f1274624374adbbb495b1701fe1eedaab028fad0f97a2409f0b79df9a791f9a","2026-09-21T04:06:11.194346+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=OBS_HELPER|status=OK|path=/usr/local/libexec/homelab-backup-observability|sha256=16dd24c8c84fe9bb3970a6cb7cd0e58dfa5243e9056f9f40f7785b8e61443581","2026-09-21T04:06:11.196372+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=local|label=OBS_CONFIG|status=OK|path=/etc/homelab-backup/observability.json|sha256=5b29bd9b76077a38f55ad2944efccdf8312c2d7dbfc8724cf54cfa81f8a9df3b","2026-09-21T04:06:11.337967+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=pve02|label=VM_PROBE|status=OK|path=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|sha256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96","2026-09-21T04:06:11.445701+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=pve02|label=VM_WORKER|status=OK|path=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|sha256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1","2026-09-21T04:06:11.561015+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=pve02|label=TRANSPORT|status=OK|path=/usr/local/libexec/homelab-cloud-quorum-payload|sha256=df9e19485e5d87112bf58bf77bdddf3b9b896418dd85ff945a31cd5b6817f352","2026-09-21T04:06:11.684450+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=pve03|label=VM_PROBE|status=OK|path=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|sha256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96","2026-09-21T04:06:11.800422+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=pve03|label=VM_WORKER|status=OK|path=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|sha256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1","2026-09-21T04:06:11.912451+03:00 pve01 homelab-backup-audit[1531640]: SOURCE|scope=pve03|label=TRANSPORT|status=OK|path=/usr/local/libexec/homelab-cloud-quorum-payload|sha256=df9e19485e5d87112bf58bf77bdddf3b9b896418dd85ff945a31cd5b6817f352","2026-09-21T04:06:12.032820+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=110|owner=pve01|state=STALE|generation=vm110-pve01-20260904T053821Z|age_seconds=1452397|sla_seconds=72000|archive_size_bytes=478357303|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm110/vm110-pve01-20260904T053821Z.json","2026-09-21T04:06:12.583550+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=111|owner=pve02|state=STALE|generation=vm111-pve02-20260903T185424Z|age_seconds=1490976|sla_seconds=72000|archive_size_bytes=369892675|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm111/vm111-pve02-20260903T185424Z.json","2026-09-21T04:06:12.699563+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=112|owner=pve01|state=STALE|generation=vm112-pve01-20260903T185707Z|age_seconds=1490890|sla_seconds=72000|archive_size_bytes=223672749|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm112/vm112-pve01-20260903T185707Z.json","2026-09-21T04:06:13.262137+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=113|owner=pve02|state=STALE|generation=vm113-pve02-20260903T185827Z|age_seconds=1490810|sla_seconds=72000|archive_size_bytes=221529138|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm113/vm113-pve02-20260903T185827Z.json","2026-09-21T04:06:13.888816+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=130|owner=pve03|state=STALE|generation=vm130-pve03-20260916T105207Z|age_seconds=394846|sla_seconds=72000|archive_size_bytes=10723620588|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json","2026-09-21T04:06:14.020966+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=150|owner=pve01|state=STALE|generation=vm150-pve01-20260904T053517Z|age_seconds=1452510|sla_seconds=72000|archive_size_bytes=887338816|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm150/vm150-pve01-20260904T053517Z.json","2026-09-21T04:06:14.631216+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=160|owner=pve02|state=STALE|generation=vm160-pve02-20260916T023620Z|age_seconds=425983|sla_seconds=72000|archive_size_bytes=3673704465|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json","2026-09-21T04:06:14.799619+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=170|owner=pve01|state=STALE|generation=vm170-pve01-20260916T103019Z|age_seconds=397136|sla_seconds=72000|archive_size_bytes=7119997483|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json","2026-09-21T04:06:14.967163+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=171|owner=pve01|state=STALE|generation=vm171-pve01-20260916T023018Z|age_seconds=426668|sla_seconds=72000|archive_size_bytes=2041990920|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json","2026-09-21T04:06:15.535243+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=190|owner=pve03|state=STALE|generation=vm190-pve03-20260903T175951Z|age_seconds=1494095|sla_seconds=72000|archive_size_bytes=1557482946|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm190/vm190-pve03-20260903T175951Z.json","2026-09-21T04:06:16.105383+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=VM|id=9130|owner=pve02|state=STALE|generation=vm9130-pve02-20260903T180554Z|age_seconds=1491584|sla_seconds=72000|archive_size_bytes=10998362942|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm9130/vm9130-pve02-20260903T180554Z.json","2026-09-21T04:06:16.374799+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=APP|id=filebrowser|owner=pve01|state=STALE|generation=app-filebrowser-pve01-20260916T113558Z|age_seconds=394198|sla_seconds=72000|archive_size_bytes=3182|restore_evidence=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json","2026-09-21T04:06:16.630071+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=APP|id=skladchik-reports-monitor|owner=pve01|state=STALE|generation=app-skladchik-reports-monitor-pve01-20260916T113630Z|age_seconds=394171|sla_seconds=72000|archive_size_bytes=174001|restore_evidence=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json","2026-09-21T04:06:16.884463+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=APP|id=immich|owner=pve01|state=STALE|generation=app-immich-pve01-20260916T113650Z|age_seconds=394113|sla_seconds=72000|archive_size_bytes=270193610|restore_evidence=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json","2026-09-21T04:06:17.133070+03:00 pve01 homelab-backup-audit[1531640]: TARGET|kind=APP|id=hbarhub|owner=pve01|state=STALE|generation=app-hbarhub-pve01-20260916T113814Z|age_seconds=394007|sla_seconds=72000|archive_size_bytes=77065509|restore_evidence=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json","2026-09-21T04:06:17.156819+03:00 pve01 homelab-backup-audit[1531640]: CONTROL|state=STALE_PROVEN|CONTROL_FRESHNESS=STALE_PROVEN|generation=control-bootstrap-v2-pve01-20260916T120057Z|age_seconds=392702|sla_seconds=86400|archive_size_bytes=9002631|restore_evidence_checked=true|source_coverage=RESTORE_EVIDENCE|retention=ACTIVE_OWNER_AWARE","2026-09-21T04:06:17.472190+03:00 pve01 homelab-backup-audit[1531640]: CLOUD|remote=pve01-mail-01-crypt|status=OK|total_bytes=1099511627776|used_bytes=806303109447|free_bytes=293208518329","2026-09-21T04:06:17.818436+03:00 pve01 homelab-backup-audit[1531640]: CLOUD|remote=pve01-mail-02-crypt|status=OK|total_bytes=1099511627776|used_bytes=806303173306|free_bytes=293208454470","2026-09-21T04:06:17.896422+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=110|logical=vm110|current_bytes=478357303|baseline_median_bytes=478357303|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896422+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=111|logical=vm111|current_bytes=369892675|baseline_median_bytes=369892675|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896530+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=112|logical=vm112|current_bytes=223672749|baseline_median_bytes=223672749|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896530+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=113|logical=vm113|current_bytes=221529138|baseline_median_bytes=221529138|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896530+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=130|logical=vm130|current_bytes=10723620588|baseline_median_bytes=10723620588|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896586+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=150|logical=vm150|current_bytes=887338816|baseline_median_bytes=887338816|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896586+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=160|logical=vm160|current_bytes=3673704465|baseline_median_bytes=3673704465|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896617+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=170|logical=vm170|current_bytes=7119997483|baseline_median_bytes=7119997483|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896644+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=171|logical=vm171|current_bytes=2041990920|baseline_median_bytes=2041990920|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896673+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=190|logical=vm190|current_bytes=1557482946|baseline_median_bytes=1557482946|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896705+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=VM|id=9130|logical=vm9130|current_bytes=10998362942|baseline_median_bytes=10998362942|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896743+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=APP|id=filebrowser|logical=app-filebrowser|current_bytes=3182|baseline_median_bytes=3182|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896804+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=APP|id=skladchik-reports-monitor|logical=app-skladchik-reports-monitor|current_bytes=174001|baseline_median_bytes=174001|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896838+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=APP|id=immich|logical=app-immich|current_bytes=270193610|baseline_median_bytes=270193610|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896871+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=APP|id=hbarhub|logical=app-hbarhub|current_bytes=77065509|baseline_median_bytes=77065509|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.896912+03:00 pve01 homelab-backup-audit[1531640]: SIZE_OBSERVABILITY|kind=CONTROL|id=control-bootstrap-v2|logical=control-bootstrap-v2|current_bytes=9002631|baseline_median_bytes=9002631|ratio_milli=1000|history_samples=4|status=OK","2026-09-21T04:06:17.899335+03:00 pve01 homelab-backup-audit[1531640]: CAPACITY_FORECAST|remote=pve01-mail-01-crypt|used_percent=73|current_status=OK|forecast_status=CRIT|span_seconds=1207858|net_growth_bytes_per_day=28795460932|days_to_80_percent=3|days_to_90_percent=7","2026-09-21T04:06:17.899335+03:00 pve01 homelab-backup-audit[1531640]: CAPACITY_FORECAST|remote=pve01-mail-02-crypt|used_percent=73|current_status=OK|forecast_status=CRIT|span_seconds=1207858|net_growth_bytes_per_day=28824952653|days_to_80_percent=3|days_to_90_percent=7","2026-09-21T04:06:17.947251+03:00 pve01 homelab-backup-audit[1531640]: OBSERVABILITY|status=ATTENTION|size_anomalies=0|size_learning=0|capacity_status=CRIT|worst_used_percent=73|worst_days_to_80=3|history_snapshots=336","2026-09-21T04:06:17.992408+03:00 pve01 homelab-backup-audit[1531640]: SYSTEMD_EXPECTATION|unit=homelab-vm-cloud-quorum-queue.timer|expected=ENABLED_ACTIVE|status=BAD|load_state=loaded|unit_file_state=disabled|active_state=inactive","2026-09-21T04:06:18.031594+03:00 pve01 homelab-backup-audit[1531640]: SYSTEMD_EXPECTATION|unit=homelab-app-cloud-quorum-queue.timer|expected=ENABLED_ACTIVE|status=BAD|load_state=loaded|unit_file_state=disabled|active_state=inactive","2026-09-21T04:06:18.063445+03:00 pve01 homelab-backup-audit[1531640]: SYSTEMD_EXPECTATION|unit=homelab-backup-audit.timer|expected=ENABLED_ACTIVE|status=OK|load_state=loaded|unit_file_state=enabled|active_state=active","2026-09-21T04:06:18.095846+03:00 pve01 homelab-backup-audit[1531640]: SYSTEMD_EXPECTATION|unit=homelab-backup-retention-gc.timer|expected=ENABLED_ACTIVE|status=OK|load_state=loaded|unit_file_state=enabled|active_state=active","2026-09-21T04:06:18.111399+03:00 pve01 homelab-backup-audit[1531640]: SYSTEMD_EXPECTATION|unit=homelab-backup-healthcheck.timer|expected=ABSENT|status=EXPECTED_ABSENT|load_state=not-found|unit_file_state=|active_state=inactive","2026-09-21T04:06:18.128489+03:00 pve01 homelab-backup-audit[1531640]: SYSTEMD_EXPECTATION|unit=homelab-backup-sla-health.timer|expected=ABSENT|status=EXPECTED_ABSENT|load_state=not-found|unit_file_state=|active_state=inactive","2026-09-21T04:06:18.146268+03:00 pve01 homelab-backup-audit[1531640]: SYSTEMD_EXPECTATION|unit=homelab-legacy-monitoring-backups.timer|expected=ABSENT|status=EXPECTED_ABSENT|load_state=not-found|unit_file_state=|active_state=inactive","2026-09-21T04:06:18.163978+03:00 pve01 homelab-backup-audit[1531640]: SYSTEMD_EXPECTATION|unit=homelab-verified-backup-health.timer|expected=ABSENT|status=EXPECTED_ABSENT|load_state=not-found|unit_file_state=|active_state=inactive","2026-09-21T04:06:18.180834+03:00 pve01 homelab-backup-audit[1531640]: SYSTEMD_EXPECTATION|unit=homelab-vm-backup-health.timer|expected=ABSENT|status=EXPECTED_ABSENT|load_state=not-found|unit_file_state=|active_state=inactive","2026-09-21T04:06:18.217167+03:00 pve01 homelab-backup-audit[1531640]: SYSTEMD_EXPECTATION|unit=homelab-control-backup.timer|expected=ENABLED_ACTIVE|status=BAD|load_state=loaded|unit_file_state=disabled|active_state=inactive","2026-09-21T04:06:18.218967+03:00 pve01 homelab-backup-audit[1531640]: LOCK|path=/run/homelab-backup-heavy.lock|state=FREE","2026-09-21T04:06:18.220397+03:00 pve01 homelab-backup-audit[1531640]: LOCK|path=/run/lock/homelab-vm-cloud-quorum-queue.lock|state=FREE","2026-09-21T04:06:18.221932+03:00 pve01 homelab-backup-audit[1531640]: LOCK|path=/run/lock/homelab-app-cloud-quorum-queue.lock|state=FREE","2026-09-21T04:06:18.223232+03:00 pve01 homelab-backup-audit[1531640]: LOCK|path=/run/lock/homelab-backup-dispatcher.lock|state=FREE","2026-09-21T04:06:18.242565+03:00 pve01 homelab-backup-audit[1531640]: STAGING|node=pve01|status=OK|used_percent=5|avail_bytes=285683359744|warn_percent=85","2026-09-21T04:06:18.575507+03:00 pve01 homelab-backup-audit[1531640]: STAGING|node=pve02|status=OK|used_percent=2|avail_bytes=146738655232|warn_percent=85","2026-09-21T04:06:18.931336+03:00 pve01 homelab-backup-audit[1531640]: STAGING|node=pve03|status=OK|used_percent=1|avail_bytes=199483838464|warn_percent=85","2026-09-21T04:06:18.935085+03:00 pve01 homelab-backup-audit[1531640]: ESCROW|status=OK|age_seconds=3555916|physical_copies=2_USER_ATTESTED|rotation_policy=ON_STATIC_SECRET_CHANGE|static_secret_drift_detection=NOT_MACHINE_PROVEN|oauth_token_churn_static_drift=NO|clean_machine_sufficiency=NOT_PROVEN","2026-09-21T04:06:18.941422+03:00 pve01 homelab-backup-audit[1531640]: RETENTION|status=ACTIVE_OWNER_AWARE|historical_gc_proven=NO|owner_aware_gc_proven=YES|vm_keep=2|app_keep=7|control_keep=4","2026-09-21T04:06:18.960683+03:00 pve01 homelab-backup-audit[1531640]: DR|DR_PROOF_STATUS=PROVEN|proof_status=PROVEN|full_site_dr=PROVEN|qemu_boot=PROVEN|ct_boot=PROVEN|clean_machine=PROVEN|provider_worm=NOT_PROVEN_SKIPPED|legacy_site_seal_valid=true|legacy_full_site_claim=true|physical_escrow_media_read_proven=false","2026-09-21T04:06:18.960683+03:00 pve01 homelab-backup-audit[1531640]: PAYLOAD_LEAKS|latest_sealed_detected=0|historical_gc_scope=OPEN","2026-09-21T04:06:18.960846+03:00 pve01 homelab-backup-audit[1531640]: RISK|severity=CRITICAL|reasons=SOURCE_DRIFT,INFRASTRUCTURE_INVARIANT,RPO_BREACH,CAPACITY_CRITICAL|backup_status=RED|dr_proof=PROVEN","2026-09-21T04:06:18.960880+03:00 pve01 homelab-backup-audit[1531640]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|RISK_SEVERITY=CRITICAL|RISK_REASONS=SOURCE_DRIFT,INFRASTRUCTURE_INVARIANT,RPO_BREACH,CAPACITY_CRITICAL|DR_PROOF_STATUS=PROVEN|CONTROL_FRESHNESS=STALE_PROVEN|LIFECYCLE_STATUS=CLOSED_PROVEN|VM_COUNT=11|APP_COUNT=4|FRESH=0|STALE=15|RESUME=0|UNVERIFIED=0|CONTROL_STATE=STALE_PROVEN|CLOUD_OK=2|SCHEDULER_OK=7|SOURCE_OK=28|SOURCE_DRIFT=1|STAGING_BAD=0|STAGING_WARN=0|PAYLOAD_LEAKS=0|RETENTION_STATUS=ACTIVE_OWNER_AWARE|OBSERVABILITY=ATTENTION|SIZE_ANOMALIES=0|CAPACITY=CRIT|GLOBAL_DR_PROVEN=YES","2026-09-21T04:06:18.968987+03:00 pve01 homelab-backup-audit[1531640]: HEALTH_WRITE|status=OK|path=/var/lib/homelab-health/backup-unified-audit.json|backup_status=RED"],"matched_count":204,"rc":0,"stderr_sha256":null,"unit":"homelab-backup-audit.service"}
===== CURRENT_HEALTH_AUTHORITY_CANDIDATES =====
HEALTH_CANDIDATE={"age_sec":64,"bytes":1540,"path":"/var/lib/homelab-health/backup-unified-audit.json","selected":{},"sha256":"a32979ff27d5b003accdc21c50489c45f0d336acbaa3b1611616bd10b72b89e3"}
HEALTH_CANDIDATE={"age_sec":3618,"bytes":567539,"path":"/var/lib/homelab-health/backup-observability-history.jsonl","selected":{},"sha256":"a182280d3fd9384a2807b58e5516744879a2e87d2aa43fcc8c53725684adb9ac"}
HEALTH_CANDIDATE={"age_sec":22545,"bytes":618,"path":"/var/lib/homelab-health/control-backup.json","selected":{},"sha256":"003b40157a1174cfd7ca6aa9aead224d9a39fae93dd7d215199b67fc3b3d1a1a"}
HEALTH_CANDIDATE={"age_sec":73621,"bytes":181,"path":"/var/lib/homelab-health/backup-boot-drill.json","selected":{},"sha256":"5cf6ed79fdec1f2901a868b05a57f17d1388189dcc2fb32e191b0ee4fd1f5d97"}
HEALTH_CANDIDATE={"age_sec":75366,"bytes":232,"path":"/var/lib/homelab-health/drift-check.txt","selected":{"STATUS":"WARN TS=2026-09-20T04:11:17Z TYPE=drift-check NOTE=drift_detected DIFF=/var/lib/homelab-baseline/drift-latest.diff BASE=/var/lib/homelab-baseline/latest-normalized.txt CURRENT=/var/lib/homelab-baseline/current-normalized.txt"},"sha256":"4bee6be6d9089d0866921628db83fc3371f2abe7ed0745e2b0da5d69038b4e04"}
HEALTH_CANDIDATE={"age_sec":1839555,"bytes":811,"path":"/var/lib/homelab-health/backup-framework.txt","selected":{"FRESHNESS":"STALE","STATUS":"FAIL"},"sha256":"192caeb6ecfd0d386aa32fcf838229854ea73f3abb7d46343cd8f1b4acf7b2ab"}
HEALTH_CANDIDATE={"age_sec":1839555,"bytes":765,"path":"/var/lib/homelab-health/backup-sla.txt","selected":{"FRESHNESS":"STALE","STATUS":"FAIL"},"sha256":"4762bb7cfb98ab1311adbd1e80287f7e18f79983656a888a056ed730ecdf0372"}
HEALTH_CANDIDATE={"age_sec":1839555,"bytes":769,"path":"/var/lib/homelab-health/vm-backup-policy.txt","selected":{"FRESHNESS":"STALE","STATUS":"FAIL"},"sha256":"947eef16fbfd3815d6f823c3bc990415a33c5611342bc0d32fe959cb9e95b799"}
===== LIVE_DRIFT_TARGET =====
DRIFT_LIVE={"bytes":3140,"path":"/usr/local/sbin/homelab-vm160-remote-backup","repo_matches":[{"live_equal":false,"repo_path":"tasks/cluster-cloud-quorum-vm160-canary-v1/assets/homelab-vm160-remote-backup","sha256":"af9daa1dde5382549e315097802ad11e7ecc3279ae918b0c629b749ff8ed6063"}],"sha256":"ffcb6b946d9c621cb9227642008c5f6bf7fb48041b3604c926c7ed2f2be47cc4"}
DRIFT_CANONICAL_HISTORY=["16e84298035f5e4ec7e2b7d39967ad5e40ea3d4b|2026-07-18T07:28:24+03:00|origin/cr-2026-0006-cluster-cloud-quorum, cr-2026-0006-cluster-cloud-quorum|CR-2026-0006: add VM160 cloud quorum canary task"]
===== ALERT_MAIL_RCA_NO_SEND =====
ALERT_UNIT={"props":{"ActiveState":"active","ExecMainExitTimestamp":"","ExecMainStartTimestamp":"Fri 2026-08-28 16:31:39 MSK","ExecMainStatus":"0","FragmentPath":"/usr/lib/systemd/system/postfix.service","Id":"postfix.service","LoadState":"loaded","OnFailure":"","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled"},"rc":0,"unit":"postfix.service"}
ALERT_UNIT={"props":{"ActiveState":"inactive","ExecMainExitTimestamp":"","ExecMainStartTimestamp":"","ExecMainStatus":"0","FragmentPath":"","Id":"alertmanager.service","LoadState":"not-found","OnFailure":"","Result":"success","SubState":"dead","TriggeredBy":"","Triggers":"","UnitFileState":""},"rc":0,"unit":"alertmanager.service"}
ALERT_UNIT={"props":{"ActiveState":"inactive","ExecMainExitTimestamp":"","ExecMainStartTimestamp":"","ExecMainStatus":"0","FragmentPath":"","Id":"prometheus-alertmanager.service","LoadState":"not-found","OnFailure":"","Result":"success","SubState":"dead","TriggeredBy":"","Triggers":"","UnitFileState":""},"rc":0,"unit":"prometheus-alertmanager.service"}
POSTFIX_CHECK={"rc":0,"stderr_sha256":null,"stdout_sha256":"af1b92e92f4024ba70a84250f106128ef5164e6550cc342776303eec3795132c"}
POSTCONF={"configured":false,"key":"relayhost","rc":0,"value_sha256":null}
POSTCONF={"configured":true,"key":"smtp_sasl_auth_enable","rc":0,"value_sha256":"9390298f3fb0c5b160498935d79cb139aef28e1c47358b4bbba61862b9c26e59"}
POSTCONF={"configured":false,"key":"smtp_tls_security_level","rc":0,"value_sha256":null}
POSTCONF={"configured":false,"key":"smtp_sasl_password_maps","rc":0,"value_sha256":null}
ROOT_ALIAS_CONFIGURED=false
ALERT_SCRIPT={"bytes":1806,"mentions_alertmanager":true,"mentions_curl":false,"mentions_mail":false,"mentions_sendmail":false,"path":"/usr/local/sbin/homelab-alertmanager-backup-run","repo_matches":[],"sha256":"6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a"}
ALERT_WIRING_PATHS=[]
===== RCA_SUMMARY =====
RCA_TARGETS={"canonical_divergence":true,"failed_units":["homelab-app-cloud-quorum-queue.service","homelab-backup-retention-gc.service","homelab-vm-cloud-quorum-queue.service"],"live_drift_target":"/usr/local/sbin/homelab-vm160-remote-backup","mail_notification_gap":true}
NO_MUTATION=true
NO_GIT_FETCH=true
NO_SERVICE_ACTION=true
NO_BACKUP_STARTED=true
NO_RESTORE_STARTED=true
NO_DELETE=true
NO_MAIL_SENT=true
NO_WORKTREE_CREATED=true
TASK_RESULT=PASS_HOMELAB_BACKUP_RCA_READONLY
HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-RCA-READONLY-1003R1","rollback_restored":null,"rollback_started":false,"status":"OK","version":1}
WORKERS2_BACKUP_RCA_END=1

OUTPUT_END
CHAT_OUTPUT_END
