CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-V2-FAILURE-RCA-READONLY-1004R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-v2-failure-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=04cc92e6fe63247f64d16ce4f140a998a0207420bcdfd5c4a0b211896b71a66a
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=1b5b0eefe46c881abf1271c36e7f983e52db25c0f823f7be9c8d8f78425c5e2d
SANITIZED_OUTPUT_SHA256=1b5b0eefe46c881abf1271c36e7f983e52db25c0f823f7be9c8d8f78425c5e2d
OUTPUT_BEGIN
WORKERS2_BACKUP_V2_FAILURE_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-V2-FAILURE-RCA-READONLY-1004R1
MODE=read-only
COMPONENT=homelab-backup-v2-failure-rca
HOST=pve01
UID=0
TIME_UTC=2026-09-21T01:11:46Z
MUTATION_BOUNDARY=NO_FETCH_NO_RESET_NO_SERVICE_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL
===== LIVE_GITEA_AND_CANONICAL =====
LOCAL_HEAD={"rc":0,"stderr_sha256":null,"stdout":"d5e2ef5639398a762283e47143c17b494f8ed579"}
LOCAL_MAIN={"rc":0,"stderr_sha256":null,"stdout":"d5e2ef5639398a762283e47143c17b494f8ed579"}
TRACKING_ORIGIN_MAIN={"rc":0,"stderr_sha256":null,"stdout":"d5e2ef5639398a762283e47143c17b494f8ed579"}
GIT_STATUS={"rc":0,"stderr_sha256":null,"stdout":"## main...origin/main"}
GITEA_LIVE_MAIN={"rc":0,"stderr_sha256":null,"stdout":"326622a4864fa9b5f43b69981733236bb42c4d7b\trefs/heads/main"}
WORKTREE_CONCURRENCY_SUMMARY={"count":69,"recent_backup_related":[{"branch":"refs/heads/cr-2026-0004-backup-tail-closure","head":"bf80410f592f8eb9eaa80695f609870fe3fb85fd","path":"/root/homelab-ops-cr-2026-0004"},{"branch":"refs/heads/cr-2026-0006-cluster-cloud-quorum","head":"16e84298035f5e4ec7e2b7d39967ad5e40ea3d4b","path":"/root/homelab-ops-cr-2026-0006"},{"branch":"refs/heads/cr-2026-0020-app-cloud-quorum","head":"f556984e410ed819b4cb1e00635dceb43f0ffc5d","path":"/root/homelab-ops-cr-2026-0020"},{"branch":"refs/heads/cr-2026-0021-vm-backup-freshness","head":"3c92036f3000189493fc47557c4cb03e7a412c82","path":"/root/homelab-ops-cr-2026-0021"},{"branch":"refs/heads/cr-2026-0022-app-cloud-quorum-freshness","head":"a497e0f15f1c8d11c3603a4cf5a6a65654cae6fb","path":"/root/homelab-ops-cr-2026-0022"},{"branch":"refs/heads/cr-2026-0024-hbarhub-cloud-quorum","head":"332e8d3f8d2742a05a8d73637cac6a2defc3a825","path":"/root/homelab-ops-cr-2026-0024"},{"branch":"refs/heads/cr-2026-0025-scrutiny-cloud-quorum","head":"73890efb9e99bf02a09daa0e547ab36b48c83b30","path":"/root/homelab-ops-cr-2026-0025"},{"branch":"refs/heads/cr-2026-0026-backup-scheduler-rpo-hardening","head":"ebc08fe5d6beef3cc927bc7d23f6ab37b35d947d","path":"/root/homelab-ops-cr-2026-0026"},{"branch":"refs/heads/cr-2026-0027-backup-lock-fd-hardening","head":"d5a14cbaef47db39f17a94e248e183f0916bdab1","path":"/root/homelab-ops-cr-2026-0027"},{"branch":"refs/heads/cr-2026-0028-backup-scheduler-fairness-granularity","head":"6e3707e90181318be3e8616b14ee553968c485dd","path":"/root/homelab-ops-cr-2026-0028"},{"branch":"refs/heads/cr-2026-0029-control-bootstrap-cloud-quorum","head":"dd81175a24ff64048b36d6ec08dcd3addfc681e9","path":"/root/homelab-ops-cr-2026-0029"},{"branch":"refs/heads/cr-2026-0031-vm-restore-evidence-hardening","head":"f8941f01f41d03b3368a33df868f7ea11311eb6a","path":"/root/homelab-ops-cr-2026-0031"},{"branch":"refs/heads/cr-2026-0032-backup-unified-monitoring","head":"8ad7fcc4537c48de47daac086087d247886c58d9","path":"/root/homelab-ops-cr-2026-0032"},{"branch":"refs/heads/cr-2026-0033-backup-retention-safe-gc","head":"42fd895f421ebc177006c5eb4173f7b4f40ad47f","path":"/root/homelab-ops-cr-2026-0033"},{"branch":"refs/heads/cr-2026-0046-reports-monitor-canonical-backup-supervision","head":"6fc45798c21bd66c561c14059e14f09b7d0a8d63","path":"/root/homelab-ops-cr-2026-0046"},{"branch":"refs/heads/cr-2026-0049-pve03-r8152-lifecycle-repair","head":"801579bd301ad0b28f462f462919660b38041344","path":"/root/homelab-ops-cr-2026-0049"},{"branch":"refs/heads/cr-2026-0050-backup-matrix-residue-retirement","head":"6c1165dccfb6e84b571f0caac6773fd427bf800d","path":"/root/homelab-ops-cr-2026-0050"},{"branch":"refs/heads/cr-2026-0054-backup-control-coverage-hardening","head":"3f02810a16b711700a9da87189c63e38b1fa324b","path":"/root/homelab-ops-cr-2026-0054"},{"branch":"refs/heads/cr-2026-0055-backup-retention-plan-stability-hotfix","head":"902126179efd488e20938a091dcee36109829b86","path":"/root/homelab-ops-cr-2026-0055"},{"branch":"refs/heads/cr-2026-0056-backup-retention-transaction-resume-hotfix","head":"b69b63f531fb71eb95ab147bbfce57591f90e0a4","path":"/root/homelab-ops-cr-2026-0056"},{"branch":"refs/heads/cr-2026-0058-legacy-backup-target-retirement","head":"4b427f5f0406cab6757cf8889fd479a62d218261","path":"/root/homelab-ops-cr-2026-0058"},{"branch":"refs/heads/cr-2026-0060-backup-retention-remote-list-failclosed-hotfix","head":"1a0e4ce27b94bbfbf7b60829974a728b5eba3ad2","path":"/root/homelab-ops-cr-2026-0060"},{"branch":"refs/heads/cr-2026-0064-backup-retention-pinned-scheduled-scope-hotfix","head":"d3a11bde0e73ddfd7049693782d44100c0c1130e","path":"/root/homelab-ops-cr-2026-0064"},{"branch":"refs/heads/cr-2026-0065-backup-retention-dynamic-pinned-prune-mailru-cleanup","head":"2c2f378aec7dc4ac23c633a31f738a850fface56","path":"/root/homelab-ops-cr-2026-0065"},{"branch":"refs/heads/cr-2026-0066-legacy-backup-target-retirement","head":"bcf018f263d9fc607eddd1796e5ef306870c6b7b","path":"/root/homelab-ops-cr-2026-0066"},{"branch":"refs/heads/cr-2026-0070-backup-target-lifecycle-registry","head":"dbf8266d5f2bc4c87024ea8f0eb5f0e8922c7519","path":"/root/homelab-ops-cr-2026-0070"},{"branch":"refs/heads/cr-2026-0071-backup-observability","head":"07cb58a803022b5538e30b201380e215a5490c84","path":"/root/homelab-ops-cr-2026-0071"},{"branch":"refs/heads/cr-2026-0072-gfs-retention","head":"34d76b1cfd4071d94290edac50a31a8c0d4c11c8","path":"/root/homelab-ops-cr-2026-0072"},{"branch":"refs/heads/cr-2026-0074-backup-delete-evidence-hardening","head":"804a2391818f842d768edf434ffeaaf129e576f5","path":"/root/homelab-ops-cr-2026-0074"},{"branch":"refs/heads/cr-2026-0018-backup-cloud-quorum-foundation","head":"b25e80a67fb97ff6c0aa272b25675b9f7c82fb96","path":"/srv/homelab-ops-cr-2026-0018"}]}
===== CURRENT_INVENTORY_FOCUS =====
INVENTORY_FOCUS=[{"name":"unbound1","node":"pve01","status":"running","type":"lxc","vmid":112},{"name":"homelab-ops-worker","node":"pve03","status":"running","type":"qemu","vmid":190},{"name":"newfi-staging","node":"pve03","status":"running","type":"qemu","vmid":211},{"name":"edge-cold-standby","node":"pve02","status":"stopped","type":"qemu","vmid":9130}]
===== BACKUP_V2_RUNTIME_SOURCE =====
SOURCE_FOCUS={"bytes":1656,"exists":true,"matches":["5:EXPECTED={'fresh_lxc_helper.py': 'dbb3e6dbb71b10797df736431d9501a752a0f8c439163c9a0bddb7f76747f011', 'producer_exec_helper.py': 'b3b97099bcd7126670f866e81faae6a42a96d869d9989e6a67f6ea6c871c90b3', 'lxc_adapter.py': '3fa7d194010d76f7c8bc95035ae8876262e9337903e310f45f45df88c266dc8d', 'qemu_adapter.py': '1906db42bd34bbe85eb46fd9797b3572d46e0a41c29d1c29be5f9df5e448f15b', 'scheduler_adapter.py': 'a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96', 'entry504.py': 'ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793'}"],"path":"/usr/local/sbin/homelab-backup-v2","sha256":"5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2"}
SOURCE_FOCUS={"bytes":309,"exists":true,"matches":["2:Description=Homelab Backup V2 manifest-driven scheduler","3:After=network-online.target","4:Wants=network-online.target","11:ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled"],"path":"/etc/systemd/system/homelab-backup-v2.service","sha256":"1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565"}
SOURCE_FOCUS={"bytes":192,"exists":true,"matches":["2:Description=Homelab Backup V2 scheduler timer","11:WantedBy=timers.target"],"path":"/etc/systemd/system/homelab-backup-v2.timer","sha256":"5469057602452b303a01ccb8057085c1b0e447973e5ed198cebf7dd5daf58e9d"}
BACKUP_V2_REFERENCED_EXISTING_PATHS=[]
===== BACKUP_V2_FAILURE_JOURNAL =====
BACKUP_V2_JOURNAL_FOCUS={"lines":["2026-09-20T15:19:49.057775+03:00 pve01 homelab-backup-v2[2298848]: SCHEDULE_TARGET=xf-hkmods","2026-09-20T15:19:49.057775+03:00 pve01 homelab-backup-v2[2298848]: SCHEDULE_RESULT=SUCCESS","2026-09-20T15:19:59.382893+03:00 pve01 homelab-backup-v2[3306237]: SCHEDULE_TARGET=guest-pve01-lxc-112","2026-09-20T15:19:59.382893+03:00 pve01 homelab-backup-v2[3306237]: SCHEDULE_RESULT=FAIL","2026-09-20T15:19:59.382893+03:00 pve01 homelab-backup-v2[3306237]: SCHEDULE_REASON=CT112_PIPELINE_FAILED","2026-09-20T15:19:59.392549+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T15:19:59.392691+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T15:56:31.790799+03:00 pve01 homelab-backup-v2[3417390]: SCHEDULE_TARGET=us-netbird","2026-09-20T15:56:31.790799+03:00 pve01 homelab-backup-v2[3417390]: SCHEDULE_RESULT=FAIL","2026-09-20T15:56:31.790799+03:00 pve01 homelab-backup-v2[3417390]: SCHEDULE_REASON=PIPE_FAILED","2026-09-20T15:56:31.797372+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T15:56:31.797541+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T16:31:22.568658+03:00 pve01 homelab-backup-v2[3535131]: SCHEDULE_TARGET=guest-pve01-lxc-112","2026-09-20T16:31:22.568658+03:00 pve01 homelab-backup-v2[3535131]: SCHEDULE_RESULT=FAIL","2026-09-20T16:31:22.568658+03:00 pve01 homelab-backup-v2[3535131]: SCHEDULE_REASON=CT112_PIPELINE_FAILED","2026-09-20T16:31:22.578409+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T16:31:22.578669+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T17:05:15.846826+03:00 pve01 homelab-backup-v2[3635207]: SCHEDULE_TARGET=md-relay","2026-09-20T17:05:15.846826+03:00 pve01 homelab-backup-v2[3635207]: SCHEDULE_RESULT=SUCCESS","2026-09-20T17:41:42.675536+03:00 pve01 homelab-backup-v2[3751454]: SCHEDULE_TARGET=guest-pve01-lxc-112","2026-09-20T17:41:42.675536+03:00 pve01 homelab-backup-v2[3751454]: SCHEDULE_RESULT=FAIL","2026-09-20T17:41:42.675536+03:00 pve01 homelab-backup-v2[3751454]: SCHEDULE_REASON=CT112_PIPELINE_FAILED","2026-09-20T17:41:42.681881+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T17:41:42.682116+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T18:16:37.584638+03:00 pve01 homelab-backup-v2[3852802]: SCHEDULE_TARGET=spb-ingress","2026-09-20T18:16:37.584638+03:00 pve01 homelab-backup-v2[3852802]: SCHEDULE_RESULT=SUCCESS","2026-09-20T18:55:11.885256+03:00 pve01 homelab-backup-v2[3975404]: SCHEDULE_TARGET=router-netcraze","2026-09-20T18:55:11.885256+03:00 pve01 homelab-backup-v2[3975404]: SCHEDULE_RESULT=SUCCESS","2026-09-20T19:38:44.368912+03:00 pve01 homelab-backup-v2[4075290]: SCHEDULE_TARGET=xf-codevipe","2026-09-20T19:38:44.368912+03:00 pve01 homelab-backup-v2[4075290]: SCHEDULE_RESULT=SUCCESS","2026-09-20T20:03:09.246625+03:00 pve01 homelab-backup-v2[5417]: SCHEDULE_TARGET=guest-pve01-lxc-112","2026-09-20T20:03:09.246625+03:00 pve01 homelab-backup-v2[5417]: SCHEDULE_RESULT=FAIL","2026-09-20T20:03:09.246625+03:00 pve01 homelab-backup-v2[5417]: SCHEDULE_REASON=CT112_PIPELINE_FAILED","2026-09-20T20:03:09.252660+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T20:03:09.252827+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T20:50:52.995548+03:00 pve01 homelab-backup-v2[107015]: SCHEDULE_TARGET=xf-zakrutim","2026-09-20T20:50:52.995548+03:00 pve01 homelab-backup-v2[107015]: SCHEDULE_RESULT=SUCCESS","2026-09-20T21:12:11.756667+03:00 pve01 vzdump[223483]: <root@pam> starting task UPID:pve01:000368FC:0BF1EB40:6AB021FB:vzdump:110:root@pam:","2026-09-20T21:12:11.772892+03:00 pve01 vzdump[223484]: INFO: starting new backup job: vzdump 110 --ionice 7 --stdexcludes 0 --node pve01 --mode snapshot --zstd 1 --compress zstd --tmpdir /var/tmp/homelab-vzdump-v2-v2-ct110-20260920T181158Z-25703b1ef77f --lockwait 0 --prune-backups 'keep-all=1' --bwlimit 32768 --remove 0","2026-09-20T21:12:11.773447+03:00 pve01 vzdump[223484]: INFO: Starting Backup of VM 110 (lxc)","2026-09-20T21:14:15.597824+03:00 pve01 vzdump[223484]: INFO: Finished Backup of VM 110 (00:02:04)","2026-09-20T21:14:15.601427+03:00 pve01 vzdump[223484]: INFO: Backup job finished successfully","2026-09-20T21:14:15.629994+03:00 pve01 vzdump[223483]: <root@pam> end task UPID:pve01:000368FC:0BF1EB40:6AB021FB:vzdump:110:root@pam: OK","2026-09-20T21:14:15.881089+03:00 pve01 homelab-backup-v2[223361]: SCHEDULE_TARGET=ct110","2026-09-20T21:14:15.881089+03:00 pve01 homelab-backup-v2[223361]: LXC_EXECUTION_DIAG={\"archive_bytes\":541730358,\"exception_class\":null,\"exception_message_sha256\":null,\"executor\":\"local-proven-pve270\",\"helper_rc\":0,\"node\":\"pve01\",\"producer_rc\":0,\"stderr_bytes_observed\":1405,\"stderr_bytes_retained\":1405,\"stderr_stream_sha256\":\"52da6b194c8151f5e9372188032c9c6fa1b1a9135609736265a2a169a9c57adb\",\"stderr_truncated\":false}","2026-09-20T21:15:20.610181+03:00 pve01 homelab-backup-v2[223361]: SCHEDULE_RESULT=SUCCESS","2026-09-20T21:51:38.078713+03:00 pve01 homelab-backup-v2[345333]: SCHEDULE_TARGET=control-bootstrap-v2","2026-09-20T21:51:38.078713+03:00 pve01 homelab-backup-v2[345333]: SCHEDULE_RESULT=SUCCESS","2026-09-20T22:29:31.800106+03:00 pve01 homelab-backup-v2[459026]: SCHEDULE_TARGET=us-netbird","2026-09-20T22:29:31.800106+03:00 pve01 homelab-backup-v2[459026]: SCHEDULE_RESULT=FAIL","2026-09-20T22:29:31.800106+03:00 pve01 homelab-backup-v2[459026]: SCHEDULE_REASON=PIPE_FAILED","2026-09-20T22:29:31.805836+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-20T22:29:31.805983+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-20T23:09:29.556447+03:00 pve01 homelab-backup-v2[593177]: SCHEDULE_TARGET=ct111","2026-09-20T23:09:29.556447+03:00 pve01 homelab-backup-v2[593177]: LXC_EXECUTION_DIAG={\"executor\":\"remote-bounded-ssh-cwd-root\",\"node\":\"pve02\",\"producer_rc\":0,\"stderr_bytes_observed\":1035,\"stderr_bytes_retained\":1035,\"stderr_stream_sha256\":\"bce35788cf8d44be1cd8cf226e02ad4d65baf010cb8e8dfc4eb7a5abb32ced63\",\"stderr_truncated\":false}","2026-09-20T23:10:21.129791+03:00 pve01 homelab-backup-v2[593177]: SCHEDULE_RESULT=SUCCESS","2026-09-20T23:38:33.285692+03:00 pve01 vzdump[681827]: <root@pam> starting task UPID:pve01:000A6783:0BFF5189:6AB04449:vzdump:210:root@pam:","2026-09-20T23:38:33.296638+03:00 pve01 vzdump[681859]: INFO: starting new backup job: vzdump 210 --dumpdir /var/tmp/homelab-vzdump-v2-v2-vm210-20260920T203832Z-458da35ec027 --compress zstd --mode snapshot","2026-09-20T23:38:33.301799+03:00 pve01 vzdump[681859]: INFO: Starting Backup of VM 210 (qemu)","2026-09-20T23:38:47.314579+03:00 pve01 vzdump[681859]: INFO: Finished Backup of VM 210 (00:00:14)","2026-09-20T23:38:47.318698+03:00 pve01 vzdump[681859]: INFO: Backup job finished successfully","2026-09-20T23:38:47.335141+03:00 pve01 vzdump[681827]: <root@pam> end task UPID:pve01:000A6783:0BFF5189:6AB04449:vzdump:210:root@pam: OK","2026-09-20T23:38:47.350575+03:00 pve01 homelab-backup-v2[681773]: SCHEDULE_TARGET=vm210","2026-09-20T23:38:47.350575+03:00 pve01 homelab-backup-v2[681773]: QEMU_EXECUTION_DIAG={\"archive_bytes\":687647191,\"archive_selected\":true,\"node\":\"pve01\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":210}","2026-09-20T23:39:58.319976+03:00 pve01 homelab-backup-v2[681773]: SCHEDULE_RESULT=SUCCESS","2026-09-21T00:12:09.285479+03:00 pve01 homelab-backup-v2[782723]: SCHEDULE_TARGET=guest-pve01-lxc-112","2026-09-21T00:12:09.285479+03:00 pve01 homelab-backup-v2[782723]: SCHEDULE_RESULT=FAIL","2026-09-21T00:12:09.285479+03:00 pve01 homelab-backup-v2[782723]: SCHEDULE_REASON=CT112_PIPELINE_FAILED","2026-09-21T00:12:09.292481+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.","2026-09-21T00:12:09.292801+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-21T00:50:11.899660+03:00 pve01 vzdump[915940]: <root@pam> starting task UPID:pve01:000DF9E5:0C05E0AE:6AB05513:vzdump:112:root@pam:","2026-09-21T00:50:11.913977+03:00 pve01 vzdump[915941]: INFO: starting new backup job: vzdump 112 --prune-backups 'keep-all=1' --bwlimit 32768 --node pve01 --tmpdir /var/tmp/homelab-vzdump-v2-v2-ct112-20260920T214958Z-30e3291e1543 --stdexcludes 0 --zstd 1 --remove 0 --mode snapshot --compress zstd --ionice 7 --lockwait 0","2026-09-21T00:50:11.914482+03:00 pve01 vzdump[915941]: INFO: Starting Backup of VM 112 (lxc)","2026-09-21T00:50:35.059514+03:00 pve01 vzdump[915941]: INFO: Finished Backup of VM 112 (00:00:24)","2026-09-21T00:50:35.063713+03:00 pve01 vzdump[915941]: INFO: Backup job finished successfully","2026-09-21T00:50:35.079365+03:00 pve01 vzdump[915940]: <root@pam> end task UPID:pve01:000DF9E5:0C05E0AE:6AB05513:vzdump:112:root@pam: OK","2026-09-21T00:50:35.131822+03:00 pve01 homelab-backup-v2[904735]: SCHEDULE_TARGET=ct112","2026-09-21T00:50:35.131822+03:00 pve01 homelab-backup-v2[904735]: LXC_EXECUTION_DIAG={\"archive_bytes\":223936825,\"exception_class\":null,\"exception_message_sha256\":null,\"executor\":\"local-proven-pve270\",\"helper_rc\":0,\"node\":\"pve01\",\"producer_rc\":0,\"stderr_bytes_observed\":1408,\"stderr_bytes_retained\":1408,\"stderr_stream_sha256\":\"92e2c2144d82520ef7dbe70184d9317d94da5e39aff3b7fb3cf3cf74685e5b68\",\"stderr_truncated\":false}","2026-09-21T00:51:16.588203+03:00 pve01 homelab-backup-v2[904735]: SCHEDULE_RESULT=SUCCESS","2026-09-21T01:26:45.553982+03:00 pve01 homelab-backup-v2[1018631]: SCHEDULE_TARGET=ct113","2026-09-21T01:26:45.553982+03:00 pve01 homelab-backup-v2[1018631]: LXC_EXECUTION_DIAG={\"executor\":\"remote-bounded-ssh-cwd-root\",\"node\":\"pve02\",\"producer_rc\":0,\"stderr_bytes_observed\":1038,\"stderr_bytes_retained\":1038,\"stderr_stream_sha256\":\"5590f20ce6d625aae5e745852a46ffd5331ef67462164b084208650f60f43eeb\",\"stderr_truncated\":false}","2026-09-21T01:27:21.219244+03:00 pve01 homelab-backup-v2[1018631]: SCHEDULE_RESULT=SUCCESS","2026-09-21T02:00:36.030805+03:00 pve01 vzdump[1131961]: <root@pam> starting task UPID:pve01:001145BA:0C0C52BB:6AB06594:vzdump:150:root@pam:","2026-09-21T02:00:36.041445+03:00 pve01 vzdump[1131962]: INFO: starting new backup job: vzdump 150 --mode snapshot --compress zstd --dumpdir /var/tmp/homelab-vzdump-v2-v2-vm150-20260920T230035Z-e0880e47ea55","2026-09-21T02:00:36.045251+03:00 pve01 vzdump[1131962]: INFO: Starting Backup of VM 150 (qemu)","2026-09-21T02:00:53.219661+03:00 pve01 vzdump[1131962]: INFO: Finished Backup of VM 150 (00:00:17)","2026-09-21T02:00:53.225223+03:00 pve01 vzdump[1131962]: INFO: Backup job finished successfully","2026-09-21T02:00:53.241129+03:00 pve01 vzdump[1131961]: <root@pam> end task UPID:pve01:001145BA:0C0C52BB:6AB06594:vzdump:150:root@pam: OK","2026-09-21T02:00:53.259679+03:00 pve01 homelab-backup-v2[1131900]: SCHEDULE_TARGET=vm150","2026-09-21T02:00:53.259679+03:00 pve01 homelab-backup-v2[1131900]: QEMU_EXECUTION_DIAG={\"archive_bytes\":948768127,\"archive_selected\":true,\"node\":\"pve01\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":150}","2026-09-21T02:02:27.509220+03:00 pve01 homelab-backup-v2[1131900]: SCHEDULE_RESULT=SUCCESS","2026-09-21T02:37:59.001120+03:00 pve01 homelab-backup-v2[1256150]: SELFTEST_REPLACEMENT486=PASS","2026-09-21T02:42:54.073760+03:00 pve01 homelab-backup-v2[1256150]: SCHEDULE_TARGET=vm130","2026-09-21T02:42:54.073760+03:00 pve01 homelab-backup-v2[1256150]: QEMU_EXECUTION_DIAG={\"archive_bytes\":10948261509,\"archive_selected\":true,\"node\":\"pve03\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":130}","2026-09-21T02:57:13.496557+03:00 pve01 homelab-backup-v2[1256150]: SCHEDULE_RESULT=SUCCESS","2026-09-21T03:16:56.429153+03:00 pve01 homelab-backup-v2[1371340]: SCHEDULE_TARGET=vm160","2026-09-21T03:16:56.429153+03:00 pve01 homelab-backup-v2[1371340]: QEMU_EXECUTION_DIAG={\"archive_bytes\":3755423628,\"archive_selected\":true,\"node\":\"pve02\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":160}","2026-09-21T03:21:40.026609+03:00 pve01 homelab-backup-v2[1371340]: SCHEDULE_RESULT=SUCCESS","2026-09-21T03:47:59.909526+03:00 pve01 vzdump[1480118]: <root@pam> starting task UPID:pve01:001695B7:0C1627DF:6AB07EBF:vzdump:171:root@pam:","2026-09-21T03:47:59.921859+03:00 pve01 vzdump[1480119]: INFO: starting new backup job: vzdump 171 --dumpdir /var/tmp/homelab-vzdump-v2-v2-vm171-20260921T004759Z-537db1552014 --compress zstd --mode snapshot","2026-09-21T03:47:59.925791+03:00 pve01 vzdump[1480119]: INFO: Starting Backup of VM 171 (qemu)","2026-09-21T03:48:43.831277+03:00 pve01 vzdump[1480119]: INFO: Finished Backup of VM 171 (00:00:44)","2026-09-21T03:48:43.836725+03:00 pve01 vzdump[1480119]: INFO: Backup job finished successfully","2026-09-21T03:48:43.872532+03:00 pve01 vzdump[1480118]: <root@pam> end task UPID:pve01:001695B7:0C1627DF:6AB07EBF:vzdump:171:root@pam: OK","2026-09-21T03:48:43.892527+03:00 pve01 homelab-backup-v2[1479892]: SCHEDULE_TARGET=vm171","2026-09-21T03:48:43.892527+03:00 pve01 homelab-backup-v2[1479892]: QEMU_EXECUTION_DIAG={\"archive_bytes\":2051667128,\"archive_selected\":true,\"node\":\"pve01\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":171}","2026-09-21T03:51:17.662338+03:00 pve01 homelab-backup-v2[1479892]: SCHEDULE_RESULT=SUCCESS"],"matched_count":107,"rc":0,"stderr_sha256":null}
===== RECENT_BACKUP_V2_STATE_FILES =====
BACKUP_V2_STATE_ROOTS=["/var/lib/homelab-backup-v2"]
RECENT_STATE_FILE={"age_sec":1229,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/vm171.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm171-20260921T004759Z-537db1552014\",\"green_verified\":false,\"last_attempt_epoch\":1789951877,\"last_success_epoch\":1789951877,\"logical_id\":\"vm171\",\"manifest_sha256\":\"bcc1c90b61c53d3e58182f571f9cd1a6dd6d612469d9f070505d82b128bdd7a2\",\"payload_sha256\":\"41520b284fdec39732e9bff8881a0ed19f543859e280755b1bbc838e52c68649\",\"replicas\":[{\"crypt\":\"pve03-mail-01-crypt\",\"physical\":\"pve03-mail-01\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-21T00:51:17Z\"}"],"sha256":"46b5f03744cd5b4c6f01a0c8e260846f538354d788b8a3527cc56db2aec5d6c0"}
RECENT_STATE_FILE={"age_sec":3006,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/vm160.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm160-20260921T001432Z-31df1acfee7a\",\"green_verified\":false,\"last_attempt_epoch\":1789950100,\"last_success_epoch\":1789950100,\"logical_id\":\"vm160\",\"manifest_sha256\":\"658eb4a8dd4060d58af8afb72bb33b0315020eb3837e36d8f5866f92ca4bb18c\",\"payload_sha256\":\"d4bbc19aef3d2e96252c2fd7b4ca7b93fa9d83f872fdff8fcca9933227d5b1b3\",\"replicas\":[{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"},{\"crypt\":\"pve01-mail-04-crypt\",\"physical\":\"pve01-mail-04\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-21T00:21:40Z\"}"],"sha256":"1219e114d204fefd63adcd3e0f82c90b80cb714b8f0253e86dd0e406f4f4abd2"}
RECENT_STATE_FILE={"age_sec":4473,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/vm130.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm130-20260920T233759Z-c9518c8be2c3\",\"green_verified\":false,\"last_attempt_epoch\":1789948633,\"last_success_epoch\":1789948633,\"logical_id\":\"vm130\",\"manifest_sha256\":\"93b2eec0d7750145384c2429e5f5a24abb0034acc086f3514375a90d637219d7\",\"payload_sha256\":\"60f90f8352115d5df0ac6af07bf07e7a3976f3c61e83732d0d7ce071a6414faa\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve03-mail-03-crypt\",\"physical\":\"pve03-mail-03\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T23:57:13Z\"}"],"sha256":"f4b2f4f2885c61740f89443d3e792fe2d57c6ff1e87daf27fbe421e0988b7e90"}
RECENT_STATE_FILE={"age_sec":7759,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/vm150.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm150-20260920T230035Z-e0880e47ea55\",\"green_verified\":false,\"last_attempt_epoch\":1789945347,\"last_success_epoch\":1789945347,\"logical_id\":\"vm150\",\"manifest_sha256\":\"7cc63a5fb78f8bc1e73247503b1896e3a31a99022fd9f43895c3de653ee7aab1\",\"payload_sha256\":\"3eaf82326949ec7d849e374379e7ea981d859343ce0aac471bfcff952ee5f20e\",\"replicas\":[{\"crypt\":\"pve03-mail-03-crypt\",\"physical\":\"pve03-mail-03\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T23:02:27Z\"}"],"sha256":"4ad4046a48ac815fae689998a8ea96d6db66dbb5b601182813815c33d9fb535e"}
RECENT_STATE_FILE={"age_sec":9865,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/ct113.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-ct113-20260920T222532Z-4f4a608def82\",\"green_verified\":false,\"last_attempt_epoch\":1789943241,\"last_success_epoch\":1789943241,\"logical_id\":\"ct113\",\"manifest_sha256\":\"a721c4dfcd8031fd52b7281dcc2e16f0de5874e0b7e3e3eb25791b4c64319e1a\",\"payload_sha256\":\"7a488bad3e70b1e4cc5246dd8d78d5bf61cb79ad92ce8f7d573eb387fcb002bd\",\"replicas\":[{\"crypt\":\"pve03-mail-03-crypt\",\"physical\":\"pve03-mail-03\"},{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T22:27:21Z\"}"],"sha256":"c78f182a0b0ac42fc1daa40ed9f89c0da1d54710b85665af9ac1f279250c9aa3"}
RECENT_STATE_FILE={"age_sec":12030,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/ct112.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-ct112-20260920T214958Z-30e3291e1543\",\"green_verified\":false,\"last_attempt_epoch\":1789941076,\"last_success_epoch\":1789941076,\"logical_id\":\"ct112\",\"manifest_sha256\":\"299de9c4c8ed701081e274631b81fc937bc36dd8b67d4a287a3f975dd7b366dd\",\"payload_sha256\":\"5b9f2079a420e709985c6c54eabcd59856dafd130107727743c401f95ffe216d\",\"replicas\":[{\"crypt\":\"pve03-mail-03-crypt\",\"physical\":\"pve03-mail-03\"},{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T21:51:16Z\"}"],"sha256":"eedf7d1687d8d6b5ee84f750f9cce85a423a4f02ccbb39aa8333fc3e354e2243"}
RECENT_STATE_FILE={"age_sec":14377,"bytes":844,"path":"/var/lib/homelab-backup-v2/state/guest-pve01-lxc-112.json","selected":["{\"consecutive_failures\":5,\"failure_backoff_seconds\":21600,\"last_attempt_epoch\":1789938729,\"last_success_epoch\":1789799511,\"last_success_generation\":\"V2-20260919T062945Z-1c175c0e0cd7\",\"last_success_green_verified\":true,\"last_success_manifest_sha256\":\"b0311f9af3f38813dfe98d64a6e9d6ae93745a1c770080440b07df34b7f66bbe\",\"last_success_payload_sha256\":\"b54e21482312be3a810a2d15e74c8f5fac642ca594f75dffa0d98d9fe60d457b\",\"last_success_replicas\":[{\"account_id\":\"e7c8100c3ee2308a64aa763264fc0f738f30f99f19a7306fc9c2554fdd308199\",\"crypt_remote\":\"pve03-mail-04-crypt\"},{\"account_id\":\"690cb55f912672f2854c3424fe9d536beb6096dadb767e894b3f52bb8fd911bf\",\"crypt_remote\":\"pve03-mail-01-crypt\"}],\"logical_id\":\"guest-pve01-lxc-112\",\"reason\":\"CT112_PIPELINE_FAILED\",\"result\":\"FAIL\",\"retry_after_epoch\":1789960329,\"schema\":1,\"updated_at_utc\":\"2026-09-20T21:12:09Z\"}"],"sha256":"f2ed991431a1b75bb4884f1f1c72881490f9cd71160f6034fb95bdad7a58be39"}
RECENT_STATE_FILE={"age_sec":16308,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/vm210.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm210-20260920T203832Z-458da35ec027\",\"green_verified\":false,\"last_attempt_epoch\":1789936798,\"last_success_epoch\":1789936798,\"logical_id\":\"vm210\",\"manifest_sha256\":\"3587cca11932a034f03c4638bb347972307cd179dc60635b2945dafc5558d875\",\"payload_sha256\":\"7e3a8eb9653712a80bf9eb919aae0004faad55ca324f8acb1c55c8a7574fc34e\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T20:39:58Z\"}"],"sha256":"862f6bdfe6cdf9272832e986075a4fff84e9edf73a17fa73ca2a556db52d0aee"}
RECENT_STATE_FILE={"age_sec":18085,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/ct111.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-ct111-20260920T200749Z-a2bac7e3ecb7\",\"green_verified\":false,\"last_attempt_epoch\":1789935021,\"last_success_epoch\":1789935021,\"logical_id\":\"ct111\",\"manifest_sha256\":\"22ff237e0a246ab51c8bd1ff8f4c1e12742a707578170f67055344c8076494f6\",\"payload_sha256\":\"0e65a2cd9ff79535f391287adca2ae6e8f87d394fb7f7a7ca8c00922677442fd\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve03-mail-03-crypt\",\"physical\":\"pve03-mail-03\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T20:10:21Z\"}"],"sha256":"12a47c12453c7fe3b66250923e1710b549a565c51dfd93d3d7d54d8b5c5c11c8"}
RECENT_STATE_FILE={"age_sec":20535,"bytes":791,"path":"/var/lib/homelab-backup-v2/state/us-netbird.json","selected":["{\"consecutive_failures\":9,\"failure_backoff_seconds\":21600,\"generation\":\"v2-us-netbird-20260920T192916Z-ca2592a81a30\",\"last_attempt_epoch\":1789932571,\"last_success_epoch\":1789704670,\"last_success_generation\":\"v2-us-netbird-20260918T041032Z-d7648be4c16b\",\"last_success_green_verified\":false,\"last_success_manifest_sha256\":\"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5\",\"last_success_payload_sha256\":\"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf\",\"last_success_replicas\":[{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"},{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"}],\"logical_id\":\"us-netbird\",\"reason\":\"PIPE_FAILED\",\"result\":\"FAIL\",\"retry_after_epoch\":1789954171,\"schema\":1,\"sealed\":false,\"updated_at_utc\":\"2026-09-20T19:29:31Z\"}"],"sha256":"57736e0734fb3f3f809aca0b5fb750f87248fb77534e613fbfa58c2ae68960c9"}
RECENT_STATE_FILE={"age_sec":22808,"bytes":354,"path":"/var/lib/homelab-backup-v2/state/control-bootstrap-v2.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"control-bootstrap-v2-pve01-20260916T120057Z\",\"last_attempt_epoch\":1789930298,\"last_success_epoch\":1789930298,\"logical_id\":\"control-bootstrap-v2\",\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T18:51:38Z\",\"verification\":\"existing_control_engine_verify\"}"],"sha256":"586acfdfb8f56db1c85a49b8e549f325aa98519709911ef2156964d3706c7b9d"}
RECENT_STATE_FILE={"age_sec":24986,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/ct110.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-ct110-20260920T181158Z-25703b1ef77f\",\"green_verified\":false,\"last_attempt_epoch\":1789928120,\"last_success_epoch\":1789928120,\"logical_id\":\"ct110\",\"manifest_sha256\":\"6f99d74ab6a71daca2b382d7e280e02ae44d1b65abe127aa5a0106e04907b5aa\",\"payload_sha256\":\"eaa23dc6f970e658b1d39615c3a45bf20091d5da875ec817b7e8fd8e1168a4e0\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T18:15:20Z\"}"],"sha256":"a340f72b5e965296245d38ebd5dba835ff795e2a7d851934811071fde2f453f3"}
RECENT_STATE_FILE={"age_sec":26453,"bytes":621,"path":"/var/lib/homelab-backup-v2/state/xf-zakrutim.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-xf-zakrutim-20260920T173658Z-a8de3568c79b\",\"green_verified\":false,\"last_attempt_epoch\":1789926652,\"last_success_epoch\":1789926652,\"logical_id\":\"xf-zakrutim\",\"manifest_sha256\":\"53835f258d47a68635dc1ef54998e019bac0f25d9fab7423fbc67ed17010cff6\",\"payload_sha256\":\"fe757c28ade0a1758d76ad4d616f559f30621fe28513aa185f83628c10708311\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T17:50:52Z\"}"],"sha256":"17757d0b482449d3a12c4a40a22bb915c682139be8eb6037631c2ce4ebe6e84c"}
RECENT_STATE_FILE={"age_sec":30782,"bytes":621,"path":"/var/lib/homelab-backup-v2/state/xf-codevipe.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-xf-codevipe-20260920T162616Z-da4e8fcf3129\",\"green_verified\":false,\"last_attempt_epoch\":1789922324,\"last_success_epoch\":1789922324,\"logical_id\":\"xf-codevipe\",\"manifest_sha256\":\"5218ed5341f7a93a9fe68a715978375f47f6181e3c58ca075de2fa7a13e408ea\",\"payload_sha256\":\"e00525a66738da126b01fc26cc27e5688ba0f2dcc6afc3276b3aa7dee23ee08b\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T16:38:44Z\"}"],"sha256":"b65a7092d18371bce1d7681224fa3d54911756c0800ecd7df3179d2589b32c5e"}
RECENT_STATE_FILE={"age_sec":33395,"bytes":629,"path":"/var/lib/homelab-backup-v2/state/router-netcraze.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-router-netcraze-20260920T155444Z-a91b2efdb4fe\",\"green_verified\":false,\"last_attempt_epoch\":1789919711,\"last_success_epoch\":1789919711,\"logical_id\":\"router-netcraze\",\"manifest_sha256\":\"fd7fd914d15f08b3d816a8bc76c233eff22b38a42494f7f9259859fb9edd3cb9\",\"payload_sha256\":\"252260b8ba622dc5a4536424abc9cde197a9e71dee67cac108f103d85a84b28c\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T15:55:11Z\"}"],"sha256":"0a360ef89a3d8ee8d2d45061f45120eb3464b800c4050300c205ff7269a9f0a0"}
RECENT_STATE_FILE={"age_sec":35709,"bytes":621,"path":"/var/lib/homelab-backup-v2/state/spb-ingress.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-spb-ingress-20260920T151616Z-9d80091d0c4c\",\"green_verified\":false,\"last_attempt_epoch\":1789917397,\"last_success_epoch\":1789917397,\"logical_id\":\"spb-ingress\",\"manifest_sha256\":\"4f54c7273374849985bc3d3df7940840ab4cc5083a3324585ce9e4978a75e410\",\"payload_sha256\":\"f775a3f6ea7f397559f4b56e5f823694187749a24240173298a65d3d637826d6\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T15:16:37Z\"}"],"sha256":"7e339f360d15e31044a00bcb7127ab5b2ae8193e50aa9dc4d70ec207196336ee"}
RECENT_STATE_FILE={"age_sec":39991,"bytes":615,"path":"/var/lib/homelab-backup-v2/state/md-relay.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-md-relay-20260920T140452Z-1943a8c57bd6\",\"green_verified\":false,\"last_attempt_epoch\":1789913115,\"last_success_epoch\":1789913115,\"logical_id\":\"md-relay\",\"manifest_sha256\":\"a80cc1c23deea8d06a036a4ba696376ef8f7d9f838a0f29e4e5d52ed5579ed12\",\"payload_sha256\":\"c168e3d7e2b0650c844e5c17aeb532bc918bf987db7f6a574f8f9559d010eccc\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T14:05:15Z\"}"],"sha256":"21e6e06ebeba772b1bdc22c2b4d47882facf9813dcb0c2308621dac7e3211988"}
RECENT_STATE_FILE={"age_sec":46317,"bytes":617,"path":"/var/lib/homelab-backup-v2/state/xf-hkmods.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-xf-hkmods-20260920T071214Z-6407219f0624\",\"green_verified\":false,\"last_attempt_epoch\":1789906789,\"last_success_epoch\":1789906789,\"logical_id\":\"xf-hkmods\",\"manifest_sha256\":\"d548aca241a964ac1056325ccdefae48ccc1d53c843dd594403badd9db8a98c2\",\"payload_sha256\":\"94955dc3f8cb1fbe5fbfa74bb42140742b7197e70ca35bedad6ddddcaf181714\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T12:19:49Z\"}"],"sha256":"7cec53bc60b4f12e3285f19f4fe6de9c92d198f61ff268997cf6a36f2b29a8fa"}
RECENT_STATE_FILE={"age_sec":70838,"bytes":621,"path":"/var/lib/homelab-backup-v2/state/xf-gamevipe.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-xf-gamevipe-20260920T015714Z-762dbd7ce0d7\",\"green_verified\":false,\"last_attempt_epoch\":1789882268,\"last_success_epoch\":1789882268,\"logical_id\":\"xf-gamevipe\",\"manifest_sha256\":\"81fa85fdb7ca2f3b3bd20be4cfb1b0c13712255d544ddec67a0f98f2fd948e98\",\"payload_sha256\":\"46d4c7e4e9f1056660e0cdc3136774865b08b8edb5ca92873cea7d41d602e4bb\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T05:31:08Z\"}"],"sha256":"3ac48c6c7f0e0e2699089eecb211687405cd99f57ca66a1e8e5fb7b0145b260e"}
RECENT_STATE_FILE={"age_sec":87487,"bytes":615,"path":"/var/lib/homelab-backup-v2/state/xf-newfi.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-xf-newfi-20260920T004616Z-6be6559a5589\",\"green_verified\":false,\"last_attempt_epoch\":1789865619,\"last_success_epoch\":1789865619,\"logical_id\":\"xf-newfi\",\"manifest_sha256\":\"f912b85107e9e1ee2425749099f0de9a9a53c909f98cd4f41356a20ff148da62\",\"payload_sha256\":\"774bc5dcb7bb3905392c2c078f7c5d4bc127d87d97c2330a5baab997c7b27aa8\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-20T00:53:39Z\"}"],"sha256":"f4e48ea4aa80dc57537510a377775a0ca216e69a8308f92334d4114ed91e76f7"}
RECENT_STATE_FILE={"age_sec":92069,"bytes":625,"path":"/var/lib/homelab-backup-v2/state/xf-kingofwolk.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-xf-kingofwolk-20260919T232652Z-cb0189b2b0ba\",\"green_verified\":false,\"last_attempt_epoch\":1789861037,\"last_success_epoch\":1789861037,\"logical_id\":\"xf-kingofwolk\",\"manifest_sha256\":\"69a9ce6a55c57121b2634c65b2826a8056b4de65127d229b8c3fbbb2980b4162\",\"payload_sha256\":\"cf32a08d5c05eeb04b4d2b2d71faef01c363c9d223b9b9a93813d1bafecca334\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-19T23:37:17Z\"}"],"sha256":"6357284997a7cf7b28e7a740e12608b617e9f57f257b35ccd43853268fb28391"}
RECENT_STATE_FILE={"age_sec":92695,"bytes":617,"path":"/var/lib/homelab-backup-v2/state/xf-dsmods.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-xf-dsmods-20260919T214716Z-c783be0bd209\",\"green_verified\":false,\"last_attempt_epoch\":1789860411,\"last_success_epoch\":1789860411,\"logical_id\":\"xf-dsmods\",\"manifest_sha256\":\"f3b4cd994ffcc6e9a7944f8569fc83df52e8bd9692f4f2067155cc3dbc6b3891\",\"payload_sha256\":\"e609e9951c2baba42ab268507ea277140f73750587a9ccd99e5924af6995f0e3\",\"replicas\":[{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"},{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-19T23:26:51Z\"}"],"sha256":"0b38d624d502ea4a7e3b2284f2a74b6b1985399791cdf8a44d467c855c2aba14"}
RECENT_STATE_FILE={"age_sec":99549,"bytes":611,"path":"/var/lib/homelab-backup-v2/state/vm9130.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm9130-20260919T210758Z-0aa987efac8c\",\"green_verified\":false,\"last_attempt_epoch\":1789853557,\"last_success_epoch\":1789853557,\"logical_id\":\"vm9130\",\"manifest_sha256\":\"4cc97d6769312b0be7e2c3cfe144da7fb68f3f5664755a7c5b8d44c1139bcd3b\",\"payload_sha256\":\"b3703c1f4616f30fc6711a7b7bd2e50af5be805bd6752f87a57be2b9ac531cb8\",\"replicas\":[{\"crypt\":\"pve01-mail-03-crypt\",\"physical\":\"pve01-mail-03\"},{\"crypt\":\"pve03-mail-02-crypt\",\"physical\":\"pve03-mail-02\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-19T21:32:37Z\"}"],"sha256":"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db"}
RECENT_STATE_FILE={"age_sec":103122,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/vm211.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm211-20260919T203132Z-82878bb22008\",\"green_verified\":false,\"last_attempt_epoch\":1789849984,\"last_success_epoch\":1789849984,\"logical_id\":\"vm211\",\"manifest_sha256\":\"af8ad542b43c8ac9f75969b4726d8e294b7c1e62ca569c2383776e5e24d9934d\",\"payload_sha256\":\"a232d511565774d22f86afe0e773b28da4a8e722ec1b2a357e55b0b62f1eeef9\",\"replicas\":[{\"crypt\":\"pve01-mail-03-crypt\",\"physical\":\"pve01-mail-03\"},{\"crypt\":\"pve03-mail-02-crypt\",\"physical\":\"pve03-mail-02\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-19T20:33:04Z\"}"],"sha256":"ecd899387f605d91b036977372cab73a69a0a4cedc050f22ece38c4cb664311b"}
RECENT_STATE_FILE={"age_sec":104899,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/vm170.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm170-20260919T195132Z-74c06ba16732\",\"green_verified\":false,\"last_attempt_epoch\":1789848207,\"last_success_epoch\":1789848207,\"logical_id\":\"vm170\",\"manifest_sha256\":\"7ef201d524d1cea100d8cf87da2d33e920354258c91ed827d634b62d770ae6f4\",\"payload_sha256\":\"22d92332a6b86141b3f67f357535c53826d0127f3e0a4f858a47ddec7777a104\",\"replicas\":[{\"crypt\":\"pve01-mail-04-crypt\",\"physical\":\"pve01-mail-04\"},{\"crypt\":\"pve03-mail-01-crypt\",\"physical\":\"pve03-mail-01\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-19T20:03:27Z\"}"],"sha256":"897ed79b3dc484f868155492fc9471a78dd901b896fdaa934d472174c6a84704"}
RECENT_STATE_FILE={"age_sec":107652,"bytes":609,"path":"/var/lib/homelab-backup-v2/state/vm190.json","selected":["{\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm190-20260919T191358Z-c42ef8cbe584\",\"green_verified\":false,\"last_attempt_epoch\":1789845454,\"last_success_epoch\":1789845454,\"logical_id\":\"vm190\",\"manifest_sha256\":\"d98b7e43bb12e3304a498629b8e2c61baf01a54a9ae2cede67de9a971ee52c44\",\"payload_sha256\":\"a81b1bdc20c5ab28375e20304087099386666e74a5ad18563eaa01751112ddc5\",\"replicas\":[{\"crypt\":\"pve01-mail-04-crypt\",\"physical\":\"pve01-mail-04\"},{\"crypt\":\"pve03-mail-01-crypt\",\"physical\":\"pve03-mail-01\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-19T19:17:34Z\"}"],"sha256":"44d1db9aaac329596eaf9b3a7ebe33daf5c68775d56fecb8cf8ed887c310f059"}
===== RETENTION_VM200_RCA =====
RETENTION_RUNTIME_FOCUS={"bytes":32975,"exists":true,"matches":["23:GFS_HELPER=\"${HOMELAB_BACKUP_GFS_POLICY_HELPER:-/usr/local/libexec/homelab-backup-gfs-policy}\"","24:GFS_HELPER_SHA_EXPECTED=6b7ec1ab3ee1c540a9113b5620c3b4b5a50de6536f3a069d31b2e1243f23bafa","25:GFS_POLICY=\"${HOMELAB_BACKUP_GFS_POLICY:-/etc/homelab-backup/gfs-retention.json}\"","30:[[ -x \"$GFS_HELPER\" && ! -L \"$GFS_HELPER\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_MISSING\\n' >&2; exit 69; }","31:[[ \"$(sha256sum \"$GFS_HELPER\"|awk '{print $1}')\" == \"$GFS_HELPER_SHA_EXPECTED\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\\n' >&2; exit 65; }","32:HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" validate >/dev/null || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID\\n' >&2; exit 66; }","33:GFS_WEEKLY=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" weekly-buckets)\"","34:GFS_MONTHLY=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" monthly-buckets)\"","35:GFS_CRITICAL_COUNT=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" critical-logicals | wc -l)\"","36:VM_KEEP=\"$(\"$REGISTRY_HELPER\" policy-keep VM)\"","37:APP_KEEP=\"$(\"$REGISTRY_HELPER\" policy-keep APP)\"","38:CONTROL_KEEP=\"$(\"$REGISTRY_HELPER\" policy-keep CONTROL)\"","47:node_for_logical(){ \"$REGISTRY_HELPER\" logical-owner \"$1\" 2>/dev/null; }","130:pinned_delete_contract_enabled(){","131:  [[ -n \"${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}\" || -n \"${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}\" ]]","133:pinned_delete_assert_file(){","134:  local f=\"${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}\" expected=\"${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}\" actual line logical generation owner state_file state_sha count=0","135:  [[ -n \"$f\" && -n \"$expected\" ]] || die PINNED_DELETE_CONTRACT_INCOMPLETE 64","136:  [[ \"$expected\" =~ ^[0-9a-f]{64}$ ]] || die PINNED_DELETE_SHA_INVALID 64","137:  [[ -f \"$f\" && ! -L \"$f\" ]] || die PINNED_DELETE_FILE_UNSAFE 64","139:  [[ \"$actual\" == \"$expected\" ]] || die PINNED_DELETE_FILE_SHA_MISMATCH 66","141:    [[ -n \"$line\" ]] || die PINNED_DELETE_EMPTY_LINE 66","144:    owner=\"$(awk -F'|' '{for(i=1;i<=NF;i++)if($i~/^owner=/){sub(/^owner=/,\"\",$i);print $i}}' <<<\"$line\")\"","147:    [[ \"$line\" == \"DELETE|logical=$logical|generation=$generation|owner=$owner|state_file=$state_file|state_sha256=$state_sha\" ]] || die PINNED_DELETE_LINE_FORMAT_INVALID 66","148:    [[ \"$(node_for_logical \"$logical\")\" == \"$owner\" ]] || die \"PINNED_DELETE_OWNER_INVALID_${logical}\" 66","149:    safe_generation \"$logical\" \"$generation\" || die \"PINNED_DELETE_GENERATION_INVALID_${logical}\" 66","150:    [[ \"$state_file\" == \"$STATE_ROOT/$logical/$generation.json\" ]] || die \"PINNED_DELETE_STATE_PATH_INVALID_${logical}\" 66","151:    [[ \"$state_sha\" =~ ^[0-9a-f]{64}$ ]] || die \"PINNED_DELETE_STATE_SHA_INVALID_${logical}\" 66","154:  (( count > 0 )) || die PINNED_DELETE_FILE_EMPTY 66","155:  [[ \"$(LC_ALL=C sort \"$f\" | uniq | wc -l)\" == \"$count\" ]] || die PINNED_DELETE_FILE_DUPLICATE 66","157:pinned_tx_identity(){","158:  local f=\"$1\" logical generation state_sha status owner expected_path","159:  [[ -s \"$f\" && ! -L \"$f\" ]] || die \"PINNED_TX_FILE_UNSAFE_$f\" 66","160:  jq -e --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" '.schema==\"homelab-backup-retention-transaction-v1\" and .task_id==$task and .change_id==$change and (.logical_id|type)==\"string\" and (.generation|type)==\"string\" and (.state_sha256|type)==\"string\" and (.status==\"PLANNED\" or .status==\"B_DELETED\" or .status==\"DONE\")' \"$f\" >/dev/null || die \"PINNED_TX_CONTENT_INVALID_$f\" 66","162:  owner=\"$(node_for_logical \"$logical\")\" || die \"PINNED_TX_OWNER_UNKNOWN_${logical}\" 66","163:  safe_generation \"$logical\" \"$generation\" || die \"PINNED_TX_GENERATION_UNSAFE_${logical}\" 66","164:  [[ \"$state_sha\" =~ ^[0-9a-f]{64}$ ]] || die \"PINNED_TX_STATE_SHA_INVALID_${logical}\" 66","166:  [[ \"$f\" == \"$expected_path\" ]] || die \"PINNED_TX_PATH_INVALID_${logical}_${generation}\" 66","167:  printf 'DELETE|logical=%s|generation=%s|owner=%s|state_file=%s|state_sha256=%s\\n' \"$logical\" \"$generation\" \"$owner\" \"$STATE_ROOT/$logical/$generation.json\" \"$state_sha\"","169:pinned_assert_transaction_tree(){","172:  [[ -d \"$root\" && ! -L \"$root\" ]] || die PINNED_TX_ROOT_UNSAFE 66","173:  [[ -z \"$(find \"$root\" -type l -print -quit)\" ]] || die PINNED_TX_SYMLINK_PRESENT 66","174:  [[ -z \"$(find \"$root\" -type f ! -name '*.json' -print -quit)\" ]] || die PINNED_TX_NONJSON_PRESENT 66","175:  while IFS= read -r f; do pinned_tx_identity \"$f\" >/dev/null; done < <(find \"$root\" -type f -name '*.json' -print | LC_ALL=C sort)","177:pinned_assert_protect_scope(){","183:    [[ \"$reason\" == GC_TRANSACTION_PLANNED || \"$reason\" == GC_TRANSACTION_B_DELETED ]] || die \"PINNED_PLAN_UNEXPECTED_PROTECT_${logical}_${generation}_${reason}\" 66","185:    [[ -s \"$txf\" && ! -L \"$txf\" ]] || die \"PINNED_PLAN_PROTECT_TX_MISSING_${logical}_${generation}\" 66","187:    [[ \"GC_TRANSACTION_$status\" == \"$reason\" ]] || die \"PINNED_PLAN_PROTECT_STATUS_MISMATCH_${logical}_${generation}\" 66","188:    grep -Fqx \"$(pinned_tx_identity \"$txf\")\" \"$HOMELAB_RETENTION_PINNED_DELETE_FILE\" || die \"PINNED_PLAN_PROTECT_NOT_PINNED_${logical}_${generation}\" 66","191:pinned_assert_delete_scope(){","193:  actual=\"$TMPROOT/pinned-delete-actual.$stage\"","194:  expected=\"$TMPROOT/pinned-delete-expected.$stage\"","195:  pinned_delete_assert_file","196:  pinned_assert_transaction_tree","197:  pinned_assert_protect_scope \"$plan_file\"","200:    while IFS= read -r f; do pinned_tx_identity \"$f\" >>\"$actual\"; done < <(find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -print | LC_ALL=C sort)","203:  LC_ALL=C sort \"$HOMELAB_RETENTION_PINNED_DELETE_FILE\" >\"$expected\"","205:  [[ \"$count_actual\" == \"$count_expected\" ]] || die \"PINNED_DELETE_SCOPE_COUNT_MISMATCH_${stage}\" 66","206:  [[ \"$(uniq \"$actual\" | wc -l)\" == \"$count_actual\" ]] || die \"PINNED_DELETE_SCOPE_DUPLICATE_${stage}\" 66","207:  cmp -s \"$actual\" \"$expected\" || die \"PINNED_DELETE_SCOPE_MISMATCH_${stage}\" 66","208:  printf 'PINNED_DELETE_SCOPE=OK|stage=%s|count=%s|sha256=%s\\n' \"$stage\" \"$count_expected\" \"$HOMELAB_RETENTION_PINNED_DELETE_SHA256\"","211:  printf 'CONTRACT|task=%s|policy=OWNER_AWARE_SAFE_GC_GFS|target_registry=CANONICAL|vm_keep=%s|app_keep=%s|control_keep=%s|min_delete_age_seconds=%s|gfs_critical=%s|gfs_weekly=%s|gfs_monthly=%s|gfs_completed_periods_only=YES|nonsealed=PROTECT|retired_preserve=PROTECT|unknown_namespace=PROTECT|owner_state_required=YES|both_clouds_required=YES|latest_restore_proven_required=YES|legacy_hb=PROTECT|legacy_pve01=PROTECT|legacy_control=PROTECT\\n' \"$TASK_ID\" \"$VM_KEEP\" \"$APP_KEEP\" \"$CONTROL_KEEP\" \"$MIN_DELETE_AGE_SECONDS\" \"$GFS_CRITICAL_COUNT\" \"$GFS_WEEKLY\" \"$GFS_MONTHLY\"","213:select_gfs(){","215:  : >\"$TMPROOT/$logical.gfs-eligible\"","216:  : >\"$TMPROOT/$logical.gfs-weekly\"","217:  : >\"$TMPROOT/$logical.gfs-monthly\"","218:  if ! HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" is-critical \"$logical\"; then return 0; fi","224:    printf '%s\\t%s\\n' \"$gen\" \"$sealed_at\" >>\"$TMPROOT/$logical.gfs-eligible\"","226:  out=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" select \"$logical\" <\"$TMPROOT/$logical.gfs-eligible\")\" || die \"GFS_SELECT_FAILED_$logical\" 66","230:      WEEKLY) printf '%s\\t%s\\n' \"$gen\" \"$bucket\" >>\"$TMPROOT/$logical.gfs-weekly\" ;;","231:      MONTHLY) printf '%s\\t%s\\n' \"$gen\" \"$bucket\" >>\"$TMPROOT/$logical.gfs-monthly\" ;;","232:      *) die \"GFS_SELECT_KIND_INVALID_$logical\" 66 ;;","240:  local now logical node keep lifecycle open_count file json schema lid gen status sha sealed_at epoch age idx tx gfs_bucket candidates=0 protected=0 kept=0 gfs_kept=0","244:    node=\"$(node_for_logical \"$logical\")\" || die \"OWNER_UNKNOWN_$logical\"","246:    if [[ \"$lifecycle\" == ACTIVE ]]; then keep=\"$(keep_for \"$logical\")\" || die \"KEEP_POLICY_UNKNOWN_$logical\"; else keep=0; fi","262:        printf 'PROTECT|logical=%s|generation=%s|owner=%s|reason=INCOMPLETE_%s\\n' \"$logical\" \"$gen\" \"$node\" \"$status\"","266:      strict_state \"$json\" \"$logical\" \"$gen\" || { printf 'PROTECT|logical=%s|generation=%s|owner=%s|reason=SEALED_STATE_NOT_STRICT\\n' \"$logical\" \"$gen\" \"$node\"; protected=$((protected+1)); continue; }","268:      epoch=\"$(date -u -d \"$sealed_at\" +%s 2>\"$TMPROOT/date.err\")\" || { printf 'PROTECT|logical=%s|generation=%s|owner=%s|reason=SEALED_TIME_INVALID\\n' \"$logical\" \"$gen\" \"$node\"; protected=$((protected+1)); continue; }","273:      while IFS=$'\\t' read -r gen file sha age node sealed_at; do printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=LOGICAL_HAS_INCOMPLETE\\n' \"$logical\" \"$gen\" \"$node\"; kept=$((kept+1)); done < <(LC_ALL=C sort -r \"$TMPROOT/$logical.rows\")","277:      while IFS=$'\\t' read -r gen file sha age node sealed_at; do printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=RETIRED_PRESERVE\\n' \"$logical\" \"$gen\" \"$node\"; kept=$((kept+1)); done < <(LC_ALL=C sort -r \"$TMPROOT/$logical.rows\")","281:      while IFS=$'\\t' read -r gen file sha age node sealed_at; do printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=LATEST_RESTORE_NOT_PROVEN\\n' \"$logical\" \"$gen\" \"$node\"; kept=$((kept+1)); done < <(LC_ALL=C sort -r \"$TMPROOT/$logical.rows\")","284:    select_gfs \"$logical\"","290:        printf 'RETIRED|logical=%s|generation=%s|owner=%s|reason=GC_ALREADY_DONE\\n' \"$logical\" \"$gen\" \"$node\"","292:        printf 'PROTECT|logical=%s|generation=%s|owner=%s|reason=GC_TRANSACTION_%s\\n' \"$logical\" \"$gen\" \"$node\" \"$tx\"; protected=$((protected+1))","294:        printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=POLICY_NEWEST_%s\\n' \"$logical\" \"$gen\" \"$node\" \"$keep\"; kept=$((kept+1))","296:        printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=MIN_AGE|age_seconds=%s\\n' \"$logical\" \"$gen\" \"$node\" \"$age\"; kept=$((kept+1))","298:        printf 'PROTECT|logical=%s|generation=%s|owner=%s|reason=REMOTE_SET_MISMATCH\\n' \"$logical\" \"$gen\" \"$node\"; protected=$((protected+1))","299:      elif gfs_bucket=\"$(awk -F'\\t' -v g=\"$gen\" '$1==g{print $2; exit}' \"$TMPROOT/$logical.gfs-weekly\")\" && [[ -n \"$gfs_bucket\" ]]; then","300:        printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=GFS_WEEKLY|bucket=%s\\n' \"$logical\" \"$gen\" \"$node\" \"$gfs_bucket\"; kept=$((kept+1)); gfs_kept=$((gfs_kept+1))","301:      elif gfs_bucket=\"$(awk -F'\\t' -v g=\"$gen\" '$1==g{print $2; exit}' \"$TMPROOT/$logical.gfs-monthly\")\" && [[ -n \"$gfs_bucket\" ]]; then","302:        printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=GFS_MONTHLY|bucket=%s\\n' \"$logical\" \"$gen\" \"$node\" \"$gfs_bucket\"; kept=$((kept+1)); gfs_kept=$((gfs_kept+1))","304:        printf 'DELETE|logical=%s|generation=%s|owner=%s|state_file=%s|state_sha256=%s|age_seconds=%s\\n' \"$logical\" \"$gen\" \"$node\" \"$file\" \"$sha\" \"$age\"","312:  printf 'PLAN_SUMMARY|delete_candidates=%s|protected=%s|kept=%s|vm_keep=%s|app_keep=%s|control_keep=%s|gfs_kept=%s|gfs_critical=%s|gfs_weekly=%s|gfs_monthly=%s\\n' \"$candidates\" \"$protected\" \"$kept\" \"$VM_KEEP\" \"$APP_KEEP\" \"$CONTROL_KEEP\" \"$gfs_kept\" \"$GFS_CRITICAL_COUNT\" \"$GFS_WEEKLY\" \"$GFS_MONTHLY\"","329:  local line=\"$1\" logical generation owner state_file state_sha age json current_sha","332:  owner=\"$(awk -F'|' '{for(i=1;i<=NF;i++)if($i~/^owner=/){sub(/^owner=/,\"\",$i);print $i}}' <<<\"$line\")\"","336:  [[ \"$(node_for_logical \"$logical\")\" == \"$owner\" ]] || die \"APPLY_OWNER_CHANGED_${logical}_${generation}\"","339:  json=\"$(node_exec \"$owner\" cat \"$state_file\")\" || die \"APPLY_STATE_READ_${logical}\" 69","353:  printf 'GC_DELETE_DONE|logical=%s|generation=%s|owner=%s\\n' \"$logical\" \"$generation\" \"$owner\"","356:  local f logical generation state_sha status owner state_file json current_sha","365:    owner=\"$(node_for_logical \"$logical\")\" || die \"TX_OWNER_UNKNOWN_$logical\"","368:    json=\"$(node_exec \"$owner\" cat \"$state_file\")\" || die \"TX_STATE_READ_${logical}\" 69","410:  local plan_file=\"$1\" canary_proven=\"$2\" deleted=\"$3\" candidates protected kept gfs_kept hist=false tmp","414:  gfs_kept=\"$(awk -F'[=|]' '/^PLAN_SUMMARY\\|/{for(i=1;i<=NF;i++)if($i==\"gfs_kept\"){print $(i+1)}}' \"$plan_file\" | tail -n 1)\"","418:  jq -n --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --argjson canary \"$canary_proven\" --argjson hist \"$hist\" --argjson candidates \"${candidates:-0}\" --argjson protected \"${protected:-0}\" --argjson kept \"${kept:-0}\" --argjson gfs_kept \"${gfs_kept:-0}\" --argjson deleted \"$deleted\" --argjson gfs_critical \"$GFS_CRITICAL_COUNT\" --argjson gfs_weekly \"$GFS_WEEKLY\" --argjson gfs_monthly \"$GFS_MONTHLY\" '{schema:\"homelab-backup-retention-v1\",status:\"OK\",task_id:$task,change_id:$change,updated_at_utc:$at,policy:{vm_keep:2,app_keep:7,control_keep:4,min_delete_age_seconds:86400,gfs:{enabled:true,critical_vm_count:$gfs_critical,weekly_completed_buckets:$gfs_weekly,monthly_completed_buckets:$gfs_monthly,completed_periods_only:true}},owner_aware_gc_proven:true,destructive_canary_proven:$canary,historical_gc_proven:$hist,last_plan:{delete_candidates:$candidates,protected:$protected,kept:$kept,gfs_kept:$gfs_kept},last_run_deleted:$deleted,legacy_namespaces_protected:[\"hb\",\"pve01\",\"control-bootstrap\"],provider_worm_proven:false}' >\"$tmp\"","442:  if pinned_delete_contract_enabled; then pinned_assert_delete_scope \"$p1\" PRELOCK; fi","445:  if pinned_delete_contract_enabled; then plan >\"$TMPROOT/pinned-pre-resume-plan.txt\"; pinned_assert_delete_scope \"$TMPROOT/pinned-pre-resume-plan.txt\" LOCKED_PRE_RESUME; fi","448:  if pinned_delete_contract_enabled; then pinned_assert_delete_scope \"$p2\" LOCKED_POST_RESUME; fi"],"path":"/usr/local/sbin/homelab-backup-retention-gc","sha256":"8adb6c10f5911f39c53775b89eac3cd1823f1ed34b4caccade5279c213e998a3"}
REPO_VM200_POLICY_HITS={"lines":[],"rc":1,"stderr_sha256":null}
===== AUDIT_SCHEDULER_EXPECTATION_RCA =====
AUDIT_RUNTIME_FOCUS={"bytes":39160,"exists":true,"matches":["18:VM_QUEUE=/usr/local/sbin/homelab-vm-cloud-quorum-queue","19:APP_QUEUE=/usr/local/sbin/homelab-app-cloud-quorum-queue","22:CONTROL_BACKUP=/usr/local/sbin/homelab-control-backup","24:CONTROL_TIMER=/etc/systemd/system/homelab-control-backup.timer","25:CONTROL_SERVICE=/etc/systemd/system/homelab-control-backup.service","28:VM_TIMER=/etc/systemd/system/homelab-vm-cloud-quorum-queue.timer","29:APP_TIMER=/etc/systemd/system/homelab-app-cloud-quorum-queue.timer","30:VM_SERVICE=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service","31:APP_SERVICE=/etc/systemd/system/homelab-app-cloud-quorum-queue.service","110:  local label=\"$1\" file=\"$2\" expected=\"$3\" actual","111:  if [[ -f \"$file\" && ! -L \"$file\" ]] && actual=\"$(sha256sum \"$file\" 2>&1 | awk 'NR==1{print $1}')\" && [[ \"$actual\" == \"$expected\" ]]; then","120:  local node=\"$1\" label=\"$2\" file=\"$3\" expected=\"$4\" out actual","122:  if [[ \"$actual\" == \"$expected\" ]]; then","147:expected_vm_count=\"${#VMIDS[@]}\"; expected_app_count=\"${#APPS[@]}\"","148:[[ \"$expected_vm_count\" == 11 && \"$expected_app_count\" == 4 ]] || die TARGET_REGISTRY_COUNT_INVALID 66","190:printf 'TARGET_REGISTRY|status=OK|path=%s|sha256=%s|vm_active=%s|app_active=%s\\n' \"${HOMELAB_BACKUP_TARGET_REGISTRY:-/etc/homelab-backup/targets.json}\" \"$(sha256sum \"${HOMELAB_BACKUP_TARGET_REGISTRY:-/etc/homelab-backup/targets.json}\"|awk '{print $1}')\" \"$expected_vm_count\" \"$expected_app_count\"","403:while IFS=$'\\t' read -r unit expected_state; do","407:  case \"$expected_state\" in","408:    ENABLED_ACTIVE)","415:      if [[ \"$load_state\" == not-found && \"$active_state\" == inactive ]]; then status=EXPECTED_ABSENT; scheduler_ok=$((scheduler_ok+1)); else status=BAD; infra_bad=$((infra_bad+1)); fi","419:  printf 'SYSTEMD_EXPECTATION|unit=%s|expected=%s|status=%s|load_state=%s|unit_file_state=%s|active_state=%s\\n' \"$unit\" \"$expected_state\" \"$status\" \"$load_state\" \"$unit_file_state\" \"$active_state\"","422:for lock in /run/homelab-backup-heavy.lock /run/lock/homelab-vm-cloud-quorum-queue.lock /run/lock/homelab-app-cloud-quorum-queue.lock /run/lock/homelab-backup-dispatcher.lock; do","509:if (( vm_seen != expected_vm_count || app_seen != expected_app_count )); then die COVERAGE_COUNT_INVALID; fi"],"path":"/usr/local/sbin/homelab-backup-audit","sha256":"1474c9ce58bbc1b3ae7437261d1bdb923232926fb2943c7c830253f8b284724c"}
UNIT_STATE={"props":{"ActiveState":"active","LastTriggerUSec":"Mon 2026-09-21 03:47:58 MSK","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","UnitFileState":"enabled"},"rc":0,"unit":"homelab-backup-v2.timer"}
UNIT_STATE={"props":{"ActiveState":"inactive","LoadState":"loaded","Result":"success","SubState":"dead","UnitFileState":"static"},"rc":0,"unit":"homelab-backup-v2.service"}
UNIT_STATE={"props":{"ActiveState":"active","LastTriggerUSec":"Mon 2026-09-21 04:06:11 MSK","LoadState":"loaded","NextElapseUSecRealtime":"Mon 2026-09-21 04:36:17 MSK","Result":"success","SubState":"waiting","UnitFileState":"enabled"},"rc":0,"unit":"homelab-backup-audit.timer"}
UNIT_STATE={"props":{"ActiveState":"active","LastTriggerUSec":"Mon 2026-09-21 03:22:42 MSK","LoadState":"loaded","NextElapseUSecRealtime":"Tue 2026-09-22 03:32:10 MSK","Result":"success","SubState":"waiting","UnitFileState":"enabled"},"rc":0,"unit":"homelab-backup-retention-gc.timer"}
UNIT_STATE={"props":{"ActiveState":"inactive","LastTriggerUSec":"","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"dead","UnitFileState":"disabled"},"rc":0,"unit":"homelab-vm-cloud-quorum-queue.timer"}
UNIT_STATE={"props":{"ActiveState":"inactive","LastTriggerUSec":"","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"dead","UnitFileState":"disabled"},"rc":0,"unit":"homelab-app-cloud-quorum-queue.timer"}
UNIT_STATE={"props":{"ActiveState":"inactive","LastTriggerUSec":"","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"dead","UnitFileState":"disabled"},"rc":0,"unit":"homelab-control-backup.timer"}
===== LOCK_AND_RUNNING_PROCESS_CHECK =====
RELEVANT_RUNNING_PROCESSES=[]
NO_MUTATION=true
NO_GIT_FETCH=true
NO_SERVICE_ACTION=true
NO_BACKUP_STARTED=true
NO_RESTORE_STARTED=true
NO_RETENTION_RUN_STARTED=true
NO_DELETE=true
NO_MAIL_SENT=true
NO_WORKTREE_CREATED=true
RCA_DURATION_SEC=0.679
TASK_RESULT=PASS_HOMELAB_BACKUP_V2_FAILURE_RCA_READONLY
HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-V2-FAILURE-RCA-READONLY-1004R1","rollback_restored":null,"rollback_started":false,"status":"OK","version":1}
WORKERS2_BACKUP_V2_FAILURE_RCA_END=1

OUTPUT_END
CHAT_OUTPUT_END
