homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity
Files
15833bfde07b318ed091667a964ecbca770190e4
homelab-public-context/runtime/latest.json
T

39 lines
70 KiB
JSON
Raw Blame History

{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "SUPPORT-260921-HOMELAB-BACKUP-V2-FAILURE-RCA-READONLY-1004R1",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "read-only",
"component": "homelab-backup-v2-failure-rca",
"started_at_utc": "2026-09-21T01:11:46Z",
"finished_at_utc": "2026-09-21T01:11:47Z",
"reference_register_checked": true,
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
"error_register_checked": true,
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
"command_sha256": "04cc92e6fe63247f64d16ce4f140a998a0207420bcdfd5c4a0b211896b71a66a",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
"change_declared": false,
"result_contract_valid": true,
"result_contract_status": null,
"result_contract_error": null,
"command_rc": 0,
"changes_made": false,
"rollback_started": false,
"rollback_restored": null,
"mutation_outcome": "NO_MUTATION",
"sanitized": true,
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "1b5b0eefe46c881abf1271c36e7f983e52db25c0f823f7be9c8d8f78425c5e2d",
"sanitized_output_sha256": "1b5b0eefe46c881abf1271c36e7f983e52db25c0f823f7be9c8d8f78425c5e2d",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "WORKERS2_BACKUP_V2_FAILURE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-V2-FAILURE-RCA-READONLY-1004R1\nMODE=read-only\nCOMPONENT=homelab-backup-v2-failure-rca\nHOST=pve01\nUID=0\nTIME_UTC=2026-09-21T01:11:46Z\nMUTATION_BOUNDARY=NO_FETCH_NO_RESET_NO_SERVICE_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL\n===== LIVE_GITEA_AND_CANONICAL =====\nLOCAL_HEAD={\"rc\":0,\"stderr_sha256\":null,\"stdout\":\"d5e2ef5639398a762283e47143c17b494f8ed579\"}\nLOCAL_MAIN={\"rc\":0,\"stderr_sha256\":null,\"stdout\":\"d5e2ef5639398a762283e47143c17b494f8ed579\"}\nTRACKING_ORIGIN_MAIN={\"rc\":0,\"stderr_sha256\":null,\"stdout\":\"d5e2ef5639398a762283e47143c17b494f8ed579\"}\nGIT_STATUS={\"rc\":0,\"stderr_sha256\":null,\"stdout\":\"## main...origin/main\"}\nGITEA_LIVE_MAIN={\"rc\":0,\"stderr_sha256\":null,\"stdout\":\"326622a4864fa9b5f43b69981733236bb42c4d7b\\trefs/heads/main\"}\nWORKTREE_CONCURRENCY_SUMMARY={\"count\":69,\"recent_backup_related\":[{\"branch\":\"refs/heads/cr-2026-0004-backup-tail-closure\",\"head\":\"bf80410f592f8eb9eaa80695f609870fe3fb85fd\",\"path\":\"/root/homelab-ops-cr-2026-0004\"},{\"branch\":\"refs/heads/cr-2026-0006-cluster-cloud-quorum\",\"head\":\"16e84298035f5e4ec7e2b7d39967ad5e40ea3d4b\",\"path\":\"/root/homelab-ops-cr-2026-0006\"},{\"branch\":\"refs/heads/cr-2026-0020-app-cloud-quorum\",\"head\":\"f556984e410ed819b4cb1e00635dceb43f0ffc5d\",\"path\":\"/root/homelab-ops-cr-2026-0020\"},{\"branch\":\"refs/heads/cr-2026-0021-vm-backup-freshness\",\"head\":\"3c92036f3000189493fc47557c4cb03e7a412c82\",\"path\":\"/root/homelab-ops-cr-2026-0021\"},{\"branch\":\"refs/heads/cr-2026-0022-app-cloud-quorum-freshness\",\"head\":\"a497e0f15f1c8d11c3603a4cf5a6a65654cae6fb\",\"path\":\"/root/homelab-ops-cr-2026-0022\"},{\"branch\":\"refs/heads/cr-2026-0024-hbarhub-cloud-quorum\",\"head\":\"332e8d3f8d2742a05a8d73637cac6a2defc3a825\",\"path\":\"/root/homelab-ops-cr-2026-0024\"},{\"branch\":\"refs/heads/cr-2026-0025-scrutiny-cloud-quorum\",\"head\":\"73890efb9e99bf02a09daa0e547ab36b48c83b30\",\"path\":\"/root/homelab-ops-cr-2026-0025\"},{\"branch\":\"refs/heads/cr-2026-0026-backup-scheduler-rpo-hardening\",\"head\":\"ebc08fe5d6beef3cc927bc7d23f6ab37b35d947d\",\"path\":\"/root/homelab-ops-cr-2026-0026\"},{\"branch\":\"refs/heads/cr-2026-0027-backup-lock-fd-hardening\",\"head\":\"d5a14cbaef47db39f17a94e248e183f0916bdab1\",\"path\":\"/root/homelab-ops-cr-2026-0027\"},{\"branch\":\"refs/heads/cr-2026-0028-backup-scheduler-fairness-granularity\",\"head\":\"6e3707e90181318be3e8616b14ee553968c485dd\",\"path\":\"/root/homelab-ops-cr-2026-0028\"},{\"branch\":\"refs/heads/cr-2026-0029-control-bootstrap-cloud-quorum\",\"head\":\"dd81175a24ff64048b36d6ec08dcd3addfc681e9\",\"path\":\"/root/homelab-ops-cr-2026-0029\"},{\"branch\":\"refs/heads/cr-2026-0031-vm-restore-evidence-hardening\",\"head\":\"f8941f01f41d03b3368a33df868f7ea11311eb6a\",\"path\":\"/root/homelab-ops-cr-2026-0031\"},{\"branch\":\"refs/heads/cr-2026-0032-backup-unified-monitoring\",\"head\":\"8ad7fcc4537c48de47daac086087d247886c58d9\",\"path\":\"/root/homelab-ops-cr-2026-0032\"},{\"branch\":\"refs/heads/cr-2026-0033-backup-retention-safe-gc\",\"head\":\"42fd895f421ebc177006c5eb4173f7b4f40ad47f\",\"path\":\"/root/homelab-ops-cr-2026-0033\"},{\"branch\":\"refs/heads/cr-2026-0046-reports-monitor-canonical-backup-supervision\",\"head\":\"6fc45798c21bd66c561c14059e14f09b7d0a8d63\",\"path\":\"/root/homelab-ops-cr-2026-0046\"},{\"branch\":\"refs/heads/cr-2026-0049-pve03-r8152-lifecycle-repair\",\"head\":\"801579bd301ad0b28f462f462919660b38041344\",\"path\":\"/root/homelab-ops-cr-2026-0049\"},{\"branch\":\"refs/heads/cr-2026-0050-backup-matrix-residue-retirement\",\"head\":\"6c1165dccfb6e84b571f0caac6773fd427bf800d\",\"path\":\"/root/homelab-ops-cr-2026-0050\"},{\"branch\":\"refs/heads/cr-2026-0054-backup-control-coverage-hardening\",\"head\":\"3f02810a16b711700a9da87189c63e38b1fa324b\",\"path\":\"/root/homelab-ops-cr-2026-0054\"},{\"branch\":\"refs/heads/cr-2026-0055-backup-retention-plan-stability-hotfix\",\"head\":\"902126179efd488e20938a091dcee36109829b86\",\"path\":\"/root/homelab-ops-cr-2026-0055\"},{\"branch\":\"refs/heads/cr-2026-0056-backup-retention-transaction-resume-hotfix\",\"head\":\"b69b63f531fb71eb95ab147bbfce57591f90e0a4\",\"path\":\"/root/homelab-ops-cr-2026-0056\"},{\"branch\":\"refs/heads/cr-2026-0058-legacy-backup-target-retirement\",\"head\":\"4b427f5f0406cab6757cf8889fd479a62d218261\",\"path\":\"/root/homelab-ops-cr-2026-0058\"},{\"branch\":\"refs/heads/cr-2026-0060-backup-retention-remote-list-failclosed-hotfix\",\"head\":\"1a0e4ce27b94bbfbf7b60829974a728b5eba3ad2\",\"path\":\"/root/homelab-ops-cr-2026-0060\"},{\"branch\":\"refs/heads/cr-2026-0064-backup-retention-pinned-scheduled-scope-hotfix\",\"head\":\"d3a11bde0e73ddfd7049693782d44100c0c1130e\",\"path\":\"/root/homelab-ops-cr-2026-0064\"},{\"branch\":\"refs/heads/cr-2026-0065-backup-retention-dynamic-pinned-prune-mailru-cleanup\",\"head\":\"2c2f378aec7dc4ac23c633a31f738a850fface56\",\"path\":\"/root/homelab-ops-cr-2026-0065\"},{\"branch\":\"refs/heads/cr-2026-0066-legacy-backup-target-retirement\",\"head\":\"bcf018f263d9fc607eddd1796e5ef306870c6b7b\",\"path\":\"/root/homelab-ops-cr-2026-0066\"},{\"branch\":\"refs/heads/cr-2026-0070-backup-target-lifecycle-registry\",\"head\":\"dbf8266d5f2bc4c87024ea8f0eb5f0e8922c7519\",\"path\":\"/root/homelab-ops-cr-2026-0070\"},{\"branch\":\"refs/heads/cr-2026-0071-backup-observability\",\"head\":\"07cb58a803022b5538e30b201380e215a5490c84\",\"path\":\"/root/homelab-ops-cr-2026-0071\"},{\"branch\":\"refs/heads/cr-2026-0072-gfs-retention\",\"head\":\"34d76b1cfd4071d94290edac50a31a8c0d4c11c8\",\"path\":\"/root/homelab-ops-cr-2026-0072\"},{\"branch\":\"refs/heads/cr-2026-0074-backup-delete-evidence-hardening\",\"head\":\"804a2391818f842d768edf434ffeaaf129e576f5\",\"path\":\"/root/homelab-ops-cr-2026-0074\"},{\"branch\":\"refs/heads/cr-2026-0018-backup-cloud-quorum-foundation\",\"head\":\"b25e80a67fb97ff6c0aa272b25675b9f7c82fb96\",\"path\":\"/srv/homelab-ops-cr-2026-0018\"}]}\n===== CURRENT_INVENTORY_FOCUS =====\nINVENTORY_FOCUS=[{\"name\":\"unbound1\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"lxc\",\"vmid\":112},{\"name\":\"homelab-ops-worker\",\"node\":\"pve03\",\"status\":\"running\",\"type\":\"qemu\",\"vmid\":190},{\"name\":\"newfi-staging\",\"node\":\"pve03\",\"status\":\"running\",\"type\":\"qemu\",\"vmid\":211},{\"name\":\"edge-cold-standby\",\"node\":\"pve02\",\"status\":\"stopped\",\"type\":\"qemu\",\"vmid\":9130}]\n===== BACKUP_V2_RUNTIME_SOURCE =====\nSOURCE_FOCUS={\"bytes\":1656,\"exists\":true,\"matches\":[\"5:EXPECTED={'fresh_lxc_helper.py': 'dbb3e6dbb71b10797df736431d9501a752a0f8c439163c9a0bddb7f76747f011', 'producer_exec_helper.py': 'b3b97099bcd7126670f866e81faae6a42a96d869d9989e6a67f6ea6c871c90b3', 'lxc_adapter.py': '3fa7d194010d76f7c8bc95035ae8876262e9337903e310f45f45df88c266dc8d', 'qemu_adapter.py': '1906db42bd34bbe85eb46fd9797b3572d46e0a41c29d1c29be5f9df5e448f15b', 'scheduler_adapter.py': 'a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96', 'entry504.py': 'ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793'}\"],\"path\":\"/usr/local/sbin/homelab-backup-v2\",\"sha256\":\"5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\"}\nSOURCE_FOCUS={\"bytes\":309,\"exists\":true,\"matches\":[\"2:Description=Homelab Backup V2 manifest-driven scheduler\",\"3:After=network-online.target\",\"4:Wants=network-online.target\",\"11:ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\"],\"path\":\"/etc/systemd/system/homelab-backup-v2.service\",\"sha256\":\"1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565\"}\nSOURCE_FOCUS={\"bytes\":192,\"exists\":true,\"matches\":[\"2:Description=Homelab Backup V2 scheduler timer\",\"11:WantedBy=timers.target\"],\"path\":\"/etc/systemd/system/homelab-backup-v2.timer\",\"sha256\":\"5469057602452b303a01ccb8057085c1b0e447973e5ed198cebf7dd5daf58e9d\"}\nBACKUP_V2_REFERENCED_EXISTING_PATHS=[]\n===== BACKUP_V2_FAILURE_JOURNAL =====\nBACKUP_V2_JOURNAL_FOCUS={\"lines\":[\"2026-09-20T15:19:49.057775+03:00 pve01 homelab-backup-v2[2298848]: SCHEDULE_TARGET=xf-hkmods\",\"2026-09-20T15:19:49.057775+03:00 pve01 homelab-backup-v2[2298848]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-20T15:19:59.382893+03:00 pve01 homelab-backup-v2[3306237]: SCHEDULE_TARGET=guest-pve01-lxc-112\",\"2026-09-20T15:19:59.382893+03:00 pve01 homelab-backup-v2[3306237]: SCHEDULE_RESULT=FAIL\",\"2026-09-20T15:19:59.382893+03:00 pve01 homelab-backup-v2[3306237]: SCHEDULE_REASON=CT112_PIPELINE_FAILED\",\"2026-09-20T15:19:59.392549+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\",\"2026-09-20T15:19:59.392691+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-20T15:56:31.790799+03:00 pve01 homelab-backup-v2[3417390]: SCHEDULE_TARGET=us-netbird\",\"2026-09-20T15:56:31.790799+03:00 pve01 homelab-backup-v2[3417390]: SCHEDULE_RESULT=FAIL\",\"2026-09-20T15:56:31.790799+03:00 pve01 homelab-backup-v2[3417390]: SCHEDULE_REASON=PIPE_FAILED\",\"2026-09-20T15:56:31.797372+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\",\"2026-09-20T15:56:31.797541+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-20T16:31:22.568658+03:00 pve01 homelab-backup-v2[3535131]: SCHEDULE_TARGET=guest-pve01-lxc-112\",\"2026-09-20T16:31:22.568658+03:00 pve01 homelab-backup-v2[3535131]: SCHEDULE_RESULT=FAIL\",\"2026-09-20T16:31:22.568658+03:00 pve01 homelab-backup-v2[3535131]: SCHEDULE_REASON=CT112_PIPELINE_FAILED\",\"2026-09-20T16:31:22.578409+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\",\"2026-09-20T16:31:22.578669+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-20T17:05:15.846826+03:00 pve01 homelab-backup-v2[3635207]: SCHEDULE_TARGET=md-relay\",\"2026-09-20T17:05:15.846826+03:00 pve01 homelab-backup-v2[3635207]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-20T17:41:42.675536+03:00 pve01 homelab-backup-v2[3751454]: SCHEDULE_TARGET=guest-pve01-lxc-112\",\"2026-09-20T17:41:42.675536+03:00 pve01 homelab-backup-v2[3751454]: SCHEDULE_RESULT=FAIL\",\"2026-09-20T17:41:42.675536+03:00 pve01 homelab-backup-v2[3751454]: SCHEDULE_REASON=CT112_PIPELINE_FAILED\",\"2026-09-20T17:41:42.681881+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\",\"2026-09-20T17:41:42.682116+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-20T18:16:37.584638+03:00 pve01 homelab-backup-v2[3852802]: SCHEDULE_TARGET=spb-ingress\",\"2026-09-20T18:16:37.584638+03:00 pve01 homelab-backup-v2[3852802]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-20T18:55:11.885256+03:00 pve01 homelab-backup-v2[3975404]: SCHEDULE_TARGET=router-netcraze\",\"2026-09-20T18:55:11.885256+03:00 pve01 homelab-backup-v2[3975404]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-20T19:38:44.368912+03:00 pve01 homelab-backup-v2[4075290]: SCHEDULE_TARGET=xf-codevipe\",\"2026-09-20T19:38:44.368912+03:00 pve01 homelab-backup-v2[4075290]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-20T20:03:09.246625+03:00 pve01 homelab-backup-v2[5417]: SCHEDULE_TARGET=guest-pve01-lxc-112\",\"2026-09-20T20:03:09.246625+03:00 pve01 homelab-backup-v2[5417]: SCHEDULE_RESULT=FAIL\",\"2026-09-20T20:03:09.246625+03:00 pve01 homelab-backup-v2[5417]: SCHEDULE_REASON=CT112_PIPELINE_FAILED\",\"2026-09-20T20:03:09.252660+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\",\"2026-09-20T20:03:09.252827+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-20T20:50:52.995548+03:00 pve01 homelab-backup-v2[107015]: SCHEDULE_TARGET=xf-zakrutim\",\"2026-09-20T20:50:52.995548+03:00 pve01 homelab-backup-v2[107015]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-20T21:12:11.756667+03:00 pve01 vzdump[223483]: <root@pam> starting task UPID:pve01:000368FC:0BF1EB40:6AB021FB:vzdump:110:root@pam:\",\"2026-09-20T21:12:11.772892+03:00 pve01 vzdump[223484]: INFO: starting new backup job: vzdump 110 --ionice 7 --stdexcludes 0 --node pve01 --mode snapshot --zstd 1 --compress zstd --tmpdir /var/tmp/homelab-vzdump-v2-v2-ct110-20260920T181158Z-25703b1ef77f --lockwait 0 --prune-backups 'keep-all=1' --bwlimit 32768 --remove 0\",\"2026-09-20T21:12:11.773447+03:00 pve01 vzdump[223484]: INFO: Starting Backup of VM 110 (lxc)\",\"2026-09-20T21:14:15.597824+03:00 pve01 vzdump[223484]: INFO: Finished Backup of VM 110 (00:02:04)\",\"2026-09-20T21:14:15.601427+03:00 pve01 vzdump[223484]: INFO: Backup job finished successfully\",\"2026-09-20T21:14:15.629994+03:00 pve01 vzdump[223483]: <root@pam> end task UPID:pve01:000368FC:0BF1EB40:6AB021FB:vzdump:110:root@pam: OK\",\"2026-09-20T21:14:15.881089+03:00 pve01 homelab-backup-v2[223361]: SCHEDULE_TARGET=ct110\",\"2026-09-20T21:14:15.881089+03:00 pve01 homelab-backup-v2[223361]: LXC_EXECUTION_DIAG={\\\"archive_bytes\\\":541730358,\\\"exception_class\\\":null,\\\"exception_message_sha256\\\":null,\\\"executor\\\":\\\"local-proven-pve270\\\",\\\"helper_rc\\\":0,\\\"node\\\":\\\"pve01\\\",\\\"producer_rc\\\":0,\\\"stderr_bytes_observed\\\":1405,\\\"stderr_bytes_retained\\\":1405,\\\"stderr_stream_sha256\\\":\\\"52da6b194c8151f5e9372188032c9c6fa1b1a9135609736265a2a169a9c57adb\\\",\\\"stderr_truncated\\\":false}\",\"2026-09-20T21:15:20.610181+03:00 pve01 homelab-backup-v2[223361]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-20T21:51:38.078713+03:00 pve01 homelab-backup-v2[345333]: SCHEDULE_TARGET=control-bootstrap-v2\",\"2026-09-20T21:51:38.078713+03:00 pve01 homelab-backup-v2[345333]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-20T22:29:31.800106+03:00 pve01 homelab-backup-v2[459026]: SCHEDULE_TARGET=us-netbird\",\"2026-09-20T22:29:31.800106+03:00 pve01 homelab-backup-v2[459026]: SCHEDULE_RESULT=FAIL\",\"2026-09-20T22:29:31.800106+03:00 pve01 homelab-backup-v2[459026]: SCHEDULE_REASON=PIPE_FAILED\",\"2026-09-20T22:29:31.805836+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\",\"2026-09-20T22:29:31.805983+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-20T23:09:29.556447+03:00 pve01 homelab-backup-v2[593177]: SCHEDULE_TARGET=ct111\",\"2026-09-20T23:09:29.556447+03:00 pve01 homelab-backup-v2[593177]: LXC_EXECUTION_DIAG={\\\"executor\\\":\\\"remote-bounded-ssh-cwd-root\\\",\\\"node\\\":\\\"pve02\\\",\\\"producer_rc\\\":0,\\\"stderr_bytes_observed\\\":1035,\\\"stderr_bytes_retained\\\":1035,\\\"stderr_stream_sha256\\\":\\\"bce35788cf8d44be1cd8cf226e02ad4d65baf010cb8e8dfc4eb7a5abb32ced63\\\",\\\"stderr_truncated\\\":false}\",\"2026-09-20T23:10:21.129791+03:00 pve01 homelab-backup-v2[593177]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-20T23:38:33.285692+03:00 pve01 vzdump[681827]: <root@pam> starting task UPID:pve01:000A6783:0BFF5189:6AB04449:vzdump:210:root@pam:\",\"2026-09-20T23:38:33.296638+03:00 pve01 vzdump[681859]: INFO: starting new backup job: vzdump 210 --dumpdir /var/tmp/homelab-vzdump-v2-v2-vm210-20260920T203832Z-458da35ec027 --compress zstd --mode snapshot\",\"2026-09-20T23:38:33.301799+03:00 pve01 vzdump[681859]: INFO: Starting Backup of VM 210 (qemu)\",\"2026-09-20T23:38:47.314579+03:00 pve01 vzdump[681859]: INFO: Finished Backup of VM 210 (00:00:14)\",\"2026-09-20T23:38:47.318698+03:00 pve01 vzdump[681859]: INFO: Backup job finished successfully\",\"2026-09-20T23:38:47.335141+03:00 pve01 vzdump[681827]: <root@pam> end task UPID:pve01:000A6783:0BFF5189:6AB04449:vzdump:210:root@pam: OK\",\"2026-09-20T23:38:47.350575+03:00 pve01 homelab-backup-v2[681773]: SCHEDULE_TARGET=vm210\",\"2026-09-20T23:38:47.350575+03:00 pve01 homelab-backup-v2[681773]: QEMU_EXECUTION_DIAG={\\\"archive_bytes\\\":687647191,\\\"archive_selected\\\":true,\\\"node\\\":\\\"pve01\\\",\\\"producer_rc\\\":0,\\\"regular_file_count\\\":2,\\\"sidecar_count\\\":1,\\\"vmid\\\":210}\",\"2026-09-20T23:39:58.319976+03:00 pve01 homelab-backup-v2[681773]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T00:12:09.285479+03:00 pve01 homelab-backup-v2[782723]: SCHEDULE_TARGET=guest-pve01-lxc-112\",\"2026-09-21T00:12:09.285479+03:00 pve01 homelab-backup-v2[782723]: SCHEDULE_RESULT=FAIL\",\"2026-09-21T00:12:09.285479+03:00 pve01 homelab-backup-v2[782723]: SCHEDULE_REASON=CT112_PIPELINE_FAILED\",\"2026-09-21T00:12:09.292481+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\",\"2026-09-21T00:12:09.292801+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-21T00:50:11.899660+03:00 pve01 vzdump[915940]: <root@pam> starting task UPID:pve01:000DF9E5:0C05E0AE:6AB05513:vzdump:112:root@pam:\",\"2026-09-21T00:50:11.913977+03:00 pve01 vzdump[915941]: INFO: starting new backup job: vzdump 112 --prune-backups 'keep-all=1' --bwlimit 32768 --node pve01 --tmpdir /var/tmp/homelab-vzdump-v2-v2-ct112-20260920T214958Z-30e3291e1543 --stdexcludes 0 --zstd 1 --remove 0 --mode snapshot --compress zstd --ionice 7 --lockwait 0\",\"2026-09-21T00:50:11.914482+03:00 pve01 vzdump[915941]: INFO: Starting Backup of VM 112 (lxc)\",\"2026-09-21T00:50:35.059514+03:00 pve01 vzdump[915941]: INFO: Finished Backup of VM 112 (00:00:24)\",\"2026-09-21T00:50:35.063713+03:00 pve01 vzdump[915941]: INFO: Backup job finished successfully\",\"2026-09-21T00:50:35.079365+03:00 pve01 vzdump[915940]: <root@pam> end task UPID:pve01:000DF9E5:0C05E0AE:6AB05513:vzdump:112:root@pam: OK\",\"2026-09-21T00:50:35.131822+03:00 pve01 homelab-backup-v2[904735]: SCHEDULE_TARGET=ct112\",\"2026-09-21T00:50:35.131822+03:00 pve01 homelab-backup-v2[904735]: LXC_EXECUTION_DIAG={\\\"archive_bytes\\\":223936825,\\\"exception_class\\\":null,\\\"exception_message_sha256\\\":null,\\\"executor\\\":\\\"local-proven-pve270\\\",\\\"helper_rc\\\":0,\\\"node\\\":\\\"pve01\\\",\\\"producer_rc\\\":0,\\\"stderr_bytes_observed\\\":1408,\\\"stderr_bytes_retained\\\":1408,\\\"stderr_stream_sha256\\\":\\\"92e2c2144d82520ef7dbe70184d9317d94da5e39aff3b7fb3cf3cf74685e5b68\\\",\\\"stderr_truncated\\\":false}\",\"2026-09-21T00:51:16.588203+03:00 pve01 homelab-backup-v2[904735]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T01:26:45.553982+03:00 pve01 homelab-backup-v2[1018631]: SCHEDULE_TARGET=ct113\",\"2026-09-21T01:26:45.553982+03:00 pve01 homelab-backup-v2[1018631]: LXC_EXECUTION_DIAG={\\\"executor\\\":\\\"remote-bounded-ssh-cwd-root\\\",\\\"node\\\":\\\"pve02\\\",\\\"producer_rc\\\":0,\\\"stderr_bytes_observed\\\":1038,\\\"stderr_bytes_retained\\\":1038,\\\"stderr_stream_sha256\\\":\\\"5590f20ce6d625aae5e745852a46ffd5331ef67462164b084208650f60f43eeb\\\",\\\"stderr_truncated\\\":false}\",\"2026-09-21T01:27:21.219244+03:00 pve01 homelab-backup-v2[1018631]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T02:00:36.030805+03:00 pve01 vzdump[1131961]: <root@pam> starting task UPID:pve01:001145BA:0C0C52BB:6AB06594:vzdump:150:root@pam:\",\"2026-09-21T02:00:36.041445+03:00 pve01 vzdump[1131962]: INFO: starting new backup job: vzdump 150 --mode snapshot --compress zstd --dumpdir /var/tmp/homelab-vzdump-v2-v2-vm150-20260920T230035Z-e0880e47ea55\",\"2026-09-21T02:00:36.045251+03:00 pve01 vzdump[1131962]: INFO: Starting Backup of VM 150 (qemu)\",\"2026-09-21T02:00:53.219661+03:00 pve01 vzdump[1131962]: INFO: Finished Backup of VM 150 (00:00:17)\",\"2026-09-21T02:00:53.225223+03:00 pve01 vzdump[1131962]: INFO: Backup job finished successfully\",\"2026-09-21T02:00:53.241129+03:00 pve01 vzdump[1131961]: <root@pam> end task UPID:pve01:001145BA:0C0C52BB:6AB06594:vzdump:150:root@pam: OK\",\"2026-09-21T02:00:53.259679+03:00 pve01 homelab-backup-v2[1131900]: SCHEDULE_TARGET=vm150\",\"2026-09-21T02:00:53.259679+03:00 pve01 homelab-backup-v2[1131900]: QEMU_EXECUTION_DIAG={\\\"archive_bytes\\\":948768127,\\\"archive_selected\\\":true,\\\"node\\\":\\\"pve01\\\",\\\"producer_rc\\\":0,\\\"regular_file_count\\\":2,\\\"sidecar_count\\\":1,\\\"vmid\\\":150}\",\"2026-09-21T02:02:27.509220+03:00 pve01 homelab-backup-v2[1131900]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T02:37:59.001120+03:00 pve01 homelab-backup-v2[1256150]: SELFTEST_REPLACEMENT486=PASS\",\"2026-09-21T02:42:54.073760+03:00 pve01 homelab-backup-v2[1256150]: SCHEDULE_TARGET=vm130\",\"2026-09-21T02:42:54.073760+03:00 pve01 homelab-backup-v2[1256150]: QEMU_EXECUTION_DIAG={\\\"archive_bytes\\\":10948261509,\\\"archive_selected\\\":true,\\\"node\\\":\\\"pve03\\\",\\\"producer_rc\\\":0,\\\"regular_file_count\\\":2,\\\"sidecar_count\\\":1,\\\"vmid\\\":130}\",\"2026-09-21T02:57:13.496557+03:00 pve01 homelab-backup-v2[1256150]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T03:16:56.429153+03:00 pve01 homelab-backup-v2[1371340]: SCHEDULE_TARGET=vm160\",\"2026-09-21T03:16:56.429153+03:00 pve01 homelab-backup-v2[1371340]: QEMU_EXECUTION_DIAG={\\\"archive_bytes\\\":3755423628,\\\"archive_selected\\\":true,\\\"node\\\":\\\"pve02\\\",\\\"producer_rc\\\":0,\\\"regular_file_count\\\":2,\\\"sidecar_count\\\":1,\\\"vmid\\\":160}\",\"2026-09-21T03:21:40.026609+03:00 pve01 homelab-backup-v2[1371340]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T03:47:59.909526+03:00 pve01 vzdump[1480118]: <root@pam> starting task UPID:pve01:001695B7:0C1627DF:6AB07EBF:vzdump:171:root@pam:\",\"2026-09-21T03:47:59.921859+03:00 pve01 vzdump[1480119]: INFO: starting new backup job: vzdump 171 --dumpdir /var/tmp/homelab-vzdump-v2-v2-vm171-20260921T004759Z-537db1552014 --compress zstd --mode snapshot\",\"2026-09-21T03:47:59.925791+03:00 pve01 vzdump[1480119]: INFO: Starting Backup of VM 171 (qemu)\",\"2026-09-21T03:48:43.831277+03:00 pve01 vzdump[1480119]: INFO: Finished Backup of VM 171 (00:00:44)\",\"2026-09-21T03:48:43.836725+03:00 pve01 vzdump[1480119]: INFO: Backup job finished successfully\",\"2026-09-21T03:48:43.872532+03:00 pve01 vzdump[1480118]: <root@pam> end task UPID:pve01:001695B7:0C1627DF:6AB07EBF:vzdump:171:root@pam: OK\",\"2026-09-21T03:48:43.892527+03:00 pve01 homelab-backup-v2[1479892]: SCHEDULE_TARGET=vm171\",\"2026-09-21T03:48:43.892527+03:00 pve01 homelab-backup-v2[1479892]: QEMU_EXECUTION_DIAG={\\\"archive_bytes\\\":2051667128,\\\"archive_selected\\\":true,\\\"node\\\":\\\"pve01\\\",\\\"producer_rc\\\":0,\\\"regular_file_count\\\":2,\\\"sidecar_count\\\":1,\\\"vmid\\\":171}\",\"2026-09-21T03:51:17.662338+03:00 pve01 homelab-backup-v2[1479892]: SCHEDULE_RESULT=SUCCESS\"],\"matched_count\":107,\"rc\":0,\"stderr_sha256\":null}\n===== RECENT_BACKUP_V2_STATE_FILES =====\nBACKUP_V2_STATE_ROOTS=[\"/var/lib/homelab-backup-v2\"]\nRECENT_STATE_FILE={\"age_sec\":1229,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/vm171.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-vm171-20260921T004759Z-537db1552014\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789951877,\\\"last_success_epoch\\\":1789951877,\\\"logical_id\\\":\\\"vm171\\\",\\\"manifest_sha256\\\":\\\"bcc1c90b61c53d3e58182f571f9cd1a6dd6d612469d9f070505d82b128bdd7a2\\\",\\\"payload_sha256\\\":\\\"41520b284fdec39732e9bff8881a0ed19f543859e280755b1bbc838e52c68649\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-01-crypt\\\",\\\"physical\\\":\\\"pve03-mail-01\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-21T00:51:17Z\\\"}\"],\"sha256\":\"46b5f03744cd5b4c6f01a0c8e260846f538354d788b8a3527cc56db2aec5d6c0\"}\nRECENT_STATE_FILE={\"age_sec\":3006,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/vm160.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-vm160-20260921T001432Z-31df1acfee7a\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789950100,\\\"last_success_epoch\\\":1789950100,\\\"logical_id\\\":\\\"vm160\\\",\\\"manifest_sha256\\\":\\\"658eb4a8dd4060d58af8afb72bb33b0315020eb3837e36d8f5866f92ca4bb18c\\\",\\\"payload_sha256\\\":\\\"d4bbc19aef3d2e96252c2fd7b4ca7b93fa9d83f872fdff8fcca9933227d5b1b3\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"},{\\\"crypt\\\":\\\"pve01-mail-04-crypt\\\",\\\"physical\\\":\\\"pve01-mail-04\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-21T00:21:40Z\\\"}\"],\"sha256\":\"1219e114d204fefd63adcd3e0f82c90b80cb714b8f0253e86dd0e406f4f4abd2\"}\nRECENT_STATE_FILE={\"age_sec\":4473,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/vm130.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-vm130-20260920T233759Z-c9518c8be2c3\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789948633,\\\"last_success_epoch\\\":1789948633,\\\"logical_id\\\":\\\"vm130\\\",\\\"manifest_sha256\\\":\\\"93b2eec0d7750145384c2429e5f5a24abb0034acc086f3514375a90d637219d7\\\",\\\"payload_sha256\\\":\\\"60f90f8352115d5df0ac6af07bf07e7a3976f3c61e83732d0d7ce071a6414faa\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve03-mail-03-crypt\\\",\\\"physical\\\":\\\"pve03-mail-03\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T23:57:13Z\\\"}\"],\"sha256\":\"f4b2f4f2885c61740f89443d3e792fe2d57c6ff1e87daf27fbe421e0988b7e90\"}\nRECENT_STATE_FILE={\"age_sec\":7759,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/vm150.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-vm150-20260920T230035Z-e0880e47ea55\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789945347,\\\"last_success_epoch\\\":1789945347,\\\"logical_id\\\":\\\"vm150\\\",\\\"manifest_sha256\\\":\\\"7cc63a5fb78f8bc1e73247503b1896e3a31a99022fd9f43895c3de653ee7aab1\\\",\\\"payload_sha256\\\":\\\"3eaf82326949ec7d849e374379e7ea981d859343ce0aac471bfcff952ee5f20e\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-03-crypt\\\",\\\"physical\\\":\\\"pve03-mail-03\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T23:02:27Z\\\"}\"],\"sha256\":\"4ad4046a48ac815fae689998a8ea96d6db66dbb5b601182813815c33d9fb535e\"}\nRECENT_STATE_FILE={\"age_sec\":9865,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/ct113.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-ct113-20260920T222532Z-4f4a608def82\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789943241,\\\"last_success_epoch\\\":1789943241,\\\"logical_id\\\":\\\"ct113\\\",\\\"manifest_sha256\\\":\\\"a721c4dfcd8031fd52b7281dcc2e16f0de5874e0b7e3e3eb25791b4c64319e1a\\\",\\\"payload_sha256\\\":\\\"7a488bad3e70b1e4cc5246dd8d78d5bf61cb79ad92ce8f7d573eb387fcb002bd\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-03-crypt\\\",\\\"physical\\\":\\\"pve03-mail-03\\\"},{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T22:27:21Z\\\"}\"],\"sha256\":\"c78f182a0b0ac42fc1daa40ed9f89c0da1d54710b85665af9ac1f279250c9aa3\"}\nRECENT_STATE_FILE={\"age_sec\":12030,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/ct112.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-ct112-20260920T214958Z-30e3291e1543\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789941076,\\\"last_success_epoch\\\":1789941076,\\\"logical_id\\\":\\\"ct112\\\",\\\"manifest_sha256\\\":\\\"299de9c4c8ed701081e274631b81fc937bc36dd8b67d4a287a3f975dd7b366dd\\\",\\\"payload_sha256\\\":\\\"5b9f2079a420e709985c6c54eabcd59856dafd130107727743c401f95ffe216d\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-03-crypt\\\",\\\"physical\\\":\\\"pve03-mail-03\\\"},{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T21:51:16Z\\\"}\"],\"sha256\":\"eedf7d1687d8d6b5ee84f750f9cce85a423a4f02ccbb39aa8333fc3e354e2243\"}\nRECENT_STATE_FILE={\"age_sec\":14377,\"bytes\":844,\"path\":\"/var/lib/homelab-backup-v2/state/guest-pve01-lxc-112.json\",\"selected\":[\"{\\\"consecutive_failures\\\":5,\\\"failure_backoff_seconds\\\":21600,\\\"last_attempt_epoch\\\":1789938729,\\\"last_success_epoch\\\":1789799511,\\\"last_success_generation\\\":\\\"V2-20260919T062945Z-1c175c0e0cd7\\\",\\\"last_success_green_verified\\\":true,\\\"last_success_manifest_sha256\\\":\\\"b0311f9af3f38813dfe98d64a6e9d6ae93745a1c770080440b07df34b7f66bbe\\\",\\\"last_success_payload_sha256\\\":\\\"b54e21482312be3a810a2d15e74c8f5fac642ca594f75dffa0d98d9fe60d457b\\\",\\\"last_success_replicas\\\":[{\\\"account_id\\\":\\\"e7c8100c3ee2308a64aa763264fc0f738f30f99f19a7306fc9c2554fdd308199\\\",\\\"crypt_remote\\\":\\\"pve03-mail-04-crypt\\\"},{\\\"account_id\\\":\\\"690cb55f912672f2854c3424fe9d536beb6096dadb767e894b3f52bb8fd911bf\\\",\\\"crypt_remote\\\":\\\"pve03-mail-01-crypt\\\"}],\\\"logical_id\\\":\\\"guest-pve01-lxc-112\\\",\\\"reason\\\":\\\"CT112_PIPELINE_FAILED\\\",\\\"result\\\":\\\"FAIL\\\",\\\"retry_after_epoch\\\":1789960329,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T21:12:09Z\\\"}\"],\"sha256\":\"f2ed991431a1b75bb4884f1f1c72881490f9cd71160f6034fb95bdad7a58be39\"}\nRECENT_STATE_FILE={\"age_sec\":16308,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/vm210.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-vm210-20260920T203832Z-458da35ec027\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789936798,\\\"last_success_epoch\\\":1789936798,\\\"logical_id\\\":\\\"vm210\\\",\\\"manifest_sha256\\\":\\\"3587cca11932a034f03c4638bb347972307cd179dc60635b2945dafc5558d875\\\",\\\"payload_sha256\\\":\\\"7e3a8eb9653712a80bf9eb919aae0004faad55ca324f8acb1c55c8a7574fc34e\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T20:39:58Z\\\"}\"],\"sha256\":\"862f6bdfe6cdf9272832e986075a4fff84e9edf73a17fa73ca2a556db52d0aee\"}\nRECENT_STATE_FILE={\"age_sec\":18085,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/ct111.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-ct111-20260920T200749Z-a2bac7e3ecb7\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789935021,\\\"last_success_epoch\\\":1789935021,\\\"logical_id\\\":\\\"ct111\\\",\\\"manifest_sha256\\\":\\\"22ff237e0a246ab51c8bd1ff8f4c1e12742a707578170f67055344c8076494f6\\\",\\\"payload_sha256\\\":\\\"0e65a2cd9ff79535f391287adca2ae6e8f87d394fb7f7a7ca8c00922677442fd\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve03-mail-03-crypt\\\",\\\"physical\\\":\\\"pve03-mail-03\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T20:10:21Z\\\"}\"],\"sha256\":\"12a47c12453c7fe3b66250923e1710b549a565c51dfd93d3d7d54d8b5c5c11c8\"}\nRECENT_STATE_FILE={\"age_sec\":20535,\"bytes\":791,\"path\":\"/var/lib/homelab-backup-v2/state/us-netbird.json\",\"selected\":[\"{\\\"consecutive_failures\\\":9,\\\"failure_backoff_seconds\\\":21600,\\\"generation\\\":\\\"v2-us-netbird-20260920T192916Z-ca2592a81a30\\\",\\\"last_attempt_epoch\\\":1789932571,\\\"last_success_epoch\\\":1789704670,\\\"last_success_generation\\\":\\\"v2-us-netbird-20260918T041032Z-d7648be4c16b\\\",\\\"last_success_green_verified\\\":false,\\\"last_success_manifest_sha256\\\":\\\"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5\\\",\\\"last_success_payload_sha256\\\":\\\"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf\\\",\\\"last_success_replicas\\\":[{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"},{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"}],\\\"logical_id\\\":\\\"us-netbird\\\",\\\"reason\\\":\\\"PIPE_FAILED\\\",\\\"result\\\":\\\"FAIL\\\",\\\"retry_after_epoch\\\":1789954171,\\\"schema\\\":1,\\\"sealed\\\":false,\\\"updated_at_utc\\\":\\\"2026-09-20T19:29:31Z\\\"}\"],\"sha256\":\"57736e0734fb3f3f809aca0b5fb750f87248fb77534e613fbfa58c2ae68960c9\"}\nRECENT_STATE_FILE={\"age_sec\":22808,\"bytes\":354,\"path\":\"/var/lib/homelab-backup-v2/state/control-bootstrap-v2.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"control-bootstrap-v2-pve01-20260916T120057Z\\\",\\\"last_attempt_epoch\\\":1789930298,\\\"last_success_epoch\\\":1789930298,\\\"logical_id\\\":\\\"control-bootstrap-v2\\\",\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T18:51:38Z\\\",\\\"verification\\\":\\\"existing_control_engine_verify\\\"}\"],\"sha256\":\"586acfdfb8f56db1c85a49b8e549f325aa98519709911ef2156964d3706c7b9d\"}\nRECENT_STATE_FILE={\"age_sec\":24986,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/ct110.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-ct110-20260920T181158Z-25703b1ef77f\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789928120,\\\"last_success_epoch\\\":1789928120,\\\"logical_id\\\":\\\"ct110\\\",\\\"manifest_sha256\\\":\\\"6f99d74ab6a71daca2b382d7e280e02ae44d1b65abe127aa5a0106e04907b5aa\\\",\\\"payload_sha256\\\":\\\"eaa23dc6f970e658b1d39615c3a45bf20091d5da875ec817b7e8fd8e1168a4e0\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T18:15:20Z\\\"}\"],\"sha256\":\"a340f72b5e965296245d38ebd5dba835ff795e2a7d851934811071fde2f453f3\"}\nRECENT_STATE_FILE={\"age_sec\":26453,\"bytes\":621,\"path\":\"/var/lib/homelab-backup-v2/state/xf-zakrutim.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-xf-zakrutim-20260920T173658Z-a8de3568c79b\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789926652,\\\"last_success_epoch\\\":1789926652,\\\"logical_id\\\":\\\"xf-zakrutim\\\",\\\"manifest_sha256\\\":\\\"53835f258d47a68635dc1ef54998e019bac0f25d9fab7423fbc67ed17010cff6\\\",\\\"payload_sha256\\\":\\\"fe757c28ade0a1758d76ad4d616f559f30621fe28513aa185f83628c10708311\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T17:50:52Z\\\"}\"],\"sha256\":\"17757d0b482449d3a12c4a40a22bb915c682139be8eb6037631c2ce4ebe6e84c\"}\nRECENT_STATE_FILE={\"age_sec\":30782,\"bytes\":621,\"path\":\"/var/lib/homelab-backup-v2/state/xf-codevipe.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-xf-codevipe-20260920T162616Z-da4e8fcf3129\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789922324,\\\"last_success_epoch\\\":1789922324,\\\"logical_id\\\":\\\"xf-codevipe\\\",\\\"manifest_sha256\\\":\\\"5218ed5341f7a93a9fe68a715978375f47f6181e3c58ca075de2fa7a13e408ea\\\",\\\"payload_sha256\\\":\\\"e00525a66738da126b01fc26cc27e5688ba0f2dcc6afc3276b3aa7dee23ee08b\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T16:38:44Z\\\"}\"],\"sha256\":\"b65a7092d18371bce1d7681224fa3d54911756c0800ecd7df3179d2589b32c5e\"}\nRECENT_STATE_FILE={\"age_sec\":33395,\"bytes\":629,\"path\":\"/var/lib/homelab-backup-v2/state/router-netcraze.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-router-netcraze-20260920T155444Z-a91b2efdb4fe\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789919711,\\\"last_success_epoch\\\":1789919711,\\\"logical_id\\\":\\\"router-netcraze\\\",\\\"manifest_sha256\\\":\\\"fd7fd914d15f08b3d816a8bc76c233eff22b38a42494f7f9259859fb9edd3cb9\\\",\\\"payload_sha256\\\":\\\"252260b8ba622dc5a4536424abc9cde197a9e71dee67cac108f103d85a84b28c\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T15:55:11Z\\\"}\"],\"sha256\":\"0a360ef89a3d8ee8d2d45061f45120eb3464b800c4050300c205ff7269a9f0a0\"}\nRECENT_STATE_FILE={\"age_sec\":35709,\"bytes\":621,\"path\":\"/var/lib/homelab-backup-v2/state/spb-ingress.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-spb-ingress-20260920T151616Z-9d80091d0c4c\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789917397,\\\"last_success_epoch\\\":1789917397,\\\"logical_id\\\":\\\"spb-ingress\\\",\\\"manifest_sha256\\\":\\\"4f54c7273374849985bc3d3df7940840ab4cc5083a3324585ce9e4978a75e410\\\",\\\"payload_sha256\\\":\\\"f775a3f6ea7f397559f4b56e5f823694187749a24240173298a65d3d637826d6\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T15:16:37Z\\\"}\"],\"sha256\":\"7e339f360d15e31044a00bcb7127ab5b2ae8193e50aa9dc4d70ec207196336ee\"}\nRECENT_STATE_FILE={\"age_sec\":39991,\"bytes\":615,\"path\":\"/var/lib/homelab-backup-v2/state/md-relay.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-md-relay-20260920T140452Z-1943a8c57bd6\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789913115,\\\"last_success_epoch\\\":1789913115,\\\"logical_id\\\":\\\"md-relay\\\",\\\"manifest_sha256\\\":\\\"a80cc1c23deea8d06a036a4ba696376ef8f7d9f838a0f29e4e5d52ed5579ed12\\\",\\\"payload_sha256\\\":\\\"c168e3d7e2b0650c844e5c17aeb532bc918bf987db7f6a574f8f9559d010eccc\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T14:05:15Z\\\"}\"],\"sha256\":\"21e6e06ebeba772b1bdc22c2b4d47882facf9813dcb0c2308621dac7e3211988\"}\nRECENT_STATE_FILE={\"age_sec\":46317,\"bytes\":617,\"path\":\"/var/lib/homelab-backup-v2/state/xf-hkmods.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-xf-hkmods-20260920T071214Z-6407219f0624\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789906789,\\\"last_success_epoch\\\":1789906789,\\\"logical_id\\\":\\\"xf-hkmods\\\",\\\"manifest_sha256\\\":\\\"d548aca241a964ac1056325ccdefae48ccc1d53c843dd594403badd9db8a98c2\\\",\\\"payload_sha256\\\":\\\"94955dc3f8cb1fbe5fbfa74bb42140742b7197e70ca35bedad6ddddcaf181714\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T12:19:49Z\\\"}\"],\"sha256\":\"7cec53bc60b4f12e3285f19f4fe6de9c92d198f61ff268997cf6a36f2b29a8fa\"}\nRECENT_STATE_FILE={\"age_sec\":70838,\"bytes\":621,\"path\":\"/var/lib/homelab-backup-v2/state/xf-gamevipe.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-xf-gamevipe-20260920T015714Z-762dbd7ce0d7\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789882268,\\\"last_success_epoch\\\":1789882268,\\\"logical_id\\\":\\\"xf-gamevipe\\\",\\\"manifest_sha256\\\":\\\"81fa85fdb7ca2f3b3bd20be4cfb1b0c13712255d544ddec67a0f98f2fd948e98\\\",\\\"payload_sha256\\\":\\\"46d4c7e4e9f1056660e0cdc3136774865b08b8edb5ca92873cea7d41d602e4bb\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T05:31:08Z\\\"}\"],\"sha256\":\"3ac48c6c7f0e0e2699089eecb211687405cd99f57ca66a1e8e5fb7b0145b260e\"}\nRECENT_STATE_FILE={\"age_sec\":87487,\"bytes\":615,\"path\":\"/var/lib/homelab-backup-v2/state/xf-newfi.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-xf-newfi-20260920T004616Z-6be6559a5589\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789865619,\\\"last_success_epoch\\\":1789865619,\\\"logical_id\\\":\\\"xf-newfi\\\",\\\"manifest_sha256\\\":\\\"f912b85107e9e1ee2425749099f0de9a9a53c909f98cd4f41356a20ff148da62\\\",\\\"payload_sha256\\\":\\\"774bc5dcb7bb3905392c2c078f7c5d4bc127d87d97c2330a5baab997c7b27aa8\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-20T00:53:39Z\\\"}\"],\"sha256\":\"f4e48ea4aa80dc57537510a377775a0ca216e69a8308f92334d4114ed91e76f7\"}\nRECENT_STATE_FILE={\"age_sec\":92069,\"bytes\":625,\"path\":\"/var/lib/homelab-backup-v2/state/xf-kingofwolk.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-xf-kingofwolk-20260919T232652Z-cb0189b2b0ba\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789861037,\\\"last_success_epoch\\\":1789861037,\\\"logical_id\\\":\\\"xf-kingofwolk\\\",\\\"manifest_sha256\\\":\\\"69a9ce6a55c57121b2634c65b2826a8056b4de65127d229b8c3fbbb2980b4162\\\",\\\"payload_sha256\\\":\\\"cf32a08d5c05eeb04b4d2b2d71faef01c363c9d223b9b9a93813d1bafecca334\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-19T23:37:17Z\\\"}\"],\"sha256\":\"6357284997a7cf7b28e7a740e12608b617e9f57f257b35ccd43853268fb28391\"}\nRECENT_STATE_FILE={\"age_sec\":92695,\"bytes\":617,\"path\":\"/var/lib/homelab-backup-v2/state/xf-dsmods.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-xf-dsmods-20260919T214716Z-c783be0bd209\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789860411,\\\"last_success_epoch\\\":1789860411,\\\"logical_id\\\":\\\"xf-dsmods\\\",\\\"manifest_sha256\\\":\\\"f3b4cd994ffcc6e9a7944f8569fc83df52e8bd9692f4f2067155cc3dbc6b3891\\\",\\\"payload_sha256\\\":\\\"e609e9951c2baba42ab268507ea277140f73750587a9ccd99e5924af6995f0e3\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve03-mail-04-crypt\\\",\\\"physical\\\":\\\"pve03-mail-04\\\"},{\\\"crypt\\\":\\\"pve01-mail-05-crypt\\\",\\\"physical\\\":\\\"pve01-mail-05\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-19T23:26:51Z\\\"}\"],\"sha256\":\"0b38d624d502ea4a7e3b2284f2a74b6b1985399791cdf8a44d467c855c2aba14\"}\nRECENT_STATE_FILE={\"age_sec\":99549,\"bytes\":611,\"path\":\"/var/lib/homelab-backup-v2/state/vm9130.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-vm9130-20260919T210758Z-0aa987efac8c\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789853557,\\\"last_success_epoch\\\":1789853557,\\\"logical_id\\\":\\\"vm9130\\\",\\\"manifest_sha256\\\":\\\"4cc97d6769312b0be7e2c3cfe144da7fb68f3f5664755a7c5b8d44c1139bcd3b\\\",\\\"payload_sha256\\\":\\\"b3703c1f4616f30fc6711a7b7bd2e50af5be805bd6752f87a57be2b9ac531cb8\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve01-mail-03-crypt\\\",\\\"physical\\\":\\\"pve01-mail-03\\\"},{\\\"crypt\\\":\\\"pve03-mail-02-crypt\\\",\\\"physical\\\":\\\"pve03-mail-02\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-19T21:32:37Z\\\"}\"],\"sha256\":\"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db\"}\nRECENT_STATE_FILE={\"age_sec\":103122,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/vm211.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-vm211-20260919T203132Z-82878bb22008\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789849984,\\\"last_success_epoch\\\":1789849984,\\\"logical_id\\\":\\\"vm211\\\",\\\"manifest_sha256\\\":\\\"af8ad542b43c8ac9f75969b4726d8e294b7c1e62ca569c2383776e5e24d9934d\\\",\\\"payload_sha256\\\":\\\"a232d511565774d22f86afe0e773b28da4a8e722ec1b2a357e55b0b62f1eeef9\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve01-mail-03-crypt\\\",\\\"physical\\\":\\\"pve01-mail-03\\\"},{\\\"crypt\\\":\\\"pve03-mail-02-crypt\\\",\\\"physical\\\":\\\"pve03-mail-02\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-19T20:33:04Z\\\"}\"],\"sha256\":\"ecd899387f605d91b036977372cab73a69a0a4cedc050f22ece38c4cb664311b\"}\nRECENT_STATE_FILE={\"age_sec\":104899,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/vm170.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-vm170-20260919T195132Z-74c06ba16732\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789848207,\\\"last_success_epoch\\\":1789848207,\\\"logical_id\\\":\\\"vm170\\\",\\\"manifest_sha256\\\":\\\"7ef201d524d1cea100d8cf87da2d33e920354258c91ed827d634b62d770ae6f4\\\",\\\"payload_sha256\\\":\\\"22d92332a6b86141b3f67f357535c53826d0127f3e0a4f858a47ddec7777a104\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve01-mail-04-crypt\\\",\\\"physical\\\":\\\"pve01-mail-04\\\"},{\\\"crypt\\\":\\\"pve03-mail-01-crypt\\\",\\\"physical\\\":\\\"pve03-mail-01\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-19T20:03:27Z\\\"}\"],\"sha256\":\"897ed79b3dc484f868155492fc9471a78dd901b896fdaa934d472174c6a84704\"}\nRECENT_STATE_FILE={\"age_sec\":107652,\"bytes\":609,\"path\":\"/var/lib/homelab-backup-v2/state/vm190.json\",\"selected\":[\"{\\\"consecutive_failures\\\":0,\\\"failure_backoff_seconds\\\":0,\\\"generation\\\":\\\"v2-vm190-20260919T191358Z-c42ef8cbe584\\\",\\\"green_verified\\\":false,\\\"last_attempt_epoch\\\":1789845454,\\\"last_success_epoch\\\":1789845454,\\\"logical_id\\\":\\\"vm190\\\",\\\"manifest_sha256\\\":\\\"d98b7e43bb12e3304a498629b8e2c61baf01a54a9ae2cede67de9a971ee52c44\\\",\\\"payload_sha256\\\":\\\"a81b1bdc20c5ab28375e20304087099386666e74a5ad18563eaa01751112ddc5\\\",\\\"replicas\\\":[{\\\"crypt\\\":\\\"pve01-mail-04-crypt\\\",\\\"physical\\\":\\\"pve01-mail-04\\\"},{\\\"crypt\\\":\\\"pve03-mail-01-crypt\\\",\\\"physical\\\":\\\"pve03-mail-01\\\"}],\\\"result\\\":\\\"SUCCESS\\\",\\\"retry_after_epoch\\\":0,\\\"schema\\\":1,\\\"updated_at_utc\\\":\\\"2026-09-19T19:17:34Z\\\"}\"],\"sha256\":\"44d1db9aaac329596eaf9b3a7ebe33daf5c68775d56fecb8cf8ed887c310f059\"}\n===== RETENTION_VM200_RCA =====\nRETENTION_RUNTIME_FOCUS={\"bytes\":32975,\"exists\":true,\"matches\":[\"23:GFS_HELPER=\\\"${HOMELAB_BACKUP_GFS_POLICY_HELPER:-/usr/local/libexec/homelab-backup-gfs-policy}\\\"\",\"24:GFS_HELPER_SHA_EXPECTED=6b7ec1ab3ee1c540a9113b5620c3b4b5a50de6536f3a069d31b2e1243f23bafa\",\"25:GFS_POLICY=\\\"${HOMELAB_BACKUP_GFS_POLICY:-/etc/homelab-backup/gfs-retention.json}\\\"\",\"30:[[ -x \\\"$GFS_HELPER\\\" && ! -L \\\"$GFS_HELPER\\\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_MISSING\\\\n' >&2; exit 69; }\",\"31:[[ \\\"$(sha256sum \\\"$GFS_HELPER\\\"|awk '{print $1}')\\\" == \\\"$GFS_HELPER_SHA_EXPECTED\\\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\\\\n' >&2; exit 65; }\",\"32:HOMELAB_BACKUP_GFS_POLICY=\\\"$GFS_POLICY\\\" \\\"$GFS_HELPER\\\" validate >/dev/null || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID\\\\n' >&2; exit 66; }\",\"33:GFS_WEEKLY=\\\"$(HOMELAB_BACKUP_GFS_POLICY=\\\"$GFS_POLICY\\\" \\\"$GFS_HELPER\\\" weekly-buckets)\\\"\",\"34:GFS_MONTHLY=\\\"$(HOMELAB_BACKUP_GFS_POLICY=\\\"$GFS_POLICY\\\" \\\"$GFS_HELPER\\\" monthly-buckets)\\\"\",\"35:GFS_CRITICAL_COUNT=\\\"$(HOMELAB_BACKUP_GFS_POLICY=\\\"$GFS_POLICY\\\" \\\"$GFS_HELPER\\\" critical-logicals | wc -l)\\\"\",\"36:VM_KEEP=\\\"$(\\\"$REGISTRY_HELPER\\\" policy-keep VM)\\\"\",\"37:APP_KEEP=\\\"$(\\\"$REGISTRY_HELPER\\\" policy-keep APP)\\\"\",\"38:CONTROL_KEEP=\\\"$(\\\"$REGISTRY_HELPER\\\" policy-keep CONTROL)\\\"\",\"47:node_for_logical(){ \\\"$REGISTRY_HELPER\\\" logical-owner \\\"$1\\\" 2>/dev/null; }\",\"130:pinned_delete_contract_enabled(){\",\"131: [[ -n \\\"${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}\\\" || -n \\\"${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}\\\" ]]\",\"133:pinned_delete_assert_file(){\",\"134: local f=\\\"${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}\\\" expected=\\\"${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}\\\" actual line logical generation owner state_file state_sha count=0\",\"135: [[ -n \\\"$f\\\" && -n \\\"$expected\\\" ]] || die PINNED_DELETE_CONTRACT_INCOMPLETE 64\",\"136: [[ \\\"$expected\\\" =~ ^[0-9a-f]{64}$ ]] || die PINNED_DELETE_SHA_INVALID 64\",\"137: [[ -f \\\"$f\\\" && ! -L \\\"$f\\\" ]] || die PINNED_DELETE_FILE_UNSAFE 64\",\"139: [[ \\\"$actual\\\" == \\\"$expected\\\" ]] || die PINNED_DELETE_FILE_SHA_MISMATCH 66\",\"141: [[ -n \\\"$line\\\" ]] || die PINNED_DELETE_EMPTY_LINE 66\",\"144: owner=\\\"$(awk -F'|' '{for(i=1;i<=NF;i++)if($i~/^owner=/){sub(/^owner=/,\\\"\\\",$i);print $i}}' <<<\\\"$line\\\")\\\"\",\"147: [[ \\\"$line\\\" == \\\"DELETE|logical=$logical|generation=$generation|owner=$owner|state_file=$state_file|state_sha256=$state_sha\\\" ]] || die PINNED_DELETE_LINE_FORMAT_INVALID 66\",\"148: [[ \\\"$(node_for_logical \\\"$logical\\\")\\\" == \\\"$owner\\\" ]] || die \\\"PINNED_DELETE_OWNER_INVALID_${logical}\\\" 66\",\"149: safe_generation \\\"$logical\\\" \\\"$generation\\\" || die \\\"PINNED_DELETE_GENERATION_INVALID_${logical}\\\" 66\",\"150: [[ \\\"$state_file\\\" == \\\"$STATE_ROOT/$logical/$generation.json\\\" ]] || die \\\"PINNED_DELETE_STATE_PATH_INVALID_${logical}\\\" 66\",\"151: [[ \\\"$state_sha\\\" =~ ^[0-9a-f]{64}$ ]] || die \\\"PINNED_DELETE_STATE_SHA_INVALID_${logical}\\\" 66\",\"154: (( count > 0 )) || die PINNED_DELETE_FILE_EMPTY 66\",\"155: [[ \\\"$(LC_ALL=C sort \\\"$f\\\" | uniq | wc -l)\\\" == \\\"$count\\\" ]] || die PINNED_DELETE_FILE_DUPLICATE 66\",\"157:pinned_tx_identity(){\",\"158: local f=\\\"$1\\\" logical generation state_sha status owner expected_path\",\"159: [[ -s \\\"$f\\\" && ! -L \\\"$f\\\" ]] || die \\\"PINNED_TX_FILE_UNSAFE_$f\\\" 66\",\"160: jq -e --arg task \\\"$TASK_ID\\\" --arg change \\\"$CHANGE_ID\\\" '.schema==\\\"homelab-backup-retention-transaction-v1\\\" and .task_id==$task and .change_id==$change and (.logical_id|type)==\\\"string\\\" and (.generation|type)==\\\"string\\\" and (.state_sha256|type)==\\\"string\\\" and (.status==\\\"PLANNED\\\" or .status==\\\"B_DELETED\\\" or .status==\\\"DONE\\\")' \\\"$f\\\" >/dev/null || die \\\"PINNED_TX_CONTENT_INVALID_$f\\\" 66\",\"162: owner=\\\"$(node_for_logical \\\"$logical\\\")\\\" || die \\\"PINNED_TX_OWNER_UNKNOWN_${logical}\\\" 66\",\"163: safe_generation \\\"$logical\\\" \\\"$generation\\\" || die \\\"PINNED_TX_GENERATION_UNSAFE_${logical}\\\" 66\",\"164: [[ \\\"$state_sha\\\" =~ ^[0-9a-f]{64}$ ]] || die \\\"PINNED_TX_STATE_SHA_INVALID_${logical}\\\" 66\",\"166: [[ \\\"$f\\\" == \\\"$expected_path\\\" ]] || die \\\"PINNED_TX_PATH_INVALID_${logical}_${generation}\\\" 66\",\"167: printf 'DELETE|logical=%s|generation=%s|owner=%s|state_file=%s|state_sha256=%s\\\\n' \\\"$logical\\\" \\\"$generation\\\" \\\"$owner\\\" \\\"$STATE_ROOT/$logical/$generation.json\\\" \\\"$state_sha\\\"\",\"169:pinned_assert_transaction_tree(){\",\"172: [[ -d \\\"$root\\\" && ! -L \\\"$root\\\" ]] || die PINNED_TX_ROOT_UNSAFE 66\",\"173: [[ -z \\\"$(find \\\"$root\\\" -type l -print -quit)\\\" ]] || die PINNED_TX_SYMLINK_PRESENT 66\",\"174: [[ -z \\\"$(find \\\"$root\\\" -type f ! -name '*.json' -print -quit)\\\" ]] || die PINNED_TX_NONJSON_PRESENT 66\",\"175: while IFS= read -r f; do pinned_tx_identity \\\"$f\\\" >/dev/null; done < <(find \\\"$root\\\" -type f -name '*.json' -print | LC_ALL=C sort)\",\"177:pinned_assert_protect_scope(){\",\"183: [[ \\\"$reason\\\" == GC_TRANSACTION_PLANNED || \\\"$reason\\\" == GC_TRANSACTION_B_DELETED ]] || die \\\"PINNED_PLAN_UNEXPECTED_PROTECT_${logical}_${generation}_${reason}\\\" 66\",\"185: [[ -s \\\"$txf\\\" && ! -L \\\"$txf\\\" ]] || die \\\"PINNED_PLAN_PROTECT_TX_MISSING_${logical}_${generation}\\\" 66\",\"187: [[ \\\"GC_TRANSACTION_$status\\\" == \\\"$reason\\\" ]] || die \\\"PINNED_PLAN_PROTECT_STATUS_MISMATCH_${logical}_${generation}\\\" 66\",\"188: grep -Fqx \\\"$(pinned_tx_identity \\\"$txf\\\")\\\" \\\"$HOMELAB_RETENTION_PINNED_DELETE_FILE\\\" || die \\\"PINNED_PLAN_PROTECT_NOT_PINNED_${logical}_${generation}\\\" 66\",\"191:pinned_assert_delete_scope(){\",\"193: actual=\\\"$TMPROOT/pinned-delete-actual.$stage\\\"\",\"194: expected=\\\"$TMPROOT/pinned-delete-expected.$stage\\\"\",\"195: pinned_delete_assert_file\",\"196: pinned_assert_transaction_tree\",\"197: pinned_assert_protect_scope \\\"$plan_file\\\"\",\"200: while IFS= read -r f; do pinned_tx_identity \\\"$f\\\" >>\\\"$actual\\\"; done < <(find \\\"$RETENTION_ROOT/transactions\\\" -type f -name '*.json' -print | LC_ALL=C sort)\",\"203: LC_ALL=C sort \\\"$HOMELAB_RETENTION_PINNED_DELETE_FILE\\\" >\\\"$expected\\\"\",\"205: [[ \\\"$count_actual\\\" == \\\"$count_expected\\\" ]] || die \\\"PINNED_DELETE_SCOPE_COUNT_MISMATCH_${stage}\\\" 66\",\"206: [[ \\\"$(uniq \\\"$actual\\\" | wc -l)\\\" == \\\"$count_actual\\\" ]] || die \\\"PINNED_DELETE_SCOPE_DUPLICATE_${stage}\\\" 66\",\"207: cmp -s \\\"$actual\\\" \\\"$expected\\\" || die \\\"PINNED_DELETE_SCOPE_MISMATCH_${stage}\\\" 66\",\"208: printf 'PINNED_DELETE_SCOPE=OK|stage=%s|count=%s|sha256=%s\\\\n' \\\"$stage\\\" \\\"$count_expected\\\" \\\"$HOMELAB_RETENTION_PINNED_DELETE_SHA256\\\"\",\"211: printf 'CONTRACT|task=%s|policy=OWNER_AWARE_SAFE_GC_GFS|target_registry=CANONICAL|vm_keep=%s|app_keep=%s|control_keep=%s|min_delete_age_seconds=%s|gfs_critical=%s|gfs_weekly=%s|gfs_monthly=%s|gfs_completed_periods_only=YES|nonsealed=PROTECT|retired_preserve=PROTECT|unknown_namespace=PROTECT|owner_state_required=YES|both_clouds_required=YES|latest_restore_proven_required=YES|legacy_hb=PROTECT|legacy_pve01=PROTECT|legacy_control=PROTECT\\\\n' \\\"$TASK_ID\\\" \\\"$VM_KEEP\\\" \\\"$APP_KEEP\\\" \\\"$CONTROL_KEEP\\\" \\\"$MIN_DELETE_AGE_SECONDS\\\" \\\"$GFS_CRITICAL_COUNT\\\" \\\"$GFS_WEEKLY\\\" \\\"$GFS_MONTHLY\\\"\",\"213:select_gfs(){\",\"215: : >\\\"$TMPROOT/$logical.gfs-eligible\\\"\",\"216: : >\\\"$TMPROOT/$logical.gfs-weekly\\\"\",\"217: : >\\\"$TMPROOT/$logical.gfs-monthly\\\"\",\"218: if ! HOMELAB_BACKUP_GFS_POLICY=\\\"$GFS_POLICY\\\" \\\"$GFS_HELPER\\\" is-critical \\\"$logical\\\"; then return 0; fi\",\"224: printf '%s\\\\t%s\\\\n' \\\"$gen\\\" \\\"$sealed_at\\\" >>\\\"$TMPROOT/$logical.gfs-eligible\\\"\",\"226: out=\\\"$(HOMELAB_BACKUP_GFS_POLICY=\\\"$GFS_POLICY\\\" \\\"$GFS_HELPER\\\" select \\\"$logical\\\" <\\\"$TMPROOT/$logical.gfs-eligible\\\")\\\" || die \\\"GFS_SELECT_FAILED_$logical\\\" 66\",\"230: WEEKLY) printf '%s\\\\t%s\\\\n' \\\"$gen\\\" \\\"$bucket\\\" >>\\\"$TMPROOT/$logical.gfs-weekly\\\" ;;\",\"231: MONTHLY) printf '%s\\\\t%s\\\\n' \\\"$gen\\\" \\\"$bucket\\\" >>\\\"$TMPROOT/$logical.gfs-monthly\\\" ;;\",\"232: *) die \\\"GFS_SELECT_KIND_INVALID_$logical\\\" 66 ;;\",\"240: local now logical node keep lifecycle open_count file json schema lid gen status sha sealed_at epoch age idx tx gfs_bucket candidates=0 protected=0 kept=0 gfs_kept=0\",\"244: node=\\\"$(node_for_logical \\\"$logical\\\")\\\" || die \\\"OWNER_UNKNOWN_$logical\\\"\",\"246: if [[ \\\"$lifecycle\\\" == ACTIVE ]]; then keep=\\\"$(keep_for \\\"$logical\\\")\\\" || die \\\"KEEP_POLICY_UNKNOWN_$logical\\\"; else keep=0; fi\",\"262: printf 'PROTECT|logical=%s|generation=%s|owner=%s|reason=INCOMPLETE_%s\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\" \\\"$status\\\"\",\"266: strict_state \\\"$json\\\" \\\"$logical\\\" \\\"$gen\\\" || { printf 'PROTECT|logical=%s|generation=%s|owner=%s|reason=SEALED_STATE_NOT_STRICT\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\"; protected=$((protected+1)); continue; }\",\"268: epoch=\\\"$(date -u -d \\\"$sealed_at\\\" +%s 2>\\\"$TMPROOT/date.err\\\")\\\" || { printf 'PROTECT|logical=%s|generation=%s|owner=%s|reason=SEALED_TIME_INVALID\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\"; protected=$((protected+1)); continue; }\",\"273: while IFS=$'\\\\t' read -r gen file sha age node sealed_at; do printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=LOGICAL_HAS_INCOMPLETE\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\"; kept=$((kept+1)); done < <(LC_ALL=C sort -r \\\"$TMPROOT/$logical.rows\\\")\",\"277: while IFS=$'\\\\t' read -r gen file sha age node sealed_at; do printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=RETIRED_PRESERVE\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\"; kept=$((kept+1)); done < <(LC_ALL=C sort -r \\\"$TMPROOT/$logical.rows\\\")\",\"281: while IFS=$'\\\\t' read -r gen file sha age node sealed_at; do printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=LATEST_RESTORE_NOT_PROVEN\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\"; kept=$((kept+1)); done < <(LC_ALL=C sort -r \\\"$TMPROOT/$logical.rows\\\")\",\"284: select_gfs \\\"$logical\\\"\",\"290: printf 'RETIRED|logical=%s|generation=%s|owner=%s|reason=GC_ALREADY_DONE\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\"\",\"292: printf 'PROTECT|logical=%s|generation=%s|owner=%s|reason=GC_TRANSACTION_%s\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\" \\\"$tx\\\"; protected=$((protected+1))\",\"294: printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=POLICY_NEWEST_%s\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\" \\\"$keep\\\"; kept=$((kept+1))\",\"296: printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=MIN_AGE|age_seconds=%s\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\" \\\"$age\\\"; kept=$((kept+1))\",\"298: printf 'PROTECT|logical=%s|generation=%s|owner=%s|reason=REMOTE_SET_MISMATCH\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\"; protected=$((protected+1))\",\"299: elif gfs_bucket=\\\"$(awk -F'\\\\t' -v g=\\\"$gen\\\" '$1==g{print $2; exit}' \\\"$TMPROOT/$logical.gfs-weekly\\\")\\\" && [[ -n \\\"$gfs_bucket\\\" ]]; then\",\"300: printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=GFS_WEEKLY|bucket=%s\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\" \\\"$gfs_bucket\\\"; kept=$((kept+1)); gfs_kept=$((gfs_kept+1))\",\"301: elif gfs_bucket=\\\"$(awk -F'\\\\t' -v g=\\\"$gen\\\" '$1==g{print $2; exit}' \\\"$TMPROOT/$logical.gfs-monthly\\\")\\\" && [[ -n \\\"$gfs_bucket\\\" ]]; then\",\"302: printf 'KEEP|logical=%s|generation=%s|owner=%s|reason=GFS_MONTHLY|bucket=%s\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\" \\\"$gfs_bucket\\\"; kept=$((kept+1)); gfs_kept=$((gfs_kept+1))\",\"304: printf 'DELETE|logical=%s|generation=%s|owner=%s|state_file=%s|state_sha256=%s|age_seconds=%s\\\\n' \\\"$logical\\\" \\\"$gen\\\" \\\"$node\\\" \\\"$file\\\" \\\"$sha\\\" \\\"$age\\\"\",\"312: printf 'PLAN_SUMMARY|delete_candidates=%s|protected=%s|kept=%s|vm_keep=%s|app_keep=%s|control_keep=%s|gfs_kept=%s|gfs_critical=%s|gfs_weekly=%s|gfs_monthly=%s\\\\n' \\\"$candidates\\\" \\\"$protected\\\" \\\"$kept\\\" \\\"$VM_KEEP\\\" \\\"$APP_KEEP\\\" \\\"$CONTROL_KEEP\\\" \\\"$gfs_kept\\\" \\\"$GFS_CRITICAL_COUNT\\\" \\\"$GFS_WEEKLY\\\" \\\"$GFS_MONTHLY\\\"\",\"329: local line=\\\"$1\\\" logical generation owner state_file state_sha age json current_sha\",\"332: owner=\\\"$(awk -F'|' '{for(i=1;i<=NF;i++)if($i~/^owner=/){sub(/^owner=/,\\\"\\\",$i);print $i}}' <<<\\\"$line\\\")\\\"\",\"336: [[ \\\"$(node_for_logical \\\"$logical\\\")\\\" == \\\"$owner\\\" ]] || die \\\"APPLY_OWNER_CHANGED_${logical}_${generation}\\\"\",\"339: json=\\\"$(node_exec \\\"$owner\\\" cat \\\"$state_file\\\")\\\" || die \\\"APPLY_STATE_READ_${logical}\\\" 69\",\"353: printf 'GC_DELETE_DONE|logical=%s|generation=%s|owner=%s\\\\n' \\\"$logical\\\" \\\"$generation\\\" \\\"$owner\\\"\",\"356: local f logical generation state_sha status owner state_file json current_sha\",\"365: owner=\\\"$(node_for_logical \\\"$logical\\\")\\\" || die \\\"TX_OWNER_UNKNOWN_$logical\\\"\",\"368: json=\\\"$(node_exec \\\"$owner\\\" cat \\\"$state_file\\\")\\\" || die \\\"TX_STATE_READ_${logical}\\\" 69\",\"410: local plan_file=\\\"$1\\\" canary_proven=\\\"$2\\\" deleted=\\\"$3\\\" candidates protected kept gfs_kept hist=false tmp\",\"414: gfs_kept=\\\"$(awk -F'[=|]' '/^PLAN_SUMMARY\\\\|/{for(i=1;i<=NF;i++)if($i==\\\"gfs_kept\\\"){print $(i+1)}}' \\\"$plan_file\\\" | tail -n 1)\\\"\",\"418: jq -n --arg task \\\"$TASK_ID\\\" --arg change \\\"$CHANGE_ID\\\" --arg at \\\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\\\" --argjson canary \\\"$canary_proven\\\" --argjson hist \\\"$hist\\\" --argjson candidates \\\"${candidates:-0}\\\" --argjson protected \\\"${protected:-0}\\\" --argjson kept \\\"${kept:-0}\\\" --argjson gfs_kept \\\"${gfs_kept:-0}\\\" --argjson deleted \\\"$deleted\\\" --argjson gfs_critical \\\"$GFS_CRITICAL_COUNT\\\" --argjson gfs_weekly \\\"$GFS_WEEKLY\\\" --argjson gfs_monthly \\\"$GFS_MONTHLY\\\" '{schema:\\\"homelab-backup-retention-v1\\\",status:\\\"OK\\\",task_id:$task,change_id:$change,updated_at_utc:$at,policy:{vm_keep:2,app_keep:7,control_keep:4,min_delete_age_seconds:86400,gfs:{enabled:true,critical_vm_count:$gfs_critical,weekly_completed_buckets:$gfs_weekly,monthly_completed_buckets:$gfs_monthly,completed_periods_only:true}},owner_aware_gc_proven:true,destructive_canary_proven:$canary,historical_gc_proven:$hist,last_plan:{delete_candidates:$candidates,protected:$protected,kept:$kept,gfs_kept:$gfs_kept},last_run_deleted:$deleted,legacy_namespaces_protected:[\\\"hb\\\",\\\"pve01\\\",\\\"control-bootstrap\\\"],provider_worm_proven:false}' >\\\"$tmp\\\"\",\"442: if pinned_delete_contract_enabled; then pinned_assert_delete_scope \\\"$p1\\\" PRELOCK; fi\",\"445: if pinned_delete_contract_enabled; then plan >\\\"$TMPROOT/pinned-pre-resume-plan.txt\\\"; pinned_assert_delete_scope \\\"$TMPROOT/pinned-pre-resume-plan.txt\\\" LOCKED_PRE_RESUME; fi\",\"448: if pinned_delete_contract_enabled; then pinned_assert_delete_scope \\\"$p2\\\" LOCKED_POST_RESUME; fi\"],\"path\":\"/usr/local/sbin/homelab-backup-retention-gc\",\"sha256\":\"8adb6c10f5911f39c53775b89eac3cd1823f1ed34b4caccade5279c213e998a3\"}\nREPO_VM200_POLICY_HITS={\"lines\":[],\"rc\":1,\"stderr_sha256\":null}\n===== AUDIT_SCHEDULER_EXPECTATION_RCA =====\nAUDIT_RUNTIME_FOCUS={\"bytes\":39160,\"exists\":true,\"matches\":[\"18:VM_QUEUE=/usr/local/sbin/homelab-vm-cloud-quorum-queue\",\"19:APP_QUEUE=/usr/local/sbin/homelab-app-cloud-quorum-queue\",\"22:CONTROL_BACKUP=/usr/local/sbin/homelab-control-backup\",\"24:CONTROL_TIMER=/etc/systemd/system/homelab-control-backup.timer\",\"25:CONTROL_SERVICE=/etc/systemd/system/homelab-control-backup.service\",\"28:VM_TIMER=/etc/systemd/system/homelab-vm-cloud-quorum-queue.timer\",\"29:APP_TIMER=/etc/systemd/system/homelab-app-cloud-quorum-queue.timer\",\"30:VM_SERVICE=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service\",\"31:APP_SERVICE=/etc/systemd/system/homelab-app-cloud-quorum-queue.service\",\"110: local label=\\\"$1\\\" file=\\\"$2\\\" expected=\\\"$3\\\" actual\",\"111: if [[ -f \\\"$file\\\" && ! -L \\\"$file\\\" ]] && actual=\\\"$(sha256sum \\\"$file\\\" 2>&1 | awk 'NR==1{print $1}')\\\" && [[ \\\"$actual\\\" == \\\"$expected\\\" ]]; then\",\"120: local node=\\\"$1\\\" label=\\\"$2\\\" file=\\\"$3\\\" expected=\\\"$4\\\" out actual\",\"122: if [[ \\\"$actual\\\" == \\\"$expected\\\" ]]; then\",\"147:expected_vm_count=\\\"${#VMIDS[@]}\\\"; expected_app_count=\\\"${#APPS[@]}\\\"\",\"148:[[ \\\"$expected_vm_count\\\" == 11 && \\\"$expected_app_count\\\" == 4 ]] || die TARGET_REGISTRY_COUNT_INVALID 66\",\"190:printf 'TARGET_REGISTRY|status=OK|path=%s|sha256=%s|vm_active=%s|app_active=%s\\\\n' \\\"${HOMELAB_BACKUP_TARGET_REGISTRY:-/etc/homelab-backup/targets.json}\\\" \\\"$(sha256sum \\\"${HOMELAB_BACKUP_TARGET_REGISTRY:-/etc/homelab-backup/targets.json}\\\"|awk '{print $1}')\\\" \\\"$expected_vm_count\\\" \\\"$expected_app_count\\\"\",\"403:while IFS=$'\\\\t' read -r unit expected_state; do\",\"407: case \\\"$expected_state\\\" in\",\"408: ENABLED_ACTIVE)\",\"415: if [[ \\\"$load_state\\\" == not-found && \\\"$active_state\\\" == inactive ]]; then status=EXPECTED_ABSENT; scheduler_ok=$((scheduler_ok+1)); else status=BAD; infra_bad=$((infra_bad+1)); fi\",\"419: printf 'SYSTEMD_EXPECTATION|unit=%s|expected=%s|status=%s|load_state=%s|unit_file_state=%s|active_state=%s\\\\n' \\\"$unit\\\" \\\"$expected_state\\\" \\\"$status\\\" \\\"$load_state\\\" \\\"$unit_file_state\\\" \\\"$active_state\\\"\",\"422:for lock in /run/homelab-backup-heavy.lock /run/lock/homelab-vm-cloud-quorum-queue.lock /run/lock/homelab-app-cloud-quorum-queue.lock /run/lock/homelab-backup-dispatcher.lock; do\",\"509:if (( vm_seen != expected_vm_count || app_seen != expected_app_count )); then die COVERAGE_COUNT_INVALID; fi\"],\"path\":\"/usr/local/sbin/homelab-backup-audit\",\"sha256\":\"1474c9ce58bbc1b3ae7437261d1bdb923232926fb2943c7c830253f8b284724c\"}\nUNIT_STATE={\"props\":{\"ActiveState\":\"active\",\"LastTriggerUSec\":\"Mon 2026-09-21 03:47:58 MSK\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"UnitFileState\":\"enabled\"},\"rc\":0,\"unit\":\"homelab-backup-v2.timer\"}\nUNIT_STATE={\"props\":{\"ActiveState\":\"inactive\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"UnitFileState\":\"static\"},\"rc\":0,\"unit\":\"homelab-backup-v2.service\"}\nUNIT_STATE={\"props\":{\"ActiveState\":\"active\",\"LastTriggerUSec\":\"Mon 2026-09-21 04:06:11 MSK\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"Mon 2026-09-21 04:36:17 MSK\",\"Result\":\"success\",\"SubState\":\"waiting\",\"UnitFileState\":\"enabled\"},\"rc\":0,\"unit\":\"homelab-backup-audit.timer\"}\nUNIT_STATE={\"props\":{\"ActiveState\":\"active\",\"LastTriggerUSec\":\"Mon 2026-09-21 03:22:42 MSK\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"Tue 2026-09-22 03:32:10 MSK\",\"Result\":\"success\",\"SubState\":\"waiting\",\"UnitFileState\":\"enabled\"},\"rc\":0,\"unit\":\"homelab-backup-retention-gc.timer\"}\nUNIT_STATE={\"props\":{\"ActiveState\":\"inactive\",\"LastTriggerUSec\":\"\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"dead\",\"UnitFileState\":\"disabled\"},\"rc\":0,\"unit\":\"homelab-vm-cloud-quorum-queue.timer\"}\nUNIT_STATE={\"props\":{\"ActiveState\":\"inactive\",\"LastTriggerUSec\":\"\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"dead\",\"UnitFileState\":\"disabled\"},\"rc\":0,\"unit\":\"homelab-app-cloud-quorum-queue.timer\"}\nUNIT_STATE={\"props\":{\"ActiveState\":\"inactive\",\"LastTriggerUSec\":\"\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"dead\",\"UnitFileState\":\"disabled\"},\"rc\":0,\"unit\":\"homelab-control-backup.timer\"}\n===== LOCK_AND_RUNNING_PROCESS_CHECK =====\nRELEVANT_RUNNING_PROCESSES=[]\nNO_MUTATION=true\nNO_GIT_FETCH=true\nNO_SERVICE_ACTION=true\nNO_BACKUP_STARTED=true\nNO_RESTORE_STARTED=true\nNO_RETENTION_RUN_STARTED=true\nNO_DELETE=true\nNO_MAIL_SENT=true\nNO_WORKTREE_CREATED=true\nRCA_DURATION_SEC=0.679\nTASK_RESULT=PASS_HOMELAB_BACKUP_V2_FAILURE_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-V2-FAILURE-RCA-READONLY-1004R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\nWORKERS2_BACKUP_V2_FAILURE_RCA_END=1\n"
}
View Git Blame Copy Permalink