From 4060389738c2c6269aef2ac7d59c10fcf1de86dd Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Sun, 27 Sep 2026 21:48:06 +0000 Subject: [PATCH] runtime: publish SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1 --- ...BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.json | 38 ++ ...-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.txt | 414 +++++++++++++ runtime/latest.json | 39 +- runtime/latest.txt | 560 +++++++++++++----- 4 files changed, 896 insertions(+), 155 deletions(-) create mode 100644 runtime/history/SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.json create mode 100644 runtime/history/SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.txt diff --git a/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.json b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.json new file mode 100644 index 00000000..3e0057bb --- /dev/null +++ b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "change", + "component": "homelab-backup-lock-aware-rpo-recovery", + "started_at_utc": "2026-09-27T20:06:48Z", + "finished_at_utc": "2026-09-27T21:48:06Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "78e9898eb6e2b60241cabb6cd7725b09ff0770faa1ebc48e7e01882c30f03029", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": true, + "result_contract_valid": true, + "result_contract_status": "OK", + "result_contract_error": null, + "command_rc": 0, + "changes_made": true, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "MUTATED", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "ec33d4b1814e274b3b42698808583b2ff788fc28ebda27f071d75672b16ab009", + "sanitized_output_sha256": "ec33d4b1814e274b3b42698808583b2ff788fc28ebda27f071d75672b16ab009", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "BACKUP_LOCK_AWARE_RPO_RECOVERY_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1\nMODE=change\nCOMPONENT=homelab-backup-lock-aware-rpo-recovery\nMUTATION_BOUNDARY=ONLY_CURRENT_V2_VERIFY_TARGET_FOR_FRESH_AUDIT_RPO_BREACHES;AUDIT_ONLY_WHILE_HEAVY_LOCK_FREE;HEAVY_LOCK_ONLY_DURING_SINGLE_VERIFY_TARGET;EXCLUDES_CT110_ALREADY_FRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1234_STATUS=OK\nSOURCE_1234_RC=0\nSOURCE_1234_DECISION=PASS_NONRPO_LOCK_RCA_COLLECTED\nSOURCE_1234_NEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT\nSOURCE_1234_LOCK_SELF_INDUCED_PROVISIONAL=true\nSOURCE_1234_AUDIT_NON_RPO_COUNT=0\nSOURCE_1234_AUDIT_RPO_COUNT=11\n=== LIVE_IDENTITY ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nSOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_WORKTREE_CLEAN=true\nRECOVERY_START_EPOCH=1790539608\nRECOVERY_START_UTC=2026-09-27T20:06:48Z\n=== INITIAL_QUIET_AUDIT ===\nQUIET_AUDIT_PASS|tag=initial|attempt=1|rc=1|issue_count=11|rpo_count=11|stamp=2614978226898\nAUDIT0_RC=1\nAUDIT0_ISSUE_COUNT=11\nAUDIT0_RPO_COUNT=11\nAUDIT0_NON_RPO_COUNT=0\nAUDIT0_RPO_SET_BEGIN\ncontrol-bootstrap-v2\nvm111\nvm112\nvm113\nvm130\nvm150\nvm160\nvm170\nvm171\nvm190\nvm9130\nAUDIT0_RPO_SET_END\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n=== ADAPTIVE_LABEL_control-bootstrap-v2 ===\nQUIET_AUDIT_PASS|tag=pre-control-bootstrap-v2|attempt=1|rc=1|issue_count=11|rpo_count=11|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=control-bootstrap-v2|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":1,\"audit_label\":\"control-bootstrap-v2\",\"verify_target\":\"control-bootstrap-v2\",\"start_epoch\":1790539621}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=197\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=6a9ad8a7e2e40c8c36cade820e9ef71c85e8b9204277681db95017579e7835ec\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=control-bootstrap-v2\nSELFTEST_REPLACEMENT486=PASS\nVERIFY_TARGET_RESULT={\"generation\":\"control-bootstrap-v2-pve01-20260916T120057Z\",\"logical_id\":\"control-bootstrap-v2\",\"payload_bytes\":0,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=control-bootstrap-v2\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=control-bootstrap-v2\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=control-bootstrap-v2-pve01-20260916T120057Z\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=0\nVERIFY_RESULT_ALLOCATOR_HEALTHY=None\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":1,\"audit_label\":\"control-bootstrap-v2\",\"verify_target\":\"control-bootstrap-v2\",\"elapsed_seconds\":5,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=control-bootstrap-v2\nQUIET_AUDIT_PASS|tag=post-control-bootstrap-v2|attempt=1|rc=1|issue_count=10|rpo_count=10|stamp=2614978226898\nTARGET_RPO_CLEARED=control-bootstrap-v2\n=== ADAPTIVE_LABEL_vm111 ===\nQUIET_AUDIT_PASS|tag=pre-vm111|attempt=1|rc=1|issue_count=10|rpo_count=10|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=ct111|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":2,\"audit_label\":\"vm111\",\"verify_target\":\"ct111\",\"start_epoch\":1790539634}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=472\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=ee74b2f99919efc17e8c41a36e9a41c82a7ecc43fc23da826d36780b37a9670e\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=ct111\nSELFTEST_REPLACEMENT486=PASS\nLXC_EXECUTION_DIAG={\"executor\":\"remote-bounded-ssh-cwd-root\",\"node\":\"pve02\",\"producer_rc\":0,\"stderr_bytes_observed\":994,\"stderr_bytes_retained\":994,\"stderr_stream_sha256\":\"7d6fe7d03b821031eecbffa0af551ee1caab9a80b983f2ab9f4ca53236b77687\",\"stderr_truncated\":false}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-ct111-20260927T200714Z-2f72200d7742\",\"logical_id\":\"ct111\",\"payload_bytes\":417826152,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=ct111\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=ct111\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-ct111-20260927T200714Z-2f72200d7742\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=417826152\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":2,\"audit_label\":\"vm111\",\"verify_target\":\"ct111\",\"elapsed_seconds\":180,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=ct111\nQUIET_AUDIT_PASS|tag=post-vm111|attempt=1|rc=1|issue_count=9|rpo_count=9|stamp=2614978226898\nTARGET_RPO_CLEARED=vm111\n=== ADAPTIVE_LABEL_vm112 ===\nQUIET_AUDIT_PASS|tag=pre-vm112|attempt=1|rc=1|issue_count=9|rpo_count=9|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=ct112|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":3,\"audit_label\":\"vm112\",\"verify_target\":\"ct112\",\"start_epoch\":1790539822}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=561\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=4f15f56422eeecd2d1a8a0c6f960a9e284d2e9ce66d6abce724251961453bccb\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=ct112\nSELFTEST_REPLACEMENT486=PASS\nLXC_EXECUTION_DIAG={\"archive_bytes\":224039854,\"exception_class\":null,\"exception_message_sha256\":null,\"executor\":\"local-proven-pve270\",\"helper_rc\":0,\"node\":\"pve01\",\"producer_rc\":0,\"stderr_bytes_observed\":1367,\"stderr_bytes_retained\":1367,\"stderr_stream_sha256\":\"4bb8f468cb6fcffd1af7303399038ec6b5cd9e126b6975ddaaac0b2f4a13f723\",\"stderr_truncated\":false}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-ct112-20260927T201022Z-6cdb5a555315\",\"logical_id\":\"ct112\",\"payload_bytes\":224039854,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=ct112\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=ct112\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-ct112-20260927T201022Z-6cdb5a555315\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=224039854\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":3,\"audit_label\":\"vm112\",\"verify_target\":\"ct112\",\"elapsed_seconds\":85,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=ct112\nQUIET_AUDIT_PASS|tag=post-vm112|attempt=1|rc=1|issue_count=8|rpo_count=8|stamp=2614978226898\nTARGET_RPO_CLEARED=vm112\n=== ADAPTIVE_LABEL_vm113 ===\nQUIET_AUDIT_PASS|tag=pre-vm113|attempt=1|rc=1|issue_count=8|rpo_count=8|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=ct113|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":4,\"audit_label\":\"vm113\",\"verify_target\":\"ct113\",\"start_epoch\":1790539915}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=472\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=fa7bc3f5137a8c4d7633f40a686625f2f3b3cf2a8bc59314e388d77e50a3e22a\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=ct113\nSELFTEST_REPLACEMENT486=PASS\nLXC_EXECUTION_DIAG={\"executor\":\"remote-bounded-ssh-cwd-root\",\"node\":\"pve02\",\"producer_rc\":0,\"stderr_bytes_observed\":997,\"stderr_bytes_retained\":997,\"stderr_stream_sha256\":\"be14e6c3e664cdc3483e88915ce87df5390e191db3cafd82b528b7ebecce4c57\",\"stderr_truncated\":false}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-ct113-20260927T201155Z-87c3bfe1d555\",\"logical_id\":\"ct113\",\"payload_bytes\":221991399,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=ct113\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=ct113\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-ct113-20260927T201155Z-87c3bfe1d555\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=221991399\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":4,\"audit_label\":\"vm113\",\"verify_target\":\"ct113\",\"elapsed_seconds\":92,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=ct113\nQUIET_AUDIT_PASS|tag=post-vm113|attempt=1|rc=1|issue_count=7|rpo_count=7|stamp=2614978226898\nTARGET_RPO_CLEARED=vm113\n=== ADAPTIVE_LABEL_vm150 ===\nQUIET_AUDIT_PASS|tag=pre-vm150|attempt=1|rc=1|issue_count=7|rpo_count=7|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=vm150|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":5,\"audit_label\":\"vm150\",\"verify_target\":\"vm150\",\"start_epoch\":1790540014}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=363\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=2b7fdf9f070f942f8fd11708790f350361d94d23b23818fc0811f7742d02a549\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm150\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":912043917,\"archive_selected\":true,\"node\":\"pve01\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":150}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm150-20260927T201335Z-2ba76459e7c5\",\"logical_id\":\"vm150\",\"payload_bytes\":912043917,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm150\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm150\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm150-20260927T201335Z-2ba76459e7c5\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=912043917\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":5,\"audit_label\":\"vm150\",\"verify_target\":\"vm150\",\"elapsed_seconds\":149,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm150\nQUIET_AUDIT_PASS|tag=post-vm150|attempt=1|rc=1|issue_count=6|rpo_count=6|stamp=2614978226898\nTARGET_RPO_CLEARED=vm150\n=== ADAPTIVE_LABEL_vm190 ===\nQUIET_AUDIT_PASS|tag=pre-vm190|attempt=1|rc=1|issue_count=6|rpo_count=6|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=vm190|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":6,\"audit_label\":\"vm190\",\"verify_target\":\"vm190\",\"start_epoch\":1790540172}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=365\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=0790485f05e9dc81ab27de10b12fd506674786b8ff7c437dcdde00dbdf3e0640\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm190\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":1645890869,\"archive_selected\":true,\"node\":\"pve03\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":190}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm190-20260927T201613Z-662906757ca1\",\"logical_id\":\"vm190\",\"payload_bytes\":1645890869,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm190\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm190\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm190-20260927T201613Z-662906757ca1\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=1645890869\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":6,\"audit_label\":\"vm190\",\"verify_target\":\"vm190\",\"elapsed_seconds\":274,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm190\nQUIET_AUDIT_PASS|tag=post-vm190|attempt=1|rc=1|issue_count=5|rpo_count=5|stamp=2614978226898\nTARGET_RPO_CLEARED=vm190\n=== ADAPTIVE_LABEL_vm170 ===\nQUIET_AUDIT_PASS|tag=pre-vm170|attempt=1|rc=1|issue_count=5|rpo_count=5|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=vm170|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":7,\"audit_label\":\"vm170\",\"verify_target\":\"vm170\",\"start_epoch\":1790540454}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=365\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=5a45794eb1a2944f763b9d381c31078b5c30c4b6d7ea97f3bee5068bc03b5ca5\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm170\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":8232066517,\"archive_selected\":true,\"node\":\"pve01\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":170}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm170-20260927T202054Z-1daa87a3cfbc\",\"logical_id\":\"vm170\",\"payload_bytes\":8232066517,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm170\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm170\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm170-20260927T202054Z-1daa87a3cfbc\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=8232066517\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":7,\"audit_label\":\"vm170\",\"verify_target\":\"vm170\",\"elapsed_seconds\":1099,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm170\nQUIET_AUDIT_PASS|tag=post-vm170|attempt=1|rc=1|issue_count=4|rpo_count=4|stamp=2616807252105\nTARGET_RPO_CLEARED=vm170\n=== ADAPTIVE_LABEL_vm171 ===\nQUIET_AUDIT_PASS|tag=pre-vm171|attempt=1|rc=1|issue_count=4|rpo_count=4|stamp=2616807252105\nTARGET_LOCK_ACQUIRED|tag=vm171|attempt=1|stamp=2616807252105\nTARGETED_REFRESH_BEGIN={\"iteration\":8,\"audit_label\":\"vm171\",\"verify_target\":\"vm171\",\"start_epoch\":1790541566}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=365\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=55865e4af68895bd93720362070400a825f2456605a5f64631fdbf3ec97fcd3e\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm171\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":2061491229,\"archive_selected\":true,\"node\":\"pve01\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":171}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm171-20260927T203926Z-61ca38fe3ac5\",\"logical_id\":\"vm171\",\"payload_bytes\":2061491229,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm171\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm171\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm171-20260927T203926Z-61ca38fe3ac5\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=2061491229\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":8,\"audit_label\":\"vm171\",\"verify_target\":\"vm171\",\"elapsed_seconds\":319,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm171\nQUIET_AUDIT_PASS|tag=post-vm171|attempt=1|rc=1|issue_count=3|rpo_count=3|stamp=2616807252105\nTARGET_RPO_CLEARED=vm171\n=== ADAPTIVE_LABEL_vm9130 ===\nQUIET_AUDIT_PASS|tag=pre-vm9130|attempt=1|rc=1|issue_count=3|rpo_count=3|stamp=2616807252105\nTARGET_LOCK_ACQUIRED|tag=vm9130|attempt=1|stamp=2616807252105\nTARGETED_REFRESH_BEGIN={\"iteration\":9,\"audit_label\":\"vm9130\",\"verify_target\":\"vm9130\",\"start_epoch\":1790541892}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=370\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=90516b2d72a05634cbd971a9dc6321061fa0ba3a9ae39be8a5f01cfc36dcf7e2\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm9130\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":10968466823,\"archive_selected\":true,\"node\":\"pve02\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":9130}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm9130-20260927T204452Z-91cc0cc7ecc5\",\"logical_id\":\"vm9130\",\"payload_bytes\":10968466823,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm9130\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm9130\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm9130-20260927T204452Z-91cc0cc7ecc5\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=10968466823\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":9,\"audit_label\":\"vm9130\",\"verify_target\":\"vm9130\",\"elapsed_seconds\":1603,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm9130\nQUIET_AUDIT_PASS|tag=post-vm9130|attempt=1|rc=1|issue_count=2|rpo_count=2|stamp=2619094297661\nTARGET_RPO_CLEARED=vm9130\n=== ADAPTIVE_LABEL_vm160 ===\nQUIET_AUDIT_PASS|tag=pre-vm160|attempt=1|rc=1|issue_count=2|rpo_count=2|stamp=2619094297661\nTARGET_LOCK_ACQUIRED|tag=vm160|attempt=1|stamp=2619094297661\nTARGETED_REFRESH_BEGIN={\"iteration\":10,\"audit_label\":\"vm160\",\"verify_target\":\"vm160\",\"start_epoch\":1790543503}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=365\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=d1a71b612d49c80016a8831df98ae906e50b267b7fdead624c68099e96b43296\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm160\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":3805401657,\"archive_selected\":true,\"node\":\"pve02\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":160}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm160-20260927T211143Z-0cadc4ac0b12\",\"logical_id\":\"vm160\",\"payload_bytes\":3805401657,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm160\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm160\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm160-20260927T211143Z-0cadc4ac0b12\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=3805401657\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":10,\"audit_label\":\"vm160\",\"verify_target\":\"vm160\",\"elapsed_seconds\":577,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm160\nQUIET_AUDIT_PASS|tag=post-vm160|attempt=1|rc=1|issue_count=1|rpo_count=1|stamp=2619094297661\nTARGET_RPO_CLEARED=vm160\n=== ADAPTIVE_LABEL_vm130 ===\nQUIET_AUDIT_PASS|tag=pre-vm130|attempt=1|rc=1|issue_count=1|rpo_count=1|stamp=2619094297661\nTARGET_LOCK_ACQUIRED|tag=vm130|attempt=1|stamp=2619094297661\nTARGETED_REFRESH_BEGIN={\"iteration\":11,\"audit_label\":\"vm130\",\"verify_target\":\"vm130\",\"start_epoch\":1790544088}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=367\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=1cdb7027b911f85d1ee9a35afca3dba5641a6891231cea2afcf01af152522157\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm130\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":11053999065,\"archive_selected\":true,\"node\":\"pve03\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":130}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm130-20260927T212128Z-0046d7efc414\",\"logical_id\":\"vm130\",\"payload_bytes\":11053999065,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm130\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm130\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm130-20260927T212128Z-0046d7efc414\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=11053999065\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":11,\"audit_label\":\"vm130\",\"verify_target\":\"vm130\",\"elapsed_seconds\":1590,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm130\nQUIET_AUDIT_PASS|tag=post-vm130|attempt=1|rc=0|issue_count=0|rpo_count=0|stamp=2621202326883\nTARGET_RPO_CLEARED=vm130\n=== FINAL_QUIET_AUDIT ===\nQUIET_AUDIT_PASS|tag=final|attempt=1|rc=0|issue_count=0|rpo_count=0|stamp=2621202326883\nFINAL_AUDIT_RC=0\nFINAL_ISSUE_COUNT=0\nFINAL_RPO_COUNT=0\nFINAL_NON_RPO_COUNT=0\nFINAL_AUDIT_OUTPUT_BEGIN\nTARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55069|generation=app-filebrowser-pve01-20260927T063003Z\nTARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=54902|generation=app-hbarhub-pve01-20260927T063154Z\nTARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55003|generation=app-immich-pve01-20260927T063041Z\nTARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55050|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nTARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=6055|generation=control-bootstrap-v2-pve01-20260916T120057Z\nTARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=7378|legacy_age_seconds=2045313|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17\nTARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=5867|replica_count=2|state_id=ct111|generation=v2-ct111-20260927T200714Z-2f72200d7742\nTARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=5774|legacy_age_seconds=2083808|replica_count=2|state_id=ct112|generation=v2-ct112-20260927T201022Z-6cdb5a555315\nTARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=5674|replica_count=2|state_id=ct113|generation=v2-ct113-20260927T201155Z-87c3bfe1d555\nTARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=3|replica_count=2|state_id=vm130|generation=v2-vm130-20260927T212128Z-0046d7efc414\nTARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=5518|legacy_age_seconds=2045429|replica_count=2|state_id=vm150|generation=v2-vm150-20260927T201335Z-2ba76459e7c5\nTARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=1601|replica_count=2|state_id=vm160|generation=v2-vm160-20260927T211143Z-0cadc4ac0b12\nTARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=4128|legacy_age_seconds=990050|replica_count=2|state_id=vm170|generation=v2-vm170-20260927T202054Z-1daa87a3cfbc\nTARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=3796|legacy_age_seconds=1019582|replica_count=2|state_id=vm171|generation=v2-vm171-20260927T203926Z-61ca38fe3ac5\nTARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=5235|replica_count=2|state_id=vm190|generation=v2-vm190-20260927T201613Z-662906757ca1\nTARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=2186|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260927T204452Z-91cc0cc7ecc5\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nFINAL_AUDIT_OUTPUT_END\nFINAL_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nRECOVERY_END_EPOCH=1790545685\nRECOVERY_ELAPSED_SECONDS=6077\n=== SUCCESS_MAIL_POLICY_WINDOW ===\nTARGETED_SUCCESS_BACKUP_EVENTS=4\nTARGETED_SUCCESS_MAIL_TO_ROOT_EVENTS=0\nSUCCESS_MAIL_POLICY_ACCEPTED=true\n=== CONTROL_PLANE_POST ===\nWRAPPER_SHA256_POST=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256_POST=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256_POST=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nV2_TIMER_ACTIVE=active\nV2_TIMER_ENABLED=enabled\nAUDIT_TIMER_ACTIVE=active\nAUDIT_TIMER_ENABLED=enabled\nCHANGES_MADE=true\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=MUTATED\nTARGETED_REFRESH_COUNT=11\nTASK_RESULT=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN\nDECISION=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN\nNEXT_GATE=FINAL_BACKUP_ACCEPTANCE_CLOSEOUT\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1\",\"status\":\"OK\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.txt b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.txt new file mode 100644 index 00000000..ad4d530e --- /dev/null +++ b/runtime/history/SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.txt @@ -0,0 +1,414 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=change +COMPONENT=homelab-backup-lock-aware-rpo-recovery +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=78e9898eb6e2b60241cabb6cd7725b09ff0770faa1ebc48e7e01882c30f03029 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=true +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=OK +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=true +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=MUTATED +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=ec33d4b1814e274b3b42698808583b2ff788fc28ebda27f071d75672b16ab009 +SANITIZED_OUTPUT_SHA256=ec33d4b1814e274b3b42698808583b2ff788fc28ebda27f071d75672b16ab009 +OUTPUT_BEGIN +BACKUP_LOCK_AWARE_RPO_RECOVERY_BEGIN=true +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1 +MODE=change +COMPONENT=homelab-backup-lock-aware-rpo-recovery +MUTATION_BOUNDARY=ONLY_CURRENT_V2_VERIFY_TARGET_FOR_FRESH_AUDIT_RPO_BREACHES;AUDIT_ONLY_WHILE_HEAVY_LOCK_FREE;HEAVY_LOCK_ONLY_DURING_SINGLE_VERIFY_TARGET;EXCLUDES_CT110_ALREADY_FRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE +=== AUTHORITY === +HOST=pve01 +CLUSTER_QUORATE=Yes +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +SOURCE_1234_STATUS=OK +SOURCE_1234_RC=0 +SOURCE_1234_DECISION=PASS_NONRPO_LOCK_RCA_COLLECTED +SOURCE_1234_NEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT +SOURCE_1234_LOCK_SELF_INDUCED_PROVISIONAL=true +SOURCE_1234_AUDIT_NON_RPO_COUNT=0 +SOURCE_1234_AUDIT_RPO_COUNT=11 +=== LIVE_IDENTITY === +WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 +VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad +AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +SOURCE_WORKTREE_CLEAN=true +RECOVERY_START_EPOCH=1790539608 +RECOVERY_START_UTC=2026-09-27T20:06:48Z +=== INITIAL_QUIET_AUDIT === +QUIET_AUDIT_PASS|tag=initial|attempt=1|rc=1|issue_count=11|rpo_count=11|stamp=2614978226898 +AUDIT0_RC=1 +AUDIT0_ISSUE_COUNT=11 +AUDIT0_RPO_COUNT=11 +AUDIT0_NON_RPO_COUNT=0 +AUDIT0_RPO_SET_BEGIN +control-bootstrap-v2 +vm111 +vm112 +vm113 +vm130 +vm150 +vm160 +vm170 +vm171 +vm190 +vm9130 +AUDIT0_RPO_SET_END +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +=== ADAPTIVE_LABEL_control-bootstrap-v2 === +QUIET_AUDIT_PASS|tag=pre-control-bootstrap-v2|attempt=1|rc=1|issue_count=11|rpo_count=11|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=control-bootstrap-v2|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":1,"audit_label":"control-bootstrap-v2","verify_target":"control-bootstrap-v2","start_epoch":1790539621} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=197 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=6a9ad8a7e2e40c8c36cade820e9ef71c85e8b9204277681db95017579e7835ec +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=control-bootstrap-v2 +SELFTEST_REPLACEMENT486=PASS +VERIFY_TARGET_RESULT={"generation":"control-bootstrap-v2-pve01-20260916T120057Z","logical_id":"control-bootstrap-v2","payload_bytes":0,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=control-bootstrap-v2 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=control-bootstrap-v2 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=control-bootstrap-v2-pve01-20260916T120057Z +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=0 +VERIFY_RESULT_ALLOCATOR_HEALTHY=None +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":1,"audit_label":"control-bootstrap-v2","verify_target":"control-bootstrap-v2","elapsed_seconds":5,"accepted":true} +TARGET_LOCK_RELEASED|tag=control-bootstrap-v2 +QUIET_AUDIT_PASS|tag=post-control-bootstrap-v2|attempt=1|rc=1|issue_count=10|rpo_count=10|stamp=2614978226898 +TARGET_RPO_CLEARED=control-bootstrap-v2 +=== ADAPTIVE_LABEL_vm111 === +QUIET_AUDIT_PASS|tag=pre-vm111|attempt=1|rc=1|issue_count=10|rpo_count=10|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=ct111|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":2,"audit_label":"vm111","verify_target":"ct111","start_epoch":1790539634} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=472 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=ee74b2f99919efc17e8c41a36e9a41c82a7ecc43fc23da826d36780b37a9670e +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=ct111 +SELFTEST_REPLACEMENT486=PASS +LXC_EXECUTION_DIAG={"executor":"remote-bounded-ssh-cwd-root","node":"pve02","producer_rc":0,"stderr_bytes_observed":994,"stderr_bytes_retained":994,"stderr_stream_sha256":"7d6fe7d03b821031eecbffa0af551ee1caab9a80b983f2ab9f4ca53236b77687","stderr_truncated":false} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct111-20260927T200714Z-2f72200d7742","logical_id":"ct111","payload_bytes":417826152,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=ct111 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=ct111 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-ct111-20260927T200714Z-2f72200d7742 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=417826152 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":2,"audit_label":"vm111","verify_target":"ct111","elapsed_seconds":180,"accepted":true} +TARGET_LOCK_RELEASED|tag=ct111 +QUIET_AUDIT_PASS|tag=post-vm111|attempt=1|rc=1|issue_count=9|rpo_count=9|stamp=2614978226898 +TARGET_RPO_CLEARED=vm111 +=== ADAPTIVE_LABEL_vm112 === +QUIET_AUDIT_PASS|tag=pre-vm112|attempt=1|rc=1|issue_count=9|rpo_count=9|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=ct112|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":3,"audit_label":"vm112","verify_target":"ct112","start_epoch":1790539822} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=561 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=4f15f56422eeecd2d1a8a0c6f960a9e284d2e9ce66d6abce724251961453bccb +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=ct112 +SELFTEST_REPLACEMENT486=PASS +LXC_EXECUTION_DIAG={"archive_bytes":224039854,"exception_class":null,"exception_message_sha256":null,"executor":"local-proven-pve270","helper_rc":0,"node":"pve01","producer_rc":0,"stderr_bytes_observed":1367,"stderr_bytes_retained":1367,"stderr_stream_sha256":"4bb8f468cb6fcffd1af7303399038ec6b5cd9e126b6975ddaaac0b2f4a13f723","stderr_truncated":false} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct112-20260927T201022Z-6cdb5a555315","logical_id":"ct112","payload_bytes":224039854,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=ct112 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=ct112 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-ct112-20260927T201022Z-6cdb5a555315 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=224039854 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":3,"audit_label":"vm112","verify_target":"ct112","elapsed_seconds":85,"accepted":true} +TARGET_LOCK_RELEASED|tag=ct112 +QUIET_AUDIT_PASS|tag=post-vm112|attempt=1|rc=1|issue_count=8|rpo_count=8|stamp=2614978226898 +TARGET_RPO_CLEARED=vm112 +=== ADAPTIVE_LABEL_vm113 === +QUIET_AUDIT_PASS|tag=pre-vm113|attempt=1|rc=1|issue_count=8|rpo_count=8|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=ct113|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":4,"audit_label":"vm113","verify_target":"ct113","start_epoch":1790539915} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=472 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=fa7bc3f5137a8c4d7633f40a686625f2f3b3cf2a8bc59314e388d77e50a3e22a +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=ct113 +SELFTEST_REPLACEMENT486=PASS +LXC_EXECUTION_DIAG={"executor":"remote-bounded-ssh-cwd-root","node":"pve02","producer_rc":0,"stderr_bytes_observed":997,"stderr_bytes_retained":997,"stderr_stream_sha256":"be14e6c3e664cdc3483e88915ce87df5390e191db3cafd82b528b7ebecce4c57","stderr_truncated":false} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct113-20260927T201155Z-87c3bfe1d555","logical_id":"ct113","payload_bytes":221991399,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=ct113 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=ct113 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-ct113-20260927T201155Z-87c3bfe1d555 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=221991399 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":4,"audit_label":"vm113","verify_target":"ct113","elapsed_seconds":92,"accepted":true} +TARGET_LOCK_RELEASED|tag=ct113 +QUIET_AUDIT_PASS|tag=post-vm113|attempt=1|rc=1|issue_count=7|rpo_count=7|stamp=2614978226898 +TARGET_RPO_CLEARED=vm113 +=== ADAPTIVE_LABEL_vm150 === +QUIET_AUDIT_PASS|tag=pre-vm150|attempt=1|rc=1|issue_count=7|rpo_count=7|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=vm150|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":5,"audit_label":"vm150","verify_target":"vm150","start_epoch":1790540014} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=363 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=2b7fdf9f070f942f8fd11708790f350361d94d23b23818fc0811f7742d02a549 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm150 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":912043917,"archive_selected":true,"node":"pve01","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":150} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm150-20260927T201335Z-2ba76459e7c5","logical_id":"vm150","payload_bytes":912043917,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm150 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm150 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm150-20260927T201335Z-2ba76459e7c5 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=912043917 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":5,"audit_label":"vm150","verify_target":"vm150","elapsed_seconds":149,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm150 +QUIET_AUDIT_PASS|tag=post-vm150|attempt=1|rc=1|issue_count=6|rpo_count=6|stamp=2614978226898 +TARGET_RPO_CLEARED=vm150 +=== ADAPTIVE_LABEL_vm190 === +QUIET_AUDIT_PASS|tag=pre-vm190|attempt=1|rc=1|issue_count=6|rpo_count=6|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=vm190|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":6,"audit_label":"vm190","verify_target":"vm190","start_epoch":1790540172} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=365 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=0790485f05e9dc81ab27de10b12fd506674786b8ff7c437dcdde00dbdf3e0640 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm190 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":1645890869,"archive_selected":true,"node":"pve03","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":190} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm190-20260927T201613Z-662906757ca1","logical_id":"vm190","payload_bytes":1645890869,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm190 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm190 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm190-20260927T201613Z-662906757ca1 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=1645890869 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":6,"audit_label":"vm190","verify_target":"vm190","elapsed_seconds":274,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm190 +QUIET_AUDIT_PASS|tag=post-vm190|attempt=1|rc=1|issue_count=5|rpo_count=5|stamp=2614978226898 +TARGET_RPO_CLEARED=vm190 +=== ADAPTIVE_LABEL_vm170 === +QUIET_AUDIT_PASS|tag=pre-vm170|attempt=1|rc=1|issue_count=5|rpo_count=5|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=vm170|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":7,"audit_label":"vm170","verify_target":"vm170","start_epoch":1790540454} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=365 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=5a45794eb1a2944f763b9d381c31078b5c30c4b6d7ea97f3bee5068bc03b5ca5 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm170 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":8232066517,"archive_selected":true,"node":"pve01","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":170} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm170-20260927T202054Z-1daa87a3cfbc","logical_id":"vm170","payload_bytes":8232066517,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm170 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm170 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm170-20260927T202054Z-1daa87a3cfbc +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=8232066517 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":7,"audit_label":"vm170","verify_target":"vm170","elapsed_seconds":1099,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm170 +QUIET_AUDIT_PASS|tag=post-vm170|attempt=1|rc=1|issue_count=4|rpo_count=4|stamp=2616807252105 +TARGET_RPO_CLEARED=vm170 +=== ADAPTIVE_LABEL_vm171 === +QUIET_AUDIT_PASS|tag=pre-vm171|attempt=1|rc=1|issue_count=4|rpo_count=4|stamp=2616807252105 +TARGET_LOCK_ACQUIRED|tag=vm171|attempt=1|stamp=2616807252105 +TARGETED_REFRESH_BEGIN={"iteration":8,"audit_label":"vm171","verify_target":"vm171","start_epoch":1790541566} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=365 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=55865e4af68895bd93720362070400a825f2456605a5f64631fdbf3ec97fcd3e +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm171 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":2061491229,"archive_selected":true,"node":"pve01","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":171} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm171-20260927T203926Z-61ca38fe3ac5","logical_id":"vm171","payload_bytes":2061491229,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm171 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm171 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm171-20260927T203926Z-61ca38fe3ac5 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=2061491229 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":8,"audit_label":"vm171","verify_target":"vm171","elapsed_seconds":319,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm171 +QUIET_AUDIT_PASS|tag=post-vm171|attempt=1|rc=1|issue_count=3|rpo_count=3|stamp=2616807252105 +TARGET_RPO_CLEARED=vm171 +=== ADAPTIVE_LABEL_vm9130 === +QUIET_AUDIT_PASS|tag=pre-vm9130|attempt=1|rc=1|issue_count=3|rpo_count=3|stamp=2616807252105 +TARGET_LOCK_ACQUIRED|tag=vm9130|attempt=1|stamp=2616807252105 +TARGETED_REFRESH_BEGIN={"iteration":9,"audit_label":"vm9130","verify_target":"vm9130","start_epoch":1790541892} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=370 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=90516b2d72a05634cbd971a9dc6321061fa0ba3a9ae39be8a5f01cfc36dcf7e2 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm9130 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":10968466823,"archive_selected":true,"node":"pve02","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":9130} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm9130-20260927T204452Z-91cc0cc7ecc5","logical_id":"vm9130","payload_bytes":10968466823,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm9130 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm9130 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm9130-20260927T204452Z-91cc0cc7ecc5 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=10968466823 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":9,"audit_label":"vm9130","verify_target":"vm9130","elapsed_seconds":1603,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm9130 +QUIET_AUDIT_PASS|tag=post-vm9130|attempt=1|rc=1|issue_count=2|rpo_count=2|stamp=2619094297661 +TARGET_RPO_CLEARED=vm9130 +=== ADAPTIVE_LABEL_vm160 === +QUIET_AUDIT_PASS|tag=pre-vm160|attempt=1|rc=1|issue_count=2|rpo_count=2|stamp=2619094297661 +TARGET_LOCK_ACQUIRED|tag=vm160|attempt=1|stamp=2619094297661 +TARGETED_REFRESH_BEGIN={"iteration":10,"audit_label":"vm160","verify_target":"vm160","start_epoch":1790543503} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=365 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=d1a71b612d49c80016a8831df98ae906e50b267b7fdead624c68099e96b43296 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm160 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":3805401657,"archive_selected":true,"node":"pve02","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":160} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm160-20260927T211143Z-0cadc4ac0b12","logical_id":"vm160","payload_bytes":3805401657,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm160 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm160 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm160-20260927T211143Z-0cadc4ac0b12 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=3805401657 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":10,"audit_label":"vm160","verify_target":"vm160","elapsed_seconds":577,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm160 +QUIET_AUDIT_PASS|tag=post-vm160|attempt=1|rc=1|issue_count=1|rpo_count=1|stamp=2619094297661 +TARGET_RPO_CLEARED=vm160 +=== ADAPTIVE_LABEL_vm130 === +QUIET_AUDIT_PASS|tag=pre-vm130|attempt=1|rc=1|issue_count=1|rpo_count=1|stamp=2619094297661 +TARGET_LOCK_ACQUIRED|tag=vm130|attempt=1|stamp=2619094297661 +TARGETED_REFRESH_BEGIN={"iteration":11,"audit_label":"vm130","verify_target":"vm130","start_epoch":1790544088} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=367 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=1cdb7027b911f85d1ee9a35afca3dba5641a6891231cea2afcf01af152522157 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm130 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":11053999065,"archive_selected":true,"node":"pve03","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":130} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm130-20260927T212128Z-0046d7efc414","logical_id":"vm130","payload_bytes":11053999065,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm130 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm130 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm130-20260927T212128Z-0046d7efc414 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=11053999065 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":11,"audit_label":"vm130","verify_target":"vm130","elapsed_seconds":1590,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm130 +QUIET_AUDIT_PASS|tag=post-vm130|attempt=1|rc=0|issue_count=0|rpo_count=0|stamp=2621202326883 +TARGET_RPO_CLEARED=vm130 +=== FINAL_QUIET_AUDIT === +QUIET_AUDIT_PASS|tag=final|attempt=1|rc=0|issue_count=0|rpo_count=0|stamp=2621202326883 +FINAL_AUDIT_RC=0 +FINAL_ISSUE_COUNT=0 +FINAL_RPO_COUNT=0 +FINAL_NON_RPO_COUNT=0 +FINAL_AUDIT_OUTPUT_BEGIN +TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55069|generation=app-filebrowser-pve01-20260927T063003Z +TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=54902|generation=app-hbarhub-pve01-20260927T063154Z +TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55003|generation=app-immich-pve01-20260927T063041Z +TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55050|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=6055|generation=control-bootstrap-v2-pve01-20260916T120057Z +TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=7378|legacy_age_seconds=2045313|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17 +TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=5867|replica_count=2|state_id=ct111|generation=v2-ct111-20260927T200714Z-2f72200d7742 +TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=5774|legacy_age_seconds=2083808|replica_count=2|state_id=ct112|generation=v2-ct112-20260927T201022Z-6cdb5a555315 +TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=5674|replica_count=2|state_id=ct113|generation=v2-ct113-20260927T201155Z-87c3bfe1d555 +TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=3|replica_count=2|state_id=vm130|generation=v2-vm130-20260927T212128Z-0046d7efc414 +TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=5518|legacy_age_seconds=2045429|replica_count=2|state_id=vm150|generation=v2-vm150-20260927T201335Z-2ba76459e7c5 +TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=1601|replica_count=2|state_id=vm160|generation=v2-vm160-20260927T211143Z-0cadc4ac0b12 +TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=4128|legacy_age_seconds=990050|replica_count=2|state_id=vm170|generation=v2-vm170-20260927T202054Z-1daa87a3cfbc +TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=3796|legacy_age_seconds=1019582|replica_count=2|state_id=vm171|generation=v2-vm171-20260927T203926Z-61ca38fe3ac5 +TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=5235|replica_count=2|state_id=vm190|generation=v2-vm190-20260927T201613Z-662906757ca1 +TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=2186|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260927T204452Z-91cc0cc7ecc5 +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +FINAL_AUDIT_OUTPUT_END +FINAL_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +RECOVERY_END_EPOCH=1790545685 +RECOVERY_ELAPSED_SECONDS=6077 +=== SUCCESS_MAIL_POLICY_WINDOW === +TARGETED_SUCCESS_BACKUP_EVENTS=4 +TARGETED_SUCCESS_MAIL_TO_ROOT_EVENTS=0 +SUCCESS_MAIL_POLICY_ACCEPTED=true +=== CONTROL_PLANE_POST === +WRAPPER_SHA256_POST=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +HELPER_SHA256_POST=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 +VZDUMP_CONF_SHA256_POST=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad +AUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +V2_TIMER_ACTIVE=active +V2_TIMER_ENABLED=enabled +AUDIT_TIMER_ACTIVE=active +AUDIT_TIMER_ENABLED=enabled +CHANGES_MADE=true +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=MUTATED +TARGETED_REFRESH_COUNT=11 +TASK_RESULT=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN +DECISION=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN +NEXT_GATE=FINAL_BACKUP_ACCEPTANCE_CLOSEOUT +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1","status":"OK","changes_made":true,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 9d955f70..3e0057bb 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1 +1,38 @@ -{"schema_version":1,"channel":"homelab-runtime","command_id":"CONTEXT-AUTO-20260927T211701Z","status":"OK","rc":0,"host":"pve01","mode":"read-only","component":"cluster-context","started_at_utc":"2026-09-27T21:17:01Z","finished_at_utc":"2026-09-27T21:17:02Z","reference_register_checked":true,"reference_sha256":"5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66","error_register_checked":true,"error_register_sha256":"3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0","changes_made":false,"sanitized":true,"secrets_included":false,"private_addresses_included":false,"output":"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK"} +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "change", + "component": "homelab-backup-lock-aware-rpo-recovery", + "started_at_utc": "2026-09-27T20:06:48Z", + "finished_at_utc": "2026-09-27T21:48:06Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "78e9898eb6e2b60241cabb6cd7725b09ff0770faa1ebc48e7e01882c30f03029", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": true, + "result_contract_valid": true, + "result_contract_status": "OK", + "result_contract_error": null, + "command_rc": 0, + "changes_made": true, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "MUTATED", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "ec33d4b1814e274b3b42698808583b2ff788fc28ebda27f071d75672b16ab009", + "sanitized_output_sha256": "ec33d4b1814e274b3b42698808583b2ff788fc28ebda27f071d75672b16ab009", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "BACKUP_LOCK_AWARE_RPO_RECOVERY_BEGIN=true\nCOMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1\nMODE=change\nCOMPONENT=homelab-backup-lock-aware-rpo-recovery\nMUTATION_BOUNDARY=ONLY_CURRENT_V2_VERIFY_TARGET_FOR_FRESH_AUDIT_RPO_BREACHES;AUDIT_ONLY_WHILE_HEAVY_LOCK_FREE;HEAVY_LOCK_ONLY_DURING_SINGLE_VERIFY_TARGET;EXCLUDES_CT110_ALREADY_FRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1234_STATUS=OK\nSOURCE_1234_RC=0\nSOURCE_1234_DECISION=PASS_NONRPO_LOCK_RCA_COLLECTED\nSOURCE_1234_NEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT\nSOURCE_1234_LOCK_SELF_INDUCED_PROVISIONAL=true\nSOURCE_1234_AUDIT_NON_RPO_COUNT=0\nSOURCE_1234_AUDIT_RPO_COUNT=11\n=== LIVE_IDENTITY ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nSOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_WORKTREE_CLEAN=true\nRECOVERY_START_EPOCH=1790539608\nRECOVERY_START_UTC=2026-09-27T20:06:48Z\n=== INITIAL_QUIET_AUDIT ===\nQUIET_AUDIT_PASS|tag=initial|attempt=1|rc=1|issue_count=11|rpo_count=11|stamp=2614978226898\nAUDIT0_RC=1\nAUDIT0_ISSUE_COUNT=11\nAUDIT0_RPO_COUNT=11\nAUDIT0_NON_RPO_COUNT=0\nAUDIT0_RPO_SET_BEGIN\ncontrol-bootstrap-v2\nvm111\nvm112\nvm113\nvm130\nvm150\nvm160\nvm170\nvm171\nvm190\nvm9130\nAUDIT0_RPO_SET_END\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n=== ADAPTIVE_LABEL_control-bootstrap-v2 ===\nQUIET_AUDIT_PASS|tag=pre-control-bootstrap-v2|attempt=1|rc=1|issue_count=11|rpo_count=11|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=control-bootstrap-v2|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":1,\"audit_label\":\"control-bootstrap-v2\",\"verify_target\":\"control-bootstrap-v2\",\"start_epoch\":1790539621}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=197\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=6a9ad8a7e2e40c8c36cade820e9ef71c85e8b9204277681db95017579e7835ec\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=control-bootstrap-v2\nSELFTEST_REPLACEMENT486=PASS\nVERIFY_TARGET_RESULT={\"generation\":\"control-bootstrap-v2-pve01-20260916T120057Z\",\"logical_id\":\"control-bootstrap-v2\",\"payload_bytes\":0,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=control-bootstrap-v2\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=control-bootstrap-v2\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=control-bootstrap-v2-pve01-20260916T120057Z\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=0\nVERIFY_RESULT_ALLOCATOR_HEALTHY=None\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":1,\"audit_label\":\"control-bootstrap-v2\",\"verify_target\":\"control-bootstrap-v2\",\"elapsed_seconds\":5,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=control-bootstrap-v2\nQUIET_AUDIT_PASS|tag=post-control-bootstrap-v2|attempt=1|rc=1|issue_count=10|rpo_count=10|stamp=2614978226898\nTARGET_RPO_CLEARED=control-bootstrap-v2\n=== ADAPTIVE_LABEL_vm111 ===\nQUIET_AUDIT_PASS|tag=pre-vm111|attempt=1|rc=1|issue_count=10|rpo_count=10|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=ct111|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":2,\"audit_label\":\"vm111\",\"verify_target\":\"ct111\",\"start_epoch\":1790539634}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=472\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=ee74b2f99919efc17e8c41a36e9a41c82a7ecc43fc23da826d36780b37a9670e\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=ct111\nSELFTEST_REPLACEMENT486=PASS\nLXC_EXECUTION_DIAG={\"executor\":\"remote-bounded-ssh-cwd-root\",\"node\":\"pve02\",\"producer_rc\":0,\"stderr_bytes_observed\":994,\"stderr_bytes_retained\":994,\"stderr_stream_sha256\":\"7d6fe7d03b821031eecbffa0af551ee1caab9a80b983f2ab9f4ca53236b77687\",\"stderr_truncated\":false}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-ct111-20260927T200714Z-2f72200d7742\",\"logical_id\":\"ct111\",\"payload_bytes\":417826152,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=ct111\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=ct111\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-ct111-20260927T200714Z-2f72200d7742\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=417826152\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":2,\"audit_label\":\"vm111\",\"verify_target\":\"ct111\",\"elapsed_seconds\":180,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=ct111\nQUIET_AUDIT_PASS|tag=post-vm111|attempt=1|rc=1|issue_count=9|rpo_count=9|stamp=2614978226898\nTARGET_RPO_CLEARED=vm111\n=== ADAPTIVE_LABEL_vm112 ===\nQUIET_AUDIT_PASS|tag=pre-vm112|attempt=1|rc=1|issue_count=9|rpo_count=9|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=ct112|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":3,\"audit_label\":\"vm112\",\"verify_target\":\"ct112\",\"start_epoch\":1790539822}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=561\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=4f15f56422eeecd2d1a8a0c6f960a9e284d2e9ce66d6abce724251961453bccb\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=ct112\nSELFTEST_REPLACEMENT486=PASS\nLXC_EXECUTION_DIAG={\"archive_bytes\":224039854,\"exception_class\":null,\"exception_message_sha256\":null,\"executor\":\"local-proven-pve270\",\"helper_rc\":0,\"node\":\"pve01\",\"producer_rc\":0,\"stderr_bytes_observed\":1367,\"stderr_bytes_retained\":1367,\"stderr_stream_sha256\":\"4bb8f468cb6fcffd1af7303399038ec6b5cd9e126b6975ddaaac0b2f4a13f723\",\"stderr_truncated\":false}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-ct112-20260927T201022Z-6cdb5a555315\",\"logical_id\":\"ct112\",\"payload_bytes\":224039854,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=ct112\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=ct112\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-ct112-20260927T201022Z-6cdb5a555315\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=224039854\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":3,\"audit_label\":\"vm112\",\"verify_target\":\"ct112\",\"elapsed_seconds\":85,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=ct112\nQUIET_AUDIT_PASS|tag=post-vm112|attempt=1|rc=1|issue_count=8|rpo_count=8|stamp=2614978226898\nTARGET_RPO_CLEARED=vm112\n=== ADAPTIVE_LABEL_vm113 ===\nQUIET_AUDIT_PASS|tag=pre-vm113|attempt=1|rc=1|issue_count=8|rpo_count=8|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=ct113|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":4,\"audit_label\":\"vm113\",\"verify_target\":\"ct113\",\"start_epoch\":1790539915}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=472\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=fa7bc3f5137a8c4d7633f40a686625f2f3b3cf2a8bc59314e388d77e50a3e22a\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=ct113\nSELFTEST_REPLACEMENT486=PASS\nLXC_EXECUTION_DIAG={\"executor\":\"remote-bounded-ssh-cwd-root\",\"node\":\"pve02\",\"producer_rc\":0,\"stderr_bytes_observed\":997,\"stderr_bytes_retained\":997,\"stderr_stream_sha256\":\"be14e6c3e664cdc3483e88915ce87df5390e191db3cafd82b528b7ebecce4c57\",\"stderr_truncated\":false}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-ct113-20260927T201155Z-87c3bfe1d555\",\"logical_id\":\"ct113\",\"payload_bytes\":221991399,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=ct113\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=ct113\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-ct113-20260927T201155Z-87c3bfe1d555\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=221991399\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":4,\"audit_label\":\"vm113\",\"verify_target\":\"ct113\",\"elapsed_seconds\":92,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=ct113\nQUIET_AUDIT_PASS|tag=post-vm113|attempt=1|rc=1|issue_count=7|rpo_count=7|stamp=2614978226898\nTARGET_RPO_CLEARED=vm113\n=== ADAPTIVE_LABEL_vm150 ===\nQUIET_AUDIT_PASS|tag=pre-vm150|attempt=1|rc=1|issue_count=7|rpo_count=7|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=vm150|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":5,\"audit_label\":\"vm150\",\"verify_target\":\"vm150\",\"start_epoch\":1790540014}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=363\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=2b7fdf9f070f942f8fd11708790f350361d94d23b23818fc0811f7742d02a549\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm150\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":912043917,\"archive_selected\":true,\"node\":\"pve01\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":150}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm150-20260927T201335Z-2ba76459e7c5\",\"logical_id\":\"vm150\",\"payload_bytes\":912043917,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm150\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm150\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm150-20260927T201335Z-2ba76459e7c5\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=912043917\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":5,\"audit_label\":\"vm150\",\"verify_target\":\"vm150\",\"elapsed_seconds\":149,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm150\nQUIET_AUDIT_PASS|tag=post-vm150|attempt=1|rc=1|issue_count=6|rpo_count=6|stamp=2614978226898\nTARGET_RPO_CLEARED=vm150\n=== ADAPTIVE_LABEL_vm190 ===\nQUIET_AUDIT_PASS|tag=pre-vm190|attempt=1|rc=1|issue_count=6|rpo_count=6|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=vm190|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":6,\"audit_label\":\"vm190\",\"verify_target\":\"vm190\",\"start_epoch\":1790540172}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=365\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=0790485f05e9dc81ab27de10b12fd506674786b8ff7c437dcdde00dbdf3e0640\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm190\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":1645890869,\"archive_selected\":true,\"node\":\"pve03\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":190}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm190-20260927T201613Z-662906757ca1\",\"logical_id\":\"vm190\",\"payload_bytes\":1645890869,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm190\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm190\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm190-20260927T201613Z-662906757ca1\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=1645890869\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":6,\"audit_label\":\"vm190\",\"verify_target\":\"vm190\",\"elapsed_seconds\":274,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm190\nQUIET_AUDIT_PASS|tag=post-vm190|attempt=1|rc=1|issue_count=5|rpo_count=5|stamp=2614978226898\nTARGET_RPO_CLEARED=vm190\n=== ADAPTIVE_LABEL_vm170 ===\nQUIET_AUDIT_PASS|tag=pre-vm170|attempt=1|rc=1|issue_count=5|rpo_count=5|stamp=2614978226898\nTARGET_LOCK_ACQUIRED|tag=vm170|attempt=1|stamp=2614978226898\nTARGETED_REFRESH_BEGIN={\"iteration\":7,\"audit_label\":\"vm170\",\"verify_target\":\"vm170\",\"start_epoch\":1790540454}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=365\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=5a45794eb1a2944f763b9d381c31078b5c30c4b6d7ea97f3bee5068bc03b5ca5\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm170\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":8232066517,\"archive_selected\":true,\"node\":\"pve01\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":170}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm170-20260927T202054Z-1daa87a3cfbc\",\"logical_id\":\"vm170\",\"payload_bytes\":8232066517,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm170\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm170\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm170-20260927T202054Z-1daa87a3cfbc\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=8232066517\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":7,\"audit_label\":\"vm170\",\"verify_target\":\"vm170\",\"elapsed_seconds\":1099,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm170\nQUIET_AUDIT_PASS|tag=post-vm170|attempt=1|rc=1|issue_count=4|rpo_count=4|stamp=2616807252105\nTARGET_RPO_CLEARED=vm170\n=== ADAPTIVE_LABEL_vm171 ===\nQUIET_AUDIT_PASS|tag=pre-vm171|attempt=1|rc=1|issue_count=4|rpo_count=4|stamp=2616807252105\nTARGET_LOCK_ACQUIRED|tag=vm171|attempt=1|stamp=2616807252105\nTARGETED_REFRESH_BEGIN={\"iteration\":8,\"audit_label\":\"vm171\",\"verify_target\":\"vm171\",\"start_epoch\":1790541566}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=365\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=55865e4af68895bd93720362070400a825f2456605a5f64631fdbf3ec97fcd3e\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm171\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":2061491229,\"archive_selected\":true,\"node\":\"pve01\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":171}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm171-20260927T203926Z-61ca38fe3ac5\",\"logical_id\":\"vm171\",\"payload_bytes\":2061491229,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm171\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm171\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm171-20260927T203926Z-61ca38fe3ac5\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=2061491229\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":8,\"audit_label\":\"vm171\",\"verify_target\":\"vm171\",\"elapsed_seconds\":319,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm171\nQUIET_AUDIT_PASS|tag=post-vm171|attempt=1|rc=1|issue_count=3|rpo_count=3|stamp=2616807252105\nTARGET_RPO_CLEARED=vm171\n=== ADAPTIVE_LABEL_vm9130 ===\nQUIET_AUDIT_PASS|tag=pre-vm9130|attempt=1|rc=1|issue_count=3|rpo_count=3|stamp=2616807252105\nTARGET_LOCK_ACQUIRED|tag=vm9130|attempt=1|stamp=2616807252105\nTARGETED_REFRESH_BEGIN={\"iteration\":9,\"audit_label\":\"vm9130\",\"verify_target\":\"vm9130\",\"start_epoch\":1790541892}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=370\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=90516b2d72a05634cbd971a9dc6321061fa0ba3a9ae39be8a5f01cfc36dcf7e2\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm9130\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":10968466823,\"archive_selected\":true,\"node\":\"pve02\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":9130}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm9130-20260927T204452Z-91cc0cc7ecc5\",\"logical_id\":\"vm9130\",\"payload_bytes\":10968466823,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm9130\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm9130\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm9130-20260927T204452Z-91cc0cc7ecc5\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=10968466823\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":9,\"audit_label\":\"vm9130\",\"verify_target\":\"vm9130\",\"elapsed_seconds\":1603,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm9130\nQUIET_AUDIT_PASS|tag=post-vm9130|attempt=1|rc=1|issue_count=2|rpo_count=2|stamp=2619094297661\nTARGET_RPO_CLEARED=vm9130\n=== ADAPTIVE_LABEL_vm160 ===\nQUIET_AUDIT_PASS|tag=pre-vm160|attempt=1|rc=1|issue_count=2|rpo_count=2|stamp=2619094297661\nTARGET_LOCK_ACQUIRED|tag=vm160|attempt=1|stamp=2619094297661\nTARGETED_REFRESH_BEGIN={\"iteration\":10,\"audit_label\":\"vm160\",\"verify_target\":\"vm160\",\"start_epoch\":1790543503}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=365\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=d1a71b612d49c80016a8831df98ae906e50b267b7fdead624c68099e96b43296\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm160\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":3805401657,\"archive_selected\":true,\"node\":\"pve02\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":160}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm160-20260927T211143Z-0cadc4ac0b12\",\"logical_id\":\"vm160\",\"payload_bytes\":3805401657,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm160\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm160\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm160-20260927T211143Z-0cadc4ac0b12\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=3805401657\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":10,\"audit_label\":\"vm160\",\"verify_target\":\"vm160\",\"elapsed_seconds\":577,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm160\nQUIET_AUDIT_PASS|tag=post-vm160|attempt=1|rc=1|issue_count=1|rpo_count=1|stamp=2619094297661\nTARGET_RPO_CLEARED=vm160\n=== ADAPTIVE_LABEL_vm130 ===\nQUIET_AUDIT_PASS|tag=pre-vm130|attempt=1|rc=1|issue_count=1|rpo_count=1|stamp=2619094297661\nTARGET_LOCK_ACQUIRED|tag=vm130|attempt=1|stamp=2619094297661\nTARGETED_REFRESH_BEGIN={\"iteration\":11,\"audit_label\":\"vm130\",\"verify_target\":\"vm130\",\"start_epoch\":1790544088}\nVERIFY_TARGET_RC=0\nVERIFY_STDOUT_BYTES=367\nVERIFY_STDERR_BYTES=0\nVERIFY_STDOUT_SHA256=1cdb7027b911f85d1ee9a35afca3dba5641a6891231cea2afcf01af152522157\nVERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVERIFY_OUTPUT_BEGIN=vm130\nSELFTEST_REPLACEMENT486=PASS\nQEMU_EXECUTION_DIAG={\"archive_bytes\":11053999065,\"archive_selected\":true,\"node\":\"pve03\",\"producer_rc\":0,\"regular_file_count\":2,\"sidecar_count\":1,\"vmid\":130}\nVERIFY_TARGET_RESULT={\"allocator_healthy\":13,\"generation\":\"v2-vm130-20260927T212128Z-0046d7efc414\",\"logical_id\":\"vm130\",\"payload_bytes\":11053999065,\"replicas\":2,\"result\":\"SUCCESS\"}\nVERIFY_OUTPUT_END=vm130\nVERIFY_RESULT_LINE_COUNT=1\nVERIFY_RESULT_LOGICAL_ID=vm130\nVERIFY_RESULT_RESULT=SUCCESS\nVERIFY_RESULT_GENERATION=v2-vm130-20260927T212128Z-0046d7efc414\nVERIFY_RESULT_REPLICAS=2\nVERIFY_RESULT_PAYLOAD_BYTES=11053999065\nVERIFY_RESULT_ALLOCATOR_HEALTHY=13\nVERIFY_RESULT_ACCEPTED=true\nTARGETED_REFRESH_END={\"iteration\":11,\"audit_label\":\"vm130\",\"verify_target\":\"vm130\",\"elapsed_seconds\":1590,\"accepted\":true}\nTARGET_LOCK_RELEASED|tag=vm130\nQUIET_AUDIT_PASS|tag=post-vm130|attempt=1|rc=0|issue_count=0|rpo_count=0|stamp=2621202326883\nTARGET_RPO_CLEARED=vm130\n=== FINAL_QUIET_AUDIT ===\nQUIET_AUDIT_PASS|tag=final|attempt=1|rc=0|issue_count=0|rpo_count=0|stamp=2621202326883\nFINAL_AUDIT_RC=0\nFINAL_ISSUE_COUNT=0\nFINAL_RPO_COUNT=0\nFINAL_NON_RPO_COUNT=0\nFINAL_AUDIT_OUTPUT_BEGIN\nTARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55069|generation=app-filebrowser-pve01-20260927T063003Z\nTARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=54902|generation=app-hbarhub-pve01-20260927T063154Z\nTARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55003|generation=app-immich-pve01-20260927T063041Z\nTARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55050|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z\nTARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=6055|generation=control-bootstrap-v2-pve01-20260916T120057Z\nTARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=7378|legacy_age_seconds=2045313|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17\nTARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=5867|replica_count=2|state_id=ct111|generation=v2-ct111-20260927T200714Z-2f72200d7742\nTARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=5774|legacy_age_seconds=2083808|replica_count=2|state_id=ct112|generation=v2-ct112-20260927T201022Z-6cdb5a555315\nTARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=5674|replica_count=2|state_id=ct113|generation=v2-ct113-20260927T201155Z-87c3bfe1d555\nTARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=3|replica_count=2|state_id=vm130|generation=v2-vm130-20260927T212128Z-0046d7efc414\nTARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=5518|legacy_age_seconds=2045429|replica_count=2|state_id=vm150|generation=v2-vm150-20260927T201335Z-2ba76459e7c5\nTARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=1601|replica_count=2|state_id=vm160|generation=v2-vm160-20260927T211143Z-0cadc4ac0b12\nTARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=4128|legacy_age_seconds=990050|replica_count=2|state_id=vm170|generation=v2-vm170-20260927T202054Z-1daa87a3cfbc\nTARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=3796|legacy_age_seconds=1019582|replica_count=2|state_id=vm171|generation=v2-vm171-20260927T203926Z-61ca38fe3ac5\nTARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=5235|replica_count=2|state_id=vm190|generation=v2-vm190-20260927T201613Z-662906757ca1\nTARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=2186|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260927T204452Z-91cc0cc7ecc5\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nFINAL_AUDIT_OUTPUT_END\nFINAL_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nRECOVERY_END_EPOCH=1790545685\nRECOVERY_ELAPSED_SECONDS=6077\n=== SUCCESS_MAIL_POLICY_WINDOW ===\nTARGETED_SUCCESS_BACKUP_EVENTS=4\nTARGETED_SUCCESS_MAIL_TO_ROOT_EVENTS=0\nSUCCESS_MAIL_POLICY_ACCEPTED=true\n=== CONTROL_PLANE_POST ===\nWRAPPER_SHA256_POST=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256_POST=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256_POST=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nV2_TIMER_ACTIVE=active\nV2_TIMER_ENABLED=enabled\nAUDIT_TIMER_ACTIVE=active\nAUDIT_TIMER_ENABLED=enabled\nCHANGES_MADE=true\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=MUTATED\nTARGETED_REFRESH_COUNT=11\nTASK_RESULT=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN\nDECISION=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN\nNEXT_GATE=FINAL_BACKUP_ACCEPTANCE_CLOSEOUT\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1\",\"status\":\"OK\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/latest.txt b/runtime/latest.txt index 3c06791f..ad4d530e 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,162 +1,414 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=CONTEXT-AUTO-[PRIVATE_IP]701Z +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1 STATUS=OK RC=0 HOST=pve01 -COMPONENT=cluster-context -REFERENCE_SHA[PRIVATE_IP]e5154c41cb[PRIVATE_IP]aca68090be[PRIVATE_IP]bf[PRIVATE_IP]df[PRIVATE_IP] -ERROR_REGISTER_SHA[PRIVATE_IP]ec0f527ed3afeed[PRIVATE_IP]ee[PRIVATE_IP]bbfb[PRIVATE_IP]af4ba7ba0 +MODE=change +COMPONENT=homelab-backup-lock-aware-rpo-recovery +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=78e9898eb6e2b60241cabb6cd7725b09ff0770faa1ebc48e7e01882c30f03029 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=true +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=OK +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=true +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=MUTATED +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=ec33d4b1814e274b3b42698808583b2ff788fc28ebda27f071d75672b16ab009 +SANITIZED_OUTPUT_SHA256=ec33d4b1814e274b3b42698808583b2ff788fc28ebda27f071d75672b16ab009 OUTPUT_BEGIN -GENERATED_AT_UTC=[PRIVATE_IP]T21:17:01Z -Cluster information -------------------- -Name: homelab -Config Version: 3 -Transport: knet -Secure auth: on +BACKUP_LOCK_AWARE_RPO_RECOVERY_BEGIN=true +COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1 +MODE=change +COMPONENT=homelab-backup-lock-aware-rpo-recovery +MUTATION_BOUNDARY=ONLY_CURRENT_V2_VERIFY_TARGET_FOR_FRESH_AUDIT_RPO_BREACHES;AUDIT_ONLY_WHILE_HEAVY_LOCK_FREE;HEAVY_LOCK_ONLY_DURING_SINGLE_VERIFY_TARGET;EXCLUDES_CT110_ALREADY_FRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE +=== AUTHORITY === +HOST=pve01 +CLUSTER_QUORATE=Yes +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +SOURCE_1234_STATUS=OK +SOURCE_1234_RC=0 +SOURCE_1234_DECISION=PASS_NONRPO_LOCK_RCA_COLLECTED +SOURCE_1234_NEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT +SOURCE_1234_LOCK_SELF_INDUCED_PROVISIONAL=true +SOURCE_1234_AUDIT_NON_RPO_COUNT=0 +SOURCE_1234_AUDIT_RPO_COUNT=11 +=== LIVE_IDENTITY === +WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 +VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad +AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +SOURCE_WORKTREE_CLEAN=true +RECOVERY_START_EPOCH=1790539608 +RECOVERY_START_UTC=2026-09-27T20:06:48Z +=== INITIAL_QUIET_AUDIT === +QUIET_AUDIT_PASS|tag=initial|attempt=1|rc=1|issue_count=11|rpo_count=11|stamp=2614978226898 +AUDIT0_RC=1 +AUDIT0_ISSUE_COUNT=11 +AUDIT0_RPO_COUNT=11 +AUDIT0_NON_RPO_COUNT=0 +AUDIT0_RPO_SET_BEGIN +control-bootstrap-v2 +vm111 +vm112 +vm113 +vm130 +vm150 +vm160 +vm170 +vm171 +vm190 +vm9130 +AUDIT0_RPO_SET_END +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +=== ADAPTIVE_LABEL_control-bootstrap-v2 === +QUIET_AUDIT_PASS|tag=pre-control-bootstrap-v2|attempt=1|rc=1|issue_count=11|rpo_count=11|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=control-bootstrap-v2|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":1,"audit_label":"control-bootstrap-v2","verify_target":"control-bootstrap-v2","start_epoch":1790539621} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=197 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=6a9ad8a7e2e40c8c36cade820e9ef71c85e8b9204277681db95017579e7835ec +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=control-bootstrap-v2 +SELFTEST_REPLACEMENT486=PASS +VERIFY_TARGET_RESULT={"generation":"control-bootstrap-v2-pve01-20260916T120057Z","logical_id":"control-bootstrap-v2","payload_bytes":0,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=control-bootstrap-v2 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=control-bootstrap-v2 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=control-bootstrap-v2-pve01-20260916T120057Z +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=0 +VERIFY_RESULT_ALLOCATOR_HEALTHY=None +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":1,"audit_label":"control-bootstrap-v2","verify_target":"control-bootstrap-v2","elapsed_seconds":5,"accepted":true} +TARGET_LOCK_RELEASED|tag=control-bootstrap-v2 +QUIET_AUDIT_PASS|tag=post-control-bootstrap-v2|attempt=1|rc=1|issue_count=10|rpo_count=10|stamp=2614978226898 +TARGET_RPO_CLEARED=control-bootstrap-v2 +=== ADAPTIVE_LABEL_vm111 === +QUIET_AUDIT_PASS|tag=pre-vm111|attempt=1|rc=1|issue_count=10|rpo_count=10|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=ct111|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":2,"audit_label":"vm111","verify_target":"ct111","start_epoch":1790539634} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=472 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=ee74b2f99919efc17e8c41a36e9a41c82a7ecc43fc23da826d36780b37a9670e +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=ct111 +SELFTEST_REPLACEMENT486=PASS +LXC_EXECUTION_DIAG={"executor":"remote-bounded-ssh-cwd-root","node":"pve02","producer_rc":0,"stderr_bytes_observed":994,"stderr_bytes_retained":994,"stderr_stream_sha256":"7d6fe7d03b821031eecbffa0af551ee1caab9a80b983f2ab9f4ca53236b77687","stderr_truncated":false} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct111-20260927T200714Z-2f72200d7742","logical_id":"ct111","payload_bytes":417826152,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=ct111 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=ct111 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-ct111-20260927T200714Z-2f72200d7742 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=417826152 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":2,"audit_label":"vm111","verify_target":"ct111","elapsed_seconds":180,"accepted":true} +TARGET_LOCK_RELEASED|tag=ct111 +QUIET_AUDIT_PASS|tag=post-vm111|attempt=1|rc=1|issue_count=9|rpo_count=9|stamp=2614978226898 +TARGET_RPO_CLEARED=vm111 +=== ADAPTIVE_LABEL_vm112 === +QUIET_AUDIT_PASS|tag=pre-vm112|attempt=1|rc=1|issue_count=9|rpo_count=9|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=ct112|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":3,"audit_label":"vm112","verify_target":"ct112","start_epoch":1790539822} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=561 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=4f15f56422eeecd2d1a8a0c6f960a9e284d2e9ce66d6abce724251961453bccb +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=ct112 +SELFTEST_REPLACEMENT486=PASS +LXC_EXECUTION_DIAG={"archive_bytes":224039854,"exception_class":null,"exception_message_sha256":null,"executor":"local-proven-pve270","helper_rc":0,"node":"pve01","producer_rc":0,"stderr_bytes_observed":1367,"stderr_bytes_retained":1367,"stderr_stream_sha256":"4bb8f468cb6fcffd1af7303399038ec6b5cd9e126b6975ddaaac0b2f4a13f723","stderr_truncated":false} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct112-20260927T201022Z-6cdb5a555315","logical_id":"ct112","payload_bytes":224039854,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=ct112 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=ct112 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-ct112-20260927T201022Z-6cdb5a555315 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=224039854 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":3,"audit_label":"vm112","verify_target":"ct112","elapsed_seconds":85,"accepted":true} +TARGET_LOCK_RELEASED|tag=ct112 +QUIET_AUDIT_PASS|tag=post-vm112|attempt=1|rc=1|issue_count=8|rpo_count=8|stamp=2614978226898 +TARGET_RPO_CLEARED=vm112 +=== ADAPTIVE_LABEL_vm113 === +QUIET_AUDIT_PASS|tag=pre-vm113|attempt=1|rc=1|issue_count=8|rpo_count=8|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=ct113|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":4,"audit_label":"vm113","verify_target":"ct113","start_epoch":1790539915} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=472 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=fa7bc3f5137a8c4d7633f40a686625f2f3b3cf2a8bc59314e388d77e50a3e22a +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=ct113 +SELFTEST_REPLACEMENT486=PASS +LXC_EXECUTION_DIAG={"executor":"remote-bounded-ssh-cwd-root","node":"pve02","producer_rc":0,"stderr_bytes_observed":997,"stderr_bytes_retained":997,"stderr_stream_sha256":"be14e6c3e664cdc3483e88915ce87df5390e191db3cafd82b528b7ebecce4c57","stderr_truncated":false} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct113-20260927T201155Z-87c3bfe1d555","logical_id":"ct113","payload_bytes":221991399,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=ct113 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=ct113 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-ct113-20260927T201155Z-87c3bfe1d555 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=221991399 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":4,"audit_label":"vm113","verify_target":"ct113","elapsed_seconds":92,"accepted":true} +TARGET_LOCK_RELEASED|tag=ct113 +QUIET_AUDIT_PASS|tag=post-vm113|attempt=1|rc=1|issue_count=7|rpo_count=7|stamp=2614978226898 +TARGET_RPO_CLEARED=vm113 +=== ADAPTIVE_LABEL_vm150 === +QUIET_AUDIT_PASS|tag=pre-vm150|attempt=1|rc=1|issue_count=7|rpo_count=7|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=vm150|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":5,"audit_label":"vm150","verify_target":"vm150","start_epoch":1790540014} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=363 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=2b7fdf9f070f942f8fd11708790f350361d94d23b23818fc0811f7742d02a549 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm150 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":912043917,"archive_selected":true,"node":"pve01","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":150} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm150-20260927T201335Z-2ba76459e7c5","logical_id":"vm150","payload_bytes":912043917,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm150 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm150 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm150-20260927T201335Z-2ba76459e7c5 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=912043917 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":5,"audit_label":"vm150","verify_target":"vm150","elapsed_seconds":149,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm150 +QUIET_AUDIT_PASS|tag=post-vm150|attempt=1|rc=1|issue_count=6|rpo_count=6|stamp=2614978226898 +TARGET_RPO_CLEARED=vm150 +=== ADAPTIVE_LABEL_vm190 === +QUIET_AUDIT_PASS|tag=pre-vm190|attempt=1|rc=1|issue_count=6|rpo_count=6|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=vm190|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":6,"audit_label":"vm190","verify_target":"vm190","start_epoch":1790540172} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=365 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=0790485f05e9dc81ab27de10b12fd506674786b8ff7c437dcdde00dbdf3e0640 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm190 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":1645890869,"archive_selected":true,"node":"pve03","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":190} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm190-20260927T201613Z-662906757ca1","logical_id":"vm190","payload_bytes":1645890869,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm190 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm190 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm190-20260927T201613Z-662906757ca1 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=1645890869 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":6,"audit_label":"vm190","verify_target":"vm190","elapsed_seconds":274,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm190 +QUIET_AUDIT_PASS|tag=post-vm190|attempt=1|rc=1|issue_count=5|rpo_count=5|stamp=2614978226898 +TARGET_RPO_CLEARED=vm190 +=== ADAPTIVE_LABEL_vm170 === +QUIET_AUDIT_PASS|tag=pre-vm170|attempt=1|rc=1|issue_count=5|rpo_count=5|stamp=2614978226898 +TARGET_LOCK_ACQUIRED|tag=vm170|attempt=1|stamp=2614978226898 +TARGETED_REFRESH_BEGIN={"iteration":7,"audit_label":"vm170","verify_target":"vm170","start_epoch":1790540454} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=365 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=5a45794eb1a2944f763b9d381c31078b5c30c4b6d7ea97f3bee5068bc03b5ca5 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm170 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":8232066517,"archive_selected":true,"node":"pve01","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":170} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm170-20260927T202054Z-1daa87a3cfbc","logical_id":"vm170","payload_bytes":8232066517,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm170 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm170 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm170-20260927T202054Z-1daa87a3cfbc +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=8232066517 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":7,"audit_label":"vm170","verify_target":"vm170","elapsed_seconds":1099,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm170 +QUIET_AUDIT_PASS|tag=post-vm170|attempt=1|rc=1|issue_count=4|rpo_count=4|stamp=2616807252105 +TARGET_RPO_CLEARED=vm170 +=== ADAPTIVE_LABEL_vm171 === +QUIET_AUDIT_PASS|tag=pre-vm171|attempt=1|rc=1|issue_count=4|rpo_count=4|stamp=2616807252105 +TARGET_LOCK_ACQUIRED|tag=vm171|attempt=1|stamp=2616807252105 +TARGETED_REFRESH_BEGIN={"iteration":8,"audit_label":"vm171","verify_target":"vm171","start_epoch":1790541566} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=365 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=55865e4af68895bd93720362070400a825f2456605a5f64631fdbf3ec97fcd3e +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm171 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":2061491229,"archive_selected":true,"node":"pve01","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":171} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm171-20260927T203926Z-61ca38fe3ac5","logical_id":"vm171","payload_bytes":2061491229,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm171 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm171 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm171-20260927T203926Z-61ca38fe3ac5 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=2061491229 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":8,"audit_label":"vm171","verify_target":"vm171","elapsed_seconds":319,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm171 +QUIET_AUDIT_PASS|tag=post-vm171|attempt=1|rc=1|issue_count=3|rpo_count=3|stamp=2616807252105 +TARGET_RPO_CLEARED=vm171 +=== ADAPTIVE_LABEL_vm9130 === +QUIET_AUDIT_PASS|tag=pre-vm9130|attempt=1|rc=1|issue_count=3|rpo_count=3|stamp=2616807252105 +TARGET_LOCK_ACQUIRED|tag=vm9130|attempt=1|stamp=2616807252105 +TARGETED_REFRESH_BEGIN={"iteration":9,"audit_label":"vm9130","verify_target":"vm9130","start_epoch":1790541892} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=370 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=90516b2d72a05634cbd971a9dc6321061fa0ba3a9ae39be8a5f01cfc36dcf7e2 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm9130 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":10968466823,"archive_selected":true,"node":"pve02","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":9130} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm9130-20260927T204452Z-91cc0cc7ecc5","logical_id":"vm9130","payload_bytes":10968466823,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm9130 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm9130 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm9130-20260927T204452Z-91cc0cc7ecc5 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=10968466823 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":9,"audit_label":"vm9130","verify_target":"vm9130","elapsed_seconds":1603,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm9130 +QUIET_AUDIT_PASS|tag=post-vm9130|attempt=1|rc=1|issue_count=2|rpo_count=2|stamp=2619094297661 +TARGET_RPO_CLEARED=vm9130 +=== ADAPTIVE_LABEL_vm160 === +QUIET_AUDIT_PASS|tag=pre-vm160|attempt=1|rc=1|issue_count=2|rpo_count=2|stamp=2619094297661 +TARGET_LOCK_ACQUIRED|tag=vm160|attempt=1|stamp=2619094297661 +TARGETED_REFRESH_BEGIN={"iteration":10,"audit_label":"vm160","verify_target":"vm160","start_epoch":1790543503} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=365 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=d1a71b612d49c80016a8831df98ae906e50b267b7fdead624c68099e96b43296 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm160 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":3805401657,"archive_selected":true,"node":"pve02","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":160} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm160-20260927T211143Z-0cadc4ac0b12","logical_id":"vm160","payload_bytes":3805401657,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm160 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm160 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm160-20260927T211143Z-0cadc4ac0b12 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=3805401657 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":10,"audit_label":"vm160","verify_target":"vm160","elapsed_seconds":577,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm160 +QUIET_AUDIT_PASS|tag=post-vm160|attempt=1|rc=1|issue_count=1|rpo_count=1|stamp=2619094297661 +TARGET_RPO_CLEARED=vm160 +=== ADAPTIVE_LABEL_vm130 === +QUIET_AUDIT_PASS|tag=pre-vm130|attempt=1|rc=1|issue_count=1|rpo_count=1|stamp=2619094297661 +TARGET_LOCK_ACQUIRED|tag=vm130|attempt=1|stamp=2619094297661 +TARGETED_REFRESH_BEGIN={"iteration":11,"audit_label":"vm130","verify_target":"vm130","start_epoch":1790544088} +VERIFY_TARGET_RC=0 +VERIFY_STDOUT_BYTES=367 +VERIFY_STDERR_BYTES=0 +VERIFY_STDOUT_SHA256=1cdb7027b911f85d1ee9a35afca3dba5641a6891231cea2afcf01af152522157 +VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VERIFY_OUTPUT_BEGIN=vm130 +SELFTEST_REPLACEMENT486=PASS +QEMU_EXECUTION_DIAG={"archive_bytes":11053999065,"archive_selected":true,"node":"pve03","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":130} +VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm130-20260927T212128Z-0046d7efc414","logical_id":"vm130","payload_bytes":11053999065,"replicas":2,"result":"SUCCESS"} +VERIFY_OUTPUT_END=vm130 +VERIFY_RESULT_LINE_COUNT=1 +VERIFY_RESULT_LOGICAL_ID=vm130 +VERIFY_RESULT_RESULT=SUCCESS +VERIFY_RESULT_GENERATION=v2-vm130-20260927T212128Z-0046d7efc414 +VERIFY_RESULT_REPLICAS=2 +VERIFY_RESULT_PAYLOAD_BYTES=11053999065 +VERIFY_RESULT_ALLOCATOR_HEALTHY=13 +VERIFY_RESULT_ACCEPTED=true +TARGETED_REFRESH_END={"iteration":11,"audit_label":"vm130","verify_target":"vm130","elapsed_seconds":1590,"accepted":true} +TARGET_LOCK_RELEASED|tag=vm130 +QUIET_AUDIT_PASS|tag=post-vm130|attempt=1|rc=0|issue_count=0|rpo_count=0|stamp=2621202326883 +TARGET_RPO_CLEARED=vm130 +=== FINAL_QUIET_AUDIT === +QUIET_AUDIT_PASS|tag=final|attempt=1|rc=0|issue_count=0|rpo_count=0|stamp=2621202326883 +FINAL_AUDIT_RC=0 +FINAL_ISSUE_COUNT=0 +FINAL_RPO_COUNT=0 +FINAL_NON_RPO_COUNT=0 +FINAL_AUDIT_OUTPUT_BEGIN +TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55069|generation=app-filebrowser-pve01-20260927T063003Z +TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=54902|generation=app-hbarhub-pve01-20260927T063154Z +TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55003|generation=app-immich-pve01-20260927T063041Z +TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55050|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z +TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=6055|generation=control-bootstrap-v2-pve01-20260916T120057Z +TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=7378|legacy_age_seconds=2045313|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17 +TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=5867|replica_count=2|state_id=ct111|generation=v2-ct111-20260927T200714Z-2f72200d7742 +TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=5774|legacy_age_seconds=2083808|replica_count=2|state_id=ct112|generation=v2-ct112-20260927T201022Z-6cdb5a555315 +TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=5674|replica_count=2|state_id=ct113|generation=v2-ct113-20260927T201155Z-87c3bfe1d555 +TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=3|replica_count=2|state_id=vm130|generation=v2-vm130-20260927T212128Z-0046d7efc414 +TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=5518|legacy_age_seconds=2045429|replica_count=2|state_id=vm150|generation=v2-vm150-20260927T201335Z-2ba76459e7c5 +TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=1601|replica_count=2|state_id=vm160|generation=v2-vm160-20260927T211143Z-0cadc4ac0b12 +TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=4128|legacy_age_seconds=990050|replica_count=2|state_id=vm170|generation=v2-vm170-20260927T202054Z-1daa87a3cfbc +TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=3796|legacy_age_seconds=1019582|replica_count=2|state_id=vm171|generation=v2-vm171-20260927T203926Z-61ca38fe3ac5 +TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=5235|replica_count=2|state_id=vm190|generation=v2-vm190-20260927T201613Z-662906757ca1 +TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=2186|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260927T204452Z-91cc0cc7ecc5 +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +FINAL_AUDIT_OUTPUT_END +FINAL_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +RECOVERY_END_EPOCH=1790545685 +RECOVERY_ELAPSED_SECONDS=6077 +=== SUCCESS_MAIL_POLICY_WINDOW === +TARGETED_SUCCESS_BACKUP_EVENTS=4 +TARGETED_SUCCESS_MAIL_TO_ROOT_EVENTS=0 +SUCCESS_MAIL_POLICY_ACCEPTED=true +=== CONTROL_PLANE_POST === +WRAPPER_SHA256_POST=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +HELPER_SHA256_POST=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 +VZDUMP_CONF_SHA256_POST=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad +AUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +V2_TIMER_ACTIVE=active +V2_TIMER_ENABLED=enabled +AUDIT_TIMER_ACTIVE=active +AUDIT_TIMER_ENABLED=enabled +CHANGES_MADE=true +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=MUTATED +TARGETED_REFRESH_COUNT=11 +TASK_RESULT=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN +DECISION=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN +NEXT_GATE=FINAL_BACKUP_ACCEPTANCE_CLOSEOUT +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1","status":"OK","changes_made":true,"rollback_started":false,"rollback_restored":null} -Quorum information ------------------- -Date: Mon Sep [PRIVATE_IP] 2026 -Quorum provider: corosync_votequorum -Nodes: 3 -Node ID: [PRIVATE_IP] -Ring ID: 1.13c -Quorate: Yes - -Votequorum information ----------------------- -Expected votes: 3 -Highest expected: 3 -Total votes: 3 -Quorum: 2 -Flags: Quorate - -Membership information ----------------------- - Nodeid Votes Name -[PRIVATE_IP] [PRIVATE_IP].11 (local) -[PRIVATE_IP] [PRIVATE_IP].13 -[PRIVATE_IP] [PRIVATE_IP].12 - -Membership information ----------------------- - Nodeid Votes Name - 1 1 pve01 (local) - 2 1 pve03 - 3 1 pve02 -[{"cpu":[PRIVATE_IP]424604,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/100","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"pvepro-prod","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":677795,"vmid":100},{"cpu":0,"disk":0,"diskread":0,"diskwrite":0,"id":"qemu/101","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":0,"memhost":0,"name":"pvepro-staging-v1","netin":0,"netout":0,"node":"pve01","status":"stopped","template":0,"type":"qemu","uptime":0,"vmid":101},{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/110","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns1","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"lxc","uptime":[PRIVATE_IP],"vmid":110},{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/111","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns2","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"lxc","uptime":[PRIVATE_IP],"vmid":111},{"cpu":[PRIVATE_IP]259212,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/112","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound1","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"lxc","uptime":[PRIVATE_IP],"vmid":112},{"cpu":0,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/113","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound2","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"lxc","uptime":[PRIVATE_IP],"vmid":113},{"cpu":[PRIVATE_IP]84935,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"edge-vm","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":130},{"cpu":[PRIVATE_IP]753181,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/150","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"snikket","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":150},{"cpu":[PRIVATE_IP]756987,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/160","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"forum-prod","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":160},{"cpu":[PRIVATE_IP]4218,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/170","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"core-apps","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":170},{"cpu":[PRIVATE_IP]83494,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/171","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"monitoring","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":171},{"cpu":[PRIVATE_IP]142468,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/190","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"homelab-ops-worker","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":190},{"cpu":[PRIVATE_IP]849209,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/210","maxcpu":2,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"newfi-prod","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":210},{"cpu":[PRIVATE_IP]741922,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/211","maxcpu":2,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"newfi-staging","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":211},{"cpu":0,"disk":0,"diskread":0,"diskwrite":0,"id":"qemu/9130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":0,"memhost":0,"name":"edge-cold-standby","netin":0,"netout":0,"node":"pve02","status":"stopped","template":0,"type":"qemu","uptime":0,"vmid":9130},{"cgroup-mode":2,"cpu":[PRIVATE_IP]92543,"disk":[PRIVATE_IP],"id":"node/pve03","level":"","maxcpu":8,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve03","status":"online","type":"node","uptime":[PRIVATE_IP]},{"cgroup-mode":2,"cpu":[PRIVATE_IP]60494,"disk":[PRIVATE_IP],"id":"node/pve02","level":"","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve02","status":"online","type":"node","uptime":[PRIVATE_IP]},{"cgroup-mode":2,"cpu":[PRIVATE_IP]445,"disk":[PRIVATE_IP],"id":"node/pve01","level":"","maxcpu":24,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve01","status":"online","type":"node","uptime":[PRIVATE_IP]},{"content":"iso,backup,import,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve03/local","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"iso,backup,import,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve02/local","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"iso,backup,import,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve01/local","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"images,rootdir","disk":[PRIVATE_IP],"id":"storage/pve03/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"images,rootdir","disk":[PRIVATE_IP],"id":"storage/pve02/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"images,rootdir","disk":[PRIVATE_IP],"id":"storage/pve01/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"id":"network/pve03/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve03","status":"ok","type":"network"},{"id":"network/pve02/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve02","status":"ok","type":"network"},{"id":"network/pve01/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve01","status":"ok","type":"network"}] -Name Type Status Total (KiB) Used (KiB) Available (KiB) % -local dir active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 38.97% -local-lvm lvmthin active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 18.33% - UNIT LOAD ACTIVE SUB DESCRIPTION -● homelab-backup-audit.service loaded failed failed Homelab unified backup and DR audit - -Legend: LOAD → Reflects whether the unit definition was properly loaded. - ACTIVE → The high-level unit activation state, i.e. generalization of SUB. - SUB → The low-level unit activation state, values depend on unit type. - -1 loaded units listed. -NEXT LEFT LAST PASSED UNIT ACTIVATES -Mon [PRIVATE_IP] 00:17:52 MSK 49s Mon [PRIVATE_IP] 00:16:49 MSK 13s ago homelab-router-watchdog.timer homelab-router-watchdog.service -Mon [PRIVATE_IP] 00:18:16 MSK 1min 13s Mon [PRIVATE_IP] 00:13:16 MSK 3min 46s ago netbird-peers-health.timer netbird-peers-health.service -Mon [PRIVATE_IP] 00:19:16 MSK 2min 13s Mon [PRIVATE_IP] 00:09:16 MSK 7min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service -Mon [PRIVATE_IP] 00:19:59 MSK 2min 56s Mon [PRIVATE_IP] 00:14:32 MSK 2min 30s ago homelab-health-metrics.timer homelab-health-metrics.service -Mon [PRIVATE_IP] 00:22:01 MSK 4min 58s Mon [PRIVATE_IP] 00:17:01 MSK 1s ago homelab-smartctl-textfile.timer homelab-smartctl-textfile.service -Mon [PRIVATE_IP] 00:23:01 MSK 5min Mon [PRIVATE_IP] 00:08:01 MSK 9min ago prometheus-node-exporter-apt.timer prometheus-node-exporter-apt.service -Mon [PRIVATE_IP] 00:23:16 MSK 6min Mon [PRIVATE_IP] 00:08:01 MSK 9min ago homelab-alertmanager-backup-health.timer homelab-alertmanager-backup-health.service -Mon [PRIVATE_IP] 00:27:01 MSK 9min Mon [PRIVATE_IP] 00:12:01 MSK 5min ago homelab-disk-space-health.timer homelab-disk-space-health.service -Mon [PRIVATE_IP] 00:27:02 MSK 9min Mon [PRIVATE_IP] 00:09:14 MSK 1 week 0 days ago fstrim.timer fstrim.service -Mon [PRIVATE_IP] 00:30:00 MSK 12min Mon [PRIVATE_IP] 00:15:01 MSK 2min 1s ago homelab-incident-journal.timer homelab-incident-journal.service -Mon [PRIVATE_IP] 00:30:01 MSK 12min Mon [PRIVATE_IP] 00:15:01 MSK 2min 1s ago homelab-pve03-staging-capacity-health.timer homelab-pve03-staging-capacity-health.service -Mon [PRIVATE_IP] 00:30:01 MSK 12min Mon [PRIVATE_IP] 00:15:01 MSK 2min 1s ago prometheus-node-exporter-nvme.timer prometheus-node-exporter-nvme.service -Mon [PRIVATE_IP] 00:30:41 MSK 13min Mon [PRIVATE_IP] 00:16:16 MSK 46s ago homelab-forum-snuffleupagus-health.timer homelab-forum-snuffleupagus-health.service -Mon [PRIVATE_IP] 00:30:48 MSK 13min Mon [PRIVATE_IP] 00:15:42 MSK 1min 20s ago homelab-external-probe-vps-health.timer homelab-external-probe-vps-health.service -Mon [PRIVATE_IP] 00:31:13 MSK 14min Mon [PRIVATE_IP] 00:15:42 MSK 1min 20s ago homelab-duty-admin-v2.timer homelab-duty-admin-v2.service -Mon [PRIVATE_IP] 00:33:27 MSK 16min Mon [PRIVATE_IP] 00:02:58 MSK 14min ago homelab-mkdocs-auto-refresh.timer homelab-mkdocs-auto-refresh.service -Mon [PRIVATE_IP] 00:35:58 MSK 18min Mon [PRIVATE_IP] 00:01:23 MSK 15min ago homelab-reference-refresh.timer homelab-reference-refresh.service -Mon [PRIVATE_IP] 00:36:06 MSK 19min Mon [PRIVATE_IP] 00:06:16 MSK 10min ago homelab-backup-audit.timer homelab-backup-audit.service -Mon [PRIVATE_IP] 00:38:07 MSK 21min Mon [PRIVATE_IP] 00:03:08 MSK 13min ago homelab-backup-v2.timer homelab-backup-v2.service -Mon [PRIVATE_IP] 01:30:00 MSK 1h 12min Sun [PRIVATE_IP] 23:30:01 MSK 47min ago homelab-app-cloud-quorum-queue.timer homelab-app-cloud-quorum-queue.service -Mon [PRIVATE_IP] 01:48:00 MSK 1h 30min Sun [PRIVATE_IP] 19:39:16 MSK 4h 37min ago homelab-mailru-trash-gc.timer homelab-mailru-trash-gc.service -Mon [PRIVATE_IP] 02:13:56 MSK 1h 56min Sun [PRIVATE_IP] 20:09:20 MSK 4h 7min ago homelab-evidence-root.timer homelab-evidence-root.service -Mon [PRIVATE_IP] 02:27:45 MSK 2h 10min Sun [PRIVATE_IP] 02:36:58 MSK 21h ago pve-daily-update.timer pve-daily-update.service -Mon [PRIVATE_IP] 03:26:09 MSK 3h 9min Sun [PRIVATE_IP] 03:26:52 MSK 20h ago homelab-backup-retention-gc.timer homelab-backup-retention-gc.service -Mon [PRIVATE_IP] 04:36:01 MSK 4h 18min Sun [PRIVATE_IP] 22:36:01 MSK 1h 41min ago homelab-vps-identity-audit.timer homelab-vps-identity-audit.service -Mon [PRIVATE_IP] 04:49:28 MSK 4h 32min Sun [PRIVATE_IP] 07:57:58 MSK 16h ago man-db.timer man-db.service -Mon [PRIVATE_IP] 06:10:06 MSK 5h 53min Sun [PRIVATE_IP] 06:25:48 MSK 17h ago apt-daily-upgrade.timer apt-daily-upgrade.service -Mon [PRIVATE_IP] 07:15:02 MSK 6h Sun [PRIVATE_IP] 07:12:16 MSK 17h ago homelab-drift-check.timer homelab-drift-check.service -Mon [PRIVATE_IP] 07:21:38 MSK 7h Sun [PRIVATE_IP] 07:32:58 MSK 16h ago homelab-service-registry-check.timer homelab-service-registry-check.service -Mon [PRIVATE_IP] 07:46:24 MSK 7h Sun [PRIVATE_IP] 07:53:32 MSK 16h ago homelab-alerting-health.timer homelab-alerting-health.service -Mon [PRIVATE_IP] 07:53:09 MSK 7h Sun [PRIVATE_IP] 07:58:49 MSK 16h ago homelab-desired-state-sync.timer homelab-desired-state-sync.service -Mon [PRIVATE_IP] 07:58:30 MSK 7h Sun [PRIVATE_IP] 07:44:58 MSK 16h ago homelab-runbook-generate.timer homelab-runbook-generate.service -Mon [PRIVATE_IP] 07:58:55 MSK 7h Sun [PRIVATE_IP] 07:52:02 MSK 16h ago homelab-dependency-map-check.timer homelab-dependency-map-check.service -Mon [PRIVATE_IP] 08:03:25 MSK 7h Sun [PRIVATE_IP] 08:08:16 MSK 16h ago homelab-overall-health.timer homelab-overall-health.service -Mon [PRIVATE_IP] 08:07:40 MSK 7h Sun [PRIVATE_IP] 08:13:22 MSK 16h ago homelab-kuma-monitor-policy.timer homelab-kuma-monitor-policy.service -Mon [PRIVATE_IP] 08:11:32 MSK 7h Sun [PRIVATE_IP] 08:11:58 MSK 16h ago homelab-duty-admin-report.timer homelab-duty-admin-report.service -Mon [PRIVATE_IP] 08:18:28 MSK 8h Sun [PRIVATE_IP] 08:26:58 MSK 15h ago homelab-final-readiness-gate.timer homelab-final-readiness-gate.service -Mon [PRIVATE_IP] 08:33:21 MSK 8h Sun [PRIVATE_IP] 08:31:58 MSK 15h ago homelab-capacity-risk.timer homelab-capacity-risk.service -Mon [PRIVATE_IP] 08:45:07 MSK 8h Sun [PRIVATE_IP] 08:46:49 MSK 15h ago homelab-golden-state-index.timer homelab-golden-state-index.service -Mon [PRIVATE_IP] 08:46:25 MSK 8h Sun [PRIVATE_IP] 08:40:58 MSK 15h ago homelab-secret-exposure-guard.timer homelab-secret-exposure-guard.service -Mon [PRIVATE_IP] 08:50:04 MSK 8h Sun [PRIVATE_IP] 08:40:18 MSK 15h ago homelab-cluster-passport.timer homelab-cluster-passport.service -Mon [PRIVATE_IP] 10:15:00 MSK 9h Sun [PRIVATE_IP] 10:15:01 MSK 14h ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service -Mon [PRIVATE_IP] 11:36:53 MSK 11h Sun [PRIVATE_IP] 19:04:32 MSK 5h 12min ago apt-daily.timer apt-daily.service -Mon [PRIVATE_IP] 16:48:01 MSK 16h Sun [PRIVATE_IP] 16:48:01 MSK 7h ago systemd-tmpfiles-clean.timer systemd-tmpfiles-clean.service -Tue [PRIVATE_IP] 00:00:00 MSK 23h Mon [PRIVATE_IP] 00:00:01 MSK 17min ago dpkg-db-backup.timer dpkg-db-backup.service -Tue [PRIVATE_IP] 00:10:26 MSK 23h Mon [PRIVATE_IP] 00:01:19 MSK 15min ago homelab-evidence-catalog.timer homelab-evidence-catalog.service -Tue [PRIVATE_IP] 00:11:11 MSK 23h Mon [PRIVATE_IP] 00:00:22 MSK 16min ago homelab-quality-gate.timer homelab-quality-gate.service -Tue [PRIVATE_IP] 00:12:33 MSK 23h Mon [PRIVATE_IP] 00:11:16 MSK 5min ago homelab-storage-capacity.timer homelab-storage-capacity.service -Tue [PRIVATE_IP] 00:19:44 MSK 24h Mon [PRIVATE_IP] 00:15:46 MSK 1min 16s ago homelab-docker-health.timer homelab-docker-health.service -Tue [PRIVATE_IP] 00:49:06 MSK 24h Mon [PRIVATE_IP] 00:04:49 MSK 12min ago logrotate.timer logrotate.service -Sun [PRIVATE_IP] 03:10:13 MSK 6 days Sun [PRIVATE_IP] 03:10:58 MSK 21h ago xfs_scrub_all.timer xfs_scrub_all.service -Sun [PRIVATE_IP] 03:10:23 MSK 6 days Sun [PRIVATE_IP] 03:10:42 MSK 21h ago e2scrub_all.timer e2scrub_all.service -Sun [PRIVATE_IP] 07:36:01 MSK 6 days Sun [PRIVATE_IP] 06:42:01 MSK 17h ago homelab-rotating-boot-drill.timer homelab-rotating-boot-drill.service -Mon [PRIVATE_IP] 00:14:21 MSK 6 days Mon [PRIVATE_IP] 00:01:49 MSK 15min ago homelab-secret-sanity.timer homelab-secret-sanity.service -- - Mon [PRIVATE_IP] 00:17:01 MSK 1s ago homelab-private-vpn-hosts-health.timer homelab-private-vpn-hosts-health.service -- - - - prometheus-node-exporter-ipmitool-sensor.timer prometheus-node-exporter-ipmitool-sensor.service -- - - - prometheus-node-exporter-mellanox-hca-temp.timer prometheus-node-exporter-mellanox-hca-temp.service -- - - - prometheus-node-exporter-smartmon.timer prometheus-node-exporter-smartmon.service - -58 timers listed. -RUNTIME_MANIFEST_ID=df[PRIVATE_IP]ac9b5abd[PRIVATE_IP]b[PRIVATE_IP]d89496fb2bde[PRIVATE_IP] -EXPERIMENT_ID=5462ee[PRIVATE_IP]ebe[PRIVATE_IP]feb2ad[PRIVATE_IP]1ad87bbac0b8108 -COLLECTION_START_AT=[PRIVATE_IP]T[PRIVATE_IP]:00 -FIRST_COMPLETE_D1_CLOSE=[PRIVATE_IP]T[PRIVATE_IP]:00 -PRODUCTION_LAUNCH_ALLOWED=False -EXECUTION_AUTHORITY=False -RISK_AUTHORITY=False -STAGE_A_HEALTH=AWAITING_PRIMARY_FINALITY -FORENSIC_PREVIOUS_EPOCH_PRESERVED=YES - -## NEWFI_TASK6_AUTH_CHECKPOINT_V[PRIVATE_IP] -- Scope: historical accepted AUTH test, not a current SMTP connection test. -- Newfi approved SMTP account/sender: aleisaev@yandex.ru. -- Endpoint used by accepted test: smtp.yandex.ru:465 with certificate verification; single PLAIN initial response accepted with code 235. -- Evidence: incident ledger ### NEWFI_TASK6_YANDEX_AUTH_V2_OBSERVATION and ### NEWFI_TASK6_AUTH_PROOF_CLOSE_V1. -- No password file written, no application configuration changed, no mail sent by the accepted AUTH test. -- Persistent transport installation and application delivery are not proven by this checkpoint. Task6 is not complete. -- AUTH challenge and missing-file incidents closed; launch/session incident remains OPEN with original cause unproven. -- This update does not change VM211, VM160, routing, board state or Git. -- Other incidents and infrastructure health were not assessed. - - -## NEWFI_PRIVATE_STRUCTURE92_WRONG_MARKER_SCOPE_[PRIVATE_IP] CLOSURE -- Status: CLOSED -- RCA: NEWFI92 inspected the pve01 NEWFI90 attempt marker, which had not been finalized because the outer shell exited immediately after the guest returned RC3 following its successful rollback. The guest marker contained the true final rollback state. -- Correction: NEWFI93 required both the guest FINISHED/ROLLED_BACK record and a fresh exact demo-state readback before reconciling the outer NEWFI90 marker. NEWFI92 itself never reserved an attempt or mutated state. - - -## NEWFI_PRIVATE_NEWFI95_NESTED_TRIPLE_QUOTE_PARSE_[PRIVATE_IP] -- Status: OPEN -- Command: NEWFI95 -- Symptom: after the content-structure precheck passed, the pve03 Python wrapper failed to parse at the PHP line beginning with $root. No Portal discovery or application mutation was reached. -- RCA: an outer raw triple-single-quoted guest Python payload contained an inner raw triple-single-quoted PHP payload. The inner delimiter terminated the outer Python string, causing PHP source to be parsed as Python. -- Safety: NEWFI95 was read-only, created no attempt marker, and did not reach the intended nested guest execution. Do not replay NEWFI95. - - -## NEWFI_PRIVATE_NEWFI95_NESTED_TRIPLE_QUOTE_PARSE_[PRIVATE_IP] CLOSURE -- Status: CLOSED -- RCA: an inner triple-single-quoted PHP literal terminated the outer triple-single-quoted Python guest payload in NEWFI95. The wrapper therefore failed parsing before the intended Portal discovery executed. -- Correction: NEWFI96 removed the nested Python payload layer entirely and passed one Python program directly to VM211 through qm guest exec stdin. The read-only Portal contract completed with QGA exit 0 and the implementation worktree remained unchanged. OUTPUT_END CHAT_OUTPUT_END