homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-09-27 21:48:06 +00:00
parent e59b6b9c90
commit 4060389738
4 changed files with 896 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.json
+38
View File
File diff suppressed because one or more lines are too long
runtime/history/SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1.txt
+414
View File
@@ -0,0 +1,414 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1
STATUS=OK
RC=0
HOST=pve01
MODE=change
COMPONENT=homelab-backup-lock-aware-rpo-recovery
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=78e9898eb6e2b60241cabb6cd7725b09ff0770faa1ebc48e7e01882c30f03029
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=true
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=OK
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=true
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=MUTATED
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=ec33d4b1814e274b3b42698808583b2ff788fc28ebda27f071d75672b16ab009
SANITIZED_OUTPUT_SHA256=ec33d4b1814e274b3b42698808583b2ff788fc28ebda27f071d75672b16ab009
OUTPUT_BEGIN
BACKUP_LOCK_AWARE_RPO_RECOVERY_BEGIN=true
COMMAND_ID=SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1
MODE=change
COMPONENT=homelab-backup-lock-aware-rpo-recovery
MUTATION_BOUNDARY=ONLY_CURRENT_V2_VERIFY_TARGET_FOR_FRESH_AUDIT_RPO_BREACHES;AUDIT_ONLY_WHILE_HEAVY_LOCK_FREE;HEAVY_LOCK_ONLY_DURING_SINGLE_VERIFY_TARGET;EXCLUDES_CT110_ALREADY_FRESH;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_TIMER_CHANGE;NO_RETENTION;NO_DELETE_EXISTING_ARCHIVES;NO_RESTORE;NO_MANUAL_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE
=== AUTHORITY ===
HOST=pve01
CLUSTER_QUORATE=Yes
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
SOURCE_1234_STATUS=OK
SOURCE_1234_RC=0
SOURCE_1234_DECISION=PASS_NONRPO_LOCK_RCA_COLLECTED
SOURCE_1234_NEXT_GATE=ADAPTIVE_RPO_RECOVERY_WITH_LOCK_AWARE_AUDIT
SOURCE_1234_LOCK_SELF_INDUCED_PROVISIONAL=true
SOURCE_1234_AUDIT_NON_RPO_COUNT=0
SOURCE_1234_AUDIT_RPO_COUNT=11
=== LIVE_IDENTITY ===
WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
SOURCE_WORKTREE_CLEAN=true
RECOVERY_START_EPOCH=1790539608
RECOVERY_START_UTC=2026-09-27T20:06:48Z
=== INITIAL_QUIET_AUDIT ===
QUIET_AUDIT_PASS|tag=initial|attempt=1|rc=1|issue_count=11|rpo_count=11|stamp=2614978226898
AUDIT0_RC=1
AUDIT0_ISSUE_COUNT=11
AUDIT0_RPO_COUNT=11
AUDIT0_NON_RPO_COUNT=0
AUDIT0_RPO_SET_BEGIN
control-bootstrap-v2
vm111
vm112
vm113
vm130
vm150
vm160
vm170
vm171
vm190
vm9130
AUDIT0_RPO_SET_END
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
=== ADAPTIVE_LABEL_control-bootstrap-v2 ===
QUIET_AUDIT_PASS|tag=pre-control-bootstrap-v2|attempt=1|rc=1|issue_count=11|rpo_count=11|stamp=2614978226898
TARGET_LOCK_ACQUIRED|tag=control-bootstrap-v2|attempt=1|stamp=2614978226898
TARGETED_REFRESH_BEGIN={"iteration":1,"audit_label":"control-bootstrap-v2","verify_target":"control-bootstrap-v2","start_epoch":1790539621}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=197
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=6a9ad8a7e2e40c8c36cade820e9ef71c85e8b9204277681db95017579e7835ec
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=control-bootstrap-v2
SELFTEST_REPLACEMENT486=PASS
VERIFY_TARGET_RESULT={"generation":"control-bootstrap-v2-pve01-20260916T120057Z","logical_id":"control-bootstrap-v2","payload_bytes":0,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=control-bootstrap-v2
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=control-bootstrap-v2
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=control-bootstrap-v2-pve01-20260916T120057Z
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=0
VERIFY_RESULT_ALLOCATOR_HEALTHY=None
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":1,"audit_label":"control-bootstrap-v2","verify_target":"control-bootstrap-v2","elapsed_seconds":5,"accepted":true}
TARGET_LOCK_RELEASED|tag=control-bootstrap-v2
QUIET_AUDIT_PASS|tag=post-control-bootstrap-v2|attempt=1|rc=1|issue_count=10|rpo_count=10|stamp=2614978226898
TARGET_RPO_CLEARED=control-bootstrap-v2
=== ADAPTIVE_LABEL_vm111 ===
QUIET_AUDIT_PASS|tag=pre-vm111|attempt=1|rc=1|issue_count=10|rpo_count=10|stamp=2614978226898
TARGET_LOCK_ACQUIRED|tag=ct111|attempt=1|stamp=2614978226898
TARGETED_REFRESH_BEGIN={"iteration":2,"audit_label":"vm111","verify_target":"ct111","start_epoch":1790539634}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=472
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=ee74b2f99919efc17e8c41a36e9a41c82a7ecc43fc23da826d36780b37a9670e
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=ct111
SELFTEST_REPLACEMENT486=PASS
LXC_EXECUTION_DIAG={"executor":"remote-bounded-ssh-cwd-root","node":"pve02","producer_rc":0,"stderr_bytes_observed":994,"stderr_bytes_retained":994,"stderr_stream_sha256":"7d6fe7d03b821031eecbffa0af551ee1caab9a80b983f2ab9f4ca53236b77687","stderr_truncated":false}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct111-20260927T200714Z-2f72200d7742","logical_id":"ct111","payload_bytes":417826152,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=ct111
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=ct111
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-ct111-20260927T200714Z-2f72200d7742
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=417826152
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":2,"audit_label":"vm111","verify_target":"ct111","elapsed_seconds":180,"accepted":true}
TARGET_LOCK_RELEASED|tag=ct111
QUIET_AUDIT_PASS|tag=post-vm111|attempt=1|rc=1|issue_count=9|rpo_count=9|stamp=2614978226898
TARGET_RPO_CLEARED=vm111
=== ADAPTIVE_LABEL_vm112 ===
QUIET_AUDIT_PASS|tag=pre-vm112|attempt=1|rc=1|issue_count=9|rpo_count=9|stamp=2614978226898
TARGET_LOCK_ACQUIRED|tag=ct112|attempt=1|stamp=2614978226898
TARGETED_REFRESH_BEGIN={"iteration":3,"audit_label":"vm112","verify_target":"ct112","start_epoch":1790539822}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=561
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=4f15f56422eeecd2d1a8a0c6f960a9e284d2e9ce66d6abce724251961453bccb
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=ct112
SELFTEST_REPLACEMENT486=PASS
LXC_EXECUTION_DIAG={"archive_bytes":224039854,"exception_class":null,"exception_message_sha256":null,"executor":"local-proven-pve270","helper_rc":0,"node":"pve01","producer_rc":0,"stderr_bytes_observed":1367,"stderr_bytes_retained":1367,"stderr_stream_sha256":"4bb8f468cb6fcffd1af7303399038ec6b5cd9e126b6975ddaaac0b2f4a13f723","stderr_truncated":false}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct112-20260927T201022Z-6cdb5a555315","logical_id":"ct112","payload_bytes":224039854,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=ct112
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=ct112
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-ct112-20260927T201022Z-6cdb5a555315
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=224039854
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":3,"audit_label":"vm112","verify_target":"ct112","elapsed_seconds":85,"accepted":true}
TARGET_LOCK_RELEASED|tag=ct112
QUIET_AUDIT_PASS|tag=post-vm112|attempt=1|rc=1|issue_count=8|rpo_count=8|stamp=2614978226898
TARGET_RPO_CLEARED=vm112
=== ADAPTIVE_LABEL_vm113 ===
QUIET_AUDIT_PASS|tag=pre-vm113|attempt=1|rc=1|issue_count=8|rpo_count=8|stamp=2614978226898
TARGET_LOCK_ACQUIRED|tag=ct113|attempt=1|stamp=2614978226898
TARGETED_REFRESH_BEGIN={"iteration":4,"audit_label":"vm113","verify_target":"ct113","start_epoch":1790539915}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=472
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=fa7bc3f5137a8c4d7633f40a686625f2f3b3cf2a8bc59314e388d77e50a3e22a
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=ct113
SELFTEST_REPLACEMENT486=PASS
LXC_EXECUTION_DIAG={"executor":"remote-bounded-ssh-cwd-root","node":"pve02","producer_rc":0,"stderr_bytes_observed":997,"stderr_bytes_retained":997,"stderr_stream_sha256":"be14e6c3e664cdc3483e88915ce87df5390e191db3cafd82b528b7ebecce4c57","stderr_truncated":false}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-ct113-20260927T201155Z-87c3bfe1d555","logical_id":"ct113","payload_bytes":221991399,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=ct113
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=ct113
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-ct113-20260927T201155Z-87c3bfe1d555
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=221991399
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":4,"audit_label":"vm113","verify_target":"ct113","elapsed_seconds":92,"accepted":true}
TARGET_LOCK_RELEASED|tag=ct113
QUIET_AUDIT_PASS|tag=post-vm113|attempt=1|rc=1|issue_count=7|rpo_count=7|stamp=2614978226898
TARGET_RPO_CLEARED=vm113
=== ADAPTIVE_LABEL_vm150 ===
QUIET_AUDIT_PASS|tag=pre-vm150|attempt=1|rc=1|issue_count=7|rpo_count=7|stamp=2614978226898
TARGET_LOCK_ACQUIRED|tag=vm150|attempt=1|stamp=2614978226898
TARGETED_REFRESH_BEGIN={"iteration":5,"audit_label":"vm150","verify_target":"vm150","start_epoch":1790540014}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=363
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=2b7fdf9f070f942f8fd11708790f350361d94d23b23818fc0811f7742d02a549
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=vm150
SELFTEST_REPLACEMENT486=PASS
QEMU_EXECUTION_DIAG={"archive_bytes":912043917,"archive_selected":true,"node":"pve01","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":150}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm150-20260927T201335Z-2ba76459e7c5","logical_id":"vm150","payload_bytes":912043917,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=vm150
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=vm150
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-vm150-20260927T201335Z-2ba76459e7c5
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=912043917
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":5,"audit_label":"vm150","verify_target":"vm150","elapsed_seconds":149,"accepted":true}
TARGET_LOCK_RELEASED|tag=vm150
QUIET_AUDIT_PASS|tag=post-vm150|attempt=1|rc=1|issue_count=6|rpo_count=6|stamp=2614978226898
TARGET_RPO_CLEARED=vm150
=== ADAPTIVE_LABEL_vm190 ===
QUIET_AUDIT_PASS|tag=pre-vm190|attempt=1|rc=1|issue_count=6|rpo_count=6|stamp=2614978226898
TARGET_LOCK_ACQUIRED|tag=vm190|attempt=1|stamp=2614978226898
TARGETED_REFRESH_BEGIN={"iteration":6,"audit_label":"vm190","verify_target":"vm190","start_epoch":1790540172}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=365
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=0790485f05e9dc81ab27de10b12fd506674786b8ff7c437dcdde00dbdf3e0640
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=vm190
SELFTEST_REPLACEMENT486=PASS
QEMU_EXECUTION_DIAG={"archive_bytes":1645890869,"archive_selected":true,"node":"pve03","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":190}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm190-20260927T201613Z-662906757ca1","logical_id":"vm190","payload_bytes":1645890869,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=vm190
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=vm190
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-vm190-20260927T201613Z-662906757ca1
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=1645890869
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":6,"audit_label":"vm190","verify_target":"vm190","elapsed_seconds":274,"accepted":true}
TARGET_LOCK_RELEASED|tag=vm190
QUIET_AUDIT_PASS|tag=post-vm190|attempt=1|rc=1|issue_count=5|rpo_count=5|stamp=2614978226898
TARGET_RPO_CLEARED=vm190
=== ADAPTIVE_LABEL_vm170 ===
QUIET_AUDIT_PASS|tag=pre-vm170|attempt=1|rc=1|issue_count=5|rpo_count=5|stamp=2614978226898
TARGET_LOCK_ACQUIRED|tag=vm170|attempt=1|stamp=2614978226898
TARGETED_REFRESH_BEGIN={"iteration":7,"audit_label":"vm170","verify_target":"vm170","start_epoch":1790540454}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=365
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=5a45794eb1a2944f763b9d381c31078b5c30c4b6d7ea97f3bee5068bc03b5ca5
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=vm170
SELFTEST_REPLACEMENT486=PASS
QEMU_EXECUTION_DIAG={"archive_bytes":8232066517,"archive_selected":true,"node":"pve01","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":170}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm170-20260927T202054Z-1daa87a3cfbc","logical_id":"vm170","payload_bytes":8232066517,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=vm170
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=vm170
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-vm170-20260927T202054Z-1daa87a3cfbc
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=8232066517
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":7,"audit_label":"vm170","verify_target":"vm170","elapsed_seconds":1099,"accepted":true}
TARGET_LOCK_RELEASED|tag=vm170
QUIET_AUDIT_PASS|tag=post-vm170|attempt=1|rc=1|issue_count=4|rpo_count=4|stamp=2616807252105
TARGET_RPO_CLEARED=vm170
=== ADAPTIVE_LABEL_vm171 ===
QUIET_AUDIT_PASS|tag=pre-vm171|attempt=1|rc=1|issue_count=4|rpo_count=4|stamp=2616807252105
TARGET_LOCK_ACQUIRED|tag=vm171|attempt=1|stamp=2616807252105
TARGETED_REFRESH_BEGIN={"iteration":8,"audit_label":"vm171","verify_target":"vm171","start_epoch":1790541566}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=365
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=55865e4af68895bd93720362070400a825f2456605a5f64631fdbf3ec97fcd3e
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=vm171
SELFTEST_REPLACEMENT486=PASS
QEMU_EXECUTION_DIAG={"archive_bytes":2061491229,"archive_selected":true,"node":"pve01","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":171}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm171-20260927T203926Z-61ca38fe3ac5","logical_id":"vm171","payload_bytes":2061491229,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=vm171
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=vm171
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-vm171-20260927T203926Z-61ca38fe3ac5
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=2061491229
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":8,"audit_label":"vm171","verify_target":"vm171","elapsed_seconds":319,"accepted":true}
TARGET_LOCK_RELEASED|tag=vm171
QUIET_AUDIT_PASS|tag=post-vm171|attempt=1|rc=1|issue_count=3|rpo_count=3|stamp=2616807252105
TARGET_RPO_CLEARED=vm171
=== ADAPTIVE_LABEL_vm9130 ===
QUIET_AUDIT_PASS|tag=pre-vm9130|attempt=1|rc=1|issue_count=3|rpo_count=3|stamp=2616807252105
TARGET_LOCK_ACQUIRED|tag=vm9130|attempt=1|stamp=2616807252105
TARGETED_REFRESH_BEGIN={"iteration":9,"audit_label":"vm9130","verify_target":"vm9130","start_epoch":1790541892}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=370
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=90516b2d72a05634cbd971a9dc6321061fa0ba3a9ae39be8a5f01cfc36dcf7e2
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=vm9130
SELFTEST_REPLACEMENT486=PASS
QEMU_EXECUTION_DIAG={"archive_bytes":10968466823,"archive_selected":true,"node":"pve02","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":9130}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm9130-20260927T204452Z-91cc0cc7ecc5","logical_id":"vm9130","payload_bytes":10968466823,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=vm9130
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=vm9130
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-vm9130-20260927T204452Z-91cc0cc7ecc5
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=10968466823
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":9,"audit_label":"vm9130","verify_target":"vm9130","elapsed_seconds":1603,"accepted":true}
TARGET_LOCK_RELEASED|tag=vm9130
QUIET_AUDIT_PASS|tag=post-vm9130|attempt=1|rc=1|issue_count=2|rpo_count=2|stamp=2619094297661
TARGET_RPO_CLEARED=vm9130
=== ADAPTIVE_LABEL_vm160 ===
QUIET_AUDIT_PASS|tag=pre-vm160|attempt=1|rc=1|issue_count=2|rpo_count=2|stamp=2619094297661
TARGET_LOCK_ACQUIRED|tag=vm160|attempt=1|stamp=2619094297661
TARGETED_REFRESH_BEGIN={"iteration":10,"audit_label":"vm160","verify_target":"vm160","start_epoch":1790543503}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=365
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=d1a71b612d49c80016a8831df98ae906e50b267b7fdead624c68099e96b43296
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=vm160
SELFTEST_REPLACEMENT486=PASS
QEMU_EXECUTION_DIAG={"archive_bytes":3805401657,"archive_selected":true,"node":"pve02","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":160}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm160-20260927T211143Z-0cadc4ac0b12","logical_id":"vm160","payload_bytes":3805401657,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=vm160
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=vm160
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-vm160-20260927T211143Z-0cadc4ac0b12
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=3805401657
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":10,"audit_label":"vm160","verify_target":"vm160","elapsed_seconds":577,"accepted":true}
TARGET_LOCK_RELEASED|tag=vm160
QUIET_AUDIT_PASS|tag=post-vm160|attempt=1|rc=1|issue_count=1|rpo_count=1|stamp=2619094297661
TARGET_RPO_CLEARED=vm160
=== ADAPTIVE_LABEL_vm130 ===
QUIET_AUDIT_PASS|tag=pre-vm130|attempt=1|rc=1|issue_count=1|rpo_count=1|stamp=2619094297661
TARGET_LOCK_ACQUIRED|tag=vm130|attempt=1|stamp=2619094297661
TARGETED_REFRESH_BEGIN={"iteration":11,"audit_label":"vm130","verify_target":"vm130","start_epoch":1790544088}
VERIFY_TARGET_RC=0
VERIFY_STDOUT_BYTES=367
VERIFY_STDERR_BYTES=0
VERIFY_STDOUT_SHA256=1cdb7027b911f85d1ee9a35afca3dba5641a6891231cea2afcf01af152522157
VERIFY_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VERIFY_OUTPUT_BEGIN=vm130
SELFTEST_REPLACEMENT486=PASS
QEMU_EXECUTION_DIAG={"archive_bytes":11053999065,"archive_selected":true,"node":"pve03","producer_rc":0,"regular_file_count":2,"sidecar_count":1,"vmid":130}
VERIFY_TARGET_RESULT={"allocator_healthy":13,"generation":"v2-vm130-20260927T212128Z-0046d7efc414","logical_id":"vm130","payload_bytes":11053999065,"replicas":2,"result":"SUCCESS"}
VERIFY_OUTPUT_END=vm130
VERIFY_RESULT_LINE_COUNT=1
VERIFY_RESULT_LOGICAL_ID=vm130
VERIFY_RESULT_RESULT=SUCCESS
VERIFY_RESULT_GENERATION=v2-vm130-20260927T212128Z-0046d7efc414
VERIFY_RESULT_REPLICAS=2
VERIFY_RESULT_PAYLOAD_BYTES=11053999065
VERIFY_RESULT_ALLOCATOR_HEALTHY=13
VERIFY_RESULT_ACCEPTED=true
TARGETED_REFRESH_END={"iteration":11,"audit_label":"vm130","verify_target":"vm130","elapsed_seconds":1590,"accepted":true}
TARGET_LOCK_RELEASED|tag=vm130
QUIET_AUDIT_PASS|tag=post-vm130|attempt=1|rc=0|issue_count=0|rpo_count=0|stamp=2621202326883
TARGET_RPO_CLEARED=vm130
=== FINAL_QUIET_AUDIT ===
QUIET_AUDIT_PASS|tag=final|attempt=1|rc=0|issue_count=0|rpo_count=0|stamp=2621202326883
FINAL_AUDIT_RC=0
FINAL_ISSUE_COUNT=0
FINAL_RPO_COUNT=0
FINAL_NON_RPO_COUNT=0
FINAL_AUDIT_OUTPUT_BEGIN
TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55069|generation=app-filebrowser-pve01-20260927T063003Z
TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=54902|generation=app-hbarhub-pve01-20260927T063154Z
TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55003|generation=app-immich-pve01-20260927T063041Z
TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=BACKUP_DUE|legacy_age_seconds=55050|generation=app-skladchik-reports-monitor-pve01-20260927T063026Z
TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=6055|generation=control-bootstrap-v2-pve01-20260916T120057Z
TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=7378|legacy_age_seconds=2045313|replica_count=2|state_id=ct110|generation=v2-ct110-20260927T194116Z-d16fe26dbd17
TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=5867|replica_count=2|state_id=ct111|generation=v2-ct111-20260927T200714Z-2f72200d7742
TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=5774|legacy_age_seconds=2083808|replica_count=2|state_id=ct112|generation=v2-ct112-20260927T201022Z-6cdb5a555315
TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=5674|replica_count=2|state_id=ct113|generation=v2-ct113-20260927T201155Z-87c3bfe1d555
TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=3|replica_count=2|state_id=vm130|generation=v2-vm130-20260927T212128Z-0046d7efc414
TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=5518|legacy_age_seconds=2045429|replica_count=2|state_id=vm150|generation=v2-vm150-20260927T201335Z-2ba76459e7c5
TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=1601|replica_count=2|state_id=vm160|generation=v2-vm160-20260927T211143Z-0cadc4ac0b12
TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=4128|legacy_age_seconds=990050|replica_count=2|state_id=vm170|generation=v2-vm170-20260927T202054Z-1daa87a3cfbc
TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=3796|legacy_age_seconds=1019582|replica_count=2|state_id=vm171|generation=v2-vm171-20260927T203926Z-61ca38fe3ac5
TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=5235|replica_count=2|state_id=vm190|generation=v2-vm190-20260927T201613Z-662906757ca1
TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=2186|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260927T204452Z-91cc0cc7ecc5
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
FINAL_AUDIT_OUTPUT_END
FINAL_AUDIT_SUMMARY=SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
RECOVERY_END_EPOCH=1790545685
RECOVERY_ELAPSED_SECONDS=6077
=== SUCCESS_MAIL_POLICY_WINDOW ===
TARGETED_SUCCESS_BACKUP_EVENTS=4
TARGETED_SUCCESS_MAIL_TO_ROOT_EVENTS=0
SUCCESS_MAIL_POLICY_ACCEPTED=true
=== CONTROL_PLANE_POST ===
WRAPPER_SHA256_POST=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
HELPER_SHA256_POST=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
VZDUMP_CONF_SHA256_POST=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
AUDIT_SHA256_POST=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
V2_TIMER_ACTIVE=active
V2_TIMER_ENABLED=enabled
AUDIT_TIMER_ACTIVE=active
AUDIT_TIMER_ENABLED=enabled
CHANGES_MADE=true
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=MUTATED
TARGETED_REFRESH_COUNT=11
TASK_RESULT=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN
DECISION=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN
NEXT_GATE=FINAL_BACKUP_ACCEPTANCE_CLOSEOUT
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260927-HOMELAB-BACKUP-LOCK-AWARE-RPO-RECOVERY-1235R1","status":"OK","changes_made":true,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+38 -1
View File
File diff suppressed because one or more lines are too long
runtime/latest.txt
+406 -154
View File
File diff suppressed because one or more lines are too long