CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-VM-RPO-REFRESH-1101R1
STATUS=FAIL
RC=10
HOST=pve01
MODE=change
COMPONENT=homelab-backup-vm-rpo-refresh
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=b67692a4f48430147fce56f9140fb1ac7e06b0b298409438f04c42cfe5b6f715
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=true
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=FAIL
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=10
CHANGES_MADE=true
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=MUTATED
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=61c35e9aa097ac930d804b57150f74d84901a62da41c2c100c94e40589b9f9e2
SANITIZED_OUTPUT_SHA256=61c35e9aa097ac930d804b57150f74d84901a62da41c2c100c94e40589b9f9e2
OUTPUT_BEGIN
WORKERS2_VM_RPO_REFRESH_BEGIN=1
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-VM-RPO-REFRESH-1101R1
MODE=change
COMPONENT=homelab-backup-vm-rpo-refresh
MUTATION_BOUNDARY=VM_BACKUP_SERVICE_ONLY_WHEN_CURRENT_RPO_VM_STALE;NO_APP_BACKUP;NO_TIMER_CHANGE;NO_KILL;NO_SIGNAL;NO_LOCK_BREAK;NO_CONFIG_CHANGE;NO_GC;NO_MAIL;NO_GIT_WRITE
RPO_BEFORE=[{"age_seconds":93352,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm110"},{"age_seconds":86451,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm111"},{"age_seconds":80396,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm112"},{"age_seconds":78231,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm113"},{"age_seconds":72839,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm130"},{"age_seconds":76125,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm150"}]
BACKUP_STATE_BEFORE={"active":"inactive","exec_main_status":0,"heavy_lock_free":true,"mainpid":0,"result":"success","sub":"dead"}
RPO_LOOP_1=[{"age_seconds":93353,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm110"},{"age_seconds":86452,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm111"},{"age_seconds":80397,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm112"},{"age_seconds":78232,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm113"},{"age_seconds":72840,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm130"},{"age_seconds":76126,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm150"}]
OWNER_WAIT label=started-by-1101 elapsed=0 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=60 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=120 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=180 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=240 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=300 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=361 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=421 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=481 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=541 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=601 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=662 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=722 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=782 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1454793,"result":"success","sub":"start"}
VM_REFRESH_RUN={"action":"start_backup_service","elapsed":832,"end_state":{"active":"inactive","exec_main_status":0,"heavy_lock_free":true,"mainpid":0,"result":"success","sub":"dead"},"iteration":1,"ok":true,"stale_before":["vm110","vm111","vm112","vm113","vm130","vm150"],"start_rc":0}
VM_REFRESH_PROGRESS_1={"cleared":[],"newly_stale":["vm160"],"remaining":["vm110","vm111","vm112","vm113","vm130","vm150","vm160"]}
RPO_LOOP_2=[{"age_seconds":94194,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm110"},{"age_seconds":87293,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm111"},{"age_seconds":81238,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm112"},{"age_seconds":79073,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm113"},{"age_seconds":73681,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm130"},{"age_seconds":76967,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm150"},{"age_seconds":72214,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm160"}]
OWNER_WAIT label=started-by-1101 elapsed=0 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1495880,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=60 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1495880,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=120 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1495880,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=180 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1495880,"result":"success","sub":"start"}
VM_REFRESH_RUN={"action":"start_backup_service","elapsed":190,"end_state":{"active":"inactive","exec_main_status":0,"heavy_lock_free":true,"mainpid":0,"result":"success","sub":"dead"},"iteration":2,"ok":true,"stale_before":["vm110","vm111","vm112","vm113","vm130","vm150","vm160"],"start_rc":0}
VM_REFRESH_PROGRESS_2={"cleared":["vm110"],"newly_stale":[],"remaining":["vm111","vm112","vm113","vm130","vm150","vm160"]}
RPO_LOOP_3=[{"age_seconds":87492,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm111"},{"age_seconds":81437,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm112"},{"age_seconds":79272,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm113"},{"age_seconds":73880,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm130"},{"age_seconds":77166,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm150"},{"age_seconds":72413,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm160"}]
OWNER_WAIT label=started-by-1101 elapsed=0 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1511723,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=60 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1511723,"result":"success","sub":"start"}
OWNER_WAIT label=started-by-1101 elapsed=120 state={"active":"activating","exec_main_status":0,"heavy_lock_free":false,"mainpid":1511723,"result":"success","sub":"start"}
VM_REFRESH_RUN={"action":"start_backup_service","elapsed":140,"end_state":{"active":"inactive","exec_main_status":0,"heavy_lock_free":true,"mainpid":0,"result":"success","sub":"dead"},"iteration":3,"ok":true,"stale_before":["vm111","vm112","vm113","vm130","vm150","vm160"],"start_rc":0}
VM_REFRESH_PROGRESS_3={"cleared":["vm111"],"newly_stale":[],"remaining":["vm112","vm113","vm130","vm150","vm160"]}
RPO_LOOP_4=[{"age_seconds":81585,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm112"},{"age_seconds":79420,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm113"},{"age_seconds":74028,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm130"},{"age_seconds":77314,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm150"},{"age_seconds":72561,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm160"}]
OWNER_WAIT label=started-by-1101 elapsed=0 state={"active":"inactive","exec_main_status":0,"heavy_lock_free":true,"mainpid":0,"result":"success","sub":"dead"}
VM_REFRESH_RUN={"action":"start_backup_service","elapsed":0,"end_state":{"active":"inactive","exec_main_status":0,"heavy_lock_free":true,"mainpid":0,"result":"success","sub":"dead"},"iteration":4,"ok":true,"stale_before":["vm112","vm113","vm130","vm150","vm160"],"start_rc":0}
VM_REFRESH_PROGRESS_4={"cleared":[],"newly_stale":[],"remaining":["vm112","vm113","vm130","vm150","vm160"]}
RPO_LOOP_5=[{"age_seconds":81593,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm112"},{"age_seconds":79428,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm113"},{"age_seconds":74036,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm130"},{"age_seconds":77322,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm150"},{"age_seconds":72569,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm160"}]
OWNER_WAIT label=started-by-1101 elapsed=0 state={"active":"inactive","exec_main_status":0,"heavy_lock_free":true,"mainpid":0,"result":"success","sub":"dead"}
VM_REFRESH_RUN={"action":"start_backup_service","elapsed":0,"end_state":{"active":"inactive","exec_main_status":0,"heavy_lock_free":true,"mainpid":0,"result":"success","sub":"dead"},"iteration":5,"ok":true,"stale_before":["vm112","vm113","vm130","vm150","vm160"],"start_rc":0}
VM_REFRESH_PROGRESS_5={"cleared":[],"newly_stale":[],"remaining":["vm112","vm113","vm130","vm150","vm160"]}
RPO_LOOP_6=[{"age_seconds":81601,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm112"},{"age_seconds":79436,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm113"},{"age_seconds":74044,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm130"},{"age_seconds":77330,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm150"},{"age_seconds":72577,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm160"}]
OWNER_WAIT label=started-by-1101 elapsed=0 state={"active":"inactive","exec_main_status":0,"heavy_lock_free":true,"mainpid":0,"result":"success","sub":"dead"}
VM_REFRESH_RUN={"action":"start_backup_service","elapsed":0,"end_state":{"active":"inactive","exec_main_status":0,"heavy_lock_free":true,"mainpid":0,"result":"success","sub":"dead"},"iteration":6,"ok":true,"stale_before":["vm112","vm113","vm130","vm150","vm160"],"start_rc":0}
VM_REFRESH_PROGRESS_6={"cleared":[],"newly_stale":[],"remaining":["vm112","vm113","vm130","vm150","vm160"]}
RPO_STALLED=[{"age_seconds":81608,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm112"},{"age_seconds":79443,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm113"},{"age_seconds":74051,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm130"},{"age_seconds":77337,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm150"},{"age_seconds":72584,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm160"}]
FAILURE=VM_RPO_REFRESH_STALLED
TASK_RESULT=FAIL_VM_RPO_REFRESH
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-VM-RPO-REFRESH-1101R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null}

OUTPUT_END
CHAT_OUTPUT_END
