CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-RPO-REFRESH-1098R1
STATUS=FAIL
RC=2
HOST=pve01
MODE=change
COMPONENT=homelab-backup-rpo-refresh
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=12198c2fba8c04dcd25b0328d5f33533ac89d5a6412d093c166650665d0e5344
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=true
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=FAIL
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=2
CHANGES_MADE=true
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=MUTATED
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=eadab1eba08cb0aa59cf3e89d0217dff44bbca0cfd40c94c3087ec0737596519
SANITIZED_OUTPUT_SHA256=eadab1eba08cb0aa59cf3e89d0217dff44bbca0cfd40c94c3087ec0737596519
OUTPUT_BEGIN
WORKERS2_RPO_REFRESH_BEGIN=1
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-RPO-REFRESH-1098R1
MODE=change
COMPONENT=homelab-backup-rpo-refresh
MUTATION_BOUNDARY=START_EXISTING_APP_AND_BACKUP_SERVICES_ONLY_WHEN_RPO_STALE;NO_TIMER_DISABLE;NO_KILL;NO_SIGNAL;NO_LOCK_BREAK;NO_CONFIG_CHANGE;NO_GIT_WRITE;NO_DELETE
RPO_BEFORE=[{"decision":"BACKUP_DUE","freshness_plan_rc":0,"kind":"APP","logical_id":"app-filebrowser"},{"decision":"BACKUP_DUE","freshness_plan_rc":0,"kind":"APP","logical_id":"app-hbarhub"},{"decision":"BACKUP_DUE","freshness_plan_rc":0,"kind":"APP","logical_id":"app-immich"},{"decision":"BACKUP_DUE","freshness_plan_rc":0,"kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"age_seconds":84882,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm110"},{"age_seconds":77981,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm111"}]
BACKUP_STATE_BEFORE={"heavy_lock_free":true,"mainpid":0}
RPO_LOOP_1=[{"decision":"BACKUP_DUE","freshness_plan_rc":0,"kind":"APP","logical_id":"app-filebrowser"},{"decision":"BACKUP_DUE","freshness_plan_rc":0,"kind":"APP","logical_id":"app-hbarhub"},{"decision":"BACKUP_DUE","freshness_plan_rc":0,"kind":"APP","logical_id":"app-immich"},{"decision":"BACKUP_DUE","freshness_plan_rc":0,"kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"age_seconds":84884,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm110"},{"age_seconds":77983,"freshness_limit_seconds":72000,"kind":"VM","logical_id":"vm111"}]
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=60 active=success sub=activating result=start
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=120 active=success sub=activating result=start
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=180 active=success sub=activating result=start
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=240 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=300 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=360 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=420 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=480 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=540 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=600 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=660 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=720 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=780 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=840 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=900 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=960 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1020 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1080 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1140 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1200 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1260 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1321 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1381 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1441 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1501 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1561 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1621 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1681 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1741 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1801 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1861 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1921 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=1981 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2041 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2101 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2161 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2221 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2281 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2341 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2401 active=success sub=activating result=start
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2461 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2521 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2581 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2642 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2702 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2762 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2822 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2882 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=2942 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3002 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3062 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3122 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3182 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3242 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3302 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3362 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3422 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3482 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3542 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3602 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3662 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3722 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3782 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3842 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3902 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=3962 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4023 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4083 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4143 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4203 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4263 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4323 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4383 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4443 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4503 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4563 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4623 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4683 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4743 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4803 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4863 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4923 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=4983 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5043 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5103 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5163 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5223 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5283 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5344 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5404 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5464 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5524 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5584 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5644 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5704 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5764 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5824 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5884 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=5944 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6004 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6064 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6124 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6184 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6244 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6304 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6364 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6424 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6484 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6544 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6604 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6664 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6725 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6785 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6845 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6905 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=6965 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=7025 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=7085 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=7145 active=success sub=inactive result=dead
UNIT_WAIT unit=homelab-app-cloud-quorum-queue.service elapsed=7205 active=success sub=inactive result=dead
APP_REFRESH_RUN={"iteration":1,"meta":{"active":"success","elapsed":7205,"result":"TIMEOUT","start_rc":0,"sub":"inactive"},"ok":false,"unit":"homelab-app-cloud-quorum-queue.service"}
FAILURE=APP_REFRESH_SERVICE_FAILED
TASK_RESULT=FAIL_RPO_REFRESH
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-RPO-REFRESH-1098R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null}

OUTPUT_END
CHAT_OUTPUT_END
