CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1
STATUS=FAIL
RC=3
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-live-preclosure-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=93eb74c1e05a7be6ae8dc468f268a7865ec7e60481c6a0e78a227a0de5eb7a13
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=3
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=cf7b386835799489aa2e31f36e3055ab433d761a4858a07309a508d301bb5f43
SANITIZED_OUTPUT_SHA256=cf7b386835799489aa2e31f36e3055ab433d761a4858a07309a508d301bb5f43
OUTPUT_BEGIN
WORKERS2_BACKUP_LIVE_PRECLOSURE_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1
MODE=read-only
COMPONENT=homelab-backup-live-preclosure-rca
ERROR_REGISTER_CHECK=OK
REFERENCE_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
BACKUP_AUDIT_UNIT_STATE=ActiveState:failed,SubState:failed,Result:exit-code,ExecMainCode:1,ExecMainStatus:1
VM_QUEUE_UNIT_STATE=ActiveState:failed,SubState:failed,Result:exit-code,ExecMainCode:1,ExecMainStatus:65
BACKUP_OWNER_UNIT_STATE=ActiveState:inactive,SubState:dead,Result:success,ExecMainCode:1,ExecMainStatus:0
TIMER_STATE=homelab-backup-v2.timer,enabled:enabled,active:active
TIMER_STATE=homelab-backup-audit.timer,enabled:enabled,active:active
TIMER_STATE=homelab-vm-cloud-quorum-queue.timer,enabled:disabled,active:inactive
FAILED_UNITS=homelab-backup-audit.service,homelab-vm-cloud-quorum-queue.service,netbird-peers-health.service
BACKUP_AUDIT_JOURNAL_SHA256=547c6183c7dc5f55ef85f218d53806ae723ebbd553e886f5c66959a4e11cd200
BACKUP_AUDIT_ERROR_EXCERPT_BEGIN=1
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-audit.service: Failed with result 'exit-code'.
Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-audit.service: Failed with result 'exit-code'.
Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-audit.service: Failed with result 'exit-code'.
Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-audit.service: Failed with result 'exit-code'.
Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-audit.service: Failed with result 'exit-code'.
Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
BACKUP_AUDIT_ERROR_EXCERPT_END=1
VM_QUEUE_JOURNAL_SHA256=a518b2e9c7ec704afa08b82d78b82e57b506de5aed4a8fad002e7d2191fedfd1
VM_QUEUE_ERROR_EXCERPT_BEGIN=1
QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE
CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876
FRESHNESS_STATUS=OK
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
FRESHNESS_STATUS=OK
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
FRESHNESS_STATUS=OK
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260916T113650Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json
QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
FRESHNESS_STATUS=OK
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876
FRESHNESS_STATUS=OK
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET
VM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171
homelab-vm-cloud-quorum-queue.service: Main process exited, code=exited, status=65/DATAERR
homelab-vm-cloud-quorum-queue.service: Failed with result 'exit-code'.
Failed to start homelab-vm-cloud-quorum-queue.service - Homelab canonical VM/CT cloud quorum backup queue.
homelab-vm-cloud-quorum-queue.service: Consumed 4.492s CPU time, 147.2M memory peak.
VM_QUEUE_ERROR_EXCERPT_END=1
SCHEDULER_SELECT_RC=0
SCHEDULER_SELECT_BEGIN=1
SELFTEST_REPLACEMENT486=PASS
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
SCHEDULER_SELECT_END=1
DECISION=HOLD_BACKUP_FINAL_CLOSURE_LIVE_RCA_REQUIRED
NEXT_GATE=RECONCILE_ONLY_BACKUP_AUDIT_VMQUEUE_RPO_FAILURES
TASK_RESULT=HOLD_HOMELAB_BACKUP_LIVE_PRECLOSURE_RCA
CHANGES_MADE_BY_1109=false
PRODUCT_MUTATION_BY_1109=false
VM_MUTATION_BY_1109=false
CLOUD_MUTATION_BY_1109=false
WORKERS2_BACKUP_LIVE_PRECLOSURE_RCA_END=1

OUTPUT_END
CHAT_OUTPUT_END
