runtime: publish SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1

runtime/history/SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1.txt

@@ -0,0 +1,123 @@
+CHAT_OUTPUT_BEGIN
+COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1
+STATUS=FAIL
+RC=3
+HOST=pve01
+MODE=read-only
+COMPONENT=homelab-backup-live-preclosure-rca
+REFERENCE_REGISTER_CHECK=OK
+REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
+ERROR_REGISTER_CHECK=OK
+ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
+COMMAND_SHA256=93eb74c1e05a7be6ae8dc468f268a7865ec7e60481c6a0e78a227a0de5eb7a13
+DUPLICATE_FAILED_COMMAND_BLOCKED=false
+EXECUTION_STARTED=true
+CHANGE_DECLARED=false
+RESULT_CONTRACT_VALID=true
+RESULT_CONTRACT_STATUS=NOT_APPLICABLE
+RESULT_CONTRACT_ERROR=NONE
+COMMAND_RC=3
+CHANGES_MADE=false
+ROLLBACK_STARTED=false
+ROLLBACK_RESTORED=null
+MUTATION_OUTCOME=NO_MUTATION
+SANITIZED=yes
+SECRETS_INCLUDED=no
+PRIVATE_ADDRESSES_INCLUDED=no
+RAW_EVIDENCE_SHA256=cf7b386835799489aa2e31f36e3055ab433d761a4858a07309a508d301bb5f43
+SANITIZED_OUTPUT_SHA256=cf7b386835799489aa2e31f36e3055ab433d761a4858a07309a508d301bb5f43
+OUTPUT_BEGIN
+WORKERS2_BACKUP_LIVE_PRECLOSURE_RCA_BEGIN=1
+COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1
+MODE=read-only
+COMPONENT=homelab-backup-live-preclosure-rca
+ERROR_REGISTER_CHECK=OK
+REFERENCE_CHECK=OK
+REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
+ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
+RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
+WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
+BACKUP_AUDIT_UNIT_STATE=ActiveState:failed,SubState:failed,Result:exit-code,ExecMainCode:1,ExecMainStatus:1
+VM_QUEUE_UNIT_STATE=ActiveState:failed,SubState:failed,Result:exit-code,ExecMainCode:1,ExecMainStatus:65
+BACKUP_OWNER_UNIT_STATE=ActiveState:inactive,SubState:dead,Result:success,ExecMainCode:1,ExecMainStatus:0
+TIMER_STATE=homelab-backup-v2.timer,enabled:enabled,active:active
+TIMER_STATE=homelab-backup-audit.timer,enabled:enabled,active:active
+TIMER_STATE=homelab-vm-cloud-quorum-queue.timer,enabled:disabled,active:inactive
+FAILED_UNITS=homelab-backup-audit.service,homelab-vm-cloud-quorum-queue.service,netbird-peers-health.service
+BACKUP_AUDIT_JOURNAL_SHA256=547c6183c7dc5f55ef85f218d53806ae723ebbd553e886f5c66959a4e11cd200
+BACKUP_AUDIT_ERROR_EXCERPT_BEGIN=1
+ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
+SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
+homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
+homelab-backup-audit.service: Failed with result 'exit-code'.
+Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
+ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
+SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
+homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
+homelab-backup-audit.service: Failed with result 'exit-code'.
+Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
+ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
+SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
+homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
+homelab-backup-audit.service: Failed with result 'exit-code'.
+Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
+ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
+SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
+homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
+homelab-backup-audit.service: Failed with result 'exit-code'.
+Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
+ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
+SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
+homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
+homelab-backup-audit.service: Failed with result 'exit-code'.
+Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
+BACKUP_AUDIT_ERROR_EXCERPT_END=1
+VM_QUEUE_JOURNAL_SHA256=a518b2e9c7ec704afa08b82d78b82e57b506de5aed4a8fad002e7d2191fedfd1
+VM_QUEUE_ERROR_EXCERPT_BEGIN=1
+QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
+QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE
+CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876
+FRESHNESS_STATUS=OK
+STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
+RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
+QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
+FRESHNESS_STATUS=OK
+STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
+RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
+QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
+FRESHNESS_STATUS=OK
+STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260916T113650Z.json
+RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json
+QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
+FRESHNESS_STATUS=OK
+STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
+RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
+QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
+GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876
+FRESHNESS_STATUS=OK
+STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json
+RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
+QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
+QUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET
+VM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171
+homelab-vm-cloud-quorum-queue.service: Main process exited, code=exited, status=65/DATAERR
+homelab-vm-cloud-quorum-queue.service: Failed with result 'exit-code'.
+Failed to start homelab-vm-cloud-quorum-queue.service - Homelab canonical VM/CT cloud quorum backup queue.
+homelab-vm-cloud-quorum-queue.service: Consumed 4.492s CPU time, 147.2M memory peak.
+VM_QUEUE_ERROR_EXCERPT_END=1
+SCHEDULER_SELECT_RC=0
+SCHEDULER_SELECT_BEGIN=1
+SELFTEST_REPLACEMENT486=PASS
+SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
+SCHEDULER_SELECT_END=1
+DECISION=HOLD_BACKUP_FINAL_CLOSURE_LIVE_RCA_REQUIRED
+NEXT_GATE=RECONCILE_ONLY_BACKUP_AUDIT_VMQUEUE_RPO_FAILURES
+TASK_RESULT=HOLD_HOMELAB_BACKUP_LIVE_PRECLOSURE_RCA
+CHANGES_MADE_BY_1109=false
+PRODUCT_MUTATION_BY_1109=false
+VM_MUTATION_BY_1109=false
+CLOUD_MUTATION_BY_1109=false
+WORKERS2_BACKUP_LIVE_PRECLOSURE_RCA_END=1
+
+OUTPUT_END
+CHAT_OUTPUT_END