homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-09-22 04:44:55 +00:00
parent 6e7a4a5a09
commit 46da3cbb9b
4 changed files with 703 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1.json
+38
View File
File diff suppressed because one or more lines are too long
runtime/history/SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1.txt
+359
View File
@@ -0,0 +1,359 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-audit-vmqueue-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=ed70f0c53596e8aa062b17b3c599cc9bc6eda7472edb9ec786f9a88a23af2eb4
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033
SANITIZED_OUTPUT_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033
OUTPUT_BEGIN
WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1
MODE=read-only
COMPONENT=homelab-backup-audit-vmqueue-rca
ERROR_REGISTER_CHECK=OK
REFERENCE_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
BACKUP_AUDIT_STATE_BEGIN=1
Result=exit-code
ExecMainStartTimestamp=Tue 2026-09-22 07:36:58 MSK
ExecMainExitTimestamp=Tue 2026-09-22 07:37:01 MSK
ExecMainCode=1
ExecMainStatus=1
ActiveState=failed
SubState=failed
StateChangeTimestamp=Tue 2026-09-22 07:37:01 MSK
BACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service
BACKUP_AUDIT_FRAGMENT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa
After=network-online.target
Wants=network-online.target
ExecStart=/usr/local/sbin/homelab-backup-audit scheduled
BACKUP_AUDIT_EXIT_EPOCH=1790051821
BACKUP_AUDIT_STATE_END=1
VM_QUEUE_STATE_BEGIN=1
Result=exit-code
ExecMainStartTimestamp=Wed 2026-09-16 20:32:57 MSK
ExecMainExitTimestamp=Wed 2026-09-16 20:33:02 MSK
ExecMainCode=1
ExecMainStatus=65
ActiveState=failed
SubState=failed
StateChangeTimestamp=Wed 2026-09-16 20:33:02 MSK
VM_QUEUE_FRAGMENT_PATH=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service
VM_QUEUE_FRAGMENT_SHA256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759
After=network-online.target
Wants=network-online.target
ExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all
VM_QUEUE_EXIT_EPOCH=1789579982
VM_QUEUE_STATE_END=1
BACKUP_OWNER_STATE_BEGIN=1
Result=success
ExecMainStartTimestamp=Tue 2026-09-22 07:42:15 MSK
ExecMainExitTimestamp=Tue 2026-09-22 07:42:15 MSK
ExecMainCode=1
ExecMainStatus=0
ActiveState=inactive
SubState=dead
StateChangeTimestamp=Tue 2026-09-22 07:42:15 MSK
BACKUP_OWNER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service
BACKUP_OWNER_FRAGMENT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565
After=network-online.target
Wants=network-online.target
ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled
BACKUP_OWNER_EXIT_EPOCH=1790052135
BACKUP_OWNER_STATE_END=1
POLICY_FIX_1108_FINISH_EPOCH=1790030296
VM_QUEUE_LAST_EXIT_EPOCH=1789579982
VM_QUEUE_FAILURE_CLASS=STALE_PRE1108_FAILED_STATE
BACKUP_AUDIT_JOURNAL_SHA256=77b32d66abed4873f79ed2ffb9bf0c4ec87762c4b78c85aa23136d3743f6b364
BACKUP_AUDIT_LATEST_CLASSIFIED_BEGIN=1
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
BACKUP_AUDIT_LATEST_CLASSIFIED_END=1
VM_QUEUE_JOURNAL_SHA256=3967f9d0fff08524f53eedf57007e2a2567de9ba59fad5b5f2948e5039cbfa6d
VM_QUEUE_LATEST_CLASSIFIED_BEGIN=1
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260915T223521Z.json
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260915T223521Z AGE_SECONDS=39387 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH
VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02
VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json
QUEUE_FRESHNESS VMID=160 DECISION=SKIP_FRESH GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=38803 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=160 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json
QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=9957 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json
QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=7668 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
QUEUE_FRESHNESS VMID=171 DECISION=SKIP_FRESH GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=39491 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=171 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=7022 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=6995 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json
QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=6937 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=6832 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH
VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02
VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json
QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_DUE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=53189 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_DUE
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json
QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=24343 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json
QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=22053 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json
QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET
VM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171
VM_QUEUE_LATEST_CLASSIFIED_END=1
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__COUNT=1
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__PATH=/usr/local/sbin/homelab-backup-audit
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
230-if CLOUDB_HEALTH.is_file():
231- txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace")
232- kv={}
233- for line in txt.splitlines():
234- if "=" in line:
235- k,v=line.split("=",1); kv[k]=v
236- cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes")
237-if not cloudb_ok:
238: issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"})
239-
240-source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
241-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
242-if hrc==0: source["local_head"]=ho.strip()
243-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10)
244-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10)
245-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED")
246-if frc==0 and fo.strip():
247- lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
248- if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
249-if source["remote_head"]!=source["local_head"]:
250- issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
SOURCE_GENERATION_NOT_RETURNED___COUNT=2
SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-app-cloud-quorum-queue
SOURCE_GENERATION_NOT_RETURNED___SHA256=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650
116- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
117- *) die "FRESHNESS_RECHECK_INVALID_${a}_${decision:-EMPTY}" 65 ;;
118- esac
119- printf 'QUEUE_BEGIN APP=%s MODE=%s DISPATCH=ONE_TARGET\n' "$a" "$decision"
120- out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"
121- printf '%s\n' "$out"
122- "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null
123- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
124: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$a" 65
125- printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$a" "$generation"
126- printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\n' "$a" "$generation"
127-}
128-
129-migrate_list(){
130- local a out
131- lock_heavy
132- for a in "${MIGRATE_APPS[@]}"; do
133- if out="$(exec 8>&-; exec 9>&-; "$WORKER" latest-sealed "$a" 2>/dev/null)"; then
134- printf 'QUEUE_REUSE_SEALED APP=%s\n%s\n' "$a" "$out"
135- continue
136- fi
SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-vm-cloud-quorum-queue
SOURCE_GENERATION_NOT_RETURNED___SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
131- flock -n 9 || die "GLOBAL_QUEUE_LOCK_BUSY" 75
132- exec 8>"$HEAVY_LOCK"
133- flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75
134- for vmid in "$@"; do
135- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")"
136- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
137- printf '%s\n' "$out"
138- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
139: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65
140- printf 'QUEUE_SEALED VMID=%s GENERATION=%s\n' "$vmid" "$generation"
141- done
142-}
143-dispatch_plan(){
144- freshness_plan
145-}
146-dispatch_one(){
147- local vmid="$1" freshness decision now_epoch out generation
148- [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64
149- expected_owner "$vmid" >/dev/null
150- exec 9>"$LOCK"
151- if ! flock -n 9; then
--
169- ;;
170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
171- *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;;
172- esac
173- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision"
174- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
175- printf '%s\n' "$out"
176- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
177: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65
178- printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$vmid" "$generation"
179- printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\n' "$vmid" "$generation"
180-}
181-
182-ACTION="${1:-}"
183-case "$ACTION" in
184- plan)
185- [[ "$#" == 1 ]] || die "USAGE_PLAN" 64
186- plan_all
187- ;;
188- freshness-plan)
189- [[ "$#" == 1 ]] || die "USAGE_FRESHNESS_PLAN" 64
SCHEDULER_SELECT_RC=0
SCHEDULER_SELECT_BEGIN=1
SELFTEST_REPLACEMENT486=PASS
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
SCHEDULER_SELECT_END=1
CURRENT_RPO_SCHEDULER_CLASS=NO_DUE_TARGET
EVIDENCE_ROOT=/var/lib/homelab-backup
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T093429Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T213739Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260901T103255Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T003249Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T133648Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T023235Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T163234Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T064255Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T193019Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T093128Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T222933Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260906T113150Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T012910Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T143756Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T032904Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T173418Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T063400Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T193647Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260910T093211Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T003012Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T133013Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T013042Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T143251Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T033047Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T174140Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T034236Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T174843Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T064613Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T193324Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T093438Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T223254Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260906T113503Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T013234Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T144115Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T033223Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T173740Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T063725Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T194007Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260910T093532Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260911T003332Z.json
EVIDENCE_ROOT=/var/lib/homelab-backup-v2
MTIME_EPOCH|608|/var/lib/homelab-backup-v2/state/vm171.json
DECISION=PASS_BACKUP_1110_BLOCKERS_CLASSIFIED
NEXT_GATE=REPAIR_ONLY_CLASSIFIED_BACKUP_AUDIT_VMQUEUE_BLOCKERS
TASK_RESULT=PASS_HOMELAB_BACKUP_AUDIT_VMQUEUE_RCA
CHANGES_MADE_BY_1110=false
PRODUCT_MUTATION_BY_1110=false
VM_MUTATION_BY_1110=false
CLOUD_MUTATION_BY_1110=false
WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_END=1
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+11 -11
View File
File diff suppressed because one or more lines are too long
runtime/latest.txt
+295 -59
View File
@@ -1,22 +1,22 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1
STATUS=FAIL
RC=3
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-live-preclosure-rca
COMPONENT=homelab-backup-audit-vmqueue-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=93eb74c1e05a7be6ae8dc468f268a7865ec7e60481c6a0e78a227a0de5eb7a13
COMMAND_SHA256=ed70f0c53596e8aa062b17b3c599cc9bc6eda7472edb9ec786f9a88a23af2eb4
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=3
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
@@ -24,100 +24,336 @@ MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=cf7b386835799489aa2e31f36e3055ab433d761a4858a07309a508d301bb5f43
SANITIZED_OUTPUT_SHA256=cf7b386835799489aa2e31f36e3055ab433d761a4858a07309a508d301bb5f43
RAW_EVIDENCE_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033
SANITIZED_OUTPUT_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033
OUTPUT_BEGIN
WORKERS2_BACKUP_LIVE_PRECLOSURE_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1
WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1
MODE=read-only
COMPONENT=homelab-backup-live-preclosure-rca
COMPONENT=homelab-backup-audit-vmqueue-rca
ERROR_REGISTER_CHECK=OK
REFERENCE_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
BACKUP_AUDIT_UNIT_STATE=ActiveState:failed,SubState:failed,Result:exit-code,ExecMainCode:1,ExecMainStatus:1
VM_QUEUE_UNIT_STATE=ActiveState:failed,SubState:failed,Result:exit-code,ExecMainCode:1,ExecMainStatus:65
BACKUP_OWNER_UNIT_STATE=ActiveState:inactive,SubState:dead,Result:success,ExecMainCode:1,ExecMainStatus:0
TIMER_STATE=homelab-backup-v2.timer,enabled:enabled,active:active
TIMER_STATE=homelab-backup-audit.timer,enabled:enabled,active:active
TIMER_STATE=homelab-vm-cloud-quorum-queue.timer,enabled:disabled,active:inactive
FAILED_UNITS=homelab-backup-audit.service,homelab-vm-cloud-quorum-queue.service,netbird-peers-health.service
BACKUP_AUDIT_JOURNAL_SHA256=547c6183c7dc5f55ef85f218d53806ae723ebbd553e886f5c66959a4e11cd200
BACKUP_AUDIT_ERROR_EXCERPT_BEGIN=1
BACKUP_AUDIT_STATE_BEGIN=1
Result=exit-code
ExecMainStartTimestamp=Tue 2026-09-22 07:36:58 MSK
ExecMainExitTimestamp=Tue 2026-09-22 07:37:01 MSK
ExecMainCode=1
ExecMainStatus=1
ActiveState=failed
SubState=failed
StateChangeTimestamp=Tue 2026-09-22 07:37:01 MSK
BACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service
BACKUP_AUDIT_FRAGMENT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa
After=network-online.target
Wants=network-online.target
ExecStart=/usr/local/sbin/homelab-backup-audit scheduled
BACKUP_AUDIT_EXIT_EPOCH=1790051821
BACKUP_AUDIT_STATE_END=1
VM_QUEUE_STATE_BEGIN=1
Result=exit-code
ExecMainStartTimestamp=Wed 2026-09-16 20:32:57 MSK
ExecMainExitTimestamp=Wed 2026-09-16 20:33:02 MSK
ExecMainCode=1
ExecMainStatus=65
ActiveState=failed
SubState=failed
StateChangeTimestamp=Wed 2026-09-16 20:33:02 MSK
VM_QUEUE_FRAGMENT_PATH=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service
VM_QUEUE_FRAGMENT_SHA256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759
After=network-online.target
Wants=network-online.target
ExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all
VM_QUEUE_EXIT_EPOCH=1789579982
VM_QUEUE_STATE_END=1
BACKUP_OWNER_STATE_BEGIN=1
Result=success
ExecMainStartTimestamp=Tue 2026-09-22 07:42:15 MSK
ExecMainExitTimestamp=Tue 2026-09-22 07:42:15 MSK
ExecMainCode=1
ExecMainStatus=0
ActiveState=inactive
SubState=dead
StateChangeTimestamp=Tue 2026-09-22 07:42:15 MSK
BACKUP_OWNER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service
BACKUP_OWNER_FRAGMENT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565
After=network-online.target
Wants=network-online.target
ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled
BACKUP_OWNER_EXIT_EPOCH=1790052135
BACKUP_OWNER_STATE_END=1
POLICY_FIX_1108_FINISH_EPOCH=1790030296
VM_QUEUE_LAST_EXIT_EPOCH=1789579982
VM_QUEUE_FAILURE_CLASS=STALE_PRE1108_FAILED_STATE
BACKUP_AUDIT_JOURNAL_SHA256=77b32d66abed4873f79ed2ffb9bf0c4ec87762c4b78c85aa23136d3743f6b364
BACKUP_AUDIT_LATEST_CLASSIFIED_BEGIN=1
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-audit.service: Failed with result 'exit-code'.
Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-audit.service: Failed with result 'exit-code'.
Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-audit.service: Failed with result 'exit-code'.
Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-audit.service: Failed with result 'exit-code'.
Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-audit.service: Failed with result 'exit-code'.
Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
BACKUP_AUDIT_ERROR_EXCERPT_END=1
VM_QUEUE_JOURNAL_SHA256=a518b2e9c7ec704afa08b82d78b82e57b506de5aed4a8fad002e7d2191fedfd1
VM_QUEUE_ERROR_EXCERPT_BEGIN=1
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
BACKUP_AUDIT_LATEST_CLASSIFIED_END=1
VM_QUEUE_JOURNAL_SHA256=3967f9d0fff08524f53eedf57007e2a2567de9ba59fad5b5f2948e5039cbfa6d
VM_QUEUE_LATEST_CLASSIFIED_BEGIN=1
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260915T223521Z.json
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260915T223521Z AGE_SECONDS=39387 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH
VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02
VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json
QUEUE_FRESHNESS VMID=160 DECISION=SKIP_FRESH GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=38803 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=160 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json
QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=9957 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json
QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=7668 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
QUEUE_FRESHNESS VMID=171 DECISION=SKIP_FRESH GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=39491 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=171 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=7022 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=6995 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json
QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=6937 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=6832 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH
VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02
VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json
QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_DUE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=53189 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_DUE
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json
QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=24343 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json
QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=22053 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE
CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876
FRESHNESS_STATUS=OK
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260916T113650Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json
QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876
FRESHNESS_STATUS=OK
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET
VM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171
homelab-vm-cloud-quorum-queue.service: Main process exited, code=exited, status=65/DATAERR
homelab-vm-cloud-quorum-queue.service: Failed with result 'exit-code'.
Failed to start homelab-vm-cloud-quorum-queue.service - Homelab canonical VM/CT cloud quorum backup queue.
homelab-vm-cloud-quorum-queue.service: Consumed 4.492s CPU time, 147.2M memory peak.
VM_QUEUE_ERROR_EXCERPT_END=1
VM_QUEUE_LATEST_CLASSIFIED_END=1
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__COUNT=1
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__PATH=/usr/local/sbin/homelab-backup-audit
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
230-if CLOUDB_HEALTH.is_file():
231- txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace")
232- kv={}
233- for line in txt.splitlines():
234- if "=" in line:
235- k,v=line.split("=",1); kv[k]=v
236- cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes")
237-if not cloudb_ok:
238: issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"})
239-
240-source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
241-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
242-if hrc==0: source["local_head"]=ho.strip()
243-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10)
244-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10)
245-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED")
246-if frc==0 and fo.strip():
247- lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
248- if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
249-if source["remote_head"]!=source["local_head"]:
250- issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
SOURCE_GENERATION_NOT_RETURNED___COUNT=2
SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-app-cloud-quorum-queue
SOURCE_GENERATION_NOT_RETURNED___SHA256=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650
116- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
117- *) die "FRESHNESS_RECHECK_INVALID_${a}_${decision:-EMPTY}" 65 ;;
118- esac
119- printf 'QUEUE_BEGIN APP=%s MODE=%s DISPATCH=ONE_TARGET\n' "$a" "$decision"
120- out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"
121- printf '%s\n' "$out"
122- "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null
123- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
124: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$a" 65
125- printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$a" "$generation"
126- printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\n' "$a" "$generation"
127-}
128-
129-migrate_list(){
130- local a out
131- lock_heavy
132- for a in "${MIGRATE_APPS[@]}"; do
133- if out="$(exec 8>&-; exec 9>&-; "$WORKER" latest-sealed "$a" 2>/dev/null)"; then
134- printf 'QUEUE_REUSE_SEALED APP=%s\n%s\n' "$a" "$out"
135- continue
136- fi
SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-vm-cloud-quorum-queue
SOURCE_GENERATION_NOT_RETURNED___SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
131- flock -n 9 || die "GLOBAL_QUEUE_LOCK_BUSY" 75
132- exec 8>"$HEAVY_LOCK"
133- flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75
134- for vmid in "$@"; do
135- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")"
136- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
137- printf '%s\n' "$out"
138- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
139: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65
140- printf 'QUEUE_SEALED VMID=%s GENERATION=%s\n' "$vmid" "$generation"
141- done
142-}
143-dispatch_plan(){
144- freshness_plan
145-}
146-dispatch_one(){
147- local vmid="$1" freshness decision now_epoch out generation
148- [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64
149- expected_owner "$vmid" >/dev/null
150- exec 9>"$LOCK"
151- if ! flock -n 9; then
--
169- ;;
170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
171- *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;;
172- esac
173- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision"
174- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
175- printf '%s\n' "$out"
176- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
177: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65
178- printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$vmid" "$generation"
179- printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\n' "$vmid" "$generation"
180-}
181-
182-ACTION="${1:-}"
183-case "$ACTION" in
184- plan)
185- [[ "$#" == 1 ]] || die "USAGE_PLAN" 64
186- plan_all
187- ;;
188- freshness-plan)
189- [[ "$#" == 1 ]] || die "USAGE_FRESHNESS_PLAN" 64
SCHEDULER_SELECT_RC=0
SCHEDULER_SELECT_BEGIN=1
SELFTEST_REPLACEMENT486=PASS
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
SCHEDULER_SELECT_END=1
DECISION=HOLD_BACKUP_FINAL_CLOSURE_LIVE_RCA_REQUIRED
NEXT_GATE=RECONCILE_ONLY_BACKUP_AUDIT_VMQUEUE_RPO_FAILURES
TASK_RESULT=HOLD_HOMELAB_BACKUP_LIVE_PRECLOSURE_RCA
CHANGES_MADE_BY_1109=false
PRODUCT_MUTATION_BY_1109=false
VM_MUTATION_BY_1109=false
CLOUD_MUTATION_BY_1109=false
WORKERS2_BACKUP_LIVE_PRECLOSURE_RCA_END=1
CURRENT_RPO_SCHEDULER_CLASS=NO_DUE_TARGET
EVIDENCE_ROOT=/var/lib/homelab-backup
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T093429Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T213739Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260901T103255Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T003249Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T133648Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T023235Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T163234Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T064255Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T193019Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T093128Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T222933Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260906T113150Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T012910Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T143756Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T032904Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T173418Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T063400Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T193647Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260910T093211Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T003012Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T133013Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T013042Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T143251Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T033047Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T174140Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T034236Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T174843Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T064613Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T193324Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T093438Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T223254Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260906T113503Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T013234Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T144115Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T033223Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T173740Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T063725Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T194007Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260910T093532Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260911T003332Z.json
EVIDENCE_ROOT=/var/lib/homelab-backup-v2
MTIME_EPOCH|608|/var/lib/homelab-backup-v2/state/vm171.json
DECISION=PASS_BACKUP_1110_BLOCKERS_CLASSIFIED
NEXT_GATE=REPAIR_ONLY_CLASSIFIED_BACKUP_AUDIT_VMQUEUE_BLOCKERS
TASK_RESULT=PASS_HOMELAB_BACKUP_AUDIT_VMQUEUE_RCA
CHANGES_MADE_BY_1110=false
PRODUCT_MUTATION_BY_1110=false
VM_MUTATION_BY_1110=false
CLOUD_MUTATION_BY_1110=false
WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_END=1
OUTPUT_END
CHAT_OUTPUT_END