CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-audit-vmqueue-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=ed70f0c53596e8aa062b17b3c599cc9bc6eda7472edb9ec786f9a88a23af2eb4
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033
SANITIZED_OUTPUT_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033
OUTPUT_BEGIN
WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1
MODE=read-only
COMPONENT=homelab-backup-audit-vmqueue-rca
ERROR_REGISTER_CHECK=OK
REFERENCE_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
BACKUP_AUDIT_STATE_BEGIN=1
Result=exit-code
ExecMainStartTimestamp=Tue 2026-09-22 07:36:58 MSK
ExecMainExitTimestamp=Tue 2026-09-22 07:37:01 MSK
ExecMainCode=1
ExecMainStatus=1
ActiveState=failed
SubState=failed
StateChangeTimestamp=Tue 2026-09-22 07:37:01 MSK
BACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service
BACKUP_AUDIT_FRAGMENT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa
After=network-online.target
Wants=network-online.target
ExecStart=/usr/local/sbin/homelab-backup-audit scheduled
BACKUP_AUDIT_EXIT_EPOCH=1790051821
BACKUP_AUDIT_STATE_END=1
VM_QUEUE_STATE_BEGIN=1
Result=exit-code
ExecMainStartTimestamp=Wed 2026-09-16 20:32:57 MSK
ExecMainExitTimestamp=Wed 2026-09-16 20:33:02 MSK
ExecMainCode=1
ExecMainStatus=65
ActiveState=failed
SubState=failed
StateChangeTimestamp=Wed 2026-09-16 20:33:02 MSK
VM_QUEUE_FRAGMENT_PATH=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service
VM_QUEUE_FRAGMENT_SHA256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759
After=network-online.target
Wants=network-online.target
ExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all
VM_QUEUE_EXIT_EPOCH=1789579982
VM_QUEUE_STATE_END=1
BACKUP_OWNER_STATE_BEGIN=1
Result=success
ExecMainStartTimestamp=Tue 2026-09-22 07:42:15 MSK
ExecMainExitTimestamp=Tue 2026-09-22 07:42:15 MSK
ExecMainCode=1
ExecMainStatus=0
ActiveState=inactive
SubState=dead
StateChangeTimestamp=Tue 2026-09-22 07:42:15 MSK
BACKUP_OWNER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service
BACKUP_OWNER_FRAGMENT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565
After=network-online.target
Wants=network-online.target
ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled
BACKUP_OWNER_EXIT_EPOCH=1790052135
BACKUP_OWNER_STATE_END=1
POLICY_FIX_1108_FINISH_EPOCH=1790030296
VM_QUEUE_LAST_EXIT_EPOCH=1789579982
VM_QUEUE_FAILURE_CLASS=STALE_PRE1108_FAILED_STATE
BACKUP_AUDIT_JOURNAL_SHA256=77b32d66abed4873f79ed2ffb9bf0c4ec87762c4b78c85aa23136d3743f6b364
BACKUP_AUDIT_LATEST_CLASSIFIED_BEGIN=1
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
BACKUP_AUDIT_LATEST_CLASSIFIED_END=1
VM_QUEUE_JOURNAL_SHA256=3967f9d0fff08524f53eedf57007e2a2567de9ba59fad5b5f2948e5039cbfa6d
VM_QUEUE_LATEST_CLASSIFIED_BEGIN=1
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260915T223521Z.json
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260915T223521Z AGE_SECONDS=39387 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH
VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02
VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json
QUEUE_FRESHNESS VMID=160 DECISION=SKIP_FRESH GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=38803 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=160 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json
QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=9957 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json
QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=7668 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
QUEUE_FRESHNESS VMID=171 DECISION=SKIP_FRESH GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=39491 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=171 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=7022 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=6995 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json
QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=6937 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=6832 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH
VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02
VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json
QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_DUE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=53189 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_DUE
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json
QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=24343 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json
QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=22053 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json
QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876
FRESHNESS_STATUS=OK
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
QUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET
VM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171
VM_QUEUE_LATEST_CLASSIFIED_END=1
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__COUNT=1
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__PATH=/usr/local/sbin/homelab-backup-audit
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
230-if CLOUDB_HEALTH.is_file():
231-    txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace")
232-    kv={}
233-    for line in txt.splitlines():
234-        if "=" in line:
235-            k,v=line.split("=",1); kv[k]=v
236-    cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes")
237-if not cloudb_ok:
238:    issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"})
239-
240-source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
241-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
242-if hrc==0: source["local_head"]=ho.strip()
243-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10)
244-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10)
245-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED")
246-if frc==0 and fo.strip():
247-    lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
248-    if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
249-if source["remote_head"]!=source["local_head"]:
250-    issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
SOURCE_GENERATION_NOT_RETURNED___COUNT=2
SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-app-cloud-quorum-queue
SOURCE_GENERATION_NOT_RETURNED___SHA256=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650
116-    BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
117-    *) die "FRESHNESS_RECHECK_INVALID_${a}_${decision:-EMPTY}" 65 ;;
118-  esac
119-  printf 'QUEUE_BEGIN APP=%s MODE=%s DISPATCH=ONE_TARGET\n' "$a" "$decision"
120-  out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"
121-  printf '%s\n' "$out"
122-  "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null
123-  generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
124:  [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$a" 65
125-  printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$a" "$generation"
126-  printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\n' "$a" "$generation"
127-}
128-
129-migrate_list(){
130-  local a out
131-  lock_heavy
132-  for a in "${MIGRATE_APPS[@]}"; do
133-    if out="$(exec 8>&-; exec 9>&-; "$WORKER" latest-sealed "$a" 2>/dev/null)"; then
134-      printf 'QUEUE_REUSE_SEALED APP=%s\n%s\n' "$a" "$out"
135-      continue
136-    fi
SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-vm-cloud-quorum-queue
SOURCE_GENERATION_NOT_RETURNED___SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
131-  flock -n 9 || die "GLOBAL_QUEUE_LOCK_BUSY" 75
132-  exec 8>"$HEAVY_LOCK"
133-  flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75
134-  for vmid in "$@"; do
135-    printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")"
136-    out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
137-    printf '%s\n' "$out"
138-    generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
139:    [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65
140-    printf 'QUEUE_SEALED VMID=%s GENERATION=%s\n' "$vmid" "$generation"
141-  done
142-}
143-dispatch_plan(){
144-  freshness_plan
145-}
146-dispatch_one(){
147-  local vmid="$1" freshness decision now_epoch out generation
148-  [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64
149-  expected_owner "$vmid" >/dev/null
150-  exec 9>"$LOCK"
151-  if ! flock -n 9; then
--
169-      ;;
170-    BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
171-    *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;;
172-  esac
173-  printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision"
174-  out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
175-  printf '%s\n' "$out"
176-  generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
177:  [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65
178-  printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$vmid" "$generation"
179-  printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\n' "$vmid" "$generation"
180-}
181-
182-ACTION="${1:-}"
183-case "$ACTION" in
184-  plan)
185-    [[ "$#" == 1 ]] || die "USAGE_PLAN" 64
186-    plan_all
187-    ;;
188-  freshness-plan)
189-    [[ "$#" == 1 ]] || die "USAGE_FRESHNESS_PLAN" 64
SCHEDULER_SELECT_RC=0
SCHEDULER_SELECT_BEGIN=1
SELFTEST_REPLACEMENT486=PASS
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
SCHEDULER_SELECT_END=1
CURRENT_RPO_SCHEDULER_CLASS=NO_DUE_TARGET
EVIDENCE_ROOT=/var/lib/homelab-backup
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T093429Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T213739Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260901T103255Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T003249Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T133648Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T023235Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T163234Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T064255Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T193019Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T093128Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T222933Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260906T113150Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T012910Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T143756Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T032904Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T173418Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T063400Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T193647Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260910T093211Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T003012Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T133013Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T013042Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T143251Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T033047Z.json
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T174140Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T034236Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T174843Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T064613Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T193324Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T093438Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T223254Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260906T113503Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T013234Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T144115Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T033223Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T173740Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T063725Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T194007Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260910T093532Z.json
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260911T003332Z.json
EVIDENCE_ROOT=/var/lib/homelab-backup-v2
MTIME_EPOCH|608|/var/lib/homelab-backup-v2/state/vm171.json
DECISION=PASS_BACKUP_1110_BLOCKERS_CLASSIFIED
NEXT_GATE=REPAIR_ONLY_CLASSIFIED_BACKUP_AUDIT_VMQUEUE_BLOCKERS
TASK_RESULT=PASS_HOMELAB_BACKUP_AUDIT_VMQUEUE_RCA
CHANGES_MADE_BY_1110=false
PRODUCT_MUTATION_BY_1110=false
VM_MUTATION_BY_1110=false
CLOUD_MUTATION_BY_1110=false
WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_END=1

OUTPUT_END
CHAT_OUTPUT_END
