{
  "schema_version": 1,
  "channel": "homelab-runtime",
  "command_id": "SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1",
  "status": "OK",
  "rc": 0,
  "host": "pve01",
  "mode": "read-only",
  "component": "homelab-backup-audit-vmqueue-rca",
  "started_at_utc": "2026-09-22T04:44:54Z",
  "finished_at_utc": "2026-09-22T04:44:54Z",
  "reference_register_checked": true,
  "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
  "error_register_checked": true,
  "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
  "command_sha256": "ed70f0c53596e8aa062b17b3c599cc9bc6eda7472edb9ec786f9a88a23af2eb4",
  "duplicate_failed_command_blocked": false,
  "block_reason": null,
  "execution_started": true,
  "change_declared": false,
  "result_contract_valid": true,
  "result_contract_status": null,
  "result_contract_error": null,
  "command_rc": 0,
  "changes_made": false,
  "rollback_started": false,
  "rollback_restored": null,
  "mutation_outcome": "NO_MUTATION",
  "sanitized": true,
  "secrets_included": false,
  "private_addresses_included": false,
  "raw_evidence_retained_locally": true,
  "raw_evidence_sha256": "374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033",
  "sanitized_output_sha256": "374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033",
  "output_truncated_in_json": false,
  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
  "output": "WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1\nMODE=read-only\nCOMPONENT=homelab-backup-audit-vmqueue-rca\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nBACKUP_AUDIT_STATE_BEGIN=1\nResult=exit-code\nExecMainStartTimestamp=Tue 2026-09-22 07:36:58 MSK\nExecMainExitTimestamp=Tue 2026-09-22 07:37:01 MSK\nExecMainCode=1\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nStateChangeTimestamp=Tue 2026-09-22 07:37:01 MSK\nBACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service\nBACKUP_AUDIT_FRAGMENT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa\nAfter=network-online.target\nWants=network-online.target\nExecStart=/usr/local/sbin/homelab-backup-audit scheduled\nBACKUP_AUDIT_EXIT_EPOCH=1790051821\nBACKUP_AUDIT_STATE_END=1\nVM_QUEUE_STATE_BEGIN=1\nResult=exit-code\nExecMainStartTimestamp=Wed 2026-09-16 20:32:57 MSK\nExecMainExitTimestamp=Wed 2026-09-16 20:33:02 MSK\nExecMainCode=1\nExecMainStatus=65\nActiveState=failed\nSubState=failed\nStateChangeTimestamp=Wed 2026-09-16 20:33:02 MSK\nVM_QUEUE_FRAGMENT_PATH=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service\nVM_QUEUE_FRAGMENT_SHA256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759\nAfter=network-online.target\nWants=network-online.target\nExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all\nVM_QUEUE_EXIT_EPOCH=1789579982\nVM_QUEUE_STATE_END=1\nBACKUP_OWNER_STATE_BEGIN=1\nResult=success\nExecMainStartTimestamp=Tue 2026-09-22 07:42:15 MSK\nExecMainExitTimestamp=Tue 2026-09-22 07:42:15 MSK\nExecMainCode=1\nExecMainStatus=0\nActiveState=inactive\nSubState=dead\nStateChangeTimestamp=Tue 2026-09-22 07:42:15 MSK\nBACKUP_OWNER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service\nBACKUP_OWNER_FRAGMENT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565\nAfter=network-online.target\nWants=network-online.target\nExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\nBACKUP_OWNER_EXIT_EPOCH=1790052135\nBACKUP_OWNER_STATE_END=1\nPOLICY_FIX_1108_FINISH_EPOCH=1790030296\nVM_QUEUE_LAST_EXIT_EPOCH=1789579982\nVM_QUEUE_FAILURE_CLASS=STALE_PRE1108_FAILED_STATE\nBACKUP_AUDIT_JOURNAL_SHA256=77b32d66abed4873f79ed2ffb9bf0c4ec87762c4b78c85aa23136d3743f6b364\nBACKUP_AUDIT_LATEST_CLASSIFIED_BEGIN=1\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nBACKUP_AUDIT_LATEST_CLASSIFIED_END=1\nVM_QUEUE_JOURNAL_SHA256=3967f9d0fff08524f53eedf57007e2a2567de9ba59fad5b5f2948e5039cbfa6d\nVM_QUEUE_LATEST_CLASSIFIED_BEGIN=1\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260915T223521Z.json\nQUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260915T223521Z AGE_SECONDS=39387 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\nGLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH\nVM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\nVM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json\nQUEUE_FRESHNESS VMID=160 DECISION=SKIP_FRESH GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=38803 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=160 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json\nQUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=9957 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json\nQUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=7668 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\nQUEUE_FRESHNESS VMID=171 DECISION=SKIP_FRESH GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=39491 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=171 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json\nQUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=7022 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json\nQUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=6995 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json\nQUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=6937 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json\nQUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=6832 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\nGLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH\nVM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\nVM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json\nQUEUE_FRESHNESS VMID=160 DECISION=BACKUP_DUE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=53189 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_DUE\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json\nQUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=24343 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json\nQUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=22053 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\nQUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json\nQUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json\nQUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json\nQUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json\nQUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\nGLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\nQUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET\nVM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171\nVM_QUEUE_LATEST_CLASSIFIED_END=1\nSOURCE_CLOUD_B_RESTORE_DRILL_BAD__COUNT=1\nSOURCE_CLOUD_B_RESTORE_DRILL_BAD__PATH=/usr/local/sbin/homelab-backup-audit\nSOURCE_CLOUD_B_RESTORE_DRILL_BAD__SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\n230-if CLOUDB_HEALTH.is_file():\n231-    txt=CLOUDB_HEALTH.read_text(encoding=\"utf-8\",errors=\"replace\")\n232-    kv={}\n233-    for line in txt.splitlines():\n234-        if \"=\" in line:\n235-            k,v=line.split(\"=\",1); kv[k]=v\n236-    cloudb_ok=(kv.get(\"STATUS\")==\"OK\" and kv.get(\"CLOUD_2_STATUS\")==\"OK\" and kv.get(\"SHA256_OK\")==\"yes\" and kv.get(\"TAR_OK\")==\"yes\" and kv.get(\"REQUIRED_FILES_OK\")==\"yes\")\n237-if not cloudb_ok:\n238:    issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"LATEST_CLOUD_B_DRILL_NOT_OK\"})\n239-\n240-source={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\n241-hrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\n242-if hrc==0: source[\"local_head\"]=ho.strip()\n243-frc,fo,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.url\"],10)\n244-prc,po,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.pushurl\"],10)\n245-source[\"pushurl_disabled\"]=(prc==0 and po.strip()==\"DISABLED\")\n246-if frc==0 and fo.strip():\n247-    lrc,lo,_=run([\"git\",\"ls-remote\",fo.strip(),\"refs/heads/\"+BRANCH],30)\n248-    if lrc==0 and lo.strip(): source[\"remote_head\"]=lo.split()[0]\n249-if source[\"remote_head\"]!=source[\"local_head\"]:\n250-    issues.append({\"severity\":\"MEDIUM\",\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\"})\nSOURCE_GENERATION_NOT_RETURNED___COUNT=2\nSOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-app-cloud-quorum-queue\nSOURCE_GENERATION_NOT_RETURNED___SHA256=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\n116-    BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\n117-    *) die \"FRESHNESS_RECHECK_INVALID_${a}_${decision:-EMPTY}\" 65 ;;\n118-  esac\n119-  printf 'QUEUE_BEGIN APP=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$a\" \"$decision\"\n120-  out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"\n121-  printf '%s\\n' \"$out\"\n122-  \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\n123-  generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n124:  [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$a\" 65\n125-  printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$a\" \"$generation\"\n126-  printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\\n' \"$a\" \"$generation\"\n127-}\n128-\n129-migrate_list(){\n130-  local a out\n131-  lock_heavy\n132-  for a in \"${MIGRATE_APPS[@]}\"; do\n133-    if out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" latest-sealed \"$a\" 2>/dev/null)\"; then\n134-      printf 'QUEUE_REUSE_SEALED APP=%s\\n%s\\n' \"$a\" \"$out\"\n135-      continue\n136-    fi\nSOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-vm-cloud-quorum-queue\nSOURCE_GENERATION_NOT_RETURNED___SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\n131-  flock -n 9 || die \"GLOBAL_QUEUE_LOCK_BUSY\" 75\n132-  exec 8>\"$HEAVY_LOCK\"\n133-  flock -n 8 || die \"GLOBAL_HEAVY_LOCK_BUSY\" 75\n134-  for vmid in \"$@\"; do\n135-    printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\"\n136-    out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n137-    printf '%s\\n' \"$out\"\n138-    generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n139:    [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$vmid\" 65\n140-    printf 'QUEUE_SEALED VMID=%s GENERATION=%s\\n' \"$vmid\" \"$generation\"\n141-  done\n142-}\n143-dispatch_plan(){\n144-  freshness_plan\n145-}\n146-dispatch_one(){\n147-  local vmid=\"$1\" freshness decision now_epoch out generation\n148-  [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64\n149-  expected_owner \"$vmid\" >/dev/null\n150-  exec 9>\"$LOCK\"\n151-  if ! flock -n 9; then\n--\n169-      ;;\n170-    BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\n171-    *) die \"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\" 65 ;;\n172-  esac\n173-  printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$decision\"\n174-  out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n175-  printf '%s\\n' \"$out\"\n176-  generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n177:  [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$vmid\" 65\n178-  printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$generation\"\n179-  printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\\n' \"$vmid\" \"$generation\"\n180-}\n181-\n182-ACTION=\"${1:-}\"\n183-case \"$ACTION\" in\n184-  plan)\n185-    [[ \"$#\" == 1 ]] || die \"USAGE_PLAN\" 64\n186-    plan_all\n187-    ;;\n188-  freshness-plan)\n189-    [[ \"$#\" == 1 ]] || die \"USAGE_FRESHNESS_PLAN\" 64\nSCHEDULER_SELECT_RC=0\nSCHEDULER_SELECT_BEGIN=1\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nSCHEDULER_SELECT_END=1\nCURRENT_RPO_SCHEDULER_CLASS=NO_DUE_TARGET\nEVIDENCE_ROOT=/var/lib/homelab-backup\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T093429Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T213739Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260901T103255Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T003249Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T133648Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T023235Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T163234Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T064255Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T193019Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T093128Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T222933Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260906T113150Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T012910Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T143756Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T032904Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T173418Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T063400Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T193647Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260910T093211Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T003012Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T133013Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T013042Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T143251Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T033047Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T174140Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T034236Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T174843Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T064613Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T193324Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T093438Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T223254Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260906T113503Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T013234Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T144115Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T033223Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T173740Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T063725Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T194007Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260910T093532Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260911T003332Z.json\nEVIDENCE_ROOT=/var/lib/homelab-backup-v2\nMTIME_EPOCH|608|/var/lib/homelab-backup-v2/state/vm171.json\nDECISION=PASS_BACKUP_1110_BLOCKERS_CLASSIFIED\nNEXT_GATE=REPAIR_ONLY_CLASSIFIED_BACKUP_AUDIT_VMQUEUE_BLOCKERS\nTASK_RESULT=PASS_HOMELAB_BACKUP_AUDIT_VMQUEUE_RCA\nCHANGES_MADE_BY_1110=false\nPRODUCT_MUTATION_BY_1110=false\nVM_MUTATION_BY_1110=false\nCLOUD_MUTATION_BY_1110=false\nWORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_END=1\n"
}