diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1.json b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1.json new file mode 100644 index 00000000..f0b0154f --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-audit-vmqueue-rca", + "started_at_utc": "2026-09-22T04:44:54Z", + "finished_at_utc": "2026-09-22T04:44:54Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "ed70f0c53596e8aa062b17b3c599cc9bc6eda7472edb9ec786f9a88a23af2eb4", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033", + "sanitized_output_sha256": "374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1\nMODE=read-only\nCOMPONENT=homelab-backup-audit-vmqueue-rca\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nBACKUP_AUDIT_STATE_BEGIN=1\nResult=exit-code\nExecMainStartTimestamp=Tue 2026-09-22 07:36:58 MSK\nExecMainExitTimestamp=Tue 2026-09-22 07:37:01 MSK\nExecMainCode=1\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nStateChangeTimestamp=Tue 2026-09-22 07:37:01 MSK\nBACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service\nBACKUP_AUDIT_FRAGMENT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa\nAfter=network-online.target\nWants=network-online.target\nExecStart=/usr/local/sbin/homelab-backup-audit scheduled\nBACKUP_AUDIT_EXIT_EPOCH=1790051821\nBACKUP_AUDIT_STATE_END=1\nVM_QUEUE_STATE_BEGIN=1\nResult=exit-code\nExecMainStartTimestamp=Wed 2026-09-16 20:32:57 MSK\nExecMainExitTimestamp=Wed 2026-09-16 20:33:02 MSK\nExecMainCode=1\nExecMainStatus=65\nActiveState=failed\nSubState=failed\nStateChangeTimestamp=Wed 2026-09-16 20:33:02 MSK\nVM_QUEUE_FRAGMENT_PATH=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service\nVM_QUEUE_FRAGMENT_SHA256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759\nAfter=network-online.target\nWants=network-online.target\nExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all\nVM_QUEUE_EXIT_EPOCH=1789579982\nVM_QUEUE_STATE_END=1\nBACKUP_OWNER_STATE_BEGIN=1\nResult=success\nExecMainStartTimestamp=Tue 2026-09-22 07:42:15 MSK\nExecMainExitTimestamp=Tue 2026-09-22 07:42:15 MSK\nExecMainCode=1\nExecMainStatus=0\nActiveState=inactive\nSubState=dead\nStateChangeTimestamp=Tue 2026-09-22 07:42:15 MSK\nBACKUP_OWNER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service\nBACKUP_OWNER_FRAGMENT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565\nAfter=network-online.target\nWants=network-online.target\nExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\nBACKUP_OWNER_EXIT_EPOCH=1790052135\nBACKUP_OWNER_STATE_END=1\nPOLICY_FIX_1108_FINISH_EPOCH=1790030296\nVM_QUEUE_LAST_EXIT_EPOCH=1789579982\nVM_QUEUE_FAILURE_CLASS=STALE_PRE1108_FAILED_STATE\nBACKUP_AUDIT_JOURNAL_SHA256=77b32d66abed4873f79ed2ffb9bf0c4ec87762c4b78c85aa23136d3743f6b364\nBACKUP_AUDIT_LATEST_CLASSIFIED_BEGIN=1\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nBACKUP_AUDIT_LATEST_CLASSIFIED_END=1\nVM_QUEUE_JOURNAL_SHA256=3967f9d0fff08524f53eedf57007e2a2567de9ba59fad5b5f2948e5039cbfa6d\nVM_QUEUE_LATEST_CLASSIFIED_BEGIN=1\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260915T223521Z.json\nQUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260915T223521Z AGE_SECONDS=39387 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\nGLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH\nVM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\nVM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json\nQUEUE_FRESHNESS VMID=160 DECISION=SKIP_FRESH GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=38803 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=160 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json\nQUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=9957 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json\nQUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=7668 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\nQUEUE_FRESHNESS VMID=171 DECISION=SKIP_FRESH GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=39491 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=171 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json\nQUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=7022 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json\nQUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=6995 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json\nQUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=6937 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json\nQUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=6832 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\nGLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH\nVM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\nVM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json\nQUEUE_FRESHNESS VMID=160 DECISION=BACKUP_DUE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=53189 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_DUE\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json\nQUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=24343 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json\nQUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=22053 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\nQUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json\nQUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json\nQUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json\nQUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json\nQUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\nGLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\nQUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET\nVM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171\nVM_QUEUE_LATEST_CLASSIFIED_END=1\nSOURCE_CLOUD_B_RESTORE_DRILL_BAD__COUNT=1\nSOURCE_CLOUD_B_RESTORE_DRILL_BAD__PATH=/usr/local/sbin/homelab-backup-audit\nSOURCE_CLOUD_B_RESTORE_DRILL_BAD__SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\n230-if CLOUDB_HEALTH.is_file():\n231- txt=CLOUDB_HEALTH.read_text(encoding=\"utf-8\",errors=\"replace\")\n232- kv={}\n233- for line in txt.splitlines():\n234- if \"=\" in line:\n235- k,v=line.split(\"=\",1); kv[k]=v\n236- cloudb_ok=(kv.get(\"STATUS\")==\"OK\" and kv.get(\"CLOUD_2_STATUS\")==\"OK\" and kv.get(\"SHA256_OK\")==\"yes\" and kv.get(\"TAR_OK\")==\"yes\" and kv.get(\"REQUIRED_FILES_OK\")==\"yes\")\n237-if not cloudb_ok:\n238: issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"LATEST_CLOUD_B_DRILL_NOT_OK\"})\n239-\n240-source={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\n241-hrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\n242-if hrc==0: source[\"local_head\"]=ho.strip()\n243-frc,fo,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.url\"],10)\n244-prc,po,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.pushurl\"],10)\n245-source[\"pushurl_disabled\"]=(prc==0 and po.strip()==\"DISABLED\")\n246-if frc==0 and fo.strip():\n247- lrc,lo,_=run([\"git\",\"ls-remote\",fo.strip(),\"refs/heads/\"+BRANCH],30)\n248- if lrc==0 and lo.strip(): source[\"remote_head\"]=lo.split()[0]\n249-if source[\"remote_head\"]!=source[\"local_head\"]:\n250- issues.append({\"severity\":\"MEDIUM\",\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\"})\nSOURCE_GENERATION_NOT_RETURNED___COUNT=2\nSOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-app-cloud-quorum-queue\nSOURCE_GENERATION_NOT_RETURNED___SHA256=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\n116- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\n117- *) die \"FRESHNESS_RECHECK_INVALID_${a}_${decision:-EMPTY}\" 65 ;;\n118- esac\n119- printf 'QUEUE_BEGIN APP=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$a\" \"$decision\"\n120- out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"\n121- printf '%s\\n' \"$out\"\n122- \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\n123- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n124: [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$a\" 65\n125- printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$a\" \"$generation\"\n126- printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\\n' \"$a\" \"$generation\"\n127-}\n128-\n129-migrate_list(){\n130- local a out\n131- lock_heavy\n132- for a in \"${MIGRATE_APPS[@]}\"; do\n133- if out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" latest-sealed \"$a\" 2>/dev/null)\"; then\n134- printf 'QUEUE_REUSE_SEALED APP=%s\\n%s\\n' \"$a\" \"$out\"\n135- continue\n136- fi\nSOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-vm-cloud-quorum-queue\nSOURCE_GENERATION_NOT_RETURNED___SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\n131- flock -n 9 || die \"GLOBAL_QUEUE_LOCK_BUSY\" 75\n132- exec 8>\"$HEAVY_LOCK\"\n133- flock -n 8 || die \"GLOBAL_HEAVY_LOCK_BUSY\" 75\n134- for vmid in \"$@\"; do\n135- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\"\n136- out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n137- printf '%s\\n' \"$out\"\n138- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n139: [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$vmid\" 65\n140- printf 'QUEUE_SEALED VMID=%s GENERATION=%s\\n' \"$vmid\" \"$generation\"\n141- done\n142-}\n143-dispatch_plan(){\n144- freshness_plan\n145-}\n146-dispatch_one(){\n147- local vmid=\"$1\" freshness decision now_epoch out generation\n148- [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64\n149- expected_owner \"$vmid\" >/dev/null\n150- exec 9>\"$LOCK\"\n151- if ! flock -n 9; then\n--\n169- ;;\n170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\n171- *) die \"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\" 65 ;;\n172- esac\n173- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$decision\"\n174- out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n175- printf '%s\\n' \"$out\"\n176- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n177: [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$vmid\" 65\n178- printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$generation\"\n179- printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\\n' \"$vmid\" \"$generation\"\n180-}\n181-\n182-ACTION=\"${1:-}\"\n183-case \"$ACTION\" in\n184- plan)\n185- [[ \"$#\" == 1 ]] || die \"USAGE_PLAN\" 64\n186- plan_all\n187- ;;\n188- freshness-plan)\n189- [[ \"$#\" == 1 ]] || die \"USAGE_FRESHNESS_PLAN\" 64\nSCHEDULER_SELECT_RC=0\nSCHEDULER_SELECT_BEGIN=1\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nSCHEDULER_SELECT_END=1\nCURRENT_RPO_SCHEDULER_CLASS=NO_DUE_TARGET\nEVIDENCE_ROOT=/var/lib/homelab-backup\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T093429Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T213739Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260901T103255Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T003249Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T133648Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T023235Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T163234Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T064255Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T193019Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T093128Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T222933Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260906T113150Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T012910Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T143756Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T032904Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T173418Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T063400Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T193647Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260910T093211Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T003012Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T133013Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T013042Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T143251Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T033047Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T174140Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T034236Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T174843Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T064613Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T193324Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T093438Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T223254Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260906T113503Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T013234Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T144115Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T033223Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T173740Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T063725Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T194007Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260910T093532Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260911T003332Z.json\nEVIDENCE_ROOT=/var/lib/homelab-backup-v2\nMTIME_EPOCH|608|/var/lib/homelab-backup-v2/state/vm171.json\nDECISION=PASS_BACKUP_1110_BLOCKERS_CLASSIFIED\nNEXT_GATE=REPAIR_ONLY_CLASSIFIED_BACKUP_AUDIT_VMQUEUE_BLOCKERS\nTASK_RESULT=PASS_HOMELAB_BACKUP_AUDIT_VMQUEUE_RCA\nCHANGES_MADE_BY_1110=false\nPRODUCT_MUTATION_BY_1110=false\nVM_MUTATION_BY_1110=false\nCLOUD_MUTATION_BY_1110=false\nWORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1.txt b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1.txt new file mode 100644 index 00000000..2f665265 --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1.txt @@ -0,0 +1,359 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-audit-vmqueue-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=ed70f0c53596e8aa062b17b3c599cc9bc6eda7472edb9ec786f9a88a23af2eb4 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033 +SANITIZED_OUTPUT_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033 +OUTPUT_BEGIN +WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1 +MODE=read-only +COMPONENT=homelab-backup-audit-vmqueue-rca +ERROR_REGISTER_CHECK=OK +REFERENCE_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 +BACKUP_AUDIT_STATE_BEGIN=1 +Result=exit-code +ExecMainStartTimestamp=Tue 2026-09-22 07:36:58 MSK +ExecMainExitTimestamp=Tue 2026-09-22 07:37:01 MSK +ExecMainCode=1 +ExecMainStatus=1 +ActiveState=failed +SubState=failed +StateChangeTimestamp=Tue 2026-09-22 07:37:01 MSK +BACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service +BACKUP_AUDIT_FRAGMENT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa +After=network-online.target +Wants=network-online.target +ExecStart=/usr/local/sbin/homelab-backup-audit scheduled +BACKUP_AUDIT_EXIT_EPOCH=1790051821 +BACKUP_AUDIT_STATE_END=1 +VM_QUEUE_STATE_BEGIN=1 +Result=exit-code +ExecMainStartTimestamp=Wed 2026-09-16 20:32:57 MSK +ExecMainExitTimestamp=Wed 2026-09-16 20:33:02 MSK +ExecMainCode=1 +ExecMainStatus=65 +ActiveState=failed +SubState=failed +StateChangeTimestamp=Wed 2026-09-16 20:33:02 MSK +VM_QUEUE_FRAGMENT_PATH=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service +VM_QUEUE_FRAGMENT_SHA256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759 +After=network-online.target +Wants=network-online.target +ExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all +VM_QUEUE_EXIT_EPOCH=1789579982 +VM_QUEUE_STATE_END=1 +BACKUP_OWNER_STATE_BEGIN=1 +Result=success +ExecMainStartTimestamp=Tue 2026-09-22 07:42:15 MSK +ExecMainExitTimestamp=Tue 2026-09-22 07:42:15 MSK +ExecMainCode=1 +ExecMainStatus=0 +ActiveState=inactive +SubState=dead +StateChangeTimestamp=Tue 2026-09-22 07:42:15 MSK +BACKUP_OWNER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service +BACKUP_OWNER_FRAGMENT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565 +After=network-online.target +Wants=network-online.target +ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled +BACKUP_OWNER_EXIT_EPOCH=1790052135 +BACKUP_OWNER_STATE_END=1 +POLICY_FIX_1108_FINISH_EPOCH=1790030296 +VM_QUEUE_LAST_EXIT_EPOCH=1789579982 +VM_QUEUE_FAILURE_CLASS=STALE_PRE1108_FAILED_STATE +BACKUP_AUDIT_JOURNAL_SHA256=77b32d66abed4873f79ed2ffb9bf0c4ec87762c4b78c85aa23136d3743f6b364 +BACKUP_AUDIT_LATEST_CLASSIFIED_BEGIN=1 +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +BACKUP_AUDIT_LATEST_CLASSIFIED_END=1 +VM_QUEUE_JOURNAL_SHA256=3967f9d0fff08524f53eedf57007e2a2567de9ba59fad5b5f2948e5039cbfa6d +VM_QUEUE_LATEST_CLASSIFIED_BEGIN=1 +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260915T223521Z.json +QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260915T223521Z AGE_SECONDS=39387 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH +GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH +VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02 +VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json +QUEUE_FRESHNESS VMID=160 DECISION=SKIP_FRESH GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=38803 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=160 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json +QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=9957 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json +QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=7668 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json +QUEUE_FRESHNESS VMID=171 DECISION=SKIP_FRESH GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=39491 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=171 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json +QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=7022 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json +QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=6995 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json +QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=6937 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json +QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=6832 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH +GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH +VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02 +VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json +QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_DUE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=53189 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_DUE +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json +QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=24343 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json +QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=22053 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json +QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json +QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json +QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json +QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json +QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH +GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876 +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json +QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET +VM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171 +VM_QUEUE_LATEST_CLASSIFIED_END=1 +SOURCE_CLOUD_B_RESTORE_DRILL_BAD__COUNT=1 +SOURCE_CLOUD_B_RESTORE_DRILL_BAD__PATH=/usr/local/sbin/homelab-backup-audit +SOURCE_CLOUD_B_RESTORE_DRILL_BAD__SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 +230-if CLOUDB_HEALTH.is_file(): +231- txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace") +232- kv={} +233- for line in txt.splitlines(): +234- if "=" in line: +235- k,v=line.split("=",1); kv[k]=v +236- cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes") +237-if not cloudb_ok: +238: issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"}) +239- +240-source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +241-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +242-if hrc==0: source["local_head"]=ho.strip() +243-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) +244-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) +245-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") +246-if frc==0 and fo.strip(): +247- lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) +248- if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] +249-if source["remote_head"]!=source["local_head"]: +250- issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) +SOURCE_GENERATION_NOT_RETURNED___COUNT=2 +SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-app-cloud-quorum-queue +SOURCE_GENERATION_NOT_RETURNED___SHA256=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650 +116- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;; +117- *) die "FRESHNESS_RECHECK_INVALID_${a}_${decision:-EMPTY}" 65 ;; +118- esac +119- printf 'QUEUE_BEGIN APP=%s MODE=%s DISPATCH=ONE_TARGET\n' "$a" "$decision" +120- out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")" +121- printf '%s\n' "$out" +122- "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null +123- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +124: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$a" 65 +125- printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$a" "$generation" +126- printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\n' "$a" "$generation" +127-} +128- +129-migrate_list(){ +130- local a out +131- lock_heavy +132- for a in "${MIGRATE_APPS[@]}"; do +133- if out="$(exec 8>&-; exec 9>&-; "$WORKER" latest-sealed "$a" 2>/dev/null)"; then +134- printf 'QUEUE_REUSE_SEALED APP=%s\n%s\n' "$a" "$out" +135- continue +136- fi +SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-vm-cloud-quorum-queue +SOURCE_GENERATION_NOT_RETURNED___SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 +131- flock -n 9 || die "GLOBAL_QUEUE_LOCK_BUSY" 75 +132- exec 8>"$HEAVY_LOCK" +133- flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75 +134- for vmid in "$@"; do +135- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")" +136- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)" +137- printf '%s\n' "$out" +138- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +139: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65 +140- printf 'QUEUE_SEALED VMID=%s GENERATION=%s\n' "$vmid" "$generation" +141- done +142-} +143-dispatch_plan(){ +144- freshness_plan +145-} +146-dispatch_one(){ +147- local vmid="$1" freshness decision now_epoch out generation +148- [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64 +149- expected_owner "$vmid" >/dev/null +150- exec 9>"$LOCK" +151- if ! flock -n 9; then +-- +169- ;; +170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;; +171- *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;; +172- esac +173- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision" +174- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)" +175- printf '%s\n' "$out" +176- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +177: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65 +178- printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$vmid" "$generation" +179- printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\n' "$vmid" "$generation" +180-} +181- +182-ACTION="${1:-}" +183-case "$ACTION" in +184- plan) +185- [[ "$#" == 1 ]] || die "USAGE_PLAN" 64 +186- plan_all +187- ;; +188- freshness-plan) +189- [[ "$#" == 1 ]] || die "USAGE_FRESHNESS_PLAN" 64 +SCHEDULER_SELECT_RC=0 +SCHEDULER_SELECT_BEGIN=1 +SELFTEST_REPLACEMENT486=PASS +SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET +SCHEDULER_SELECT_END=1 +CURRENT_RPO_SCHEDULER_CLASS=NO_DUE_TARGET +EVIDENCE_ROOT=/var/lib/homelab-backup +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T093429Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T213739Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260901T103255Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T003249Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T133648Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T023235Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T163234Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T064255Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T193019Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T093128Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T222933Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260906T113150Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T012910Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T143756Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T032904Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T173418Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T063400Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T193647Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260910T093211Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T003012Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T133013Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T013042Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T143251Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T033047Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T174140Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T034236Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T174843Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T064613Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T193324Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T093438Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T223254Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260906T113503Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T013234Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T144115Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T033223Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T173740Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T063725Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T194007Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260910T093532Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260911T003332Z.json +EVIDENCE_ROOT=/var/lib/homelab-backup-v2 +MTIME_EPOCH|608|/var/lib/homelab-backup-v2/state/vm171.json +DECISION=PASS_BACKUP_1110_BLOCKERS_CLASSIFIED +NEXT_GATE=REPAIR_ONLY_CLASSIFIED_BACKUP_AUDIT_VMQUEUE_BLOCKERS +TASK_RESULT=PASS_HOMELAB_BACKUP_AUDIT_VMQUEUE_RCA +CHANGES_MADE_BY_1110=false +PRODUCT_MUTATION_BY_1110=false +VM_MUTATION_BY_1110=false +CLOUD_MUTATION_BY_1110=false +WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 90402d24..f0b0154f 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1", - "status": "FAIL", - "rc": 3, + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1", + "status": "OK", + "rc": 0, "host": "pve01", "mode": "read-only", - "component": "homelab-backup-live-preclosure-rca", - "started_at_utc": "2026-09-22T04:37:17Z", - "finished_at_utc": "2026-09-22T04:37:17Z", + "component": "homelab-backup-audit-vmqueue-rca", + "started_at_utc": "2026-09-22T04:44:54Z", + "finished_at_utc": "2026-09-22T04:44:54Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "93eb74c1e05a7be6ae8dc468f268a7865ec7e60481c6a0e78a227a0de5eb7a13", + "command_sha256": "ed70f0c53596e8aa062b17b3c599cc9bc6eda7472edb9ec786f9a88a23af2eb4", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -21,7 +21,7 @@ "result_contract_valid": true, "result_contract_status": null, "result_contract_error": null, - "command_rc": 3, + "command_rc": 0, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "cf7b386835799489aa2e31f36e3055ab433d761a4858a07309a508d301bb5f43", - "sanitized_output_sha256": "cf7b386835799489aa2e31f36e3055ab433d761a4858a07309a508d301bb5f43", + "raw_evidence_sha256": "374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033", + "sanitized_output_sha256": "374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_BACKUP_LIVE_PRECLOSURE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1\nMODE=read-only\nCOMPONENT=homelab-backup-live-preclosure-rca\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nBACKUP_AUDIT_UNIT_STATE=ActiveState:failed,SubState:failed,Result:exit-code,ExecMainCode:1,ExecMainStatus:1\nVM_QUEUE_UNIT_STATE=ActiveState:failed,SubState:failed,Result:exit-code,ExecMainCode:1,ExecMainStatus:65\nBACKUP_OWNER_UNIT_STATE=ActiveState:inactive,SubState:dead,Result:success,ExecMainCode:1,ExecMainStatus:0\nTIMER_STATE=homelab-backup-v2.timer,enabled:enabled,active:active\nTIMER_STATE=homelab-backup-audit.timer,enabled:enabled,active:active\nTIMER_STATE=homelab-vm-cloud-quorum-queue.timer,enabled:disabled,active:inactive\nFAILED_UNITS=homelab-backup-audit.service,homelab-vm-cloud-quorum-queue.service,netbird-peers-health.service\nBACKUP_AUDIT_JOURNAL_SHA256=547c6183c7dc5f55ef85f218d53806ae723ebbd553e886f5c66959a4e11cd200\nBACKUP_AUDIT_ERROR_EXCERPT_BEGIN=1\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nhomelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-audit.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nhomelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-audit.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nhomelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-audit.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nhomelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-audit.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nhomelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-audit.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nBACKUP_AUDIT_ERROR_EXCERPT_END=1\nVM_QUEUE_JOURNAL_SHA256=a518b2e9c7ec704afa08b82d78b82e57b506de5aed4a8fad002e7d2191fedfd1\nVM_QUEUE_ERROR_EXCERPT_BEGIN=1\nQUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE\nCLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876\nFRESHNESS_STATUS=OK\nSTATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json\nQUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nFRESHNESS_STATUS=OK\nSTATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json\nQUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nFRESHNESS_STATUS=OK\nSTATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260916T113650Z.json\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json\nQUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nFRESHNESS_STATUS=OK\nSTATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json\nQUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nGLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876\nFRESHNESS_STATUS=OK\nSTATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\nQUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET\nVM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171\nhomelab-vm-cloud-quorum-queue.service: Main process exited, code=exited, status=65/DATAERR\nhomelab-vm-cloud-quorum-queue.service: Failed with result 'exit-code'.\nFailed to start homelab-vm-cloud-quorum-queue.service - Homelab canonical VM/CT cloud quorum backup queue.\nhomelab-vm-cloud-quorum-queue.service: Consumed 4.492s CPU time, 147.2M memory peak.\nVM_QUEUE_ERROR_EXCERPT_END=1\nSCHEDULER_SELECT_RC=0\nSCHEDULER_SELECT_BEGIN=1\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nSCHEDULER_SELECT_END=1\nDECISION=HOLD_BACKUP_FINAL_CLOSURE_LIVE_RCA_REQUIRED\nNEXT_GATE=RECONCILE_ONLY_BACKUP_AUDIT_VMQUEUE_RPO_FAILURES\nTASK_RESULT=HOLD_HOMELAB_BACKUP_LIVE_PRECLOSURE_RCA\nCHANGES_MADE_BY_1109=false\nPRODUCT_MUTATION_BY_1109=false\nVM_MUTATION_BY_1109=false\nCLOUD_MUTATION_BY_1109=false\nWORKERS2_BACKUP_LIVE_PRECLOSURE_RCA_END=1\n" + "output": "WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1\nMODE=read-only\nCOMPONENT=homelab-backup-audit-vmqueue-rca\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nBACKUP_AUDIT_STATE_BEGIN=1\nResult=exit-code\nExecMainStartTimestamp=Tue 2026-09-22 07:36:58 MSK\nExecMainExitTimestamp=Tue 2026-09-22 07:37:01 MSK\nExecMainCode=1\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nStateChangeTimestamp=Tue 2026-09-22 07:37:01 MSK\nBACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service\nBACKUP_AUDIT_FRAGMENT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa\nAfter=network-online.target\nWants=network-online.target\nExecStart=/usr/local/sbin/homelab-backup-audit scheduled\nBACKUP_AUDIT_EXIT_EPOCH=1790051821\nBACKUP_AUDIT_STATE_END=1\nVM_QUEUE_STATE_BEGIN=1\nResult=exit-code\nExecMainStartTimestamp=Wed 2026-09-16 20:32:57 MSK\nExecMainExitTimestamp=Wed 2026-09-16 20:33:02 MSK\nExecMainCode=1\nExecMainStatus=65\nActiveState=failed\nSubState=failed\nStateChangeTimestamp=Wed 2026-09-16 20:33:02 MSK\nVM_QUEUE_FRAGMENT_PATH=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service\nVM_QUEUE_FRAGMENT_SHA256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759\nAfter=network-online.target\nWants=network-online.target\nExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all\nVM_QUEUE_EXIT_EPOCH=1789579982\nVM_QUEUE_STATE_END=1\nBACKUP_OWNER_STATE_BEGIN=1\nResult=success\nExecMainStartTimestamp=Tue 2026-09-22 07:42:15 MSK\nExecMainExitTimestamp=Tue 2026-09-22 07:42:15 MSK\nExecMainCode=1\nExecMainStatus=0\nActiveState=inactive\nSubState=dead\nStateChangeTimestamp=Tue 2026-09-22 07:42:15 MSK\nBACKUP_OWNER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service\nBACKUP_OWNER_FRAGMENT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565\nAfter=network-online.target\nWants=network-online.target\nExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\nBACKUP_OWNER_EXIT_EPOCH=1790052135\nBACKUP_OWNER_STATE_END=1\nPOLICY_FIX_1108_FINISH_EPOCH=1790030296\nVM_QUEUE_LAST_EXIT_EPOCH=1789579982\nVM_QUEUE_FAILURE_CLASS=STALE_PRE1108_FAILED_STATE\nBACKUP_AUDIT_JOURNAL_SHA256=77b32d66abed4873f79ed2ffb9bf0c4ec87762c4b78c85aa23136d3743f6b364\nBACKUP_AUDIT_LATEST_CLASSIFIED_BEGIN=1\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK\nISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING\nBACKUP_AUDIT_LATEST_CLASSIFIED_END=1\nVM_QUEUE_JOURNAL_SHA256=3967f9d0fff08524f53eedf57007e2a2567de9ba59fad5b5f2948e5039cbfa6d\nVM_QUEUE_LATEST_CLASSIFIED_BEGIN=1\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260915T223521Z.json\nQUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260915T223521Z AGE_SECONDS=39387 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\nGLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH\nVM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\nVM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json\nQUEUE_FRESHNESS VMID=160 DECISION=SKIP_FRESH GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=38803 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=160 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json\nQUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=9957 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json\nQUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=7668 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\nQUEUE_FRESHNESS VMID=171 DECISION=SKIP_FRESH GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=39491 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=171 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json\nQUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=7022 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json\nQUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=6995 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json\nQUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=6937 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json\nQUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=6832 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\nGLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH\nVM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\nVM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json\nQUEUE_FRESHNESS VMID=160 DECISION=BACKUP_DUE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=53189 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_DUE\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json\nQUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=24343 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json\nQUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=22053 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\nQUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json\nQUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json\nQUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json\nQUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json\nQUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\nGLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876\nFRESHNESS_STATUS=OK\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\nQUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\nQUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET\nVM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171\nVM_QUEUE_LATEST_CLASSIFIED_END=1\nSOURCE_CLOUD_B_RESTORE_DRILL_BAD__COUNT=1\nSOURCE_CLOUD_B_RESTORE_DRILL_BAD__PATH=/usr/local/sbin/homelab-backup-audit\nSOURCE_CLOUD_B_RESTORE_DRILL_BAD__SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\n230-if CLOUDB_HEALTH.is_file():\n231- txt=CLOUDB_HEALTH.read_text(encoding=\"utf-8\",errors=\"replace\")\n232- kv={}\n233- for line in txt.splitlines():\n234- if \"=\" in line:\n235- k,v=line.split(\"=\",1); kv[k]=v\n236- cloudb_ok=(kv.get(\"STATUS\")==\"OK\" and kv.get(\"CLOUD_2_STATUS\")==\"OK\" and kv.get(\"SHA256_OK\")==\"yes\" and kv.get(\"TAR_OK\")==\"yes\" and kv.get(\"REQUIRED_FILES_OK\")==\"yes\")\n237-if not cloudb_ok:\n238: issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"LATEST_CLOUD_B_DRILL_NOT_OK\"})\n239-\n240-source={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\n241-hrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\n242-if hrc==0: source[\"local_head\"]=ho.strip()\n243-frc,fo,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.url\"],10)\n244-prc,po,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.pushurl\"],10)\n245-source[\"pushurl_disabled\"]=(prc==0 and po.strip()==\"DISABLED\")\n246-if frc==0 and fo.strip():\n247- lrc,lo,_=run([\"git\",\"ls-remote\",fo.strip(),\"refs/heads/\"+BRANCH],30)\n248- if lrc==0 and lo.strip(): source[\"remote_head\"]=lo.split()[0]\n249-if source[\"remote_head\"]!=source[\"local_head\"]:\n250- issues.append({\"severity\":\"MEDIUM\",\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\"})\nSOURCE_GENERATION_NOT_RETURNED___COUNT=2\nSOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-app-cloud-quorum-queue\nSOURCE_GENERATION_NOT_RETURNED___SHA256=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\n116- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\n117- *) die \"FRESHNESS_RECHECK_INVALID_${a}_${decision:-EMPTY}\" 65 ;;\n118- esac\n119- printf 'QUEUE_BEGIN APP=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$a\" \"$decision\"\n120- out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"\n121- printf '%s\\n' \"$out\"\n122- \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\n123- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n124: [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$a\" 65\n125- printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$a\" \"$generation\"\n126- printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\\n' \"$a\" \"$generation\"\n127-}\n128-\n129-migrate_list(){\n130- local a out\n131- lock_heavy\n132- for a in \"${MIGRATE_APPS[@]}\"; do\n133- if out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" latest-sealed \"$a\" 2>/dev/null)\"; then\n134- printf 'QUEUE_REUSE_SEALED APP=%s\\n%s\\n' \"$a\" \"$out\"\n135- continue\n136- fi\nSOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-vm-cloud-quorum-queue\nSOURCE_GENERATION_NOT_RETURNED___SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\n131- flock -n 9 || die \"GLOBAL_QUEUE_LOCK_BUSY\" 75\n132- exec 8>\"$HEAVY_LOCK\"\n133- flock -n 8 || die \"GLOBAL_HEAVY_LOCK_BUSY\" 75\n134- for vmid in \"$@\"; do\n135- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\"\n136- out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n137- printf '%s\\n' \"$out\"\n138- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n139: [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$vmid\" 65\n140- printf 'QUEUE_SEALED VMID=%s GENERATION=%s\\n' \"$vmid\" \"$generation\"\n141- done\n142-}\n143-dispatch_plan(){\n144- freshness_plan\n145-}\n146-dispatch_one(){\n147- local vmid=\"$1\" freshness decision now_epoch out generation\n148- [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64\n149- expected_owner \"$vmid\" >/dev/null\n150- exec 9>\"$LOCK\"\n151- if ! flock -n 9; then\n--\n169- ;;\n170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\n171- *) die \"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\" 65 ;;\n172- esac\n173- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$decision\"\n174- out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n175- printf '%s\\n' \"$out\"\n176- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n177: [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$vmid\" 65\n178- printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$generation\"\n179- printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\\n' \"$vmid\" \"$generation\"\n180-}\n181-\n182-ACTION=\"${1:-}\"\n183-case \"$ACTION\" in\n184- plan)\n185- [[ \"$#\" == 1 ]] || die \"USAGE_PLAN\" 64\n186- plan_all\n187- ;;\n188- freshness-plan)\n189- [[ \"$#\" == 1 ]] || die \"USAGE_FRESHNESS_PLAN\" 64\nSCHEDULER_SELECT_RC=0\nSCHEDULER_SELECT_BEGIN=1\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nSCHEDULER_SELECT_END=1\nCURRENT_RPO_SCHEDULER_CLASS=NO_DUE_TARGET\nEVIDENCE_ROOT=/var/lib/homelab-backup\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T093429Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T213739Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260901T103255Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T003249Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T133648Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T023235Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T163234Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T064255Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T193019Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T093128Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T222933Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260906T113150Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T012910Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T143756Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T032904Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T173418Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T063400Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T193647Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260910T093211Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T003012Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T133013Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T013042Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T143251Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T033047Z.json\nMTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T174140Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T034236Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T174843Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T064613Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T193324Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T093438Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T223254Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260906T113503Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T013234Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T144115Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T033223Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T173740Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T063725Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T194007Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260910T093532Z.json\nMTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260911T003332Z.json\nEVIDENCE_ROOT=/var/lib/homelab-backup-v2\nMTIME_EPOCH|608|/var/lib/homelab-backup-v2/state/vm171.json\nDECISION=PASS_BACKUP_1110_BLOCKERS_CLASSIFIED\nNEXT_GATE=REPAIR_ONLY_CLASSIFIED_BACKUP_AUDIT_VMQUEUE_BLOCKERS\nTASK_RESULT=PASS_HOMELAB_BACKUP_AUDIT_VMQUEUE_RCA\nCHANGES_MADE_BY_1110=false\nPRODUCT_MUTATION_BY_1110=false\nVM_MUTATION_BY_1110=false\nCLOUD_MUTATION_BY_1110=false\nWORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 0bc17537..2f665265 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1 -STATUS=FAIL -RC=3 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1 +STATUS=OK +RC=0 HOST=pve01 MODE=read-only -COMPONENT=homelab-backup-live-preclosure-rca +COMPONENT=homelab-backup-audit-vmqueue-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=93eb74c1e05a7be6ae8dc468f268a7865ec7e60481c6a0e78a227a0de5eb7a13 +COMMAND_SHA256=ed70f0c53596e8aa062b17b3c599cc9bc6eda7472edb9ec786f9a88a23af2eb4 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=3 +COMMAND_RC=0 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,100 +24,336 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=cf7b386835799489aa2e31f36e3055ab433d761a4858a07309a508d301bb5f43 -SANITIZED_OUTPUT_SHA256=cf7b386835799489aa2e31f36e3055ab433d761a4858a07309a508d301bb5f43 +RAW_EVIDENCE_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033 +SANITIZED_OUTPUT_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033 OUTPUT_BEGIN -WORKERS2_BACKUP_LIVE_PRECLOSURE_RCA_BEGIN=1 -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-LIVE-PRECLOSURE-RCA-1109R1 +WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1 MODE=read-only -COMPONENT=homelab-backup-live-preclosure-rca +COMPONENT=homelab-backup-audit-vmqueue-rca ERROR_REGISTER_CHECK=OK REFERENCE_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 -BACKUP_AUDIT_UNIT_STATE=ActiveState:failed,SubState:failed,Result:exit-code,ExecMainCode:1,ExecMainStatus:1 -VM_QUEUE_UNIT_STATE=ActiveState:failed,SubState:failed,Result:exit-code,ExecMainCode:1,ExecMainStatus:65 -BACKUP_OWNER_UNIT_STATE=ActiveState:inactive,SubState:dead,Result:success,ExecMainCode:1,ExecMainStatus:0 -TIMER_STATE=homelab-backup-v2.timer,enabled:enabled,active:active -TIMER_STATE=homelab-backup-audit.timer,enabled:enabled,active:active -TIMER_STATE=homelab-vm-cloud-quorum-queue.timer,enabled:disabled,active:inactive -FAILED_UNITS=homelab-backup-audit.service,homelab-vm-cloud-quorum-queue.service,netbird-peers-health.service -BACKUP_AUDIT_JOURNAL_SHA256=547c6183c7dc5f55ef85f218d53806ae723ebbd553e886f5c66959a4e11cd200 -BACKUP_AUDIT_ERROR_EXCERPT_BEGIN=1 +BACKUP_AUDIT_STATE_BEGIN=1 +Result=exit-code +ExecMainStartTimestamp=Tue 2026-09-22 07:36:58 MSK +ExecMainExitTimestamp=Tue 2026-09-22 07:37:01 MSK +ExecMainCode=1 +ExecMainStatus=1 +ActiveState=failed +SubState=failed +StateChangeTimestamp=Tue 2026-09-22 07:37:01 MSK +BACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service +BACKUP_AUDIT_FRAGMENT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa +After=network-online.target +Wants=network-online.target +ExecStart=/usr/local/sbin/homelab-backup-audit scheduled +BACKUP_AUDIT_EXIT_EPOCH=1790051821 +BACKUP_AUDIT_STATE_END=1 +VM_QUEUE_STATE_BEGIN=1 +Result=exit-code +ExecMainStartTimestamp=Wed 2026-09-16 20:32:57 MSK +ExecMainExitTimestamp=Wed 2026-09-16 20:33:02 MSK +ExecMainCode=1 +ExecMainStatus=65 +ActiveState=failed +SubState=failed +StateChangeTimestamp=Wed 2026-09-16 20:33:02 MSK +VM_QUEUE_FRAGMENT_PATH=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service +VM_QUEUE_FRAGMENT_SHA256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759 +After=network-online.target +Wants=network-online.target +ExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all +VM_QUEUE_EXIT_EPOCH=1789579982 +VM_QUEUE_STATE_END=1 +BACKUP_OWNER_STATE_BEGIN=1 +Result=success +ExecMainStartTimestamp=Tue 2026-09-22 07:42:15 MSK +ExecMainExitTimestamp=Tue 2026-09-22 07:42:15 MSK +ExecMainCode=1 +ExecMainStatus=0 +ActiveState=inactive +SubState=dead +StateChangeTimestamp=Tue 2026-09-22 07:42:15 MSK +BACKUP_OWNER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service +BACKUP_OWNER_FRAGMENT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565 +After=network-online.target +Wants=network-online.target +ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled +BACKUP_OWNER_EXIT_EPOCH=1790052135 +BACKUP_OWNER_STATE_END=1 +POLICY_FIX_1108_FINISH_EPOCH=1790030296 +VM_QUEUE_LAST_EXIT_EPOCH=1789579982 +VM_QUEUE_FAILURE_CLASS=STALE_PRE1108_FAILED_STATE +BACKUP_AUDIT_JOURNAL_SHA256=77b32d66abed4873f79ed2ffb9bf0c4ec87762c4b78c85aa23136d3743f6b364 +BACKUP_AUDIT_LATEST_CLASSIFIED_BEGIN=1 ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING -homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -homelab-backup-audit.service: Failed with result 'exit-code'. -Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK -SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING -homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -homelab-backup-audit.service: Failed with result 'exit-code'. -Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK -SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING -homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -homelab-backup-audit.service: Failed with result 'exit-code'. -Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING -homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -homelab-backup-audit.service: Failed with result 'exit-code'. -Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING -homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -homelab-backup-audit.service: Failed with result 'exit-code'. -Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -BACKUP_AUDIT_ERROR_EXCERPT_END=1 -VM_QUEUE_JOURNAL_SHA256=a518b2e9c7ec704afa08b82d78b82e57b506de5aed4a8fad002e7d2191fedfd1 -VM_QUEUE_ERROR_EXCERPT_BEGIN=1 +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK +ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING +BACKUP_AUDIT_LATEST_CLASSIFIED_END=1 +VM_QUEUE_JOURNAL_SHA256=3967f9d0fff08524f53eedf57007e2a2567de9ba59fad5b5f2948e5039cbfa6d +VM_QUEUE_LATEST_CLASSIFIED_BEGIN=1 +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260915T223521Z.json +QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260915T223521Z AGE_SECONDS=39387 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH +GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH +VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02 +VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json +QUEUE_FRESHNESS VMID=160 DECISION=SKIP_FRESH GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=38803 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=160 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json +QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=9957 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json +QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=7668 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json +QUEUE_FRESHNESS VMID=171 DECISION=SKIP_FRESH GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=39491 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=171 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json +QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=7022 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json +QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=6995 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json +QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=6937 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json +QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=6832 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH +GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH +VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02 +VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json +QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_DUE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=53189 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_DUE +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json +QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=24343 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json +QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=22053 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH +FRESHNESS_STATUS=OK +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE -CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876 FRESHNESS_STATUS=OK -STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH FRESHNESS_STATUS=OK -STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH FRESHNESS_STATUS=OK -STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260916T113650Z.json RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH FRESHNESS_STATUS=OK -STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 +QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876 FRESHNESS_STATUS=OK -STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000 QUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET VM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171 -homelab-vm-cloud-quorum-queue.service: Main process exited, code=exited, status=65/DATAERR -homelab-vm-cloud-quorum-queue.service: Failed with result 'exit-code'. -Failed to start homelab-vm-cloud-quorum-queue.service - Homelab canonical VM/CT cloud quorum backup queue. -homelab-vm-cloud-quorum-queue.service: Consumed 4.492s CPU time, 147.2M memory peak. -VM_QUEUE_ERROR_EXCERPT_END=1 +VM_QUEUE_LATEST_CLASSIFIED_END=1 +SOURCE_CLOUD_B_RESTORE_DRILL_BAD__COUNT=1 +SOURCE_CLOUD_B_RESTORE_DRILL_BAD__PATH=/usr/local/sbin/homelab-backup-audit +SOURCE_CLOUD_B_RESTORE_DRILL_BAD__SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 +230-if CLOUDB_HEALTH.is_file(): +231- txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace") +232- kv={} +233- for line in txt.splitlines(): +234- if "=" in line: +235- k,v=line.split("=",1); kv[k]=v +236- cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes") +237-if not cloudb_ok: +238: issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"}) +239- +240-source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +241-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +242-if hrc==0: source["local_head"]=ho.strip() +243-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) +244-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) +245-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") +246-if frc==0 and fo.strip(): +247- lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) +248- if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] +249-if source["remote_head"]!=source["local_head"]: +250- issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) +SOURCE_GENERATION_NOT_RETURNED___COUNT=2 +SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-app-cloud-quorum-queue +SOURCE_GENERATION_NOT_RETURNED___SHA256=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650 +116- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;; +117- *) die "FRESHNESS_RECHECK_INVALID_${a}_${decision:-EMPTY}" 65 ;; +118- esac +119- printf 'QUEUE_BEGIN APP=%s MODE=%s DISPATCH=ONE_TARGET\n' "$a" "$decision" +120- out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")" +121- printf '%s\n' "$out" +122- "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null +123- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +124: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$a" 65 +125- printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$a" "$generation" +126- printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\n' "$a" "$generation" +127-} +128- +129-migrate_list(){ +130- local a out +131- lock_heavy +132- for a in "${MIGRATE_APPS[@]}"; do +133- if out="$(exec 8>&-; exec 9>&-; "$WORKER" latest-sealed "$a" 2>/dev/null)"; then +134- printf 'QUEUE_REUSE_SEALED APP=%s\n%s\n' "$a" "$out" +135- continue +136- fi +SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-vm-cloud-quorum-queue +SOURCE_GENERATION_NOT_RETURNED___SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 +131- flock -n 9 || die "GLOBAL_QUEUE_LOCK_BUSY" 75 +132- exec 8>"$HEAVY_LOCK" +133- flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75 +134- for vmid in "$@"; do +135- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")" +136- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)" +137- printf '%s\n' "$out" +138- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +139: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65 +140- printf 'QUEUE_SEALED VMID=%s GENERATION=%s\n' "$vmid" "$generation" +141- done +142-} +143-dispatch_plan(){ +144- freshness_plan +145-} +146-dispatch_one(){ +147- local vmid="$1" freshness decision now_epoch out generation +148- [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64 +149- expected_owner "$vmid" >/dev/null +150- exec 9>"$LOCK" +151- if ! flock -n 9; then +-- +169- ;; +170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;; +171- *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;; +172- esac +173- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision" +174- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)" +175- printf '%s\n' "$out" +176- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +177: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65 +178- printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$vmid" "$generation" +179- printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\n' "$vmid" "$generation" +180-} +181- +182-ACTION="${1:-}" +183-case "$ACTION" in +184- plan) +185- [[ "$#" == 1 ]] || die "USAGE_PLAN" 64 +186- plan_all +187- ;; +188- freshness-plan) +189- [[ "$#" == 1 ]] || die "USAGE_FRESHNESS_PLAN" 64 SCHEDULER_SELECT_RC=0 SCHEDULER_SELECT_BEGIN=1 SELFTEST_REPLACEMENT486=PASS SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET SCHEDULER_SELECT_END=1 -DECISION=HOLD_BACKUP_FINAL_CLOSURE_LIVE_RCA_REQUIRED -NEXT_GATE=RECONCILE_ONLY_BACKUP_AUDIT_VMQUEUE_RPO_FAILURES -TASK_RESULT=HOLD_HOMELAB_BACKUP_LIVE_PRECLOSURE_RCA -CHANGES_MADE_BY_1109=false -PRODUCT_MUTATION_BY_1109=false -VM_MUTATION_BY_1109=false -CLOUD_MUTATION_BY_1109=false -WORKERS2_BACKUP_LIVE_PRECLOSURE_RCA_END=1 +CURRENT_RPO_SCHEDULER_CLASS=NO_DUE_TARGET +EVIDENCE_ROOT=/var/lib/homelab-backup +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T093429Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T213739Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260901T103255Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T003249Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T133648Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T023235Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T163234Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T064255Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T193019Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T093128Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T222933Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260906T113150Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T012910Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T143756Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T032904Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T173418Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T063400Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T193647Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260910T093211Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T003012Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T133013Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T013042Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T143251Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T033047Z.json +MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T174140Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T034236Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T174843Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T064613Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T193324Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T093438Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T223254Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260906T113503Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T013234Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T144115Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T033223Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T173740Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T063725Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T194007Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260910T093532Z.json +MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260911T003332Z.json +EVIDENCE_ROOT=/var/lib/homelab-backup-v2 +MTIME_EPOCH|608|/var/lib/homelab-backup-v2/state/vm171.json +DECISION=PASS_BACKUP_1110_BLOCKERS_CLASSIFIED +NEXT_GATE=REPAIR_ONLY_CLASSIFIED_BACKUP_AUDIT_VMQUEUE_BLOCKERS +TASK_RESULT=PASS_HOMELAB_BACKUP_AUDIT_VMQUEUE_RCA +CHANGES_MADE_BY_1110=false +PRODUCT_MUTATION_BY_1110=false +VM_MUTATION_BY_1110=false +CLOUD_MUTATION_BY_1110=false +WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END