homelab-runtime-publisher
360 lines
25 KiB
Plaintext
360 lines
25 KiB
Plaintext
CHAT_OUTPUT_BEGIN
|
|
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1
|
|
STATUS=OK
|
|
RC=0
|
|
HOST=pve01
|
|
MODE=read-only
|
|
COMPONENT=homelab-backup-audit-vmqueue-rca
|
|
REFERENCE_REGISTER_CHECK=OK
|
|
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
|
ERROR_REGISTER_CHECK=OK
|
|
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
|
COMMAND_SHA256=ed70f0c53596e8aa062b17b3c599cc9bc6eda7472edb9ec786f9a88a23af2eb4
|
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
|
EXECUTION_STARTED=true
|
|
CHANGE_DECLARED=false
|
|
RESULT_CONTRACT_VALID=true
|
|
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
|
RESULT_CONTRACT_ERROR=NONE
|
|
COMMAND_RC=0
|
|
CHANGES_MADE=false
|
|
ROLLBACK_STARTED=false
|
|
ROLLBACK_RESTORED=null
|
|
MUTATION_OUTCOME=NO_MUTATION
|
|
SANITIZED=yes
|
|
SECRETS_INCLUDED=no
|
|
PRIVATE_ADDRESSES_INCLUDED=no
|
|
RAW_EVIDENCE_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033
|
|
SANITIZED_OUTPUT_SHA256=374fca16fcb577fbdd36d6d6ef0338586c0a609fc46e7830dedc40a9a6060033
|
|
OUTPUT_BEGIN
|
|
WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_BEGIN=1
|
|
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-AUDIT-VMQUEUE-RCA-1110R1
|
|
MODE=read-only
|
|
COMPONENT=homelab-backup-audit-vmqueue-rca
|
|
ERROR_REGISTER_CHECK=OK
|
|
REFERENCE_CHECK=OK
|
|
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
|
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
|
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
|
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
|
|
BACKUP_AUDIT_STATE_BEGIN=1
|
|
Result=exit-code
|
|
ExecMainStartTimestamp=Tue 2026-09-22 07:36:58 MSK
|
|
ExecMainExitTimestamp=Tue 2026-09-22 07:37:01 MSK
|
|
ExecMainCode=1
|
|
ExecMainStatus=1
|
|
ActiveState=failed
|
|
SubState=failed
|
|
StateChangeTimestamp=Tue 2026-09-22 07:37:01 MSK
|
|
BACKUP_AUDIT_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-audit.service
|
|
BACKUP_AUDIT_FRAGMENT_SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
ExecStart=/usr/local/sbin/homelab-backup-audit scheduled
|
|
BACKUP_AUDIT_EXIT_EPOCH=1790051821
|
|
BACKUP_AUDIT_STATE_END=1
|
|
VM_QUEUE_STATE_BEGIN=1
|
|
Result=exit-code
|
|
ExecMainStartTimestamp=Wed 2026-09-16 20:32:57 MSK
|
|
ExecMainExitTimestamp=Wed 2026-09-16 20:33:02 MSK
|
|
ExecMainCode=1
|
|
ExecMainStatus=65
|
|
ActiveState=failed
|
|
SubState=failed
|
|
StateChangeTimestamp=Wed 2026-09-16 20:33:02 MSK
|
|
VM_QUEUE_FRAGMENT_PATH=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service
|
|
VM_QUEUE_FRAGMENT_SHA256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
ExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all
|
|
VM_QUEUE_EXIT_EPOCH=1789579982
|
|
VM_QUEUE_STATE_END=1
|
|
BACKUP_OWNER_STATE_BEGIN=1
|
|
Result=success
|
|
ExecMainStartTimestamp=Tue 2026-09-22 07:42:15 MSK
|
|
ExecMainExitTimestamp=Tue 2026-09-22 07:42:15 MSK
|
|
ExecMainCode=1
|
|
ExecMainStatus=0
|
|
ActiveState=inactive
|
|
SubState=dead
|
|
StateChangeTimestamp=Tue 2026-09-22 07:42:15 MSK
|
|
BACKUP_OWNER_FRAGMENT_PATH=/etc/systemd/system/homelab-backup-v2.service
|
|
BACKUP_OWNER_FRAGMENT_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled
|
|
BACKUP_OWNER_EXIT_EPOCH=1790052135
|
|
BACKUP_OWNER_STATE_END=1
|
|
POLICY_FIX_1108_FINISH_EPOCH=1790030296
|
|
VM_QUEUE_LAST_EXIT_EPOCH=1789579982
|
|
VM_QUEUE_FAILURE_CLASS=STALE_PRE1108_FAILED_STATE
|
|
BACKUP_AUDIT_JOURNAL_SHA256=77b32d66abed4873f79ed2ffb9bf0c4ec87762c4b78c85aa23136d3743f6b364
|
|
BACKUP_AUDIT_LATEST_CLASSIFIED_BEGIN=1
|
|
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
|
|
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
|
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
|
|
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
|
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
|
ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
|
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
|
|
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
|
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
|
|
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
|
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
|
|
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
|
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
|
|
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
|
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
|
|
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
|
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
|
|
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
|
ISSUE|severity=CRITICAL|code=CLOUD_B_RESTORE_DRILL_BAD|logical_id=app-skladchik-reports-monitor|root_cause=LATEST_CLOUD_B_DRILL_NOT_OK
|
|
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=BAD|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
|
BACKUP_AUDIT_LATEST_CLASSIFIED_END=1
|
|
VM_QUEUE_JOURNAL_SHA256=3967f9d0fff08524f53eedf57007e2a2567de9ba59fad5b5f2948e5039cbfa6d
|
|
VM_QUEUE_LATEST_CLASSIFIED_BEGIN=1
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260915T223521Z.json
|
|
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260915T223521Z AGE_SECONDS=39387 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
|
|
GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH
|
|
VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02
|
|
VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json
|
|
QUEUE_FRESHNESS VMID=160 DECISION=SKIP_FRESH GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=38803 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN VMID=160 DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json
|
|
QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=9957 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json
|
|
QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=7668 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
|
|
QUEUE_FRESHNESS VMID=171 DECISION=SKIP_FRESH GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=39491 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN VMID=171 DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
|
|
QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=7022 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
|
|
QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=6995 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json
|
|
QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=6937 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
|
|
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=6832 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
|
|
GLOBAL_DISPATCH DECISION=NOOP REASON=ALL_FRESH
|
|
VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02
|
|
VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json
|
|
QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_DUE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=53189 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_DUE
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json
|
|
QUEUE_FRESHNESS VMID=170 DECISION=SKIP_FRESH GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=24343 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN VMID=170 DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json
|
|
QUEUE_FRESHNESS VMID=130 DECISION=SKIP_FRESH GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=22053 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN VMID=130 DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
|
|
QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53876 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_DUE
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json
|
|
QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260916T113558Z AGE_SECONDS=21408 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json
|
|
QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260916T113630Z AGE_SECONDS=21381 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json
|
|
QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260916T113650Z AGE_SECONDS=21323 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json
|
|
QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260916T113814Z AGE_SECONDS=21217 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH
|
|
GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_DUE PRIORITY=3 AGE_SECONDS=53876
|
|
FRESHNESS_STATUS=OK
|
|
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json
|
|
QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_DUE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=53879 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000
|
|
QUEUE_BEGIN VMID=171 OWNER=pve01 MODE=BACKUP_DUE DISPATCH=ONE_TARGET
|
|
VM_QUORUM_QUEUE_ERROR=GENERATION_NOT_RETURNED_171
|
|
VM_QUEUE_LATEST_CLASSIFIED_END=1
|
|
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__COUNT=1
|
|
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__PATH=/usr/local/sbin/homelab-backup-audit
|
|
SOURCE_CLOUD_B_RESTORE_DRILL_BAD__SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
|
|
230-if CLOUDB_HEALTH.is_file():
|
|
231- txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace")
|
|
232- kv={}
|
|
233- for line in txt.splitlines():
|
|
234- if "=" in line:
|
|
235- k,v=line.split("=",1); kv[k]=v
|
|
236- cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes")
|
|
237-if not cloudb_ok:
|
|
238: issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"})
|
|
239-
|
|
240-source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
|
|
241-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
|
|
242-if hrc==0: source["local_head"]=ho.strip()
|
|
243-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10)
|
|
244-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10)
|
|
245-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED")
|
|
246-if frc==0 and fo.strip():
|
|
247- lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
|
|
248- if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
|
|
249-if source["remote_head"]!=source["local_head"]:
|
|
250- issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
|
|
SOURCE_GENERATION_NOT_RETURNED___COUNT=2
|
|
SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-app-cloud-quorum-queue
|
|
SOURCE_GENERATION_NOT_RETURNED___SHA256=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650
|
|
116- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
|
|
117- *) die "FRESHNESS_RECHECK_INVALID_${a}_${decision:-EMPTY}" 65 ;;
|
|
118- esac
|
|
119- printf 'QUEUE_BEGIN APP=%s MODE=%s DISPATCH=ONE_TARGET\n' "$a" "$decision"
|
|
120- out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"
|
|
121- printf '%s\n' "$out"
|
|
122- "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null
|
|
123- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
|
|
124: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$a" 65
|
|
125- printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$a" "$generation"
|
|
126- printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\n' "$a" "$generation"
|
|
127-}
|
|
128-
|
|
129-migrate_list(){
|
|
130- local a out
|
|
131- lock_heavy
|
|
132- for a in "${MIGRATE_APPS[@]}"; do
|
|
133- if out="$(exec 8>&-; exec 9>&-; "$WORKER" latest-sealed "$a" 2>/dev/null)"; then
|
|
134- printf 'QUEUE_REUSE_SEALED APP=%s\n%s\n' "$a" "$out"
|
|
135- continue
|
|
136- fi
|
|
SOURCE_GENERATION_NOT_RETURNED___PATH=/usr/local/sbin/homelab-vm-cloud-quorum-queue
|
|
SOURCE_GENERATION_NOT_RETURNED___SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
|
|
131- flock -n 9 || die "GLOBAL_QUEUE_LOCK_BUSY" 75
|
|
132- exec 8>"$HEAVY_LOCK"
|
|
133- flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75
|
|
134- for vmid in "$@"; do
|
|
135- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")"
|
|
136- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
|
|
137- printf '%s\n' "$out"
|
|
138- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
|
|
139: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65
|
|
140- printf 'QUEUE_SEALED VMID=%s GENERATION=%s\n' "$vmid" "$generation"
|
|
141- done
|
|
142-}
|
|
143-dispatch_plan(){
|
|
144- freshness_plan
|
|
145-}
|
|
146-dispatch_one(){
|
|
147- local vmid="$1" freshness decision now_epoch out generation
|
|
148- [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64
|
|
149- expected_owner "$vmid" >/dev/null
|
|
150- exec 9>"$LOCK"
|
|
151- if ! flock -n 9; then
|
|
--
|
|
169- ;;
|
|
170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
|
|
171- *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;;
|
|
172- esac
|
|
173- printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision"
|
|
174- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
|
|
175- printf '%s\n' "$out"
|
|
176- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
|
|
177: [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$vmid" 65
|
|
178- printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$vmid" "$generation"
|
|
179- printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\n' "$vmid" "$generation"
|
|
180-}
|
|
181-
|
|
182-ACTION="${1:-}"
|
|
183-case "$ACTION" in
|
|
184- plan)
|
|
185- [[ "$#" == 1 ]] || die "USAGE_PLAN" 64
|
|
186- plan_all
|
|
187- ;;
|
|
188- freshness-plan)
|
|
189- [[ "$#" == 1 ]] || die "USAGE_FRESHNESS_PLAN" 64
|
|
SCHEDULER_SELECT_RC=0
|
|
SCHEDULER_SELECT_BEGIN=1
|
|
SELFTEST_REPLACEMENT486=PASS
|
|
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
|
|
SCHEDULER_SELECT_END=1
|
|
CURRENT_RPO_SCHEDULER_CLASS=NO_DUE_TARGET
|
|
EVIDENCE_ROOT=/var/lib/homelab-backup
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T093429Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260831T213739Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260901T103255Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T003249Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260902T133648Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T023235Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260903T163234Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T064255Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260904T193019Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T093128Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260905T222933Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260906T113150Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T012910Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260907T143756Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T032904Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260908T173418Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T063400Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260909T193647Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260910T093211Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T003012Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260911T133013Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T013042Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260912T143251Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T033047Z.json
|
|
MTIME_EPOCH|458|/var/lib/homelab-backup/retention-safe-gc/transactions/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260913T174140Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T034236Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260903T174843Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T064613Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260904T193324Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T093438Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260905T223254Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260906T113503Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T013234Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260907T144115Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T033223Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260908T173740Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T063725Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260909T194007Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260910T093532Z.json
|
|
MTIME_EPOCH|410|/var/lib/homelab-backup/retention-safe-gc/transactions/vm171/vm171-pve01-20260911T003332Z.json
|
|
EVIDENCE_ROOT=/var/lib/homelab-backup-v2
|
|
MTIME_EPOCH|608|/var/lib/homelab-backup-v2/state/vm171.json
|
|
DECISION=PASS_BACKUP_1110_BLOCKERS_CLASSIFIED
|
|
NEXT_GATE=REPAIR_ONLY_CLASSIFIED_BACKUP_AUDIT_VMQUEUE_BLOCKERS
|
|
TASK_RESULT=PASS_HOMELAB_BACKUP_AUDIT_VMQUEUE_RCA
|
|
CHANGES_MADE_BY_1110=false
|
|
PRODUCT_MUTATION_BY_1110=false
|
|
VM_MUTATION_BY_1110=false
|
|
CLOUD_MUTATION_BY_1110=false
|
|
WORKERS2_BACKUP_AUDIT_VMQUEUE_RCA_END=1
|
|
|
|
OUTPUT_END
|
|
CHAT_OUTPUT_END
|