CHAT_OUTPUT_BEGIN
COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=newfi-dr221-vm210-producer-membership-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=f147173d66cd34c3f641e8e304fe06d95889d3fb0886845c883d37b7494f102f
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=df560dcc7c495f7172790e5839527d5a14299e541ad404e0fee0a82517a452eb
SANITIZED_OUTPUT_SHA256=4c78ca1b8648c6cbd612ae432ff927d18627ba6573375c6afc3af25de784b531
OUTPUT_BEGIN
NEWFI416_BEGIN=true
COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1
MODE=read-only
COMPONENT=newfi-dr221-vm210-producer-membership-rca
SCOPE=PROVE_OR_REFUTE_VM210_MEMBERSHIP_IN_CANONICAL_VM_CLOUD_QUORUM_PRODUCER
MUTATION_BOUNDARY=NONE;NO_BACKUP;NO_DISPATCH;NO_SYSTEMD_ACTION;NO_CLOUD_WRITE;NO_VM_WRITE;NO_GIT_WRITE
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
QUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
FRESHNESS_SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96
NODE_WORKER_SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1
VM210_ALL_TARGET_ROWS=2
VM210_PVE_TARGET_COUNT=1
VM210_XF_TARGET_COUNT=1
VM210_PVE_ENABLED=true
VM210_XF_ENABLED=true
PRODUCER_MEMBERSHIP_SOURCE_BEGIN=true
PRODUCER_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue|SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
9-FRESHNESS_SECONDS=72000
10-DUE_SECONDS=43200
11-SSH_BASE=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]
12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)
13-MIGRATE_VMIDS=(160 170 130 171 180 190 150 9130 111 112 113)
14-
15-die(){ printf 'VM_QUORUM_QUEUE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; }
16-(( DUE_SECONDS > 0 && DUE_SECONDS < FRESHNESS_SECONDS )) || die DUE_THRESHOLD_INVALID 65
17:expected_owner(){
18-  case "$1" in
19-    110|112|150|170|171) printf 'pve01\n' ;;
20-    111|113|160|180|9130) printf 'pve02\n' ;;
21-    130|190|200) printf 'pve03\n' ;;
22:    *) die "VMID_NOT_ALLOWED_$1" 64 ;;
23-  esac
24-}
25-observed_owner(){
--
45-}
46-_run_worker_unlocked() {
47-  local vmid="$1" action="$2" generation="${3:-}" owner observed host
48:  owner="$(expected_owner "$vmid")"
49-  observed="$(observed_owner "$vmid")"
50-  [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65
51-  if [[ "$owner" == "pve01" ]]; then
--
61-}
62-run_probe(){
63-  local vmid="$1" owner observed host
64:  owner="$(expected_owner "$vmid")"
65-  observed="$(observed_owner "$vmid")"
66-  [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65
67-  if [[ "$owner" == "pve01" ]]; then
--
73-}
74-plan_all(){
75-  local vmid
76:  for vmid in "${ALL_VMIDS[@]}"; do
77:    printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\n' "$vmid" "$(expected_owner "$vmid")" "$(observed_owner "$vmid")"
78-  done
79-}
80-freshness_decision(){
--
116-freshness_plan(){
117-  local vmid now_epoch out decision
118-  [[ -x "$PROBE" ]] || die "FRESHNESS_PROBE_NOT_INSTALLED" 69
119:  for vmid in "${ALL_VMIDS[@]}"; do
120-    now_epoch="$(date -u +%s)"
121-    out="$(freshness_decision "$vmid" "$now_epoch")"
122-    printf '%s\n' "$out"
--
132-  exec 8>"$HEAVY_LOCK"
133-  flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75
134-  for vmid in "$@"; do
135:    printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")"
136-    out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
137-    printf '%s\n' "$out"
138-    generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
--
146-dispatch_one(){
147-  local vmid="$1" freshness decision now_epoch out generation
148-  [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64
149:  expected_owner "$vmid" >/dev/null
150-  exec 9>"$LOCK"
151-  if ! flock -n 9; then
152-    printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\n' "$vmid"
--
170-    BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
171-    *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;;
172-  esac
173:  printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision"
174-  out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
175-  printf '%s\n' "$out"
176-  generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96
5-RESTORE_ROOT=/var/lib/homelab-backup/vm-restore-validation
6-HOST="$(hostname -s)"
7-die(){ printf 'VM_FRESHNESS_PROBE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; }
8:expected_owner(){
9-case "$1" in
10-110|112|150|170|171) printf 'pve01\n' ;;
11-111|113|160|180|9130) printf 'pve02\n' ;;
12-130|190|200) printf 'pve03\n' ;;
13:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
14-esac
15-}
16-assert_local_identity(){
17-local vmid="$1" owner
18:owner="$(expected_owner "$vmid")"
19-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65
20-}
21-state_scan(){
PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1
21-active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ "$rc" == 1 ]] && return 0; die "PGREP_FAILED_$rc" 69; fi; }
22-snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }
23-vm_lock(){ qm config "$1" | awk -F': ' '$1=="lock"{print $2; found=1} END{if(!found) print "none"}'; }
24:expected_owner(){
25-case "$1" in
26-110|112|150|170|171) printf 'pve01\n' ;;
27-111|113|160|180|9130) printf 'pve02\n' ;;
28-130|190|200) printf 'pve03\n' ;;
29:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
30-esac
31-}
32:expected_type(){
33-case "$1" in
34:110|111|112|113|200) printf 'lxc\n' ;;
35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;;
36:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
37-esac
38-}
39-expected_status(){
--
45-}
46-assert_local_identity(){
47-local vmid="$1" owner type status expect_status
48:owner="$(expected_owner "$vmid")"
49:type="$(expected_type "$vmid")"
50-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65
51-if [[ "$type" == "qemu" ]]; then
52-[[ -s "/etc/pve/qemu-server/${vmid}.conf" ]] || die "QEMU_CONFIG_MISSING_$vmid" 66
--
153-plan(){
154-local vmid="$1" owner type status
155-assert_local_identity "$vmid"
156:owner="$(expected_owner "$vmid")"
157:type="$(expected_type "$vmid")"
158-status="$(expected_status "$vmid")"
159-printf 'PLAN_STATUS=OK\nVMID=%s\nOWNER=%s\nTYPE=%s\nEXPECTED_STATUS=%s\n' "$vmid" "$owner" "$type" "$status"
160-}
PRODUCER_MEMBERSHIP_SOURCE_END=true
VM210_FRESHNESS_DIRECT_RC=64
VM210_FRESHNESS_STDOUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VM210_FRESHNESS_STDERR_SHA256=b0d9e4d7ad61d749dfaf399ab3ad606613113a88a2c3e7085ff17e2463dc8b9e
VM210_FRESHNESS_NOT_ALLOWED=true
QUEUE_PLAN_RC=0
QUEUE_PLAN_HAS_VM210=false
QUEUE_PLAN_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
STATIC_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue
STATIC_CONTAINS_210_LITERAL=false
STATIC_CONTAINS_VM9130_LITERAL=true
STATIC_LINE=12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)
STATIC_LINE=22:    *) die "VMID_NOT_ALLOWED_$1" 64 ;;
STATIC_LINE=76:  for vmid in "${ALL_VMIDS[@]}"; do
STATIC_LINE=119:  for vmid in "${ALL_VMIDS[@]}"; do
STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe
STATIC_CONTAINS_210_LITERAL=false
STATIC_CONTAINS_VM9130_LITERAL=true
STATIC_LINE=13:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker
STATIC_CONTAINS_210_LITERAL=false
STATIC_CONTAINS_VM9130_LITERAL=true
STATIC_LINE=29:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
STATIC_LINE=34:110|111|112|113|200) printf 'lxc\n' ;;
STATIC_LINE=35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;;
STATIC_LINE=36:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
OWNER_STATE|service=activating|sub=start|heavy_lock=BUSY
DECISION=PASS_DR221_416_VM210_PRODUCER_MEMBERSHIP_MISSING_PROVEN
NEXT_GATE=WAIT_OWNER_THEN_REPAIR_VM210_CLOUD_QUORUM_PRODUCER_MEMBERSHIP_AND_FINAL_DR
CHANGES_MADE_BY_416=false
PRODUCT_MUTATION_BY_416=false
VM_MUTATION_BY_416=false
CLOUD_MUTATION_BY_416=false
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
NEWFI416_END=true

OUTPUT_END
CHAT_OUTPUT_END
