homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-09-22 14:05:21 +00:00
parent 91e577b7bf
commit 0273560f43
4 changed files with 472 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.json
+38
View File
File diff suppressed because one or more lines are too long
runtime/history/NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.txt
+216
View File
@@ -0,0 +1,216 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=newfi-dr221-vm210-producer-membership-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=f147173d66cd34c3f641e8e304fe06d95889d3fb0886845c883d37b7494f102f
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=df560dcc7c495f7172790e5839527d5a14299e541ad404e0fee0a82517a452eb
SANITIZED_OUTPUT_SHA256=4c78ca1b8648c6cbd612ae432ff927d18627ba6573375c6afc3af25de784b531
OUTPUT_BEGIN
NEWFI416_BEGIN=true
COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1
MODE=read-only
COMPONENT=newfi-dr221-vm210-producer-membership-rca
SCOPE=PROVE_OR_REFUTE_VM210_MEMBERSHIP_IN_CANONICAL_VM_CLOUD_QUORUM_PRODUCER
MUTATION_BOUNDARY=NONE;NO_BACKUP;NO_DISPATCH;NO_SYSTEMD_ACTION;NO_CLOUD_WRITE;NO_VM_WRITE;NO_GIT_WRITE
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
QUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
FRESHNESS_SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96
NODE_WORKER_SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1
VM210_ALL_TARGET_ROWS=2
VM210_PVE_TARGET_COUNT=1
VM210_XF_TARGET_COUNT=1
VM210_PVE_ENABLED=true
VM210_XF_ENABLED=true
PRODUCER_MEMBERSHIP_SOURCE_BEGIN=true
PRODUCER_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue|SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
9-FRESHNESS_SECONDS=72000
10-DUE_SECONDS=43200
11-SSH_BASE=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]
12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)
13-MIGRATE_VMIDS=(160 170 130 171 180 190 150 9130 111 112 113)
14-
15-die(){ printf 'VM_QUORUM_QUEUE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; }
16-(( DUE_SECONDS > 0 && DUE_SECONDS < FRESHNESS_SECONDS )) || die DUE_THRESHOLD_INVALID 65
17:expected_owner(){
18- case "$1" in
19- 110|112|150|170|171) printf 'pve01\n' ;;
20- 111|113|160|180|9130) printf 'pve02\n' ;;
21- 130|190|200) printf 'pve03\n' ;;
22: *) die "VMID_NOT_ALLOWED_$1" 64 ;;
23- esac
24-}
25-observed_owner(){
--
45-}
46-_run_worker_unlocked() {
47- local vmid="$1" action="$2" generation="${3:-}" owner observed host
48: owner="$(expected_owner "$vmid")"
49- observed="$(observed_owner "$vmid")"
50- [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65
51- if [[ "$owner" == "pve01" ]]; then
--
61-}
62-run_probe(){
63- local vmid="$1" owner observed host
64: owner="$(expected_owner "$vmid")"
65- observed="$(observed_owner "$vmid")"
66- [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65
67- if [[ "$owner" == "pve01" ]]; then
--
73-}
74-plan_all(){
75- local vmid
76: for vmid in "${ALL_VMIDS[@]}"; do
77: printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\n' "$vmid" "$(expected_owner "$vmid")" "$(observed_owner "$vmid")"
78- done
79-}
80-freshness_decision(){
--
116-freshness_plan(){
117- local vmid now_epoch out decision
118- [[ -x "$PROBE" ]] || die "FRESHNESS_PROBE_NOT_INSTALLED" 69
119: for vmid in "${ALL_VMIDS[@]}"; do
120- now_epoch="$(date -u +%s)"
121- out="$(freshness_decision "$vmid" "$now_epoch")"
122- printf '%s\n' "$out"
--
132- exec 8>"$HEAVY_LOCK"
133- flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75
134- for vmid in "$@"; do
135: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")"
136- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
137- printf '%s\n' "$out"
138- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
--
146-dispatch_one(){
147- local vmid="$1" freshness decision now_epoch out generation
148- [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64
149: expected_owner "$vmid" >/dev/null
150- exec 9>"$LOCK"
151- if ! flock -n 9; then
152- printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\n' "$vmid"
--
170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
171- *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;;
172- esac
173: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision"
174- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
175- printf '%s\n' "$out"
176- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96
5-RESTORE_ROOT=/var/lib/homelab-backup/vm-restore-validation
6-HOST="$(hostname -s)"
7-die(){ printf 'VM_FRESHNESS_PROBE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; }
8:expected_owner(){
9-case "$1" in
10-110|112|150|170|171) printf 'pve01\n' ;;
11-111|113|160|180|9130) printf 'pve02\n' ;;
12-130|190|200) printf 'pve03\n' ;;
13:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
14-esac
15-}
16-assert_local_identity(){
17-local vmid="$1" owner
18:owner="$(expected_owner "$vmid")"
19-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65
20-}
21-state_scan(){
PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1
21-active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ "$rc" == 1 ]] && return 0; die "PGREP_FAILED_$rc" 69; fi; }
22-snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }
23-vm_lock(){ qm config "$1" | awk -F': ' '$1=="lock"{print $2; found=1} END{if(!found) print "none"}'; }
24:expected_owner(){
25-case "$1" in
26-110|112|150|170|171) printf 'pve01\n' ;;
27-111|113|160|180|9130) printf 'pve02\n' ;;
28-130|190|200) printf 'pve03\n' ;;
29:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
30-esac
31-}
32:expected_type(){
33-case "$1" in
34:110|111|112|113|200) printf 'lxc\n' ;;
35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;;
36:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
37-esac
38-}
39-expected_status(){
--
45-}
46-assert_local_identity(){
47-local vmid="$1" owner type status expect_status
48:owner="$(expected_owner "$vmid")"
49:type="$(expected_type "$vmid")"
50-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65
51-if [[ "$type" == "qemu" ]]; then
52-[[ -s "/etc/pve/qemu-server/${vmid}.conf" ]] || die "QEMU_CONFIG_MISSING_$vmid" 66
--
153-plan(){
154-local vmid="$1" owner type status
155-assert_local_identity "$vmid"
156:owner="$(expected_owner "$vmid")"
157:type="$(expected_type "$vmid")"
158-status="$(expected_status "$vmid")"
159-printf 'PLAN_STATUS=OK\nVMID=%s\nOWNER=%s\nTYPE=%s\nEXPECTED_STATUS=%s\n' "$vmid" "$owner" "$type" "$status"
160-}
PRODUCER_MEMBERSHIP_SOURCE_END=true
VM210_FRESHNESS_DIRECT_RC=64
VM210_FRESHNESS_STDOUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VM210_FRESHNESS_STDERR_SHA256=b0d9e4d7ad61d749dfaf399ab3ad606613113a88a2c3e7085ff17e2463dc8b9e
VM210_FRESHNESS_NOT_ALLOWED=true
QUEUE_PLAN_RC=0
QUEUE_PLAN_HAS_VM210=false
QUEUE_PLAN_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
STATIC_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue
STATIC_CONTAINS_210_LITERAL=false
STATIC_CONTAINS_VM9130_LITERAL=true
STATIC_LINE=12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)
STATIC_LINE=22: *) die "VMID_NOT_ALLOWED_$1" 64 ;;
STATIC_LINE=76: for vmid in "${ALL_VMIDS[@]}"; do
STATIC_LINE=119: for vmid in "${ALL_VMIDS[@]}"; do
STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe
STATIC_CONTAINS_210_LITERAL=false
STATIC_CONTAINS_VM9130_LITERAL=true
STATIC_LINE=13:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker
STATIC_CONTAINS_210_LITERAL=false
STATIC_CONTAINS_VM9130_LITERAL=true
STATIC_LINE=29:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
STATIC_LINE=34:110|111|112|113|200) printf 'lxc\n' ;;
STATIC_LINE=35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;;
STATIC_LINE=36:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
OWNER_STATE|service=activating|sub=start|heavy_lock=BUSY
DECISION=PASS_DR221_416_VM210_PRODUCER_MEMBERSHIP_MISSING_PROVEN
NEXT_GATE=WAIT_OWNER_THEN_REPAIR_VM210_CLOUD_QUORUM_PRODUCER_MEMBERSHIP_AND_FINAL_DR
CHANGES_MADE_BY_416=false
PRODUCT_MUTATION_BY_416=false
VM_MUTATION_BY_416=false
CLOUD_MUTATION_BY_416=false
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
NEWFI416_END=true
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+18 -18
View File
File diff suppressed because one or more lines are too long
runtime/latest.txt
+200 -21
View File
@@ -1,37 +1,216 @@
CHAT_OUTPUT_BEGIN CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260922-SPB-HOME-INGRESS-CERT-VHOST-FIX-9123R1 COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1
STATUS=FAIL STATUS=OK
RC=76 RC=0
HOST=pve01 HOST=pve01
MODE=change MODE=read-only
COMPONENT=spb-home-ingress-cert-vhost-fix COMPONENT=newfi-dr221-vm210-producer-membership-rca
REFERENCE_REGISTER_CHECK=OK REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=fbcb4259c7383951413ad138ffac6c866a9b89f23e6a2a3a3d15b96ae602a946 COMMAND_SHA256=f147173d66cd34c3f641e8e304fe06d95889d3fb0886845c883d37b7494f102f
DUPLICATE_FAILED_COMMAND_BLOCKED=false DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true EXECUTION_STARTED=true
CHANGE_DECLARED=true CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=false RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=UNKNOWN RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=INVALID_CONTRACT:ValueError RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=46 COMMAND_RC=0
CHANGES_MADE=unknown CHANGES_MADE=false
ROLLBACK_STARTED=unknown ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null ROLLBACK_RESTORED=null
MUTATION_OUTCOME=UNKNOWN MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes SANITIZED=yes
SECRETS_INCLUDED=no SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=ef0e7c0fe3c4a7aa29d15ee109d164fd9c85355601ad5532b8acac3178f1070d RAW_EVIDENCE_SHA256=df560dcc7c495f7172790e5839527d5a14299e541ad404e0fee0a82517a452eb
SANITIZED_OUTPUT_SHA256=ef0e7c0fe3c4a7aa29d15ee109d164fd9c85355601ad5532b8acac3178f1070d SANITIZED_OUTPUT_SHA256=4c78ca1b8648c6cbd612ae432ff927d18627ba6573375c6afc3af25de784b531
OUTPUT_BEGIN OUTPUT_BEGIN
SPB9121_CHANGE={"preflight":{"sha_gate":"PASS","data_mount_hash":"ec0ac67820f88633","certbot_layout":"PASS","new_objects_absent":"PASS","nginx_test":"PASS","gitread_policy":"PASS","gitread_backend_hash":"160a494a6106d479"},"certs":{"git-read.gram1.ru":{"certbot_rc":1,"stdout_sha":null,"stderr_sha":"04e28386d4c46be6"}},"vhosts":{},"probes":{},"changes_made":false,"rollback_started":true,"rollback_restored":true,"status":"FAIL","rollback":{"started":true,"reason":"CERTBOT_FAIL_git-read.gram1.ru","removed_files":[],"cert_delete":{},"reload":{"test_rc":0,"reload_rc":0,"stderr_sha":null},"restored":true}} NEWFI416_BEGIN=true
TASK_COMPLETE=true COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1
TASK_RESULT=FAIL_SPB_HOME_INGRESS_CERT_VHOST_FIX MODE=read-only
DECISION=HOLD_SPB_ON_AND_READ_ROLLBACK_RESULT COMPONENT=newfi-dr221-vm210-producer-membership-rca
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-SPB-HOME-INGRESS-CERT-VHOST-FIX-9123R1","status":"FAIL","changes_made":false,"rollback_started":true,"rollback_restored":true} SCOPE=PROVE_OR_REFUTE_VM210_MEMBERSHIP_IN_CANONICAL_VM_CLOUD_QUORUM_PRODUCER
MUTATION_BOUNDARY=NONE;NO_BACKUP;NO_DISPATCH;NO_SYSTEMD_ACTION;NO_CLOUD_WRITE;NO_VM_WRITE;NO_GIT_WRITE
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
QUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
FRESHNESS_SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96
NODE_WORKER_SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1
VM210_ALL_TARGET_ROWS=2
VM210_PVE_TARGET_COUNT=1
VM210_XF_TARGET_COUNT=1
VM210_PVE_ENABLED=true
VM210_XF_ENABLED=true
PRODUCER_MEMBERSHIP_SOURCE_BEGIN=true
PRODUCER_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue|SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
9-FRESHNESS_SECONDS=72000
10-DUE_SECONDS=43200
11-SSH_BASE=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]
12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)
13-MIGRATE_VMIDS=(160 170 130 171 180 190 150 9130 111 112 113)
14-
15-die(){ printf 'VM_QUORUM_QUEUE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; }
16-(( DUE_SECONDS > 0 && DUE_SECONDS < FRESHNESS_SECONDS )) || die DUE_THRESHOLD_INVALID 65
17:expected_owner(){
18- case "$1" in
19- 110|112|150|170|171) printf 'pve01\n' ;;
20- 111|113|160|180|9130) printf 'pve02\n' ;;
21- 130|190|200) printf 'pve03\n' ;;
22: *) die "VMID_NOT_ALLOWED_$1" 64 ;;
23- esac
24-}
25-observed_owner(){
--
45-}
46-_run_worker_unlocked() {
47- local vmid="$1" action="$2" generation="${3:-}" owner observed host
48: owner="$(expected_owner "$vmid")"
49- observed="$(observed_owner "$vmid")"
50- [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65
51- if [[ "$owner" == "pve01" ]]; then
--
61-}
62-run_probe(){
63- local vmid="$1" owner observed host
64: owner="$(expected_owner "$vmid")"
65- observed="$(observed_owner "$vmid")"
66- [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65
67- if [[ "$owner" == "pve01" ]]; then
--
73-}
74-plan_all(){
75- local vmid
76: for vmid in "${ALL_VMIDS[@]}"; do
77: printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\n' "$vmid" "$(expected_owner "$vmid")" "$(observed_owner "$vmid")"
78- done
79-}
80-freshness_decision(){
--
116-freshness_plan(){
117- local vmid now_epoch out decision
118- [[ -x "$PROBE" ]] || die "FRESHNESS_PROBE_NOT_INSTALLED" 69
119: for vmid in "${ALL_VMIDS[@]}"; do
120- now_epoch="$(date -u +%s)"
121- out="$(freshness_decision "$vmid" "$now_epoch")"
122- printf '%s\n' "$out"
--
132- exec 8>"$HEAVY_LOCK"
133- flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75
134- for vmid in "$@"; do
135: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")"
136- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
137- printf '%s\n' "$out"
138- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
--
146-dispatch_one(){
147- local vmid="$1" freshness decision now_epoch out generation
148- [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64
149: expected_owner "$vmid" >/dev/null
150- exec 9>"$LOCK"
151- if ! flock -n 9; then
152- printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\n' "$vmid"
--
170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;
171- *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;;
172- esac
173: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision"
174- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)"
175- printf '%s\n' "$out"
176- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96
5-RESTORE_ROOT=/var/lib/homelab-backup/vm-restore-validation
6-HOST="$(hostname -s)"
7-die(){ printf 'VM_FRESHNESS_PROBE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; }
8:expected_owner(){
9-case "$1" in
10-110|112|150|170|171) printf 'pve01\n' ;;
11-111|113|160|180|9130) printf 'pve02\n' ;;
12-130|190|200) printf 'pve03\n' ;;
13:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
14-esac
15-}
16-assert_local_identity(){
17-local vmid="$1" owner
18:owner="$(expected_owner "$vmid")"
19-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65
20-}
21-state_scan(){
PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1
21-active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ "$rc" == 1 ]] && return 0; die "PGREP_FAILED_$rc" 69; fi; }
22-snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }
23-vm_lock(){ qm config "$1" | awk -F': ' '$1=="lock"{print $2; found=1} END{if(!found) print "none"}'; }
24:expected_owner(){
25-case "$1" in
26-110|112|150|170|171) printf 'pve01\n' ;;
27-111|113|160|180|9130) printf 'pve02\n' ;;
28-130|190|200) printf 'pve03\n' ;;
29:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
30-esac
31-}
32:expected_type(){
33-case "$1" in
34:110|111|112|113|200) printf 'lxc\n' ;;
35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;;
36:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
37-esac
38-}
39-expected_status(){
--
45-}
46-assert_local_identity(){
47-local vmid="$1" owner type status expect_status
48:owner="$(expected_owner "$vmid")"
49:type="$(expected_type "$vmid")"
50-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65
51-if [[ "$type" == "qemu" ]]; then
52-[[ -s "/etc/pve/qemu-server/${vmid}.conf" ]] || die "QEMU_CONFIG_MISSING_$vmid" 66
--
153-plan(){
154-local vmid="$1" owner type status
155-assert_local_identity "$vmid"
156:owner="$(expected_owner "$vmid")"
157:type="$(expected_type "$vmid")"
158-status="$(expected_status "$vmid")"
159-printf 'PLAN_STATUS=OK\nVMID=%s\nOWNER=%s\nTYPE=%s\nEXPECTED_STATUS=%s\n' "$vmid" "$owner" "$type" "$status"
160-}
PRODUCER_MEMBERSHIP_SOURCE_END=true
VM210_FRESHNESS_DIRECT_RC=64
VM210_FRESHNESS_STDOUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
VM210_FRESHNESS_STDERR_SHA256=b0d9e4d7ad61d749dfaf399ab3ad606613113a88a2c3e7085ff17e2463dc8b9e
VM210_FRESHNESS_NOT_ALLOWED=true
QUEUE_PLAN_RC=0
QUEUE_PLAN_HAS_VM210=false
QUEUE_PLAN_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
STATIC_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue
STATIC_CONTAINS_210_LITERAL=false
STATIC_CONTAINS_VM9130_LITERAL=true
STATIC_LINE=12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)
STATIC_LINE=22: *) die "VMID_NOT_ALLOWED_$1" 64 ;;
STATIC_LINE=76: for vmid in "${ALL_VMIDS[@]}"; do
STATIC_LINE=119: for vmid in "${ALL_VMIDS[@]}"; do
STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe
STATIC_CONTAINS_210_LITERAL=false
STATIC_CONTAINS_VM9130_LITERAL=true
STATIC_LINE=13:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker
STATIC_CONTAINS_210_LITERAL=false
STATIC_CONTAINS_VM9130_LITERAL=true
STATIC_LINE=29:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
STATIC_LINE=34:110|111|112|113|200) printf 'lxc\n' ;;
STATIC_LINE=35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;;
STATIC_LINE=36:*) die "VMID_NOT_ALLOWED_$1" 64 ;;
OWNER_STATE|service=activating|sub=start|heavy_lock=BUSY
DECISION=PASS_DR221_416_VM210_PRODUCER_MEMBERSHIP_MISSING_PROVEN
NEXT_GATE=WAIT_OWNER_THEN_REPAIR_VM210_CLOUD_QUORUM_PRODUCER_MEMBERSHIP_AND_FINAL_DR
CHANGES_MADE_BY_416=false
PRODUCT_MUTATION_BY_416=false
VM_MUTATION_BY_416=false
CLOUD_MUTATION_BY_416=false
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
NEWFI416_END=true
OUTPUT_END OUTPUT_END
CHAT_OUTPUT_END CHAT_OUTPUT_END