From 0273560f4346aaf0155a3699078427aa56a2d3e4 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 22 Sep 2026 14:05:21 +0000 Subject: [PATCH] runtime: publish NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1 --- ...21-VM210-PRODUCER-MEMBERSHIP-RCA416R1.json | 38 +++ ...221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.txt | 216 +++++++++++++++++ runtime/latest.json | 36 +-- runtime/latest.txt | 221 ++++++++++++++++-- 4 files changed, 472 insertions(+), 39 deletions(-) create mode 100644 runtime/history/NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.json create mode 100644 runtime/history/NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.txt diff --git a/runtime/history/NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.json b/runtime/history/NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.json new file mode 100644 index 00000000..6ac14e23 --- /dev/null +++ b/runtime/history/NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "newfi-dr221-vm210-producer-membership-rca", + "started_at_utc": "2026-09-22T14:05:20Z", + "finished_at_utc": "2026-09-22T14:05:20Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "f147173d66cd34c3f641e8e304fe06d95889d3fb0886845c883d37b7494f102f", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "df560dcc7c495f7172790e5839527d5a14299e541ad404e0fee0a82517a452eb", + "sanitized_output_sha256": "4c78ca1b8648c6cbd612ae432ff927d18627ba6573375c6afc3af25de784b531", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "NEWFI416_BEGIN=true\nCOMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1\nMODE=read-only\nCOMPONENT=newfi-dr221-vm210-producer-membership-rca\nSCOPE=PROVE_OR_REFUTE_VM210_MEMBERSHIP_IN_CANONICAL_VM_CLOUD_QUORUM_PRODUCER\nMUTATION_BOUNDARY=NONE;NO_BACKUP;NO_DISPATCH;NO_SYSTEMD_ACTION;NO_CLOUD_WRITE;NO_VM_WRITE;NO_GIT_WRITE\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nFRESHNESS_SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\nNODE_WORKER_SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\nVM210_ALL_TARGET_ROWS=2\nVM210_PVE_TARGET_COUNT=1\nVM210_XF_TARGET_COUNT=1\nVM210_PVE_ENABLED=true\nVM210_XF_ENABLED=true\nPRODUCER_MEMBERSHIP_SOURCE_BEGIN=true\nPRODUCER_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue|SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\n9-FRESHNESS_SECONDS=72000\n10-DUE_SECONDS=43200\n11-SSH_BASE=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\n12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)\n13-MIGRATE_VMIDS=(160 170 130 171 180 190 150 9130 111 112 113)\n14-\n15-die(){ printf 'VM_QUORUM_QUEUE_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }\n16-(( DUE_SECONDS > 0 && DUE_SECONDS < FRESHNESS_SECONDS )) || die DUE_THRESHOLD_INVALID 65\n17:expected_owner(){\n18- case \"$1\" in\n19- 110|112|150|170|171) printf 'pve01\\n' ;;\n20- 111|113|160|180|9130) printf 'pve02\\n' ;;\n21- 130|190|200) printf 'pve03\\n' ;;\n22: *) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n23- esac\n24-}\n25-observed_owner(){\n--\n45-}\n46-_run_worker_unlocked() {\n47- local vmid=\"$1\" action=\"$2\" generation=\"${3:-}\" owner observed host\n48: owner=\"$(expected_owner \"$vmid\")\"\n49- observed=\"$(observed_owner \"$vmid\")\"\n50- [[ \"$observed\" == \"$owner\" ]] || die \"CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}\" 65\n51- if [[ \"$owner\" == \"pve01\" ]]; then\n--\n61-}\n62-run_probe(){\n63- local vmid=\"$1\" owner observed host\n64: owner=\"$(expected_owner \"$vmid\")\"\n65- observed=\"$(observed_owner \"$vmid\")\"\n66- [[ \"$observed\" == \"$owner\" ]] || die \"CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}\" 65\n67- if [[ \"$owner\" == \"pve01\" ]]; then\n--\n73-}\n74-plan_all(){\n75- local vmid\n76: for vmid in \"${ALL_VMIDS[@]}\"; do\n77: printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$(observed_owner \"$vmid\")\"\n78- done\n79-}\n80-freshness_decision(){\n--\n116-freshness_plan(){\n117- local vmid now_epoch out decision\n118- [[ -x \"$PROBE\" ]] || die \"FRESHNESS_PROBE_NOT_INSTALLED\" 69\n119: for vmid in \"${ALL_VMIDS[@]}\"; do\n120- now_epoch=\"$(date -u +%s)\"\n121- out=\"$(freshness_decision \"$vmid\" \"$now_epoch\")\"\n122- printf '%s\\n' \"$out\"\n--\n132- exec 8>\"$HEAVY_LOCK\"\n133- flock -n 8 || die \"GLOBAL_HEAVY_LOCK_BUSY\" 75\n134- for vmid in \"$@\"; do\n135: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\"\n136- out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n137- printf '%s\\n' \"$out\"\n138- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n--\n146-dispatch_one(){\n147- local vmid=\"$1\" freshness decision now_epoch out generation\n148- [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64\n149: expected_owner \"$vmid\" >/dev/null\n150- exec 9>\"$LOCK\"\n151- if ! flock -n 9; then\n152- printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\\n' \"$vmid\"\n--\n170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\n171- *) die \"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\" 65 ;;\n172- esac\n173: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$decision\"\n174- out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n175- printf '%s\\n' \"$out\"\n176- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nPRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\n5-RESTORE_ROOT=/var/lib/homelab-backup/vm-restore-validation\n6-HOST=\"$(hostname -s)\"\n7-die(){ printf 'VM_FRESHNESS_PROBE_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }\n8:expected_owner(){\n9-case \"$1\" in\n10-110|112|150|170|171) printf 'pve01\\n' ;;\n11-111|113|160|180|9130) printf 'pve02\\n' ;;\n12-130|190|200) printf 'pve03\\n' ;;\n13:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n14-esac\n15-}\n16-assert_local_identity(){\n17-local vmid=\"$1\" owner\n18:owner=\"$(expected_owner \"$vmid\")\"\n19-[[ \"$HOST\" == \"$owner\" ]] || die \"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\" 65\n20-}\n21-state_scan(){\nPRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\n21-active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ \"$rc\" == 1 ]] && return 0; die \"PGREP_FAILED_$rc\" 69; fi; }\n22-snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }\n23-vm_lock(){ qm config \"$1\" | awk -F': ' '$1==\"lock\"{print $2; found=1} END{if(!found) print \"none\"}'; }\n24:expected_owner(){\n25-case \"$1\" in\n26-110|112|150|170|171) printf 'pve01\\n' ;;\n27-111|113|160|180|9130) printf 'pve02\\n' ;;\n28-130|190|200) printf 'pve03\\n' ;;\n29:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n30-esac\n31-}\n32:expected_type(){\n33-case \"$1\" in\n34:110|111|112|113|200) printf 'lxc\\n' ;;\n35:130|150|160|170|171|180|190|9130) printf 'qemu\\n' ;;\n36:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n37-esac\n38-}\n39-expected_status(){\n--\n45-}\n46-assert_local_identity(){\n47-local vmid=\"$1\" owner type status expect_status\n48:owner=\"$(expected_owner \"$vmid\")\"\n49:type=\"$(expected_type \"$vmid\")\"\n50-[[ \"$HOST\" == \"$owner\" ]] || die \"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\" 65\n51-if [[ \"$type\" == \"qemu\" ]]; then\n52-[[ -s \"/etc/pve/qemu-server/${vmid}.conf\" ]] || die \"QEMU_CONFIG_MISSING_$vmid\" 66\n--\n153-plan(){\n154-local vmid=\"$1\" owner type status\n155-assert_local_identity \"$vmid\"\n156:owner=\"$(expected_owner \"$vmid\")\"\n157:type=\"$(expected_type \"$vmid\")\"\n158-status=\"$(expected_status \"$vmid\")\"\n159-printf 'PLAN_STATUS=OK\\nVMID=%s\\nOWNER=%s\\nTYPE=%s\\nEXPECTED_STATUS=%s\\n' \"$vmid\" \"$owner\" \"$type\" \"$status\"\n160-}\nPRODUCER_MEMBERSHIP_SOURCE_END=true\nVM210_FRESHNESS_DIRECT_RC=64\nVM210_FRESHNESS_STDOUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVM210_FRESHNESS_STDERR_SHA256=b0d9e4d7ad61d749dfaf399ab3ad606613113a88a2c3e7085ff17e2463dc8b9e\nVM210_FRESHNESS_NOT_ALLOWED=true\nQUEUE_PLAN_RC=0\nQUEUE_PLAN_HAS_VM210=false\nQUEUE_PLAN_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nSTATIC_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)\nSTATIC_LINE=22: *) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_LINE=76: for vmid in \"${ALL_VMIDS[@]}\"; do\nSTATIC_LINE=119: for vmid in \"${ALL_VMIDS[@]}\"; do\nSTATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=13:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=29:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_LINE=34:110|111|112|113|200) printf 'lxc\\n' ;;\nSTATIC_LINE=35:130|150|160|170|171|180|190|9130) printf 'qemu\\n' ;;\nSTATIC_LINE=36:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nOWNER_STATE|service=activating|sub=start|heavy_lock=BUSY\nDECISION=PASS_DR221_416_VM210_PRODUCER_MEMBERSHIP_MISSING_PROVEN\nNEXT_GATE=WAIT_OWNER_THEN_REPAIR_VM210_CLOUD_QUORUM_PRODUCER_MEMBERSHIP_AND_FINAL_DR\nCHANGES_MADE_BY_416=false\nPRODUCT_MUTATION_BY_416=false\nVM_MUTATION_BY_416=false\nCLOUD_MUTATION_BY_416=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nNEWFI416_END=true\n" +} diff --git a/runtime/history/NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.txt b/runtime/history/NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.txt new file mode 100644 index 00000000..9d6c8458 --- /dev/null +++ b/runtime/history/NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1.txt @@ -0,0 +1,216 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=newfi-dr221-vm210-producer-membership-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=f147173d66cd34c3f641e8e304fe06d95889d3fb0886845c883d37b7494f102f +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=df560dcc7c495f7172790e5839527d5a14299e541ad404e0fee0a82517a452eb +SANITIZED_OUTPUT_SHA256=4c78ca1b8648c6cbd612ae432ff927d18627ba6573375c6afc3af25de784b531 +OUTPUT_BEGIN +NEWFI416_BEGIN=true +COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1 +MODE=read-only +COMPONENT=newfi-dr221-vm210-producer-membership-rca +SCOPE=PROVE_OR_REFUTE_VM210_MEMBERSHIP_IN_CANONICAL_VM_CLOUD_QUORUM_PRODUCER +MUTATION_BOUNDARY=NONE;NO_BACKUP;NO_DISPATCH;NO_SYSTEMD_ACTION;NO_CLOUD_WRITE;NO_VM_WRITE;NO_GIT_WRITE +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +QUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 +FRESHNESS_SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96 +NODE_WORKER_SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 +VM210_ALL_TARGET_ROWS=2 +VM210_PVE_TARGET_COUNT=1 +VM210_XF_TARGET_COUNT=1 +VM210_PVE_ENABLED=true +VM210_XF_ENABLED=true +PRODUCER_MEMBERSHIP_SOURCE_BEGIN=true +PRODUCER_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue|SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 +9-FRESHNESS_SECONDS=72000 +10-DUE_SECONDS=43200 +11-SSH_BASE=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] +12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200) +13-MIGRATE_VMIDS=(160 170 130 171 180 190 150 9130 111 112 113) +14- +15-die(){ printf 'VM_QUORUM_QUEUE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; } +16-(( DUE_SECONDS > 0 && DUE_SECONDS < FRESHNESS_SECONDS )) || die DUE_THRESHOLD_INVALID 65 +17:expected_owner(){ +18- case "$1" in +19- 110|112|150|170|171) printf 'pve01\n' ;; +20- 111|113|160|180|9130) printf 'pve02\n' ;; +21- 130|190|200) printf 'pve03\n' ;; +22: *) die "VMID_NOT_ALLOWED_$1" 64 ;; +23- esac +24-} +25-observed_owner(){ +-- +45-} +46-_run_worker_unlocked() { +47- local vmid="$1" action="$2" generation="${3:-}" owner observed host +48: owner="$(expected_owner "$vmid")" +49- observed="$(observed_owner "$vmid")" +50- [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65 +51- if [[ "$owner" == "pve01" ]]; then +-- +61-} +62-run_probe(){ +63- local vmid="$1" owner observed host +64: owner="$(expected_owner "$vmid")" +65- observed="$(observed_owner "$vmid")" +66- [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65 +67- if [[ "$owner" == "pve01" ]]; then +-- +73-} +74-plan_all(){ +75- local vmid +76: for vmid in "${ALL_VMIDS[@]}"; do +77: printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\n' "$vmid" "$(expected_owner "$vmid")" "$(observed_owner "$vmid")" +78- done +79-} +80-freshness_decision(){ +-- +116-freshness_plan(){ +117- local vmid now_epoch out decision +118- [[ -x "$PROBE" ]] || die "FRESHNESS_PROBE_NOT_INSTALLED" 69 +119: for vmid in "${ALL_VMIDS[@]}"; do +120- now_epoch="$(date -u +%s)" +121- out="$(freshness_decision "$vmid" "$now_epoch")" +122- printf '%s\n' "$out" +-- +132- exec 8>"$HEAVY_LOCK" +133- flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75 +134- for vmid in "$@"; do +135: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")" +136- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)" +137- printf '%s\n' "$out" +138- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +-- +146-dispatch_one(){ +147- local vmid="$1" freshness decision now_epoch out generation +148- [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64 +149: expected_owner "$vmid" >/dev/null +150- exec 9>"$LOCK" +151- if ! flock -n 9; then +152- printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\n' "$vmid" +-- +170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;; +171- *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;; +172- esac +173: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision" +174- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)" +175- printf '%s\n' "$out" +176- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96 +5-RESTORE_ROOT=/var/lib/homelab-backup/vm-restore-validation +6-HOST="$(hostname -s)" +7-die(){ printf 'VM_FRESHNESS_PROBE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; } +8:expected_owner(){ +9-case "$1" in +10-110|112|150|170|171) printf 'pve01\n' ;; +11-111|113|160|180|9130) printf 'pve02\n' ;; +12-130|190|200) printf 'pve03\n' ;; +13:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +14-esac +15-} +16-assert_local_identity(){ +17-local vmid="$1" owner +18:owner="$(expected_owner "$vmid")" +19-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65 +20-} +21-state_scan(){ +PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 +21-active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ "$rc" == 1 ]] && return 0; die "PGREP_FAILED_$rc" 69; fi; } +22-snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; } +23-vm_lock(){ qm config "$1" | awk -F': ' '$1=="lock"{print $2; found=1} END{if(!found) print "none"}'; } +24:expected_owner(){ +25-case "$1" in +26-110|112|150|170|171) printf 'pve01\n' ;; +27-111|113|160|180|9130) printf 'pve02\n' ;; +28-130|190|200) printf 'pve03\n' ;; +29:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +30-esac +31-} +32:expected_type(){ +33-case "$1" in +34:110|111|112|113|200) printf 'lxc\n' ;; +35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;; +36:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +37-esac +38-} +39-expected_status(){ +-- +45-} +46-assert_local_identity(){ +47-local vmid="$1" owner type status expect_status +48:owner="$(expected_owner "$vmid")" +49:type="$(expected_type "$vmid")" +50-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65 +51-if [[ "$type" == "qemu" ]]; then +52-[[ -s "/etc/pve/qemu-server/${vmid}.conf" ]] || die "QEMU_CONFIG_MISSING_$vmid" 66 +-- +153-plan(){ +154-local vmid="$1" owner type status +155-assert_local_identity "$vmid" +156:owner="$(expected_owner "$vmid")" +157:type="$(expected_type "$vmid")" +158-status="$(expected_status "$vmid")" +159-printf 'PLAN_STATUS=OK\nVMID=%s\nOWNER=%s\nTYPE=%s\nEXPECTED_STATUS=%s\n' "$vmid" "$owner" "$type" "$status" +160-} +PRODUCER_MEMBERSHIP_SOURCE_END=true +VM210_FRESHNESS_DIRECT_RC=64 +VM210_FRESHNESS_STDOUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VM210_FRESHNESS_STDERR_SHA256=b0d9e4d7ad61d749dfaf399ab3ad606613113a88a2c3e7085ff17e2463dc8b9e +VM210_FRESHNESS_NOT_ALLOWED=true +QUEUE_PLAN_RC=0 +QUEUE_PLAN_HAS_VM210=false +QUEUE_PLAN_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +STATIC_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue +STATIC_CONTAINS_210_LITERAL=false +STATIC_CONTAINS_VM9130_LITERAL=true +STATIC_LINE=12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200) +STATIC_LINE=22: *) die "VMID_NOT_ALLOWED_$1" 64 ;; +STATIC_LINE=76: for vmid in "${ALL_VMIDS[@]}"; do +STATIC_LINE=119: for vmid in "${ALL_VMIDS[@]}"; do +STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe +STATIC_CONTAINS_210_LITERAL=false +STATIC_CONTAINS_VM9130_LITERAL=true +STATIC_LINE=13:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker +STATIC_CONTAINS_210_LITERAL=false +STATIC_CONTAINS_VM9130_LITERAL=true +STATIC_LINE=29:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +STATIC_LINE=34:110|111|112|113|200) printf 'lxc\n' ;; +STATIC_LINE=35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;; +STATIC_LINE=36:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +OWNER_STATE|service=activating|sub=start|heavy_lock=BUSY +DECISION=PASS_DR221_416_VM210_PRODUCER_MEMBERSHIP_MISSING_PROVEN +NEXT_GATE=WAIT_OWNER_THEN_REPAIR_VM210_CLOUD_QUORUM_PRODUCER_MEMBERSHIP_AND_FINAL_DR +CHANGES_MADE_BY_416=false +PRODUCT_MUTATION_BY_416=false +VM_MUTATION_BY_416=false +CLOUD_MUTATION_BY_416=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +NEWFI416_END=true + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 1f3cb38e..6ac14e23 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260922-SPB-HOME-INGRESS-CERT-VHOST-FIX-9123R1", - "status": "FAIL", - "rc": 76, + "command_id": "NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "spb-home-ingress-cert-vhost-fix", - "started_at_utc": "2026-09-22T14:03:25Z", - "finished_at_utc": "2026-09-22T14:03:29Z", + "mode": "read-only", + "component": "newfi-dr221-vm210-producer-membership-rca", + "started_at_utc": "2026-09-22T14:05:20Z", + "finished_at_utc": "2026-09-22T14:05:20Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "fbcb4259c7383951413ad138ffac6c866a9b89f23e6a2a3a3d15b96ae602a946", + "command_sha256": "f147173d66cd34c3f641e8e304fe06d95889d3fb0886845c883d37b7494f102f", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, - "result_contract_valid": false, + "change_declared": false, + "result_contract_valid": true, "result_contract_status": null, - "result_contract_error": "INVALID_CONTRACT:ValueError", - "command_rc": 46, - "changes_made": null, - "rollback_started": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, "rollback_restored": null, - "mutation_outcome": "UNKNOWN", + "mutation_outcome": "NO_MUTATION", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "ef0e7c0fe3c4a7aa29d15ee109d164fd9c85355601ad5532b8acac3178f1070d", - "sanitized_output_sha256": "ef0e7c0fe3c4a7aa29d15ee109d164fd9c85355601ad5532b8acac3178f1070d", + "raw_evidence_sha256": "df560dcc7c495f7172790e5839527d5a14299e541ad404e0fee0a82517a452eb", + "sanitized_output_sha256": "4c78ca1b8648c6cbd612ae432ff927d18627ba6573375c6afc3af25de784b531", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "SPB9121_CHANGE={\"preflight\":{\"sha_gate\":\"PASS\",\"data_mount_hash\":\"ec0ac67820f88633\",\"certbot_layout\":\"PASS\",\"new_objects_absent\":\"PASS\",\"nginx_test\":\"PASS\",\"gitread_policy\":\"PASS\",\"gitread_backend_hash\":\"160a494a6106d479\"},\"certs\":{\"git-read.gram1.ru\":{\"certbot_rc\":1,\"stdout_sha\":null,\"stderr_sha\":\"04e28386d4c46be6\"}},\"vhosts\":{},\"probes\":{},\"changes_made\":false,\"rollback_started\":true,\"rollback_restored\":true,\"status\":\"FAIL\",\"rollback\":{\"started\":true,\"reason\":\"CERTBOT_FAIL_git-read.gram1.ru\",\"removed_files\":[],\"cert_delete\":{},\"reload\":{\"test_rc\":0,\"reload_rc\":0,\"stderr_sha\":null},\"restored\":true}}\nTASK_COMPLETE=true\nTASK_RESULT=FAIL_SPB_HOME_INGRESS_CERT_VHOST_FIX\nDECISION=HOLD_SPB_ON_AND_READ_ROLLBACK_RESULT\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-SPB-HOME-INGRESS-CERT-VHOST-FIX-9123R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":true,\"rollback_restored\":true}\n" + "output": "NEWFI416_BEGIN=true\nCOMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1\nMODE=read-only\nCOMPONENT=newfi-dr221-vm210-producer-membership-rca\nSCOPE=PROVE_OR_REFUTE_VM210_MEMBERSHIP_IN_CANONICAL_VM_CLOUD_QUORUM_PRODUCER\nMUTATION_BOUNDARY=NONE;NO_BACKUP;NO_DISPATCH;NO_SYSTEMD_ACTION;NO_CLOUD_WRITE;NO_VM_WRITE;NO_GIT_WRITE\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nFRESHNESS_SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\nNODE_WORKER_SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\nVM210_ALL_TARGET_ROWS=2\nVM210_PVE_TARGET_COUNT=1\nVM210_XF_TARGET_COUNT=1\nVM210_PVE_ENABLED=true\nVM210_XF_ENABLED=true\nPRODUCER_MEMBERSHIP_SOURCE_BEGIN=true\nPRODUCER_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue|SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\n9-FRESHNESS_SECONDS=72000\n10-DUE_SECONDS=43200\n11-SSH_BASE=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\n12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)\n13-MIGRATE_VMIDS=(160 170 130 171 180 190 150 9130 111 112 113)\n14-\n15-die(){ printf 'VM_QUORUM_QUEUE_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }\n16-(( DUE_SECONDS > 0 && DUE_SECONDS < FRESHNESS_SECONDS )) || die DUE_THRESHOLD_INVALID 65\n17:expected_owner(){\n18- case \"$1\" in\n19- 110|112|150|170|171) printf 'pve01\\n' ;;\n20- 111|113|160|180|9130) printf 'pve02\\n' ;;\n21- 130|190|200) printf 'pve03\\n' ;;\n22: *) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n23- esac\n24-}\n25-observed_owner(){\n--\n45-}\n46-_run_worker_unlocked() {\n47- local vmid=\"$1\" action=\"$2\" generation=\"${3:-}\" owner observed host\n48: owner=\"$(expected_owner \"$vmid\")\"\n49- observed=\"$(observed_owner \"$vmid\")\"\n50- [[ \"$observed\" == \"$owner\" ]] || die \"CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}\" 65\n51- if [[ \"$owner\" == \"pve01\" ]]; then\n--\n61-}\n62-run_probe(){\n63- local vmid=\"$1\" owner observed host\n64: owner=\"$(expected_owner \"$vmid\")\"\n65- observed=\"$(observed_owner \"$vmid\")\"\n66- [[ \"$observed\" == \"$owner\" ]] || die \"CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}\" 65\n67- if [[ \"$owner\" == \"pve01\" ]]; then\n--\n73-}\n74-plan_all(){\n75- local vmid\n76: for vmid in \"${ALL_VMIDS[@]}\"; do\n77: printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$(observed_owner \"$vmid\")\"\n78- done\n79-}\n80-freshness_decision(){\n--\n116-freshness_plan(){\n117- local vmid now_epoch out decision\n118- [[ -x \"$PROBE\" ]] || die \"FRESHNESS_PROBE_NOT_INSTALLED\" 69\n119: for vmid in \"${ALL_VMIDS[@]}\"; do\n120- now_epoch=\"$(date -u +%s)\"\n121- out=\"$(freshness_decision \"$vmid\" \"$now_epoch\")\"\n122- printf '%s\\n' \"$out\"\n--\n132- exec 8>\"$HEAVY_LOCK\"\n133- flock -n 8 || die \"GLOBAL_HEAVY_LOCK_BUSY\" 75\n134- for vmid in \"$@\"; do\n135: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\"\n136- out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n137- printf '%s\\n' \"$out\"\n138- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n--\n146-dispatch_one(){\n147- local vmid=\"$1\" freshness decision now_epoch out generation\n148- [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64\n149: expected_owner \"$vmid\" >/dev/null\n150- exec 9>\"$LOCK\"\n151- if ! flock -n 9; then\n152- printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\\n' \"$vmid\"\n--\n170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\n171- *) die \"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\" 65 ;;\n172- esac\n173: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$decision\"\n174- out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n175- printf '%s\\n' \"$out\"\n176- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nPRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\n5-RESTORE_ROOT=/var/lib/homelab-backup/vm-restore-validation\n6-HOST=\"$(hostname -s)\"\n7-die(){ printf 'VM_FRESHNESS_PROBE_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }\n8:expected_owner(){\n9-case \"$1\" in\n10-110|112|150|170|171) printf 'pve01\\n' ;;\n11-111|113|160|180|9130) printf 'pve02\\n' ;;\n12-130|190|200) printf 'pve03\\n' ;;\n13:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n14-esac\n15-}\n16-assert_local_identity(){\n17-local vmid=\"$1\" owner\n18:owner=\"$(expected_owner \"$vmid\")\"\n19-[[ \"$HOST\" == \"$owner\" ]] || die \"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\" 65\n20-}\n21-state_scan(){\nPRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\n21-active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ \"$rc\" == 1 ]] && return 0; die \"PGREP_FAILED_$rc\" 69; fi; }\n22-snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }\n23-vm_lock(){ qm config \"$1\" | awk -F': ' '$1==\"lock\"{print $2; found=1} END{if(!found) print \"none\"}'; }\n24:expected_owner(){\n25-case \"$1\" in\n26-110|112|150|170|171) printf 'pve01\\n' ;;\n27-111|113|160|180|9130) printf 'pve02\\n' ;;\n28-130|190|200) printf 'pve03\\n' ;;\n29:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n30-esac\n31-}\n32:expected_type(){\n33-case \"$1\" in\n34:110|111|112|113|200) printf 'lxc\\n' ;;\n35:130|150|160|170|171|180|190|9130) printf 'qemu\\n' ;;\n36:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n37-esac\n38-}\n39-expected_status(){\n--\n45-}\n46-assert_local_identity(){\n47-local vmid=\"$1\" owner type status expect_status\n48:owner=\"$(expected_owner \"$vmid\")\"\n49:type=\"$(expected_type \"$vmid\")\"\n50-[[ \"$HOST\" == \"$owner\" ]] || die \"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\" 65\n51-if [[ \"$type\" == \"qemu\" ]]; then\n52-[[ -s \"/etc/pve/qemu-server/${vmid}.conf\" ]] || die \"QEMU_CONFIG_MISSING_$vmid\" 66\n--\n153-plan(){\n154-local vmid=\"$1\" owner type status\n155-assert_local_identity \"$vmid\"\n156:owner=\"$(expected_owner \"$vmid\")\"\n157:type=\"$(expected_type \"$vmid\")\"\n158-status=\"$(expected_status \"$vmid\")\"\n159-printf 'PLAN_STATUS=OK\\nVMID=%s\\nOWNER=%s\\nTYPE=%s\\nEXPECTED_STATUS=%s\\n' \"$vmid\" \"$owner\" \"$type\" \"$status\"\n160-}\nPRODUCER_MEMBERSHIP_SOURCE_END=true\nVM210_FRESHNESS_DIRECT_RC=64\nVM210_FRESHNESS_STDOUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVM210_FRESHNESS_STDERR_SHA256=b0d9e4d7ad61d749dfaf399ab3ad606613113a88a2c3e7085ff17e2463dc8b9e\nVM210_FRESHNESS_NOT_ALLOWED=true\nQUEUE_PLAN_RC=0\nQUEUE_PLAN_HAS_VM210=false\nQUEUE_PLAN_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nSTATIC_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)\nSTATIC_LINE=22: *) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_LINE=76: for vmid in \"${ALL_VMIDS[@]}\"; do\nSTATIC_LINE=119: for vmid in \"${ALL_VMIDS[@]}\"; do\nSTATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=13:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=29:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_LINE=34:110|111|112|113|200) printf 'lxc\\n' ;;\nSTATIC_LINE=35:130|150|160|170|171|180|190|9130) printf 'qemu\\n' ;;\nSTATIC_LINE=36:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nOWNER_STATE|service=activating|sub=start|heavy_lock=BUSY\nDECISION=PASS_DR221_416_VM210_PRODUCER_MEMBERSHIP_MISSING_PROVEN\nNEXT_GATE=WAIT_OWNER_THEN_REPAIR_VM210_CLOUD_QUORUM_PRODUCER_MEMBERSHIP_AND_FINAL_DR\nCHANGES_MADE_BY_416=false\nPRODUCT_MUTATION_BY_416=false\nVM_MUTATION_BY_416=false\nCLOUD_MUTATION_BY_416=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nNEWFI416_END=true\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 7bd4bb35..9d6c8458 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,37 +1,216 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260922-SPB-HOME-INGRESS-CERT-VHOST-FIX-9123R1 -STATUS=FAIL -RC=76 +COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=spb-home-ingress-cert-vhost-fix +MODE=read-only +COMPONENT=newfi-dr221-vm210-producer-membership-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=fbcb4259c7383951413ad138ffac6c866a9b89f23e6a2a3a3d15b96ae602a946 +COMMAND_SHA256=f147173d66cd34c3f641e8e304fe06d95889d3fb0886845c883d37b7494f102f DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true -RESULT_CONTRACT_VALID=false -RESULT_CONTRACT_STATUS=UNKNOWN -RESULT_CONTRACT_ERROR=INVALID_CONTRACT:ValueError -COMMAND_RC=46 -CHANGES_MADE=unknown -ROLLBACK_STARTED=unknown +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=UNKNOWN +MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=ef0e7c0fe3c4a7aa29d15ee109d164fd9c85355601ad5532b8acac3178f1070d -SANITIZED_OUTPUT_SHA256=ef0e7c0fe3c4a7aa29d15ee109d164fd9c85355601ad5532b8acac3178f1070d +RAW_EVIDENCE_SHA256=df560dcc7c495f7172790e5839527d5a14299e541ad404e0fee0a82517a452eb +SANITIZED_OUTPUT_SHA256=4c78ca1b8648c6cbd612ae432ff927d18627ba6573375c6afc3af25de784b531 OUTPUT_BEGIN -SPB9121_CHANGE={"preflight":{"sha_gate":"PASS","data_mount_hash":"ec0ac67820f88633","certbot_layout":"PASS","new_objects_absent":"PASS","nginx_test":"PASS","gitread_policy":"PASS","gitread_backend_hash":"160a494a6106d479"},"certs":{"git-read.gram1.ru":{"certbot_rc":1,"stdout_sha":null,"stderr_sha":"04e28386d4c46be6"}},"vhosts":{},"probes":{},"changes_made":false,"rollback_started":true,"rollback_restored":true,"status":"FAIL","rollback":{"started":true,"reason":"CERTBOT_FAIL_git-read.gram1.ru","removed_files":[],"cert_delete":{},"reload":{"test_rc":0,"reload_rc":0,"stderr_sha":null},"restored":true}} -TASK_COMPLETE=true -TASK_RESULT=FAIL_SPB_HOME_INGRESS_CERT_VHOST_FIX -DECISION=HOLD_SPB_ON_AND_READ_ROLLBACK_RESULT -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-SPB-HOME-INGRESS-CERT-VHOST-FIX-9123R1","status":"FAIL","changes_made":false,"rollback_started":true,"rollback_restored":true} +NEWFI416_BEGIN=true +COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1 +MODE=read-only +COMPONENT=newfi-dr221-vm210-producer-membership-rca +SCOPE=PROVE_OR_REFUTE_VM210_MEMBERSHIP_IN_CANONICAL_VM_CLOUD_QUORUM_PRODUCER +MUTATION_BOUNDARY=NONE;NO_BACKUP;NO_DISPATCH;NO_SYSTEMD_ACTION;NO_CLOUD_WRITE;NO_VM_WRITE;NO_GIT_WRITE +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +QUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 +FRESHNESS_SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96 +NODE_WORKER_SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 +VM210_ALL_TARGET_ROWS=2 +VM210_PVE_TARGET_COUNT=1 +VM210_XF_TARGET_COUNT=1 +VM210_PVE_ENABLED=true +VM210_XF_ENABLED=true +PRODUCER_MEMBERSHIP_SOURCE_BEGIN=true +PRODUCER_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue|SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 +9-FRESHNESS_SECONDS=72000 +10-DUE_SECONDS=43200 +11-SSH_BASE=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] +12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200) +13-MIGRATE_VMIDS=(160 170 130 171 180 190 150 9130 111 112 113) +14- +15-die(){ printf 'VM_QUORUM_QUEUE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; } +16-(( DUE_SECONDS > 0 && DUE_SECONDS < FRESHNESS_SECONDS )) || die DUE_THRESHOLD_INVALID 65 +17:expected_owner(){ +18- case "$1" in +19- 110|112|150|170|171) printf 'pve01\n' ;; +20- 111|113|160|180|9130) printf 'pve02\n' ;; +21- 130|190|200) printf 'pve03\n' ;; +22: *) die "VMID_NOT_ALLOWED_$1" 64 ;; +23- esac +24-} +25-observed_owner(){ +-- +45-} +46-_run_worker_unlocked() { +47- local vmid="$1" action="$2" generation="${3:-}" owner observed host +48: owner="$(expected_owner "$vmid")" +49- observed="$(observed_owner "$vmid")" +50- [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65 +51- if [[ "$owner" == "pve01" ]]; then +-- +61-} +62-run_probe(){ +63- local vmid="$1" owner observed host +64: owner="$(expected_owner "$vmid")" +65- observed="$(observed_owner "$vmid")" +66- [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65 +67- if [[ "$owner" == "pve01" ]]; then +-- +73-} +74-plan_all(){ +75- local vmid +76: for vmid in "${ALL_VMIDS[@]}"; do +77: printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\n' "$vmid" "$(expected_owner "$vmid")" "$(observed_owner "$vmid")" +78- done +79-} +80-freshness_decision(){ +-- +116-freshness_plan(){ +117- local vmid now_epoch out decision +118- [[ -x "$PROBE" ]] || die "FRESHNESS_PROBE_NOT_INSTALLED" 69 +119: for vmid in "${ALL_VMIDS[@]}"; do +120- now_epoch="$(date -u +%s)" +121- out="$(freshness_decision "$vmid" "$now_epoch")" +122- printf '%s\n' "$out" +-- +132- exec 8>"$HEAVY_LOCK" +133- flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75 +134- for vmid in "$@"; do +135: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")" +136- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)" +137- printf '%s\n' "$out" +138- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +-- +146-dispatch_one(){ +147- local vmid="$1" freshness decision now_epoch out generation +148- [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64 +149: expected_owner "$vmid" >/dev/null +150- exec 9>"$LOCK" +151- if ! flock -n 9; then +152- printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\n' "$vmid" +-- +170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;; +171- *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;; +172- esac +173: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision" +174- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)" +175- printf '%s\n' "$out" +176- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96 +5-RESTORE_ROOT=/var/lib/homelab-backup/vm-restore-validation +6-HOST="$(hostname -s)" +7-die(){ printf 'VM_FRESHNESS_PROBE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; } +8:expected_owner(){ +9-case "$1" in +10-110|112|150|170|171) printf 'pve01\n' ;; +11-111|113|160|180|9130) printf 'pve02\n' ;; +12-130|190|200) printf 'pve03\n' ;; +13:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +14-esac +15-} +16-assert_local_identity(){ +17-local vmid="$1" owner +18:owner="$(expected_owner "$vmid")" +19-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65 +20-} +21-state_scan(){ +PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 +21-active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ "$rc" == 1 ]] && return 0; die "PGREP_FAILED_$rc" 69; fi; } +22-snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; } +23-vm_lock(){ qm config "$1" | awk -F': ' '$1=="lock"{print $2; found=1} END{if(!found) print "none"}'; } +24:expected_owner(){ +25-case "$1" in +26-110|112|150|170|171) printf 'pve01\n' ;; +27-111|113|160|180|9130) printf 'pve02\n' ;; +28-130|190|200) printf 'pve03\n' ;; +29:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +30-esac +31-} +32:expected_type(){ +33-case "$1" in +34:110|111|112|113|200) printf 'lxc\n' ;; +35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;; +36:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +37-esac +38-} +39-expected_status(){ +-- +45-} +46-assert_local_identity(){ +47-local vmid="$1" owner type status expect_status +48:owner="$(expected_owner "$vmid")" +49:type="$(expected_type "$vmid")" +50-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65 +51-if [[ "$type" == "qemu" ]]; then +52-[[ -s "/etc/pve/qemu-server/${vmid}.conf" ]] || die "QEMU_CONFIG_MISSING_$vmid" 66 +-- +153-plan(){ +154-local vmid="$1" owner type status +155-assert_local_identity "$vmid" +156:owner="$(expected_owner "$vmid")" +157:type="$(expected_type "$vmid")" +158-status="$(expected_status "$vmid")" +159-printf 'PLAN_STATUS=OK\nVMID=%s\nOWNER=%s\nTYPE=%s\nEXPECTED_STATUS=%s\n' "$vmid" "$owner" "$type" "$status" +160-} +PRODUCER_MEMBERSHIP_SOURCE_END=true +VM210_FRESHNESS_DIRECT_RC=64 +VM210_FRESHNESS_STDOUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +VM210_FRESHNESS_STDERR_SHA256=b0d9e4d7ad61d749dfaf399ab3ad606613113a88a2c3e7085ff17e2463dc8b9e +VM210_FRESHNESS_NOT_ALLOWED=true +QUEUE_PLAN_RC=0 +QUEUE_PLAN_HAS_VM210=false +QUEUE_PLAN_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +STATIC_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue +STATIC_CONTAINS_210_LITERAL=false +STATIC_CONTAINS_VM9130_LITERAL=true +STATIC_LINE=12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200) +STATIC_LINE=22: *) die "VMID_NOT_ALLOWED_$1" 64 ;; +STATIC_LINE=76: for vmid in "${ALL_VMIDS[@]}"; do +STATIC_LINE=119: for vmid in "${ALL_VMIDS[@]}"; do +STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe +STATIC_CONTAINS_210_LITERAL=false +STATIC_CONTAINS_VM9130_LITERAL=true +STATIC_LINE=13:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker +STATIC_CONTAINS_210_LITERAL=false +STATIC_CONTAINS_VM9130_LITERAL=true +STATIC_LINE=29:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +STATIC_LINE=34:110|111|112|113|200) printf 'lxc\n' ;; +STATIC_LINE=35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;; +STATIC_LINE=36:*) die "VMID_NOT_ALLOWED_$1" 64 ;; +OWNER_STATE|service=activating|sub=start|heavy_lock=BUSY +DECISION=PASS_DR221_416_VM210_PRODUCER_MEMBERSHIP_MISSING_PROVEN +NEXT_GATE=WAIT_OWNER_THEN_REPAIR_VM210_CLOUD_QUORUM_PRODUCER_MEMBERSHIP_AND_FINAL_DR +CHANGES_MADE_BY_416=false +PRODUCT_MUTATION_BY_416=false +VM_MUTATION_BY_416=false +CLOUD_MUTATION_BY_416=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +NEWFI416_END=true OUTPUT_END CHAT_OUTPUT_END