homelab-public-context/runtime/latest.json

{
  "schema_version": 1,
  "channel": "homelab-runtime",
  "command_id": "NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1",
  "status": "OK",
  "rc": 0,
  "host": "pve01",
  "mode": "read-only",
  "component": "newfi-dr221-vm210-producer-membership-rca",
  "started_at_utc": "2026-09-22T14:05:20Z",
  "finished_at_utc": "2026-09-22T14:05:20Z",
  "reference_register_checked": true,
  "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
  "error_register_checked": true,
  "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
  "command_sha256": "f147173d66cd34c3f641e8e304fe06d95889d3fb0886845c883d37b7494f102f",
  "duplicate_failed_command_blocked": false,
  "block_reason": null,
  "execution_started": true,
  "change_declared": false,
  "result_contract_valid": true,
  "result_contract_status": null,
  "result_contract_error": null,
  "command_rc": 0,
  "changes_made": false,
  "rollback_started": false,
  "rollback_restored": null,
  "mutation_outcome": "NO_MUTATION",
  "sanitized": true,
  "secrets_included": false,
  "private_addresses_included": false,
  "raw_evidence_retained_locally": true,
  "raw_evidence_sha256": "df560dcc7c495f7172790e5839527d5a14299e541ad404e0fee0a82517a452eb",
  "sanitized_output_sha256": "4c78ca1b8648c6cbd612ae432ff927d18627ba6573375c6afc3af25de784b531",
  "output_truncated_in_json": false,
  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
  "output": "NEWFI416_BEGIN=true\nCOMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1\nMODE=read-only\nCOMPONENT=newfi-dr221-vm210-producer-membership-rca\nSCOPE=PROVE_OR_REFUTE_VM210_MEMBERSHIP_IN_CANONICAL_VM_CLOUD_QUORUM_PRODUCER\nMUTATION_BOUNDARY=NONE;NO_BACKUP;NO_DISPATCH;NO_SYSTEMD_ACTION;NO_CLOUD_WRITE;NO_VM_WRITE;NO_GIT_WRITE\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nFRESHNESS_SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\nNODE_WORKER_SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\nVM210_ALL_TARGET_ROWS=2\nVM210_PVE_TARGET_COUNT=1\nVM210_XF_TARGET_COUNT=1\nVM210_PVE_ENABLED=true\nVM210_XF_ENABLED=true\nPRODUCER_MEMBERSHIP_SOURCE_BEGIN=true\nPRODUCER_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue|SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\n9-FRESHNESS_SECONDS=72000\n10-DUE_SECONDS=43200\n11-SSH_BASE=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\n12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)\n13-MIGRATE_VMIDS=(160 170 130 171 180 190 150 9130 111 112 113)\n14-\n15-die(){ printf 'VM_QUORUM_QUEUE_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }\n16-(( DUE_SECONDS > 0 && DUE_SECONDS < FRESHNESS_SECONDS )) || die DUE_THRESHOLD_INVALID 65\n17:expected_owner(){\n18-  case \"$1\" in\n19-    110|112|150|170|171) printf 'pve01\\n' ;;\n20-    111|113|160|180|9130) printf 'pve02\\n' ;;\n21-    130|190|200) printf 'pve03\\n' ;;\n22:    *) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n23-  esac\n24-}\n25-observed_owner(){\n--\n45-}\n46-_run_worker_unlocked() {\n47-  local vmid=\"$1\" action=\"$2\" generation=\"${3:-}\" owner observed host\n48:  owner=\"$(expected_owner \"$vmid\")\"\n49-  observed=\"$(observed_owner \"$vmid\")\"\n50-  [[ \"$observed\" == \"$owner\" ]] || die \"CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}\" 65\n51-  if [[ \"$owner\" == \"pve01\" ]]; then\n--\n61-}\n62-run_probe(){\n63-  local vmid=\"$1\" owner observed host\n64:  owner=\"$(expected_owner \"$vmid\")\"\n65-  observed=\"$(observed_owner \"$vmid\")\"\n66-  [[ \"$observed\" == \"$owner\" ]] || die \"CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}\" 65\n67-  if [[ \"$owner\" == \"pve01\" ]]; then\n--\n73-}\n74-plan_all(){\n75-  local vmid\n76:  for vmid in \"${ALL_VMIDS[@]}\"; do\n77:    printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$(observed_owner \"$vmid\")\"\n78-  done\n79-}\n80-freshness_decision(){\n--\n116-freshness_plan(){\n117-  local vmid now_epoch out decision\n118-  [[ -x \"$PROBE\" ]] || die \"FRESHNESS_PROBE_NOT_INSTALLED\" 69\n119:  for vmid in \"${ALL_VMIDS[@]}\"; do\n120-    now_epoch=\"$(date -u +%s)\"\n121-    out=\"$(freshness_decision \"$vmid\" \"$now_epoch\")\"\n122-    printf '%s\\n' \"$out\"\n--\n132-  exec 8>\"$HEAVY_LOCK\"\n133-  flock -n 8 || die \"GLOBAL_HEAVY_LOCK_BUSY\" 75\n134-  for vmid in \"$@\"; do\n135:    printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\"\n136-    out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n137-    printf '%s\\n' \"$out\"\n138-    generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n--\n146-dispatch_one(){\n147-  local vmid=\"$1\" freshness decision now_epoch out generation\n148-  [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64\n149:  expected_owner \"$vmid\" >/dev/null\n150-  exec 9>\"$LOCK\"\n151-  if ! flock -n 9; then\n152-    printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\\n' \"$vmid\"\n--\n170-    BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\n171-    *) die \"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\" 65 ;;\n172-  esac\n173:  printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$decision\"\n174-  out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n175-  printf '%s\\n' \"$out\"\n176-  generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nPRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\n5-RESTORE_ROOT=/var/lib/homelab-backup/vm-restore-validation\n6-HOST=\"$(hostname -s)\"\n7-die(){ printf 'VM_FRESHNESS_PROBE_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }\n8:expected_owner(){\n9-case \"$1\" in\n10-110|112|150|170|171) printf 'pve01\\n' ;;\n11-111|113|160|180|9130) printf 'pve02\\n' ;;\n12-130|190|200) printf 'pve03\\n' ;;\n13:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n14-esac\n15-}\n16-assert_local_identity(){\n17-local vmid=\"$1\" owner\n18:owner=\"$(expected_owner \"$vmid\")\"\n19-[[ \"$HOST\" == \"$owner\" ]] || die \"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\" 65\n20-}\n21-state_scan(){\nPRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\n21-active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ \"$rc\" == 1 ]] && return 0; die \"PGREP_FAILED_$rc\" 69; fi; }\n22-snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }\n23-vm_lock(){ qm config \"$1\" | awk -F': ' '$1==\"lock\"{print $2; found=1} END{if(!found) print \"none\"}'; }\n24:expected_owner(){\n25-case \"$1\" in\n26-110|112|150|170|171) printf 'pve01\\n' ;;\n27-111|113|160|180|9130) printf 'pve02\\n' ;;\n28-130|190|200) printf 'pve03\\n' ;;\n29:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n30-esac\n31-}\n32:expected_type(){\n33-case \"$1\" in\n34:110|111|112|113|200) printf 'lxc\\n' ;;\n35:130|150|160|170|171|180|190|9130) printf 'qemu\\n' ;;\n36:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n37-esac\n38-}\n39-expected_status(){\n--\n45-}\n46-assert_local_identity(){\n47-local vmid=\"$1\" owner type status expect_status\n48:owner=\"$(expected_owner \"$vmid\")\"\n49:type=\"$(expected_type \"$vmid\")\"\n50-[[ \"$HOST\" == \"$owner\" ]] || die \"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\" 65\n51-if [[ \"$type\" == \"qemu\" ]]; then\n52-[[ -s \"/etc/pve/qemu-server/${vmid}.conf\" ]] || die \"QEMU_CONFIG_MISSING_$vmid\" 66\n--\n153-plan(){\n154-local vmid=\"$1\" owner type status\n155-assert_local_identity \"$vmid\"\n156:owner=\"$(expected_owner \"$vmid\")\"\n157:type=\"$(expected_type \"$vmid\")\"\n158-status=\"$(expected_status \"$vmid\")\"\n159-printf 'PLAN_STATUS=OK\\nVMID=%s\\nOWNER=%s\\nTYPE=%s\\nEXPECTED_STATUS=%s\\n' \"$vmid\" \"$owner\" \"$type\" \"$status\"\n160-}\nPRODUCER_MEMBERSHIP_SOURCE_END=true\nVM210_FRESHNESS_DIRECT_RC=64\nVM210_FRESHNESS_STDOUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVM210_FRESHNESS_STDERR_SHA256=b0d9e4d7ad61d749dfaf399ab3ad606613113a88a2c3e7085ff17e2463dc8b9e\nVM210_FRESHNESS_NOT_ALLOWED=true\nQUEUE_PLAN_RC=0\nQUEUE_PLAN_HAS_VM210=false\nQUEUE_PLAN_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nSTATIC_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)\nSTATIC_LINE=22:    *) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_LINE=76:  for vmid in \"${ALL_VMIDS[@]}\"; do\nSTATIC_LINE=119:  for vmid in \"${ALL_VMIDS[@]}\"; do\nSTATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=13:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=29:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_LINE=34:110|111|112|113|200) printf 'lxc\\n' ;;\nSTATIC_LINE=35:130|150|160|170|171|180|190|9130) printf 'qemu\\n' ;;\nSTATIC_LINE=36:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nOWNER_STATE|service=activating|sub=start|heavy_lock=BUSY\nDECISION=PASS_DR221_416_VM210_PRODUCER_MEMBERSHIP_MISSING_PROVEN\nNEXT_GATE=WAIT_OWNER_THEN_REPAIR_VM210_CLOUD_QUORUM_PRODUCER_MEMBERSHIP_AND_FINAL_DR\nCHANGES_MADE_BY_416=false\nPRODUCT_MUTATION_BY_416=false\nVM_MUTATION_BY_416=false\nCLOUD_MUTATION_BY_416=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nNEWFI416_END=true\n"
}