homelab-runtime-publisher
39 lines
42 KiB
JSON
39 lines
42 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"channel": "homelab-runtime",
|
|
"command_id": "SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1",
|
|
"status": "OK",
|
|
"rc": 0,
|
|
"host": "pve01",
|
|
"mode": "read-only",
|
|
"component": "homelab-backup-vm9130-rca",
|
|
"started_at_utc": "2026-09-21T03:53:27Z",
|
|
"finished_at_utc": "2026-09-21T03:53:38Z",
|
|
"reference_register_checked": true,
|
|
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
|
|
"error_register_checked": true,
|
|
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
|
|
"command_sha256": "a15d8b5f33489f8d1eb5674ec7b4201fe98d7a2525f0dcb5d5cf67b6b072a193",
|
|
"duplicate_failed_command_blocked": false,
|
|
"block_reason": null,
|
|
"execution_started": true,
|
|
"change_declared": false,
|
|
"result_contract_valid": true,
|
|
"result_contract_status": null,
|
|
"result_contract_error": null,
|
|
"command_rc": 0,
|
|
"changes_made": false,
|
|
"rollback_started": false,
|
|
"rollback_restored": null,
|
|
"mutation_outcome": "NO_MUTATION",
|
|
"sanitized": true,
|
|
"secrets_included": false,
|
|
"private_addresses_included": false,
|
|
"raw_evidence_retained_locally": true,
|
|
"raw_evidence_sha256": "dec8f24d8c18ccf29284612a9706bdfba40963983766a7e3552aca0819069bbd",
|
|
"sanitized_output_sha256": "2d97d7e8351c12124c172903bbb35fcbec82ac9dc28d5ffcd538b8b1c5eca523",
|
|
"output_truncated_in_json": false,
|
|
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
|
"output": "WORKERS2_BACKUP_VM9130_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1\nMODE=read-only\nCOMPONENT=homelab-backup-vm9130-rca\nHOST=pve01\nUID=0\nMUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL\nWORKTREE_HEAD=47b9be72cf2487ba66295d7da998016bbeea57be\nWORKTREE_HEAD_EXPECTED=true\nWORKTREE_CLEAN=true\nWORKTREE_STATUS_LINES=[]\nAPP_FRESHNESS_PLAN={\"lines\":[\"QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\"],\"rc\":0,\"stderr\":[]}\nVM9130_STATE={\"age_seconds\":109251,\"consecutive_failures\":0,\"exists\":true,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm9130-20260919T210758Z-0aa987efac8c\",\"last_attempt_epoch\":1789853557,\"last_success_epoch\":1789853557,\"logical_id\":\"vm9130\",\"replica_count\":2,\"result\":\"SUCCESS\",\"sha256\":\"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db\"}\nVM_QUEUE_FRESHNESS_PLAN={\"rc\":65,\"stderr\":[\"VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\",\"VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\"],\"stdout_sha256\":\"251ff72648a76f16b91a5f77b421a20fab3f0113e79c118408803793f4befa16\",\"vm9130_lines\":[]}\nDISPATCHER_PLAN={\"lines\":[\"FRESHNESS_STATUS=OK\",\"VMID=160\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm160-pve02-20260916T023620Z\",\"UPDATED_AT_UTC=2026-09-16T02:46:28Z\",\"SEALED_EPOCH=1789526788\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm160/vm160-pve02-20260916T023620Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json\",\"QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_STALE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=436023 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_STALE\",\"FRESHNESS_STATUS=OK\",\"VMID=170\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm170-pve01-20260916T103019Z\",\"UPDATED_AT_UTC=2026-09-16T10:47:15Z\",\"SEALED_EPOCH=1789555635\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm170/vm170-pve01-20260916T103019Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json\",\"QUEUE_FRESHNESS VMID=170 DECISION=BACKUP_STALE GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=407177 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=170 DECISION=BACKUP_STALE\",\"FRESHNESS_STATUS=OK\",\"VMID=130\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm130-pve03-20260916T105207Z\",\"UPDATED_AT_UTC=2026-09-16T11:25:25Z\",\"SEALED_EPOCH=1789557925\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm130/vm130-pve03-20260916T105207Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json\",\"QUEUE_FRESHNESS VMID=130 DECISION=BACKUP_STALE GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=404888 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=130 DECISION=BACKUP_STALE\",\"FRESHNESS_STATUS=OK\",\"VMID=171\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm171-pve01-20260916T023018Z\",\"UPDATED_AT_UTC=2026-09-16T02:35:03Z\",\"SEALED_EPOCH=1789526103\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\",\"QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_STALE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=436710 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_STALE\",\"CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_STALE PRIORITY=2 AGE_SECONDS=436710\",\"FRESHNESS_STATUS=OK\",\"APP=filebrowser\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-filebrowser-pve01-20260921T033108Z\",\"UPDATED_AT_UTC=2026-09-21T03:31:18Z\",\"SEALED_EPOCH=1789961478\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260921T033108Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260921T033108Z.json\",\"QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260921T033108Z AGE_SECONDS=1337 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\",\"FRESHNESS_STATUS=OK\",\"APP=skladchik-reports-monitor\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"UPDATED_AT_UTC=2026-09-21T03:31:48Z\",\"SEALED_EPOCH=1789961508\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260921T033135Z AGE_SECONDS=1307 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\",\"FRESHNESS_STATUS=OK\",\"APP=immich\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-immich-pve01-20260921T033154Z\",\"UPDATED_AT_UTC=2026-09-21T03:32:40Z\",\"SEALED_EPOCH=1789961560\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260921T033154Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260921T033154Z.json\",\"QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260921T033154Z AGE_SECONDS=1255 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\",\"FRESHNESS_STATUS=OK\",\"APP=hbarhub\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-hbarhub-pve01-20260921T033309Z\",\"UPDATED_AT_UTC=2026-09-21T03:34:20Z\",\"SEALED_EPOCH=1789961660\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260921T033309Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260921T033309Z.json\",\"QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260921T033309Z AGE_SECONDS=1155 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\",\"CLASS_DISPATCH KIND=APP DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1\",\"GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_STALE PRIORITY=2 AGE_SECONDS=436710\"],\"rc\":0,\"stderr\":[\"VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\",\"VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\"],\"stdout_sha256\":\"beaf5c144e0d201cbeef18657cd58484b241ff667de66cb9a6483d41cd29ef0b\"}\nVM9130_REGISTRY_ENTRIES=[{\"id\":\"9130\",\"kind\":\"VM\",\"lifecycle\":\"ACTIVE\",\"logical_id\":\"vm9130\",\"owner\":\"pve02\"}]\nVM9130_CLUSTER_RESOURCE={\"name\":\"edge-cold-standby\",\"node\":\"pve02\",\"status\":\"stopped\",\"template\":0,\"type\":\"qemu\",\"vmid\":9130}\nVM9130_OWNER_READONLY={\"lines\":[\"pve02\",\"status: stopped\",\"agent: enabled=1\",\"boot: order=scsi0\",\"ciuser: debian\",\"cores: 4\",\"cpu: host\",\"description: CR0023 cold standby of VM130. Keep stopped; all NICs link_down=1.\",\"ide2: local-lvm:vm-9130-cloudinit,media=cdrom,size=4M\",\"ipconfig0: ip=[PRIVATE_IP]/24,gw=[PRIVATE_IP]\",\"memory: 12288\",\"meta: creation-qemu=11.0.0,ctime=1781460976\",\"name: edge-cold-standby\",\"nameserver: [PRIVATE_IP] [PRIVATE_IP]\",\"net0: virtio=BC:24:11:7F:04:D6,bridge=vmbr0,firewall=1,link_down=1\",\"onboot: 0\",\"ostype: l26\",\"protection: 1\",\"scsi0: local-lvm:vm-9130-disk-0,discard=on,size=96G,ssd=1\",\"scsi1: local-lvm:vm-9130-disk-1,backup=1,discard=on,size=150G,ssd=1\",\"scsihw: virtio-scsi-pci\",\"searchdomain: gram1.ru\",\"serial0: socket\",\"smbios1: uuid=9327a861-e9f4-4450-b13f-a1e767a83c86\",\"sshkeys: ssh-rsa%20AAAAB3NzaC1yc2EAAAADAQABAAACAQC13sgRxsdexPSyrcz3pfeVJoGajhrb3mJI2gdB2HNWOj6SS%2FcHHiJBKUjg80s6rhZJD%2FtpyLOj%2BQbrkJR3L3FdrgwgCvPcYrBujuZ2TLaRIKIDlNilnRkmVQmUBahgAyrnaQBBl1DVFQsfF0NLF0PYjvPF3789giWqCfv4L14vJSFxvp3wEOjkY1hiwZ8jffMD0R99UW4FTqm29KEFx5ywt%2BdrOT6sMtkBYCIR%2F5JGCB0riH%2F9xn904TIasSMBDz2IeEFpO73bd72JtmnxFyaWaBllKn%2Bu9t18ahMx5vSn2UerGGZbbrZBigxNIwOaHdvdyxADSR3QU90T6nzUv5Fov3vIoOt4uhVaMsYew25hpzm8%2Byp77liUu9mfuMWzo0Q7XJrNfhuCJ9pcrBeqopdThFC36c72JgksWT9Lhfef7bMbEa%2BIWF0wL3L%2FALp1QVA1cbQWPVAF1x3UEHnXuxJCj2x8z4Xli2SJu7TS16qF9Lv1GqVYMeZvW48LwXSbwWcui3B4ldSuo5ksmPeggcTuHzxy5TyQk9sJjlz10Pw9A5zbbuYZDo56n17c1couUm1a3EQ25CimrECygWX2OKcAEdmoiWAihKHyAuLFnSRUELbNrFcFJIb4JkrGk7edWdF%2Btw3jOHlHDcJew08xyoutlN1P9iFTrotMbQjyNTrGgQ%3D%3D%20root%40pve01%0A\",\"startup: order=40\",\"vga: serial0\",\"vmgenid: 110339cf-08fb-4da8-bc7d-2265b8080f2c\",\"91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 /usr/local/libexec/homelab-vm-cloud-quorum-node-worker\"],\"rc\":0,\"stderr\":[]}\nVM_QUEUE_DISPATCH_SOURCE={\"lines_140_220\":[{\"line\":140,\"text\":\" printf 'QUEUE_SEALED VMID=%s GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":141,\"text\":\" done\"},{\"line\":142,\"text\":\"}\"},{\"line\":143,\"text\":\"dispatch_plan(){\"},{\"line\":144,\"text\":\" freshness_plan\"},{\"line\":145,\"text\":\"}\"},{\"line\":146,\"text\":\"dispatch_one(){\"},{\"line\":147,\"text\":\" local vmid=\\\"$1\\\" freshness decision now_epoch out generation\"},{\"line\":148,\"text\":\" [[ \\\"${HOMELAB_BACKUP_DISPATCHER:-}\\\" == 1 ]] || die \\\"DISPATCH_ONE_DIRECT_FORBIDDEN\\\" 64\"},{\"line\":149,\"text\":\" expected_owner \\\"$vmid\\\" >/dev/null\"},{\"line\":150,\"text\":\" exec 9>\\\"$LOCK\\\"\"},{\"line\":151,\"text\":\" if ! flock -n 9; then\"},{\"line\":152,\"text\":\" printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\\\\n' \\\"$vmid\\\"\"},{\"line\":153,\"text\":\" return 0\"},{\"line\":154,\"text\":\" fi\"},{\"line\":155,\"text\":\" exec 8>\\\"$HEAVY_LOCK\\\"\"},{\"line\":156,\"text\":\" if ! flock -n 8; then\"},{\"line\":157,\"text\":\" printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_HEAVY_LOCK_BUSY VMID=%s\\\\n' \\\"$vmid\\\"\"},{\"line\":158,\"text\":\" return 0\"},{\"line\":159,\"text\":\" fi\"},{\"line\":160,\"text\":\" now_epoch=\\\"$(date -u +%s)\\\"\"},{\"line\":161,\"text\":\" freshness=\\\"$(exec 8>&-; exec 9>&-; freshness_decision \\\"$vmid\\\" \\\"$now_epoch\\\")\\\"\"},{\"line\":162,\"text\":\" printf '%s\\\\n' \\\"$freshness\\\"\"},{\"line\":163,\"text\":\" decision=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":164,\"text\":\" case \\\"$decision\\\" in\"},{\"line\":165,\"text\":\" SKIP_FRESH)\"},{\"line\":166,\"text\":\" generation=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":167,\"text\":\" printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SKIP_FRESH GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":168,\"text\":\" return 0\"},{\"line\":169,\"text\":\" ;;\"},{\"line\":170,\"text\":\" BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\"},{\"line\":171,\"text\":\" *) die \\\"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\\\" 65 ;;\"},{\"line\":172,\"text\":\" esac\"},{\"line\":173,\"text\":\" printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\\\n' \\\"$vmid\\\" \\\"$(expected_owner \\\"$vmid\\\")\\\" \\\"$decision\\\"\"},{\"line\":174,\"text\":\" out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":175,\"text\":\" printf '%s\\\\n' \\\"$out\\\"\"},{\"line\":176,\"text\":\" generation=\\\"$(awk -F= '$1==\\\"GENERATION\\\"{print $2}' <<<\\\"$out\\\" | tail -1)\\\"\"},{\"line\":177,\"text\":\" [[ -n \\\"$generation\\\" ]] || die \\\"GENERATION_NOT_RETURNED_$vmid\\\" 65\"},{\"line\":178,\"text\":\" printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":179,\"text\":\" printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":180,\"text\":\"}\"},{\"line\":181,\"text\":\"\"},{\"line\":182,\"text\":\"ACTION=\\\"${1:-}\\\"\"},{\"line\":183,\"text\":\"case \\\"$ACTION\\\" in\"},{\"line\":184,\"text\":\" plan)\"},{\"line\":185,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_PLAN\\\" 64\"},{\"line\":186,\"text\":\" plan_all\"},{\"line\":187,\"text\":\" ;;\"},{\"line\":188,\"text\":\" freshness-plan)\"},{\"line\":189,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_FRESHNESS_PLAN\\\" 64\"},{\"line\":190,\"text\":\" freshness_plan\"},{\"line\":191,\"text\":\" ;;\"},{\"line\":192,\"text\":\" dispatch-plan)\"},{\"line\":193,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_DISPATCH_PLAN\\\" 64\"},{\"line\":194,\"text\":\" dispatch_plan\"},{\"line\":195,\"text\":\" ;;\"},{\"line\":196,\"text\":\" dispatch-one)\"},{\"line\":197,\"text\":\" [[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_DISPATCH_ONE\\\" 64\"},{\"line\":198,\"text\":\" dispatch_one \\\"$2\\\"\"},{\"line\":199,\"text\":\" ;;\"},{\"line\":200,\"text\":\" canary)\"},{\"line\":201,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_CANARY\\\" 64\"},{\"line\":202,\"text\":\" backup_list_force 110\"},{\"line\":203,\"text\":\" ;;\"},{\"line\":204,\"text\":\" migrate)\"},{\"line\":205,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_MIGRATE\\\" 64\"},{\"line\":206,\"text\":\" backup_list_force \\\"${MIGRATE_VMIDS[@]}\\\"\"},{\"line\":207,\"text\":\" ;;\"},{\"line\":208,\"text\":\" all)\"},{\"line\":209,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_ALL\\\" 64\"},{\"line\":210,\"text\":\" [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die \\\"DISPATCHER_MISSING\\\" 69\"},{\"line\":211,\"text\":\" /usr/local/sbin/homelab-backup-dispatcher run VM_TIMER\"},{\"line\":212,\"text\":\" ;;\"},{\"line\":213,\"text\":\" verify)\"},{\"line\":214,\"text\":\" [[ \\\"$#\\\" == 3 ]] || die \\\"USAGE_VERIFY\\\" 64\"},{\"line\":215,\"text\":\" run_worker \\\"$2\\\" verify \\\"$3\\\"\"},{\"line\":216,\"text\":\" ;;\"},{\"line\":217,\"text\":\" *)\"},{\"line\":218,\"text\":\" die \\\"ACTION_INVALID\\\" 64\"},{\"line\":219,\"text\":\" ;;\"},{\"line\":220,\"text\":\"esac\"}],\"sha256\":\"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\"}\nDISPATCHER_RUN_SOURCE={\"lines_35_60\":[{\"line\":35,\"text\":\"plan(){ select_global; }\"},{\"line\":36,\"text\":\"run(){\"},{\"line\":37,\"text\":\" local trigger=\\\"$1\\\" i=0 actions=0 scan pick decision kind target out rc last_sealed=\\\"\\\"\"},{\"line\":38,\"text\":\" exec 7>\\\"$LOCK\\\"\"},{\"line\":39,\"text\":\" if ! flock -n 7; then printf 'DISPATCHER_DEFERRED TRIGGER=%s REASON=DISPATCHER_LOCK_BUSY\\\\n' \\\"$trigger\\\"; return 0; fi\"},{\"line\":40,\"text\":\" while (( i < MAX_ITERATIONS )); do\"},{\"line\":41,\"text\":\" i=$((i+1)); scan=\\\"$(select_global)\\\"; printf '%s\\\\n' \\\"$scan\\\"; pick=\\\"$(grep '^GLOBAL_DISPATCH ' <<<\\\"$scan\\\" | tail -1)\\\"; [[ -n \\\"$pick\\\" ]] || die GLOBAL_DISPATCH_RESULT_MISSING 65\"},{\"line\":42,\"text\":\" decision=\\\"\\\"; kind=\\\"\\\"; target=\\\"\\\"; for token in $pick; do case \\\"$token\\\" in KIND=*) kind=\\\"${token#KIND=}\\\";; TARGET=*) target=\\\"${token#TARGET=}\\\";; DECISION=*) decision=\\\"${token#DECISION=}\\\";; esac; done\"},{\"line\":43,\"text\":\" if [[ \\\"$decision\\\" == NOOP ]]; then printf 'DISPATCHER_DONE TRIGGER=%s ACTIONS=%s ITERATIONS=%s\\\\n' \\\"$trigger\\\" \\\"$actions\\\" \\\"$i\\\"; return 0; fi\"},{\"line\":44,\"text\":\" [[ \\\"$kind\\\" == VM || \\\"$kind\\\" == APP ]] || die GLOBAL_KIND_INVALID 65; [[ -n \\\"$target\\\" ]] || die GLOBAL_TARGET_MISSING 65\"},{\"line\":45,\"text\":\" if [[ \\\"$last_sealed\\\" == \\\"$kind:$target\\\" ]]; then die \\\"NO_PROGRESS_AFTER_SEALED_${kind}_${target}\\\" 65; fi\"},{\"line\":46,\"text\":\" if [[ \\\"$kind\\\" == VM ]]; then if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$VMQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; else if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$APPQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; fi\"},{\"line\":47,\"text\":\" printf '%s\\\\n' \\\"$out\\\"; (( rc == 0 )) || return \\\"$rc\\\"\"},{\"line\":48,\"text\":\" if grep -q '^QUEUE_DEFERRED ' <<<\\\"$out\\\"; then printf 'DISPATCHER_DEFERRED TRIGGER=%s KIND=%s TARGET=%s REASON=QUEUE_OR_HEAVY_BUSY\\\\n' \\\"$trigger\\\" \\\"$kind\\\" \\\"$target\\\"; return 0; fi\"},{\"line\":49,\"text\":\" if grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SEALED ' <<<\\\"$out\\\"; then last_sealed=\\\"$kind:$target\\\"; actions=$((actions+1)); elif grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SKIP_FRESH ' <<<\\\"$out\\\"; then last_sealed=\\\"\\\"; actions=$((actions+1)); else die \\\"DISPATCH_RESULT_UNCLASSIFIED_${kind}_${target}\\\" 65; fi\"},{\"line\":50,\"text\":\" done\"},{\"line\":51,\"text\":\" die MAX_ITERATIONS_EXCEEDED 65\"},{\"line\":52,\"text\":\"}\"},{\"line\":53,\"text\":\"case \\\"${1:-}\\\" in plan) [[ \\\"$#\\\" == 1 ]] || die USAGE_PLAN 64; plan;; run) [[ \\\"$#\\\" == 2 ]] || die USAGE_RUN 64; run \\\"$2\\\";; *) die ACTION_INVALID 64;; esac\"}],\"sha256\":\"9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c\"}\nVM_WORKER_BACKUP_GUARDS={\"hits\":[{\"line\":6,\"text\":\"STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\"},{\"line\":8,\"text\":\"RESTORE_STATE_ROOT=/var/lib/homelab-backup/vm-restore-validation\"},{\"line\":11,\"text\":\"REMOTE_A='pve01-mail-01-crypt:homelab-backups'\"},{\"line\":12,\"text\":\"REMOTE_B='pve01-mail-02-crypt:homelab-backups'\"},{\"line\":14,\"text\":\"die(){ printf 'VM_QUORUM_WORKER_ERROR=%s\\\\n' \\\"$1\\\" >&2; exit \\\"${2:-65}\\\"; }\"},{\"line\":15,\"text\":\"for tool in jq sha256sum zstd tar rclone stat find sort awk date mkfifo tee wc vma; do command -v \\\"$tool\\\" >/dev/null 2>&1 || die \\\"TOOL_MISSING_${tool}\\\" 69; done\"},{\"line\":16,\"text\":\"[[ -x \\\"$TRANSPORT\\\" && ! -L \\\"$TRANSPORT\\\" ]] || die TRANSPORT_MISSING 69\"},{\"line\":17,\"text\":\"[[ \\\"$(sha256sum \\\"$TRANSPORT\\\"|awk '{print $1}')\\\" == \\\"$TRANSPORT_SHA_EXPECTED\\\" ]] || die TRANSPORT_SHA_MISMATCH 65\"},{\"line\":21,\"text\":\"active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ \\\"$rc\\\" == 1 ]] && return 0; die \\\"PGREP_FAILED_$rc\\\" 69; fi; }\"},{\"line\":22,\"text\":\"snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }\"},{\"line\":23,\"text\":\"vm_lock(){ qm config \\\"$1\\\" | awk -F': ' '$1==\\\"lock\\\"{print $2; found=1} END{if(!found) print \\\"none\\\"}'; }\"},{\"line\":24,\"text\":\"expected_owner(){\"},{\"line\":27,\"text\":\"111|113|160|180|9130) printf 'pve02\\\\n' ;;\"},{\"line\":29,\"text\":\"*) die \\\"VMID_NOT_ALLOWED_$1\\\" 64 ;;\"},{\"line\":35,\"text\":\"130|150|160|170|171|180|190|9130) printf 'qemu\\\\n' ;;\"},{\"line\":36,\"text\":\"*) die \\\"VMID_NOT_ALLOWED_$1\\\" 64 ;;\"},{\"line\":39,\"text\":\"expected_status(){\"},{\"line\":40,\"text\":\"if [[ \\\"$1\\\" == \\\"9130\\\" ]]; then printf 'stopped\\\\n'; else printf 'running\\\\n'; fi\"},{\"line\":42,\"text\":\"current_status(){\"},{\"line\":43,\"text\":\"local vmid=\\\"$1\\\" type=\\\"$2\\\"\"},{\"line\":44,\"text\":\"if [[ \\\"$type\\\" == \\\"qemu\\\" ]]; then qm status \\\"$vmid\\\" | awk '{print $2}'; else pct status \\\"$vmid\\\" | awk '{print $2}'; fi\"},{\"line\":47,\"text\":\"local vmid=\\\"$1\\\" owner type status expect_status\"},{\"line\":48,\"text\":\"owner=\\\"$(expected_owner \\\"$vmid\\\")\\\"\"},{\"line\":49,\"text\":\"type=\\\"$(expected_type \\\"$vmid\\\")\\\"\"},{\"line\":50,\"text\":\"[[ \\\"$HOST\\\" == \\\"$owner\\\" ]] || die \\\"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\\\" 65\"},{\"line\":52,\"text\":\"[[ -s \\\"/etc/pve/qemu-server/${vmid}.conf\\\" ]] || die \\\"QEMU_CONFIG_MISSING_$vmid\\\" 66\"},{\"line\":54,\"text\":\"[[ -s \\\"/etc/pve/lxc/${vmid}.conf\\\" ]] || die \\\"LXC_CONFIG_MISSING_$vmid\\\" 66\"},{\"line\":56,\"text\":\"status=\\\"$(current_status \\\"$vmid\\\" \\\"$type\\\")\\\"\"},{\"line\":57,\"text\":\"expect_status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":58,\"text\":\"[[ \\\"$status\\\" == \\\"$expect_status\\\" ]] || die \\\"STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\\\" 66\"},{\"line\":59,\"text\":\"if [[ \\\"$vmid\\\" == \\\"160\\\" ]]; then [[ \\\"$(snapshot_count_160)\\\" == \\\"0\\\" ]] || die \\\"VM160_SNAPSHOT_COUNT_CHANGED\\\" 66; fi\"},{\"line\":60,\"text\":\"if [[ \\\"$vmid\\\" == \\\"9130\\\" ]]; then [[ \\\"$(vm_lock 9130)\\\" == \\\"none\\\" ]] || die \\\"VM9130_LOCK_PRESENT\\\" 66; fi\"},{\"line\":62,\"text\":\"VM_STATUS=\\\"$status\\\"\"},{\"line\":65,\"text\":\"local vmid=\\\"$1\\\" type=\\\"$2\\\" status expect_status\"},{\"line\":66,\"text\":\"status=\\\"$(current_status \\\"$vmid\\\" \\\"$type\\\")\\\"\"},{\"line\":67,\"text\":\"expect_status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":68,\"text\":\"[[ \\\"$status\\\" == \\\"$expect_status\\\" ]] || die \\\"POST_STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\\\" 66\"},{\"line\":69,\"text\":\"if [[ \\\"$vmid\\\" == \\\"160\\\" ]]; then [[ \\\"$(snapshot_count_160)\\\" == \\\"0\\\" ]] || die \\\"VM160_SNAPSHOT_COUNT_CHANGED_POST\\\" 66; fi\"},{\"line\":70,\"text\":\"if [[ \\\"$vmid\\\" == \\\"9130\\\" ]]; then [[ \\\"$(vm_lock 9130)\\\" == \\\"none\\\" ]] || die \\\"VM9130_LOCK_PRESENT_POST\\\" 66; fi\"},{\"line\":72,\"text\":\"latest_incomplete_state(){\"},{\"line\":73,\"text\":\"local vmid=\\\"$1\\\" dir=\\\"$STATE_ROOT/vm${vmid}\\\" f\"},{\"line\":76,\"text\":\"jq -e '.status!=\\\"SEALED\\\"' \\\"$f\\\" >/dev/null 2>&1 && printf '%s\\\\n' \\\"$f\\\"\"},{\"line\":79,\"text\":\"restore_dir(){ printf '%s\\\\n' \\\"$RESTORE_STATE_ROOT/vm$1\\\"; }\"},{\"line\":81,\"text\":\" local vmid=\\\"$1\\\" gen=\\\"$2\\\" f state sha\"},{\"line\":82,\"text\":\" f=\\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; state=\\\"$STATE_ROOT/vm${vmid}/$gen.json\\\"\"},{\"line\":83,\"text\":\" [[ -s \\\"$f\\\" && ! -L \\\"$f\\\" && -s \\\"$state\\\" && ! -L \\\"$state\\\" ]] || return 1\"},{\"line\":84,\"text\":\" sha=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\" || return 1\"},{\"line\":85,\"text\":\" jq -e --arg l \\\"vm${vmid}\\\" --arg g \\\"$gen\\\" --arg s \\\"$state\\\" --arg sha \\\"$sha\\\" '.schema==\\\"homelab-vm-restore-validation-v1\\\" and .logical_id==$l and .generation==$g and .transport_state==\\\"SEALED\\\" and .transport_state_file==$s and .restore_source==\\\"Cloud A\\\" and .parts_manifest_verified==true and .payload_sha256==$sha and .payload_sha256_verified==true and .payload_decompression_verified==true and .proxmox_config_extract_verified==true and .archive_recovery_readback_verified==true and .bounded_staging==true and .full_local_payload_written==false and .full_restore_proven==false and .result==\\\"OK\\\"' \\\"$f\\\" >/dev/null\"},{\"line\":88,\"text\":\" local vmid=\\\"$1\\\" gen=\\\"$2\\\" state=\\\"$3\\\" work=\\\"$4\\\" base manifest expected_msha got_msha expected_count expected_size idx name size psha count total\"},{\"line\":89,\"text\":\" base=\\\"$(jq -er '.remote_a_base' \\\"$state\\\")\\\"; expected_msha=\\\"$(jq -er '.parts_manifest_sha256' \\\"$state\\\")\\\"; expected_count=\\\"$(jq -er '.part_count' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"line\":90,\"text\":\" [[ \\\"$base\\\" == \\\"${REMOTE_A%/}/vm${vmid}/$gen\\\" ]] || die \\\"RESTORE_A_BASE_MISMATCH_$vmid\\\" 70\"},{\"line\":92,\"text\":\" got_msha=\\\"$(sha256sum \\\"$manifest\\\"|awk '{print $1}')\\\"; [[ \\\"$got_msha\\\" == \\\"$expected_msha\\\" ]] || die \\\"RESTORE_A_MANIFEST_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":95,\"text\":\" [[ \\\"$idx\\\" =~ ^[0-9]{8}$ && \\\"$name\\\" == part-????????.bin && \\\"$size\\\" =~ ^[1-9][0-9]*$ && \\\"$psha\\\" =~ ^[0-9a-f]{64}$ ]] || die \\\"RESTORE_A_MANIFEST_ROW_INVALID_$vmid\\\" 70\"},{\"line\":96,\"text\":\" (( size <= 536870912 )) || die \\\"RESTORE_A_PART_EXCEEDS_BOUND_$vmid\\\" 70\"},{\"line\":97,\"text\":\" [[ \\\"$idx\\\" == \\\"$(printf '%08d' \\\"$count\\\")\\\" ]] || die \\\"RESTORE_A_MANIFEST_INDEX_INVALID_$vmid\\\" 70\"},{\"line\":100,\"text\":\" [[ \\\"$count\\\" == \\\"$expected_count\\\" ]] || die \\\"RESTORE_A_PART_COUNT_MISMATCH_$vmid\\\" 70\"},{\"line\":101,\"text\":\" [[ \\\"$total\\\" == \\\"$expected_size\\\" ]] || die \\\"RESTORE_A_MANIFEST_TOTAL_SIZE_MISMATCH_$vmid\\\" 70\"},{\"line\":105,\"text\":\" local vmid=\\\"$1\\\" base=\\\"$2\\\" manifest=\\\"$3\\\" work=\\\"$4\\\" idx name size psha part actual count\"},{\"line\":109,\"text\":\" [[ -s \\\"$part\\\" && ! -L \\\"$part\\\" && \\\"$(stat -c %s \\\"$part\\\")\\\" == \\\"$size\\\" ]] || die \\\"RESTORE_A_PART_SIZE_MISMATCH_$vmid\\\" 70\"},{\"line\":110,\"text\":\" actual=\\\"$(sha256sum \\\"$part\\\"|awk '{print $1}')\\\"; [[ \\\"$actual\\\" == \\\"$psha\\\" ]] || die \\\"RESTORE_A_PART_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":115,\"text\":\" local vmid=\\\"$1\\\" gen=\\\"$2\\\" logical state work payload_name expected_sha expected_size base manifest fifo sha_out sha_pid pipe_rc sha_rc actual_sha config config_sha members member_count cfg_member level out tmp\"},{\"line\":116,\"text\":\" assert_local_identity \\\"$vmid\\\"; logical=\\\"vm${vmid}\\\"; state=\\\"$STATE_ROOT/$logical/$gen.json\\\"\"},{\"line\":117,\"text\":\" [[ -s \\\"$state\\\" && ! -L \\\"$state\\\" ]] || die \\\"RESTORE_STATE_MISSING_$vmid\\\" 65\"},{\"line\":118,\"text\":\" jq -e '.schema==\\\"homelab-cloud-quorum-payload-v1\\\" and .status==\\\"SEALED\\\" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_payload_present==false and .local_cleanup==\\\"OK\\\"' \\\"$state\\\" >/dev/null || die \\\"RESTORE_REQUIRES_STRICT_SEALED_$vmid\\\" 65\"},{\"line\":119,\"text\":\" if restore_ok \\\"$vmid\\\" \\\"$gen\\\"; then printf 'RESTORE_VALIDATION_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESULT=ALREADY_VALIDATED\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; return 0; fi\"},{\"line\":120,\"text\":\" [[ -z \\\"$(active_backup_restore)\\\" ]] || die ACTIVE_BACKUP_RESTORE_PRESENT 75\"},{\"line\":121,\"text\":\" payload_name=\\\"$(jq -er '.payload_name' \\\"$state\\\")\\\"; expected_sha=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"line\":122,\"text\":\" [[ \\\"$payload_name\\\" == \\\"vzdump-lxc-${vmid}-\\\"*.tar.zst || \\\"$payload_name\\\" == \\\"vzdump-qemu-${vmid}-\\\"*.vma.zst ]] || die \\\"RESTORE_PAYLOAD_NAME_INVALID_$vmid\\\" 70\"},{\"line\":123,\"text\":\" work=\\\"$RESTORE_WORK_ROOT/$logical/$gen\\\"; if [[ -e \\\"$work\\\" ]]; then [[ -d \\\"$work\\\" && ! -L \\\"$work\\\" ]] || die \\\"RESTORE_WORK_UNSAFE_$vmid\\\" 70; rm -rf --one-file-system \\\"$work\\\"; fi; mkdir -p \\\"$work\\\"; chmod 0700 \\\"$RESTORE_WORK_ROOT\\\" \\\"$RESTORE_WORK_ROOT/$logical\\\" \\\"$work\\\"; ACTIVE_RESTORE_STAGE=\\\"$work\\\"\"},{\"line\":124,\"text\":\" base=\\\"$(prepare_cloud_a_manifest \\\"$vmid\\\" \\\"$gen\\\" \\\"$state\\\" \\\"$work\\\")\\\"; manifest=\\\"$work/PARTS.tsv\\\"; fifo=\\\"$work/archive.sha.fifo\\\"; sha_out=\\\"$work/archive.sha256\\\"; mkfifo -m 0600 \\\"$fifo\\\"\"},{\"line\":127,\"text\":\" members=\\\"$work/members.txt\\\"; if stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | tee \\\"$fifo\\\" | zstd -dc | tar -tf - >\\\"$members\\\"; then pipe_rc=0; else pipe_rc=$?; fi\"},{\"line\":129,\"text\":\" if stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | tee \\\"$fifo\\\" | zstd -t - >/dev/null; then pipe_rc=0; else pipe_rc=$?; fi\"},{\"line\":132,\"text\":\" (( pipe_rc == 0 && sha_rc == 0 )) || die \\\"RESTORE_A_STREAM_VALIDATION_FAILED_${vmid}_${pipe_rc}_${sha_rc}\\\" 70\"},{\"line\":133,\"text\":\" actual_sha=\\\"$(awk 'NR==1{print $1}' \\\"$sha_out\\\")\\\"; [[ \\\"$actual_sha\\\" == \\\"$expected_sha\\\" ]] || die \\\"RESTORE_A_PAYLOAD_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":136,\"text\":\" [[ -s \\\"$members\\\" ]] || die \\\"RESTORE_LXC_ARCHIVE_EMPTY_$vmid\\\" 70\"},{\"line\":137,\"text\":\" awk '$0 ~ /^\\\\// || $0 ~ /(^|\\\\/)\\\\.\\\\.(\\\\/|$)/ {bad=1} END{exit bad}' \\\"$members\\\" || die \\\"RESTORE_LXC_MEMBER_UNSAFE_$vmid\\\" 70\"},{\"line\":138,\"text\":\" member_count=\\\"$(wc -l <\\\"$members\\\")\\\"; [[ \\\"$member_count\\\" =~ ^[1-9][0-9]*$ ]] || die \\\"RESTORE_LXC_MEMBER_COUNT_INVALID_$vmid\\\" 70\"},{\"line\":139,\"text\":\" cfg_member=\\\"$(awk '$0==\\\"./etc/vzdump/pct.conf\\\" || $0==\\\"etc/vzdump/pct.conf\\\" {print; exit}' \\\"$members\\\")\\\"; [[ -n \\\"$cfg_member\\\" ]] || die \\\"RESTORE_LXC_CONFIG_MEMBER_MISSING_$vmid\\\" 70\"},{\"line\":140,\"text\":\" if ( set +o pipefail; stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | zstd -dc | tar -xOf - \\\"$cfg_member\\\" >\\\"$config\\\" ); then :; else die \\\"RESTORE_LXC_CONFIG_EXTRACT_FAILED_$vmid\\\" 70; fi\"},{\"line\":143,\"text\":\" if ( set +o pipefail; stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | zstd -dc | vma config - >\\\"$config\\\" ); then :; else die \\\"RESTORE_QEMU_CONFIG_EXTRACT_FAILED_$vmid\\\" 70; fi\"},{\"line\":146,\"text\":\" [[ -s \\\"$config\\\" && ! -L \\\"$config\\\" ]] || die \\\"RESTORE_CONFIG_EMPTY_$vmid\\\" 70; config_sha=\\\"$(sha256sum \\\"$config\\\"|awk '{print $1}')\\\"\"},{\"line\":147,\"text\":\" out=\\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; mkdir -p \\\"$(dirname \\\"$out\\\")\\\"; chmod 0700 \\\"$RESTORE_STATE_ROOT\\\" \\\"$(dirname \\\"$out\\\")\\\"; tmp=\\\"${out}.new.$$\\\"\"},{\"line\":148,\"text\":\" jq -n --arg logical \\\"$logical\\\" --arg generation \\\"$gen\\\" --argjson vmid \\\"$vmid\\\" --arg vm_type \\\"$VM_TYPE\\\" --arg state \\\"$state\\\" --arg sha \\\"$expected_sha\\\" --arg config_sha \\\"$config_sha\\\" --arg level \\\"$level\\\" --argjson member_count \\\"$member_count\\\" --arg at \\\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\\\" '{schema:\\\"homelab-vm-restore-validation-v1\\\",logical_id:$logical,generation:$generation,vmid:$vmid,vm_type:$vm_type,transport_state:\\\"SEALED\\\",transport_state_file:$state,restore_source:\\\"Cloud A\\\",parts_manifest_verified:true,payload_sha256:$sha,payload_sha256_verified:true,payload_decompression_verified:true,proxmox_config_extract_verified:true,proxmox_config_sha256:$config_sha,lxc_member_count:$member_count,archive_recovery_readback_verified:true,bounded_staging:true,max_staged_part_bytes:536870912,full_local_payload_written:false,full_restore_proven:false,validation_level:$level,result:\\\"OK\\\",validated_at_utc:$at}' >\\\"$tmp\\\"\"},{\"line\":150,\"text\":\" restore_ok \\\"$vmid\\\" \\\"$gen\\\" || die \\\"RESTORE_EVIDENCE_POSTWRITE_INVALID_$vmid\\\" 70\"},{\"line\":151,\"text\":\" printf 'RESTORE_VALIDATION_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$out\\\"\"},{\"line\":154,\"text\":\"local vmid=\\\"$1\\\" owner type status\"},{\"line\":155,\"text\":\"assert_local_identity \\\"$vmid\\\"\"},{\"line\":156,\"text\":\"owner=\\\"$(expected_owner \\\"$vmid\\\")\\\"\"},{\"line\":157,\"text\":\"type=\\\"$(expected_type \\\"$vmid\\\")\\\"\"},{\"line\":158,\"text\":\"status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":159,\"text\":\"printf 'PLAN_STATUS=OK\\\\nVMID=%s\\\\nOWNER=%s\\\\nTYPE=%s\\\\nEXPECTED_STATUS=%s\\\\n' \\\"$vmid\\\" \\\"$owner\\\" \\\"$type\\\" \\\"$status\\\"\"},{\"line\":162,\"text\":\" local vmid=\\\"$1\\\" generation=\\\"$2\\\"\"},{\"line\":163,\"text\":\" assert_local_identity \\\"$vmid\\\"\"},{\"line\":164,\"text\":\" restore_validate \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":165,\"text\":\" assert_post_identity \\\"$vmid\\\" \\\"$VM_TYPE\\\"\"},{\"line\":166,\"text\":\" printf 'VERIFY_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$(restore_dir \\\"$vmid\\\")/$generation.json\\\"\"},{\"line\":168,\"text\":\"backup_one(){\"},{\"line\":169,\"text\":\"local vmid=\\\"$1\\\" pending_count pending state generation payload gen_dir archive_count archive log_file before_status\"},{\"line\":170,\"text\":\"local -a vzdump_args\"},{\"line\":171,\"text\":\"assert_local_identity \\\"$vmid\\\"\"},{\"line\":172,\"text\":\"[[ -x \\\"$TRANSPORT\\\" ]] || die \\\"TRANSPORT_NOT_INSTALLED\\\" 69\"},{\"line\":173,\"text\":\"[[ -d /mnt/staging && ! -L /mnt/staging ]] || die \\\"STAGING_UNSAFE\\\" 66\"},{\"line\":176,\"text\":\"[[ -z \\\"$(active_backup_restore)\\\" ]] || die \\\"ACTIVE_BACKUP_RESTORE_PRESENT\\\" 75\"},{\"line\":179,\"text\":\"mapfile -t pending < <(latest_incomplete_state \\\"$vmid\\\")\"},{\"line\":181,\"text\":\"(( pending_count <= 1 )) || die \\\"MULTIPLE_INCOMPLETE_GENERATIONS_$vmid\\\" 65\"},{\"line\":183,\"text\":\"state=\\\"${pending[0]}\\\"\"},{\"line\":184,\"text\":\"generation=\\\"$(jq -er '.generation' \\\"$state\\\")\\\"\"},{\"line\":185,\"text\":\"payload=\\\"$(jq -er '.payload_path' \\\"$state\\\")\\\"\"},{\"line\":186,\"text\":\"printf 'RESUME=YES\\\\nVMID=%s\\\\nGENERATION=%s\\\\nSTATE=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$state\\\"\"},{\"line\":187,\"text\":\"\\\"$TRANSPORT\\\" backup-delete \\\"vm${vmid}\\\" \\\"$generation\\\" \\\"$payload\\\" \\\"$REMOTE_A\\\" \\\"$REMOTE_B\\\" \\\"$STATE_ROOT\\\"\"},{\"line\":189,\"text\":\"generation=\\\"vm${vmid}-${HOST}-$(date -u +%Y%m%dT%H%M%SZ)\\\"\"},{\"line\":190,\"text\":\"gen_dir=\\\"$STAGE_ROOT/vm${vmid}/$generation\\\"\"},{\"line\":191,\"text\":\"[[ ! -e \\\"$gen_dir\\\" ]] || die \\\"GENERATION_DIR_EXISTS_$generation\\\" 65\"},{\"line\":194,\"text\":\"log_file=\\\"$gen_dir/vzdump.log\\\"\"},{\"line\":195,\"text\":\"printf 'RESUME=NO\\\\nVMID=%s\\\\nGENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":196,\"text\":\"vzdump_args=(--dumpdir \\\"$gen_dir\\\" --mode snapshot --compress zstd --ionice 7)\"},{\"line\":198,\"text\":\"[[ -d \\\"$LXC_TMP_ROOT\\\" && ! -L \\\"$LXC_TMP_ROOT\\\" ]] || die \\\"LXC_TMP_ROOT_UNSAFE\\\" 66\"},{\"line\":199,\"text\":\"[[ \\\"$(find \\\"$LXC_TMP_ROOT\\\" -maxdepth 0 -printf '%m')\\\" == \\\"1777\\\" ]] || die \\\"LXC_TMP_ROOT_MODE_UNSAFE\\\" 66\"},{\"line\":200,\"text\":\"vzdump_args+=(--tmpdir \\\"$LXC_TMP_ROOT\\\")\"},{\"line\":202,\"text\":\"if ! ( umask 0022; exec vzdump \\\"$vmid\\\" \\\"${vzdump_args[@]}\\\" ) >\\\"$log_file\\\" 2>&1; then\"},{\"line\":204,\"text\":\"die \\\"VZDUMP_FAILED_$vmid\\\" 70\"},{\"line\":206,\"text\":\"mapfile -t archives < <(find \\\"$gen_dir\\\" -maxdepth 1 -type f -name \\\"vzdump-*-${vmid}-*.zst\\\" -print)\"},{\"line\":208,\"text\":\"[[ \\\"$archive_count\\\" == \\\"1\\\" ]] || { rm -rf -- \\\"$gen_dir\\\"; die \\\"VZDUMP_ARCHIVE_COUNT_${vmid}_${archive_count}\\\" 70; }\"},{\"line\":210,\"text\":\"[[ -s \\\"$archive\\\" && ! -L \\\"$archive\\\" ]] || die \\\"VZDUMP_ARCHIVE_UNSAFE_$vmid\\\" 70\"},{\"line\":212,\"text\":\"\\\"$TRANSPORT\\\" backup-delete \\\"vm${vmid}\\\" \\\"$generation\\\" \\\"$archive\\\" \\\"$REMOTE_A\\\" \\\"$REMOTE_B\\\" \\\"$STATE_ROOT\\\"\"},{\"line\":215,\"text\":\"rmdir -- \\\"$STAGE_ROOT/vm${vmid}\\\" 2>/dev/null || true\"},{\"line\":217,\"text\":\"assert_post_identity \\\"$vmid\\\" \\\"$VM_TYPE\\\"\"},{\"line\":218,\"text\":\"state=\\\"$STATE_ROOT/vm${vmid}/$generation.json\\\"\"},{\"line\":219,\"text\":\"jq -e '.status==\\\"SEALED\\\" and .cloud_quorum==2 and .local_payload_present==false and .local_cleanup==\\\"OK\\\" and .seal_written==true' \\\"$state\\\" >/dev/null\"},{\"line\":220,\"text\":\" restore_validate \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":221,\"text\":\" printf 'BACKUP_STATUS=SEALED\\\\nVMID=%s\\\\nGENERATION=%s\\\\nOWNER=%s\\\\nTYPE=%s\\\\nPOST_STATUS=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$HOST\\\" \\\"$VM_TYPE\\\" \\\"$(expected_status \\\"$vmid\\\")\\\" \\\"$(restore_dir \\\"$vmid\\\")/$generation.json\\\"\"},{\"line\":225,\"text\":\"[[ -d /mnt/staging && ! -L /mnt/staging ]] || die \\\"STAGING_UNSAFE\\\" 66\"},{\"line\":226,\"text\":\"[[ -z \\\"$(active_backup_restore)\\\" ]] || die \\\"ACTIVE_BACKUP_RESTORE_PRESENT\\\" 75\"},{\"line\":230,\"text\":\"[[ \\\"$free\\\" =~ ^[1-9][0-9]+$ && \\\"$a_free\\\" =~ ^[1-9][0-9]+$ && \\\"$b_free\\\" =~ ^[1-9][0-9]+$ ]] || die \\\"CAPACITY_VALUE_INVALID\\\" 65\"},{\"line\":231,\"text\":\"printf 'HOST_PREFLIGHT_STATUS=OK\\\\nHOST=%s\\\\nSTAGING_FREE_BYTES=%s\\\\nREMOTE_A_FREE_BYTES=%s\\\\nREMOTE_B_FREE_BYTES=%s\\\\nACTIVE_BACKUP_RESTORE=0\\\\n' \\\"$HOST\\\" \\\"$free\\\" \\\"$a_free\\\" \\\"$b_free\\\"\"},{\"line\":234,\"text\":\" local vmid=\\\"$1\\\" dir latest gen\"},{\"line\":235,\"text\":\" assert_local_identity \\\"$vmid\\\"; dir=\\\"$STATE_ROOT/vm${vmid}\\\"\"},{\"line\":236,\"text\":\" [[ -d \\\"$dir\\\" ]] || die \\\"STATE_DIR_MISSING_$vmid\\\" 65\"},{\"line\":238,\"text\":\" [[ -n \\\"$latest\\\" && -s \\\"$latest\\\" && ! -L \\\"$latest\\\" ]] || die \\\"LATEST_STATE_MISSING_$vmid\\\" 65\"},{\"line\":239,\"text\":\" jq -e '.status==\\\"SEALED\\\" and .cloud_quorum==2 and .local_payload_present==false and .local_cleanup==\\\"OK\\\" and .seal_written==true' \\\"$latest\\\" >/dev/null || die \\\"LATEST_STATE_NOT_SEALED_$vmid\\\" 65\"},{\"line\":240,\"text\":\" gen=\\\"$(jq -er '.generation' \\\"$latest\\\")\\\"; restore_ok \\\"$vmid\\\" \\\"$gen\\\" || die \\\"LATEST_RESTORE_NOT_OK_$vmid\\\" 65\"},{\"line\":241,\"text\":\" printf 'LATEST_SEALED_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nSTATE_FILE=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$latest\\\" \\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"\"},{\"line\":246,\"text\":\"[[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_HOST_PREFLIGHT\\\" 64\"},{\"line\":250,\"text\":\"[[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_LATEST_SEALED\\\" 64\"},{\"line\":254,\"text\":\"[[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_PLAN\\\" 64\"},{\"line\":257,\"text\":\"backup)\"},{\"line\":258,\"text\":\"[[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_BACKUP\\\" 64\"},{\"line\":259,\"text\":\"backup_one \\\"$2\\\"\"},{\"line\":262,\"text\":\" [[ \\\"$#\\\" == 3 ]] || die \\\"USAGE_RESTORE_VALIDATE\\\" 64\"},{\"line\":266,\"text\":\"[[ \\\"$#\\\" == 3 ]] || die \\\"USAGE_VERIFY\\\" 64\"},{\"line\":270,\"text\":\"die \\\"ACTION_INVALID\\\" 64\"}],\"sha256\":\"91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\"}\nUNIT_STATE={\"homelab-app-cloud-quorum-queue.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-retention-gc.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-v2.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"2\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-backup-v2.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"active\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"waiting\"},\"UnitFileState\":{\"rc\":0,\"value\":\"enabled\"}}}\nRCA_CLASSIFICATION={\"cluster_status\":\"stopped\",\"dispatcher_plan_mentions_9130\":false,\"dispatcher_plan_rc\":0,\"freshness_plan_decision\":null,\"owner_node\":\"pve02\",\"queue_plan_rc\":65,\"worker_contains_status_or_stopped_guard\":true}\nNO_MUTATION=true\nNO_REPO_WRITE=true\nNO_GIT_FETCH=true\nNO_GIT_COMMIT=true\nNO_GIT_PUSH_UPDATE=true\nNO_SYSTEMD_ACTION=true\nNO_BACKUP_STARTED=true\nNO_RESTORE_STARTED=true\nNO_RETENTION_STARTED=true\nNO_DELETE=true\nNO_MAIL_SENT=true\nDURATION_SEC=11.169\nTASK_RESULT=PASS_HOMELAB_BACKUP_VM9130_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\nWORKERS2_BACKUP_VM9130_RCA_END=1\n"
|
|
}
|