diff --git a/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1.json b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1.json new file mode 100644 index 00000000..5f90b1b0 --- /dev/null +++ b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-vm9130-rca", + "started_at_utc": "2026-09-21T03:53:27Z", + "finished_at_utc": "2026-09-21T03:53:38Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "a15d8b5f33489f8d1eb5674ec7b4201fe98d7a2525f0dcb5d5cf67b6b072a193", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "dec8f24d8c18ccf29284612a9706bdfba40963983766a7e3552aca0819069bbd", + "sanitized_output_sha256": "2d97d7e8351c12124c172903bbb35fcbec82ac9dc28d5ffcd538b8b1c5eca523", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_VM9130_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1\nMODE=read-only\nCOMPONENT=homelab-backup-vm9130-rca\nHOST=pve01\nUID=0\nMUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL\nWORKTREE_HEAD=47b9be72cf2487ba66295d7da998016bbeea57be\nWORKTREE_HEAD_EXPECTED=true\nWORKTREE_CLEAN=true\nWORKTREE_STATUS_LINES=[]\nAPP_FRESHNESS_PLAN={\"lines\":[\"QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\"],\"rc\":0,\"stderr\":[]}\nVM9130_STATE={\"age_seconds\":109251,\"consecutive_failures\":0,\"exists\":true,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm9130-20260919T210758Z-0aa987efac8c\",\"last_attempt_epoch\":1789853557,\"last_success_epoch\":1789853557,\"logical_id\":\"vm9130\",\"replica_count\":2,\"result\":\"SUCCESS\",\"sha256\":\"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db\"}\nVM_QUEUE_FRESHNESS_PLAN={\"rc\":65,\"stderr\":[\"VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\",\"VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\"],\"stdout_sha256\":\"251ff72648a76f16b91a5f77b421a20fab3f0113e79c118408803793f4befa16\",\"vm9130_lines\":[]}\nDISPATCHER_PLAN={\"lines\":[\"FRESHNESS_STATUS=OK\",\"VMID=160\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm160-pve02-20260916T023620Z\",\"UPDATED_AT_UTC=2026-09-16T02:46:28Z\",\"SEALED_EPOCH=1789526788\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm160/vm160-pve02-20260916T023620Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json\",\"QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_STALE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=436023 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_STALE\",\"FRESHNESS_STATUS=OK\",\"VMID=170\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm170-pve01-20260916T103019Z\",\"UPDATED_AT_UTC=2026-09-16T10:47:15Z\",\"SEALED_EPOCH=1789555635\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm170/vm170-pve01-20260916T103019Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json\",\"QUEUE_FRESHNESS VMID=170 DECISION=BACKUP_STALE GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=407177 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=170 DECISION=BACKUP_STALE\",\"FRESHNESS_STATUS=OK\",\"VMID=130\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm130-pve03-20260916T105207Z\",\"UPDATED_AT_UTC=2026-09-16T11:25:25Z\",\"SEALED_EPOCH=1789557925\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm130/vm130-pve03-20260916T105207Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json\",\"QUEUE_FRESHNESS VMID=130 DECISION=BACKUP_STALE GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=404888 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=130 DECISION=BACKUP_STALE\",\"FRESHNESS_STATUS=OK\",\"VMID=171\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm171-pve01-20260916T023018Z\",\"UPDATED_AT_UTC=2026-09-16T02:35:03Z\",\"SEALED_EPOCH=1789526103\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\",\"QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_STALE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=436710 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_STALE\",\"CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_STALE PRIORITY=2 AGE_SECONDS=436710\",\"FRESHNESS_STATUS=OK\",\"APP=filebrowser\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-filebrowser-pve01-20260921T033108Z\",\"UPDATED_AT_UTC=2026-09-21T03:31:18Z\",\"SEALED_EPOCH=1789961478\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260921T033108Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260921T033108Z.json\",\"QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260921T033108Z AGE_SECONDS=1337 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\",\"FRESHNESS_STATUS=OK\",\"APP=skladchik-reports-monitor\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"UPDATED_AT_UTC=2026-09-21T03:31:48Z\",\"SEALED_EPOCH=1789961508\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260921T033135Z AGE_SECONDS=1307 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\",\"FRESHNESS_STATUS=OK\",\"APP=immich\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-immich-pve01-20260921T033154Z\",\"UPDATED_AT_UTC=2026-09-21T03:32:40Z\",\"SEALED_EPOCH=1789961560\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260921T033154Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260921T033154Z.json\",\"QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260921T033154Z AGE_SECONDS=1255 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\",\"FRESHNESS_STATUS=OK\",\"APP=hbarhub\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-hbarhub-pve01-20260921T033309Z\",\"UPDATED_AT_UTC=2026-09-21T03:34:20Z\",\"SEALED_EPOCH=1789961660\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260921T033309Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260921T033309Z.json\",\"QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260921T033309Z AGE_SECONDS=1155 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\",\"CLASS_DISPATCH KIND=APP DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1\",\"GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_STALE PRIORITY=2 AGE_SECONDS=436710\"],\"rc\":0,\"stderr\":[\"VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\",\"VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\"],\"stdout_sha256\":\"beaf5c144e0d201cbeef18657cd58484b241ff667de66cb9a6483d41cd29ef0b\"}\nVM9130_REGISTRY_ENTRIES=[{\"id\":\"9130\",\"kind\":\"VM\",\"lifecycle\":\"ACTIVE\",\"logical_id\":\"vm9130\",\"owner\":\"pve02\"}]\nVM9130_CLUSTER_RESOURCE={\"name\":\"edge-cold-standby\",\"node\":\"pve02\",\"status\":\"stopped\",\"template\":0,\"type\":\"qemu\",\"vmid\":9130}\nVM9130_OWNER_READONLY={\"lines\":[\"pve02\",\"status: stopped\",\"agent: enabled=1\",\"boot: order=scsi0\",\"ciuser: debian\",\"cores: 4\",\"cpu: host\",\"description: CR0023 cold standby of VM130. Keep stopped; all NICs link_down=1.\",\"ide2: local-lvm:vm-9130-cloudinit,media=cdrom,size=4M\",\"ipconfig0: ip=[PRIVATE_IP]/24,gw=[PRIVATE_IP]\",\"memory: 12288\",\"meta: creation-qemu=11.0.0,ctime=1781460976\",\"name: edge-cold-standby\",\"nameserver: [PRIVATE_IP] [PRIVATE_IP]\",\"net0: virtio=BC:24:11:7F:04:D6,bridge=vmbr0,firewall=1,link_down=1\",\"onboot: 0\",\"ostype: l26\",\"protection: 1\",\"scsi0: local-lvm:vm-9130-disk-0,discard=on,size=96G,ssd=1\",\"scsi1: local-lvm:vm-9130-disk-1,backup=1,discard=on,size=150G,ssd=1\",\"scsihw: virtio-scsi-pci\",\"searchdomain: gram1.ru\",\"serial0: socket\",\"smbios1: uuid=9327a861-e9f4-4450-b13f-a1e767a83c86\",\"sshkeys: ssh-rsa%20AAAAB3NzaC1yc2EAAAADAQABAAACAQC13sgRxsdexPSyrcz3pfeVJoGajhrb3mJI2gdB2HNWOj6SS%2FcHHiJBKUjg80s6rhZJD%2FtpyLOj%2BQbrkJR3L3FdrgwgCvPcYrBujuZ2TLaRIKIDlNilnRkmVQmUBahgAyrnaQBBl1DVFQsfF0NLF0PYjvPF3789giWqCfv4L14vJSFxvp3wEOjkY1hiwZ8jffMD0R99UW4FTqm29KEFx5ywt%2BdrOT6sMtkBYCIR%2F5JGCB0riH%2F9xn904TIasSMBDz2IeEFpO73bd72JtmnxFyaWaBllKn%2Bu9t18ahMx5vSn2UerGGZbbrZBigxNIwOaHdvdyxADSR3QU90T6nzUv5Fov3vIoOt4uhVaMsYew25hpzm8%2Byp77liUu9mfuMWzo0Q7XJrNfhuCJ9pcrBeqopdThFC36c72JgksWT9Lhfef7bMbEa%2BIWF0wL3L%2FALp1QVA1cbQWPVAF1x3UEHnXuxJCj2x8z4Xli2SJu7TS16qF9Lv1GqVYMeZvW48LwXSbwWcui3B4ldSuo5ksmPeggcTuHzxy5TyQk9sJjlz10Pw9A5zbbuYZDo56n17c1couUm1a3EQ25CimrECygWX2OKcAEdmoiWAihKHyAuLFnSRUELbNrFcFJIb4JkrGk7edWdF%2Btw3jOHlHDcJew08xyoutlN1P9iFTrotMbQjyNTrGgQ%3D%3D%20root%40pve01%0A\",\"startup: order=40\",\"vga: serial0\",\"vmgenid: 110339cf-08fb-4da8-bc7d-2265b8080f2c\",\"91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 /usr/local/libexec/homelab-vm-cloud-quorum-node-worker\"],\"rc\":0,\"stderr\":[]}\nVM_QUEUE_DISPATCH_SOURCE={\"lines_140_220\":[{\"line\":140,\"text\":\" printf 'QUEUE_SEALED VMID=%s GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":141,\"text\":\" done\"},{\"line\":142,\"text\":\"}\"},{\"line\":143,\"text\":\"dispatch_plan(){\"},{\"line\":144,\"text\":\" freshness_plan\"},{\"line\":145,\"text\":\"}\"},{\"line\":146,\"text\":\"dispatch_one(){\"},{\"line\":147,\"text\":\" local vmid=\\\"$1\\\" freshness decision now_epoch out generation\"},{\"line\":148,\"text\":\" [[ \\\"${HOMELAB_BACKUP_DISPATCHER:-}\\\" == 1 ]] || die \\\"DISPATCH_ONE_DIRECT_FORBIDDEN\\\" 64\"},{\"line\":149,\"text\":\" expected_owner \\\"$vmid\\\" >/dev/null\"},{\"line\":150,\"text\":\" exec 9>\\\"$LOCK\\\"\"},{\"line\":151,\"text\":\" if ! flock -n 9; then\"},{\"line\":152,\"text\":\" printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\\\\n' \\\"$vmid\\\"\"},{\"line\":153,\"text\":\" return 0\"},{\"line\":154,\"text\":\" fi\"},{\"line\":155,\"text\":\" exec 8>\\\"$HEAVY_LOCK\\\"\"},{\"line\":156,\"text\":\" if ! flock -n 8; then\"},{\"line\":157,\"text\":\" printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_HEAVY_LOCK_BUSY VMID=%s\\\\n' \\\"$vmid\\\"\"},{\"line\":158,\"text\":\" return 0\"},{\"line\":159,\"text\":\" fi\"},{\"line\":160,\"text\":\" now_epoch=\\\"$(date -u +%s)\\\"\"},{\"line\":161,\"text\":\" freshness=\\\"$(exec 8>&-; exec 9>&-; freshness_decision \\\"$vmid\\\" \\\"$now_epoch\\\")\\\"\"},{\"line\":162,\"text\":\" printf '%s\\\\n' \\\"$freshness\\\"\"},{\"line\":163,\"text\":\" decision=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":164,\"text\":\" case \\\"$decision\\\" in\"},{\"line\":165,\"text\":\" SKIP_FRESH)\"},{\"line\":166,\"text\":\" generation=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":167,\"text\":\" printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SKIP_FRESH GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":168,\"text\":\" return 0\"},{\"line\":169,\"text\":\" ;;\"},{\"line\":170,\"text\":\" BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\"},{\"line\":171,\"text\":\" *) die \\\"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\\\" 65 ;;\"},{\"line\":172,\"text\":\" esac\"},{\"line\":173,\"text\":\" printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\\\n' \\\"$vmid\\\" \\\"$(expected_owner \\\"$vmid\\\")\\\" \\\"$decision\\\"\"},{\"line\":174,\"text\":\" out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":175,\"text\":\" printf '%s\\\\n' \\\"$out\\\"\"},{\"line\":176,\"text\":\" generation=\\\"$(awk -F= '$1==\\\"GENERATION\\\"{print $2}' <<<\\\"$out\\\" | tail -1)\\\"\"},{\"line\":177,\"text\":\" [[ -n \\\"$generation\\\" ]] || die \\\"GENERATION_NOT_RETURNED_$vmid\\\" 65\"},{\"line\":178,\"text\":\" printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":179,\"text\":\" printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":180,\"text\":\"}\"},{\"line\":181,\"text\":\"\"},{\"line\":182,\"text\":\"ACTION=\\\"${1:-}\\\"\"},{\"line\":183,\"text\":\"case \\\"$ACTION\\\" in\"},{\"line\":184,\"text\":\" plan)\"},{\"line\":185,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_PLAN\\\" 64\"},{\"line\":186,\"text\":\" plan_all\"},{\"line\":187,\"text\":\" ;;\"},{\"line\":188,\"text\":\" freshness-plan)\"},{\"line\":189,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_FRESHNESS_PLAN\\\" 64\"},{\"line\":190,\"text\":\" freshness_plan\"},{\"line\":191,\"text\":\" ;;\"},{\"line\":192,\"text\":\" dispatch-plan)\"},{\"line\":193,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_DISPATCH_PLAN\\\" 64\"},{\"line\":194,\"text\":\" dispatch_plan\"},{\"line\":195,\"text\":\" ;;\"},{\"line\":196,\"text\":\" dispatch-one)\"},{\"line\":197,\"text\":\" [[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_DISPATCH_ONE\\\" 64\"},{\"line\":198,\"text\":\" dispatch_one \\\"$2\\\"\"},{\"line\":199,\"text\":\" ;;\"},{\"line\":200,\"text\":\" canary)\"},{\"line\":201,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_CANARY\\\" 64\"},{\"line\":202,\"text\":\" backup_list_force 110\"},{\"line\":203,\"text\":\" ;;\"},{\"line\":204,\"text\":\" migrate)\"},{\"line\":205,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_MIGRATE\\\" 64\"},{\"line\":206,\"text\":\" backup_list_force \\\"${MIGRATE_VMIDS[@]}\\\"\"},{\"line\":207,\"text\":\" ;;\"},{\"line\":208,\"text\":\" all)\"},{\"line\":209,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_ALL\\\" 64\"},{\"line\":210,\"text\":\" [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die \\\"DISPATCHER_MISSING\\\" 69\"},{\"line\":211,\"text\":\" /usr/local/sbin/homelab-backup-dispatcher run VM_TIMER\"},{\"line\":212,\"text\":\" ;;\"},{\"line\":213,\"text\":\" verify)\"},{\"line\":214,\"text\":\" [[ \\\"$#\\\" == 3 ]] || die \\\"USAGE_VERIFY\\\" 64\"},{\"line\":215,\"text\":\" run_worker \\\"$2\\\" verify \\\"$3\\\"\"},{\"line\":216,\"text\":\" ;;\"},{\"line\":217,\"text\":\" *)\"},{\"line\":218,\"text\":\" die \\\"ACTION_INVALID\\\" 64\"},{\"line\":219,\"text\":\" ;;\"},{\"line\":220,\"text\":\"esac\"}],\"sha256\":\"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\"}\nDISPATCHER_RUN_SOURCE={\"lines_35_60\":[{\"line\":35,\"text\":\"plan(){ select_global; }\"},{\"line\":36,\"text\":\"run(){\"},{\"line\":37,\"text\":\" local trigger=\\\"$1\\\" i=0 actions=0 scan pick decision kind target out rc last_sealed=\\\"\\\"\"},{\"line\":38,\"text\":\" exec 7>\\\"$LOCK\\\"\"},{\"line\":39,\"text\":\" if ! flock -n 7; then printf 'DISPATCHER_DEFERRED TRIGGER=%s REASON=DISPATCHER_LOCK_BUSY\\\\n' \\\"$trigger\\\"; return 0; fi\"},{\"line\":40,\"text\":\" while (( i < MAX_ITERATIONS )); do\"},{\"line\":41,\"text\":\" i=$((i+1)); scan=\\\"$(select_global)\\\"; printf '%s\\\\n' \\\"$scan\\\"; pick=\\\"$(grep '^GLOBAL_DISPATCH ' <<<\\\"$scan\\\" | tail -1)\\\"; [[ -n \\\"$pick\\\" ]] || die GLOBAL_DISPATCH_RESULT_MISSING 65\"},{\"line\":42,\"text\":\" decision=\\\"\\\"; kind=\\\"\\\"; target=\\\"\\\"; for token in $pick; do case \\\"$token\\\" in KIND=*) kind=\\\"${token#KIND=}\\\";; TARGET=*) target=\\\"${token#TARGET=}\\\";; DECISION=*) decision=\\\"${token#DECISION=}\\\";; esac; done\"},{\"line\":43,\"text\":\" if [[ \\\"$decision\\\" == NOOP ]]; then printf 'DISPATCHER_DONE TRIGGER=%s ACTIONS=%s ITERATIONS=%s\\\\n' \\\"$trigger\\\" \\\"$actions\\\" \\\"$i\\\"; return 0; fi\"},{\"line\":44,\"text\":\" [[ \\\"$kind\\\" == VM || \\\"$kind\\\" == APP ]] || die GLOBAL_KIND_INVALID 65; [[ -n \\\"$target\\\" ]] || die GLOBAL_TARGET_MISSING 65\"},{\"line\":45,\"text\":\" if [[ \\\"$last_sealed\\\" == \\\"$kind:$target\\\" ]]; then die \\\"NO_PROGRESS_AFTER_SEALED_${kind}_${target}\\\" 65; fi\"},{\"line\":46,\"text\":\" if [[ \\\"$kind\\\" == VM ]]; then if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$VMQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; else if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$APPQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; fi\"},{\"line\":47,\"text\":\" printf '%s\\\\n' \\\"$out\\\"; (( rc == 0 )) || return \\\"$rc\\\"\"},{\"line\":48,\"text\":\" if grep -q '^QUEUE_DEFERRED ' <<<\\\"$out\\\"; then printf 'DISPATCHER_DEFERRED TRIGGER=%s KIND=%s TARGET=%s REASON=QUEUE_OR_HEAVY_BUSY\\\\n' \\\"$trigger\\\" \\\"$kind\\\" \\\"$target\\\"; return 0; fi\"},{\"line\":49,\"text\":\" if grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SEALED ' <<<\\\"$out\\\"; then last_sealed=\\\"$kind:$target\\\"; actions=$((actions+1)); elif grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SKIP_FRESH ' <<<\\\"$out\\\"; then last_sealed=\\\"\\\"; actions=$((actions+1)); else die \\\"DISPATCH_RESULT_UNCLASSIFIED_${kind}_${target}\\\" 65; fi\"},{\"line\":50,\"text\":\" done\"},{\"line\":51,\"text\":\" die MAX_ITERATIONS_EXCEEDED 65\"},{\"line\":52,\"text\":\"}\"},{\"line\":53,\"text\":\"case \\\"${1:-}\\\" in plan) [[ \\\"$#\\\" == 1 ]] || die USAGE_PLAN 64; plan;; run) [[ \\\"$#\\\" == 2 ]] || die USAGE_RUN 64; run \\\"$2\\\";; *) die ACTION_INVALID 64;; esac\"}],\"sha256\":\"9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c\"}\nVM_WORKER_BACKUP_GUARDS={\"hits\":[{\"line\":6,\"text\":\"STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\"},{\"line\":8,\"text\":\"RESTORE_STATE_ROOT=/var/lib/homelab-backup/vm-restore-validation\"},{\"line\":11,\"text\":\"REMOTE_A='pve01-mail-01-crypt:homelab-backups'\"},{\"line\":12,\"text\":\"REMOTE_B='pve01-mail-02-crypt:homelab-backups'\"},{\"line\":14,\"text\":\"die(){ printf 'VM_QUORUM_WORKER_ERROR=%s\\\\n' \\\"$1\\\" >&2; exit \\\"${2:-65}\\\"; }\"},{\"line\":15,\"text\":\"for tool in jq sha256sum zstd tar rclone stat find sort awk date mkfifo tee wc vma; do command -v \\\"$tool\\\" >/dev/null 2>&1 || die \\\"TOOL_MISSING_${tool}\\\" 69; done\"},{\"line\":16,\"text\":\"[[ -x \\\"$TRANSPORT\\\" && ! -L \\\"$TRANSPORT\\\" ]] || die TRANSPORT_MISSING 69\"},{\"line\":17,\"text\":\"[[ \\\"$(sha256sum \\\"$TRANSPORT\\\"|awk '{print $1}')\\\" == \\\"$TRANSPORT_SHA_EXPECTED\\\" ]] || die TRANSPORT_SHA_MISMATCH 65\"},{\"line\":21,\"text\":\"active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ \\\"$rc\\\" == 1 ]] && return 0; die \\\"PGREP_FAILED_$rc\\\" 69; fi; }\"},{\"line\":22,\"text\":\"snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }\"},{\"line\":23,\"text\":\"vm_lock(){ qm config \\\"$1\\\" | awk -F': ' '$1==\\\"lock\\\"{print $2; found=1} END{if(!found) print \\\"none\\\"}'; }\"},{\"line\":24,\"text\":\"expected_owner(){\"},{\"line\":27,\"text\":\"111|113|160|180|9130) printf 'pve02\\\\n' ;;\"},{\"line\":29,\"text\":\"*) die \\\"VMID_NOT_ALLOWED_$1\\\" 64 ;;\"},{\"line\":35,\"text\":\"130|150|160|170|171|180|190|9130) printf 'qemu\\\\n' ;;\"},{\"line\":36,\"text\":\"*) die \\\"VMID_NOT_ALLOWED_$1\\\" 64 ;;\"},{\"line\":39,\"text\":\"expected_status(){\"},{\"line\":40,\"text\":\"if [[ \\\"$1\\\" == \\\"9130\\\" ]]; then printf 'stopped\\\\n'; else printf 'running\\\\n'; fi\"},{\"line\":42,\"text\":\"current_status(){\"},{\"line\":43,\"text\":\"local vmid=\\\"$1\\\" type=\\\"$2\\\"\"},{\"line\":44,\"text\":\"if [[ \\\"$type\\\" == \\\"qemu\\\" ]]; then qm status \\\"$vmid\\\" | awk '{print $2}'; else pct status \\\"$vmid\\\" | awk '{print $2}'; fi\"},{\"line\":47,\"text\":\"local vmid=\\\"$1\\\" owner type status expect_status\"},{\"line\":48,\"text\":\"owner=\\\"$(expected_owner \\\"$vmid\\\")\\\"\"},{\"line\":49,\"text\":\"type=\\\"$(expected_type \\\"$vmid\\\")\\\"\"},{\"line\":50,\"text\":\"[[ \\\"$HOST\\\" == \\\"$owner\\\" ]] || die \\\"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\\\" 65\"},{\"line\":52,\"text\":\"[[ -s \\\"/etc/pve/qemu-server/${vmid}.conf\\\" ]] || die \\\"QEMU_CONFIG_MISSING_$vmid\\\" 66\"},{\"line\":54,\"text\":\"[[ -s \\\"/etc/pve/lxc/${vmid}.conf\\\" ]] || die \\\"LXC_CONFIG_MISSING_$vmid\\\" 66\"},{\"line\":56,\"text\":\"status=\\\"$(current_status \\\"$vmid\\\" \\\"$type\\\")\\\"\"},{\"line\":57,\"text\":\"expect_status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":58,\"text\":\"[[ \\\"$status\\\" == \\\"$expect_status\\\" ]] || die \\\"STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\\\" 66\"},{\"line\":59,\"text\":\"if [[ \\\"$vmid\\\" == \\\"160\\\" ]]; then [[ \\\"$(snapshot_count_160)\\\" == \\\"0\\\" ]] || die \\\"VM160_SNAPSHOT_COUNT_CHANGED\\\" 66; fi\"},{\"line\":60,\"text\":\"if [[ \\\"$vmid\\\" == \\\"9130\\\" ]]; then [[ \\\"$(vm_lock 9130)\\\" == \\\"none\\\" ]] || die \\\"VM9130_LOCK_PRESENT\\\" 66; fi\"},{\"line\":62,\"text\":\"VM_STATUS=\\\"$status\\\"\"},{\"line\":65,\"text\":\"local vmid=\\\"$1\\\" type=\\\"$2\\\" status expect_status\"},{\"line\":66,\"text\":\"status=\\\"$(current_status \\\"$vmid\\\" \\\"$type\\\")\\\"\"},{\"line\":67,\"text\":\"expect_status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":68,\"text\":\"[[ \\\"$status\\\" == \\\"$expect_status\\\" ]] || die \\\"POST_STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\\\" 66\"},{\"line\":69,\"text\":\"if [[ \\\"$vmid\\\" == \\\"160\\\" ]]; then [[ \\\"$(snapshot_count_160)\\\" == \\\"0\\\" ]] || die \\\"VM160_SNAPSHOT_COUNT_CHANGED_POST\\\" 66; fi\"},{\"line\":70,\"text\":\"if [[ \\\"$vmid\\\" == \\\"9130\\\" ]]; then [[ \\\"$(vm_lock 9130)\\\" == \\\"none\\\" ]] || die \\\"VM9130_LOCK_PRESENT_POST\\\" 66; fi\"},{\"line\":72,\"text\":\"latest_incomplete_state(){\"},{\"line\":73,\"text\":\"local vmid=\\\"$1\\\" dir=\\\"$STATE_ROOT/vm${vmid}\\\" f\"},{\"line\":76,\"text\":\"jq -e '.status!=\\\"SEALED\\\"' \\\"$f\\\" >/dev/null 2>&1 && printf '%s\\\\n' \\\"$f\\\"\"},{\"line\":79,\"text\":\"restore_dir(){ printf '%s\\\\n' \\\"$RESTORE_STATE_ROOT/vm$1\\\"; }\"},{\"line\":81,\"text\":\" local vmid=\\\"$1\\\" gen=\\\"$2\\\" f state sha\"},{\"line\":82,\"text\":\" f=\\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; state=\\\"$STATE_ROOT/vm${vmid}/$gen.json\\\"\"},{\"line\":83,\"text\":\" [[ -s \\\"$f\\\" && ! -L \\\"$f\\\" && -s \\\"$state\\\" && ! -L \\\"$state\\\" ]] || return 1\"},{\"line\":84,\"text\":\" sha=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\" || return 1\"},{\"line\":85,\"text\":\" jq -e --arg l \\\"vm${vmid}\\\" --arg g \\\"$gen\\\" --arg s \\\"$state\\\" --arg sha \\\"$sha\\\" '.schema==\\\"homelab-vm-restore-validation-v1\\\" and .logical_id==$l and .generation==$g and .transport_state==\\\"SEALED\\\" and .transport_state_file==$s and .restore_source==\\\"Cloud A\\\" and .parts_manifest_verified==true and .payload_sha256==$sha and .payload_sha256_verified==true and .payload_decompression_verified==true and .proxmox_config_extract_verified==true and .archive_recovery_readback_verified==true and .bounded_staging==true and .full_local_payload_written==false and .full_restore_proven==false and .result==\\\"OK\\\"' \\\"$f\\\" >/dev/null\"},{\"line\":88,\"text\":\" local vmid=\\\"$1\\\" gen=\\\"$2\\\" state=\\\"$3\\\" work=\\\"$4\\\" base manifest expected_msha got_msha expected_count expected_size idx name size psha count total\"},{\"line\":89,\"text\":\" base=\\\"$(jq -er '.remote_a_base' \\\"$state\\\")\\\"; expected_msha=\\\"$(jq -er '.parts_manifest_sha256' \\\"$state\\\")\\\"; expected_count=\\\"$(jq -er '.part_count' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"line\":90,\"text\":\" [[ \\\"$base\\\" == \\\"${REMOTE_A%/}/vm${vmid}/$gen\\\" ]] || die \\\"RESTORE_A_BASE_MISMATCH_$vmid\\\" 70\"},{\"line\":92,\"text\":\" got_msha=\\\"$(sha256sum \\\"$manifest\\\"|awk '{print $1}')\\\"; [[ \\\"$got_msha\\\" == \\\"$expected_msha\\\" ]] || die \\\"RESTORE_A_MANIFEST_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":95,\"text\":\" [[ \\\"$idx\\\" =~ ^[0-9]{8}$ && \\\"$name\\\" == part-????????.bin && \\\"$size\\\" =~ ^[1-9][0-9]*$ && \\\"$psha\\\" =~ ^[0-9a-f]{64}$ ]] || die \\\"RESTORE_A_MANIFEST_ROW_INVALID_$vmid\\\" 70\"},{\"line\":96,\"text\":\" (( size <= 536870912 )) || die \\\"RESTORE_A_PART_EXCEEDS_BOUND_$vmid\\\" 70\"},{\"line\":97,\"text\":\" [[ \\\"$idx\\\" == \\\"$(printf '%08d' \\\"$count\\\")\\\" ]] || die \\\"RESTORE_A_MANIFEST_INDEX_INVALID_$vmid\\\" 70\"},{\"line\":100,\"text\":\" [[ \\\"$count\\\" == \\\"$expected_count\\\" ]] || die \\\"RESTORE_A_PART_COUNT_MISMATCH_$vmid\\\" 70\"},{\"line\":101,\"text\":\" [[ \\\"$total\\\" == \\\"$expected_size\\\" ]] || die \\\"RESTORE_A_MANIFEST_TOTAL_SIZE_MISMATCH_$vmid\\\" 70\"},{\"line\":105,\"text\":\" local vmid=\\\"$1\\\" base=\\\"$2\\\" manifest=\\\"$3\\\" work=\\\"$4\\\" idx name size psha part actual count\"},{\"line\":109,\"text\":\" [[ -s \\\"$part\\\" && ! -L \\\"$part\\\" && \\\"$(stat -c %s \\\"$part\\\")\\\" == \\\"$size\\\" ]] || die \\\"RESTORE_A_PART_SIZE_MISMATCH_$vmid\\\" 70\"},{\"line\":110,\"text\":\" actual=\\\"$(sha256sum \\\"$part\\\"|awk '{print $1}')\\\"; [[ \\\"$actual\\\" == \\\"$psha\\\" ]] || die \\\"RESTORE_A_PART_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":115,\"text\":\" local vmid=\\\"$1\\\" gen=\\\"$2\\\" logical state work payload_name expected_sha expected_size base manifest fifo sha_out sha_pid pipe_rc sha_rc actual_sha config config_sha members member_count cfg_member level out tmp\"},{\"line\":116,\"text\":\" assert_local_identity \\\"$vmid\\\"; logical=\\\"vm${vmid}\\\"; state=\\\"$STATE_ROOT/$logical/$gen.json\\\"\"},{\"line\":117,\"text\":\" [[ -s \\\"$state\\\" && ! -L \\\"$state\\\" ]] || die \\\"RESTORE_STATE_MISSING_$vmid\\\" 65\"},{\"line\":118,\"text\":\" jq -e '.schema==\\\"homelab-cloud-quorum-payload-v1\\\" and .status==\\\"SEALED\\\" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_payload_present==false and .local_cleanup==\\\"OK\\\"' \\\"$state\\\" >/dev/null || die \\\"RESTORE_REQUIRES_STRICT_SEALED_$vmid\\\" 65\"},{\"line\":119,\"text\":\" if restore_ok \\\"$vmid\\\" \\\"$gen\\\"; then printf 'RESTORE_VALIDATION_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESULT=ALREADY_VALIDATED\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; return 0; fi\"},{\"line\":120,\"text\":\" [[ -z \\\"$(active_backup_restore)\\\" ]] || die ACTIVE_BACKUP_RESTORE_PRESENT 75\"},{\"line\":121,\"text\":\" payload_name=\\\"$(jq -er '.payload_name' \\\"$state\\\")\\\"; expected_sha=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"line\":122,\"text\":\" [[ \\\"$payload_name\\\" == \\\"vzdump-lxc-${vmid}-\\\"*.tar.zst || \\\"$payload_name\\\" == \\\"vzdump-qemu-${vmid}-\\\"*.vma.zst ]] || die \\\"RESTORE_PAYLOAD_NAME_INVALID_$vmid\\\" 70\"},{\"line\":123,\"text\":\" work=\\\"$RESTORE_WORK_ROOT/$logical/$gen\\\"; if [[ -e \\\"$work\\\" ]]; then [[ -d \\\"$work\\\" && ! -L \\\"$work\\\" ]] || die \\\"RESTORE_WORK_UNSAFE_$vmid\\\" 70; rm -rf --one-file-system \\\"$work\\\"; fi; mkdir -p \\\"$work\\\"; chmod 0700 \\\"$RESTORE_WORK_ROOT\\\" \\\"$RESTORE_WORK_ROOT/$logical\\\" \\\"$work\\\"; ACTIVE_RESTORE_STAGE=\\\"$work\\\"\"},{\"line\":124,\"text\":\" base=\\\"$(prepare_cloud_a_manifest \\\"$vmid\\\" \\\"$gen\\\" \\\"$state\\\" \\\"$work\\\")\\\"; manifest=\\\"$work/PARTS.tsv\\\"; fifo=\\\"$work/archive.sha.fifo\\\"; sha_out=\\\"$work/archive.sha256\\\"; mkfifo -m 0600 \\\"$fifo\\\"\"},{\"line\":127,\"text\":\" members=\\\"$work/members.txt\\\"; if stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | tee \\\"$fifo\\\" | zstd -dc | tar -tf - >\\\"$members\\\"; then pipe_rc=0; else pipe_rc=$?; fi\"},{\"line\":129,\"text\":\" if stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | tee \\\"$fifo\\\" | zstd -t - >/dev/null; then pipe_rc=0; else pipe_rc=$?; fi\"},{\"line\":132,\"text\":\" (( pipe_rc == 0 && sha_rc == 0 )) || die \\\"RESTORE_A_STREAM_VALIDATION_FAILED_${vmid}_${pipe_rc}_${sha_rc}\\\" 70\"},{\"line\":133,\"text\":\" actual_sha=\\\"$(awk 'NR==1{print $1}' \\\"$sha_out\\\")\\\"; [[ \\\"$actual_sha\\\" == \\\"$expected_sha\\\" ]] || die \\\"RESTORE_A_PAYLOAD_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":136,\"text\":\" [[ -s \\\"$members\\\" ]] || die \\\"RESTORE_LXC_ARCHIVE_EMPTY_$vmid\\\" 70\"},{\"line\":137,\"text\":\" awk '$0 ~ /^\\\\// || $0 ~ /(^|\\\\/)\\\\.\\\\.(\\\\/|$)/ {bad=1} END{exit bad}' \\\"$members\\\" || die \\\"RESTORE_LXC_MEMBER_UNSAFE_$vmid\\\" 70\"},{\"line\":138,\"text\":\" member_count=\\\"$(wc -l <\\\"$members\\\")\\\"; [[ \\\"$member_count\\\" =~ ^[1-9][0-9]*$ ]] || die \\\"RESTORE_LXC_MEMBER_COUNT_INVALID_$vmid\\\" 70\"},{\"line\":139,\"text\":\" cfg_member=\\\"$(awk '$0==\\\"./etc/vzdump/pct.conf\\\" || $0==\\\"etc/vzdump/pct.conf\\\" {print; exit}' \\\"$members\\\")\\\"; [[ -n \\\"$cfg_member\\\" ]] || die \\\"RESTORE_LXC_CONFIG_MEMBER_MISSING_$vmid\\\" 70\"},{\"line\":140,\"text\":\" if ( set +o pipefail; stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | zstd -dc | tar -xOf - \\\"$cfg_member\\\" >\\\"$config\\\" ); then :; else die \\\"RESTORE_LXC_CONFIG_EXTRACT_FAILED_$vmid\\\" 70; fi\"},{\"line\":143,\"text\":\" if ( set +o pipefail; stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | zstd -dc | vma config - >\\\"$config\\\" ); then :; else die \\\"RESTORE_QEMU_CONFIG_EXTRACT_FAILED_$vmid\\\" 70; fi\"},{\"line\":146,\"text\":\" [[ -s \\\"$config\\\" && ! -L \\\"$config\\\" ]] || die \\\"RESTORE_CONFIG_EMPTY_$vmid\\\" 70; config_sha=\\\"$(sha256sum \\\"$config\\\"|awk '{print $1}')\\\"\"},{\"line\":147,\"text\":\" out=\\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; mkdir -p \\\"$(dirname \\\"$out\\\")\\\"; chmod 0700 \\\"$RESTORE_STATE_ROOT\\\" \\\"$(dirname \\\"$out\\\")\\\"; tmp=\\\"${out}.new.$$\\\"\"},{\"line\":148,\"text\":\" jq -n --arg logical \\\"$logical\\\" --arg generation \\\"$gen\\\" --argjson vmid \\\"$vmid\\\" --arg vm_type \\\"$VM_TYPE\\\" --arg state \\\"$state\\\" --arg sha \\\"$expected_sha\\\" --arg config_sha \\\"$config_sha\\\" --arg level \\\"$level\\\" --argjson member_count \\\"$member_count\\\" --arg at \\\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\\\" '{schema:\\\"homelab-vm-restore-validation-v1\\\",logical_id:$logical,generation:$generation,vmid:$vmid,vm_type:$vm_type,transport_state:\\\"SEALED\\\",transport_state_file:$state,restore_source:\\\"Cloud A\\\",parts_manifest_verified:true,payload_sha256:$sha,payload_sha256_verified:true,payload_decompression_verified:true,proxmox_config_extract_verified:true,proxmox_config_sha256:$config_sha,lxc_member_count:$member_count,archive_recovery_readback_verified:true,bounded_staging:true,max_staged_part_bytes:536870912,full_local_payload_written:false,full_restore_proven:false,validation_level:$level,result:\\\"OK\\\",validated_at_utc:$at}' >\\\"$tmp\\\"\"},{\"line\":150,\"text\":\" restore_ok \\\"$vmid\\\" \\\"$gen\\\" || die \\\"RESTORE_EVIDENCE_POSTWRITE_INVALID_$vmid\\\" 70\"},{\"line\":151,\"text\":\" printf 'RESTORE_VALIDATION_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$out\\\"\"},{\"line\":154,\"text\":\"local vmid=\\\"$1\\\" owner type status\"},{\"line\":155,\"text\":\"assert_local_identity \\\"$vmid\\\"\"},{\"line\":156,\"text\":\"owner=\\\"$(expected_owner \\\"$vmid\\\")\\\"\"},{\"line\":157,\"text\":\"type=\\\"$(expected_type \\\"$vmid\\\")\\\"\"},{\"line\":158,\"text\":\"status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":159,\"text\":\"printf 'PLAN_STATUS=OK\\\\nVMID=%s\\\\nOWNER=%s\\\\nTYPE=%s\\\\nEXPECTED_STATUS=%s\\\\n' \\\"$vmid\\\" \\\"$owner\\\" \\\"$type\\\" \\\"$status\\\"\"},{\"line\":162,\"text\":\" local vmid=\\\"$1\\\" generation=\\\"$2\\\"\"},{\"line\":163,\"text\":\" assert_local_identity \\\"$vmid\\\"\"},{\"line\":164,\"text\":\" restore_validate \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":165,\"text\":\" assert_post_identity \\\"$vmid\\\" \\\"$VM_TYPE\\\"\"},{\"line\":166,\"text\":\" printf 'VERIFY_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$(restore_dir \\\"$vmid\\\")/$generation.json\\\"\"},{\"line\":168,\"text\":\"backup_one(){\"},{\"line\":169,\"text\":\"local vmid=\\\"$1\\\" pending_count pending state generation payload gen_dir archive_count archive log_file before_status\"},{\"line\":170,\"text\":\"local -a vzdump_args\"},{\"line\":171,\"text\":\"assert_local_identity \\\"$vmid\\\"\"},{\"line\":172,\"text\":\"[[ -x \\\"$TRANSPORT\\\" ]] || die \\\"TRANSPORT_NOT_INSTALLED\\\" 69\"},{\"line\":173,\"text\":\"[[ -d /mnt/staging && ! -L /mnt/staging ]] || die \\\"STAGING_UNSAFE\\\" 66\"},{\"line\":176,\"text\":\"[[ -z \\\"$(active_backup_restore)\\\" ]] || die \\\"ACTIVE_BACKUP_RESTORE_PRESENT\\\" 75\"},{\"line\":179,\"text\":\"mapfile -t pending < <(latest_incomplete_state \\\"$vmid\\\")\"},{\"line\":181,\"text\":\"(( pending_count <= 1 )) || die \\\"MULTIPLE_INCOMPLETE_GENERATIONS_$vmid\\\" 65\"},{\"line\":183,\"text\":\"state=\\\"${pending[0]}\\\"\"},{\"line\":184,\"text\":\"generation=\\\"$(jq -er '.generation' \\\"$state\\\")\\\"\"},{\"line\":185,\"text\":\"payload=\\\"$(jq -er '.payload_path' \\\"$state\\\")\\\"\"},{\"line\":186,\"text\":\"printf 'RESUME=YES\\\\nVMID=%s\\\\nGENERATION=%s\\\\nSTATE=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$state\\\"\"},{\"line\":187,\"text\":\"\\\"$TRANSPORT\\\" backup-delete \\\"vm${vmid}\\\" \\\"$generation\\\" \\\"$payload\\\" \\\"$REMOTE_A\\\" \\\"$REMOTE_B\\\" \\\"$STATE_ROOT\\\"\"},{\"line\":189,\"text\":\"generation=\\\"vm${vmid}-${HOST}-$(date -u +%Y%m%dT%H%M%SZ)\\\"\"},{\"line\":190,\"text\":\"gen_dir=\\\"$STAGE_ROOT/vm${vmid}/$generation\\\"\"},{\"line\":191,\"text\":\"[[ ! -e \\\"$gen_dir\\\" ]] || die \\\"GENERATION_DIR_EXISTS_$generation\\\" 65\"},{\"line\":194,\"text\":\"log_file=\\\"$gen_dir/vzdump.log\\\"\"},{\"line\":195,\"text\":\"printf 'RESUME=NO\\\\nVMID=%s\\\\nGENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":196,\"text\":\"vzdump_args=(--dumpdir \\\"$gen_dir\\\" --mode snapshot --compress zstd --ionice 7)\"},{\"line\":198,\"text\":\"[[ -d \\\"$LXC_TMP_ROOT\\\" && ! -L \\\"$LXC_TMP_ROOT\\\" ]] || die \\\"LXC_TMP_ROOT_UNSAFE\\\" 66\"},{\"line\":199,\"text\":\"[[ \\\"$(find \\\"$LXC_TMP_ROOT\\\" -maxdepth 0 -printf '%m')\\\" == \\\"1777\\\" ]] || die \\\"LXC_TMP_ROOT_MODE_UNSAFE\\\" 66\"},{\"line\":200,\"text\":\"vzdump_args+=(--tmpdir \\\"$LXC_TMP_ROOT\\\")\"},{\"line\":202,\"text\":\"if ! ( umask 0022; exec vzdump \\\"$vmid\\\" \\\"${vzdump_args[@]}\\\" ) >\\\"$log_file\\\" 2>&1; then\"},{\"line\":204,\"text\":\"die \\\"VZDUMP_FAILED_$vmid\\\" 70\"},{\"line\":206,\"text\":\"mapfile -t archives < <(find \\\"$gen_dir\\\" -maxdepth 1 -type f -name \\\"vzdump-*-${vmid}-*.zst\\\" -print)\"},{\"line\":208,\"text\":\"[[ \\\"$archive_count\\\" == \\\"1\\\" ]] || { rm -rf -- \\\"$gen_dir\\\"; die \\\"VZDUMP_ARCHIVE_COUNT_${vmid}_${archive_count}\\\" 70; }\"},{\"line\":210,\"text\":\"[[ -s \\\"$archive\\\" && ! -L \\\"$archive\\\" ]] || die \\\"VZDUMP_ARCHIVE_UNSAFE_$vmid\\\" 70\"},{\"line\":212,\"text\":\"\\\"$TRANSPORT\\\" backup-delete \\\"vm${vmid}\\\" \\\"$generation\\\" \\\"$archive\\\" \\\"$REMOTE_A\\\" \\\"$REMOTE_B\\\" \\\"$STATE_ROOT\\\"\"},{\"line\":215,\"text\":\"rmdir -- \\\"$STAGE_ROOT/vm${vmid}\\\" 2>/dev/null || true\"},{\"line\":217,\"text\":\"assert_post_identity \\\"$vmid\\\" \\\"$VM_TYPE\\\"\"},{\"line\":218,\"text\":\"state=\\\"$STATE_ROOT/vm${vmid}/$generation.json\\\"\"},{\"line\":219,\"text\":\"jq -e '.status==\\\"SEALED\\\" and .cloud_quorum==2 and .local_payload_present==false and .local_cleanup==\\\"OK\\\" and .seal_written==true' \\\"$state\\\" >/dev/null\"},{\"line\":220,\"text\":\" restore_validate \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":221,\"text\":\" printf 'BACKUP_STATUS=SEALED\\\\nVMID=%s\\\\nGENERATION=%s\\\\nOWNER=%s\\\\nTYPE=%s\\\\nPOST_STATUS=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$HOST\\\" \\\"$VM_TYPE\\\" \\\"$(expected_status \\\"$vmid\\\")\\\" \\\"$(restore_dir \\\"$vmid\\\")/$generation.json\\\"\"},{\"line\":225,\"text\":\"[[ -d /mnt/staging && ! -L /mnt/staging ]] || die \\\"STAGING_UNSAFE\\\" 66\"},{\"line\":226,\"text\":\"[[ -z \\\"$(active_backup_restore)\\\" ]] || die \\\"ACTIVE_BACKUP_RESTORE_PRESENT\\\" 75\"},{\"line\":230,\"text\":\"[[ \\\"$free\\\" =~ ^[1-9][0-9]+$ && \\\"$a_free\\\" =~ ^[1-9][0-9]+$ && \\\"$b_free\\\" =~ ^[1-9][0-9]+$ ]] || die \\\"CAPACITY_VALUE_INVALID\\\" 65\"},{\"line\":231,\"text\":\"printf 'HOST_PREFLIGHT_STATUS=OK\\\\nHOST=%s\\\\nSTAGING_FREE_BYTES=%s\\\\nREMOTE_A_FREE_BYTES=%s\\\\nREMOTE_B_FREE_BYTES=%s\\\\nACTIVE_BACKUP_RESTORE=0\\\\n' \\\"$HOST\\\" \\\"$free\\\" \\\"$a_free\\\" \\\"$b_free\\\"\"},{\"line\":234,\"text\":\" local vmid=\\\"$1\\\" dir latest gen\"},{\"line\":235,\"text\":\" assert_local_identity \\\"$vmid\\\"; dir=\\\"$STATE_ROOT/vm${vmid}\\\"\"},{\"line\":236,\"text\":\" [[ -d \\\"$dir\\\" ]] || die \\\"STATE_DIR_MISSING_$vmid\\\" 65\"},{\"line\":238,\"text\":\" [[ -n \\\"$latest\\\" && -s \\\"$latest\\\" && ! -L \\\"$latest\\\" ]] || die \\\"LATEST_STATE_MISSING_$vmid\\\" 65\"},{\"line\":239,\"text\":\" jq -e '.status==\\\"SEALED\\\" and .cloud_quorum==2 and .local_payload_present==false and .local_cleanup==\\\"OK\\\" and .seal_written==true' \\\"$latest\\\" >/dev/null || die \\\"LATEST_STATE_NOT_SEALED_$vmid\\\" 65\"},{\"line\":240,\"text\":\" gen=\\\"$(jq -er '.generation' \\\"$latest\\\")\\\"; restore_ok \\\"$vmid\\\" \\\"$gen\\\" || die \\\"LATEST_RESTORE_NOT_OK_$vmid\\\" 65\"},{\"line\":241,\"text\":\" printf 'LATEST_SEALED_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nSTATE_FILE=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$latest\\\" \\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"\"},{\"line\":246,\"text\":\"[[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_HOST_PREFLIGHT\\\" 64\"},{\"line\":250,\"text\":\"[[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_LATEST_SEALED\\\" 64\"},{\"line\":254,\"text\":\"[[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_PLAN\\\" 64\"},{\"line\":257,\"text\":\"backup)\"},{\"line\":258,\"text\":\"[[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_BACKUP\\\" 64\"},{\"line\":259,\"text\":\"backup_one \\\"$2\\\"\"},{\"line\":262,\"text\":\" [[ \\\"$#\\\" == 3 ]] || die \\\"USAGE_RESTORE_VALIDATE\\\" 64\"},{\"line\":266,\"text\":\"[[ \\\"$#\\\" == 3 ]] || die \\\"USAGE_VERIFY\\\" 64\"},{\"line\":270,\"text\":\"die \\\"ACTION_INVALID\\\" 64\"}],\"sha256\":\"91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\"}\nUNIT_STATE={\"homelab-app-cloud-quorum-queue.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-retention-gc.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-v2.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"2\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-backup-v2.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"active\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"waiting\"},\"UnitFileState\":{\"rc\":0,\"value\":\"enabled\"}}}\nRCA_CLASSIFICATION={\"cluster_status\":\"stopped\",\"dispatcher_plan_mentions_9130\":false,\"dispatcher_plan_rc\":0,\"freshness_plan_decision\":null,\"owner_node\":\"pve02\",\"queue_plan_rc\":65,\"worker_contains_status_or_stopped_guard\":true}\nNO_MUTATION=true\nNO_REPO_WRITE=true\nNO_GIT_FETCH=true\nNO_GIT_COMMIT=true\nNO_GIT_PUSH_UPDATE=true\nNO_SYSTEMD_ACTION=true\nNO_BACKUP_STARTED=true\nNO_RESTORE_STARTED=true\nNO_RETENTION_STARTED=true\nNO_DELETE=true\nNO_MAIL_SENT=true\nDURATION_SEC=11.169\nTASK_RESULT=PASS_HOMELAB_BACKUP_VM9130_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\nWORKERS2_BACKUP_VM9130_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1.txt b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1.txt new file mode 100644 index 00000000..cb86a8cf --- /dev/null +++ b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1.txt @@ -0,0 +1,70 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-vm9130-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=a15d8b5f33489f8d1eb5674ec7b4201fe98d7a2525f0dcb5d5cf67b6b072a193 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=dec8f24d8c18ccf29284612a9706bdfba40963983766a7e3552aca0819069bbd +SANITIZED_OUTPUT_SHA256=2d97d7e8351c12124c172903bbb35fcbec82ac9dc28d5ffcd538b8b1c5eca523 +OUTPUT_BEGIN +WORKERS2_BACKUP_VM9130_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1 +MODE=read-only +COMPONENT=homelab-backup-vm9130-rca +HOST=pve01 +UID=0 +MUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL +WORKTREE_HEAD=47b9be72cf2487ba66295d7da998016bbeea57be +WORKTREE_HEAD_EXPECTED=true +WORKTREE_CLEAN=true +WORKTREE_STATUS_LINES=[] +APP_FRESHNESS_PLAN={"lines":["QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH"],"rc":0,"stderr":[]} +VM9130_STATE={"age_seconds":109251,"consecutive_failures":0,"exists":true,"failure_backoff_seconds":0,"generation":"v2-vm9130-20260919T210758Z-0aa987efac8c","last_attempt_epoch":1789853557,"last_success_epoch":1789853557,"logical_id":"vm9130","replica_count":2,"result":"SUCCESS","sha256":"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db"} +VM_QUEUE_FRESHNESS_PLAN={"rc":65,"stderr":["VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02","VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY"],"stdout_sha256":"251ff72648a76f16b91a5f77b421a20fab3f0113e79c118408803793f4befa16","vm9130_lines":[]} +DISPATCHER_PLAN={"lines":["FRESHNESS_STATUS=OK","VMID=160","DECISION=SEALED_PROVEN","GENERATION=vm160-pve02-20260916T023620Z","UPDATED_AT_UTC=2026-09-16T02:46:28Z","SEALED_EPOCH=1789526788","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm160/vm160-pve02-20260916T023620Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json","QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_STALE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=436023 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_STALE","FRESHNESS_STATUS=OK","VMID=170","DECISION=SEALED_PROVEN","GENERATION=vm170-pve01-20260916T103019Z","UPDATED_AT_UTC=2026-09-16T10:47:15Z","SEALED_EPOCH=1789555635","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm170/vm170-pve01-20260916T103019Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json","QUEUE_FRESHNESS VMID=170 DECISION=BACKUP_STALE GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=407177 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN VMID=170 DECISION=BACKUP_STALE","FRESHNESS_STATUS=OK","VMID=130","DECISION=SEALED_PROVEN","GENERATION=vm130-pve03-20260916T105207Z","UPDATED_AT_UTC=2026-09-16T11:25:25Z","SEALED_EPOCH=1789557925","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm130/vm130-pve03-20260916T105207Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json","QUEUE_FRESHNESS VMID=130 DECISION=BACKUP_STALE GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=404888 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN VMID=130 DECISION=BACKUP_STALE","FRESHNESS_STATUS=OK","VMID=171","DECISION=SEALED_PROVEN","GENERATION=vm171-pve01-20260916T023018Z","UPDATED_AT_UTC=2026-09-16T02:35:03Z","SEALED_EPOCH=1789526103","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json","QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_STALE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=436710 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_STALE","CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_STALE PRIORITY=2 AGE_SECONDS=436710","FRESHNESS_STATUS=OK","APP=filebrowser","DECISION=SEALED_PROVEN","GENERATION=app-filebrowser-pve01-20260921T033108Z","UPDATED_AT_UTC=2026-09-21T03:31:18Z","SEALED_EPOCH=1789961478","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260921T033108Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260921T033108Z.json","QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260921T033108Z AGE_SECONDS=1337 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH","FRESHNESS_STATUS=OK","APP=skladchik-reports-monitor","DECISION=SEALED_PROVEN","GENERATION=app-skladchik-reports-monitor-pve01-20260921T033135Z","UPDATED_AT_UTC=2026-09-21T03:31:48Z","SEALED_EPOCH=1789961508","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.json","QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260921T033135Z AGE_SECONDS=1307 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH","FRESHNESS_STATUS=OK","APP=immich","DECISION=SEALED_PROVEN","GENERATION=app-immich-pve01-20260921T033154Z","UPDATED_AT_UTC=2026-09-21T03:32:40Z","SEALED_EPOCH=1789961560","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260921T033154Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260921T033154Z.json","QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260921T033154Z AGE_SECONDS=1255 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH","FRESHNESS_STATUS=OK","APP=hbarhub","DECISION=SEALED_PROVEN","GENERATION=app-hbarhub-pve01-20260921T033309Z","UPDATED_AT_UTC=2026-09-21T03:34:20Z","SEALED_EPOCH=1789961660","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260921T033309Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260921T033309Z.json","QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260921T033309Z AGE_SECONDS=1155 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH","CLASS_DISPATCH KIND=APP DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1","GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_STALE PRIORITY=2 AGE_SECONDS=436710"],"rc":0,"stderr":["VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02","VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY"],"stdout_sha256":"beaf5c144e0d201cbeef18657cd58484b241ff667de66cb9a6483d41cd29ef0b"} +VM9130_REGISTRY_ENTRIES=[{"id":"9130","kind":"VM","lifecycle":"ACTIVE","logical_id":"vm9130","owner":"pve02"}] +VM9130_CLUSTER_RESOURCE={"name":"edge-cold-standby","node":"pve02","status":"stopped","template":0,"type":"qemu","vmid":9130} +VM9130_OWNER_READONLY={"lines":["pve02","status: stopped","agent: enabled=1","boot: order=scsi0","ciuser: debian","cores: 4","cpu: host","description: CR0023 cold standby of VM130. Keep stopped; all NICs link_down=1.","ide2: local-lvm:vm-9130-cloudinit,media=cdrom,size=4M","ipconfig0: ip=[PRIVATE_IP]/24,gw=[PRIVATE_IP]","memory: 12288","meta: creation-qemu=11.0.0,ctime=1781460976","name: edge-cold-standby","nameserver: [PRIVATE_IP] [PRIVATE_IP]","net0: virtio=BC:24:11:7F:04:D6,bridge=vmbr0,firewall=1,link_down=1","onboot: 0","ostype: l26","protection: 1","scsi0: local-lvm:vm-9130-disk-0,discard=on,size=96G,ssd=1","scsi1: local-lvm:vm-9130-disk-1,backup=1,discard=on,size=150G,ssd=1","scsihw: virtio-scsi-pci","searchdomain: gram1.ru","serial0: socket","smbios1: uuid=9327a861-e9f4-4450-b13f-a1e767a83c86","sshkeys: ssh-rsa%20AAAAB3NzaC1yc2EAAAADAQABAAACAQC13sgRxsdexPSyrcz3pfeVJoGajhrb3mJI2gdB2HNWOj6SS%2FcHHiJBKUjg80s6rhZJD%2FtpyLOj%2BQbrkJR3L3FdrgwgCvPcYrBujuZ2TLaRIKIDlNilnRkmVQmUBahgAyrnaQBBl1DVFQsfF0NLF0PYjvPF3789giWqCfv4L14vJSFxvp3wEOjkY1hiwZ8jffMD0R99UW4FTqm29KEFx5ywt%2BdrOT6sMtkBYCIR%2F5JGCB0riH%2F9xn904TIasSMBDz2IeEFpO73bd72JtmnxFyaWaBllKn%2Bu9t18ahMx5vSn2UerGGZbbrZBigxNIwOaHdvdyxADSR3QU90T6nzUv5Fov3vIoOt4uhVaMsYew25hpzm8%2Byp77liUu9mfuMWzo0Q7XJrNfhuCJ9pcrBeqopdThFC36c72JgksWT9Lhfef7bMbEa%2BIWF0wL3L%2FALp1QVA1cbQWPVAF1x3UEHnXuxJCj2x8z4Xli2SJu7TS16qF9Lv1GqVYMeZvW48LwXSbwWcui3B4ldSuo5ksmPeggcTuHzxy5TyQk9sJjlz10Pw9A5zbbuYZDo56n17c1couUm1a3EQ25CimrECygWX2OKcAEdmoiWAihKHyAuLFnSRUELbNrFcFJIb4JkrGk7edWdF%2Btw3jOHlHDcJew08xyoutlN1P9iFTrotMbQjyNTrGgQ%3D%3D%20root%40pve01%0A","startup: order=40","vga: serial0","vmgenid: 110339cf-08fb-4da8-bc7d-2265b8080f2c","91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 /usr/local/libexec/homelab-vm-cloud-quorum-node-worker"],"rc":0,"stderr":[]} +VM_QUEUE_DISPATCH_SOURCE={"lines_140_220":[{"line":140,"text":" printf 'QUEUE_SEALED VMID=%s GENERATION=%s\\n' \"$vmid\" \"$generation\""},{"line":141,"text":" done"},{"line":142,"text":"}"},{"line":143,"text":"dispatch_plan(){"},{"line":144,"text":" freshness_plan"},{"line":145,"text":"}"},{"line":146,"text":"dispatch_one(){"},{"line":147,"text":" local vmid=\"$1\" freshness decision now_epoch out generation"},{"line":148,"text":" [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64"},{"line":149,"text":" expected_owner \"$vmid\" >/dev/null"},{"line":150,"text":" exec 9>\"$LOCK\""},{"line":151,"text":" if ! flock -n 9; then"},{"line":152,"text":" printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\\n' \"$vmid\""},{"line":153,"text":" return 0"},{"line":154,"text":" fi"},{"line":155,"text":" exec 8>\"$HEAVY_LOCK\""},{"line":156,"text":" if ! flock -n 8; then"},{"line":157,"text":" printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_HEAVY_LOCK_BUSY VMID=%s\\n' \"$vmid\""},{"line":158,"text":" return 0"},{"line":159,"text":" fi"},{"line":160,"text":" now_epoch=\"$(date -u +%s)\""},{"line":161,"text":" freshness=\"$(exec 8>&-; exec 9>&-; freshness_decision \"$vmid\" \"$now_epoch\")\""},{"line":162,"text":" printf '%s\\n' \"$freshness\""},{"line":163,"text":" decision=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\""},{"line":164,"text":" case \"$decision\" in"},{"line":165,"text":" SKIP_FRESH)"},{"line":166,"text":" generation=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\""},{"line":167,"text":" printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SKIP_FRESH GENERATION=%s\\n' \"$vmid\" \"$generation\""},{"line":168,"text":" return 0"},{"line":169,"text":" ;;"},{"line":170,"text":" BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;"},{"line":171,"text":" *) die \"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\" 65 ;;"},{"line":172,"text":" esac"},{"line":173,"text":" printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$decision\""},{"line":174,"text":" out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\""},{"line":175,"text":" printf '%s\\n' \"$out\""},{"line":176,"text":" generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\""},{"line":177,"text":" [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$vmid\" 65"},{"line":178,"text":" printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$generation\""},{"line":179,"text":" printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\\n' \"$vmid\" \"$generation\""},{"line":180,"text":"}"},{"line":181,"text":""},{"line":182,"text":"ACTION=\"${1:-}\""},{"line":183,"text":"case \"$ACTION\" in"},{"line":184,"text":" plan)"},{"line":185,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_PLAN\" 64"},{"line":186,"text":" plan_all"},{"line":187,"text":" ;;"},{"line":188,"text":" freshness-plan)"},{"line":189,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_FRESHNESS_PLAN\" 64"},{"line":190,"text":" freshness_plan"},{"line":191,"text":" ;;"},{"line":192,"text":" dispatch-plan)"},{"line":193,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_DISPATCH_PLAN\" 64"},{"line":194,"text":" dispatch_plan"},{"line":195,"text":" ;;"},{"line":196,"text":" dispatch-one)"},{"line":197,"text":" [[ \"$#\" == 2 ]] || die \"USAGE_DISPATCH_ONE\" 64"},{"line":198,"text":" dispatch_one \"$2\""},{"line":199,"text":" ;;"},{"line":200,"text":" canary)"},{"line":201,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_CANARY\" 64"},{"line":202,"text":" backup_list_force 110"},{"line":203,"text":" ;;"},{"line":204,"text":" migrate)"},{"line":205,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_MIGRATE\" 64"},{"line":206,"text":" backup_list_force \"${MIGRATE_VMIDS[@]}\""},{"line":207,"text":" ;;"},{"line":208,"text":" all)"},{"line":209,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_ALL\" 64"},{"line":210,"text":" [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die \"DISPATCHER_MISSING\" 69"},{"line":211,"text":" /usr/local/sbin/homelab-backup-dispatcher run VM_TIMER"},{"line":212,"text":" ;;"},{"line":213,"text":" verify)"},{"line":214,"text":" [[ \"$#\" == 3 ]] || die \"USAGE_VERIFY\" 64"},{"line":215,"text":" run_worker \"$2\" verify \"$3\""},{"line":216,"text":" ;;"},{"line":217,"text":" *)"},{"line":218,"text":" die \"ACTION_INVALID\" 64"},{"line":219,"text":" ;;"},{"line":220,"text":"esac"}],"sha256":"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612"} +DISPATCHER_RUN_SOURCE={"lines_35_60":[{"line":35,"text":"plan(){ select_global; }"},{"line":36,"text":"run(){"},{"line":37,"text":" local trigger=\"$1\" i=0 actions=0 scan pick decision kind target out rc last_sealed=\"\""},{"line":38,"text":" exec 7>\"$LOCK\""},{"line":39,"text":" if ! flock -n 7; then printf 'DISPATCHER_DEFERRED TRIGGER=%s REASON=DISPATCHER_LOCK_BUSY\\n' \"$trigger\"; return 0; fi"},{"line":40,"text":" while (( i < MAX_ITERATIONS )); do"},{"line":41,"text":" i=$((i+1)); scan=\"$(select_global)\"; printf '%s\\n' \"$scan\"; pick=\"$(grep '^GLOBAL_DISPATCH ' <<<\"$scan\" | tail -1)\"; [[ -n \"$pick\" ]] || die GLOBAL_DISPATCH_RESULT_MISSING 65"},{"line":42,"text":" decision=\"\"; kind=\"\"; target=\"\"; for token in $pick; do case \"$token\" in KIND=*) kind=\"${token#KIND=}\";; TARGET=*) target=\"${token#TARGET=}\";; DECISION=*) decision=\"${token#DECISION=}\";; esac; done"},{"line":43,"text":" if [[ \"$decision\" == NOOP ]]; then printf 'DISPATCHER_DONE TRIGGER=%s ACTIONS=%s ITERATIONS=%s\\n' \"$trigger\" \"$actions\" \"$i\"; return 0; fi"},{"line":44,"text":" [[ \"$kind\" == VM || \"$kind\" == APP ]] || die GLOBAL_KIND_INVALID 65; [[ -n \"$target\" ]] || die GLOBAL_TARGET_MISSING 65"},{"line":45,"text":" if [[ \"$last_sealed\" == \"$kind:$target\" ]]; then die \"NO_PROGRESS_AFTER_SEALED_${kind}_${target}\" 65; fi"},{"line":46,"text":" if [[ \"$kind\" == VM ]]; then if out=\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \"$VMQ\" dispatch-one \"$target\" 2>&1)\"; then rc=0; else rc=$?; fi; else if out=\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \"$APPQ\" dispatch-one \"$target\" 2>&1)\"; then rc=0; else rc=$?; fi; fi"},{"line":47,"text":" printf '%s\\n' \"$out\"; (( rc == 0 )) || return \"$rc\""},{"line":48,"text":" if grep -q '^QUEUE_DEFERRED ' <<<\"$out\"; then printf 'DISPATCHER_DEFERRED TRIGGER=%s KIND=%s TARGET=%s REASON=QUEUE_OR_HEAVY_BUSY\\n' \"$trigger\" \"$kind\" \"$target\"; return 0; fi"},{"line":49,"text":" if grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SEALED ' <<<\"$out\"; then last_sealed=\"$kind:$target\"; actions=$((actions+1)); elif grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SKIP_FRESH ' <<<\"$out\"; then last_sealed=\"\"; actions=$((actions+1)); else die \"DISPATCH_RESULT_UNCLASSIFIED_${kind}_${target}\" 65; fi"},{"line":50,"text":" done"},{"line":51,"text":" die MAX_ITERATIONS_EXCEEDED 65"},{"line":52,"text":"}"},{"line":53,"text":"case \"${1:-}\" in plan) [[ \"$#\" == 1 ]] || die USAGE_PLAN 64; plan;; run) [[ \"$#\" == 2 ]] || die USAGE_RUN 64; run \"$2\";; *) die ACTION_INVALID 64;; esac"}],"sha256":"9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c"} +VM_WORKER_BACKUP_GUARDS={"hits":[{"line":6,"text":"STATE_ROOT=/var/lib/homelab-backup/cloud-quorum"},{"line":8,"text":"RESTORE_STATE_ROOT=/var/lib/homelab-backup/vm-restore-validation"},{"line":11,"text":"REMOTE_A='pve01-mail-01-crypt:homelab-backups'"},{"line":12,"text":"REMOTE_B='pve01-mail-02-crypt:homelab-backups'"},{"line":14,"text":"die(){ printf 'VM_QUORUM_WORKER_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }"},{"line":15,"text":"for tool in jq sha256sum zstd tar rclone stat find sort awk date mkfifo tee wc vma; do command -v \"$tool\" >/dev/null 2>&1 || die \"TOOL_MISSING_${tool}\" 69; done"},{"line":16,"text":"[[ -x \"$TRANSPORT\" && ! -L \"$TRANSPORT\" ]] || die TRANSPORT_MISSING 69"},{"line":17,"text":"[[ \"$(sha256sum \"$TRANSPORT\"|awk '{print $1}')\" == \"$TRANSPORT_SHA_EXPECTED\" ]] || die TRANSPORT_SHA_MISMATCH 65"},{"line":21,"text":"active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ \"$rc\" == 1 ]] && return 0; die \"PGREP_FAILED_$rc\" 69; fi; }"},{"line":22,"text":"snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }"},{"line":23,"text":"vm_lock(){ qm config \"$1\" | awk -F': ' '$1==\"lock\"{print $2; found=1} END{if(!found) print \"none\"}'; }"},{"line":24,"text":"expected_owner(){"},{"line":27,"text":"111|113|160|180|9130) printf 'pve02\\n' ;;"},{"line":29,"text":"*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;"},{"line":35,"text":"130|150|160|170|171|180|190|9130) printf 'qemu\\n' ;;"},{"line":36,"text":"*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;"},{"line":39,"text":"expected_status(){"},{"line":40,"text":"if [[ \"$1\" == \"9130\" ]]; then printf 'stopped\\n'; else printf 'running\\n'; fi"},{"line":42,"text":"current_status(){"},{"line":43,"text":"local vmid=\"$1\" type=\"$2\""},{"line":44,"text":"if [[ \"$type\" == \"qemu\" ]]; then qm status \"$vmid\" | awk '{print $2}'; else pct status \"$vmid\" | awk '{print $2}'; fi"},{"line":47,"text":"local vmid=\"$1\" owner type status expect_status"},{"line":48,"text":"owner=\"$(expected_owner \"$vmid\")\""},{"line":49,"text":"type=\"$(expected_type \"$vmid\")\""},{"line":50,"text":"[[ \"$HOST\" == \"$owner\" ]] || die \"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\" 65"},{"line":52,"text":"[[ -s \"/etc/pve/qemu-server/${vmid}.conf\" ]] || die \"QEMU_CONFIG_MISSING_$vmid\" 66"},{"line":54,"text":"[[ -s \"/etc/pve/lxc/${vmid}.conf\" ]] || die \"LXC_CONFIG_MISSING_$vmid\" 66"},{"line":56,"text":"status=\"$(current_status \"$vmid\" \"$type\")\""},{"line":57,"text":"expect_status=\"$(expected_status \"$vmid\")\""},{"line":58,"text":"[[ \"$status\" == \"$expect_status\" ]] || die \"STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\" 66"},{"line":59,"text":"if [[ \"$vmid\" == \"160\" ]]; then [[ \"$(snapshot_count_160)\" == \"0\" ]] || die \"VM160_SNAPSHOT_COUNT_CHANGED\" 66; fi"},{"line":60,"text":"if [[ \"$vmid\" == \"9130\" ]]; then [[ \"$(vm_lock 9130)\" == \"none\" ]] || die \"VM9130_LOCK_PRESENT\" 66; fi"},{"line":62,"text":"VM_STATUS=\"$status\""},{"line":65,"text":"local vmid=\"$1\" type=\"$2\" status expect_status"},{"line":66,"text":"status=\"$(current_status \"$vmid\" \"$type\")\""},{"line":67,"text":"expect_status=\"$(expected_status \"$vmid\")\""},{"line":68,"text":"[[ \"$status\" == \"$expect_status\" ]] || die \"POST_STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\" 66"},{"line":69,"text":"if [[ \"$vmid\" == \"160\" ]]; then [[ \"$(snapshot_count_160)\" == \"0\" ]] || die \"VM160_SNAPSHOT_COUNT_CHANGED_POST\" 66; fi"},{"line":70,"text":"if [[ \"$vmid\" == \"9130\" ]]; then [[ \"$(vm_lock 9130)\" == \"none\" ]] || die \"VM9130_LOCK_PRESENT_POST\" 66; fi"},{"line":72,"text":"latest_incomplete_state(){"},{"line":73,"text":"local vmid=\"$1\" dir=\"$STATE_ROOT/vm${vmid}\" f"},{"line":76,"text":"jq -e '.status!=\"SEALED\"' \"$f\" >/dev/null 2>&1 && printf '%s\\n' \"$f\""},{"line":79,"text":"restore_dir(){ printf '%s\\n' \"$RESTORE_STATE_ROOT/vm$1\"; }"},{"line":81,"text":" local vmid=\"$1\" gen=\"$2\" f state sha"},{"line":82,"text":" f=\"$(restore_dir \"$vmid\")/$gen.json\"; state=\"$STATE_ROOT/vm${vmid}/$gen.json\""},{"line":83,"text":" [[ -s \"$f\" && ! -L \"$f\" && -s \"$state\" && ! -L \"$state\" ]] || return 1"},{"line":84,"text":" sha=\"$(jq -er '.archive_sha256' \"$state\")\" || return 1"},{"line":85,"text":" jq -e --arg l \"vm${vmid}\" --arg g \"$gen\" --arg s \"$state\" --arg sha \"$sha\" '.schema==\"homelab-vm-restore-validation-v1\" and .logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .transport_state_file==$s and .restore_source==\"Cloud A\" and .parts_manifest_verified==true and .payload_sha256==$sha and .payload_sha256_verified==true and .payload_decompression_verified==true and .proxmox_config_extract_verified==true and .archive_recovery_readback_verified==true and .bounded_staging==true and .full_local_payload_written==false and .full_restore_proven==false and .result==\"OK\"' \"$f\" >/dev/null"},{"line":88,"text":" local vmid=\"$1\" gen=\"$2\" state=\"$3\" work=\"$4\" base manifest expected_msha got_msha expected_count expected_size idx name size psha count total"},{"line":89,"text":" base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"; expected_count=\"$(jq -er '.part_count' \"$state\")\"; expected_size=\"$(jq -er '.archive_size_bytes' \"$state\")\""},{"line":90,"text":" [[ \"$base\" == \"${REMOTE_A%/}/vm${vmid}/$gen\" ]] || die \"RESTORE_A_BASE_MISMATCH_$vmid\" 70"},{"line":92,"text":" got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die \"RESTORE_A_MANIFEST_SHA_MISMATCH_$vmid\" 70"},{"line":95,"text":" [[ \"$idx\" =~ ^[0-9]{8}$ && \"$name\" == part-????????.bin && \"$size\" =~ ^[1-9][0-9]*$ && \"$psha\" =~ ^[0-9a-f]{64}$ ]] || die \"RESTORE_A_MANIFEST_ROW_INVALID_$vmid\" 70"},{"line":96,"text":" (( size <= 536870912 )) || die \"RESTORE_A_PART_EXCEEDS_BOUND_$vmid\" 70"},{"line":97,"text":" [[ \"$idx\" == \"$(printf '%08d' \"$count\")\" ]] || die \"RESTORE_A_MANIFEST_INDEX_INVALID_$vmid\" 70"},{"line":100,"text":" [[ \"$count\" == \"$expected_count\" ]] || die \"RESTORE_A_PART_COUNT_MISMATCH_$vmid\" 70"},{"line":101,"text":" [[ \"$total\" == \"$expected_size\" ]] || die \"RESTORE_A_MANIFEST_TOTAL_SIZE_MISMATCH_$vmid\" 70"},{"line":105,"text":" local vmid=\"$1\" base=\"$2\" manifest=\"$3\" work=\"$4\" idx name size psha part actual count"},{"line":109,"text":" [[ -s \"$part\" && ! -L \"$part\" && \"$(stat -c %s \"$part\")\" == \"$size\" ]] || die \"RESTORE_A_PART_SIZE_MISMATCH_$vmid\" 70"},{"line":110,"text":" actual=\"$(sha256sum \"$part\"|awk '{print $1}')\"; [[ \"$actual\" == \"$psha\" ]] || die \"RESTORE_A_PART_SHA_MISMATCH_$vmid\" 70"},{"line":115,"text":" local vmid=\"$1\" gen=\"$2\" logical state work payload_name expected_sha expected_size base manifest fifo sha_out sha_pid pipe_rc sha_rc actual_sha config config_sha members member_count cfg_member level out tmp"},{"line":116,"text":" assert_local_identity \"$vmid\"; logical=\"vm${vmid}\"; state=\"$STATE_ROOT/$logical/$gen.json\""},{"line":117,"text":" [[ -s \"$state\" && ! -L \"$state\" ]] || die \"RESTORE_STATE_MISSING_$vmid\" 65"},{"line":118,"text":" jq -e '.schema==\"homelab-cloud-quorum-payload-v1\" and .status==\"SEALED\" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_payload_present==false and .local_cleanup==\"OK\"' \"$state\" >/dev/null || die \"RESTORE_REQUIRES_STRICT_SEALED_$vmid\" 65"},{"line":119,"text":" if restore_ok \"$vmid\" \"$gen\"; then printf 'RESTORE_VALIDATION_STATUS=OK\\nVMID=%s\\nGENERATION=%s\\nRESULT=ALREADY_VALIDATED\\nRESTORE_EVIDENCE=%s\\n' \"$vmid\" \"$gen\" \"$(restore_dir \"$vmid\")/$gen.json\"; return 0; fi"},{"line":120,"text":" [[ -z \"$(active_backup_restore)\" ]] || die ACTIVE_BACKUP_RESTORE_PRESENT 75"},{"line":121,"text":" payload_name=\"$(jq -er '.payload_name' \"$state\")\"; expected_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; expected_size=\"$(jq -er '.archive_size_bytes' \"$state\")\""},{"line":122,"text":" [[ \"$payload_name\" == \"vzdump-lxc-${vmid}-\"*.tar.zst || \"$payload_name\" == \"vzdump-qemu-${vmid}-\"*.vma.zst ]] || die \"RESTORE_PAYLOAD_NAME_INVALID_$vmid\" 70"},{"line":123,"text":" work=\"$RESTORE_WORK_ROOT/$logical/$gen\"; if [[ -e \"$work\" ]]; then [[ -d \"$work\" && ! -L \"$work\" ]] || die \"RESTORE_WORK_UNSAFE_$vmid\" 70; rm -rf --one-file-system \"$work\"; fi; mkdir -p \"$work\"; chmod 0700 \"$RESTORE_WORK_ROOT\" \"$RESTORE_WORK_ROOT/$logical\" \"$work\"; ACTIVE_RESTORE_STAGE=\"$work\""},{"line":124,"text":" base=\"$(prepare_cloud_a_manifest \"$vmid\" \"$gen\" \"$state\" \"$work\")\"; manifest=\"$work/PARTS.tsv\"; fifo=\"$work/archive.sha.fifo\"; sha_out=\"$work/archive.sha256\"; mkfifo -m 0600 \"$fifo\""},{"line":127,"text":" members=\"$work/members.txt\"; if stream_cloud_a \"$vmid\" \"$base\" \"$manifest\" \"$work\" | tee \"$fifo\" | zstd -dc | tar -tf - >\"$members\"; then pipe_rc=0; else pipe_rc=$?; fi"},{"line":129,"text":" if stream_cloud_a \"$vmid\" \"$base\" \"$manifest\" \"$work\" | tee \"$fifo\" | zstd -t - >/dev/null; then pipe_rc=0; else pipe_rc=$?; fi"},{"line":132,"text":" (( pipe_rc == 0 && sha_rc == 0 )) || die \"RESTORE_A_STREAM_VALIDATION_FAILED_${vmid}_${pipe_rc}_${sha_rc}\" 70"},{"line":133,"text":" actual_sha=\"$(awk 'NR==1{print $1}' \"$sha_out\")\"; [[ \"$actual_sha\" == \"$expected_sha\" ]] || die \"RESTORE_A_PAYLOAD_SHA_MISMATCH_$vmid\" 70"},{"line":136,"text":" [[ -s \"$members\" ]] || die \"RESTORE_LXC_ARCHIVE_EMPTY_$vmid\" 70"},{"line":137,"text":" awk '$0 ~ /^\\// || $0 ~ /(^|\\/)\\.\\.(\\/|$)/ {bad=1} END{exit bad}' \"$members\" || die \"RESTORE_LXC_MEMBER_UNSAFE_$vmid\" 70"},{"line":138,"text":" member_count=\"$(wc -l <\"$members\")\"; [[ \"$member_count\" =~ ^[1-9][0-9]*$ ]] || die \"RESTORE_LXC_MEMBER_COUNT_INVALID_$vmid\" 70"},{"line":139,"text":" cfg_member=\"$(awk '$0==\"./etc/vzdump/pct.conf\" || $0==\"etc/vzdump/pct.conf\" {print; exit}' \"$members\")\"; [[ -n \"$cfg_member\" ]] || die \"RESTORE_LXC_CONFIG_MEMBER_MISSING_$vmid\" 70"},{"line":140,"text":" if ( set +o pipefail; stream_cloud_a \"$vmid\" \"$base\" \"$manifest\" \"$work\" | zstd -dc | tar -xOf - \"$cfg_member\" >\"$config\" ); then :; else die \"RESTORE_LXC_CONFIG_EXTRACT_FAILED_$vmid\" 70; fi"},{"line":143,"text":" if ( set +o pipefail; stream_cloud_a \"$vmid\" \"$base\" \"$manifest\" \"$work\" | zstd -dc | vma config - >\"$config\" ); then :; else die \"RESTORE_QEMU_CONFIG_EXTRACT_FAILED_$vmid\" 70; fi"},{"line":146,"text":" [[ -s \"$config\" && ! -L \"$config\" ]] || die \"RESTORE_CONFIG_EMPTY_$vmid\" 70; config_sha=\"$(sha256sum \"$config\"|awk '{print $1}')\""},{"line":147,"text":" out=\"$(restore_dir \"$vmid\")/$gen.json\"; mkdir -p \"$(dirname \"$out\")\"; chmod 0700 \"$RESTORE_STATE_ROOT\" \"$(dirname \"$out\")\"; tmp=\"${out}.new.$$\""},{"line":148,"text":" jq -n --arg logical \"$logical\" --arg generation \"$gen\" --argjson vmid \"$vmid\" --arg vm_type \"$VM_TYPE\" --arg state \"$state\" --arg sha \"$expected_sha\" --arg config_sha \"$config_sha\" --arg level \"$level\" --argjson member_count \"$member_count\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" '{schema:\"homelab-vm-restore-validation-v1\",logical_id:$logical,generation:$generation,vmid:$vmid,vm_type:$vm_type,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",parts_manifest_verified:true,payload_sha256:$sha,payload_sha256_verified:true,payload_decompression_verified:true,proxmox_config_extract_verified:true,proxmox_config_sha256:$config_sha,lxc_member_count:$member_count,archive_recovery_readback_verified:true,bounded_staging:true,max_staged_part_bytes:536870912,full_local_payload_written:false,full_restore_proven:false,validation_level:$level,result:\"OK\",validated_at_utc:$at}' >\"$tmp\""},{"line":150,"text":" restore_ok \"$vmid\" \"$gen\" || die \"RESTORE_EVIDENCE_POSTWRITE_INVALID_$vmid\" 70"},{"line":151,"text":" printf 'RESTORE_VALIDATION_STATUS=OK\\nVMID=%s\\nGENERATION=%s\\nRESTORE_EVIDENCE=%s\\n' \"$vmid\" \"$gen\" \"$out\""},{"line":154,"text":"local vmid=\"$1\" owner type status"},{"line":155,"text":"assert_local_identity \"$vmid\""},{"line":156,"text":"owner=\"$(expected_owner \"$vmid\")\""},{"line":157,"text":"type=\"$(expected_type \"$vmid\")\""},{"line":158,"text":"status=\"$(expected_status \"$vmid\")\""},{"line":159,"text":"printf 'PLAN_STATUS=OK\\nVMID=%s\\nOWNER=%s\\nTYPE=%s\\nEXPECTED_STATUS=%s\\n' \"$vmid\" \"$owner\" \"$type\" \"$status\""},{"line":162,"text":" local vmid=\"$1\" generation=\"$2\""},{"line":163,"text":" assert_local_identity \"$vmid\""},{"line":164,"text":" restore_validate \"$vmid\" \"$generation\""},{"line":165,"text":" assert_post_identity \"$vmid\" \"$VM_TYPE\""},{"line":166,"text":" printf 'VERIFY_STATUS=OK\\nVMID=%s\\nGENERATION=%s\\nRESTORE_EVIDENCE=%s\\n' \"$vmid\" \"$generation\" \"$(restore_dir \"$vmid\")/$generation.json\""},{"line":168,"text":"backup_one(){"},{"line":169,"text":"local vmid=\"$1\" pending_count pending state generation payload gen_dir archive_count archive log_file before_status"},{"line":170,"text":"local -a vzdump_args"},{"line":171,"text":"assert_local_identity \"$vmid\""},{"line":172,"text":"[[ -x \"$TRANSPORT\" ]] || die \"TRANSPORT_NOT_INSTALLED\" 69"},{"line":173,"text":"[[ -d /mnt/staging && ! -L /mnt/staging ]] || die \"STAGING_UNSAFE\" 66"},{"line":176,"text":"[[ -z \"$(active_backup_restore)\" ]] || die \"ACTIVE_BACKUP_RESTORE_PRESENT\" 75"},{"line":179,"text":"mapfile -t pending < <(latest_incomplete_state \"$vmid\")"},{"line":181,"text":"(( pending_count <= 1 )) || die \"MULTIPLE_INCOMPLETE_GENERATIONS_$vmid\" 65"},{"line":183,"text":"state=\"${pending[0]}\""},{"line":184,"text":"generation=\"$(jq -er '.generation' \"$state\")\""},{"line":185,"text":"payload=\"$(jq -er '.payload_path' \"$state\")\""},{"line":186,"text":"printf 'RESUME=YES\\nVMID=%s\\nGENERATION=%s\\nSTATE=%s\\n' \"$vmid\" \"$generation\" \"$state\""},{"line":187,"text":"\"$TRANSPORT\" backup-delete \"vm${vmid}\" \"$generation\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\""},{"line":189,"text":"generation=\"vm${vmid}-${HOST}-$(date -u +%Y%m%dT%H%M%SZ)\""},{"line":190,"text":"gen_dir=\"$STAGE_ROOT/vm${vmid}/$generation\""},{"line":191,"text":"[[ ! -e \"$gen_dir\" ]] || die \"GENERATION_DIR_EXISTS_$generation\" 65"},{"line":194,"text":"log_file=\"$gen_dir/vzdump.log\""},{"line":195,"text":"printf 'RESUME=NO\\nVMID=%s\\nGENERATION=%s\\n' \"$vmid\" \"$generation\""},{"line":196,"text":"vzdump_args=(--dumpdir \"$gen_dir\" --mode snapshot --compress zstd --ionice 7)"},{"line":198,"text":"[[ -d \"$LXC_TMP_ROOT\" && ! -L \"$LXC_TMP_ROOT\" ]] || die \"LXC_TMP_ROOT_UNSAFE\" 66"},{"line":199,"text":"[[ \"$(find \"$LXC_TMP_ROOT\" -maxdepth 0 -printf '%m')\" == \"1777\" ]] || die \"LXC_TMP_ROOT_MODE_UNSAFE\" 66"},{"line":200,"text":"vzdump_args+=(--tmpdir \"$LXC_TMP_ROOT\")"},{"line":202,"text":"if ! ( umask 0022; exec vzdump \"$vmid\" \"${vzdump_args[@]}\" ) >\"$log_file\" 2>&1; then"},{"line":204,"text":"die \"VZDUMP_FAILED_$vmid\" 70"},{"line":206,"text":"mapfile -t archives < <(find \"$gen_dir\" -maxdepth 1 -type f -name \"vzdump-*-${vmid}-*.zst\" -print)"},{"line":208,"text":"[[ \"$archive_count\" == \"1\" ]] || { rm -rf -- \"$gen_dir\"; die \"VZDUMP_ARCHIVE_COUNT_${vmid}_${archive_count}\" 70; }"},{"line":210,"text":"[[ -s \"$archive\" && ! -L \"$archive\" ]] || die \"VZDUMP_ARCHIVE_UNSAFE_$vmid\" 70"},{"line":212,"text":"\"$TRANSPORT\" backup-delete \"vm${vmid}\" \"$generation\" \"$archive\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\""},{"line":215,"text":"rmdir -- \"$STAGE_ROOT/vm${vmid}\" 2>/dev/null || true"},{"line":217,"text":"assert_post_identity \"$vmid\" \"$VM_TYPE\""},{"line":218,"text":"state=\"$STATE_ROOT/vm${vmid}/$generation.json\""},{"line":219,"text":"jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .local_payload_present==false and .local_cleanup==\"OK\" and .seal_written==true' \"$state\" >/dev/null"},{"line":220,"text":" restore_validate \"$vmid\" \"$generation\""},{"line":221,"text":" printf 'BACKUP_STATUS=SEALED\\nVMID=%s\\nGENERATION=%s\\nOWNER=%s\\nTYPE=%s\\nPOST_STATUS=%s\\nRESTORE_EVIDENCE=%s\\n' \"$vmid\" \"$generation\" \"$HOST\" \"$VM_TYPE\" \"$(expected_status \"$vmid\")\" \"$(restore_dir \"$vmid\")/$generation.json\""},{"line":225,"text":"[[ -d /mnt/staging && ! -L /mnt/staging ]] || die \"STAGING_UNSAFE\" 66"},{"line":226,"text":"[[ -z \"$(active_backup_restore)\" ]] || die \"ACTIVE_BACKUP_RESTORE_PRESENT\" 75"},{"line":230,"text":"[[ \"$free\" =~ ^[1-9][0-9]+$ && \"$a_free\" =~ ^[1-9][0-9]+$ && \"$b_free\" =~ ^[1-9][0-9]+$ ]] || die \"CAPACITY_VALUE_INVALID\" 65"},{"line":231,"text":"printf 'HOST_PREFLIGHT_STATUS=OK\\nHOST=%s\\nSTAGING_FREE_BYTES=%s\\nREMOTE_A_FREE_BYTES=%s\\nREMOTE_B_FREE_BYTES=%s\\nACTIVE_BACKUP_RESTORE=0\\n' \"$HOST\" \"$free\" \"$a_free\" \"$b_free\""},{"line":234,"text":" local vmid=\"$1\" dir latest gen"},{"line":235,"text":" assert_local_identity \"$vmid\"; dir=\"$STATE_ROOT/vm${vmid}\""},{"line":236,"text":" [[ -d \"$dir\" ]] || die \"STATE_DIR_MISSING_$vmid\" 65"},{"line":238,"text":" [[ -n \"$latest\" && -s \"$latest\" && ! -L \"$latest\" ]] || die \"LATEST_STATE_MISSING_$vmid\" 65"},{"line":239,"text":" jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .local_payload_present==false and .local_cleanup==\"OK\" and .seal_written==true' \"$latest\" >/dev/null || die \"LATEST_STATE_NOT_SEALED_$vmid\" 65"},{"line":240,"text":" gen=\"$(jq -er '.generation' \"$latest\")\"; restore_ok \"$vmid\" \"$gen\" || die \"LATEST_RESTORE_NOT_OK_$vmid\" 65"},{"line":241,"text":" printf 'LATEST_SEALED_STATUS=OK\\nVMID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$vmid\" \"$gen\" \"$latest\" \"$(restore_dir \"$vmid\")/$gen.json\""},{"line":246,"text":"[[ \"$#\" == 1 ]] || die \"USAGE_HOST_PREFLIGHT\" 64"},{"line":250,"text":"[[ \"$#\" == 2 ]] || die \"USAGE_LATEST_SEALED\" 64"},{"line":254,"text":"[[ \"$#\" == 2 ]] || die \"USAGE_PLAN\" 64"},{"line":257,"text":"backup)"},{"line":258,"text":"[[ \"$#\" == 2 ]] || die \"USAGE_BACKUP\" 64"},{"line":259,"text":"backup_one \"$2\""},{"line":262,"text":" [[ \"$#\" == 3 ]] || die \"USAGE_RESTORE_VALIDATE\" 64"},{"line":266,"text":"[[ \"$#\" == 3 ]] || die \"USAGE_VERIFY\" 64"},{"line":270,"text":"die \"ACTION_INVALID\" 64"}],"sha256":"91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1"} +UNIT_STATE={"homelab-app-cloud-quorum-queue.timer":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainStatus":{"rc":0,"value":""},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"disabled"}},"homelab-backup-retention-gc.timer":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainStatus":{"rc":0,"value":""},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"disabled"}},"homelab-backup-v2.service":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainStatus":{"rc":0,"value":"2"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"static"}},"homelab-backup-v2.timer":{"ActiveState":{"rc":0,"value":"active"},"ExecMainStatus":{"rc":0,"value":""},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"waiting"},"UnitFileState":{"rc":0,"value":"enabled"}}} +RCA_CLASSIFICATION={"cluster_status":"stopped","dispatcher_plan_mentions_9130":false,"dispatcher_plan_rc":0,"freshness_plan_decision":null,"owner_node":"pve02","queue_plan_rc":65,"worker_contains_status_or_stopped_guard":true} +NO_MUTATION=true +NO_REPO_WRITE=true +NO_GIT_FETCH=true +NO_GIT_COMMIT=true +NO_GIT_PUSH_UPDATE=true +NO_SYSTEMD_ACTION=true +NO_BACKUP_STARTED=true +NO_RESTORE_STARTED=true +NO_RETENTION_STARTED=true +NO_DELETE=true +NO_MAIL_SENT=true +DURATION_SEC=11.169 +TASK_RESULT=PASS_HOMELAB_BACKUP_VM9130_RCA_READONLY +HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1","rollback_restored":null,"rollback_started":false,"status":"OK","version":1} +WORKERS2_BACKUP_VM9130_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index ce1f7084..5f90b1b0 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260921-HOMELAB-BACKUP-TARGETED-CONTINUATION-1047R1", - "status": "FAIL", - "rc": 1, + "command_id": "SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "homelab-backup-targeted-continuation", - "started_at_utc": "2026-09-21T03:50:39Z", - "finished_at_utc": "2026-09-21T03:50:41Z", + "mode": "read-only", + "component": "homelab-backup-vm9130-rca", + "started_at_utc": "2026-09-21T03:53:27Z", + "finished_at_utc": "2026-09-21T03:53:38Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "82608ea4b9da22360373565f61912c6ab5f62cdf8ca0c7dc9020cfaba9117924", + "command_sha256": "a15d8b5f33489f8d1eb5674ec7b4201fe98d7a2525f0dcb5d5cf67b6b072a193", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, + "change_declared": false, "result_contract_valid": true, - "result_contract_status": "FAIL", + "result_contract_status": null, "result_contract_error": null, - "command_rc": 1, - "changes_made": true, - "rollback_started": true, - "rollback_restored": true, - "mutation_outcome": "ROLLED_BACK", + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "cc7d08ca13508be1d1a71150dfcf653da21a6bac9973cb87d778b15ba1655360", - "sanitized_output_sha256": "cc7d08ca13508be1d1a71150dfcf653da21a6bac9973cb87d778b15ba1655360", + "raw_evidence_sha256": "dec8f24d8c18ccf29284612a9706bdfba40963983766a7e3552aca0819069bbd", + "sanitized_output_sha256": "2d97d7e8351c12124c172903bbb35fcbec82ac9dc28d5ffcd538b8b1c5eca523", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_BACKUP_TARGETED_CONTINUATION_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-TARGETED-CONTINUATION-1047R1\nMODE=change\nCOMPONENT=homelab-backup-targeted-continuation\nMUTATION_BOUNDARY=PRESERVE_1045_SOURCE_AND_APP_GENERATIONS;TARGET_VM9130_VIA_PROVEN_DISPATCH_ONE;INSTALL_COMMITTED_APP_SCHEDULER_RETENTION_AUDIT;STABLE_RETENTION_PLAN_SAFE_GC;ENABLE_TIMERS;NO_MAIL_NO_ALERTMANAGER_CHANGE_NO_SOURCE_PUSH_NO_RESTORE_DRILL\nAPP_PRE_CONTINUATION_DECISIONS={\"filebrowser\":\"SKIP_FRESH\",\"hbarhub\":\"SKIP_FRESH\",\"immich\":\"SKIP_FRESH\",\"skladchik-reports-monitor\":\"SKIP_FRESH\"}\nVM9130_BEFORE={\"age_seconds\":109083,\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm9130-20260919T210758Z-0aa987efac8c\",\"last_attempt_epoch\":1789853557,\"last_success_epoch\":1789853557,\"replica_count\":2,\"result\":\"SUCCESS\",\"sha256\":\"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db\"}\nFAILURE=VM9130_DISPATCH_ONE_RC_65\nVM_BACKUP_STARTED=true\nVM_BACKUP_FRESH=false\nRUNTIME_CHANGED=false\nRETENTION_GC_STARTED=false\nRETENTION_GC_SUCCESS=false\nROLLBACK_STARTED_INTERNAL=true\nROLLBACK_RESTORED_INTERNAL=true\nTASK_RESULT=FAIL_HOMELAB_BACKUP_TARGETED_CONTINUATION\nHOMELAB_RESULT_CONTRACT={\"changes_made\":true,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-TARGETED-CONTINUATION-1047R1\",\"rollback_restored\":true,\"rollback_started\":true,\"status\":\"FAIL\",\"version\":1}\nWORKERS2_BACKUP_TARGETED_CONTINUATION_END=1\n" + "output": "WORKERS2_BACKUP_VM9130_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1\nMODE=read-only\nCOMPONENT=homelab-backup-vm9130-rca\nHOST=pve01\nUID=0\nMUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL\nWORKTREE_HEAD=47b9be72cf2487ba66295d7da998016bbeea57be\nWORKTREE_HEAD_EXPECTED=true\nWORKTREE_CLEAN=true\nWORKTREE_STATUS_LINES=[]\nAPP_FRESHNESS_PLAN={\"lines\":[\"QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\"],\"rc\":0,\"stderr\":[]}\nVM9130_STATE={\"age_seconds\":109251,\"consecutive_failures\":0,\"exists\":true,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm9130-20260919T210758Z-0aa987efac8c\",\"last_attempt_epoch\":1789853557,\"last_success_epoch\":1789853557,\"logical_id\":\"vm9130\",\"replica_count\":2,\"result\":\"SUCCESS\",\"sha256\":\"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db\"}\nVM_QUEUE_FRESHNESS_PLAN={\"rc\":65,\"stderr\":[\"VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\",\"VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\"],\"stdout_sha256\":\"251ff72648a76f16b91a5f77b421a20fab3f0113e79c118408803793f4befa16\",\"vm9130_lines\":[]}\nDISPATCHER_PLAN={\"lines\":[\"FRESHNESS_STATUS=OK\",\"VMID=160\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm160-pve02-20260916T023620Z\",\"UPDATED_AT_UTC=2026-09-16T02:46:28Z\",\"SEALED_EPOCH=1789526788\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm160/vm160-pve02-20260916T023620Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json\",\"QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_STALE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=436023 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_STALE\",\"FRESHNESS_STATUS=OK\",\"VMID=170\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm170-pve01-20260916T103019Z\",\"UPDATED_AT_UTC=2026-09-16T10:47:15Z\",\"SEALED_EPOCH=1789555635\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm170/vm170-pve01-20260916T103019Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json\",\"QUEUE_FRESHNESS VMID=170 DECISION=BACKUP_STALE GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=407177 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=170 DECISION=BACKUP_STALE\",\"FRESHNESS_STATUS=OK\",\"VMID=130\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm130-pve03-20260916T105207Z\",\"UPDATED_AT_UTC=2026-09-16T11:25:25Z\",\"SEALED_EPOCH=1789557925\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm130/vm130-pve03-20260916T105207Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json\",\"QUEUE_FRESHNESS VMID=130 DECISION=BACKUP_STALE GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=404888 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=130 DECISION=BACKUP_STALE\",\"FRESHNESS_STATUS=OK\",\"VMID=171\",\"DECISION=SEALED_PROVEN\",\"GENERATION=vm171-pve01-20260916T023018Z\",\"UPDATED_AT_UTC=2026-09-16T02:35:03Z\",\"SEALED_EPOCH=1789526103\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\",\"QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_STALE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=436710 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_STALE\",\"CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_STALE PRIORITY=2 AGE_SECONDS=436710\",\"FRESHNESS_STATUS=OK\",\"APP=filebrowser\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-filebrowser-pve01-20260921T033108Z\",\"UPDATED_AT_UTC=2026-09-21T03:31:18Z\",\"SEALED_EPOCH=1789961478\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260921T033108Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260921T033108Z.json\",\"QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260921T033108Z AGE_SECONDS=1337 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\",\"FRESHNESS_STATUS=OK\",\"APP=skladchik-reports-monitor\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"UPDATED_AT_UTC=2026-09-21T03:31:48Z\",\"SEALED_EPOCH=1789961508\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260921T033135Z AGE_SECONDS=1307 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\",\"FRESHNESS_STATUS=OK\",\"APP=immich\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-immich-pve01-20260921T033154Z\",\"UPDATED_AT_UTC=2026-09-21T03:32:40Z\",\"SEALED_EPOCH=1789961560\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260921T033154Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260921T033154Z.json\",\"QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260921T033154Z AGE_SECONDS=1255 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\",\"FRESHNESS_STATUS=OK\",\"APP=hbarhub\",\"DECISION=SEALED_PROVEN\",\"GENERATION=app-hbarhub-pve01-20260921T033309Z\",\"UPDATED_AT_UTC=2026-09-21T03:34:20Z\",\"SEALED_EPOCH=1789961660\",\"STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260921T033309Z.json\",\"RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260921T033309Z.json\",\"QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260921T033309Z AGE_SECONDS=1155 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000\",\"QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\",\"CLASS_DISPATCH KIND=APP DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1\",\"GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_STALE PRIORITY=2 AGE_SECONDS=436710\"],\"rc\":0,\"stderr\":[\"VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02\",\"VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY\"],\"stdout_sha256\":\"beaf5c144e0d201cbeef18657cd58484b241ff667de66cb9a6483d41cd29ef0b\"}\nVM9130_REGISTRY_ENTRIES=[{\"id\":\"9130\",\"kind\":\"VM\",\"lifecycle\":\"ACTIVE\",\"logical_id\":\"vm9130\",\"owner\":\"pve02\"}]\nVM9130_CLUSTER_RESOURCE={\"name\":\"edge-cold-standby\",\"node\":\"pve02\",\"status\":\"stopped\",\"template\":0,\"type\":\"qemu\",\"vmid\":9130}\nVM9130_OWNER_READONLY={\"lines\":[\"pve02\",\"status: stopped\",\"agent: enabled=1\",\"boot: order=scsi0\",\"ciuser: debian\",\"cores: 4\",\"cpu: host\",\"description: CR0023 cold standby of VM130. Keep stopped; all NICs link_down=1.\",\"ide2: local-lvm:vm-9130-cloudinit,media=cdrom,size=4M\",\"ipconfig0: ip=[PRIVATE_IP]/24,gw=[PRIVATE_IP]\",\"memory: 12288\",\"meta: creation-qemu=11.0.0,ctime=1781460976\",\"name: edge-cold-standby\",\"nameserver: [PRIVATE_IP] [PRIVATE_IP]\",\"net0: virtio=BC:24:11:7F:04:D6,bridge=vmbr0,firewall=1,link_down=1\",\"onboot: 0\",\"ostype: l26\",\"protection: 1\",\"scsi0: local-lvm:vm-9130-disk-0,discard=on,size=96G,ssd=1\",\"scsi1: local-lvm:vm-9130-disk-1,backup=1,discard=on,size=150G,ssd=1\",\"scsihw: virtio-scsi-pci\",\"searchdomain: gram1.ru\",\"serial0: socket\",\"smbios1: uuid=9327a861-e9f4-4450-b13f-a1e767a83c86\",\"sshkeys: ssh-rsa%20AAAAB3NzaC1yc2EAAAADAQABAAACAQC13sgRxsdexPSyrcz3pfeVJoGajhrb3mJI2gdB2HNWOj6SS%2FcHHiJBKUjg80s6rhZJD%2FtpyLOj%2BQbrkJR3L3FdrgwgCvPcYrBujuZ2TLaRIKIDlNilnRkmVQmUBahgAyrnaQBBl1DVFQsfF0NLF0PYjvPF3789giWqCfv4L14vJSFxvp3wEOjkY1hiwZ8jffMD0R99UW4FTqm29KEFx5ywt%2BdrOT6sMtkBYCIR%2F5JGCB0riH%2F9xn904TIasSMBDz2IeEFpO73bd72JtmnxFyaWaBllKn%2Bu9t18ahMx5vSn2UerGGZbbrZBigxNIwOaHdvdyxADSR3QU90T6nzUv5Fov3vIoOt4uhVaMsYew25hpzm8%2Byp77liUu9mfuMWzo0Q7XJrNfhuCJ9pcrBeqopdThFC36c72JgksWT9Lhfef7bMbEa%2BIWF0wL3L%2FALp1QVA1cbQWPVAF1x3UEHnXuxJCj2x8z4Xli2SJu7TS16qF9Lv1GqVYMeZvW48LwXSbwWcui3B4ldSuo5ksmPeggcTuHzxy5TyQk9sJjlz10Pw9A5zbbuYZDo56n17c1couUm1a3EQ25CimrECygWX2OKcAEdmoiWAihKHyAuLFnSRUELbNrFcFJIb4JkrGk7edWdF%2Btw3jOHlHDcJew08xyoutlN1P9iFTrotMbQjyNTrGgQ%3D%3D%20root%40pve01%0A\",\"startup: order=40\",\"vga: serial0\",\"vmgenid: 110339cf-08fb-4da8-bc7d-2265b8080f2c\",\"91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 /usr/local/libexec/homelab-vm-cloud-quorum-node-worker\"],\"rc\":0,\"stderr\":[]}\nVM_QUEUE_DISPATCH_SOURCE={\"lines_140_220\":[{\"line\":140,\"text\":\" printf 'QUEUE_SEALED VMID=%s GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":141,\"text\":\" done\"},{\"line\":142,\"text\":\"}\"},{\"line\":143,\"text\":\"dispatch_plan(){\"},{\"line\":144,\"text\":\" freshness_plan\"},{\"line\":145,\"text\":\"}\"},{\"line\":146,\"text\":\"dispatch_one(){\"},{\"line\":147,\"text\":\" local vmid=\\\"$1\\\" freshness decision now_epoch out generation\"},{\"line\":148,\"text\":\" [[ \\\"${HOMELAB_BACKUP_DISPATCHER:-}\\\" == 1 ]] || die \\\"DISPATCH_ONE_DIRECT_FORBIDDEN\\\" 64\"},{\"line\":149,\"text\":\" expected_owner \\\"$vmid\\\" >/dev/null\"},{\"line\":150,\"text\":\" exec 9>\\\"$LOCK\\\"\"},{\"line\":151,\"text\":\" if ! flock -n 9; then\"},{\"line\":152,\"text\":\" printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\\\\n' \\\"$vmid\\\"\"},{\"line\":153,\"text\":\" return 0\"},{\"line\":154,\"text\":\" fi\"},{\"line\":155,\"text\":\" exec 8>\\\"$HEAVY_LOCK\\\"\"},{\"line\":156,\"text\":\" if ! flock -n 8; then\"},{\"line\":157,\"text\":\" printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_HEAVY_LOCK_BUSY VMID=%s\\\\n' \\\"$vmid\\\"\"},{\"line\":158,\"text\":\" return 0\"},{\"line\":159,\"text\":\" fi\"},{\"line\":160,\"text\":\" now_epoch=\\\"$(date -u +%s)\\\"\"},{\"line\":161,\"text\":\" freshness=\\\"$(exec 8>&-; exec 9>&-; freshness_decision \\\"$vmid\\\" \\\"$now_epoch\\\")\\\"\"},{\"line\":162,\"text\":\" printf '%s\\\\n' \\\"$freshness\\\"\"},{\"line\":163,\"text\":\" decision=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":164,\"text\":\" case \\\"$decision\\\" in\"},{\"line\":165,\"text\":\" SKIP_FRESH)\"},{\"line\":166,\"text\":\" generation=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":167,\"text\":\" printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SKIP_FRESH GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":168,\"text\":\" return 0\"},{\"line\":169,\"text\":\" ;;\"},{\"line\":170,\"text\":\" BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\"},{\"line\":171,\"text\":\" *) die \\\"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\\\" 65 ;;\"},{\"line\":172,\"text\":\" esac\"},{\"line\":173,\"text\":\" printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\\\n' \\\"$vmid\\\" \\\"$(expected_owner \\\"$vmid\\\")\\\" \\\"$decision\\\"\"},{\"line\":174,\"text\":\" out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":175,\"text\":\" printf '%s\\\\n' \\\"$out\\\"\"},{\"line\":176,\"text\":\" generation=\\\"$(awk -F= '$1==\\\"GENERATION\\\"{print $2}' <<<\\\"$out\\\" | tail -1)\\\"\"},{\"line\":177,\"text\":\" [[ -n \\\"$generation\\\" ]] || die \\\"GENERATION_NOT_RETURNED_$vmid\\\" 65\"},{\"line\":178,\"text\":\" printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":179,\"text\":\" printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":180,\"text\":\"}\"},{\"line\":181,\"text\":\"\"},{\"line\":182,\"text\":\"ACTION=\\\"${1:-}\\\"\"},{\"line\":183,\"text\":\"case \\\"$ACTION\\\" in\"},{\"line\":184,\"text\":\" plan)\"},{\"line\":185,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_PLAN\\\" 64\"},{\"line\":186,\"text\":\" plan_all\"},{\"line\":187,\"text\":\" ;;\"},{\"line\":188,\"text\":\" freshness-plan)\"},{\"line\":189,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_FRESHNESS_PLAN\\\" 64\"},{\"line\":190,\"text\":\" freshness_plan\"},{\"line\":191,\"text\":\" ;;\"},{\"line\":192,\"text\":\" dispatch-plan)\"},{\"line\":193,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_DISPATCH_PLAN\\\" 64\"},{\"line\":194,\"text\":\" dispatch_plan\"},{\"line\":195,\"text\":\" ;;\"},{\"line\":196,\"text\":\" dispatch-one)\"},{\"line\":197,\"text\":\" [[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_DISPATCH_ONE\\\" 64\"},{\"line\":198,\"text\":\" dispatch_one \\\"$2\\\"\"},{\"line\":199,\"text\":\" ;;\"},{\"line\":200,\"text\":\" canary)\"},{\"line\":201,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_CANARY\\\" 64\"},{\"line\":202,\"text\":\" backup_list_force 110\"},{\"line\":203,\"text\":\" ;;\"},{\"line\":204,\"text\":\" migrate)\"},{\"line\":205,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_MIGRATE\\\" 64\"},{\"line\":206,\"text\":\" backup_list_force \\\"${MIGRATE_VMIDS[@]}\\\"\"},{\"line\":207,\"text\":\" ;;\"},{\"line\":208,\"text\":\" all)\"},{\"line\":209,\"text\":\" [[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_ALL\\\" 64\"},{\"line\":210,\"text\":\" [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die \\\"DISPATCHER_MISSING\\\" 69\"},{\"line\":211,\"text\":\" /usr/local/sbin/homelab-backup-dispatcher run VM_TIMER\"},{\"line\":212,\"text\":\" ;;\"},{\"line\":213,\"text\":\" verify)\"},{\"line\":214,\"text\":\" [[ \\\"$#\\\" == 3 ]] || die \\\"USAGE_VERIFY\\\" 64\"},{\"line\":215,\"text\":\" run_worker \\\"$2\\\" verify \\\"$3\\\"\"},{\"line\":216,\"text\":\" ;;\"},{\"line\":217,\"text\":\" *)\"},{\"line\":218,\"text\":\" die \\\"ACTION_INVALID\\\" 64\"},{\"line\":219,\"text\":\" ;;\"},{\"line\":220,\"text\":\"esac\"}],\"sha256\":\"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\"}\nDISPATCHER_RUN_SOURCE={\"lines_35_60\":[{\"line\":35,\"text\":\"plan(){ select_global; }\"},{\"line\":36,\"text\":\"run(){\"},{\"line\":37,\"text\":\" local trigger=\\\"$1\\\" i=0 actions=0 scan pick decision kind target out rc last_sealed=\\\"\\\"\"},{\"line\":38,\"text\":\" exec 7>\\\"$LOCK\\\"\"},{\"line\":39,\"text\":\" if ! flock -n 7; then printf 'DISPATCHER_DEFERRED TRIGGER=%s REASON=DISPATCHER_LOCK_BUSY\\\\n' \\\"$trigger\\\"; return 0; fi\"},{\"line\":40,\"text\":\" while (( i < MAX_ITERATIONS )); do\"},{\"line\":41,\"text\":\" i=$((i+1)); scan=\\\"$(select_global)\\\"; printf '%s\\\\n' \\\"$scan\\\"; pick=\\\"$(grep '^GLOBAL_DISPATCH ' <<<\\\"$scan\\\" | tail -1)\\\"; [[ -n \\\"$pick\\\" ]] || die GLOBAL_DISPATCH_RESULT_MISSING 65\"},{\"line\":42,\"text\":\" decision=\\\"\\\"; kind=\\\"\\\"; target=\\\"\\\"; for token in $pick; do case \\\"$token\\\" in KIND=*) kind=\\\"${token#KIND=}\\\";; TARGET=*) target=\\\"${token#TARGET=}\\\";; DECISION=*) decision=\\\"${token#DECISION=}\\\";; esac; done\"},{\"line\":43,\"text\":\" if [[ \\\"$decision\\\" == NOOP ]]; then printf 'DISPATCHER_DONE TRIGGER=%s ACTIONS=%s ITERATIONS=%s\\\\n' \\\"$trigger\\\" \\\"$actions\\\" \\\"$i\\\"; return 0; fi\"},{\"line\":44,\"text\":\" [[ \\\"$kind\\\" == VM || \\\"$kind\\\" == APP ]] || die GLOBAL_KIND_INVALID 65; [[ -n \\\"$target\\\" ]] || die GLOBAL_TARGET_MISSING 65\"},{\"line\":45,\"text\":\" if [[ \\\"$last_sealed\\\" == \\\"$kind:$target\\\" ]]; then die \\\"NO_PROGRESS_AFTER_SEALED_${kind}_${target}\\\" 65; fi\"},{\"line\":46,\"text\":\" if [[ \\\"$kind\\\" == VM ]]; then if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$VMQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; else if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$APPQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; fi\"},{\"line\":47,\"text\":\" printf '%s\\\\n' \\\"$out\\\"; (( rc == 0 )) || return \\\"$rc\\\"\"},{\"line\":48,\"text\":\" if grep -q '^QUEUE_DEFERRED ' <<<\\\"$out\\\"; then printf 'DISPATCHER_DEFERRED TRIGGER=%s KIND=%s TARGET=%s REASON=QUEUE_OR_HEAVY_BUSY\\\\n' \\\"$trigger\\\" \\\"$kind\\\" \\\"$target\\\"; return 0; fi\"},{\"line\":49,\"text\":\" if grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SEALED ' <<<\\\"$out\\\"; then last_sealed=\\\"$kind:$target\\\"; actions=$((actions+1)); elif grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SKIP_FRESH ' <<<\\\"$out\\\"; then last_sealed=\\\"\\\"; actions=$((actions+1)); else die \\\"DISPATCH_RESULT_UNCLASSIFIED_${kind}_${target}\\\" 65; fi\"},{\"line\":50,\"text\":\" done\"},{\"line\":51,\"text\":\" die MAX_ITERATIONS_EXCEEDED 65\"},{\"line\":52,\"text\":\"}\"},{\"line\":53,\"text\":\"case \\\"${1:-}\\\" in plan) [[ \\\"$#\\\" == 1 ]] || die USAGE_PLAN 64; plan;; run) [[ \\\"$#\\\" == 2 ]] || die USAGE_RUN 64; run \\\"$2\\\";; *) die ACTION_INVALID 64;; esac\"}],\"sha256\":\"9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c\"}\nVM_WORKER_BACKUP_GUARDS={\"hits\":[{\"line\":6,\"text\":\"STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\"},{\"line\":8,\"text\":\"RESTORE_STATE_ROOT=/var/lib/homelab-backup/vm-restore-validation\"},{\"line\":11,\"text\":\"REMOTE_A='pve01-mail-01-crypt:homelab-backups'\"},{\"line\":12,\"text\":\"REMOTE_B='pve01-mail-02-crypt:homelab-backups'\"},{\"line\":14,\"text\":\"die(){ printf 'VM_QUORUM_WORKER_ERROR=%s\\\\n' \\\"$1\\\" >&2; exit \\\"${2:-65}\\\"; }\"},{\"line\":15,\"text\":\"for tool in jq sha256sum zstd tar rclone stat find sort awk date mkfifo tee wc vma; do command -v \\\"$tool\\\" >/dev/null 2>&1 || die \\\"TOOL_MISSING_${tool}\\\" 69; done\"},{\"line\":16,\"text\":\"[[ -x \\\"$TRANSPORT\\\" && ! -L \\\"$TRANSPORT\\\" ]] || die TRANSPORT_MISSING 69\"},{\"line\":17,\"text\":\"[[ \\\"$(sha256sum \\\"$TRANSPORT\\\"|awk '{print $1}')\\\" == \\\"$TRANSPORT_SHA_EXPECTED\\\" ]] || die TRANSPORT_SHA_MISMATCH 65\"},{\"line\":21,\"text\":\"active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ \\\"$rc\\\" == 1 ]] && return 0; die \\\"PGREP_FAILED_$rc\\\" 69; fi; }\"},{\"line\":22,\"text\":\"snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }\"},{\"line\":23,\"text\":\"vm_lock(){ qm config \\\"$1\\\" | awk -F': ' '$1==\\\"lock\\\"{print $2; found=1} END{if(!found) print \\\"none\\\"}'; }\"},{\"line\":24,\"text\":\"expected_owner(){\"},{\"line\":27,\"text\":\"111|113|160|180|9130) printf 'pve02\\\\n' ;;\"},{\"line\":29,\"text\":\"*) die \\\"VMID_NOT_ALLOWED_$1\\\" 64 ;;\"},{\"line\":35,\"text\":\"130|150|160|170|171|180|190|9130) printf 'qemu\\\\n' ;;\"},{\"line\":36,\"text\":\"*) die \\\"VMID_NOT_ALLOWED_$1\\\" 64 ;;\"},{\"line\":39,\"text\":\"expected_status(){\"},{\"line\":40,\"text\":\"if [[ \\\"$1\\\" == \\\"9130\\\" ]]; then printf 'stopped\\\\n'; else printf 'running\\\\n'; fi\"},{\"line\":42,\"text\":\"current_status(){\"},{\"line\":43,\"text\":\"local vmid=\\\"$1\\\" type=\\\"$2\\\"\"},{\"line\":44,\"text\":\"if [[ \\\"$type\\\" == \\\"qemu\\\" ]]; then qm status \\\"$vmid\\\" | awk '{print $2}'; else pct status \\\"$vmid\\\" | awk '{print $2}'; fi\"},{\"line\":47,\"text\":\"local vmid=\\\"$1\\\" owner type status expect_status\"},{\"line\":48,\"text\":\"owner=\\\"$(expected_owner \\\"$vmid\\\")\\\"\"},{\"line\":49,\"text\":\"type=\\\"$(expected_type \\\"$vmid\\\")\\\"\"},{\"line\":50,\"text\":\"[[ \\\"$HOST\\\" == \\\"$owner\\\" ]] || die \\\"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\\\" 65\"},{\"line\":52,\"text\":\"[[ -s \\\"/etc/pve/qemu-server/${vmid}.conf\\\" ]] || die \\\"QEMU_CONFIG_MISSING_$vmid\\\" 66\"},{\"line\":54,\"text\":\"[[ -s \\\"/etc/pve/lxc/${vmid}.conf\\\" ]] || die \\\"LXC_CONFIG_MISSING_$vmid\\\" 66\"},{\"line\":56,\"text\":\"status=\\\"$(current_status \\\"$vmid\\\" \\\"$type\\\")\\\"\"},{\"line\":57,\"text\":\"expect_status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":58,\"text\":\"[[ \\\"$status\\\" == \\\"$expect_status\\\" ]] || die \\\"STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\\\" 66\"},{\"line\":59,\"text\":\"if [[ \\\"$vmid\\\" == \\\"160\\\" ]]; then [[ \\\"$(snapshot_count_160)\\\" == \\\"0\\\" ]] || die \\\"VM160_SNAPSHOT_COUNT_CHANGED\\\" 66; fi\"},{\"line\":60,\"text\":\"if [[ \\\"$vmid\\\" == \\\"9130\\\" ]]; then [[ \\\"$(vm_lock 9130)\\\" == \\\"none\\\" ]] || die \\\"VM9130_LOCK_PRESENT\\\" 66; fi\"},{\"line\":62,\"text\":\"VM_STATUS=\\\"$status\\\"\"},{\"line\":65,\"text\":\"local vmid=\\\"$1\\\" type=\\\"$2\\\" status expect_status\"},{\"line\":66,\"text\":\"status=\\\"$(current_status \\\"$vmid\\\" \\\"$type\\\")\\\"\"},{\"line\":67,\"text\":\"expect_status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":68,\"text\":\"[[ \\\"$status\\\" == \\\"$expect_status\\\" ]] || die \\\"POST_STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\\\" 66\"},{\"line\":69,\"text\":\"if [[ \\\"$vmid\\\" == \\\"160\\\" ]]; then [[ \\\"$(snapshot_count_160)\\\" == \\\"0\\\" ]] || die \\\"VM160_SNAPSHOT_COUNT_CHANGED_POST\\\" 66; fi\"},{\"line\":70,\"text\":\"if [[ \\\"$vmid\\\" == \\\"9130\\\" ]]; then [[ \\\"$(vm_lock 9130)\\\" == \\\"none\\\" ]] || die \\\"VM9130_LOCK_PRESENT_POST\\\" 66; fi\"},{\"line\":72,\"text\":\"latest_incomplete_state(){\"},{\"line\":73,\"text\":\"local vmid=\\\"$1\\\" dir=\\\"$STATE_ROOT/vm${vmid}\\\" f\"},{\"line\":76,\"text\":\"jq -e '.status!=\\\"SEALED\\\"' \\\"$f\\\" >/dev/null 2>&1 && printf '%s\\\\n' \\\"$f\\\"\"},{\"line\":79,\"text\":\"restore_dir(){ printf '%s\\\\n' \\\"$RESTORE_STATE_ROOT/vm$1\\\"; }\"},{\"line\":81,\"text\":\" local vmid=\\\"$1\\\" gen=\\\"$2\\\" f state sha\"},{\"line\":82,\"text\":\" f=\\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; state=\\\"$STATE_ROOT/vm${vmid}/$gen.json\\\"\"},{\"line\":83,\"text\":\" [[ -s \\\"$f\\\" && ! -L \\\"$f\\\" && -s \\\"$state\\\" && ! -L \\\"$state\\\" ]] || return 1\"},{\"line\":84,\"text\":\" sha=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\" || return 1\"},{\"line\":85,\"text\":\" jq -e --arg l \\\"vm${vmid}\\\" --arg g \\\"$gen\\\" --arg s \\\"$state\\\" --arg sha \\\"$sha\\\" '.schema==\\\"homelab-vm-restore-validation-v1\\\" and .logical_id==$l and .generation==$g and .transport_state==\\\"SEALED\\\" and .transport_state_file==$s and .restore_source==\\\"Cloud A\\\" and .parts_manifest_verified==true and .payload_sha256==$sha and .payload_sha256_verified==true and .payload_decompression_verified==true and .proxmox_config_extract_verified==true and .archive_recovery_readback_verified==true and .bounded_staging==true and .full_local_payload_written==false and .full_restore_proven==false and .result==\\\"OK\\\"' \\\"$f\\\" >/dev/null\"},{\"line\":88,\"text\":\" local vmid=\\\"$1\\\" gen=\\\"$2\\\" state=\\\"$3\\\" work=\\\"$4\\\" base manifest expected_msha got_msha expected_count expected_size idx name size psha count total\"},{\"line\":89,\"text\":\" base=\\\"$(jq -er '.remote_a_base' \\\"$state\\\")\\\"; expected_msha=\\\"$(jq -er '.parts_manifest_sha256' \\\"$state\\\")\\\"; expected_count=\\\"$(jq -er '.part_count' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"line\":90,\"text\":\" [[ \\\"$base\\\" == \\\"${REMOTE_A%/}/vm${vmid}/$gen\\\" ]] || die \\\"RESTORE_A_BASE_MISMATCH_$vmid\\\" 70\"},{\"line\":92,\"text\":\" got_msha=\\\"$(sha256sum \\\"$manifest\\\"|awk '{print $1}')\\\"; [[ \\\"$got_msha\\\" == \\\"$expected_msha\\\" ]] || die \\\"RESTORE_A_MANIFEST_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":95,\"text\":\" [[ \\\"$idx\\\" =~ ^[0-9]{8}$ && \\\"$name\\\" == part-????????.bin && \\\"$size\\\" =~ ^[1-9][0-9]*$ && \\\"$psha\\\" =~ ^[0-9a-f]{64}$ ]] || die \\\"RESTORE_A_MANIFEST_ROW_INVALID_$vmid\\\" 70\"},{\"line\":96,\"text\":\" (( size <= 536870912 )) || die \\\"RESTORE_A_PART_EXCEEDS_BOUND_$vmid\\\" 70\"},{\"line\":97,\"text\":\" [[ \\\"$idx\\\" == \\\"$(printf '%08d' \\\"$count\\\")\\\" ]] || die \\\"RESTORE_A_MANIFEST_INDEX_INVALID_$vmid\\\" 70\"},{\"line\":100,\"text\":\" [[ \\\"$count\\\" == \\\"$expected_count\\\" ]] || die \\\"RESTORE_A_PART_COUNT_MISMATCH_$vmid\\\" 70\"},{\"line\":101,\"text\":\" [[ \\\"$total\\\" == \\\"$expected_size\\\" ]] || die \\\"RESTORE_A_MANIFEST_TOTAL_SIZE_MISMATCH_$vmid\\\" 70\"},{\"line\":105,\"text\":\" local vmid=\\\"$1\\\" base=\\\"$2\\\" manifest=\\\"$3\\\" work=\\\"$4\\\" idx name size psha part actual count\"},{\"line\":109,\"text\":\" [[ -s \\\"$part\\\" && ! -L \\\"$part\\\" && \\\"$(stat -c %s \\\"$part\\\")\\\" == \\\"$size\\\" ]] || die \\\"RESTORE_A_PART_SIZE_MISMATCH_$vmid\\\" 70\"},{\"line\":110,\"text\":\" actual=\\\"$(sha256sum \\\"$part\\\"|awk '{print $1}')\\\"; [[ \\\"$actual\\\" == \\\"$psha\\\" ]] || die \\\"RESTORE_A_PART_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":115,\"text\":\" local vmid=\\\"$1\\\" gen=\\\"$2\\\" logical state work payload_name expected_sha expected_size base manifest fifo sha_out sha_pid pipe_rc sha_rc actual_sha config config_sha members member_count cfg_member level out tmp\"},{\"line\":116,\"text\":\" assert_local_identity \\\"$vmid\\\"; logical=\\\"vm${vmid}\\\"; state=\\\"$STATE_ROOT/$logical/$gen.json\\\"\"},{\"line\":117,\"text\":\" [[ -s \\\"$state\\\" && ! -L \\\"$state\\\" ]] || die \\\"RESTORE_STATE_MISSING_$vmid\\\" 65\"},{\"line\":118,\"text\":\" jq -e '.schema==\\\"homelab-cloud-quorum-payload-v1\\\" and .status==\\\"SEALED\\\" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_payload_present==false and .local_cleanup==\\\"OK\\\"' \\\"$state\\\" >/dev/null || die \\\"RESTORE_REQUIRES_STRICT_SEALED_$vmid\\\" 65\"},{\"line\":119,\"text\":\" if restore_ok \\\"$vmid\\\" \\\"$gen\\\"; then printf 'RESTORE_VALIDATION_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESULT=ALREADY_VALIDATED\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; return 0; fi\"},{\"line\":120,\"text\":\" [[ -z \\\"$(active_backup_restore)\\\" ]] || die ACTIVE_BACKUP_RESTORE_PRESENT 75\"},{\"line\":121,\"text\":\" payload_name=\\\"$(jq -er '.payload_name' \\\"$state\\\")\\\"; expected_sha=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"line\":122,\"text\":\" [[ \\\"$payload_name\\\" == \\\"vzdump-lxc-${vmid}-\\\"*.tar.zst || \\\"$payload_name\\\" == \\\"vzdump-qemu-${vmid}-\\\"*.vma.zst ]] || die \\\"RESTORE_PAYLOAD_NAME_INVALID_$vmid\\\" 70\"},{\"line\":123,\"text\":\" work=\\\"$RESTORE_WORK_ROOT/$logical/$gen\\\"; if [[ -e \\\"$work\\\" ]]; then [[ -d \\\"$work\\\" && ! -L \\\"$work\\\" ]] || die \\\"RESTORE_WORK_UNSAFE_$vmid\\\" 70; rm -rf --one-file-system \\\"$work\\\"; fi; mkdir -p \\\"$work\\\"; chmod 0700 \\\"$RESTORE_WORK_ROOT\\\" \\\"$RESTORE_WORK_ROOT/$logical\\\" \\\"$work\\\"; ACTIVE_RESTORE_STAGE=\\\"$work\\\"\"},{\"line\":124,\"text\":\" base=\\\"$(prepare_cloud_a_manifest \\\"$vmid\\\" \\\"$gen\\\" \\\"$state\\\" \\\"$work\\\")\\\"; manifest=\\\"$work/PARTS.tsv\\\"; fifo=\\\"$work/archive.sha.fifo\\\"; sha_out=\\\"$work/archive.sha256\\\"; mkfifo -m 0600 \\\"$fifo\\\"\"},{\"line\":127,\"text\":\" members=\\\"$work/members.txt\\\"; if stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | tee \\\"$fifo\\\" | zstd -dc | tar -tf - >\\\"$members\\\"; then pipe_rc=0; else pipe_rc=$?; fi\"},{\"line\":129,\"text\":\" if stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | tee \\\"$fifo\\\" | zstd -t - >/dev/null; then pipe_rc=0; else pipe_rc=$?; fi\"},{\"line\":132,\"text\":\" (( pipe_rc == 0 && sha_rc == 0 )) || die \\\"RESTORE_A_STREAM_VALIDATION_FAILED_${vmid}_${pipe_rc}_${sha_rc}\\\" 70\"},{\"line\":133,\"text\":\" actual_sha=\\\"$(awk 'NR==1{print $1}' \\\"$sha_out\\\")\\\"; [[ \\\"$actual_sha\\\" == \\\"$expected_sha\\\" ]] || die \\\"RESTORE_A_PAYLOAD_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":136,\"text\":\" [[ -s \\\"$members\\\" ]] || die \\\"RESTORE_LXC_ARCHIVE_EMPTY_$vmid\\\" 70\"},{\"line\":137,\"text\":\" awk '$0 ~ /^\\\\// || $0 ~ /(^|\\\\/)\\\\.\\\\.(\\\\/|$)/ {bad=1} END{exit bad}' \\\"$members\\\" || die \\\"RESTORE_LXC_MEMBER_UNSAFE_$vmid\\\" 70\"},{\"line\":138,\"text\":\" member_count=\\\"$(wc -l <\\\"$members\\\")\\\"; [[ \\\"$member_count\\\" =~ ^[1-9][0-9]*$ ]] || die \\\"RESTORE_LXC_MEMBER_COUNT_INVALID_$vmid\\\" 70\"},{\"line\":139,\"text\":\" cfg_member=\\\"$(awk '$0==\\\"./etc/vzdump/pct.conf\\\" || $0==\\\"etc/vzdump/pct.conf\\\" {print; exit}' \\\"$members\\\")\\\"; [[ -n \\\"$cfg_member\\\" ]] || die \\\"RESTORE_LXC_CONFIG_MEMBER_MISSING_$vmid\\\" 70\"},{\"line\":140,\"text\":\" if ( set +o pipefail; stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | zstd -dc | tar -xOf - \\\"$cfg_member\\\" >\\\"$config\\\" ); then :; else die \\\"RESTORE_LXC_CONFIG_EXTRACT_FAILED_$vmid\\\" 70; fi\"},{\"line\":143,\"text\":\" if ( set +o pipefail; stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | zstd -dc | vma config - >\\\"$config\\\" ); then :; else die \\\"RESTORE_QEMU_CONFIG_EXTRACT_FAILED_$vmid\\\" 70; fi\"},{\"line\":146,\"text\":\" [[ -s \\\"$config\\\" && ! -L \\\"$config\\\" ]] || die \\\"RESTORE_CONFIG_EMPTY_$vmid\\\" 70; config_sha=\\\"$(sha256sum \\\"$config\\\"|awk '{print $1}')\\\"\"},{\"line\":147,\"text\":\" out=\\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; mkdir -p \\\"$(dirname \\\"$out\\\")\\\"; chmod 0700 \\\"$RESTORE_STATE_ROOT\\\" \\\"$(dirname \\\"$out\\\")\\\"; tmp=\\\"${out}.new.$$\\\"\"},{\"line\":148,\"text\":\" jq -n --arg logical \\\"$logical\\\" --arg generation \\\"$gen\\\" --argjson vmid \\\"$vmid\\\" --arg vm_type \\\"$VM_TYPE\\\" --arg state \\\"$state\\\" --arg sha \\\"$expected_sha\\\" --arg config_sha \\\"$config_sha\\\" --arg level \\\"$level\\\" --argjson member_count \\\"$member_count\\\" --arg at \\\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\\\" '{schema:\\\"homelab-vm-restore-validation-v1\\\",logical_id:$logical,generation:$generation,vmid:$vmid,vm_type:$vm_type,transport_state:\\\"SEALED\\\",transport_state_file:$state,restore_source:\\\"Cloud A\\\",parts_manifest_verified:true,payload_sha256:$sha,payload_sha256_verified:true,payload_decompression_verified:true,proxmox_config_extract_verified:true,proxmox_config_sha256:$config_sha,lxc_member_count:$member_count,archive_recovery_readback_verified:true,bounded_staging:true,max_staged_part_bytes:536870912,full_local_payload_written:false,full_restore_proven:false,validation_level:$level,result:\\\"OK\\\",validated_at_utc:$at}' >\\\"$tmp\\\"\"},{\"line\":150,\"text\":\" restore_ok \\\"$vmid\\\" \\\"$gen\\\" || die \\\"RESTORE_EVIDENCE_POSTWRITE_INVALID_$vmid\\\" 70\"},{\"line\":151,\"text\":\" printf 'RESTORE_VALIDATION_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$out\\\"\"},{\"line\":154,\"text\":\"local vmid=\\\"$1\\\" owner type status\"},{\"line\":155,\"text\":\"assert_local_identity \\\"$vmid\\\"\"},{\"line\":156,\"text\":\"owner=\\\"$(expected_owner \\\"$vmid\\\")\\\"\"},{\"line\":157,\"text\":\"type=\\\"$(expected_type \\\"$vmid\\\")\\\"\"},{\"line\":158,\"text\":\"status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":159,\"text\":\"printf 'PLAN_STATUS=OK\\\\nVMID=%s\\\\nOWNER=%s\\\\nTYPE=%s\\\\nEXPECTED_STATUS=%s\\\\n' \\\"$vmid\\\" \\\"$owner\\\" \\\"$type\\\" \\\"$status\\\"\"},{\"line\":162,\"text\":\" local vmid=\\\"$1\\\" generation=\\\"$2\\\"\"},{\"line\":163,\"text\":\" assert_local_identity \\\"$vmid\\\"\"},{\"line\":164,\"text\":\" restore_validate \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":165,\"text\":\" assert_post_identity \\\"$vmid\\\" \\\"$VM_TYPE\\\"\"},{\"line\":166,\"text\":\" printf 'VERIFY_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$(restore_dir \\\"$vmid\\\")/$generation.json\\\"\"},{\"line\":168,\"text\":\"backup_one(){\"},{\"line\":169,\"text\":\"local vmid=\\\"$1\\\" pending_count pending state generation payload gen_dir archive_count archive log_file before_status\"},{\"line\":170,\"text\":\"local -a vzdump_args\"},{\"line\":171,\"text\":\"assert_local_identity \\\"$vmid\\\"\"},{\"line\":172,\"text\":\"[[ -x \\\"$TRANSPORT\\\" ]] || die \\\"TRANSPORT_NOT_INSTALLED\\\" 69\"},{\"line\":173,\"text\":\"[[ -d /mnt/staging && ! -L /mnt/staging ]] || die \\\"STAGING_UNSAFE\\\" 66\"},{\"line\":176,\"text\":\"[[ -z \\\"$(active_backup_restore)\\\" ]] || die \\\"ACTIVE_BACKUP_RESTORE_PRESENT\\\" 75\"},{\"line\":179,\"text\":\"mapfile -t pending < <(latest_incomplete_state \\\"$vmid\\\")\"},{\"line\":181,\"text\":\"(( pending_count <= 1 )) || die \\\"MULTIPLE_INCOMPLETE_GENERATIONS_$vmid\\\" 65\"},{\"line\":183,\"text\":\"state=\\\"${pending[0]}\\\"\"},{\"line\":184,\"text\":\"generation=\\\"$(jq -er '.generation' \\\"$state\\\")\\\"\"},{\"line\":185,\"text\":\"payload=\\\"$(jq -er '.payload_path' \\\"$state\\\")\\\"\"},{\"line\":186,\"text\":\"printf 'RESUME=YES\\\\nVMID=%s\\\\nGENERATION=%s\\\\nSTATE=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$state\\\"\"},{\"line\":187,\"text\":\"\\\"$TRANSPORT\\\" backup-delete \\\"vm${vmid}\\\" \\\"$generation\\\" \\\"$payload\\\" \\\"$REMOTE_A\\\" \\\"$REMOTE_B\\\" \\\"$STATE_ROOT\\\"\"},{\"line\":189,\"text\":\"generation=\\\"vm${vmid}-${HOST}-$(date -u +%Y%m%dT%H%M%SZ)\\\"\"},{\"line\":190,\"text\":\"gen_dir=\\\"$STAGE_ROOT/vm${vmid}/$generation\\\"\"},{\"line\":191,\"text\":\"[[ ! -e \\\"$gen_dir\\\" ]] || die \\\"GENERATION_DIR_EXISTS_$generation\\\" 65\"},{\"line\":194,\"text\":\"log_file=\\\"$gen_dir/vzdump.log\\\"\"},{\"line\":195,\"text\":\"printf 'RESUME=NO\\\\nVMID=%s\\\\nGENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":196,\"text\":\"vzdump_args=(--dumpdir \\\"$gen_dir\\\" --mode snapshot --compress zstd --ionice 7)\"},{\"line\":198,\"text\":\"[[ -d \\\"$LXC_TMP_ROOT\\\" && ! -L \\\"$LXC_TMP_ROOT\\\" ]] || die \\\"LXC_TMP_ROOT_UNSAFE\\\" 66\"},{\"line\":199,\"text\":\"[[ \\\"$(find \\\"$LXC_TMP_ROOT\\\" -maxdepth 0 -printf '%m')\\\" == \\\"1777\\\" ]] || die \\\"LXC_TMP_ROOT_MODE_UNSAFE\\\" 66\"},{\"line\":200,\"text\":\"vzdump_args+=(--tmpdir \\\"$LXC_TMP_ROOT\\\")\"},{\"line\":202,\"text\":\"if ! ( umask 0022; exec vzdump \\\"$vmid\\\" \\\"${vzdump_args[@]}\\\" ) >\\\"$log_file\\\" 2>&1; then\"},{\"line\":204,\"text\":\"die \\\"VZDUMP_FAILED_$vmid\\\" 70\"},{\"line\":206,\"text\":\"mapfile -t archives < <(find \\\"$gen_dir\\\" -maxdepth 1 -type f -name \\\"vzdump-*-${vmid}-*.zst\\\" -print)\"},{\"line\":208,\"text\":\"[[ \\\"$archive_count\\\" == \\\"1\\\" ]] || { rm -rf -- \\\"$gen_dir\\\"; die \\\"VZDUMP_ARCHIVE_COUNT_${vmid}_${archive_count}\\\" 70; }\"},{\"line\":210,\"text\":\"[[ -s \\\"$archive\\\" && ! -L \\\"$archive\\\" ]] || die \\\"VZDUMP_ARCHIVE_UNSAFE_$vmid\\\" 70\"},{\"line\":212,\"text\":\"\\\"$TRANSPORT\\\" backup-delete \\\"vm${vmid}\\\" \\\"$generation\\\" \\\"$archive\\\" \\\"$REMOTE_A\\\" \\\"$REMOTE_B\\\" \\\"$STATE_ROOT\\\"\"},{\"line\":215,\"text\":\"rmdir -- \\\"$STAGE_ROOT/vm${vmid}\\\" 2>/dev/null || true\"},{\"line\":217,\"text\":\"assert_post_identity \\\"$vmid\\\" \\\"$VM_TYPE\\\"\"},{\"line\":218,\"text\":\"state=\\\"$STATE_ROOT/vm${vmid}/$generation.json\\\"\"},{\"line\":219,\"text\":\"jq -e '.status==\\\"SEALED\\\" and .cloud_quorum==2 and .local_payload_present==false and .local_cleanup==\\\"OK\\\" and .seal_written==true' \\\"$state\\\" >/dev/null\"},{\"line\":220,\"text\":\" restore_validate \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":221,\"text\":\" printf 'BACKUP_STATUS=SEALED\\\\nVMID=%s\\\\nGENERATION=%s\\\\nOWNER=%s\\\\nTYPE=%s\\\\nPOST_STATUS=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$HOST\\\" \\\"$VM_TYPE\\\" \\\"$(expected_status \\\"$vmid\\\")\\\" \\\"$(restore_dir \\\"$vmid\\\")/$generation.json\\\"\"},{\"line\":225,\"text\":\"[[ -d /mnt/staging && ! -L /mnt/staging ]] || die \\\"STAGING_UNSAFE\\\" 66\"},{\"line\":226,\"text\":\"[[ -z \\\"$(active_backup_restore)\\\" ]] || die \\\"ACTIVE_BACKUP_RESTORE_PRESENT\\\" 75\"},{\"line\":230,\"text\":\"[[ \\\"$free\\\" =~ ^[1-9][0-9]+$ && \\\"$a_free\\\" =~ ^[1-9][0-9]+$ && \\\"$b_free\\\" =~ ^[1-9][0-9]+$ ]] || die \\\"CAPACITY_VALUE_INVALID\\\" 65\"},{\"line\":231,\"text\":\"printf 'HOST_PREFLIGHT_STATUS=OK\\\\nHOST=%s\\\\nSTAGING_FREE_BYTES=%s\\\\nREMOTE_A_FREE_BYTES=%s\\\\nREMOTE_B_FREE_BYTES=%s\\\\nACTIVE_BACKUP_RESTORE=0\\\\n' \\\"$HOST\\\" \\\"$free\\\" \\\"$a_free\\\" \\\"$b_free\\\"\"},{\"line\":234,\"text\":\" local vmid=\\\"$1\\\" dir latest gen\"},{\"line\":235,\"text\":\" assert_local_identity \\\"$vmid\\\"; dir=\\\"$STATE_ROOT/vm${vmid}\\\"\"},{\"line\":236,\"text\":\" [[ -d \\\"$dir\\\" ]] || die \\\"STATE_DIR_MISSING_$vmid\\\" 65\"},{\"line\":238,\"text\":\" [[ -n \\\"$latest\\\" && -s \\\"$latest\\\" && ! -L \\\"$latest\\\" ]] || die \\\"LATEST_STATE_MISSING_$vmid\\\" 65\"},{\"line\":239,\"text\":\" jq -e '.status==\\\"SEALED\\\" and .cloud_quorum==2 and .local_payload_present==false and .local_cleanup==\\\"OK\\\" and .seal_written==true' \\\"$latest\\\" >/dev/null || die \\\"LATEST_STATE_NOT_SEALED_$vmid\\\" 65\"},{\"line\":240,\"text\":\" gen=\\\"$(jq -er '.generation' \\\"$latest\\\")\\\"; restore_ok \\\"$vmid\\\" \\\"$gen\\\" || die \\\"LATEST_RESTORE_NOT_OK_$vmid\\\" 65\"},{\"line\":241,\"text\":\" printf 'LATEST_SEALED_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nSTATE_FILE=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$latest\\\" \\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"\"},{\"line\":246,\"text\":\"[[ \\\"$#\\\" == 1 ]] || die \\\"USAGE_HOST_PREFLIGHT\\\" 64\"},{\"line\":250,\"text\":\"[[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_LATEST_SEALED\\\" 64\"},{\"line\":254,\"text\":\"[[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_PLAN\\\" 64\"},{\"line\":257,\"text\":\"backup)\"},{\"line\":258,\"text\":\"[[ \\\"$#\\\" == 2 ]] || die \\\"USAGE_BACKUP\\\" 64\"},{\"line\":259,\"text\":\"backup_one \\\"$2\\\"\"},{\"line\":262,\"text\":\" [[ \\\"$#\\\" == 3 ]] || die \\\"USAGE_RESTORE_VALIDATE\\\" 64\"},{\"line\":266,\"text\":\"[[ \\\"$#\\\" == 3 ]] || die \\\"USAGE_VERIFY\\\" 64\"},{\"line\":270,\"text\":\"die \\\"ACTION_INVALID\\\" 64\"}],\"sha256\":\"91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\"}\nUNIT_STATE={\"homelab-app-cloud-quorum-queue.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-retention-gc.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-v2.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"2\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-backup-v2.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"active\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"waiting\"},\"UnitFileState\":{\"rc\":0,\"value\":\"enabled\"}}}\nRCA_CLASSIFICATION={\"cluster_status\":\"stopped\",\"dispatcher_plan_mentions_9130\":false,\"dispatcher_plan_rc\":0,\"freshness_plan_decision\":null,\"owner_node\":\"pve02\",\"queue_plan_rc\":65,\"worker_contains_status_or_stopped_guard\":true}\nNO_MUTATION=true\nNO_REPO_WRITE=true\nNO_GIT_FETCH=true\nNO_GIT_COMMIT=true\nNO_GIT_PUSH_UPDATE=true\nNO_SYSTEMD_ACTION=true\nNO_BACKUP_STARTED=true\nNO_RESTORE_STARTED=true\nNO_RETENTION_STARTED=true\nNO_DELETE=true\nNO_MAIL_SENT=true\nDURATION_SEC=11.169\nTASK_RESULT=PASS_HOMELAB_BACKUP_VM9130_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\nWORKERS2_BACKUP_VM9130_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index a053a18f..cb86a8cf 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,50 +1,70 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-TARGETED-CONTINUATION-1047R1 -STATUS=FAIL -RC=1 +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=homelab-backup-targeted-continuation +MODE=read-only +COMPONENT=homelab-backup-vm9130-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=82608ea4b9da22360373565f61912c6ab5f62cdf8ca0c7dc9020cfaba9117924 +COMMAND_SHA256=a15d8b5f33489f8d1eb5674ec7b4201fe98d7a2525f0dcb5d5cf67b6b072a193 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true +CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=1 -CHANGES_MADE=true -ROLLBACK_STARTED=true -ROLLBACK_RESTORED=true -MUTATION_OUTCOME=ROLLED_BACK +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=cc7d08ca13508be1d1a71150dfcf653da21a6bac9973cb87d778b15ba1655360 -SANITIZED_OUTPUT_SHA256=cc7d08ca13508be1d1a71150dfcf653da21a6bac9973cb87d778b15ba1655360 +RAW_EVIDENCE_SHA256=dec8f24d8c18ccf29284612a9706bdfba40963983766a7e3552aca0819069bbd +SANITIZED_OUTPUT_SHA256=2d97d7e8351c12124c172903bbb35fcbec82ac9dc28d5ffcd538b8b1c5eca523 OUTPUT_BEGIN -WORKERS2_BACKUP_TARGETED_CONTINUATION_BEGIN=1 -COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-TARGETED-CONTINUATION-1047R1 -MODE=change -COMPONENT=homelab-backup-targeted-continuation -MUTATION_BOUNDARY=PRESERVE_1045_SOURCE_AND_APP_GENERATIONS;TARGET_VM9130_VIA_PROVEN_DISPATCH_ONE;INSTALL_COMMITTED_APP_SCHEDULER_RETENTION_AUDIT;STABLE_RETENTION_PLAN_SAFE_GC;ENABLE_TIMERS;NO_MAIL_NO_ALERTMANAGER_CHANGE_NO_SOURCE_PUSH_NO_RESTORE_DRILL -APP_PRE_CONTINUATION_DECISIONS={"filebrowser":"SKIP_FRESH","hbarhub":"SKIP_FRESH","immich":"SKIP_FRESH","skladchik-reports-monitor":"SKIP_FRESH"} -VM9130_BEFORE={"age_seconds":109083,"consecutive_failures":0,"failure_backoff_seconds":0,"generation":"v2-vm9130-20260919T210758Z-0aa987efac8c","last_attempt_epoch":1789853557,"last_success_epoch":1789853557,"replica_count":2,"result":"SUCCESS","sha256":"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db"} -FAILURE=VM9130_DISPATCH_ONE_RC_65 -VM_BACKUP_STARTED=true -VM_BACKUP_FRESH=false -RUNTIME_CHANGED=false -RETENTION_GC_STARTED=false -RETENTION_GC_SUCCESS=false -ROLLBACK_STARTED_INTERNAL=true -ROLLBACK_RESTORED_INTERNAL=true -TASK_RESULT=FAIL_HOMELAB_BACKUP_TARGETED_CONTINUATION -HOMELAB_RESULT_CONTRACT={"changes_made":true,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-TARGETED-CONTINUATION-1047R1","rollback_restored":true,"rollback_started":true,"status":"FAIL","version":1} -WORKERS2_BACKUP_TARGETED_CONTINUATION_END=1 +WORKERS2_BACKUP_VM9130_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1 +MODE=read-only +COMPONENT=homelab-backup-vm9130-rca +HOST=pve01 +UID=0 +MUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL +WORKTREE_HEAD=47b9be72cf2487ba66295d7da998016bbeea57be +WORKTREE_HEAD_EXPECTED=true +WORKTREE_CLEAN=true +WORKTREE_STATUS_LINES=[] +APP_FRESHNESS_PLAN={"lines":["QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH"],"rc":0,"stderr":[]} +VM9130_STATE={"age_seconds":109251,"consecutive_failures":0,"exists":true,"failure_backoff_seconds":0,"generation":"v2-vm9130-20260919T210758Z-0aa987efac8c","last_attempt_epoch":1789853557,"last_success_epoch":1789853557,"logical_id":"vm9130","replica_count":2,"result":"SUCCESS","sha256":"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db"} +VM_QUEUE_FRESHNESS_PLAN={"rc":65,"stderr":["VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02","VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY"],"stdout_sha256":"251ff72648a76f16b91a5f77b421a20fab3f0113e79c118408803793f4befa16","vm9130_lines":[]} +DISPATCHER_PLAN={"lines":["FRESHNESS_STATUS=OK","VMID=160","DECISION=SEALED_PROVEN","GENERATION=vm160-pve02-20260916T023620Z","UPDATED_AT_UTC=2026-09-16T02:46:28Z","SEALED_EPOCH=1789526788","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm160/vm160-pve02-20260916T023620Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json","QUEUE_FRESHNESS VMID=160 DECISION=BACKUP_STALE GENERATION=vm160-pve02-20260916T023620Z AGE_SECONDS=436023 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN VMID=160 DECISION=BACKUP_STALE","FRESHNESS_STATUS=OK","VMID=170","DECISION=SEALED_PROVEN","GENERATION=vm170-pve01-20260916T103019Z","UPDATED_AT_UTC=2026-09-16T10:47:15Z","SEALED_EPOCH=1789555635","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm170/vm170-pve01-20260916T103019Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json","QUEUE_FRESHNESS VMID=170 DECISION=BACKUP_STALE GENERATION=vm170-pve01-20260916T103019Z AGE_SECONDS=407177 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN VMID=170 DECISION=BACKUP_STALE","FRESHNESS_STATUS=OK","VMID=130","DECISION=SEALED_PROVEN","GENERATION=vm130-pve03-20260916T105207Z","UPDATED_AT_UTC=2026-09-16T11:25:25Z","SEALED_EPOCH=1789557925","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm130/vm130-pve03-20260916T105207Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json","QUEUE_FRESHNESS VMID=130 DECISION=BACKUP_STALE GENERATION=vm130-pve03-20260916T105207Z AGE_SECONDS=404888 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN VMID=130 DECISION=BACKUP_STALE","FRESHNESS_STATUS=OK","VMID=171","DECISION=SEALED_PROVEN","GENERATION=vm171-pve01-20260916T023018Z","UPDATED_AT_UTC=2026-09-16T02:35:03Z","SEALED_EPOCH=1789526103","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm171/vm171-pve01-20260916T023018Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json","QUEUE_FRESHNESS VMID=171 DECISION=BACKUP_STALE GENERATION=vm171-pve01-20260916T023018Z AGE_SECONDS=436710 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN VMID=171 DECISION=BACKUP_STALE","CLASS_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_STALE PRIORITY=2 AGE_SECONDS=436710","FRESHNESS_STATUS=OK","APP=filebrowser","DECISION=SEALED_PROVEN","GENERATION=app-filebrowser-pve01-20260921T033108Z","UPDATED_AT_UTC=2026-09-21T03:31:18Z","SEALED_EPOCH=1789961478","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-filebrowser/app-filebrowser-pve01-20260921T033108Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260921T033108Z.json","QUEUE_FRESHNESS APP=filebrowser DECISION=SKIP_FRESH GENERATION=app-filebrowser-pve01-20260921T033108Z AGE_SECONDS=1337 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH","FRESHNESS_STATUS=OK","APP=skladchik-reports-monitor","DECISION=SEALED_PROVEN","GENERATION=app-skladchik-reports-monitor-pve01-20260921T033135Z","UPDATED_AT_UTC=2026-09-21T03:31:48Z","SEALED_EPOCH=1789961508","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.json","QUEUE_FRESHNESS APP=skladchik-reports-monitor DECISION=SKIP_FRESH GENERATION=app-skladchik-reports-monitor-pve01-20260921T033135Z AGE_SECONDS=1307 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH","FRESHNESS_STATUS=OK","APP=immich","DECISION=SEALED_PROVEN","GENERATION=app-immich-pve01-20260921T033154Z","UPDATED_AT_UTC=2026-09-21T03:32:40Z","SEALED_EPOCH=1789961560","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-immich/app-immich-pve01-20260921T033154Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260921T033154Z.json","QUEUE_FRESHNESS APP=immich DECISION=SKIP_FRESH GENERATION=app-immich-pve01-20260921T033154Z AGE_SECONDS=1255 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH","FRESHNESS_STATUS=OK","APP=hbarhub","DECISION=SEALED_PROVEN","GENERATION=app-hbarhub-pve01-20260921T033309Z","UPDATED_AT_UTC=2026-09-21T03:34:20Z","SEALED_EPOCH=1789961660","STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-hbarhub/app-hbarhub-pve01-20260921T033309Z.json","RESTORE_EVIDENCE=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260921T033309Z.json","QUEUE_FRESHNESS APP=hbarhub DECISION=SKIP_FRESH GENERATION=app-hbarhub-pve01-20260921T033309Z AGE_SECONDS=1155 DUE_SECONDS=43200 FRESHNESS_SECONDS=72000","QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH","CLASS_DISPATCH KIND=APP DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1","GLOBAL_DISPATCH KIND=VM TARGET=171 DECISION=BACKUP_STALE PRIORITY=2 AGE_SECONDS=436710"],"rc":0,"stderr":["VM_QUORUM_QUEUE_ERROR=CLUSTER_OWNER_MISMATCH_180__EXPECTED_pve02","VM_QUORUM_QUEUE_ERROR=FRESHNESS_DECISION_INVALID_180_EMPTY"],"stdout_sha256":"beaf5c144e0d201cbeef18657cd58484b241ff667de66cb9a6483d41cd29ef0b"} +VM9130_REGISTRY_ENTRIES=[{"id":"9130","kind":"VM","lifecycle":"ACTIVE","logical_id":"vm9130","owner":"pve02"}] +VM9130_CLUSTER_RESOURCE={"name":"edge-cold-standby","node":"pve02","status":"stopped","template":0,"type":"qemu","vmid":9130} +VM9130_OWNER_READONLY={"lines":["pve02","status: stopped","agent: enabled=1","boot: order=scsi0","ciuser: debian","cores: 4","cpu: host","description: CR0023 cold standby of VM130. Keep stopped; all NICs link_down=1.","ide2: local-lvm:vm-9130-cloudinit,media=cdrom,size=4M","ipconfig0: ip=[PRIVATE_IP]/24,gw=[PRIVATE_IP]","memory: 12288","meta: creation-qemu=11.0.0,ctime=1781460976","name: edge-cold-standby","nameserver: [PRIVATE_IP] [PRIVATE_IP]","net0: virtio=BC:24:11:7F:04:D6,bridge=vmbr0,firewall=1,link_down=1","onboot: 0","ostype: l26","protection: 1","scsi0: local-lvm:vm-9130-disk-0,discard=on,size=96G,ssd=1","scsi1: local-lvm:vm-9130-disk-1,backup=1,discard=on,size=150G,ssd=1","scsihw: virtio-scsi-pci","searchdomain: gram1.ru","serial0: socket","smbios1: uuid=9327a861-e9f4-4450-b13f-a1e767a83c86","sshkeys: ssh-rsa%20AAAAB3NzaC1yc2EAAAADAQABAAACAQC13sgRxsdexPSyrcz3pfeVJoGajhrb3mJI2gdB2HNWOj6SS%2FcHHiJBKUjg80s6rhZJD%2FtpyLOj%2BQbrkJR3L3FdrgwgCvPcYrBujuZ2TLaRIKIDlNilnRkmVQmUBahgAyrnaQBBl1DVFQsfF0NLF0PYjvPF3789giWqCfv4L14vJSFxvp3wEOjkY1hiwZ8jffMD0R99UW4FTqm29KEFx5ywt%2BdrOT6sMtkBYCIR%2F5JGCB0riH%2F9xn904TIasSMBDz2IeEFpO73bd72JtmnxFyaWaBllKn%2Bu9t18ahMx5vSn2UerGGZbbrZBigxNIwOaHdvdyxADSR3QU90T6nzUv5Fov3vIoOt4uhVaMsYew25hpzm8%2Byp77liUu9mfuMWzo0Q7XJrNfhuCJ9pcrBeqopdThFC36c72JgksWT9Lhfef7bMbEa%2BIWF0wL3L%2FALp1QVA1cbQWPVAF1x3UEHnXuxJCj2x8z4Xli2SJu7TS16qF9Lv1GqVYMeZvW48LwXSbwWcui3B4ldSuo5ksmPeggcTuHzxy5TyQk9sJjlz10Pw9A5zbbuYZDo56n17c1couUm1a3EQ25CimrECygWX2OKcAEdmoiWAihKHyAuLFnSRUELbNrFcFJIb4JkrGk7edWdF%2Btw3jOHlHDcJew08xyoutlN1P9iFTrotMbQjyNTrGgQ%3D%3D%20root%40pve01%0A","startup: order=40","vga: serial0","vmgenid: 110339cf-08fb-4da8-bc7d-2265b8080f2c","91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 /usr/local/libexec/homelab-vm-cloud-quorum-node-worker"],"rc":0,"stderr":[]} +VM_QUEUE_DISPATCH_SOURCE={"lines_140_220":[{"line":140,"text":" printf 'QUEUE_SEALED VMID=%s GENERATION=%s\\n' \"$vmid\" \"$generation\""},{"line":141,"text":" done"},{"line":142,"text":"}"},{"line":143,"text":"dispatch_plan(){"},{"line":144,"text":" freshness_plan"},{"line":145,"text":"}"},{"line":146,"text":"dispatch_one(){"},{"line":147,"text":" local vmid=\"$1\" freshness decision now_epoch out generation"},{"line":148,"text":" [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64"},{"line":149,"text":" expected_owner \"$vmid\" >/dev/null"},{"line":150,"text":" exec 9>\"$LOCK\""},{"line":151,"text":" if ! flock -n 9; then"},{"line":152,"text":" printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\\n' \"$vmid\""},{"line":153,"text":" return 0"},{"line":154,"text":" fi"},{"line":155,"text":" exec 8>\"$HEAVY_LOCK\""},{"line":156,"text":" if ! flock -n 8; then"},{"line":157,"text":" printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_HEAVY_LOCK_BUSY VMID=%s\\n' \"$vmid\""},{"line":158,"text":" return 0"},{"line":159,"text":" fi"},{"line":160,"text":" now_epoch=\"$(date -u +%s)\""},{"line":161,"text":" freshness=\"$(exec 8>&-; exec 9>&-; freshness_decision \"$vmid\" \"$now_epoch\")\""},{"line":162,"text":" printf '%s\\n' \"$freshness\""},{"line":163,"text":" decision=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\""},{"line":164,"text":" case \"$decision\" in"},{"line":165,"text":" SKIP_FRESH)"},{"line":166,"text":" generation=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\""},{"line":167,"text":" printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SKIP_FRESH GENERATION=%s\\n' \"$vmid\" \"$generation\""},{"line":168,"text":" return 0"},{"line":169,"text":" ;;"},{"line":170,"text":" BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;"},{"line":171,"text":" *) die \"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\" 65 ;;"},{"line":172,"text":" esac"},{"line":173,"text":" printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$decision\""},{"line":174,"text":" out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\""},{"line":175,"text":" printf '%s\\n' \"$out\""},{"line":176,"text":" generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\""},{"line":177,"text":" [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$vmid\" 65"},{"line":178,"text":" printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$generation\""},{"line":179,"text":" printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\\n' \"$vmid\" \"$generation\""},{"line":180,"text":"}"},{"line":181,"text":""},{"line":182,"text":"ACTION=\"${1:-}\""},{"line":183,"text":"case \"$ACTION\" in"},{"line":184,"text":" plan)"},{"line":185,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_PLAN\" 64"},{"line":186,"text":" plan_all"},{"line":187,"text":" ;;"},{"line":188,"text":" freshness-plan)"},{"line":189,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_FRESHNESS_PLAN\" 64"},{"line":190,"text":" freshness_plan"},{"line":191,"text":" ;;"},{"line":192,"text":" dispatch-plan)"},{"line":193,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_DISPATCH_PLAN\" 64"},{"line":194,"text":" dispatch_plan"},{"line":195,"text":" ;;"},{"line":196,"text":" dispatch-one)"},{"line":197,"text":" [[ \"$#\" == 2 ]] || die \"USAGE_DISPATCH_ONE\" 64"},{"line":198,"text":" dispatch_one \"$2\""},{"line":199,"text":" ;;"},{"line":200,"text":" canary)"},{"line":201,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_CANARY\" 64"},{"line":202,"text":" backup_list_force 110"},{"line":203,"text":" ;;"},{"line":204,"text":" migrate)"},{"line":205,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_MIGRATE\" 64"},{"line":206,"text":" backup_list_force \"${MIGRATE_VMIDS[@]}\""},{"line":207,"text":" ;;"},{"line":208,"text":" all)"},{"line":209,"text":" [[ \"$#\" == 1 ]] || die \"USAGE_ALL\" 64"},{"line":210,"text":" [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die \"DISPATCHER_MISSING\" 69"},{"line":211,"text":" /usr/local/sbin/homelab-backup-dispatcher run VM_TIMER"},{"line":212,"text":" ;;"},{"line":213,"text":" verify)"},{"line":214,"text":" [[ \"$#\" == 3 ]] || die \"USAGE_VERIFY\" 64"},{"line":215,"text":" run_worker \"$2\" verify \"$3\""},{"line":216,"text":" ;;"},{"line":217,"text":" *)"},{"line":218,"text":" die \"ACTION_INVALID\" 64"},{"line":219,"text":" ;;"},{"line":220,"text":"esac"}],"sha256":"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612"} +DISPATCHER_RUN_SOURCE={"lines_35_60":[{"line":35,"text":"plan(){ select_global; }"},{"line":36,"text":"run(){"},{"line":37,"text":" local trigger=\"$1\" i=0 actions=0 scan pick decision kind target out rc last_sealed=\"\""},{"line":38,"text":" exec 7>\"$LOCK\""},{"line":39,"text":" if ! flock -n 7; then printf 'DISPATCHER_DEFERRED TRIGGER=%s REASON=DISPATCHER_LOCK_BUSY\\n' \"$trigger\"; return 0; fi"},{"line":40,"text":" while (( i < MAX_ITERATIONS )); do"},{"line":41,"text":" i=$((i+1)); scan=\"$(select_global)\"; printf '%s\\n' \"$scan\"; pick=\"$(grep '^GLOBAL_DISPATCH ' <<<\"$scan\" | tail -1)\"; [[ -n \"$pick\" ]] || die GLOBAL_DISPATCH_RESULT_MISSING 65"},{"line":42,"text":" decision=\"\"; kind=\"\"; target=\"\"; for token in $pick; do case \"$token\" in KIND=*) kind=\"${token#KIND=}\";; TARGET=*) target=\"${token#TARGET=}\";; DECISION=*) decision=\"${token#DECISION=}\";; esac; done"},{"line":43,"text":" if [[ \"$decision\" == NOOP ]]; then printf 'DISPATCHER_DONE TRIGGER=%s ACTIONS=%s ITERATIONS=%s\\n' \"$trigger\" \"$actions\" \"$i\"; return 0; fi"},{"line":44,"text":" [[ \"$kind\" == VM || \"$kind\" == APP ]] || die GLOBAL_KIND_INVALID 65; [[ -n \"$target\" ]] || die GLOBAL_TARGET_MISSING 65"},{"line":45,"text":" if [[ \"$last_sealed\" == \"$kind:$target\" ]]; then die \"NO_PROGRESS_AFTER_SEALED_${kind}_${target}\" 65; fi"},{"line":46,"text":" if [[ \"$kind\" == VM ]]; then if out=\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \"$VMQ\" dispatch-one \"$target\" 2>&1)\"; then rc=0; else rc=$?; fi; else if out=\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \"$APPQ\" dispatch-one \"$target\" 2>&1)\"; then rc=0; else rc=$?; fi; fi"},{"line":47,"text":" printf '%s\\n' \"$out\"; (( rc == 0 )) || return \"$rc\""},{"line":48,"text":" if grep -q '^QUEUE_DEFERRED ' <<<\"$out\"; then printf 'DISPATCHER_DEFERRED TRIGGER=%s KIND=%s TARGET=%s REASON=QUEUE_OR_HEAVY_BUSY\\n' \"$trigger\" \"$kind\" \"$target\"; return 0; fi"},{"line":49,"text":" if grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SEALED ' <<<\"$out\"; then last_sealed=\"$kind:$target\"; actions=$((actions+1)); elif grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SKIP_FRESH ' <<<\"$out\"; then last_sealed=\"\"; actions=$((actions+1)); else die \"DISPATCH_RESULT_UNCLASSIFIED_${kind}_${target}\" 65; fi"},{"line":50,"text":" done"},{"line":51,"text":" die MAX_ITERATIONS_EXCEEDED 65"},{"line":52,"text":"}"},{"line":53,"text":"case \"${1:-}\" in plan) [[ \"$#\" == 1 ]] || die USAGE_PLAN 64; plan;; run) [[ \"$#\" == 2 ]] || die USAGE_RUN 64; run \"$2\";; *) die ACTION_INVALID 64;; esac"}],"sha256":"9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c"} +VM_WORKER_BACKUP_GUARDS={"hits":[{"line":6,"text":"STATE_ROOT=/var/lib/homelab-backup/cloud-quorum"},{"line":8,"text":"RESTORE_STATE_ROOT=/var/lib/homelab-backup/vm-restore-validation"},{"line":11,"text":"REMOTE_A='pve01-mail-01-crypt:homelab-backups'"},{"line":12,"text":"REMOTE_B='pve01-mail-02-crypt:homelab-backups'"},{"line":14,"text":"die(){ printf 'VM_QUORUM_WORKER_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }"},{"line":15,"text":"for tool in jq sha256sum zstd tar rclone stat find sort awk date mkfifo tee wc vma; do command -v \"$tool\" >/dev/null 2>&1 || die \"TOOL_MISSING_${tool}\" 69; done"},{"line":16,"text":"[[ -x \"$TRANSPORT\" && ! -L \"$TRANSPORT\" ]] || die TRANSPORT_MISSING 69"},{"line":17,"text":"[[ \"$(sha256sum \"$TRANSPORT\"|awk '{print $1}')\" == \"$TRANSPORT_SHA_EXPECTED\" ]] || die TRANSPORT_SHA_MISMATCH 65"},{"line":21,"text":"active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ \"$rc\" == 1 ]] && return 0; die \"PGREP_FAILED_$rc\" 69; fi; }"},{"line":22,"text":"snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }"},{"line":23,"text":"vm_lock(){ qm config \"$1\" | awk -F': ' '$1==\"lock\"{print $2; found=1} END{if(!found) print \"none\"}'; }"},{"line":24,"text":"expected_owner(){"},{"line":27,"text":"111|113|160|180|9130) printf 'pve02\\n' ;;"},{"line":29,"text":"*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;"},{"line":35,"text":"130|150|160|170|171|180|190|9130) printf 'qemu\\n' ;;"},{"line":36,"text":"*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;"},{"line":39,"text":"expected_status(){"},{"line":40,"text":"if [[ \"$1\" == \"9130\" ]]; then printf 'stopped\\n'; else printf 'running\\n'; fi"},{"line":42,"text":"current_status(){"},{"line":43,"text":"local vmid=\"$1\" type=\"$2\""},{"line":44,"text":"if [[ \"$type\" == \"qemu\" ]]; then qm status \"$vmid\" | awk '{print $2}'; else pct status \"$vmid\" | awk '{print $2}'; fi"},{"line":47,"text":"local vmid=\"$1\" owner type status expect_status"},{"line":48,"text":"owner=\"$(expected_owner \"$vmid\")\""},{"line":49,"text":"type=\"$(expected_type \"$vmid\")\""},{"line":50,"text":"[[ \"$HOST\" == \"$owner\" ]] || die \"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\" 65"},{"line":52,"text":"[[ -s \"/etc/pve/qemu-server/${vmid}.conf\" ]] || die \"QEMU_CONFIG_MISSING_$vmid\" 66"},{"line":54,"text":"[[ -s \"/etc/pve/lxc/${vmid}.conf\" ]] || die \"LXC_CONFIG_MISSING_$vmid\" 66"},{"line":56,"text":"status=\"$(current_status \"$vmid\" \"$type\")\""},{"line":57,"text":"expect_status=\"$(expected_status \"$vmid\")\""},{"line":58,"text":"[[ \"$status\" == \"$expect_status\" ]] || die \"STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\" 66"},{"line":59,"text":"if [[ \"$vmid\" == \"160\" ]]; then [[ \"$(snapshot_count_160)\" == \"0\" ]] || die \"VM160_SNAPSHOT_COUNT_CHANGED\" 66; fi"},{"line":60,"text":"if [[ \"$vmid\" == \"9130\" ]]; then [[ \"$(vm_lock 9130)\" == \"none\" ]] || die \"VM9130_LOCK_PRESENT\" 66; fi"},{"line":62,"text":"VM_STATUS=\"$status\""},{"line":65,"text":"local vmid=\"$1\" type=\"$2\" status expect_status"},{"line":66,"text":"status=\"$(current_status \"$vmid\" \"$type\")\""},{"line":67,"text":"expect_status=\"$(expected_status \"$vmid\")\""},{"line":68,"text":"[[ \"$status\" == \"$expect_status\" ]] || die \"POST_STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\" 66"},{"line":69,"text":"if [[ \"$vmid\" == \"160\" ]]; then [[ \"$(snapshot_count_160)\" == \"0\" ]] || die \"VM160_SNAPSHOT_COUNT_CHANGED_POST\" 66; fi"},{"line":70,"text":"if [[ \"$vmid\" == \"9130\" ]]; then [[ \"$(vm_lock 9130)\" == \"none\" ]] || die \"VM9130_LOCK_PRESENT_POST\" 66; fi"},{"line":72,"text":"latest_incomplete_state(){"},{"line":73,"text":"local vmid=\"$1\" dir=\"$STATE_ROOT/vm${vmid}\" f"},{"line":76,"text":"jq -e '.status!=\"SEALED\"' \"$f\" >/dev/null 2>&1 && printf '%s\\n' \"$f\""},{"line":79,"text":"restore_dir(){ printf '%s\\n' \"$RESTORE_STATE_ROOT/vm$1\"; }"},{"line":81,"text":" local vmid=\"$1\" gen=\"$2\" f state sha"},{"line":82,"text":" f=\"$(restore_dir \"$vmid\")/$gen.json\"; state=\"$STATE_ROOT/vm${vmid}/$gen.json\""},{"line":83,"text":" [[ -s \"$f\" && ! -L \"$f\" && -s \"$state\" && ! -L \"$state\" ]] || return 1"},{"line":84,"text":" sha=\"$(jq -er '.archive_sha256' \"$state\")\" || return 1"},{"line":85,"text":" jq -e --arg l \"vm${vmid}\" --arg g \"$gen\" --arg s \"$state\" --arg sha \"$sha\" '.schema==\"homelab-vm-restore-validation-v1\" and .logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .transport_state_file==$s and .restore_source==\"Cloud A\" and .parts_manifest_verified==true and .payload_sha256==$sha and .payload_sha256_verified==true and .payload_decompression_verified==true and .proxmox_config_extract_verified==true and .archive_recovery_readback_verified==true and .bounded_staging==true and .full_local_payload_written==false and .full_restore_proven==false and .result==\"OK\"' \"$f\" >/dev/null"},{"line":88,"text":" local vmid=\"$1\" gen=\"$2\" state=\"$3\" work=\"$4\" base manifest expected_msha got_msha expected_count expected_size idx name size psha count total"},{"line":89,"text":" base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"; expected_count=\"$(jq -er '.part_count' \"$state\")\"; expected_size=\"$(jq -er '.archive_size_bytes' \"$state\")\""},{"line":90,"text":" [[ \"$base\" == \"${REMOTE_A%/}/vm${vmid}/$gen\" ]] || die \"RESTORE_A_BASE_MISMATCH_$vmid\" 70"},{"line":92,"text":" got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die \"RESTORE_A_MANIFEST_SHA_MISMATCH_$vmid\" 70"},{"line":95,"text":" [[ \"$idx\" =~ ^[0-9]{8}$ && \"$name\" == part-????????.bin && \"$size\" =~ ^[1-9][0-9]*$ && \"$psha\" =~ ^[0-9a-f]{64}$ ]] || die \"RESTORE_A_MANIFEST_ROW_INVALID_$vmid\" 70"},{"line":96,"text":" (( size <= 536870912 )) || die \"RESTORE_A_PART_EXCEEDS_BOUND_$vmid\" 70"},{"line":97,"text":" [[ \"$idx\" == \"$(printf '%08d' \"$count\")\" ]] || die \"RESTORE_A_MANIFEST_INDEX_INVALID_$vmid\" 70"},{"line":100,"text":" [[ \"$count\" == \"$expected_count\" ]] || die \"RESTORE_A_PART_COUNT_MISMATCH_$vmid\" 70"},{"line":101,"text":" [[ \"$total\" == \"$expected_size\" ]] || die \"RESTORE_A_MANIFEST_TOTAL_SIZE_MISMATCH_$vmid\" 70"},{"line":105,"text":" local vmid=\"$1\" base=\"$2\" manifest=\"$3\" work=\"$4\" idx name size psha part actual count"},{"line":109,"text":" [[ -s \"$part\" && ! -L \"$part\" && \"$(stat -c %s \"$part\")\" == \"$size\" ]] || die \"RESTORE_A_PART_SIZE_MISMATCH_$vmid\" 70"},{"line":110,"text":" actual=\"$(sha256sum \"$part\"|awk '{print $1}')\"; [[ \"$actual\" == \"$psha\" ]] || die \"RESTORE_A_PART_SHA_MISMATCH_$vmid\" 70"},{"line":115,"text":" local vmid=\"$1\" gen=\"$2\" logical state work payload_name expected_sha expected_size base manifest fifo sha_out sha_pid pipe_rc sha_rc actual_sha config config_sha members member_count cfg_member level out tmp"},{"line":116,"text":" assert_local_identity \"$vmid\"; logical=\"vm${vmid}\"; state=\"$STATE_ROOT/$logical/$gen.json\""},{"line":117,"text":" [[ -s \"$state\" && ! -L \"$state\" ]] || die \"RESTORE_STATE_MISSING_$vmid\" 65"},{"line":118,"text":" jq -e '.schema==\"homelab-cloud-quorum-payload-v1\" and .status==\"SEALED\" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_payload_present==false and .local_cleanup==\"OK\"' \"$state\" >/dev/null || die \"RESTORE_REQUIRES_STRICT_SEALED_$vmid\" 65"},{"line":119,"text":" if restore_ok \"$vmid\" \"$gen\"; then printf 'RESTORE_VALIDATION_STATUS=OK\\nVMID=%s\\nGENERATION=%s\\nRESULT=ALREADY_VALIDATED\\nRESTORE_EVIDENCE=%s\\n' \"$vmid\" \"$gen\" \"$(restore_dir \"$vmid\")/$gen.json\"; return 0; fi"},{"line":120,"text":" [[ -z \"$(active_backup_restore)\" ]] || die ACTIVE_BACKUP_RESTORE_PRESENT 75"},{"line":121,"text":" payload_name=\"$(jq -er '.payload_name' \"$state\")\"; expected_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; expected_size=\"$(jq -er '.archive_size_bytes' \"$state\")\""},{"line":122,"text":" [[ \"$payload_name\" == \"vzdump-lxc-${vmid}-\"*.tar.zst || \"$payload_name\" == \"vzdump-qemu-${vmid}-\"*.vma.zst ]] || die \"RESTORE_PAYLOAD_NAME_INVALID_$vmid\" 70"},{"line":123,"text":" work=\"$RESTORE_WORK_ROOT/$logical/$gen\"; if [[ -e \"$work\" ]]; then [[ -d \"$work\" && ! -L \"$work\" ]] || die \"RESTORE_WORK_UNSAFE_$vmid\" 70; rm -rf --one-file-system \"$work\"; fi; mkdir -p \"$work\"; chmod 0700 \"$RESTORE_WORK_ROOT\" \"$RESTORE_WORK_ROOT/$logical\" \"$work\"; ACTIVE_RESTORE_STAGE=\"$work\""},{"line":124,"text":" base=\"$(prepare_cloud_a_manifest \"$vmid\" \"$gen\" \"$state\" \"$work\")\"; manifest=\"$work/PARTS.tsv\"; fifo=\"$work/archive.sha.fifo\"; sha_out=\"$work/archive.sha256\"; mkfifo -m 0600 \"$fifo\""},{"line":127,"text":" members=\"$work/members.txt\"; if stream_cloud_a \"$vmid\" \"$base\" \"$manifest\" \"$work\" | tee \"$fifo\" | zstd -dc | tar -tf - >\"$members\"; then pipe_rc=0; else pipe_rc=$?; fi"},{"line":129,"text":" if stream_cloud_a \"$vmid\" \"$base\" \"$manifest\" \"$work\" | tee \"$fifo\" | zstd -t - >/dev/null; then pipe_rc=0; else pipe_rc=$?; fi"},{"line":132,"text":" (( pipe_rc == 0 && sha_rc == 0 )) || die \"RESTORE_A_STREAM_VALIDATION_FAILED_${vmid}_${pipe_rc}_${sha_rc}\" 70"},{"line":133,"text":" actual_sha=\"$(awk 'NR==1{print $1}' \"$sha_out\")\"; [[ \"$actual_sha\" == \"$expected_sha\" ]] || die \"RESTORE_A_PAYLOAD_SHA_MISMATCH_$vmid\" 70"},{"line":136,"text":" [[ -s \"$members\" ]] || die \"RESTORE_LXC_ARCHIVE_EMPTY_$vmid\" 70"},{"line":137,"text":" awk '$0 ~ /^\\// || $0 ~ /(^|\\/)\\.\\.(\\/|$)/ {bad=1} END{exit bad}' \"$members\" || die \"RESTORE_LXC_MEMBER_UNSAFE_$vmid\" 70"},{"line":138,"text":" member_count=\"$(wc -l <\"$members\")\"; [[ \"$member_count\" =~ ^[1-9][0-9]*$ ]] || die \"RESTORE_LXC_MEMBER_COUNT_INVALID_$vmid\" 70"},{"line":139,"text":" cfg_member=\"$(awk '$0==\"./etc/vzdump/pct.conf\" || $0==\"etc/vzdump/pct.conf\" {print; exit}' \"$members\")\"; [[ -n \"$cfg_member\" ]] || die \"RESTORE_LXC_CONFIG_MEMBER_MISSING_$vmid\" 70"},{"line":140,"text":" if ( set +o pipefail; stream_cloud_a \"$vmid\" \"$base\" \"$manifest\" \"$work\" | zstd -dc | tar -xOf - \"$cfg_member\" >\"$config\" ); then :; else die \"RESTORE_LXC_CONFIG_EXTRACT_FAILED_$vmid\" 70; fi"},{"line":143,"text":" if ( set +o pipefail; stream_cloud_a \"$vmid\" \"$base\" \"$manifest\" \"$work\" | zstd -dc | vma config - >\"$config\" ); then :; else die \"RESTORE_QEMU_CONFIG_EXTRACT_FAILED_$vmid\" 70; fi"},{"line":146,"text":" [[ -s \"$config\" && ! -L \"$config\" ]] || die \"RESTORE_CONFIG_EMPTY_$vmid\" 70; config_sha=\"$(sha256sum \"$config\"|awk '{print $1}')\""},{"line":147,"text":" out=\"$(restore_dir \"$vmid\")/$gen.json\"; mkdir -p \"$(dirname \"$out\")\"; chmod 0700 \"$RESTORE_STATE_ROOT\" \"$(dirname \"$out\")\"; tmp=\"${out}.new.$$\""},{"line":148,"text":" jq -n --arg logical \"$logical\" --arg generation \"$gen\" --argjson vmid \"$vmid\" --arg vm_type \"$VM_TYPE\" --arg state \"$state\" --arg sha \"$expected_sha\" --arg config_sha \"$config_sha\" --arg level \"$level\" --argjson member_count \"$member_count\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" '{schema:\"homelab-vm-restore-validation-v1\",logical_id:$logical,generation:$generation,vmid:$vmid,vm_type:$vm_type,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",parts_manifest_verified:true,payload_sha256:$sha,payload_sha256_verified:true,payload_decompression_verified:true,proxmox_config_extract_verified:true,proxmox_config_sha256:$config_sha,lxc_member_count:$member_count,archive_recovery_readback_verified:true,bounded_staging:true,max_staged_part_bytes:536870912,full_local_payload_written:false,full_restore_proven:false,validation_level:$level,result:\"OK\",validated_at_utc:$at}' >\"$tmp\""},{"line":150,"text":" restore_ok \"$vmid\" \"$gen\" || die \"RESTORE_EVIDENCE_POSTWRITE_INVALID_$vmid\" 70"},{"line":151,"text":" printf 'RESTORE_VALIDATION_STATUS=OK\\nVMID=%s\\nGENERATION=%s\\nRESTORE_EVIDENCE=%s\\n' \"$vmid\" \"$gen\" \"$out\""},{"line":154,"text":"local vmid=\"$1\" owner type status"},{"line":155,"text":"assert_local_identity \"$vmid\""},{"line":156,"text":"owner=\"$(expected_owner \"$vmid\")\""},{"line":157,"text":"type=\"$(expected_type \"$vmid\")\""},{"line":158,"text":"status=\"$(expected_status \"$vmid\")\""},{"line":159,"text":"printf 'PLAN_STATUS=OK\\nVMID=%s\\nOWNER=%s\\nTYPE=%s\\nEXPECTED_STATUS=%s\\n' \"$vmid\" \"$owner\" \"$type\" \"$status\""},{"line":162,"text":" local vmid=\"$1\" generation=\"$2\""},{"line":163,"text":" assert_local_identity \"$vmid\""},{"line":164,"text":" restore_validate \"$vmid\" \"$generation\""},{"line":165,"text":" assert_post_identity \"$vmid\" \"$VM_TYPE\""},{"line":166,"text":" printf 'VERIFY_STATUS=OK\\nVMID=%s\\nGENERATION=%s\\nRESTORE_EVIDENCE=%s\\n' \"$vmid\" \"$generation\" \"$(restore_dir \"$vmid\")/$generation.json\""},{"line":168,"text":"backup_one(){"},{"line":169,"text":"local vmid=\"$1\" pending_count pending state generation payload gen_dir archive_count archive log_file before_status"},{"line":170,"text":"local -a vzdump_args"},{"line":171,"text":"assert_local_identity \"$vmid\""},{"line":172,"text":"[[ -x \"$TRANSPORT\" ]] || die \"TRANSPORT_NOT_INSTALLED\" 69"},{"line":173,"text":"[[ -d /mnt/staging && ! -L /mnt/staging ]] || die \"STAGING_UNSAFE\" 66"},{"line":176,"text":"[[ -z \"$(active_backup_restore)\" ]] || die \"ACTIVE_BACKUP_RESTORE_PRESENT\" 75"},{"line":179,"text":"mapfile -t pending < <(latest_incomplete_state \"$vmid\")"},{"line":181,"text":"(( pending_count <= 1 )) || die \"MULTIPLE_INCOMPLETE_GENERATIONS_$vmid\" 65"},{"line":183,"text":"state=\"${pending[0]}\""},{"line":184,"text":"generation=\"$(jq -er '.generation' \"$state\")\""},{"line":185,"text":"payload=\"$(jq -er '.payload_path' \"$state\")\""},{"line":186,"text":"printf 'RESUME=YES\\nVMID=%s\\nGENERATION=%s\\nSTATE=%s\\n' \"$vmid\" \"$generation\" \"$state\""},{"line":187,"text":"\"$TRANSPORT\" backup-delete \"vm${vmid}\" \"$generation\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\""},{"line":189,"text":"generation=\"vm${vmid}-${HOST}-$(date -u +%Y%m%dT%H%M%SZ)\""},{"line":190,"text":"gen_dir=\"$STAGE_ROOT/vm${vmid}/$generation\""},{"line":191,"text":"[[ ! -e \"$gen_dir\" ]] || die \"GENERATION_DIR_EXISTS_$generation\" 65"},{"line":194,"text":"log_file=\"$gen_dir/vzdump.log\""},{"line":195,"text":"printf 'RESUME=NO\\nVMID=%s\\nGENERATION=%s\\n' \"$vmid\" \"$generation\""},{"line":196,"text":"vzdump_args=(--dumpdir \"$gen_dir\" --mode snapshot --compress zstd --ionice 7)"},{"line":198,"text":"[[ -d \"$LXC_TMP_ROOT\" && ! -L \"$LXC_TMP_ROOT\" ]] || die \"LXC_TMP_ROOT_UNSAFE\" 66"},{"line":199,"text":"[[ \"$(find \"$LXC_TMP_ROOT\" -maxdepth 0 -printf '%m')\" == \"1777\" ]] || die \"LXC_TMP_ROOT_MODE_UNSAFE\" 66"},{"line":200,"text":"vzdump_args+=(--tmpdir \"$LXC_TMP_ROOT\")"},{"line":202,"text":"if ! ( umask 0022; exec vzdump \"$vmid\" \"${vzdump_args[@]}\" ) >\"$log_file\" 2>&1; then"},{"line":204,"text":"die \"VZDUMP_FAILED_$vmid\" 70"},{"line":206,"text":"mapfile -t archives < <(find \"$gen_dir\" -maxdepth 1 -type f -name \"vzdump-*-${vmid}-*.zst\" -print)"},{"line":208,"text":"[[ \"$archive_count\" == \"1\" ]] || { rm -rf -- \"$gen_dir\"; die \"VZDUMP_ARCHIVE_COUNT_${vmid}_${archive_count}\" 70; }"},{"line":210,"text":"[[ -s \"$archive\" && ! -L \"$archive\" ]] || die \"VZDUMP_ARCHIVE_UNSAFE_$vmid\" 70"},{"line":212,"text":"\"$TRANSPORT\" backup-delete \"vm${vmid}\" \"$generation\" \"$archive\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\""},{"line":215,"text":"rmdir -- \"$STAGE_ROOT/vm${vmid}\" 2>/dev/null || true"},{"line":217,"text":"assert_post_identity \"$vmid\" \"$VM_TYPE\""},{"line":218,"text":"state=\"$STATE_ROOT/vm${vmid}/$generation.json\""},{"line":219,"text":"jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .local_payload_present==false and .local_cleanup==\"OK\" and .seal_written==true' \"$state\" >/dev/null"},{"line":220,"text":" restore_validate \"$vmid\" \"$generation\""},{"line":221,"text":" printf 'BACKUP_STATUS=SEALED\\nVMID=%s\\nGENERATION=%s\\nOWNER=%s\\nTYPE=%s\\nPOST_STATUS=%s\\nRESTORE_EVIDENCE=%s\\n' \"$vmid\" \"$generation\" \"$HOST\" \"$VM_TYPE\" \"$(expected_status \"$vmid\")\" \"$(restore_dir \"$vmid\")/$generation.json\""},{"line":225,"text":"[[ -d /mnt/staging && ! -L /mnt/staging ]] || die \"STAGING_UNSAFE\" 66"},{"line":226,"text":"[[ -z \"$(active_backup_restore)\" ]] || die \"ACTIVE_BACKUP_RESTORE_PRESENT\" 75"},{"line":230,"text":"[[ \"$free\" =~ ^[1-9][0-9]+$ && \"$a_free\" =~ ^[1-9][0-9]+$ && \"$b_free\" =~ ^[1-9][0-9]+$ ]] || die \"CAPACITY_VALUE_INVALID\" 65"},{"line":231,"text":"printf 'HOST_PREFLIGHT_STATUS=OK\\nHOST=%s\\nSTAGING_FREE_BYTES=%s\\nREMOTE_A_FREE_BYTES=%s\\nREMOTE_B_FREE_BYTES=%s\\nACTIVE_BACKUP_RESTORE=0\\n' \"$HOST\" \"$free\" \"$a_free\" \"$b_free\""},{"line":234,"text":" local vmid=\"$1\" dir latest gen"},{"line":235,"text":" assert_local_identity \"$vmid\"; dir=\"$STATE_ROOT/vm${vmid}\""},{"line":236,"text":" [[ -d \"$dir\" ]] || die \"STATE_DIR_MISSING_$vmid\" 65"},{"line":238,"text":" [[ -n \"$latest\" && -s \"$latest\" && ! -L \"$latest\" ]] || die \"LATEST_STATE_MISSING_$vmid\" 65"},{"line":239,"text":" jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .local_payload_present==false and .local_cleanup==\"OK\" and .seal_written==true' \"$latest\" >/dev/null || die \"LATEST_STATE_NOT_SEALED_$vmid\" 65"},{"line":240,"text":" gen=\"$(jq -er '.generation' \"$latest\")\"; restore_ok \"$vmid\" \"$gen\" || die \"LATEST_RESTORE_NOT_OK_$vmid\" 65"},{"line":241,"text":" printf 'LATEST_SEALED_STATUS=OK\\nVMID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$vmid\" \"$gen\" \"$latest\" \"$(restore_dir \"$vmid\")/$gen.json\""},{"line":246,"text":"[[ \"$#\" == 1 ]] || die \"USAGE_HOST_PREFLIGHT\" 64"},{"line":250,"text":"[[ \"$#\" == 2 ]] || die \"USAGE_LATEST_SEALED\" 64"},{"line":254,"text":"[[ \"$#\" == 2 ]] || die \"USAGE_PLAN\" 64"},{"line":257,"text":"backup)"},{"line":258,"text":"[[ \"$#\" == 2 ]] || die \"USAGE_BACKUP\" 64"},{"line":259,"text":"backup_one \"$2\""},{"line":262,"text":" [[ \"$#\" == 3 ]] || die \"USAGE_RESTORE_VALIDATE\" 64"},{"line":266,"text":"[[ \"$#\" == 3 ]] || die \"USAGE_VERIFY\" 64"},{"line":270,"text":"die \"ACTION_INVALID\" 64"}],"sha256":"91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1"} +UNIT_STATE={"homelab-app-cloud-quorum-queue.timer":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainStatus":{"rc":0,"value":""},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"disabled"}},"homelab-backup-retention-gc.timer":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainStatus":{"rc":0,"value":""},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"disabled"}},"homelab-backup-v2.service":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainStatus":{"rc":0,"value":"2"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"static"}},"homelab-backup-v2.timer":{"ActiveState":{"rc":0,"value":"active"},"ExecMainStatus":{"rc":0,"value":""},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"waiting"},"UnitFileState":{"rc":0,"value":"enabled"}}} +RCA_CLASSIFICATION={"cluster_status":"stopped","dispatcher_plan_mentions_9130":false,"dispatcher_plan_rc":0,"freshness_plan_decision":null,"owner_node":"pve02","queue_plan_rc":65,"worker_contains_status_or_stopped_guard":true} +NO_MUTATION=true +NO_REPO_WRITE=true +NO_GIT_FETCH=true +NO_GIT_COMMIT=true +NO_GIT_PUSH_UPDATE=true +NO_SYSTEMD_ACTION=true +NO_BACKUP_STARTED=true +NO_RESTORE_STARTED=true +NO_RETENTION_STARTED=true +NO_DELETE=true +NO_MAIL_SENT=true +DURATION_SEC=11.169 +TASK_RESULT=PASS_HOMELAB_BACKUP_VM9130_RCA_READONLY +HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-VM9130-RCA-READONLY-1048R1","rollback_restored":null,"rollback_started":false,"status":"OK","version":1} +WORKERS2_BACKUP_VM9130_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END