homelab-runtime-publisher
39 lines
76 KiB
JSON
39 lines
76 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"channel": "homelab-runtime",
|
|
"command_id": "NEWFI-260921-NEWFDOM-DR221-PREFLIGHT399",
|
|
"status": "OK",
|
|
"rc": 0,
|
|
"host": "pve01",
|
|
"mode": "read-only",
|
|
"component": "newfi-dr221-current-backup-restore-contract-preflight",
|
|
"started_at_utc": "2026-09-21T16:06:51Z",
|
|
"finished_at_utc": "2026-09-21T16:06:53Z",
|
|
"reference_register_checked": true,
|
|
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
|
|
"error_register_checked": true,
|
|
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
|
|
"command_sha256": "d156bf6eb6af9791575c95444afc7064500a83372f377c6b64c92c05a8f1640b",
|
|
"duplicate_failed_command_blocked": false,
|
|
"block_reason": null,
|
|
"execution_started": true,
|
|
"change_declared": false,
|
|
"result_contract_valid": true,
|
|
"result_contract_status": null,
|
|
"result_contract_error": null,
|
|
"command_rc": 0,
|
|
"changes_made": false,
|
|
"rollback_started": false,
|
|
"rollback_restored": null,
|
|
"mutation_outcome": "NO_MUTATION",
|
|
"sanitized": true,
|
|
"secrets_included": false,
|
|
"private_addresses_included": false,
|
|
"raw_evidence_retained_locally": true,
|
|
"raw_evidence_sha256": "63a70305030d9344d1975c09a2466596b73b6ebfea50a04eb0e2c68606d8a4be",
|
|
"sanitized_output_sha256": "63a70305030d9344d1975c09a2466596b73b6ebfea50a04eb0e2c68606d8a4be",
|
|
"output_truncated_in_json": false,
|
|
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
|
"output": "DR221_PREFLIGHT399_BEGIN=true\nCOMMAND_ID=NEWFI-260921-NEWFDOM-DR221-PREFLIGHT399\nIMMUTABLE_PLAN_CHECK=PASS\nPREDECESSOR_PROD220_CLOSED=true\nPREDECESSOR_COMMAND_ID=NEWFI-260921-NEWFDOM-PROD220-RECOVERY397\nGIT_STATE={\"branch\":\"implementation/newfi-closed-mvp-20260906\",\"clean\":true,\"head\":\"9c4814db67759dc09a5469d254c41a7b73f4d89a\",\"rcs\":[0,0,0,0],\"remote\":\"9c4814db67759dc09a5469d254c41a7b73f4d89a\"}\nVM210_CLUSTER={\"rc\":0,\"rows\":[{\"maxdisk\":53687091200,\"maxmem\":4294967296,\"name\":\"newfi-prod\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"qemu\",\"uptime\":1271347,\"vmid\":210}]}\nVM210_QM_STATUS={\"rc\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout\":\"status: running\"}\nVM210_CONFIG_SAFE={\"config\":{\"agent\":\"enabled=1\",\"boot\":\"order=scsi0\",\"cores\":\"2\",\"ide2\":\"local-lvm:vm-210-cloudinit,media=cdrom\",\"memory\":\"4096\",\"name\":\"newfi-prod\",\"onboot\":\"1\",\"ostype\":\"l26\",\"scsi0\":\"local-lvm:vm-210-disk-0,discard=on,size=50G,ssd=1\",\"scsihw\":\"virtio-scsi-pci\",\"serial0\":\"socket\"},\"rc\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"}\nRESTORE_TEMP_VMID_CANDIDATES=[9210,9211,9212,9213,9214,9215,9216,9217,9218,9219]\nPVE_STORAGE_SAFE={\"rc\":255,\"rows\":[],\"stderr_sha256\":\"392a23116a78a702472c91f3d369a097fc3e924b89800aa3a8b26910c9f685ec\"}\nBACKUP_RESTORE_HELPERS=[{\"bytes\":7394,\"contract_lines\":[{\"line\":12,\"text\":\"source \\\"$CONFIG\\\"\"},{\"line\":27,\"text\":\"MANIFEST=\\\"$STATE_DIR/manifest-${RUN_STAMP}.txt\\\"\"},{\"line\":92,\"text\":\"for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do\"},{\"line\":100,\"text\":\"for VMID in $VMIDS; do\"},{\"line\":102,\"text\":\"echo \\\"=== VM $VMID PRECHECK ===\\\"\"},{\"line\":104,\"text\":\"STATUS=$(qm status \\\"$VMID\\\" | awk '{print $2}')\"},{\"line\":106,\"text\":\"echo \\\"VM $VMID is not running: $STATUS\\\" >&2\"},{\"line\":110,\"text\":\"timeout 30 qm guest cmd \\\"$VMID\\\" ping >/dev/null\"},{\"line\":112,\"text\":\"BEFORE_FILE=\\\"$STATE_DIR/vm${VMID}-before-${RUN_STAMP}.txt\\\"\"},{\"line\":114,\"text\":\"qm guest exec \\\"$VMID\\\" -- /bin/sh -lc '\"},{\"line\":130,\"text\":\"-name \\\"vzdump-qemu-${VMID}-*.vma.zst\\\" \\\\\"},{\"line\":136,\"text\":\"echo \\\"=== VZDUMP VM $VMID ===\\\"\"},{\"line\":138,\"text\":\"vzdump \\\"$VMID\\\" \\\\\"},{\"line\":147,\"text\":\"ARCHIVE=$(\"},{\"line\":151,\"text\":\"-name \\\"vzdump-qemu-${VMID}-*.vma.zst\\\" \\\\\"},{\"line\":159,\"text\":\"[[ -n \\\"$ARCHIVE\\\" && -f \\\"$ARCHIVE\\\" ]] || {\"},{\"line\":160,\"text\":\"echo \\\"New archive for VM $VMID not found\\\" >&2\"},{\"line\":164,\"text\":\"echo \\\"ARCHIVE=$ARCHIVE\\\"\"},{\"line\":165,\"text\":\"du -h \\\"$ARCHIVE\\\"\"},{\"line\":168,\"text\":\"zstd -t \\\"$ARCHIVE\\\"\"},{\"line\":170,\"text\":\"SHA_FILE=\\\"${ARCHIVE}.sha256\\\"\"},{\"line\":172,\"text\":\"cd \\\"$(dirname \\\"$ARCHIVE\\\")\\\"\"},{\"line\":173,\"text\":\"sha256sum \\\"$(basename \\\"$ARCHIVE\\\")\\\"\"},{\"line\":177,\"text\":\"echo \\\"=== VM $VMID POSTCHECK ===\\\"\"},{\"line\":179,\"text\":\"qm status \\\"$VMID\\\"\"},{\"line\":180,\"text\":\"timeout 30 qm guest cmd \\\"$VMID\\\" ping >/dev/null\"},{\"line\":182,\"text\":\"AFTER_FILE=\\\"$STATE_DIR/vm${VMID}-after-${RUN_STAMP}.txt\\\"\"},{\"line\":184,\"text\":\"qm guest exec \\\"$VMID\\\" -- /bin/sh -lc '\"},{\"line\":198,\"text\":\"CREATED_ARCHIVES+=(\\\"$ARCHIVE\\\")\"},{\"line\":201,\"text\":\"echo \\\"VMID=$VMID\\\"\"},{\"line\":202,\"text\":\"echo \\\"ARCHIVE=$ARCHIVE\\\"\"},{\"line\":203,\"text\":\"echo \\\"SIZE_BYTES=$(stat -c %s \\\"$ARCHIVE\\\")\\\"\"},{\"line\":206,\"text\":\"} >>\\\"$MANIFEST\\\"\"},{\"line\":219,\"text\":\"for ARCHIVE in \\\"${CREATED_ARCHIVES[@]}\\\"; do\"},{\"line\":220,\"text\":\"NAME=$(basename \\\"$ARCHIVE\\\")\"},{\"line\":223,\"text\":\"--archive \\\\\"},{\"line\":229,\"text\":\"\\\"$ARCHIVE\\\" \\\\\"},{\"line\":230,\"text\":\"\\\"${ARCHIVE}.sha256\\\" \\\\\"},{\"line\":254,\"text\":\"echo \\\"=== CLOUD UPLOAD CHUNKED ===\\\"\"},{\"line\":255,\"text\":\"for REMOTE in \\\"$CLOUD_REMOTE_1\\\" \\\"$CLOUD_REMOTE_2\\\"; do\"},{\"line\":256,\"text\":\"DEST_ROOT=\\\"${REMOTE}${CLOUD_BASE}/${TIER}/${RUN_STAMP}\\\"\"},{\"line\":257,\"text\":\"echo \\\"REMOTE=$REMOTE\\\"\"},{\"line\":259,\"text\":\"for ARCHIVE in \\\"${CREATED_ARCHIVES[@]}\\\"; do\"},{\"line\":260,\"text\":\"NAME=$(basename \\\"$ARCHIVE\\\")\"},{\"line\":261,\"text\":\"CHUNK_TMP_BASE=\\\"$BACKUP_DIR/.rclone-chunks\\\" homelab-rclone-copy-chunked \\\"$ARCHIVE\\\" \\\"${DEST_ROOT}/${NAME}\\\"\"},{\"line\":283,\"text\":\"echo \\\"MANIFEST=$MANIFEST\\\"\"}],\"gid\":0,\"mode\":\"0700\",\"path\":\"/usr/local/sbin/homelab-vm-full-backup\",\"sha256\":\"ec16bde2cccf105bc2bd0550517ec8bbca2cbf116645b0bf3cfa15f8ec5ad022\",\"shebang\":\"#!/usr/bin/env bash\",\"uid\":0},{\"bytes\":12618,\"contract_lines\":[{\"line\":5,\"text\":\"source /etc/homelab-vm-full-backup/config.env\"},{\"line\":7,\"text\":\"LOCK=/run/lock/homelab-vm-full-restore-drill.lock\"},{\"line\":10,\"text\":\"echo \\\"Another restore drill is already running\\\" >&2\"},{\"line\":16,\"text\":\"LOG=\\\"$LOG_DIR/restore-drill-${RUN_STAMP}.log\\\"\"},{\"line\":17,\"text\":\"HEALTH=\\\"$STATE_DIR/latest-restore-drill.txt\\\"\"},{\"line\":19,\"text\":\"WORK=\\\"$STATE_DIR/restore-drill-${RUN_STAMP}\\\"\"},{\"line\":30,\"text\":\"local vmid end_epoch duration\"},{\"line\":32,\"text\":\"for vmid in \\\"${CREATED_VMIDS[@]}\\\"; do\"},{\"line\":33,\"text\":\"if qm status \\\"$vmid\\\" >/dev/null 2>&1; then\"},{\"line\":34,\"text\":\"qm stop \\\"$vmid\\\" --skiplock 1 >/dev/null 2>&1 || true\"},{\"line\":37,\"text\":\"[[ \\\"$(qm status \\\"$vmid\\\" 2>/dev/null |\"},{\"line\":43,\"text\":\"qm destroy \\\"$vmid\\\" \\\\\"},{\"line\":66,\"text\":\"cp -f \\\"$HEALTH\\\" /var/lib/homelab-health/vm170-vm171-full-restore-drill.txt\"},{\"line\":77,\"text\":\"-name \\\"vzdump-qemu-${source_vmid}-*.vma.zst\\\" \\\\\"},{\"line\":85,\"text\":\"local vmid=$1\"},{\"line\":95,\"text\":\"qm set \\\"$vmid\\\" \\\"--$key\\\" \\\"${value},link_down=1\\\"\"},{\"line\":96,\"text\":\"done < <(qm config \\\"$vmid\\\")\"},{\"line\":100,\"text\":\"local vmid=$1\"},{\"line\":103,\"text\":\"until timeout 10 qm guest cmd \\\"$vmid\\\" ping >/dev/null 2>&1; do\"},{\"line\":105,\"text\":\"echo \\\"QGA timeout for VM $vmid\\\" >&2\"},{\"line\":114,\"text\":\"local vmid=$1\"},{\"line\":117,\"text\":\"until qm guest exec \\\"$vmid\\\" -- /bin/sh -lc '\"},{\"line\":122,\"text\":\"echo \\\"Docker timeout for VM $vmid\\\" >&2\"},{\"line\":131,\"text\":\"local vmid=$1\"},{\"line\":133,\"text\":\"local tmp=\\\"$WORK/qga-${vmid}-${RANDOM}.json\\\"\"},{\"line\":135,\"text\":\"qm guest exec \\\"$vmid\\\" -- \\\"$@\\\" >\\\"$tmp\\\"\"},{\"line\":141,\"text\":\"with open(sys.argv[1], encoding=\\\"utf-8\\\") as fh:\"},{\"line\":270,\"text\":\"local archive\"},{\"line\":272,\"text\":\"archive=$(latest_archive \\\"$source_vmid\\\")\"},{\"line\":274,\"text\":\"[[ -n \\\"$archive\\\" && -f \\\"$archive\\\" ]] || {\"},{\"line\":275,\"text\":\"echo \\\"Archive not found for VM $source_vmid\\\" >&2\"},{\"line\":279,\"text\":\"[[ -f \\\"${archive}.sha256\\\" ]] || {\"},{\"line\":280,\"text\":\"echo \\\"Checksum file missing: ${archive}.sha256\\\" >&2\"},{\"line\":288,\"text\":\"echo \\\"ARCHIVE=$archive\\\"\"},{\"line\":291,\"text\":\"if qm status \\\"$test_vmid\\\" >/dev/null 2>&1; then\"},{\"line\":292,\"text\":\"echo \\\"Test VMID $test_vmid already exists; refusing restore\\\" >&2\"},{\"line\":297,\"text\":\"cd \\\"$(dirname \\\"$archive\\\")\\\"\"},{\"line\":298,\"text\":\"sha256sum -c \\\"$(basename \\\"${archive}.sha256\\\")\\\"\"},{\"line\":301,\"text\":\"zstd -t \\\"$archive\\\"\"},{\"line\":303,\"text\":\"qmrestore \\\\\"},{\"line\":304,\"text\":\"\\\"$archive\\\" \\\\\"},{\"line\":306,\"text\":\"--storage \\\"$RESTORE_STORAGE\\\" \\\\\"},{\"line\":313,\"text\":\"qm set \\\"$test_vmid\\\" \\\\\"},{\"line\":314,\"text\":\"--name \\\"restore-test-${source_vmid}-${RUN_STAMP}\\\"\"},{\"line\":317,\"text\":\"qm config \\\"$test_vmid\\\"\"},{\"line\":319,\"text\":\"if qm config \\\"$test_vmid\\\" |\"},{\"line\":326,\"text\":\"qm start \\\"$test_vmid\\\"\"},{\"line\":331,\"text\":\"qm guest cmd \\\"$test_vmid\\\" get-host-name\"},{\"line\":332,\"text\":\"qm guest cmd \\\"$test_vmid\\\" get-osinfo\"},{\"line\":336,\"text\":\"echo \\\"=== RESTORE NETWORK ISOLATION PREP ===\\\"\"},{\"line\":339,\"text\":\"RESTORE_TEST_IP=\\\"$(qm config \\\"$RESTORE_TEST_VMID\\\" | sed -n \\\"s/^ipconfig0: ip=\\\\([^,/]*\\\\).*/\\\\1/p\\\" | head -n1)\\\"\"},{\"line\":343,\"text\":\"qm guest exec \\\"$RESTORE_TEST_VMID\\\" --timeout 180 -- bash -lc \\\"ip addr add ${RESTORE_TEST_IP}/32 dev lo 2>/dev/null || true; docker start dockge-agent bookstack-db bookstack prometheus 2>/dev/null || true; sleep 60; docker start dockge-agent bookstack-db bookst...\"},{\"line\":348,\"text\":\"# RESTORE DOCKER CHECK OVERRIDE V2 - isolated restore drill semantics\"},{\"line\":350,\"text\":\"local vmid=\\\"$1\\\"\"},{\"line\":352,\"text\":\"restore_ip=\\\"$(qm config \\\"$vmid\\\" | sed -n 's/^ipconfig0: ip=\\\\([^,/]*\\\\).*/\\\\1/p' | head -n1 || true)\\\"\"},{\"line\":354,\"text\":\"echo \\\"RESTORE_TEST_VMID=$vmid\\\"\"},{\"line\":358,\"text\":\"qm guest exec \\\"$vmid\\\" --timeout 120 -- bash -lc \\\"ip link set lo up || true; ip addr add ${restore_ip}/32 dev lo 2>/dev/null || true\\\" || true\"},{\"line\":361,\"text\":\"qm guest exec \\\"$vmid\\\" --timeout 120 -- bash -lc \\\"systemctl is-active --quiet docker\\\" >/dev/null\"},{\"line\":362,\"text\":\"qm guest exec \\\"$vmid\\\" --timeout 120 -- bash -lc \\\"docker start bookstack-db bookstack dockge-agent prometheus >/dev/null 2>&1 || true\\\" >/dev/null 2>&1 || true\"},{\"line\":368,\"text\":\"out=\\\"$(qm guest exec \\\"$vmid\\\" --timeout 120 -- bash -lc 'docker ps -a --format \\\"{{.Names}}|{{.Status}}\\\" | sort; echo \\\"__TOTAL__=$(docker ps -a -q | wc -l)\\\"; echo \\\"__RUNNING__=$(docker ps -q | wc -l)\\\"; bad=\\\"$(docker ps -a --format \\\"{{.Names}}|{{.Status}}\\\" | awk ...\"},{\"line\":375,\"text\":\"qm guest exec \\\"$vmid\\\" --timeout 120 -- bash -lc \\\"docker start bookstack-db bookstack dockge-agent prometheus >/dev/null 2>&1 || true\\\" >/dev/null 2>&1 || true\"},{\"line\":395,\"text\":\"qm stop \\\"$test_vmid\\\" --skiplock 1 >/dev/null 2>&1 || true; echo \\\"RESTORE_TEST_VM_${source_vmid}_STOPPED_AFTER_CHECK=yes\\\"\"},{\"line\":424,\"text\":\"cp -f \\\"$HEALTH\\\" /var/lib/homelab-health/vm170-vm171-full-restore-drill.txt\"},{\"line\":428,\"text\":\"for vmid in \\\"${CREATED_VMIDS[@]}\\\"; do\"},{\"line\":429,\"text\":\"qm stop \\\"$vmid\\\" --skiplock 1 || true\"},{\"line\":432,\"text\":\"[[ \\\"$(qm status \\\"$vmid\\\" 2>/dev/null |\"},{\"line\":437,\"text\":\"qm destroy \\\"$vmid\\\" \\\\\"}],\"gid\":0,\"mode\":\"0700\",\"path\":\"/usr/local/sbin/homelab-vm-full-restore-drill\",\"sha256\":\"57bba1b7a774b4cfe904dfeb671028bc0427abf04d463e28d8605dbc142faf13\",\"shebang\":\"#!/usr/bin/env bash\",\"uid\":0},{\"bytes\":8681,\"contract_lines\":[{\"line\":5,\"text\":\"WORKER=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\"},{\"line\":6,\"text\":\"PROBE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe\"},{\"line\":7,\"text\":\"LOCK=/run/lock/homelab-vm-cloud-quorum-queue.lock\"},{\"line\":26,\"text\":\"local vmid=\\\"$1\\\"\"},{\"line\":27,\"text\":\"pvesh get /cluster/resources --type vm --output-format json | jq -er --argjson id \\\"$vmid\\\" '.[]|select(.vmid==$id)|.node'\"},{\"line\":47,\"text\":\"local vmid=\\\"$1\\\" action=\\\"$2\\\" generation=\\\"${3:-}\\\" owner observed host\"},{\"line\":48,\"text\":\"owner=\\\"$(expected_owner \\\"$vmid\\\")\\\"\"},{\"line\":49,\"text\":\"observed=\\\"$(observed_owner \\\"$vmid\\\")\\\"\"},{\"line\":50,\"text\":\"[[ \\\"$observed\\\" == \\\"$owner\\\" ]] || die \\\"CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}\\\" 65\"},{\"line\":51,\"text\":\"if [[ \\\"$owner\\\" == \\\"pve01\\\" ]]; then\"},{\"line\":52,\"text\":\"if [[ \\\"$action\\\" == \\\"verify\\\" ]]; then \\\"$WORKER\\\" verify \\\"$vmid\\\" \\\"$generation\\\"; else \\\"$WORKER\\\" \\\"$action\\\" \\\"$vmid\\\"; fi\"},{\"line\":54,\"text\":\"host=\\\"$(owner_host \\\"$owner\\\")\\\"\"},{\"line\":56,\"text\":\"ssh \\\"${SSH_BASE[@]}\\\" \\\"$host\\\" \\\"$WORKER\\\" verify \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":58,\"text\":\"ssh \\\"${SSH_BASE[@]}\\\" \\\"$host\\\" \\\"$WORKER\\\" \\\"$action\\\" \\\"$vmid\\\"\"},{\"line\":63,\"text\":\"local vmid=\\\"$1\\\" owner observed host\"},{\"line\":64,\"text\":\"owner=\\\"$(expected_owner \\\"$vmid\\\")\\\"\"},{\"line\":65,\"text\":\"observed=\\\"$(observed_owner \\\"$vmid\\\")\\\"\"},{\"line\":66,\"text\":\"[[ \\\"$observed\\\" == \\\"$owner\\\" ]] || die \\\"CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}\\\" 65\"},{\"line\":67,\"text\":\"if [[ \\\"$owner\\\" == \\\"pve01\\\" ]]; then\"},{\"line\":68,\"text\":\"\\\"$PROBE\\\" decision \\\"$vmid\\\"\"},{\"line\":70,\"text\":\"host=\\\"$(owner_host \\\"$owner\\\")\\\"\"},{\"line\":71,\"text\":\"ssh \\\"${SSH_BASE[@]}\\\" \\\"$host\\\" \\\"$PROBE\\\" decision \\\"$vmid\\\"\"},{\"line\":75,\"text\":\"local vmid\"},{\"line\":76,\"text\":\"for vmid in \\\"${ALL_VMIDS[@]}\\\"; do\"},{\"line\":77,\"text\":\"printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\\\\n' \\\"$vmid\\\" \\\"$(expected_owner \\\"$vmid\\\")\\\" \\\"$(observed_owner \\\"$vmid\\\")\\\"\"},{\"line\":81,\"text\":\"local vmid=\\\"$1\\\" now_epoch=\\\"$2\\\" out decision generation sealed_epoch age\"},{\"line\":82,\"text\":\"out=\\\"$(run_probe \\\"$vmid\\\")\\\"\"},{\"line\":87,\"text\":\"generation=\\\"$(awk -F= '$1==\\\"GENERATION\\\"{print $2}' <<<\\\"$out\\\" | tail -1)\\\"\"},{\"line\":88,\"text\":\"printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_RESUME GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":91,\"text\":\"generation=\\\"$(awk -F= '$1==\\\"GENERATION\\\"{print $2}' <<<\\\"$out\\\" | tail -1)\\\"\"},{\"line\":94,\"text\":\"printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_EPOCH_INVALID\\\\n' \\\"$vmid\\\"\"},{\"line\":99,\"text\":\"printf 'QUEUE_FRESHNESS VMID=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":101,\"text\":\"printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":103,\"text\":\"printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":105,\"text\":\"printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\"\"},{\"line\":109,\"text\":\"printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED\\\\n' \\\"$vmid\\\"\"},{\"line\":112,\"text\":\"die \\\"FRESHNESS_DECISION_INVALID_${vmid}_${decision:-EMPTY}\\\" 65\"},{\"line\":117,\"text\":\"local vmid now_epoch out decision\"},{\"line\":119,\"text\":\"for vmid in \\\"${ALL_VMIDS[@]}\\\"; do\"},{\"line\":121,\"text\":\"out=\\\"$(freshness_decision \\\"$vmid\\\" \\\"$now_epoch\\\")\\\"\"},{\"line\":124,\"text\":\"[[ -n \\\"$decision\\\" ]] || die \\\"FRESHNESS_PLAN_DECISION_MISSING_$vmid\\\" 65\"},{\"line\":125,\"text\":\"printf 'QUEUE_FRESHNESS_PLAN VMID=%s DECISION=%s\\\\n' \\\"$vmid\\\" \\\"$decision\\\"\"},{\"line\":129,\"text\":\"local vmid out generation\"},{\"line\":134,\"text\":\"for vmid in \\\"$@\\\"; do\"},{\"line\":135,\"text\":\"printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\\\\n' \\\"$vmid\\\" \\\"$(expected_owner \\\"$vmid\\\")\\\"\"},{\"line\":136,\"text\":\"out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":138,\"text\":\"generation=\\\"$(awk -F= '$1==\\\"GENERATION\\\"{print $2}' <<<\\\"$out\\\" | tail -1)\\\"\"},{\"line\":139,\"text\":\"[[ -n \\\"$generation\\\" ]] || die \\\"GENERATION_NOT_RETURNED_$vmid\\\" 65\"},{\"line\":140,\"text\":\"printf 'QUEUE_SEALED VMID=%s GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":147,\"text\":\"local vmid=\\\"$1\\\" freshness decision now_epoch out generation\"},{\"line\":149,\"text\":\"expected_owner \\\"$vmid\\\" >/dev/null\"},{\"line\":152,\"text\":\"printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\\\\n' \\\"$vmid\\\"\"},{\"line\":157,\"text\":\"printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_HEAVY_LOCK_BUSY VMID=%s\\\\n' \\\"$vmid\\\"\"},{\"line\":161,\"text\":\"freshness=\\\"$(exec 8>&-; exec 9>&-; freshness_decision \\\"$vmid\\\" \\\"$now_epoch\\\")\\\"\"},{\"line\":166,\"text\":\"generation=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":167,\"text\":\"printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SKIP_FRESH GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":171,\"text\":\"*) die \\\"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\\\" 65 ;;\"},{\"line\":173,\"text\":\"printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\\\n' \\\"$vmid\\\" \\\"$(expected_owner \\\"$vmid\\\")\\\" \\\"$decision\\\"\"},{\"line\":174,\"text\":\"out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":176,\"text\":\"generation=\\\"$(awk -F= '$1==\\\"GENERATION\\\"{print $2}' <<<\\\"$out\\\" | tail -1)\\\"\"},{\"line\":177,\"text\":\"[[ -n \\\"$generation\\\" ]] || die \\\"GENERATION_NOT_RETURNED_$vmid\\\" 65\"},{\"line\":178,\"text\":\"printf 'QUEUE_SEALED VMID=%s GENERATION=%s DISPATCH=ONE_TARGET\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":179,\"text\":\"printf 'QUEUE_DISPATCH_RESULT KIND=VM VMID=%s RESULT=SEALED GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"}],\"gid\":0,\"mode\":\"0755\",\"path\":\"/usr/local/sbin/homelab-vm-cloud-quorum-queue\",\"sha256\":\"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\",\"shebang\":\"#!/usr/bin/env bash\",\"uid\":0},{\"bytes\":17702,\"contract_lines\":[{\"line\":4,\"text\":\"TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\"},{\"line\":6,\"text\":\"STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\"},{\"line\":7,\"text\":\"STAGE_ROOT=/mnt/staging/homelab-vm-cloud-quorum\"},{\"line\":8,\"text\":\"RESTORE_STATE_ROOT=/var/lib/homelab-backup/vm-restore-validation\"},{\"line\":9,\"text\":\"RESTORE_WORK_ROOT=/mnt/staging/homelab-vm-restore-validation\"},{\"line\":15,\"text\":\"for tool in jq sha256sum zstd tar rclone stat find sort awk date mkfifo tee wc vma; do command -v \\\"$tool\\\" >/dev/null 2>&1 || die \\\"TOOL_MISSING_${tool}\\\" 69; done\"},{\"line\":21,\"text\":\"active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ \\\"$rc\\\" == 1 ]] && return 0; die \\\"PGREP_FAILED_$rc\\\" 69; fi; }\"},{\"line\":22,\"text\":\"snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }\"},{\"line\":23,\"text\":\"vm_lock(){ qm config \\\"$1\\\" | awk -F': ' '$1==\\\"lock\\\"{print $2; found=1} END{if(!found) print \\\"none\\\"}'; }\"},{\"line\":43,\"text\":\"local vmid=\\\"$1\\\" type=\\\"$2\\\"\"},{\"line\":44,\"text\":\"if [[ \\\"$type\\\" == \\\"qemu\\\" ]]; then qm status \\\"$vmid\\\" | awk '{print $2}'; else pct status \\\"$vmid\\\" | awk '{print $2}'; fi\"},{\"line\":47,\"text\":\"local vmid=\\\"$1\\\" owner type status expect_status\"},{\"line\":48,\"text\":\"owner=\\\"$(expected_owner \\\"$vmid\\\")\\\"\"},{\"line\":49,\"text\":\"type=\\\"$(expected_type \\\"$vmid\\\")\\\"\"},{\"line\":50,\"text\":\"[[ \\\"$HOST\\\" == \\\"$owner\\\" ]] || die \\\"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\\\" 65\"},{\"line\":52,\"text\":\"[[ -s \\\"/etc/pve/qemu-server/${vmid}.conf\\\" ]] || die \\\"QEMU_CONFIG_MISSING_$vmid\\\" 66\"},{\"line\":54,\"text\":\"[[ -s \\\"/etc/pve/lxc/${vmid}.conf\\\" ]] || die \\\"LXC_CONFIG_MISSING_$vmid\\\" 66\"},{\"line\":56,\"text\":\"status=\\\"$(current_status \\\"$vmid\\\" \\\"$type\\\")\\\"\"},{\"line\":57,\"text\":\"expect_status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":58,\"text\":\"[[ \\\"$status\\\" == \\\"$expect_status\\\" ]] || die \\\"STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\\\" 66\"},{\"line\":59,\"text\":\"if [[ \\\"$vmid\\\" == \\\"160\\\" ]]; then [[ \\\"$(snapshot_count_160)\\\" == \\\"0\\\" ]] || die \\\"VM160_SNAPSHOT_COUNT_CHANGED\\\" 66; fi\"},{\"line\":60,\"text\":\"if [[ \\\"$vmid\\\" == \\\"9130\\\" ]]; then [[ \\\"$(vm_lock 9130)\\\" == \\\"none\\\" ]] || die \\\"VM9130_LOCK_PRESENT\\\" 66; fi\"},{\"line\":65,\"text\":\"local vmid=\\\"$1\\\" type=\\\"$2\\\" status expect_status\"},{\"line\":66,\"text\":\"status=\\\"$(current_status \\\"$vmid\\\" \\\"$type\\\")\\\"\"},{\"line\":67,\"text\":\"expect_status=\\\"$(expected_status \\\"$vmid\\\")\\\"\"},{\"line\":68,\"text\":\"[[ \\\"$status\\\" == \\\"$expect_status\\\" ]] || die \\\"POST_STATUS_MISMATCH_${vmid}_${status}_EXPECTED_${expect_status}\\\" 66\"},{\"line\":69,\"text\":\"if [[ \\\"$vmid\\\" == \\\"160\\\" ]]; then [[ \\\"$(snapshot_count_160)\\\" == \\\"0\\\" ]] || die \\\"VM160_SNAPSHOT_COUNT_CHANGED_POST\\\" 66; fi\"},{\"line\":70,\"text\":\"if [[ \\\"$vmid\\\" == \\\"9130\\\" ]]; then [[ \\\"$(vm_lock 9130)\\\" == \\\"none\\\" ]] || die \\\"VM9130_LOCK_PRESENT_POST\\\" 66; fi\"},{\"line\":73,\"text\":\"local vmid=\\\"$1\\\" dir=\\\"$STATE_ROOT/vm${vmid}\\\" f\"},{\"line\":81,\"text\":\"local vmid=\\\"$1\\\" gen=\\\"$2\\\" f state sha\"},{\"line\":82,\"text\":\"f=\\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; state=\\\"$STATE_ROOT/vm${vmid}/$gen.json\\\"\"},{\"line\":83,\"text\":\"[[ -s \\\"$f\\\" && ! -L \\\"$f\\\" && -s \\\"$state\\\" && ! -L \\\"$state\\\" ]] || return 1\"},{\"line\":84,\"text\":\"sha=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\" || return 1\"},{\"line\":85,\"text\":\"jq -e --arg l \\\"vm${vmid}\\\" --arg g \\\"$gen\\\" --arg s \\\"$state\\\" --arg sha \\\"$sha\\\" '.schema==\\\"homelab-vm-restore-validation-v1\\\" and .logical_id==$l and .generation==$g and .transport_state==\\\"SEALED\\\" and .transport_state_file==$s and .restore_source==\\\"Cloud A\\\" and .par...\"},{\"line\":88,\"text\":\"local vmid=\\\"$1\\\" gen=\\\"$2\\\" state=\\\"$3\\\" work=\\\"$4\\\" base manifest expected_msha got_msha expected_count expected_size idx name size psha count total\"},{\"line\":89,\"text\":\"base=\\\"$(jq -er '.remote_a_base' \\\"$state\\\")\\\"; expected_msha=\\\"$(jq -er '.parts_manifest_sha256' \\\"$state\\\")\\\"; expected_count=\\\"$(jq -er '.part_count' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"line\":90,\"text\":\"[[ \\\"$base\\\" == \\\"${REMOTE_A%/}/vm${vmid}/$gen\\\" ]] || die \\\"RESTORE_A_BASE_MISMATCH_$vmid\\\" 70\"},{\"line\":91,\"text\":\"manifest=\\\"$work/PARTS.tsv\\\"; rclone cat \\\"${base%/}/PARTS.tsv\\\" >\\\"$manifest\\\"\"},{\"line\":92,\"text\":\"got_msha=\\\"$(sha256sum \\\"$manifest\\\"|awk '{print $1}')\\\"; [[ \\\"$got_msha\\\" == \\\"$expected_msha\\\" ]] || die \\\"RESTORE_A_MANIFEST_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":95,\"text\":\"[[ \\\"$idx\\\" =~ ^[0-9]{8}$ && \\\"$name\\\" == part-????????.bin && \\\"$size\\\" =~ ^[1-9][0-9]*$ && \\\"$psha\\\" =~ ^[0-9a-f]{64}$ ]] || die \\\"RESTORE_A_MANIFEST_ROW_INVALID_$vmid\\\" 70\"},{\"line\":96,\"text\":\"(( size <= 536870912 )) || die \\\"RESTORE_A_PART_EXCEEDS_BOUND_$vmid\\\" 70\"},{\"line\":97,\"text\":\"[[ \\\"$idx\\\" == \\\"$(printf '%08d' \\\"$count\\\")\\\" ]] || die \\\"RESTORE_A_MANIFEST_INDEX_INVALID_$vmid\\\" 70\"},{\"line\":99,\"text\":\"done <\\\"$manifest\\\"\"},{\"line\":100,\"text\":\"[[ \\\"$count\\\" == \\\"$expected_count\\\" ]] || die \\\"RESTORE_A_PART_COUNT_MISMATCH_$vmid\\\" 70\"},{\"line\":101,\"text\":\"[[ \\\"$total\\\" == \\\"$expected_size\\\" ]] || die \\\"RESTORE_A_MANIFEST_TOTAL_SIZE_MISMATCH_$vmid\\\" 70\"},{\"line\":105,\"text\":\"local vmid=\\\"$1\\\" base=\\\"$2\\\" manifest=\\\"$3\\\" work=\\\"$4\\\" idx name size psha part actual count\"},{\"line\":106,\"text\":\"count=0; part=\\\"$work/current.part\\\"\"},{\"line\":108,\"text\":\"rm -f -- \\\"$part\\\"; rclone cat \\\"${base%/}/payload.parts/$name\\\" >\\\"$part\\\"\"},{\"line\":109,\"text\":\"[[ -s \\\"$part\\\" && ! -L \\\"$part\\\" && \\\"$(stat -c %s \\\"$part\\\")\\\" == \\\"$size\\\" ]] || die \\\"RESTORE_A_PART_SIZE_MISMATCH_$vmid\\\" 70\"},{\"line\":110,\"text\":\"actual=\\\"$(sha256sum \\\"$part\\\"|awk '{print $1}')\\\"; [[ \\\"$actual\\\" == \\\"$psha\\\" ]] || die \\\"RESTORE_A_PART_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":111,\"text\":\"cat -- \\\"$part\\\"; rm -f -- \\\"$part\\\"; count=$((count+1))\"},{\"line\":112,\"text\":\"done <\\\"$manifest\\\"\"},{\"line\":115,\"text\":\"local vmid=\\\"$1\\\" gen=\\\"$2\\\" logical state work payload_name expected_sha expected_size base manifest fifo sha_out sha_pid pipe_rc sha_rc actual_sha config config_sha members member_count cfg_member level out tmp\"},{\"line\":116,\"text\":\"assert_local_identity \\\"$vmid\\\"; logical=\\\"vm${vmid}\\\"; state=\\\"$STATE_ROOT/$logical/$gen.json\\\"\"},{\"line\":117,\"text\":\"[[ -s \\\"$state\\\" && ! -L \\\"$state\\\" ]] || die \\\"RESTORE_STATE_MISSING_$vmid\\\" 65\"},{\"line\":118,\"text\":\"jq -e '.schema==\\\"homelab-cloud-quorum-payload-v1\\\" and .status==\\\"SEALED\\\" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_veri...\"},{\"line\":119,\"text\":\"if restore_ok \\\"$vmid\\\" \\\"$gen\\\"; then printf 'RESTORE_VALIDATION_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESULT=ALREADY_VALIDATED\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; return 0; fi\"},{\"line\":121,\"text\":\"payload_name=\\\"$(jq -er '.payload_name' \\\"$state\\\")\\\"; expected_sha=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"line\":122,\"text\":\"[[ \\\"$payload_name\\\" == \\\"vzdump-lxc-${vmid}-\\\"*.tar.zst || \\\"$payload_name\\\" == \\\"vzdump-qemu-${vmid}-\\\"*.vma.zst ]] || die \\\"RESTORE_PAYLOAD_NAME_INVALID_$vmid\\\" 70\"},{\"line\":123,\"text\":\"work=\\\"$RESTORE_WORK_ROOT/$logical/$gen\\\"; if [[ -e \\\"$work\\\" ]]; then [[ -d \\\"$work\\\" && ! -L \\\"$work\\\" ]] || die \\\"RESTORE_WORK_UNSAFE_$vmid\\\" 70; rm -rf --one-file-system \\\"$work\\\"; fi; mkdir -p \\\"$work\\\"; chmod 0700 \\\"$RESTORE_WORK_ROOT\\\" \\\"$RESTORE_WORK_ROOT/$logical\\\" \\\"$w...\"},{\"line\":124,\"text\":\"base=\\\"$(prepare_cloud_a_manifest \\\"$vmid\\\" \\\"$gen\\\" \\\"$state\\\" \\\"$work\\\")\\\"; manifest=\\\"$work/PARTS.tsv\\\"; fifo=\\\"$work/archive.sha.fifo\\\"; sha_out=\\\"$work/archive.sha256\\\"; mkfifo -m 0600 \\\"$fifo\\\"\"},{\"line\":127,\"text\":\"members=\\\"$work/members.txt\\\"; if stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | tee \\\"$fifo\\\" | zstd -dc | tar -tf - >\\\"$members\\\"; then pipe_rc=0; else pipe_rc=$?; fi\"},{\"line\":129,\"text\":\"if stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | tee \\\"$fifo\\\" | zstd -t - >/dev/null; then pipe_rc=0; else pipe_rc=$?; fi\"},{\"line\":132,\"text\":\"(( pipe_rc == 0 && sha_rc == 0 )) || die \\\"RESTORE_A_STREAM_VALIDATION_FAILED_${vmid}_${pipe_rc}_${sha_rc}\\\" 70\"},{\"line\":133,\"text\":\"actual_sha=\\\"$(awk 'NR==1{print $1}' \\\"$sha_out\\\")\\\"; [[ \\\"$actual_sha\\\" == \\\"$expected_sha\\\" ]] || die \\\"RESTORE_A_PAYLOAD_SHA_MISMATCH_$vmid\\\" 70\"},{\"line\":136,\"text\":\"[[ -s \\\"$members\\\" ]] || die \\\"RESTORE_LXC_ARCHIVE_EMPTY_$vmid\\\" 70\"},{\"line\":137,\"text\":\"awk '$0 ~ /^\\\\// || $0 ~ /(^|\\\\/)\\\\.\\\\.(\\\\/|$)/ {bad=1} END{exit bad}' \\\"$members\\\" || die \\\"RESTORE_LXC_MEMBER_UNSAFE_$vmid\\\" 70\"},{\"line\":138,\"text\":\"member_count=\\\"$(wc -l <\\\"$members\\\")\\\"; [[ \\\"$member_count\\\" =~ ^[1-9][0-9]*$ ]] || die \\\"RESTORE_LXC_MEMBER_COUNT_INVALID_$vmid\\\" 70\"},{\"line\":139,\"text\":\"cfg_member=\\\"$(awk '$0==\\\"./etc/vzdump/pct.conf\\\" || $0==\\\"etc/vzdump/pct.conf\\\" {print; exit}' \\\"$members\\\")\\\"; [[ -n \\\"$cfg_member\\\" ]] || die \\\"RESTORE_LXC_CONFIG_MEMBER_MISSING_$vmid\\\" 70\"},{\"line\":140,\"text\":\"if ( set +o pipefail; stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | zstd -dc | tar -xOf - \\\"$cfg_member\\\" >\\\"$config\\\" ); then :; else die \\\"RESTORE_LXC_CONFIG_EXTRACT_FAILED_$vmid\\\" 70; fi\"},{\"line\":141,\"text\":\"level=\\\"cloud-a-full-readback+part-sha+archive-sha+zstd+tar-list+config\\\"\"},{\"line\":143,\"text\":\"if ( set +o pipefail; stream_cloud_a \\\"$vmid\\\" \\\"$base\\\" \\\"$manifest\\\" \\\"$work\\\" | zstd -dc | vma config - >\\\"$config\\\" ); then :; else die \\\"RESTORE_QEMU_CONFIG_EXTRACT_FAILED_$vmid\\\" 70; fi\"},{\"line\":144,\"text\":\"level=\\\"cloud-a-full-readback+part-sha+archive-sha+zstd+vma-config\\\"\"},{\"line\":146,\"text\":\"[[ -s \\\"$config\\\" && ! -L \\\"$config\\\" ]] || die \\\"RESTORE_CONFIG_EMPTY_$vmid\\\" 70; config_sha=\\\"$(sha256sum \\\"$config\\\"|awk '{print $1}')\\\"\"},{\"line\":147,\"text\":\"out=\\\"$(restore_dir \\\"$vmid\\\")/$gen.json\\\"; mkdir -p \\\"$(dirname \\\"$out\\\")\\\"; chmod 0700 \\\"$RESTORE_STATE_ROOT\\\" \\\"$(dirname \\\"$out\\\")\\\"; tmp=\\\"${out}.new.$$\\\"\"},{\"line\":148,\"text\":\"jq -n --arg logical \\\"$logical\\\" --arg generation \\\"$gen\\\" --argjson vmid \\\"$vmid\\\" --arg vm_type \\\"$VM_TYPE\\\" --arg state \\\"$state\\\" --arg sha \\\"$expected_sha\\\" --arg config_sha \\\"$config_sha\\\" --arg level \\\"$level\\\" --argjson member_count \\\"$member_count\\\" --arg at \\\"$(date -u...\"},{\"line\":150,\"text\":\"restore_ok \\\"$vmid\\\" \\\"$gen\\\" || die \\\"RESTORE_EVIDENCE_POSTWRITE_INVALID_$vmid\\\" 70\"},{\"line\":151,\"text\":\"printf 'RESTORE_VALIDATION_STATUS=OK\\\\nVMID=%s\\\\nGENERATION=%s\\\\nRESTORE_EVIDENCE=%s\\\\n' \\\"$vmid\\\" \\\"$gen\\\" \\\"$out\\\"\"},{\"line\":154,\"text\":\"local vmid=\\\"$1\\\" owner type status\"},{\"line\":155,\"text\":\"assert_local_identity \\\"$vmid\\\"\"}],\"gid\":0,\"mode\":\"0750\",\"path\":\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"sha256\":\"91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\",\"shebang\":\"#!/usr/bin/env bash\",\"uid\":0},{\"bytes\":38039,\"contract_lines\":[{\"line\":5,\"text\":\"SCHEMA=\\\"homelab-cloud-quorum-payload-v1\\\"\"},{\"line\":19,\"text\":\"TMP_BASE=\\\"${HOMELAB_QUORUM_TMP_BASE:-/mnt/staging/homelab-cloud-quorum-tmp}\\\"\"},{\"line\":33,\"text\":\"if [[ -n \\\"$WORK\\\" && \\\"$WORK\\\" == \\\"$TMP_BASE\\\"/.quorum.* && -d \\\"$WORK\\\" ]]; then\"},{\"line\":41,\"text\":\"for tool in jq rclone sha256sum dd stat sort mktemp flock awk cmp date; do\"},{\"line\":46,\"text\":\"usage(){\"},{\"line\":47,\"text\":\"printf 'usage: %s backup-delete LOGICAL_ID GENERATION PAYLOAD REMOTE_A_ROOT REMOTE_B_ROOT STATE_ROOT\\\\n' \\\"$0\\\" >&2\"},{\"line\":48,\"text\":\"printf ' %s verify-sealed LOGICAL_ID GENERATION REMOTE_A_ROOT REMOTE_B_ROOT STATE_ROOT\\\\n' \\\"$0\\\" >&2\"},{\"line\":55,\"text\":\"[[ \\\"$#\\\" -eq 7 ]] || { usage; die \\\"ARGUMENT_COUNT_INVALID\\\" 64; }\"},{\"line\":57,\"text\":\"GENERATION=\\\"$3\\\"\"},{\"line\":64,\"text\":\"[[ \\\"$#\\\" -eq 6 ]] || { usage; die \\\"ARGUMENT_COUNT_INVALID\\\" 64; }\"},{\"line\":66,\"text\":\"GENERATION=\\\"$3\\\"\"},{\"line\":73,\"text\":\"usage\"},{\"line\":79,\"text\":\"<SENSITIVE_LINE_WITHHELD>\"},{\"line\":84,\"text\":\"if [[ \\\"$TEST_MODE\\\" != \\\"1\\\" && \\\"$STATE_ROOT\\\" != \\\"/var/lib/homelab-backup/cloud-quorum\\\" ]]; then\"},{\"line\":89,\"text\":\"STATE_FILE=\\\"$STATE_DIR/$GENERATION.json\\\"\"},{\"line\":90,\"text\":\"REMOTE_A_BASE=\\\"${REMOTE_A_ROOT%/}/$LOGICAL_ID/$GENERATION\\\"\"},{\"line\":91,\"text\":\"REMOTE_B_BASE=\\\"${REMOTE_B_ROOT%/}/$LOGICAL_ID/$GENERATION\\\"\"},{\"line\":106,\"text\":\"STATE_FILE=\\\"$STATE_DIR/$GENERATION.json\\\"\"},{\"line\":121,\"text\":\"WORK=\\\"$(mktemp -d \\\"$TMP_BASE/.quorum.XXXXXX\\\")\\\"\"},{\"line\":122,\"text\":\"[[ \\\"$WORK\\\" == \\\"$TMP_BASE\\\"/.quorum.* ]] || die \\\"WORK_PATH_INVALID\\\" 70\"},{\"line\":154,\"text\":\"--arg generation \\\"$GENERATION\\\" \\\\\"},{\"line\":160,\"text\":\".generation==$generation and\"},{\"line\":190,\"text\":\"--arg generation \\\"$GENERATION\\\" \\\\\"},{\"line\":202,\"text\":\"'{schema:$schema,status:$status,logical_id:$logical_id,generation:$generation,\"},{\"line\":380,\"text\":\"RCLONE_BIN=\\\"$(command -v rclone)\\\" || die \\\"RCLONE_MISSING\\\" 69\"},{\"line\":384,\"text\":\"RCLONE_BIN=\\\"$(command -v rclone)\\\" || die \\\"RCLONE_MISSING\\\" 69\"},{\"line\":397,\"text\":\"root_listing=\\\"$WORK/remote-${label}-root.lsf\\\"\"},{\"line\":398,\"text\":\"logical_listing=\\\"$WORK/remote-${label}-logical.lsf\\\"\"},{\"line\":403,\"text\":\"if ! awk -v target=\\\"$LOGICAL_ID/\\\" '$0==target{found=1} END{exit !found}' \\\"$root_listing\\\"; then\"},{\"line\":410,\"text\":\"if awk -v target=\\\"$GENERATION/\\\" '$0==target{found=1} END{exit !found}' \\\"$logical_listing\\\"; then\"},{\"line\":560,\"text\":\"PARTS_MANIFEST=\\\"$WORK/PARTS.tsv\\\"\"},{\"line\":561,\"text\":\"PART_FILE=\\\"$WORK/current.part\\\"\"},{\"line\":567,\"text\":\"printf 'part-%08d.bin\\\\n' \\\"$idx\\\"\"},{\"line\":614,\"text\":\"$2!=sprintf(\\\"part-%08d.bin\\\",NR-1) {exit 1}\"},{\"line\":630,\"text\":\"printf '%s\\\\n' \\\"${base%/}/payload.parts\\\"\"},{\"line\":636,\"text\":\"printf '%s\\\\n' \\\"${base%/}/PARTS.tsv\\\"\"},{\"line\":642,\"text\":\"[[ \\\"$CURRENT_PART_NAME\\\" == part-????????.bin ]] || die \\\"CURRENT_PART_NAME_INVALID\\\" 65\"},{\"line\":676,\"text\":\"$2!=sprintf(\\\"part-%08d.bin\\\",NR-1) {exit 1}\"},{\"line\":700,\"text\":\"remote_manifest=\\\"$WORK/remote-${label}-PARTS.tsv\\\"\"},{\"line\":701,\"text\":\"hash_out=\\\"$WORK/remote-${label}-reconstructed.sha256\\\"\"},{\"line\":710,\"text\":\"[[ \\\"$idx\\\" =~ ^[0-9]{8}$ && \\\"$name\\\" == part-????????.bin && \\\"$size\\\" =~ ^[1-9][0-9]*$ && \\\"$sha\\\" =~ ^[0-9a-f]{64}$ ]] || die \\\"REMOTE_${label}_MANIFEST_ROW_INVALID\\\" 65\"},{\"line\":831,\"text\":\"schema:\\\"homelab-cloud-quorum-seal-v1\\\",\"},{\"line\":834,\"text\":\"generation:.generation,\"},{\"line\":851,\"text\":\".schema==\\\"homelab-cloud-quorum-seal-v1\\\" and\"},{\"line\":854,\"text\":\"(.generation|type==\\\"string\\\" and length>0) and\"}],\"gid\":0,\"mode\":\"0750\",\"path\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"sha256\":\"df9e19485e5d87112bf58bf77bdddf3b9b896418dd85ff945a31cd5b6817f352\",\"shebang\":\"#!/usr/bin/env bash\",\"uid\":0},{\"bytes\":1656,\"contract_lines\":[],\"gid\":0,\"mode\":\"0750\",\"path\":\"/usr/local/sbin/homelab-backup-v2\",\"sha256\":\"5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\",\"shebang\":\"#!/usr/bin/python3\",\"uid\":0}]\nBACKUP_RESTORE_HELPER_MISSING=[]\nBACKUP_RESTORE_UNITS=[{\"bytes\":240,\"contract_lines\":[{\"line\":2,\"text\":\"Description=Homelab canonical VM/CT cloud quorum backup queue\"},{\"line\":3,\"text\":\"After=network-online.target\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":8,\"text\":\"ExecStart=/usr/local/sbin/homelab-vm-cloud-quorum-queue all\"}],\"gid\":0,\"mode\":\"0644\",\"path\":\"/etc/systemd/system/homelab-vm-cloud-quorum-queue.service\",\"sha256\":\"6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759\",\"shebang\":null,\"uid\":0},{\"bytes\":244,\"contract_lines\":[{\"line\":2,\"text\":\"Description=Run canonical Homelab VM/CT cloud quorum freshness queue every 4 hours\"},{\"line\":8,\"text\":\"Unit=homelab-vm-cloud-quorum-queue.service\"},{\"line\":11,\"text\":\"WantedBy=timers.target\"}],\"gid\":0,\"mode\":\"0644\",\"path\":\"/etc/systemd/system/homelab-vm-cloud-quorum-queue.timer\",\"sha256\":\"9a78c9ea507116cbb9f5645fe697e34964c905b017cc98b17977cd5006aa5050\",\"shebang\":null,\"uid\":0}]\nBACKUPV2_CONFIG_CONTRACTS=[{\"meta\":{\"bytes\":5105,\"mode\":\"0600\",\"path\":\"/etc/homelab-backup-v2/targets.json\",\"sha256\":\"a723befc7fd42d70af9088406598f18156dc3b4c76d8b34c4bb725dfaa48dd38\"},\"safe\":{\"targets\":[{\"enabled\":true,\"logical_id\":\"ct110\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":110},{\"enabled\":true,\"logical_id\":\"ct111\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":111},{\"enabled\":true,\"logical_id\":\"ct112\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":112},{\"enabled\":true,\"logical_id\":\"ct113\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":113},{\"enabled\":true,\"logical_id\":\"vm130\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":130},{\"enabled\":true,\"logical_id\":\"vm150\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":150},{\"enabled\":true,\"logical_id\":\"vm160\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":160},{\"enabled\":true,\"logical_id\":\"vm170\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":170},{\"enabled\":true,\"logical_id\":\"vm171\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":171},{\"enabled\":true,\"logical_id\":\"vm190\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":190},{\"enabled\":true,\"logical_id\":\"vm210\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":210},{\"enabled\":true,\"logical_id\":\"vm211\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":211},{\"enabled\":true,\"logical_id\":\"vm9130\",\"scope\":\"pve\",\"type\":\"pve_guest\",\"vmid\":9130},{\"enabled\":true,\"logical_id\":\"node-pve01\",\"node\":\"pve01\",\"scope\":\"node\",\"type\":\"pve_node_bundle\"},{\"enabled\":true,\"logical_id\":\"node-pve02\",\"node\":\"pve02\",\"scope\":\"node\",\"type\":\"pve_node_bundle\"},{\"enabled\":true,\"logical_id\":\"node-pve03\",\"node\":\"pve03\",\"scope\":\"node\",\"type\":\"pve_node_bundle\"},{\"enabled\":true,\"logical_id\":\"md-relay\",\"scope\":\"vps\",\"type\":\"vps_md\"},{\"enabled\":true,\"logical_id\":\"spb-ingress\",\"scope\":\"vps\",\"type\":\"vps_spb\"},{\"enabled\":true,\"logical_id\":\"us-netbird\",\"scope\":\"vps\",\"type\":\"vps_us\"},{\"enabled\":true,\"logical_id\":\"router-netcraze\",\"scope\":\"router\",\"type\":\"router\"},{\"enabled\":true,\"logical_id\":\"xf-codevipe\",\"node\":\"pve02\",\"scope\":\"xenforo\",\"type\":\"xenforo\",\"vmid\":160},{\"enabled\":true,\"logical_id\":\"xf-dsmods\",\"node\":\"pve02\",\"scope\":\"xenforo\",\"type\":\"xenforo\",\"vmid\":160},{\"enabled\":true,\"logical_id\":\"xf-kingofwolk\",\"node\":\"pve02\",\"scope\":\"xenforo\",\"type\":\"xenforo\",\"vmid\":160},{\"enabled\":true,\"logical_id\":\"xf-newfi\",\"node\":\"pve01\",\"scope\":\"xenforo\",\"type\":\"xenforo\",\"vmid\":210},{\"enabled\":true,\"logical_id\":\"xf-gamevipe\",\"node\":\"pve02\",\"scope\":\"xenforo\",\"type\":\"xenforo\",\"vmid\":160},{\"enabled\":true,\"logical_id\":\"xf-hkmods\",\"node\":\"pve02\",\"scope\":\"xenforo\",\"type\":\"xenforo\",\"vmid\":160},{\"enabled\":true,\"logical_id\":\"xf-zakrutim\",\"node\":\"pve02\",\"scope\":\"xenforo\",\"type\":\"xenforo\",\"vmid\":160},{\"enabled\":true,\"logical_id\":\"control-bootstrap-v2\",\"scope\":\"control\",\"type\":\"control\"}]}},{\"meta\":{\"bytes\":5067,\"mode\":\"0600\",\"path\":\"/etc/homelab-backup-v2/production-acceptance.json\",\"sha256\":\"708a481f6719ff1ed8d0afec1902650ed36ac631e980611b015419f504f16df0\"},\"safe\":{\"accepted_release\":\"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1\",\"overlay503\":{\"base_release\":\"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1\",\"lxc_ct110_proof\":{\"generation\":\"v2-ct110-20260917T061845Z-07b0eba22842\",\"node\":\"pve01\",\"replicas\":2},\"lxc_ct111_proof\":{\"generation\":\"v2-ct111-20260917T062204Z-e4a17ab027e0\",\"node\":\"pve02\",\"replicas\":2},\"qemu_historical_proofs\":[\"283R1_VM210_VZDUMP_RC0_TWO_REPLICAS\",\"294R1_QEMU_LOG_SIDECAR_OBSERVED\",\"295R1_QEMU_SIDECARS_REMOVED_1\"]},\"overlay504\":{\"base_release\":\"/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1\"}}}]\nRCLONE_BINARY_SHA256=20df3d5eac5d4e187ce638e4216354e7fa6778c3c51f0caf48c48acca9686354\nRCLONE_REMOTES_SAFE={\"count\":26,\"crypt_count\":13,\"crypt_names\":[\"pve01-mail-01-crypt\",\"pve01-mail-02-crypt\",\"pve01-mail-03-crypt\",\"pve01-mail-04-crypt\",\"pve01-mail-05-crypt\",\"pve02-mail-01-crypt\",\"pve02-mail-02-crypt\",\"pve02-mail-03-crypt\",\"pve02-mail-04-crypt\",\"pve03-mail-01-crypt\",\"pve03-mail-02-crypt\",\"pve03-mail-03-crypt\",\"pve03-mail-04-crypt\"],\"rc\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"}\nCURRENT_REPO_VERIFIERS=[{\"bytes\":1037,\"contract_lines\":[{\"line\":12,\"text\":\"\\\"permission_group_id\\\":\\\"resource\\\",\"},{\"line\":18,\"text\":\"assert a[\\\"resources_kb\\\"]==\\\"public_read_only\\\"\"},{\"line\":20,\"text\":\"assert a[\\\"gallery\\\"]==\\\"registered_only\\\"\"},{\"line\":23,\"text\":\"assert a[\\\"future_medical\\\"]==\\\"private_by_default_owner_scoped_at_MEDICAL218\\\"\"},{\"line\":24,\"text\":\"assert d[\\\"mutation\\\"][\\\"guest_resource_viewUpdateAttach\\\"]==\\\"remain_deny\\\"\"}],\"name\":\"verify-access203.py\",\"sha256\":\"c96393d93d3671f7b625737798c5c46fa67b8985bf90bb4e7668d37cce645acc\"},{\"bytes\":4599,\"contract_lines\":[{\"line\":11,\"text\":\"test \\\"$(runuser -u xfp_newfi -- php8.3 \\\"$R/cmd.php\\\" --version)\\\" = \\\"XenForo 2.3.12\\\"\"},{\"line\":21,\"text\":\"errors=$(mariadb --defaults-extra-file=/etc/newfi/db.cnf --batch --skip-column-names -e \\\"SELECT COUNT(*) FROM xf_error_log\\\")\"},{\"line\":26,\"text\":\"printf \\\"ADDON_ROWS_BEGIN\\\\n%s\\\\nADDON_ROWS_END\\\\nLANGUAGE_ROWS_BEGIN\\\\n%s\\\\nLANGUAGE_ROWS_END\\\\nRU_PHRASE_ROWS_BEGIN\\\\n%s\\\\nRU_PHRASE_ROWS_END\\\\nDEFAULT_LANGUAGE_ID=%s\\\\nRU_PHRASE_COUNT=%s\\\\nDB_TABLES=%s\\\\nBOARD_ROW=%s|%s|%s\\\\nERROR_LOG_COUNT=%s\\\\nMANUAL_JOB_COUNT=%s\\\\n\\\" \\\"$a...\"},{\"line\":36,\"text\":\"test \\\"$errors\\\" -eq 0\"},{\"line\":45,\"text\":\"test \\\"$(systemctl is-active cron)\\\" = active\"}],\"name\":\"verify-addons-language.sh\",\"sha256\":\"c825959b40cb6713aa7932e290d0e1b104ad2e3173d361bf6de617669f0324f0\"},{\"bytes\":1165,\"contract_lines\":[{\"line\":7,\"text\":\"if len(b)<24 or b[:8]!=b'\\\\x89PNG\\\\r\\\\n\\\\x1a\\\\n' or b[12:16]!=b'IHDR': raise RuntimeError('png')\"},{\"line\":10,\"text\":\"if cfg['checkpoint']!='NEWFI-260907-BRAND198E': raise RuntimeError('checkpoint')\"},{\"line\":11,\"text\":\"if cfg['roadmap_before']!=24 or cfg['roadmap_after_pass']!=23: raise RuntimeError('counter')\"},{\"line\":12,\"text\":\"if cfg['manual_profile_assets']!={'avatar':'skipped_user_already_installed','profile_banner':'skipped_user_already_installed'}: raise RuntimeError('manual')\"},{\"line\":13,\"text\":\"if cfg['header_logo']!='deferred_to_STYLE199_no_manual_asset_overwrite': raise RuntimeError('logo')\"},{\"line\":16,\"text\":\"if len(b)!=s['bytes'] or hashlib.sha256(b).hexdigest()!=s['sha256'] or png_size(b)!=(s['width'],s['height']): raise RuntimeError(key)\"},{\"line\":17,\"text\":\"if s['sha256'][:12] not in s['public_path']: raise RuntimeError('cache-bust:'+key)\"}],\"name\":\"verify-brand-assets.py\",\"sha256\":\"efaf9bd3af3ec7d28a8ffd3f3ddf88d25f9e76bbd1c5e262e902523716ede6af\"},{\"bytes\":933,\"contract_lines\":[],\"name\":\"verify-community202.py\",\"sha256\":\"0f4dd2f53788550a12e3d33622ee1b094680037794fe7ae05a58295185699fcb\"},{\"bytes\":6212,\"contract_lines\":[{\"line\":4,\"text\":\"REG=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\"},{\"line\":13,\"text\":\"printf '\\\\033[31mERROR_REGISTER_CHECK=OK\\\\033[0m\\\\n'\"},{\"line\":30,\"text\":\"(\\\"/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\\\",\\\"ERROR_REGISTER\\\"),\"},{\"line\":124,\"text\":\"\\\"ORDER BY resource_category_id\\\"\"},{\"line\":130,\"text\":\"\\\"ORDER BY resource_category_id\\\"\"},{\"line\":205,\"text\":\"\\\"resources\\\":q(\\\"SELECT COUNT(*) FROM xf_rm_resource\\\"),\"},{\"line\":206,\"text\":\"\\\"errors\\\":q(\\\"SELECT COUNT(*) FROM xf_error_log\\\"),\"},{\"line\":225,\"text\":\"and proof[\\\"resources\\\"]==\\\"0\\\"\"},{\"line\":226,\"text\":\"and proof[\\\"errors\\\"]==\\\"0\\\"\"}],\"name\":\"verify-content-structure.sh\",\"sha256\":\"e08bf27cf6a66d438773ce2a7239593804cb06d312c4622b3d576a9c53d27fba\"},{\"bytes\":4890,\"contract_lines\":[{\"line\":7,\"text\":\"GUARD='test \\\"$(id -u):$(hostname -s)\\\" = 0:pve03 || exit 72; for f in /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md /etc/pve/31_HOMELAB_REFERENCE.md; do test -f \\\"$f\\\" && test -r \\\"$f\\\" && test -s \\\"$f\\\" && cat -- \\\"$f\\\" >/dev/null || exit 73; done; '\"},{\"line\":12,\"text\":\"raise RuntimeError(p.stderr.decode(\\\"utf-8\\\",\\\"replace\\\")[:3000])\"},{\"line\":15,\"text\":\"rows=json.loads(run([\\\"/usr/bin/pvesh\\\",\\\"get\\\",\\\"/cluster/resources\\\",\\\"--type\\\",\\\"vm\\\",\\\"--output-format\\\",\\\"json\\\"],t=30).decode())\"},{\"line\":31,\"text\":\"$addon=$db->fetchRow(\\\"SELECT addon_id,active,is_processing,last_pending_action FROM xf_addon WHERE addon_id='Newfi/Dogs'\\\");\"},{\"line\":32,\"text\":\"$route=$db->fetchRow(\\\"SELECT route_type,route_prefix,controller FROM xf_route WHERE addon_id='Newfi/Dogs' AND route_type='public' AND route_prefix='dogs'\\\");\"},{\"line\":36,\"text\":\"'route_rows'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_route WHERE addon_id='Newfi/Dogs'\\\"),\"},{\"line\":37,\"text\":\"'template_rows'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_template WHERE addon_id='Newfi/Dogs'\\\"),\"},{\"line\":38,\"text\":\"'template_exact'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_template WHERE addon_id='Newfi/Dogs' AND type='public' AND title='newfi_dogs'\\\"),\"},{\"line\":41,\"text\":\"'dogs'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_newfi_dog\\\"),\"},{\"line\":42,\"text\":\"'errors'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_error_log\\\"),\"},{\"line\":43,\"text\":\"'jobs'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_job WHERE unique_key LIKE '%Newfi/Dogs%' OR execute_data LIKE '%Newfi/Dogs%'\\\"),\"},{\"line\":47,\"text\":\"'controller_loads'=>class_exists('Newfi\\\\\\\\Dogs\\\\\\\\Pub\\\\\\\\Controller\\\\\\\\Dogs'),\"},{\"line\":48,\"text\":\"'view_loads'=>class_exists('Newfi\\\\\\\\Dogs\\\\\\\\Pub\\\\\\\\View\\\\\\\\Dogs'),\"},{\"line\":49,\"text\":\"'entity_loads'=>class_exists('Newfi\\\\\\\\Dogs\\\\\\\\Entity\\\\\\\\Dog')\"},{\"line\":51,\"text\":\"echo \\\"NEWFI_DOGS_VERIFY=\\\".json_encode($out,JSON_UNESCAPED_SLASHES).PHP_EOL;\"},{\"line\":60,\"text\":\"line=next((x for x in str(d.get(\\\"out-data\\\",\\\"\\\")).splitlines() if x.startswith(\\\"NEWFI_DOGS_VERIFY=\\\")),None)\"},{\"line\":72,\"text\":\"str(a.get(\\\"addon_id\\\"))==\\\"Newfi/Dogs\\\" and str(a.get(\\\"active\\\"))==\\\"1\\\" and str(a.get(\\\"is_processing\\\"))==\\\"0\\\"\"},{\"line\":74,\"text\":\"and route.get(\\\"route_prefix\\\")==\\\"dogs\\\" and route.get(\\\"controller\\\")==\\\"Newfi\\\\\\\\Dogs:Dogs\\\"\"},{\"line\":77,\"text\":\"and r[\\\"dogs\\\"]==0 and r[\\\"errors\\\"]==0 and r[\\\"jobs\\\"]==0\"},{\"line\":78,\"text\":\"and r[\\\"board\\\"]==\\\"0\\\" and r[\\\"index\\\"]==\\\"portal/\\\" and r[\\\"style\\\"]==\\\"4\\\"\"},{\"line\":82,\"text\":\"print(\\\"NEWFI_DOGS_VERIFY=\\\"+json.dumps(r,ensure_ascii=False,sort_keys=True))\"}],\"name\":\"verify-dogs-staging.py\",\"sha256\":\"ce4d3de672356fae4f44beab659d8fc708e814480a9d02b97291db63bd053100\"},{\"bytes\":5621,\"contract_lines\":[{\"line\":7,\"text\":\"GUARD='test \\\"$(id -u):$(hostname -s)\\\" = 0:pve03 || exit 72; for f in /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md /etc/pve/31_HOMELAB_REFERENCE.md; do test -f \\\"$f\\\" && test -r \\\"$f\\\" && test -s \\\"$f\\\" && cat -- \\\"$f\\\" >/dev/null || exit 73; done; '\"},{\"line\":12,\"text\":\"raise RuntimeError(p.stderr.decode(\\\"utf-8\\\",\\\"replace\\\")[:3000])\"},{\"line\":15,\"text\":\"rows=json.loads(run([\\\"/usr/bin/pvesh\\\",\\\"get\\\",\\\"/cluster/resources\\\",\\\"--type\\\",\\\"vm\\\",\\\"--output-format\\\",\\\"json\\\"],t=30).decode())\"},{\"line\":34,\"text\":\"$addon=$db->fetchRow(\\\"SELECT addon_id,active,is_processing,last_pending_action FROM xf_addon WHERE addon_id='Newfi/Events'\\\");\"},{\"line\":35,\"text\":\"$route=$db->fetchRow(\\\"SELECT route_type,route_prefix,controller FROM xf_route WHERE addon_id='Newfi/Events' AND route_type='public' AND route_prefix='events'\\\");\"},{\"line\":39,\"text\":\"'route_rows'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_route WHERE addon_id='Newfi/Events'\\\"),\"},{\"line\":40,\"text\":\"'template_rows'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_template WHERE addon_id='Newfi/Events'\\\"),\"},{\"line\":41,\"text\":\"'template_exact'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_template WHERE addon_id='Newfi/Events' AND type='public' AND title='newfi_events'\\\"),\"},{\"line\":43,\"text\":\"'events'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_newfi_event\\\"),\"},{\"line\":46,\"text\":\"'errors'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_error_log\\\"),\"},{\"line\":50,\"text\":\"'jobs'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_job WHERE unique_key LIKE '%Newfi/Events%' OR execute_data LIKE '%Newfi/Events%'\\\"),\"},{\"line\":51,\"text\":\"'controller_loads'=>class_exists('Newfi\\\\\\\\Events\\\\\\\\Pub\\\\\\\\Controller\\\\\\\\Events'),\"},{\"line\":52,\"text\":\"'view_loads'=>class_exists('Newfi\\\\\\\\Events\\\\\\\\Pub\\\\\\\\View\\\\\\\\Events'),\"},{\"line\":53,\"text\":\"'event_entity_loads'=>class_exists('Newfi\\\\\\\\Events\\\\\\\\Entity\\\\\\\\Event'),\"},{\"line\":54,\"text\":\"'source_entity_loads'=>class_exists('Newfi\\\\\\\\Events\\\\\\\\Entity\\\\\\\\EventSource'),\"},{\"line\":55,\"text\":\"'change_entity_loads'=>class_exists('Newfi\\\\\\\\Events\\\\\\\\Entity\\\\\\\\EventChange')\"},{\"line\":57,\"text\":\"echo \\\"NEWFI_EVENTS_VERIFY=\\\".json_encode($out,JSON_UNESCAPED_SLASHES).PHP_EOL;\"},{\"line\":66,\"text\":\"line=next((x for x in str(d.get(\\\"out-data\\\",\\\"\\\")).splitlines() if x.startswith(\\\"NEWFI_EVENTS_VERIFY=\\\")),None)\"},{\"line\":79,\"text\":\"str(a.get(\\\"addon_id\\\"))==\\\"Newfi/Events\\\" and str(a.get(\\\"active\\\"))==\\\"1\\\" and str(a.get(\\\"is_processing\\\"))==\\\"0\\\"\"},{\"line\":81,\"text\":\"and route.get(\\\"route_prefix\\\")==\\\"events\\\" and route.get(\\\"controller\\\")==\\\"Newfi\\\\\\\\Events:Events\\\"\"},{\"line\":84,\"text\":\"and r[\\\"events\\\"]==0 and r[\\\"sources\\\"]==0 and r[\\\"changes\\\"]==0 and r[\\\"errors\\\"]==0 and r[\\\"jobs\\\"]==0\"},{\"line\":85,\"text\":\"and r[\\\"board\\\"]==\\\"0\\\" and r[\\\"index\\\"]==\\\"portal/\\\" and r[\\\"style\\\"]==\\\"4\\\"\"},{\"line\":89,\"text\":\"print(\\\"NEWFI_EVENTS_VERIFY=\\\"+json.dumps(r,ensure_ascii=False,sort_keys=True))\"}],\"name\":\"verify-events-staging.py\",\"sha256\":\"7235e9dda979c8a6091ac424ebaf241781bac0de94b5148acd7f677485b32ff0\"},{\"bytes\":893,\"contract_lines\":[{\"line\":21,\"text\":\"assert pr[\\\"portal_templates_xml_sha256\\\"]==\\\"a15da1480f674df015cf8443dc4d4cb540334e26cc8fcfc06e4eb34aaa455921\\\"\"}],\"name\":\"verify-forum201.py\",\"sha256\":\"5349228435019a6c7ffc4867f620ac25336cb48fa4042b5793d03899c1aa1b2f\"},{\"bytes\":5019,\"contract_lines\":[{\"line\":4,\"text\":\"REG=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\"},{\"line\":11,\"text\":\"BOUNDED=/var/lib/homelab-backup/vm-restore-validation/vm211/$GEN.json\"},{\"line\":12,\"text\":\"FULL=/var/lib/homelab-backup/vm-restore-validation/vm211/${GEN}-newfi-full.json\"},{\"line\":17,\"text\":\"printf '\\\\033[31mERROR_REGISTER_CHECK=OK\\\\033[0m\\\\n\\\\033[31mREFERENCE_CHECK=OK\\\\033[0m\\\\n'\"},{\"line\":48,\"text\":\"print(\\\"\\\\033[31mERROR_REGISTER_CHECK=OK\\\\033[0m\\\")\"},{\"line\":53,\"text\":\"print(\\\"STOP=RESTORE_EVIDENCE_INVALID:\\\"+path)\"},{\"line\":78,\"text\":\"\\\"restored_guest_proof\\\",\"},{\"line\":97,\"text\":\"and bounded.get(\\\"schema\\\")==\\\"homelab-vm-restore-validation-v1\\\"\"},{\"line\":105,\"text\":\"and full.get(\\\"schema\\\")==\\\"newfi-foundation-full-restore-v1\\\"\"},{\"line\":115,\"text\":\"and full.get(\\\"full_restore_proven\\\") is True\"},{\"line\":130,\"text\":\"and guest.get(\\\"xf_error_log_zero\\\") is True\"},{\"line\":134,\"text\":\"print(\\\"NEWFI_FOUNDATION_RESTORE_VERIFY=FAIL\\\")\"},{\"line\":144,\"text\":\"print(\\\"NEWFI_FOUNDATION_RESTORE_VERIFY=FAIL_TEST_VM_RESIDUE\\\")\"},{\"line\":162,\"text\":\"print(\\\"NEWFI_FOUNDATION_RESTORE_VERIFY=FAIL_TEST_LV_RESIDUE\\\")\"},{\"line\":181,\"text\":\"print(\\\"NEWFI_FOUNDATION_RESTORE_VERIFY=FAIL_REMOTE_SEAL\\\")\"},{\"line\":184,\"text\":\"print(\\\"NEWFI_FOUNDATION_RESTORE_VERIFY=PASS\\\")\"},{\"line\":187,\"text\":\"print(\\\"FULL_RESTORE_PROVEN=true\\\")\"}],\"name\":\"verify-foundation-restore.sh\",\"sha256\":\"cc8b821ffd64c8b2d780eb6efaf96f3724230c2bfe50c7db08c4162f79dc2186\"},{\"bytes\":6256,\"contract_lines\":[{\"line\":8,\"text\":\"GUARD='test \\\"$(id -u):$(hostname -s)\\\" = 0:pve03 || exit 72; for f in /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md /etc/pve/31_HOMELAB_REFERENCE.md; do test -f \\\"$f\\\" && test -r \\\"$f\\\" && test -s \\\"$f\\\" && cat -- \\\"$f\\\" >/dev/null || exit 73; done; '\"},{\"line\":13,\"text\":\"raise RuntimeError(p.stderr.decode(\\\"utf-8\\\",\\\"replace\\\")[:3000])\"},{\"line\":17,\"text\":\"for f in [\\\"/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\\\",\\\"/etc/pve/31_HOMELAB_REFERENCE.md\\\"]:\"},{\"line\":22,\"text\":\"print(\\\"\\\\x1b[31mERROR_REGISTER_CHECK=OK\\\\x1b[0m\\\")\"},{\"line\":26,\"text\":\"rows=json.loads(run([\\\"/usr/bin/pvesh\\\",\\\"get\\\",\\\"/cluster/resources\\\",\\\"--type\\\",\\\"vm\\\",\\\"--output-format\\\",\\\"json\\\"],t=30).decode())\"},{\"line\":33,\"text\":\"portal=(root/\\\"src/addons/Newfi/Portal/_data/templates.xml\\\").read_text(encoding=\\\"utf-8\\\")\"},{\"line\":34,\"text\":\"portal_ok=(\\\"\\u0413\\u0430\\u043b\\u0435\\u0440\\u0435\\u044f\\\" in portal and \\\"link('media')\\\" in portal and \\\"http://\\\" not in portal and \\\"https://\\\" not in portal)\"},{\"line\":46,\"text\":\"$related=$db->fetchAll(\\\"SELECT job_id,unique_key,execute_class FROM xf_job WHERE unique_key LIKE '%Newfi/%' OR unique_key LIKE '%Gallery%' OR unique_key LIKE '%XFMG%' OR execute_class LIKE 'XFMG\\\\\\\\\\\\\\\\%' ORDER BY job_id\\\");\"},{\"line\":68,\"text\":\"'dogs'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_newfi_dog\\\"),\"},{\"line\":71,\"text\":\"'errors'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_error_log\\\"),\"},{\"line\":78,\"text\":\"'dog_entity'=>class_exists('Newfi\\\\\\\\Dogs\\\\\\\\Entity\\\\\\\\Dog'),\"},{\"line\":81,\"text\":\"echo \\\"NEWFI_GALLERY_VERIFY=\\\".json_encode($out,JSON_UNESCAPED_SLASHES).PHP_EOL;\"},{\"line\":90,\"text\":\"line=next((x for x in str(d.get(\\\"out-data\\\",\\\"\\\")).splitlines() if x.startswith(\\\"NEWFI_GALLERY_VERIFY=\\\")),None)\"},{\"line\":99,\"text\":\"portal_ok\"},{\"line\":114,\"text\":\"and r[\\\"dogs\\\"]==0\"},{\"line\":117,\"text\":\"and r[\\\"errors\\\"]==0\"},{\"line\":119,\"text\":\"and r[\\\"index\\\"]==\\\"portal/\\\"\"},{\"line\":128,\"text\":\"r[\\\"portal_native_media_link\\\"]=portal_ok\"},{\"line\":130,\"text\":\"print(\\\"NEWFI_GALLERY_VERIFY=\\\"+json.dumps(r,ensure_ascii=False,sort_keys=True))\"}],\"name\":\"verify-gallery-staging.py\",\"sha256\":\"22db19841305aecacec79e6a12a8f048b0c1efa0d68404b3f376a7aa8c980e45\"},{\"bytes\":3731,\"contract_lines\":[{\"line\":9,\"text\":\"if p.returncode: raise RuntimeError(p.stderr.decode(\\\"utf-8\\\",\\\"replace\\\")[:3000])\"},{\"line\":11,\"text\":\"rows=json.loads(run([\\\"/usr/bin/pvesh\\\",\\\"get\\\",\\\"/cluster/resources\\\",\\\"--type\\\",\\\"vm\\\",\\\"--output-format\\\",\\\"json\\\"],t=30).decode())\"},{\"line\":18,\"text\":\"$cat=$db->fetchRow(\\\"SELECT resource_category_id,parent_category_id,HEX(title) AS title_hex,HEX(description) AS description_hex,resource_count,lft,rgt,depth FROM xf_rm_category WHERE resource_category_id=1\\\");\"},{\"line\":20,\"text\":\"'resources'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_rm_resource\\\"),\"},{\"line\":21,\"text\":\"'updates'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_rm_resource_update\\\"),\"},{\"line\":22,\"text\":\"'versions'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_rm_resource_version\\\"),\"},{\"line\":23,\"text\":\"'errors'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_error_log\\\"),\"},{\"line\":29,\"text\":\"guard='test \\\"$(id -u):$(hostname -s)\\\" = 0:pve03 || exit 72; for f in /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md /etc/pve/31_HOMELAB_REFERENCE.md; do test -f \\\"$f\\\" && test -r \\\"$f\\\" && test -s \\\"$f\\\" && cat -- \\\"$f\\\" >/dev/null || exit 73; done; exec /usr/sbin/qm gu...\"},{\"line\":37,\"text\":\"ok=(r[\\\"category_count\\\"]==1 and str(c.get(\\\"resource_category_id\\\"))==\\\"1\\\" and str(c.get(\\\"parent_category_id\\\"))==\\\"0\\\"\"},{\"line\":39,\"text\":\"and int(c.get(\\\"resource_count\\\",0))==0 and int(c.get(\\\"lft\\\",0))==1 and int(c.get(\\\"rgt\\\",0))==2 and int(c.get(\\\"depth\\\",0))==0\"},{\"line\":40,\"text\":\"and r[\\\"resources\\\"]==0 and r[\\\"updates\\\"]==0 and r[\\\"versions\\\"]==0 and r[\\\"errors\\\"]==0\"},{\"line\":41,\"text\":\"and r[\\\"board\\\"]==\\\"0\\\" and r[\\\"index\\\"]==\\\"portal/\\\" and r[\\\"style\\\"]==\\\"4\\\")\"}],\"name\":\"verify-kb-xfrm.py\",\"sha256\":\"4402881a2e6e19192f97b96a4a83c57d8ce078cfbdb7d1480a3c9ceca8077ca4\"},{\"bytes\":1240,\"contract_lines\":[{\"line\":7,\"text\":\"assert d[\\\"root\\\"]=={\\\"mode\\\":\\\"container_only\\\",\\\"resource_category_id\\\":1,\\\"title\\\":\\\"\\u0411\\u0430\\u0437\\u0430 \\u0437\\u043d\\u0430\\u043d\\u0438\\u0439\\\"}\"}],\"name\":\"verify-kb204.py\",\"sha256\":\"0192589f8a497302edbe738ae04ccad7c878c9b30f08e94203ba5cff5fccbf87\"},{\"bytes\":5256,\"contract_lines\":[{\"line\":7,\"text\":\"GUARD='test \\\"$(id -u):$(hostname -s)\\\" = 0:pve03 || exit 72; for f in /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md /etc/pve/31_HOMELAB_REFERENCE.md; do test -f \\\"$f\\\" && test -r \\\"$f\\\" && test -s \\\"$f\\\" && cat -- \\\"$f\\\" >/dev/null || exit 73; done; '\"},{\"line\":14,\"text\":\"if p.returncode: raise RuntimeError(\\\"subprocess:\\\"+Path(a[0]).name)\"},{\"line\":21,\"text\":\"raise RuntimeError(\\\"qga\\\")\"},{\"line\":26,\"text\":\"if b'\\\"result\\\":\\\"PASS\\\"' not in c: raise RuntimeError(\\\"content\\\")\"},{\"line\":27,\"text\":\"p=run([\\\"/usr/bin/python3\\\",str(ROOT/\\\"scripts/verify-portal-staging.py\\\")],t=420).stdout.decode(\\\"utf-8\\\",\\\"strict\\\")\"},{\"line\":28,\"text\":\"pl=next((x for x in p.splitlines() if x.startswith(\\\"NEWFI_PORTAL_VERIFY=\\\")),None)\"},{\"line\":29,\"text\":\"if not pl or json.loads(pl.split(\\\"=\\\",1)[1]).get(\\\"result\\\")!=\\\"PASS\\\": raise RuntimeError(\\\"portal\\\")\"},{\"line\":32,\"text\":\"nav=(ROOT/\\\"src/addons/Newfi/Portal/_data/navigation.xml\\\").read_text(encoding=\\\"utf-8\\\")\"},{\"line\":33,\"text\":\"phr=(ROOT/\\\"src/addons/Newfi/Portal/_data/phrases.xml\\\").read_text(encoding=\\\"utf-8\\\")\"},{\"line\":34,\"text\":\"tpl=(ROOT/\\\"src/addons/Newfi/Portal/_data/templates.xml\\\").read_text(encoding=\\\"utf-8\\\")\"},{\"line\":35,\"text\":\"if 'navigation_id=\\\"newfiPortal\\\"' not in nav or 'display_order=\\\"150\\\"' not in nav or \\\"{{ link('portal') }}\\\" not in nav:\"},{\"line\":36,\"text\":\"raise RuntimeError(\\\"nav-source\\\")\"},{\"line\":37,\"text\":\"if 'title=\\\"nav.newfiPortal\\\"' not in phr or \\\"<![CDATA[Newfi]]>\\\" not in phr:\"},{\"line\":38,\"text\":\"raise RuntimeError(\\\"phrase-source\\\")\"},{\"line\":39,\"text\":\"for key in (\\\"whats_new_new_posts\\\",\\\"xfrm_whats_new_overview_new_resources\\\"):\"},{\"line\":41,\"text\":\"raise RuntimeError(\\\"widget-source:\\\"+key)\"},{\"line\":44,\"text\":\"rows=json.loads(run([\\\"/usr/bin/pvesh\\\",\\\"get\\\",\\\"/cluster/resources\\\",\\\"--type\\\",\\\"vm\\\",\\\"--output-format\\\",\\\"json\\\"],t=30).stdout.decode())\"},{\"line\":45,\"text\":\"if any(str(x.get(\\\"vmid\\\"))==\\\"210\\\" for x in rows if isinstance(x,dict)): raise RuntimeError(\\\"vm210\\\")\"},{\"line\":47,\"text\":\"if len(b)!=PARKED_BYTES or hashlib.sha256(b).hexdigest()!=PARKED_SHA: raise RuntimeError(\\\"parked\\\")\"},{\"line\":55,\"text\":\"$a=$app->addOnManager()->getById('Newfi/Portal');\"},{\"line\":56,\"text\":\"$nav=$db->fetchRow(\\\"SELECT navigation_id,parent_navigation_id,display_order,navigation_type_id,enabled,addon_id,type_config FROM xf_navigation WHERE navigation_id='newfiPortal'\\\");\"},{\"line\":57,\"text\":\"$keys=$db->fetchAllColumn(\\\"SELECT widget_key FROM xf_widget WHERE widget_key IN ('whats_new_new_posts','xfrm_whats_new_overview_new_resources') ORDER BY widget_key\\\");\"},{\"line\":59,\"text\":\"$phrase=(string)\\\\XF::phrase('nav.newfiPortal');\"},{\"line\":60,\"text\":\"$html=$app->templater()->renderTemplate('public:newfi_portal', []);\"},{\"line\":70,\"text\":\"'errors'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_error_log\\\"),\"},{\"line\":79,\"text\":\"and navr[\\\"navigation_id\\\"]==\\\"newfiPortal\\\" and navr[\\\"parent_navigation_id\\\"]==\\\"\\\"\"},{\"line\":81,\"text\":\"and int(navr[\\\"enabled\\\"])==1 and navr[\\\"addon_id\\\"]==\\\"Newfi/Portal\\\"\"},{\"line\":83,\"text\":\"and sorted(r[\\\"widget_keys\\\"])==[\\\"whats_new_new_posts\\\",\\\"xfrm_whats_new_overview_new_resources\\\"]\"},{\"line\":86,\"text\":\"and r[\\\"errors\\\"]==0 and r[\\\"board_active\\\"]==\\\"0\\\" and r[\\\"index_route\\\"]==\\\"portal/\\\"):\"},{\"line\":87,\"text\":\"raise RuntimeError(\\\"runtime\\\")\"}],\"name\":\"verify-portal-navigation.py\",\"sha256\":\"25c0e846c93b1cfaa23072ad062da6ce32f7df823db642dac18d1150dae05cec\"},{\"bytes\":5003,\"contract_lines\":[{\"line\":7,\"text\":\"GUARD='test \\\"$(id -u):$(hostname -s)\\\" = 0:pve03 || exit 72; for f in /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md /etc/pve/31_HOMELAB_REFERENCE.md; do test -f \\\"$f\\\" && test -r \\\"$f\\\" && test -s \\\"$f\\\" && cat -- \\\"$f\\\" >/dev/null || exit 73; done; '\"},{\"line\":11,\"text\":\"\\\"src/addons/Newfi/Portal/addon.json\\\",\"},{\"line\":12,\"text\":\"\\\"src/addons/Newfi/Portal/Pub/Controller/Portal.php\\\",\"},{\"line\":13,\"text\":\"\\\"src/addons/Newfi/Portal/Pub/View/Portal.php\\\",\"},{\"line\":14,\"text\":\"\\\"src/addons/Newfi/Portal/_data/routes.xml\\\",\"},{\"line\":15,\"text\":\"\\\"src/addons/Newfi/Portal/_data/templates.xml\\\",\"},{\"line\":20,\"text\":\"if p.returncode: raise RuntimeError(\\\"subprocess:\\\"+Path(a[0]).name)\"},{\"line\":26,\"text\":\"raise RuntimeError(\\\"qga\\\")\"},{\"line\":29,\"text\":\"if not (ROOT/r).is_file(): raise RuntimeError(\\\"missing:\\\"+r)\"},{\"line\":31,\"text\":\"raise RuntimeError(\\\"content\\\")\"},{\"line\":32,\"text\":\"rows=json.loads(run([\\\"/usr/bin/pvesh\\\",\\\"get\\\",\\\"/cluster/resources\\\",\\\"--type\\\",\\\"vm\\\",\\\"--output-format\\\",\\\"json\\\"],t=30).stdout.decode())\"},{\"line\":33,\"text\":\"if any(str(x.get(\\\"vmid\\\"))==\\\"210\\\" for x in rows if isinstance(x,dict)): raise RuntimeError(\\\"vm210\\\")\"},{\"line\":35,\"text\":\"if len(b)!=PARKED_BYTES or hashlib.sha256(b).hexdigest()!=PARKED_SHA: raise RuntimeError(\\\"parked\\\")\"},{\"line\":42,\"text\":\"$a=$app->addOnManager()->getById('Newfi/Portal');\"},{\"line\":43,\"text\":\"$row=$db->fetchRow(\\\"SELECT addon_id,active,is_processing,last_pending_action FROM xf_addon WHERE addon_id='Newfi/Portal'\\\");\"},{\"line\":48,\"text\":\"'route_rows'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_route WHERE addon_id='Newfi/Portal' AND route_type='public' AND route_prefix='portal'\\\"),\"},{\"line\":49,\"text\":\"'route_controller'=>(string)$db->fetchOne(\\\"SELECT controller FROM xf_route WHERE addon_id='Newfi/Portal' AND route_type='public' AND route_prefix='portal' LIMIT 1\\\"),\"},{\"line\":50,\"text\":\"'template_rows'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_template WHERE addon_id='Newfi/Portal' AND type='public' AND title='newfi_portal'\\\"),\"},{\"line\":53,\"text\":\"'errors'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_error_log\\\"),\"},{\"line\":56,\"text\":\"'resources'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_rm_resource\\\"),\"},{\"line\":57,\"text\":\"'controller_loads'=>class_exists('Newfi\\\\\\\\Portal\\\\\\\\Pub\\\\\\\\Controller\\\\\\\\Portal'),\"},{\"line\":58,\"text\":\"'view_loads'=>class_exists('Newfi\\\\\\\\Portal\\\\\\\\Pub\\\\\\\\View\\\\\\\\Portal')\"},{\"line\":67,\"text\":\"and r[\\\"route_rows\\\"]==1 and r[\\\"route_controller\\\"]==\\\"Newfi\\\\\\\\Portal:Portal\\\"\"},{\"line\":68,\"text\":\"and r[\\\"template_rows\\\"]==1 and r[\\\"index_route\\\"]==\\\"portal/\\\" and r[\\\"board_active\\\"]==\\\"0\\\"\"},{\"line\":69,\"text\":\"and r[\\\"errors\\\"]==0 and r[\\\"threads\\\"]==0 and r[\\\"posts\\\"]==0 and r[\\\"resources\\\"]==0\"},{\"line\":71,\"text\":\"raise RuntimeError(\\\"runtime\\\")\"},{\"line\":75,\"text\":\"if guest!=repo: raise RuntimeError(\\\"hash\\\")\"},{\"line\":76,\"text\":\"print(\\\"NEWFI_PORTAL_VERIFY=\\\"+json.dumps({\\\"result\\\":\\\"PASS\\\",\\\"runtime\\\":r,\\\"public_parked_exact\\\":True,\\\"vm210_absent\\\":True},sort_keys=True))\"}],\"name\":\"verify-portal-staging.py\",\"sha256\":\"07cdf461aa7126a9519b209f431c0dbff093b3984f372fa855bb6bf53920fab9\"},{\"bytes\":10992,\"contract_lines\":[{\"line\":11,\"text\":\"CONFIG=ROOT/\\\"config/newfi-pwa-staging.json\\\"\"},{\"line\":38,\"text\":\"'for f in /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md /etc/pve/31_HOMELAB_REFERENCE.md; do '\"},{\"line\":43,\"text\":\"class VerifyError(RuntimeError):\"},{\"line\":49,\"text\":\"print(\\\"NEWFI_PWA_VERIFY=\\\"+json.dumps(d,ensure_ascii=False,sort_keys=True))\"},{\"line\":61,\"text\":\"raise VerifyError(\\\"QGA_TRANSPORT_RC\\\")\"},{\"line\":65,\"text\":\"raise VerifyError(\\\"QGA_JSON\\\")\"},{\"line\":67,\"text\":\"raise VerifyError(\\\"QGA_NOT_EXITED\\\")\"},{\"line\":69,\"text\":\"raise VerifyError(\\\"QGA_TRUNCATED\\\")\"},{\"line\":72,\"text\":\"raise VerifyError(\\\"QGA_EXITCODE\\\")\"},{\"line\":75,\"text\":\"GUEST='\\\\nimport hashlib\\\\nimport json\\\\nimport subprocess\\\\n\\\\nENV={\\\"PATH\\\":\\\"/usr/sbin:/usr/bin:/sbin:/bin\\\",\\\"LC_ALL\\\":\\\"C\\\"}\\\\nDB=\\\"/etc/newfi/db.cnf\\\"\\\\n\\\\ndef run(argv, timeout=40):\\\\n p=subprocess.run(\\\\n argv,stdout=subprocess.PIPE,stderr=subprocess.PIPE,\\\\n ...\"},{\"line\":79,\"text\":\"\\\"/usr/bin/pvesh\\\",\\\"get\\\",\\\"/cluster/resources\\\",\"},{\"line\":83,\"text\":\"raise VerifyError(\\\"CLUSTER_QUERY\\\")\"},{\"line\":91,\"text\":\"raise VerifyError(\\\"VM210_PRESENT\\\")\"},{\"line\":96,\"text\":\"raise VerifyError(\\\"VM211_NOT_RUNNING_PVE03\\\")\"},{\"line\":104,\"text\":\"raise VerifyError(\\\"PUBLIC_FETCH\\\")\"},{\"line\":106,\"text\":\"raise VerifyError(\\\"PUBLIC_NOT_PARKED_EXACT\\\")\"},{\"line\":112,\"text\":\"\\\"checkpoint\\\":\\\"native-xenforo-pwa-staging\\\",\"},{\"line\":114,\"text\":\"\\\"custom_pwa_layer\\\":False,\"},{\"line\":124,\"text\":\"raise VerifyError(\\\"CONFIG:\\\"+k)\"},{\"line\":126,\"text\":\"compile(GUEST,\\\"<verify-pwa-staging-guest>\\\",\\\"exec\\\")\"},{\"line\":135,\"text\":\"raise VerifyError(\\\"GUEST_RC:\\\"+str(rc)+\\\":\\\"+err[-600:])\"},{\"line\":137,\"text\":\"raise VerifyError(\\\"GUEST_STDERR:\\\"+err[-600:])\"},{\"line\":138,\"text\":\"marker=next((x for x in out.splitlines() if x.startswith(\\\"NEWFI174_PWA_GUEST=\\\")),None)\"},{\"line\":140,\"text\":\"raise VerifyError(\\\"GUEST_MARKER\\\")\"},{\"line\":145,\"text\":\"\\\"index\\\":\\\"portal/\\\",\"},{\"line\":147,\"text\":\"\\\"errors\\\":0,\"},{\"line\":166,\"text\":\"raise VerifyError(\\\"GUEST_STATE:\\\"+k+\\\":\\\"+repr(g.get(k)))\"},{\"line\":182,\"text\":\"raise VerifyError(\\\"HTTP_MANIFEST_FETCH\\\")\"},{\"line\":185,\"text\":\"raise VerifyError(\\\"HTTP_MANIFEST_BODY\\\")\"},{\"line\":190,\"text\":\"\\\"id\\\":\\\"/index.php?_pwa=1\\\",\"}],\"name\":\"verify-pwa-staging.py\",\"sha256\":\"ad9fd930cd5b52ffd836fcf41f4859bbac910540423d1b8692102ec4c8054f0e\"},{\"bytes\":4335,\"contract_lines\":[{\"line\":6,\"text\":\"TARGETS={\\\"forum\\\":[\\\"viewContent\\\",\\\"viewOthers\\\"],\\\"general\\\":[\\\"viewNode\\\",\\\"viewMemberList\\\",\\\"viewProfile\\\",\\\"search\\\"],\\\"profilePost\\\":[\\\"view\\\"],\\\"resource\\\":[\\\"view\\\",\\\"viewUpdateAttach\\\"],\\\"xfmg\\\":[\\\"view\\\",\\\"viewComments\\\",\\\"viewNote\\\"]}\"},{\"line\":11,\"text\":\"raise RuntimeError(p.stderr.decode(\\\"utf-8\\\",\\\"replace\\\")[:3000])\"},{\"line\":13,\"text\":\"rows=json.loads(run([\\\"/usr/bin/pvesh\\\",\\\"get\\\",\\\"/cluster/resources\\\",\\\"--type\\\",\\\"vm\\\",\\\"--output-format\\\",\\\"json\\\"],t=30).decode())\"},{\"line\":46,\"text\":\"'errors'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_error_log\\\"),\"},{\"line\":52,\"text\":\"guard='test \\\"$(id -u):$(hostname -s)\\\" = 0:pve03 || exit 72; for f in /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md /etc/pve/31_HOMELAB_REFERENCE.md; do test -f \\\"$f\\\" && test -r \\\"$f\\\" && test -s \\\"$f\\\" && cat -- \\\"$f\\\" >/dev/null || exit 73; done; exec /usr/sbin/qm gu...\"},{\"line\":63,\"text\":\"if r[\\\"result\\\"]!=\\\"PASS\\\" or r[\\\"boardDb\\\"]!=\\\"0\\\" or r[\\\"boardRuntime\\\"] is not False or r[\\\"index\\\"]!=\\\"portal/\\\" or r[\\\"styleDb\\\"]!=\\\"4\\\" or r[\\\"styleRuntime\\\"]!=4 or r[\\\"errors\\\"]!=0 or r[\\\"contentPermissionRows\\\"]!=0:\"}],\"name\":\"verify-rights-privacy.py\",\"sha256\":\"2916a5e942f2315a5e90d06cd66a067b84c546c74d7f354e07f7daed306efca6\"},{\"bytes\":1863,\"contract_lines\":[],\"name\":\"verify-staging-host.sh\",\"sha256\":\"9c210b305b07fc4d922f73bccd73c5ec0778337c049c96e076864c93e05ff2d6\"},{\"bytes\":10983,\"contract_lines\":[{\"line\":150,\"text\":\").read_text(\\\"utf-8\\\",errors=\\\"replace\\\")\"},{\"line\":276,\"text\":\"vhost=pathlib.Path(\\\"/etc/nginx/sites-enabled/xf-newfi.conf\\\").read_text(\\\"utf-8\\\",errors=\\\"replace\\\")\"},{\"line\":277,\"text\":\"pool=pathlib.Path(\\\"/etc/php/8.3/fpm/pool.d/xf-newfi.conf\\\").read_text(\\\"utf-8\\\",errors=\\\"replace\\\")\"},{\"line\":311,\"text\":\"\\\"--execute=SELECT COUNT(*) FROM xf_error_log\\\"\"},{\"line\":314,\"text\":\"errors=int(o.strip()) if rc==0 else -1\"},{\"line\":316,\"text\":\"errors=-1\"},{\"line\":317,\"text\":\"facts[\\\"xf_error_log_count\\\"]=errors\"},{\"line\":318,\"text\":\"check(\\\"xf_error_log_zero\\\",errors==0)\"}],\"name\":\"verify-staging-security.sh\",\"sha256\":\"adec12a6ef5ca9ecf14a2c4cb12c810629948f985553c509361264b033af98cf\"},{\"bytes\":4857,\"contract_lines\":[{\"line\":8,\"text\":\"GUARD='test \\\"$(id -u):$(hostname -s)\\\" = 0:pve03 || exit 72; for f in /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md /etc/pve/31_HOMELAB_REFERENCE.md; do test -f \\\"$f\\\" && test -r \\\"$f\\\" && test -s \\\"$f\\\" && cat -- \\\"$f\\\" >/dev/null || exit 73; done; '\"},{\"line\":15,\"text\":\"if p.returncode: raise RuntimeError(\\\"subprocess:\\\"+Path(a[0]).name)\"},{\"line\":22,\"text\":\"raise RuntimeError(\\\"qga\\\")\"},{\"line\":35,\"text\":\"raise RuntimeError(\\\"declaration\\\")\"},{\"line\":40,\"text\":\"[\\\"/usr/bin/python3\\\",str(ROOT/\\\"scripts/verify-portal-staging.py\\\")],\"},{\"line\":41,\"text\":\"[\\\"/usr/bin/python3\\\",str(ROOT/\\\"scripts/verify-portal-navigation.py\\\")]\"},{\"line\":45,\"text\":\"rows=json.loads(run([\\\"/usr/bin/pvesh\\\",\\\"get\\\",\\\"/cluster/resources\\\",\\\"--type\\\",\\\"vm\\\",\\\"--output-format\\\",\\\"json\\\"],t=30).stdout.decode())\"},{\"line\":46,\"text\":\"if any(str(x.get(\\\"vmid\\\"))==\\\"210\\\" for x in rows if isinstance(x,dict)): raise RuntimeError(\\\"vm210\\\")\"},{\"line\":48,\"text\":\"if len(body)!=PARKED_BYTES or hashlib.sha256(body).hexdigest()!=PARKED_SHA: raise RuntimeError(\\\"parked\\\")\"},{\"line\":75,\"text\":\"'errors'=>(int)$db->fetchOne(\\\"SELECT COUNT(*) FROM xf_error_log\\\")\"},{\"line\":81,\"text\":\"raise RuntimeError(\\\"style-match\\\")\"},{\"line\":98,\"text\":\"and r[\\\"index_route\\\"]==\\\"portal/\\\"\"},{\"line\":99,\"text\":\"and r[\\\"errors\\\"]==0\"},{\"line\":101,\"text\":\"raise RuntimeError(\\\"style-runtime\\\")\"}],\"name\":\"verify-style-staging.py\",\"sha256\":\"2a3bc464da8d0dee93e2fd16a8edfd6a7f750a8d28653d5f5bf77691fb86afad\"},{\"bytes\":694,\"contract_lines\":[{\"line\":14,\"text\":\"assert \\\"brand198/newfidom-pwa-192.db0307c1a668.png\\\" in v[\\\"publicLogoUrl\\\"][\\\"default\\\"]\"}],\"name\":\"verify-style199.py\",\"sha256\":\"4fc81cc1d99aac2302c0a0e5e73cbf82959fb2df099cfbeb14db6cb8a40bcca6\"},{\"bytes\":3787,\"contract_lines\":[],\"name\":\"verify-web-foundation.sh\",\"sha256\":\"f7ad539ed2b5c3d7f09aaea0e84b82679a608b60d105271bcb62f0443bb2d84b\"},{\"bytes\":4035,\"contract_lines\":[{\"line\":6,\"text\":\"RUNNER_EXPECTED=$(sha256sum \\\"$ROOT/deploy/cron/newfi-xenforo-run-jobs.sh\\\" | awk \\\"{print \\\\$1}\\\")\"},{\"line\":7,\"text\":\"CRON_EXPECTED=$(sha256sum \\\"$ROOT/deploy/cron/newfi-xenforo-run-jobs.cron\\\" | awk \\\"{print \\\\$1}\\\")\"},{\"line\":14,\"text\":\"test \\\"$(runuser -u xfp_newfi -- php8.3 \\\"$R/cmd.php\\\" --version)\\\" = \\\"XenForo 2.3.12\\\"\"},{\"line\":15,\"text\":\"runuser -u xfp_newfi -- php8.3 \\\"$R/cmd.php\\\" list --raw | grep -Eq \\\"^xf:install[[:space:]]+Installs XenForo$\\\"\"},{\"line\":18,\"text\":\"errors=$(mariadb --defaults-extra-file=/etc/newfi/db.cnf --batch --skip-column-names -e \\\"SELECT COUNT(*) FROM xf_error_log\\\")\"},{\"line\":22,\"text\":\"printf \\\"DB_TABLES=%s\\\\nERROR_LOG_COUNT=%s\\\\nMANUAL_JOB_COUNT=%s\\\\nADMIN_USER=%s\\\\n\\\" \\\"$tables\\\" \\\"$errors\\\" \\\"$manual\\\" \\\"$admin\\\"\"},{\"line\":25,\"text\":\"test \\\"$errors\\\" -eq 0\"},{\"line\":35,\"text\":\"test \\\"$(systemctl is-active cron)\\\" = active\"},{\"line\":36,\"text\":\"test \\\"$(systemctl is-enabled cron)\\\" = enabled\"},{\"line\":37,\"text\":\"test \\\"$(stat -c %U:%G /root/scripts/newfi-xenforo-run-jobs.sh)\\\" = root:root\"},{\"line\":38,\"text\":\"test \\\"$(stat -c %a /root/scripts/newfi-xenforo-run-jobs.sh)\\\" = 700\"},{\"line\":39,\"text\":\"test \\\"$(stat -c %U:%G /etc/cron.d/newfi-xenforo-run-jobs)\\\" = root:root\"},{\"line\":40,\"text\":\"test \\\"$(stat -c %a /etc/cron.d/newfi-xenforo-run-jobs)\\\" = 644\"},{\"line\":41,\"text\":\"printf \\\"RUNNER_LIVE_SHA256=%s\\\\n\\\" \\\"$(sha256sum /root/scripts/newfi-xenforo-run-jobs.sh | cut -d\\\" \\\" -f1)\\\"\"},{\"line\":42,\"text\":\"printf \\\"CRON_LIVE_SHA256=%s\\\\n\\\" \\\"$(sha256sum /etc/cron.d/newfi-xenforo-run-jobs | cut -d\\\" \\\" -f1)\\\"\"},{\"line\":60,\"text\":\"grep -q \\\"CRON_LIVE_SHA256=$CRON_EXPECTED\\\" <<<\\\"$out\\\"\"}],\"name\":\"verify-xenforo-core.sh\",\"sha256\":\"6b6bd2362bdc703f28a5f3d765b26ee2deaefbd74b036c95b2c28b9cd358f277\"}]\nBACKUP_UNIT_STATE={\"homelab-vm-cloud-quorum-queue.service\":{\"active\":\"failed\",\"active_rc\":3,\"enabled\":\"static\",\"enabled_rc\":0},\"homelab-vm-cloud-quorum-queue.timer\":{\"active\":\"inactive\",\"active_rc\":3,\"enabled\":\"disabled\",\"enabled_rc\":1}}\nDECISION=PASS_DR221_PREFLIGHT_CURRENT_BACKUP_RESTORE_CONTRACT_CAPTURED\nNEXT_GATE=RUN_DR221_FINAL_CURRENT_PRODUCTION_BACKUP_RESTORE_ACCEPTANCE\nACTIVE_MAJOR_STAGE=DR221\nMAJOR_STAGES_REMAINING=1\nHELPERS_EXECUTED=0\nCLOUD_READS=0\nCLOUD_WRITES=0\nVM_MUTATIONS=0\nPRODUCTION_MUTATION=false\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"NEWFI-260921-NEWFDOM-DR221-PREFLIGHT399\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\nDR221_PREFLIGHT399_END=true\n"
|
|
}
|