homelab-runtime-publisher
39 lines
20 KiB
JSON
39 lines
20 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"channel": "homelab-runtime",
|
|
"command_id": "SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1",
|
|
"status": "OK",
|
|
"rc": 0,
|
|
"host": "pve01",
|
|
"mode": "read-only",
|
|
"component": "homelab-backup-post1045-v2-rca",
|
|
"started_at_utc": "2026-09-21T03:46:39Z",
|
|
"finished_at_utc": "2026-09-21T03:46:41Z",
|
|
"reference_register_checked": true,
|
|
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
|
|
"error_register_checked": true,
|
|
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
|
|
"command_sha256": "e885553c8f307a102ca6681869f3474d99c8bd6c2674156eda9ad554a27a5741",
|
|
"duplicate_failed_command_blocked": false,
|
|
"block_reason": null,
|
|
"execution_started": true,
|
|
"change_declared": false,
|
|
"result_contract_valid": true,
|
|
"result_contract_status": null,
|
|
"result_contract_error": null,
|
|
"command_rc": 0,
|
|
"changes_made": false,
|
|
"rollback_started": false,
|
|
"rollback_restored": null,
|
|
"mutation_outcome": "NO_MUTATION",
|
|
"sanitized": true,
|
|
"secrets_included": false,
|
|
"private_addresses_included": false,
|
|
"raw_evidence_retained_locally": true,
|
|
"raw_evidence_sha256": "9bbb3ae7025709b275ab9507dcf0e7ce1dce96ea6f8d3b503011b80397be29af",
|
|
"sanitized_output_sha256": "9bbb3ae7025709b275ab9507dcf0e7ce1dce96ea6f8d3b503011b80397be29af",
|
|
"output_truncated_in_json": false,
|
|
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
|
"output": "WORKERS2_BACKUP_POST1045_V2_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1\nMODE=read-only\nCOMPONENT=homelab-backup-post1045-v2-rca\nHOST=pve01\nUID=0\nMUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL\nWORKTREE_HEAD=47b9be72cf2487ba66295d7da998016bbeea57be\nWORKTREE_HEAD_EXPECTED=true\nWORKTREE_CLEAN=true\nWORKTREE_STATUS_LINES=[]\nSOURCE_COMMIT_SUMMARY={\"lines\":[\"47b9be7 CR-2026-1005: close V2 backup scheduling and retention gaps\",\" .../backup-closure-remediation-1045.md | 3 +\",\" changes/CR-2026-1005/runtime/SHA256SUMS | 4 +\",\" .../runtime/homelab-app-cloud-quorum-queue.service | 10 +\",\" .../CR-2026-1005/runtime/homelab-backup-audit-v2 | 200 +++++++++\",\" .../runtime/homelab-backup-retention-gc | 464 +++++++++++++++++++++\",\" .../runtime/homelab-backup-v2-app-scheduled | 27 ++\",\" 6 files changed, 708 insertions(+)\"],\"rc\":0}\nSOURCE_RUNTIME_TREE=[{\"bytes\":503,\"name\":\"SHA256SUMS\",\"sha256\":\"921dce0c32c2246bf5cf7cacd7e23f3d2677084165d79e05e303682dc14d35ed\"},{\"bytes\":244,\"name\":\"homelab-app-cloud-quorum-queue.service\",\"sha256\":\"afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a\"},{\"bytes\":11451,\"name\":\"homelab-backup-audit-v2\",\"sha256\":\"67fff795217e3e9c3a32c5a634a0401efa2b74c533b934f0a04b52a4cd9b6594\"},{\"bytes\":32975,\"name\":\"homelab-backup-retention-gc\",\"sha256\":\"a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909\"},{\"bytes\":1372,\"name\":\"homelab-backup-v2-app-scheduled\",\"sha256\":\"52951cd5c00a60b1a2f7d8c44d7e25feae39706820ad56964ef013e6451ca1ba\"}]\nPOST1045_LIVE_RUNTIME={\"app_scheduler\":{\"exists\":false,\"path\":\"/usr/local/sbin/homelab-backup-v2-app-scheduled\",\"sha256\":null},\"app_service\":{\"baseline_match\":true,\"exists\":true,\"path\":\"/etc/systemd/system/homelab-app-cloud-quorum-queue.service\",\"sha256\":\"70b78ba529c270ec61d26a123d656385a8b2e2e4206ccf623111fa6ed3854602\"},\"audit\":{\"baseline_match\":true,\"exists\":true,\"path\":\"/usr/local/sbin/homelab-backup-audit\",\"sha256\":\"1474c9ce58bbc1b3ae7437261d1bdb923232926fb2943c7c830253f8b284724c\"},\"retention\":{\"baseline_match\":true,\"exists\":true,\"path\":\"/usr/local/sbin/homelab-backup-retention-gc\",\"sha256\":\"8adb6c10f5911f39c53775b89eac3cd1823f1ed34b4caccade5279c213e998a3\"}}\nPOST1045_UNIT_STATE={\"homelab-app-cloud-quorum-queue.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"0\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"0\"},\"ExecStart\":{\"rc\":0,\"value\":\"{ path=/usr/local/sbin/homelab-app-cloud-quorum-queue ; argv[]=/usr/local/sbin/homelab-app-cloud-quorum-queue all ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-app-cloud-quorum-queue.service\"},\"InvocationID\":{\"rc\":0,\"value\":\"\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-app-cloud-quorum-queue.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"ExecStart\":{\"rc\":0,\"value\":\"\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-app-cloud-quorum-queue.timer\"},\"InvocationID\":{\"rc\":0,\"value\":\"\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-retention-gc.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"1\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"66\"},\"ExecStart\":{\"rc\":0,\"value\":\"{ path=/usr/local/sbin/homelab-backup-retention-gc ; argv[]=/usr/local/sbin/homelab-backup-retention-gc scheduled ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-retention-gc.service\"},\"InvocationID\":{\"rc\":0,\"value\":\"426b1424d1f047e8adbb452ece2b3b80\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-backup-retention-gc.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"ExecStart\":{\"rc\":0,\"value\":\"\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-retention-gc.timer\"},\"InvocationID\":{\"rc\":0,\"value\":\"\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-v2.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"failed\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"1\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"2\"},\"ExecStart\":{\"rc\":0,\"value\":\"{ path=/usr/local/sbin/homelab-backup-v2 ; argv[]=/usr/local/sbin/homelab-backup-v2 scheduled ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-v2.service\"},\"InvocationID\":{\"rc\":0,\"value\":\"43c58a804cea48c2818a509fd2e75f48\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"exit-code\"},\"SubState\":{\"rc\":0,\"value\":\"failed\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-backup-v2.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"active\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"ExecStart\":{\"rc\":0,\"value\":\"\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-v2.timer\"},\"InvocationID\":{\"rc\":0,\"value\":\"af3b714a30a541b0b91f4e1d2e2b5980\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"waiting\"},\"UnitFileState\":{\"rc\":0,\"value\":\"enabled\"}}}\nAPP_FRESHNESS_PLAN_POST1045={\"lines\":[\"QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\"],\"rc\":0,\"stderr\":[]}\nVM9130_STATE={\"age_seconds\":108844,\"consecutive_failures\":0,\"exists\":true,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm9130-20260919T210758Z-0aa987efac8c\",\"last_attempt_epoch\":1789853557,\"last_success_epoch\":1789853557,\"logical_id\":\"vm9130\",\"replica_count\":2,\"result\":\"SUCCESS\",\"sha256\":\"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db\"}\nV2_SERVICE_JOURNAL_1045_WINDOW={\"lines\":[\"-- No entries --\"],\"rc\":0,\"stderr\":[]}\nV2_SERVICE_ROOT_CAUSE_LINES=[]\nV2_WRAPPER_SOURCE={\"path\":\"/usr/local/sbin/homelab-backup-v2\",\"relevant_lines\":[{\"line\":4,\"text\":\"ACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b')\"}],\"sha256\":\"5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\"}\nVM_QUEUE_SOURCE={\"path\":\"/usr/local/sbin/homelab-vm-cloud-quorum-queue\",\"relevant_lines\":[{\"line\":5,\"text\":\"WORKER=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\"},{\"line\":18,\"text\":\" case \\\"$1\\\" in\"},{\"line\":30,\"text\":\" case \\\"$1\\\" in\"},{\"line\":36,\"text\":\"run_worker() {\"},{\"line\":43,\"text\":\" _run_worker_unlocked \\\"$@\\\"\"},{\"line\":46,\"text\":\"_run_worker_unlocked() {\"},{\"line\":52,\"text\":\" if [[ \\\"$action\\\" == \\\"verify\\\" ]]; then \\\"$WORKER\\\" verify \\\"$vmid\\\" \\\"$generation\\\"; else \\\"$WORKER\\\" \\\"$action\\\" \\\"$vmid\\\"; fi\"},{\"line\":56,\"text\":\" ssh \\\"${SSH_BASE[@]}\\\" \\\"$host\\\" \\\"$WORKER\\\" verify \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":58,\"text\":\" ssh \\\"${SSH_BASE[@]}\\\" \\\"$host\\\" \\\"$WORKER\\\" \\\"$action\\\" \\\"$vmid\\\"\"},{\"line\":85,\"text\":\" case \\\"$decision\\\" in\"},{\"line\":88,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_RESUME GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":94,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_EPOCH_INVALID\\\\n' \\\"$vmid\\\"\"},{\"line\":99,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":101,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":103,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":105,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\"\"},{\"line\":109,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED\\\\n' \\\"$vmid\\\"\"},{\"line\":123,\"text\":\" decision=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\\\"\\\",$i);print $i}}' <<<\\\"$out\\\" | tail -1)\\\"\"},{\"line\":125,\"text\":\" printf 'QUEUE_FRESHNESS_PLAN VMID=%s DECISION=%s\\\\n' \\\"$vmid\\\" \\\"$decision\\\"\"},{\"line\":128,\"text\":\"backup_list_force(){\"},{\"line\":136,\"text\":\" out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":148,\"text\":\" [[ \\\"${HOMELAB_BACKUP_DISPATCHER:-}\\\" == 1 ]] || die \\\"DISPATCH_ONE_DIRECT_FORBIDDEN\\\" 64\"},{\"line\":163,\"text\":\" decision=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":164,\"text\":\" case \\\"$decision\\\" in\"},{\"line\":166,\"text\":\" generation=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":174,\"text\":\" out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":183,\"text\":\"case \\\"$ACTION\\\" in\"},{\"line\":188,\"text\":\" freshness-plan)\"},{\"line\":196,\"text\":\" dispatch-one)\"},{\"line\":202,\"text\":\" backup_list_force 110\"},{\"line\":206,\"text\":\" backup_list_force \\\"${MIGRATE_VMIDS[@]}\\\"\"},{\"line\":208,\"text\":\" all)\"},{\"line\":210,\"text\":\" [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die \\\"DISPATCHER_MISSING\\\" 69\"},{\"line\":211,\"text\":\" /usr/local/sbin/homelab-backup-dispatcher run VM_TIMER\"},{\"line\":215,\"text\":\" run_worker \\\"$2\\\" verify \\\"$3\\\"\"}],\"sha256\":\"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\"}\nDISPATCHER_SOURCE={\"path\":\"/usr/local/sbin/homelab-backup-dispatcher\",\"relevant_lines\":[{\"line\":4,\"text\":\"VMQ=/usr/local/sbin/homelab-vm-cloud-quorum-queue\"},{\"line\":5,\"text\":\"APPQ=/usr/local/sbin/homelab-app-cloud-quorum-queue\"},{\"line\":9,\"text\":\"rank(){ case \\\"$1\\\" in BACKUP_RESUME) printf 0;; BACKUP_UNVERIFIED) printf 1;; BACKUP_STALE) printf 2;; BACKUP_DUE) printf 3;; NOOP) printf 9;; *) return 1;; esac; }\"},{\"line\":11,\"text\":\" local kind=\\\"$1\\\" q=\\\"$2\\\" plan line decision target age pri best_target=\\\"\\\" best_decision=NOOP best_age=-1 best_pri=99\"},{\"line\":15,\"text\":\" if [[ \\\"$kind\\\" == VM ]]; then [[ \\\"$line\\\" == QUEUE_FRESHNESS\\\\ VMID=* ]] || continue; else [[ \\\"$line\\\" == QUEUE_FRESHNESS\\\\ APP=* ]] || continue; fi\"},{\"line\":16,\"text\":\" decision=\\\"\\\"; target=\\\"\\\"; age=-1\"},{\"line\":17,\"text\":\" for token in $line; do case \\\"$token\\\" in VMID=*) target=\\\"${token#VMID=}\\\";; APP=*) target=\\\"${token#APP=}\\\";; DECISION=*) decision=\\\"${token#DECISION=}\\\";; AGE_SECONDS=*) age=\\\"${token#AGE_SECONDS=}\\\";; esac; done\"},{\"line\":18,\"text\":\" [[ -n \\\"$target\\\" && -n \\\"$decision\\\" ]] || die \\\"SCAN_RECORD_INVALID_$kind\\\" 65\"},{\"line\":19,\"text\":\" case \\\"$decision\\\" in SKIP_FRESH) continue;; BACKUP_RESUME|BACKUP_UNVERIFIED) pri=\\\"$(rank \\\"$decision\\\")\\\";; BACKUP_STALE|BACKUP_DUE) pri=\\\"$(rank \\\"$decision\\\")\\\"; [[ \\\"$age\\\" =~ ^[0-9]+$ ]] || die \\\"AGE_INVALID_${kind}_${target}_${decision}\\\" 65;; *) die \\\"SCAN_DECISION_INVALID_${kind}_${target}_${decision}\\\" 65;; esac\"},{\"line\":20,\"text\":\" if [[ -z \\\"$best_target\\\" || \\\"$pri\\\" -lt \\\"$best_pri\\\" || ( \\\"$pri\\\" -eq \\\"$best_pri\\\" && ( \\\"$pri\\\" -eq 2 || \\\"$pri\\\" -eq 3 ) && \\\"$age\\\" -gt \\\"$best_age\\\" ) ]]; then best_target=\\\"$target\\\"; best_decision=\\\"$decision\\\"; best_age=\\\"$age\\\"; best_pri=\\\"$pri\\\"; fi\"},{\"line\":21,\"text\":\" done <<<\\\"$plan\\\"\"},{\"line\":22,\"text\":\" if [[ -z \\\"$best_target\\\" ]]; then printf 'CLASS_DISPATCH KIND=%s DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1\\\\n' \\\"$kind\\\"; else printf 'CLASS_DISPATCH KIND=%s TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$kind\\\" \\\"$best_target\\\" \\\"$best_decision\\\" \\\"$best_pri\\\" \\\"$best_age\\\"; fi\"},{\"line\":28,\"text\":\" for token in $vline; do case \\\"$token\\\" in TARGET=*) vt=\\\"${token#TARGET=}\\\";; DECISION=*) vd=\\\"${token#DECISION=}\\\";; PRIORITY=*) vp=\\\"${token#PRIORITY=}\\\";; AGE_SECONDS=*) va=\\\"${token#AGE_SECONDS=}\\\";; esac; done\"},{\"line\":29,\"text\":\" for token in $aline; do case \\\"$token\\\" in TARGET=*) at=\\\"${token#TARGET=}\\\";; DECISION=*) ad=\\\"${token#DECISION=}\\\";; PRIORITY=*) ap=\\\"${token#PRIORITY=}\\\";; AGE_SECONDS=*) aa=\\\"${token#AGE_SECONDS=}\\\";; esac; done\"},{\"line\":31,\"text\":\" if [[ \\\"$vd\\\" == NOOP ]]; then printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$at\\\" \\\"$ad\\\" \\\"$ap\\\" \\\"$aa\\\"; return 0; fi\"},{\"line\":32,\"text\":\" if [[ \\\"$ad\\\" == NOOP ]]; then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$vt\\\" \\\"$vd\\\" \\\"$vp\\\" \\\"$va\\\"; return 0; fi\"},{\"line\":33,\"text\":\" if (( vp < ap )); then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$vt\\\" \\\"$vd\\\" \\\"$vp\\\" \\\"$va\\\"; elif (( ap < vp )); then printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$at\\\" \\\"$ad\\\" \\\"$ap\\\" \\\"$aa\\\"; elif (( vp == ap && ( vp == 2 || vp == 3 ) && va > aa )); then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$vt\\\" \\\"$vd\\\" \\\"$vp\\\" \\\"$va\\\"; else printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$at\\\" \\\"$ad\\\" \\\"$ap\\\" \\\"$aa\\\"; fi\"},{\"line\":37,\"text\":\" local trigger=\\\"$1\\\" i=0 actions=0 scan pick decision kind target out rc last_sealed=\\\"\\\"\"},{\"line\":42,\"text\":\" decision=\\\"\\\"; kind=\\\"\\\"; target=\\\"\\\"; for token in $pick; do case \\\"$token\\\" in KIND=*) kind=\\\"${token#KIND=}\\\";; TARGET=*) target=\\\"${token#TARGET=}\\\";; DECISION=*) decision=\\\"${token#DECISION=}\\\";; esac; done\"},{\"line\":43,\"text\":\" if [[ \\\"$decision\\\" == NOOP ]]; then printf 'DISPATCHER_DONE TRIGGER=%s ACTIONS=%s ITERATIONS=%s\\\\n' \\\"$trigger\\\" \\\"$actions\\\" \\\"$i\\\"; return 0; fi\"},{\"line\":44,\"text\":\" [[ \\\"$kind\\\" == VM || \\\"$kind\\\" == APP ]] || die GLOBAL_KIND_INVALID 65; [[ -n \\\"$target\\\" ]] || die GLOBAL_TARGET_MISSING 65\"},{\"line\":45,\"text\":\" if [[ \\\"$last_sealed\\\" == \\\"$kind:$target\\\" ]]; then die \\\"NO_PROGRESS_AFTER_SEALED_${kind}_${target}\\\" 65; fi\"},{\"line\":46,\"text\":\" if [[ \\\"$kind\\\" == VM ]]; then if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$VMQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; else if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$APPQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; fi\"},{\"line\":48,\"text\":\" if grep -q '^QUEUE_DEFERRED ' <<<\\\"$out\\\"; then printf 'DISPATCHER_DEFERRED TRIGGER=%s KIND=%s TARGET=%s REASON=QUEUE_OR_HEAVY_BUSY\\\\n' \\\"$trigger\\\" \\\"$kind\\\" \\\"$target\\\"; return 0; fi\"},{\"line\":49,\"text\":\" if grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SEALED ' <<<\\\"$out\\\"; then last_sealed=\\\"$kind:$target\\\"; actions=$((actions+1)); elif grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SKIP_FRESH ' <<<\\\"$out\\\"; then last_sealed=\\\"\\\"; actions=$((actions+1)); else die \\\"DISPATCH_RESULT_UNCLASSIFIED_${kind}_${target}\\\" 65; fi\"},{\"line\":50,\"text\":\" done\"},{\"line\":53,\"text\":\"case \\\"${1:-}\\\" in plan) [[ \\\"$#\\\" == 1 ]] || die USAGE_PLAN 64; plan;; run) [[ \\\"$#\\\" == 2 ]] || die USAGE_RUN 64; run \\\"$2\\\";; *) die ACTION_INVALID 64;; esac\"}],\"sha256\":\"9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c\"}\nCORE_RUNTIME_ROLLBACK_VERIFIED=true\nAPP_CATCHUP_PRESERVED=true\nSOURCE_COMMIT_PRESERVED=true\nRETENTION_GC_RAN=false\nPOST1045_RCA_DECISION={\"app_catchup_preserved\":true,\"core_runtime_rollback_verified\":true,\"journal_has_root_cause_lines\":false,\"source_commit_preserved\":true,\"vm9130_needs_targeted_continuation\":true}\nNO_MUTATION=true\nNO_REPO_WRITE=true\nNO_GIT_FETCH=true\nNO_GIT_COMMIT=true\nNO_GIT_PUSH_UPDATE=true\nNO_SYSTEMD_ACTION=true\nNO_BACKUP_STARTED=true\nNO_RESTORE_STARTED=true\nNO_RETENTION_STARTED=true\nNO_DELETE=true\nNO_MAIL_SENT=true\nDURATION_SEC=1.467\nTASK_RESULT=PASS_HOMELAB_BACKUP_POST1045_V2_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\nWORKERS2_BACKUP_POST1045_V2_RCA_END=1\n"
|
|
}
|