From 1fc2ac2c62a24dde60ae187836e865bdbdf3c1ee Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Mon, 21 Sep 2026 03:46:41 +0000 Subject: [PATCH] runtime: publish SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1 --- ...ACKUP-POST1045-V2-RCA-READONLY-1046R1.json | 38 ++++++++++ ...BACKUP-POST1045-V2-RCA-READONLY-1046R1.txt | 74 ++++++++++++++++++ runtime/latest.json | 34 ++++----- runtime/latest.txt | 75 ++++++++++++++----- 4 files changed, 186 insertions(+), 35 deletions(-) create mode 100644 runtime/history/SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1.json create mode 100644 runtime/history/SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1.txt diff --git a/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1.json b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1.json new file mode 100644 index 00000000..4bc8b570 --- /dev/null +++ b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-post1045-v2-rca", + "started_at_utc": "2026-09-21T03:46:39Z", + "finished_at_utc": "2026-09-21T03:46:41Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "e885553c8f307a102ca6681869f3474d99c8bd6c2674156eda9ad554a27a5741", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "9bbb3ae7025709b275ab9507dcf0e7ce1dce96ea6f8d3b503011b80397be29af", + "sanitized_output_sha256": "9bbb3ae7025709b275ab9507dcf0e7ce1dce96ea6f8d3b503011b80397be29af", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_POST1045_V2_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1\nMODE=read-only\nCOMPONENT=homelab-backup-post1045-v2-rca\nHOST=pve01\nUID=0\nMUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL\nWORKTREE_HEAD=47b9be72cf2487ba66295d7da998016bbeea57be\nWORKTREE_HEAD_EXPECTED=true\nWORKTREE_CLEAN=true\nWORKTREE_STATUS_LINES=[]\nSOURCE_COMMIT_SUMMARY={\"lines\":[\"47b9be7 CR-2026-1005: close V2 backup scheduling and retention gaps\",\" .../backup-closure-remediation-1045.md | 3 +\",\" changes/CR-2026-1005/runtime/SHA256SUMS | 4 +\",\" .../runtime/homelab-app-cloud-quorum-queue.service | 10 +\",\" .../CR-2026-1005/runtime/homelab-backup-audit-v2 | 200 +++++++++\",\" .../runtime/homelab-backup-retention-gc | 464 +++++++++++++++++++++\",\" .../runtime/homelab-backup-v2-app-scheduled | 27 ++\",\" 6 files changed, 708 insertions(+)\"],\"rc\":0}\nSOURCE_RUNTIME_TREE=[{\"bytes\":503,\"name\":\"SHA256SUMS\",\"sha256\":\"921dce0c32c2246bf5cf7cacd7e23f3d2677084165d79e05e303682dc14d35ed\"},{\"bytes\":244,\"name\":\"homelab-app-cloud-quorum-queue.service\",\"sha256\":\"afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a\"},{\"bytes\":11451,\"name\":\"homelab-backup-audit-v2\",\"sha256\":\"67fff795217e3e9c3a32c5a634a0401efa2b74c533b934f0a04b52a4cd9b6594\"},{\"bytes\":32975,\"name\":\"homelab-backup-retention-gc\",\"sha256\":\"a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909\"},{\"bytes\":1372,\"name\":\"homelab-backup-v2-app-scheduled\",\"sha256\":\"52951cd5c00a60b1a2f7d8c44d7e25feae39706820ad56964ef013e6451ca1ba\"}]\nPOST1045_LIVE_RUNTIME={\"app_scheduler\":{\"exists\":false,\"path\":\"/usr/local/sbin/homelab-backup-v2-app-scheduled\",\"sha256\":null},\"app_service\":{\"baseline_match\":true,\"exists\":true,\"path\":\"/etc/systemd/system/homelab-app-cloud-quorum-queue.service\",\"sha256\":\"70b78ba529c270ec61d26a123d656385a8b2e2e4206ccf623111fa6ed3854602\"},\"audit\":{\"baseline_match\":true,\"exists\":true,\"path\":\"/usr/local/sbin/homelab-backup-audit\",\"sha256\":\"1474c9ce58bbc1b3ae7437261d1bdb923232926fb2943c7c830253f8b284724c\"},\"retention\":{\"baseline_match\":true,\"exists\":true,\"path\":\"/usr/local/sbin/homelab-backup-retention-gc\",\"sha256\":\"8adb6c10f5911f39c53775b89eac3cd1823f1ed34b4caccade5279c213e998a3\"}}\nPOST1045_UNIT_STATE={\"homelab-app-cloud-quorum-queue.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"0\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"0\"},\"ExecStart\":{\"rc\":0,\"value\":\"{ path=/usr/local/sbin/homelab-app-cloud-quorum-queue ; argv[]=/usr/local/sbin/homelab-app-cloud-quorum-queue all ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-app-cloud-quorum-queue.service\"},\"InvocationID\":{\"rc\":0,\"value\":\"\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-app-cloud-quorum-queue.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"ExecStart\":{\"rc\":0,\"value\":\"\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-app-cloud-quorum-queue.timer\"},\"InvocationID\":{\"rc\":0,\"value\":\"\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-retention-gc.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"1\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"66\"},\"ExecStart\":{\"rc\":0,\"value\":\"{ path=/usr/local/sbin/homelab-backup-retention-gc ; argv[]=/usr/local/sbin/homelab-backup-retention-gc scheduled ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-retention-gc.service\"},\"InvocationID\":{\"rc\":0,\"value\":\"426b1424d1f047e8adbb452ece2b3b80\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-backup-retention-gc.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"ExecStart\":{\"rc\":0,\"value\":\"\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-retention-gc.timer\"},\"InvocationID\":{\"rc\":0,\"value\":\"\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-v2.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"failed\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"1\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"2\"},\"ExecStart\":{\"rc\":0,\"value\":\"{ path=/usr/local/sbin/homelab-backup-v2 ; argv[]=/usr/local/sbin/homelab-backup-v2 scheduled ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-v2.service\"},\"InvocationID\":{\"rc\":0,\"value\":\"43c58a804cea48c2818a509fd2e75f48\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"exit-code\"},\"SubState\":{\"rc\":0,\"value\":\"failed\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-backup-v2.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"active\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"ExecStart\":{\"rc\":0,\"value\":\"\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-v2.timer\"},\"InvocationID\":{\"rc\":0,\"value\":\"af3b714a30a541b0b91f4e1d2e2b5980\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"waiting\"},\"UnitFileState\":{\"rc\":0,\"value\":\"enabled\"}}}\nAPP_FRESHNESS_PLAN_POST1045={\"lines\":[\"QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\"],\"rc\":0,\"stderr\":[]}\nVM9130_STATE={\"age_seconds\":108844,\"consecutive_failures\":0,\"exists\":true,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm9130-20260919T210758Z-0aa987efac8c\",\"last_attempt_epoch\":1789853557,\"last_success_epoch\":1789853557,\"logical_id\":\"vm9130\",\"replica_count\":2,\"result\":\"SUCCESS\",\"sha256\":\"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db\"}\nV2_SERVICE_JOURNAL_1045_WINDOW={\"lines\":[\"-- No entries --\"],\"rc\":0,\"stderr\":[]}\nV2_SERVICE_ROOT_CAUSE_LINES=[]\nV2_WRAPPER_SOURCE={\"path\":\"/usr/local/sbin/homelab-backup-v2\",\"relevant_lines\":[{\"line\":4,\"text\":\"ACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b')\"}],\"sha256\":\"5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\"}\nVM_QUEUE_SOURCE={\"path\":\"/usr/local/sbin/homelab-vm-cloud-quorum-queue\",\"relevant_lines\":[{\"line\":5,\"text\":\"WORKER=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\"},{\"line\":18,\"text\":\" case \\\"$1\\\" in\"},{\"line\":30,\"text\":\" case \\\"$1\\\" in\"},{\"line\":36,\"text\":\"run_worker() {\"},{\"line\":43,\"text\":\" _run_worker_unlocked \\\"$@\\\"\"},{\"line\":46,\"text\":\"_run_worker_unlocked() {\"},{\"line\":52,\"text\":\" if [[ \\\"$action\\\" == \\\"verify\\\" ]]; then \\\"$WORKER\\\" verify \\\"$vmid\\\" \\\"$generation\\\"; else \\\"$WORKER\\\" \\\"$action\\\" \\\"$vmid\\\"; fi\"},{\"line\":56,\"text\":\" ssh \\\"${SSH_BASE[@]}\\\" \\\"$host\\\" \\\"$WORKER\\\" verify \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":58,\"text\":\" ssh \\\"${SSH_BASE[@]}\\\" \\\"$host\\\" \\\"$WORKER\\\" \\\"$action\\\" \\\"$vmid\\\"\"},{\"line\":85,\"text\":\" case \\\"$decision\\\" in\"},{\"line\":88,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_RESUME GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":94,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_EPOCH_INVALID\\\\n' \\\"$vmid\\\"\"},{\"line\":99,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":101,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":103,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":105,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\"\"},{\"line\":109,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED\\\\n' \\\"$vmid\\\"\"},{\"line\":123,\"text\":\" decision=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\\\"\\\",$i);print $i}}' <<<\\\"$out\\\" | tail -1)\\\"\"},{\"line\":125,\"text\":\" printf 'QUEUE_FRESHNESS_PLAN VMID=%s DECISION=%s\\\\n' \\\"$vmid\\\" \\\"$decision\\\"\"},{\"line\":128,\"text\":\"backup_list_force(){\"},{\"line\":136,\"text\":\" out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":148,\"text\":\" [[ \\\"${HOMELAB_BACKUP_DISPATCHER:-}\\\" == 1 ]] || die \\\"DISPATCH_ONE_DIRECT_FORBIDDEN\\\" 64\"},{\"line\":163,\"text\":\" decision=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":164,\"text\":\" case \\\"$decision\\\" in\"},{\"line\":166,\"text\":\" generation=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":174,\"text\":\" out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":183,\"text\":\"case \\\"$ACTION\\\" in\"},{\"line\":188,\"text\":\" freshness-plan)\"},{\"line\":196,\"text\":\" dispatch-one)\"},{\"line\":202,\"text\":\" backup_list_force 110\"},{\"line\":206,\"text\":\" backup_list_force \\\"${MIGRATE_VMIDS[@]}\\\"\"},{\"line\":208,\"text\":\" all)\"},{\"line\":210,\"text\":\" [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die \\\"DISPATCHER_MISSING\\\" 69\"},{\"line\":211,\"text\":\" /usr/local/sbin/homelab-backup-dispatcher run VM_TIMER\"},{\"line\":215,\"text\":\" run_worker \\\"$2\\\" verify \\\"$3\\\"\"}],\"sha256\":\"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\"}\nDISPATCHER_SOURCE={\"path\":\"/usr/local/sbin/homelab-backup-dispatcher\",\"relevant_lines\":[{\"line\":4,\"text\":\"VMQ=/usr/local/sbin/homelab-vm-cloud-quorum-queue\"},{\"line\":5,\"text\":\"APPQ=/usr/local/sbin/homelab-app-cloud-quorum-queue\"},{\"line\":9,\"text\":\"rank(){ case \\\"$1\\\" in BACKUP_RESUME) printf 0;; BACKUP_UNVERIFIED) printf 1;; BACKUP_STALE) printf 2;; BACKUP_DUE) printf 3;; NOOP) printf 9;; *) return 1;; esac; }\"},{\"line\":11,\"text\":\" local kind=\\\"$1\\\" q=\\\"$2\\\" plan line decision target age pri best_target=\\\"\\\" best_decision=NOOP best_age=-1 best_pri=99\"},{\"line\":15,\"text\":\" if [[ \\\"$kind\\\" == VM ]]; then [[ \\\"$line\\\" == QUEUE_FRESHNESS\\\\ VMID=* ]] || continue; else [[ \\\"$line\\\" == QUEUE_FRESHNESS\\\\ APP=* ]] || continue; fi\"},{\"line\":16,\"text\":\" decision=\\\"\\\"; target=\\\"\\\"; age=-1\"},{\"line\":17,\"text\":\" for token in $line; do case \\\"$token\\\" in VMID=*) target=\\\"${token#VMID=}\\\";; APP=*) target=\\\"${token#APP=}\\\";; DECISION=*) decision=\\\"${token#DECISION=}\\\";; AGE_SECONDS=*) age=\\\"${token#AGE_SECONDS=}\\\";; esac; done\"},{\"line\":18,\"text\":\" [[ -n \\\"$target\\\" && -n \\\"$decision\\\" ]] || die \\\"SCAN_RECORD_INVALID_$kind\\\" 65\"},{\"line\":19,\"text\":\" case \\\"$decision\\\" in SKIP_FRESH) continue;; BACKUP_RESUME|BACKUP_UNVERIFIED) pri=\\\"$(rank \\\"$decision\\\")\\\";; BACKUP_STALE|BACKUP_DUE) pri=\\\"$(rank \\\"$decision\\\")\\\"; [[ \\\"$age\\\" =~ ^[0-9]+$ ]] || die \\\"AGE_INVALID_${kind}_${target}_${decision}\\\" 65;; *) die \\\"SCAN_DECISION_INVALID_${kind}_${target}_${decision}\\\" 65;; esac\"},{\"line\":20,\"text\":\" if [[ -z \\\"$best_target\\\" || \\\"$pri\\\" -lt \\\"$best_pri\\\" || ( \\\"$pri\\\" -eq \\\"$best_pri\\\" && ( \\\"$pri\\\" -eq 2 || \\\"$pri\\\" -eq 3 ) && \\\"$age\\\" -gt \\\"$best_age\\\" ) ]]; then best_target=\\\"$target\\\"; best_decision=\\\"$decision\\\"; best_age=\\\"$age\\\"; best_pri=\\\"$pri\\\"; fi\"},{\"line\":21,\"text\":\" done <<<\\\"$plan\\\"\"},{\"line\":22,\"text\":\" if [[ -z \\\"$best_target\\\" ]]; then printf 'CLASS_DISPATCH KIND=%s DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1\\\\n' \\\"$kind\\\"; else printf 'CLASS_DISPATCH KIND=%s TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$kind\\\" \\\"$best_target\\\" \\\"$best_decision\\\" \\\"$best_pri\\\" \\\"$best_age\\\"; fi\"},{\"line\":28,\"text\":\" for token in $vline; do case \\\"$token\\\" in TARGET=*) vt=\\\"${token#TARGET=}\\\";; DECISION=*) vd=\\\"${token#DECISION=}\\\";; PRIORITY=*) vp=\\\"${token#PRIORITY=}\\\";; AGE_SECONDS=*) va=\\\"${token#AGE_SECONDS=}\\\";; esac; done\"},{\"line\":29,\"text\":\" for token in $aline; do case \\\"$token\\\" in TARGET=*) at=\\\"${token#TARGET=}\\\";; DECISION=*) ad=\\\"${token#DECISION=}\\\";; PRIORITY=*) ap=\\\"${token#PRIORITY=}\\\";; AGE_SECONDS=*) aa=\\\"${token#AGE_SECONDS=}\\\";; esac; done\"},{\"line\":31,\"text\":\" if [[ \\\"$vd\\\" == NOOP ]]; then printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$at\\\" \\\"$ad\\\" \\\"$ap\\\" \\\"$aa\\\"; return 0; fi\"},{\"line\":32,\"text\":\" if [[ \\\"$ad\\\" == NOOP ]]; then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$vt\\\" \\\"$vd\\\" \\\"$vp\\\" \\\"$va\\\"; return 0; fi\"},{\"line\":33,\"text\":\" if (( vp < ap )); then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$vt\\\" \\\"$vd\\\" \\\"$vp\\\" \\\"$va\\\"; elif (( ap < vp )); then printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$at\\\" \\\"$ad\\\" \\\"$ap\\\" \\\"$aa\\\"; elif (( vp == ap && ( vp == 2 || vp == 3 ) && va > aa )); then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$vt\\\" \\\"$vd\\\" \\\"$vp\\\" \\\"$va\\\"; else printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$at\\\" \\\"$ad\\\" \\\"$ap\\\" \\\"$aa\\\"; fi\"},{\"line\":37,\"text\":\" local trigger=\\\"$1\\\" i=0 actions=0 scan pick decision kind target out rc last_sealed=\\\"\\\"\"},{\"line\":42,\"text\":\" decision=\\\"\\\"; kind=\\\"\\\"; target=\\\"\\\"; for token in $pick; do case \\\"$token\\\" in KIND=*) kind=\\\"${token#KIND=}\\\";; TARGET=*) target=\\\"${token#TARGET=}\\\";; DECISION=*) decision=\\\"${token#DECISION=}\\\";; esac; done\"},{\"line\":43,\"text\":\" if [[ \\\"$decision\\\" == NOOP ]]; then printf 'DISPATCHER_DONE TRIGGER=%s ACTIONS=%s ITERATIONS=%s\\\\n' \\\"$trigger\\\" \\\"$actions\\\" \\\"$i\\\"; return 0; fi\"},{\"line\":44,\"text\":\" [[ \\\"$kind\\\" == VM || \\\"$kind\\\" == APP ]] || die GLOBAL_KIND_INVALID 65; [[ -n \\\"$target\\\" ]] || die GLOBAL_TARGET_MISSING 65\"},{\"line\":45,\"text\":\" if [[ \\\"$last_sealed\\\" == \\\"$kind:$target\\\" ]]; then die \\\"NO_PROGRESS_AFTER_SEALED_${kind}_${target}\\\" 65; fi\"},{\"line\":46,\"text\":\" if [[ \\\"$kind\\\" == VM ]]; then if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$VMQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; else if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$APPQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; fi\"},{\"line\":48,\"text\":\" if grep -q '^QUEUE_DEFERRED ' <<<\\\"$out\\\"; then printf 'DISPATCHER_DEFERRED TRIGGER=%s KIND=%s TARGET=%s REASON=QUEUE_OR_HEAVY_BUSY\\\\n' \\\"$trigger\\\" \\\"$kind\\\" \\\"$target\\\"; return 0; fi\"},{\"line\":49,\"text\":\" if grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SEALED ' <<<\\\"$out\\\"; then last_sealed=\\\"$kind:$target\\\"; actions=$((actions+1)); elif grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SKIP_FRESH ' <<<\\\"$out\\\"; then last_sealed=\\\"\\\"; actions=$((actions+1)); else die \\\"DISPATCH_RESULT_UNCLASSIFIED_${kind}_${target}\\\" 65; fi\"},{\"line\":50,\"text\":\" done\"},{\"line\":53,\"text\":\"case \\\"${1:-}\\\" in plan) [[ \\\"$#\\\" == 1 ]] || die USAGE_PLAN 64; plan;; run) [[ \\\"$#\\\" == 2 ]] || die USAGE_RUN 64; run \\\"$2\\\";; *) die ACTION_INVALID 64;; esac\"}],\"sha256\":\"9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c\"}\nCORE_RUNTIME_ROLLBACK_VERIFIED=true\nAPP_CATCHUP_PRESERVED=true\nSOURCE_COMMIT_PRESERVED=true\nRETENTION_GC_RAN=false\nPOST1045_RCA_DECISION={\"app_catchup_preserved\":true,\"core_runtime_rollback_verified\":true,\"journal_has_root_cause_lines\":false,\"source_commit_preserved\":true,\"vm9130_needs_targeted_continuation\":true}\nNO_MUTATION=true\nNO_REPO_WRITE=true\nNO_GIT_FETCH=true\nNO_GIT_COMMIT=true\nNO_GIT_PUSH_UPDATE=true\nNO_SYSTEMD_ACTION=true\nNO_BACKUP_STARTED=true\nNO_RESTORE_STARTED=true\nNO_RETENTION_STARTED=true\nNO_DELETE=true\nNO_MAIL_SENT=true\nDURATION_SEC=1.467\nTASK_RESULT=PASS_HOMELAB_BACKUP_POST1045_V2_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\nWORKERS2_BACKUP_POST1045_V2_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1.txt b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1.txt new file mode 100644 index 00000000..dbd824cf --- /dev/null +++ b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1.txt @@ -0,0 +1,74 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-post1045-v2-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=e885553c8f307a102ca6681869f3474d99c8bd6c2674156eda9ad554a27a5741 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=9bbb3ae7025709b275ab9507dcf0e7ce1dce96ea6f8d3b503011b80397be29af +SANITIZED_OUTPUT_SHA256=9bbb3ae7025709b275ab9507dcf0e7ce1dce96ea6f8d3b503011b80397be29af +OUTPUT_BEGIN +WORKERS2_BACKUP_POST1045_V2_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1 +MODE=read-only +COMPONENT=homelab-backup-post1045-v2-rca +HOST=pve01 +UID=0 +MUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL +WORKTREE_HEAD=47b9be72cf2487ba66295d7da998016bbeea57be +WORKTREE_HEAD_EXPECTED=true +WORKTREE_CLEAN=true +WORKTREE_STATUS_LINES=[] +SOURCE_COMMIT_SUMMARY={"lines":["47b9be7 CR-2026-1005: close V2 backup scheduling and retention gaps"," .../backup-closure-remediation-1045.md | 3 +"," changes/CR-2026-1005/runtime/SHA256SUMS | 4 +"," .../runtime/homelab-app-cloud-quorum-queue.service | 10 +"," .../CR-2026-1005/runtime/homelab-backup-audit-v2 | 200 +++++++++"," .../runtime/homelab-backup-retention-gc | 464 +++++++++++++++++++++"," .../runtime/homelab-backup-v2-app-scheduled | 27 ++"," 6 files changed, 708 insertions(+)"],"rc":0} +SOURCE_RUNTIME_TREE=[{"bytes":503,"name":"SHA256SUMS","sha256":"921dce0c32c2246bf5cf7cacd7e23f3d2677084165d79e05e303682dc14d35ed"},{"bytes":244,"name":"homelab-app-cloud-quorum-queue.service","sha256":"afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a"},{"bytes":11451,"name":"homelab-backup-audit-v2","sha256":"67fff795217e3e9c3a32c5a634a0401efa2b74c533b934f0a04b52a4cd9b6594"},{"bytes":32975,"name":"homelab-backup-retention-gc","sha256":"a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909"},{"bytes":1372,"name":"homelab-backup-v2-app-scheduled","sha256":"52951cd5c00a60b1a2f7d8c44d7e25feae39706820ad56964ef013e6451ca1ba"}] +POST1045_LIVE_RUNTIME={"app_scheduler":{"exists":false,"path":"/usr/local/sbin/homelab-backup-v2-app-scheduled","sha256":null},"app_service":{"baseline_match":true,"exists":true,"path":"/etc/systemd/system/homelab-app-cloud-quorum-queue.service","sha256":"70b78ba529c270ec61d26a123d656385a8b2e2e4206ccf623111fa6ed3854602"},"audit":{"baseline_match":true,"exists":true,"path":"/usr/local/sbin/homelab-backup-audit","sha256":"1474c9ce58bbc1b3ae7437261d1bdb923232926fb2943c7c830253f8b284724c"},"retention":{"baseline_match":true,"exists":true,"path":"/usr/local/sbin/homelab-backup-retention-gc","sha256":"8adb6c10f5911f39c53775b89eac3cd1823f1ed34b4caccade5279c213e998a3"}} +POST1045_UNIT_STATE={"homelab-app-cloud-quorum-queue.service":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainCode":{"rc":0,"value":"0"},"ExecMainStatus":{"rc":0,"value":"0"},"ExecStart":{"rc":0,"value":"{ path=/usr/local/sbin/homelab-app-cloud-quorum-queue ; argv[]=/usr/local/sbin/homelab-app-cloud-quorum-queue all ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }"},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-app-cloud-quorum-queue.service"},"InvocationID":{"rc":0,"value":""},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"static"}},"homelab-app-cloud-quorum-queue.timer":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainCode":{"rc":0,"value":""},"ExecMainStatus":{"rc":0,"value":""},"ExecStart":{"rc":0,"value":""},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-app-cloud-quorum-queue.timer"},"InvocationID":{"rc":0,"value":""},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"disabled"}},"homelab-backup-retention-gc.service":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainCode":{"rc":0,"value":"1"},"ExecMainStatus":{"rc":0,"value":"66"},"ExecStart":{"rc":0,"value":"{ path=/usr/local/sbin/homelab-backup-retention-gc ; argv[]=/usr/local/sbin/homelab-backup-retention-gc scheduled ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }"},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-backup-retention-gc.service"},"InvocationID":{"rc":0,"value":"426b1424d1f047e8adbb452ece2b3b80"},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"static"}},"homelab-backup-retention-gc.timer":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainCode":{"rc":0,"value":""},"ExecMainStatus":{"rc":0,"value":""},"ExecStart":{"rc":0,"value":""},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-backup-retention-gc.timer"},"InvocationID":{"rc":0,"value":""},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"disabled"}},"homelab-backup-v2.service":{"ActiveState":{"rc":0,"value":"failed"},"ExecMainCode":{"rc":0,"value":"1"},"ExecMainStatus":{"rc":0,"value":"2"},"ExecStart":{"rc":0,"value":"{ path=/usr/local/sbin/homelab-backup-v2 ; argv[]=/usr/local/sbin/homelab-backup-v2 scheduled ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }"},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-backup-v2.service"},"InvocationID":{"rc":0,"value":"43c58a804cea48c2818a509fd2e75f48"},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"exit-code"},"SubState":{"rc":0,"value":"failed"},"UnitFileState":{"rc":0,"value":"static"}},"homelab-backup-v2.timer":{"ActiveState":{"rc":0,"value":"active"},"ExecMainCode":{"rc":0,"value":""},"ExecMainStatus":{"rc":0,"value":""},"ExecStart":{"rc":0,"value":""},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-backup-v2.timer"},"InvocationID":{"rc":0,"value":"af3b714a30a541b0b91f4e1d2e2b5980"},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"waiting"},"UnitFileState":{"rc":0,"value":"enabled"}}} +APP_FRESHNESS_PLAN_POST1045={"lines":["QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH"],"rc":0,"stderr":[]} +VM9130_STATE={"age_seconds":108844,"consecutive_failures":0,"exists":true,"failure_backoff_seconds":0,"generation":"v2-vm9130-20260919T210758Z-0aa987efac8c","last_attempt_epoch":1789853557,"last_success_epoch":1789853557,"logical_id":"vm9130","replica_count":2,"result":"SUCCESS","sha256":"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db"} +V2_SERVICE_JOURNAL_1045_WINDOW={"lines":["-- No entries --"],"rc":0,"stderr":[]} +V2_SERVICE_ROOT_CAUSE_LINES=[] +V2_WRAPPER_SOURCE={"path":"/usr/local/sbin/homelab-backup-v2","relevant_lines":[{"line":4,"text":"ACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b')"}],"sha256":"5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2"} +VM_QUEUE_SOURCE={"path":"/usr/local/sbin/homelab-vm-cloud-quorum-queue","relevant_lines":[{"line":5,"text":"WORKER=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker"},{"line":18,"text":" case \"$1\" in"},{"line":30,"text":" case \"$1\" in"},{"line":36,"text":"run_worker() {"},{"line":43,"text":" _run_worker_unlocked \"$@\""},{"line":46,"text":"_run_worker_unlocked() {"},{"line":52,"text":" if [[ \"$action\" == \"verify\" ]]; then \"$WORKER\" verify \"$vmid\" \"$generation\"; else \"$WORKER\" \"$action\" \"$vmid\"; fi"},{"line":56,"text":" ssh \"${SSH_BASE[@]}\" \"$host\" \"$WORKER\" verify \"$vmid\" \"$generation\""},{"line":58,"text":" ssh \"${SSH_BASE[@]}\" \"$host\" \"$WORKER\" \"$action\" \"$vmid\""},{"line":85,"text":" case \"$decision\" in"},{"line":88,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_RESUME GENERATION=%s\\n' \"$vmid\" \"$generation\""},{"line":94,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_EPOCH_INVALID\\n' \"$vmid\""},{"line":99,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$vmid\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\""},{"line":101,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$vmid\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\""},{"line":103,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$vmid\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\""},{"line":105,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\\n' \"$vmid\" \"$generation\" \"$age\""},{"line":109,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED\\n' \"$vmid\""},{"line":123,"text":" decision=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$out\" | tail -1)\""},{"line":125,"text":" printf 'QUEUE_FRESHNESS_PLAN VMID=%s DECISION=%s\\n' \"$vmid\" \"$decision\""},{"line":128,"text":"backup_list_force(){"},{"line":136,"text":" out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\""},{"line":148,"text":" [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64"},{"line":163,"text":" decision=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\""},{"line":164,"text":" case \"$decision\" in"},{"line":166,"text":" generation=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\""},{"line":174,"text":" out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\""},{"line":183,"text":"case \"$ACTION\" in"},{"line":188,"text":" freshness-plan)"},{"line":196,"text":" dispatch-one)"},{"line":202,"text":" backup_list_force 110"},{"line":206,"text":" backup_list_force \"${MIGRATE_VMIDS[@]}\""},{"line":208,"text":" all)"},{"line":210,"text":" [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die \"DISPATCHER_MISSING\" 69"},{"line":211,"text":" /usr/local/sbin/homelab-backup-dispatcher run VM_TIMER"},{"line":215,"text":" run_worker \"$2\" verify \"$3\""}],"sha256":"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612"} +DISPATCHER_SOURCE={"path":"/usr/local/sbin/homelab-backup-dispatcher","relevant_lines":[{"line":4,"text":"VMQ=/usr/local/sbin/homelab-vm-cloud-quorum-queue"},{"line":5,"text":"APPQ=/usr/local/sbin/homelab-app-cloud-quorum-queue"},{"line":9,"text":"rank(){ case \"$1\" in BACKUP_RESUME) printf 0;; BACKUP_UNVERIFIED) printf 1;; BACKUP_STALE) printf 2;; BACKUP_DUE) printf 3;; NOOP) printf 9;; *) return 1;; esac; }"},{"line":11,"text":" local kind=\"$1\" q=\"$2\" plan line decision target age pri best_target=\"\" best_decision=NOOP best_age=-1 best_pri=99"},{"line":15,"text":" if [[ \"$kind\" == VM ]]; then [[ \"$line\" == QUEUE_FRESHNESS\\ VMID=* ]] || continue; else [[ \"$line\" == QUEUE_FRESHNESS\\ APP=* ]] || continue; fi"},{"line":16,"text":" decision=\"\"; target=\"\"; age=-1"},{"line":17,"text":" for token in $line; do case \"$token\" in VMID=*) target=\"${token#VMID=}\";; APP=*) target=\"${token#APP=}\";; DECISION=*) decision=\"${token#DECISION=}\";; AGE_SECONDS=*) age=\"${token#AGE_SECONDS=}\";; esac; done"},{"line":18,"text":" [[ -n \"$target\" && -n \"$decision\" ]] || die \"SCAN_RECORD_INVALID_$kind\" 65"},{"line":19,"text":" case \"$decision\" in SKIP_FRESH) continue;; BACKUP_RESUME|BACKUP_UNVERIFIED) pri=\"$(rank \"$decision\")\";; BACKUP_STALE|BACKUP_DUE) pri=\"$(rank \"$decision\")\"; [[ \"$age\" =~ ^[0-9]+$ ]] || die \"AGE_INVALID_${kind}_${target}_${decision}\" 65;; *) die \"SCAN_DECISION_INVALID_${kind}_${target}_${decision}\" 65;; esac"},{"line":20,"text":" if [[ -z \"$best_target\" || \"$pri\" -lt \"$best_pri\" || ( \"$pri\" -eq \"$best_pri\" && ( \"$pri\" -eq 2 || \"$pri\" -eq 3 ) && \"$age\" -gt \"$best_age\" ) ]]; then best_target=\"$target\"; best_decision=\"$decision\"; best_age=\"$age\"; best_pri=\"$pri\"; fi"},{"line":21,"text":" done <<<\"$plan\""},{"line":22,"text":" if [[ -z \"$best_target\" ]]; then printf 'CLASS_DISPATCH KIND=%s DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1\\n' \"$kind\"; else printf 'CLASS_DISPATCH KIND=%s TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$kind\" \"$best_target\" \"$best_decision\" \"$best_pri\" \"$best_age\"; fi"},{"line":28,"text":" for token in $vline; do case \"$token\" in TARGET=*) vt=\"${token#TARGET=}\";; DECISION=*) vd=\"${token#DECISION=}\";; PRIORITY=*) vp=\"${token#PRIORITY=}\";; AGE_SECONDS=*) va=\"${token#AGE_SECONDS=}\";; esac; done"},{"line":29,"text":" for token in $aline; do case \"$token\" in TARGET=*) at=\"${token#TARGET=}\";; DECISION=*) ad=\"${token#DECISION=}\";; PRIORITY=*) ap=\"${token#PRIORITY=}\";; AGE_SECONDS=*) aa=\"${token#AGE_SECONDS=}\";; esac; done"},{"line":31,"text":" if [[ \"$vd\" == NOOP ]]; then printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$at\" \"$ad\" \"$ap\" \"$aa\"; return 0; fi"},{"line":32,"text":" if [[ \"$ad\" == NOOP ]]; then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$vt\" \"$vd\" \"$vp\" \"$va\"; return 0; fi"},{"line":33,"text":" if (( vp < ap )); then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$vt\" \"$vd\" \"$vp\" \"$va\"; elif (( ap < vp )); then printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$at\" \"$ad\" \"$ap\" \"$aa\"; elif (( vp == ap && ( vp == 2 || vp == 3 ) && va > aa )); then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$vt\" \"$vd\" \"$vp\" \"$va\"; else printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$at\" \"$ad\" \"$ap\" \"$aa\"; fi"},{"line":37,"text":" local trigger=\"$1\" i=0 actions=0 scan pick decision kind target out rc last_sealed=\"\""},{"line":42,"text":" decision=\"\"; kind=\"\"; target=\"\"; for token in $pick; do case \"$token\" in KIND=*) kind=\"${token#KIND=}\";; TARGET=*) target=\"${token#TARGET=}\";; DECISION=*) decision=\"${token#DECISION=}\";; esac; done"},{"line":43,"text":" if [[ \"$decision\" == NOOP ]]; then printf 'DISPATCHER_DONE TRIGGER=%s ACTIONS=%s ITERATIONS=%s\\n' \"$trigger\" \"$actions\" \"$i\"; return 0; fi"},{"line":44,"text":" [[ \"$kind\" == VM || \"$kind\" == APP ]] || die GLOBAL_KIND_INVALID 65; [[ -n \"$target\" ]] || die GLOBAL_TARGET_MISSING 65"},{"line":45,"text":" if [[ \"$last_sealed\" == \"$kind:$target\" ]]; then die \"NO_PROGRESS_AFTER_SEALED_${kind}_${target}\" 65; fi"},{"line":46,"text":" if [[ \"$kind\" == VM ]]; then if out=\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \"$VMQ\" dispatch-one \"$target\" 2>&1)\"; then rc=0; else rc=$?; fi; else if out=\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \"$APPQ\" dispatch-one \"$target\" 2>&1)\"; then rc=0; else rc=$?; fi; fi"},{"line":48,"text":" if grep -q '^QUEUE_DEFERRED ' <<<\"$out\"; then printf 'DISPATCHER_DEFERRED TRIGGER=%s KIND=%s TARGET=%s REASON=QUEUE_OR_HEAVY_BUSY\\n' \"$trigger\" \"$kind\" \"$target\"; return 0; fi"},{"line":49,"text":" if grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SEALED ' <<<\"$out\"; then last_sealed=\"$kind:$target\"; actions=$((actions+1)); elif grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SKIP_FRESH ' <<<\"$out\"; then last_sealed=\"\"; actions=$((actions+1)); else die \"DISPATCH_RESULT_UNCLASSIFIED_${kind}_${target}\" 65; fi"},{"line":50,"text":" done"},{"line":53,"text":"case \"${1:-}\" in plan) [[ \"$#\" == 1 ]] || die USAGE_PLAN 64; plan;; run) [[ \"$#\" == 2 ]] || die USAGE_RUN 64; run \"$2\";; *) die ACTION_INVALID 64;; esac"}],"sha256":"9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c"} +CORE_RUNTIME_ROLLBACK_VERIFIED=true +APP_CATCHUP_PRESERVED=true +SOURCE_COMMIT_PRESERVED=true +RETENTION_GC_RAN=false +POST1045_RCA_DECISION={"app_catchup_preserved":true,"core_runtime_rollback_verified":true,"journal_has_root_cause_lines":false,"source_commit_preserved":true,"vm9130_needs_targeted_continuation":true} +NO_MUTATION=true +NO_REPO_WRITE=true +NO_GIT_FETCH=true +NO_GIT_COMMIT=true +NO_GIT_PUSH_UPDATE=true +NO_SYSTEMD_ACTION=true +NO_BACKUP_STARTED=true +NO_RESTORE_STARTED=true +NO_RETENTION_STARTED=true +NO_DELETE=true +NO_MAIL_SENT=true +DURATION_SEC=1.467 +TASK_RESULT=PASS_HOMELAB_BACKUP_POST1045_V2_RCA_READONLY +HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1","rollback_restored":null,"rollback_started":false,"status":"OK","version":1} +WORKERS2_BACKUP_POST1045_V2_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 5dd9cbb4..4bc8b570 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SIGNALBOT-260921-ORDER-NORMALIZATION-DURABLE-STAGE-052FIX2", - "status": "FAIL", - "rc": 76, + "command_id": "SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "signalbot-order-normalization-durable-stage", - "started_at_utc": "2026-09-21T03:46:30Z", - "finished_at_utc": "2026-09-21T03:46:30Z", + "mode": "read-only", + "component": "homelab-backup-post1045-v2-rca", + "started_at_utc": "2026-09-21T03:46:39Z", + "finished_at_utc": "2026-09-21T03:46:41Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "00d0ae6b43d4637e2c1d8387e2cd4211a08d98d93bdc0a3ee12cea754f2d79f2", + "command_sha256": "e885553c8f307a102ca6681869f3474d99c8bd6c2674156eda9ad554a27a5741", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, - "result_contract_valid": false, + "change_declared": false, + "result_contract_valid": true, "result_contract_status": null, - "result_contract_error": "INVALID_CONTRACT:ValueError", + "result_contract_error": null, "command_rc": 0, - "changes_made": null, - "rollback_started": null, + "changes_made": false, + "rollback_started": false, "rollback_restored": null, - "mutation_outcome": "UNKNOWN", + "mutation_outcome": "NO_MUTATION", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "a87a39311ef0a510d26715190ede5d4c8e8f78ad2565d8047a48c25772926fde", - "sanitized_output_sha256": "a87a39311ef0a510d26715190ede5d4c8e8f78ad2565d8047a48c25772926fde", + "raw_evidence_sha256": "9bbb3ae7025709b275ab9507dcf0e7ce1dce96ea6f8d3b503011b80397be29af", + "sanitized_output_sha256": "9bbb3ae7025709b275ab9507dcf0e7ce1dce96ea6f8d3b503011b80397be29af", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "FIX2={\"phase\":\"rollback\",\"error\":\"targeted: AssertionError: assert is \\nE + where = ReconciliationWindow(window_id='1472770bf9bc5ff2323b035e21fec2a6f0f91161c2773593409d4ee46ce776f6', reconnect_event_seq... 13, tzinfo=datetime.timezone.utc), resolved_at=datetime.datetime(2026, 9, 20, 0, 0, 20, tzinfo=datetime.timezone.utc)).status\\nE + and = ReconciliationStatus.RECONCILED\\n\\ntests/test_bitfinex_reconciliation.py:200: AssertionError\\n=========================== short test summary info ============================\\nFAILED tests/test_bitfinex_reconciliation.py::test_http_or_validation_failure_marks_window_unresolved\\n1 failed in 0.17s\\n\",\"restored\":true}\nTASK_RESULT=HOLD_FIX2_ROLLED_BACK\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SIGNALBOT-260921-ORDER-NORMALIZATION-DURABLE-STAGE-052FIX2\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":true,\"rollback_restored\":true}\n" + "output": "WORKERS2_BACKUP_POST1045_V2_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1\nMODE=read-only\nCOMPONENT=homelab-backup-post1045-v2-rca\nHOST=pve01\nUID=0\nMUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL\nWORKTREE_HEAD=47b9be72cf2487ba66295d7da998016bbeea57be\nWORKTREE_HEAD_EXPECTED=true\nWORKTREE_CLEAN=true\nWORKTREE_STATUS_LINES=[]\nSOURCE_COMMIT_SUMMARY={\"lines\":[\"47b9be7 CR-2026-1005: close V2 backup scheduling and retention gaps\",\" .../backup-closure-remediation-1045.md | 3 +\",\" changes/CR-2026-1005/runtime/SHA256SUMS | 4 +\",\" .../runtime/homelab-app-cloud-quorum-queue.service | 10 +\",\" .../CR-2026-1005/runtime/homelab-backup-audit-v2 | 200 +++++++++\",\" .../runtime/homelab-backup-retention-gc | 464 +++++++++++++++++++++\",\" .../runtime/homelab-backup-v2-app-scheduled | 27 ++\",\" 6 files changed, 708 insertions(+)\"],\"rc\":0}\nSOURCE_RUNTIME_TREE=[{\"bytes\":503,\"name\":\"SHA256SUMS\",\"sha256\":\"921dce0c32c2246bf5cf7cacd7e23f3d2677084165d79e05e303682dc14d35ed\"},{\"bytes\":244,\"name\":\"homelab-app-cloud-quorum-queue.service\",\"sha256\":\"afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a\"},{\"bytes\":11451,\"name\":\"homelab-backup-audit-v2\",\"sha256\":\"67fff795217e3e9c3a32c5a634a0401efa2b74c533b934f0a04b52a4cd9b6594\"},{\"bytes\":32975,\"name\":\"homelab-backup-retention-gc\",\"sha256\":\"a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909\"},{\"bytes\":1372,\"name\":\"homelab-backup-v2-app-scheduled\",\"sha256\":\"52951cd5c00a60b1a2f7d8c44d7e25feae39706820ad56964ef013e6451ca1ba\"}]\nPOST1045_LIVE_RUNTIME={\"app_scheduler\":{\"exists\":false,\"path\":\"/usr/local/sbin/homelab-backup-v2-app-scheduled\",\"sha256\":null},\"app_service\":{\"baseline_match\":true,\"exists\":true,\"path\":\"/etc/systemd/system/homelab-app-cloud-quorum-queue.service\",\"sha256\":\"70b78ba529c270ec61d26a123d656385a8b2e2e4206ccf623111fa6ed3854602\"},\"audit\":{\"baseline_match\":true,\"exists\":true,\"path\":\"/usr/local/sbin/homelab-backup-audit\",\"sha256\":\"1474c9ce58bbc1b3ae7437261d1bdb923232926fb2943c7c830253f8b284724c\"},\"retention\":{\"baseline_match\":true,\"exists\":true,\"path\":\"/usr/local/sbin/homelab-backup-retention-gc\",\"sha256\":\"8adb6c10f5911f39c53775b89eac3cd1823f1ed34b4caccade5279c213e998a3\"}}\nPOST1045_UNIT_STATE={\"homelab-app-cloud-quorum-queue.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"0\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"0\"},\"ExecStart\":{\"rc\":0,\"value\":\"{ path=/usr/local/sbin/homelab-app-cloud-quorum-queue ; argv[]=/usr/local/sbin/homelab-app-cloud-quorum-queue all ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-app-cloud-quorum-queue.service\"},\"InvocationID\":{\"rc\":0,\"value\":\"\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-app-cloud-quorum-queue.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"ExecStart\":{\"rc\":0,\"value\":\"\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-app-cloud-quorum-queue.timer\"},\"InvocationID\":{\"rc\":0,\"value\":\"\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-retention-gc.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"1\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"66\"},\"ExecStart\":{\"rc\":0,\"value\":\"{ path=/usr/local/sbin/homelab-backup-retention-gc ; argv[]=/usr/local/sbin/homelab-backup-retention-gc scheduled ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-retention-gc.service\"},\"InvocationID\":{\"rc\":0,\"value\":\"426b1424d1f047e8adbb452ece2b3b80\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-backup-retention-gc.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"inactive\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"ExecStart\":{\"rc\":0,\"value\":\"\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-retention-gc.timer\"},\"InvocationID\":{\"rc\":0,\"value\":\"\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"dead\"},\"UnitFileState\":{\"rc\":0,\"value\":\"disabled\"}},\"homelab-backup-v2.service\":{\"ActiveState\":{\"rc\":0,\"value\":\"failed\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"1\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"2\"},\"ExecStart\":{\"rc\":0,\"value\":\"{ path=/usr/local/sbin/homelab-backup-v2 ; argv[]=/usr/local/sbin/homelab-backup-v2 scheduled ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-v2.service\"},\"InvocationID\":{\"rc\":0,\"value\":\"43c58a804cea48c2818a509fd2e75f48\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"exit-code\"},\"SubState\":{\"rc\":0,\"value\":\"failed\"},\"UnitFileState\":{\"rc\":0,\"value\":\"static\"}},\"homelab-backup-v2.timer\":{\"ActiveState\":{\"rc\":0,\"value\":\"active\"},\"ExecMainCode\":{\"rc\":0,\"value\":\"\"},\"ExecMainStatus\":{\"rc\":0,\"value\":\"\"},\"ExecStart\":{\"rc\":0,\"value\":\"\"},\"FragmentPath\":{\"rc\":0,\"value\":\"/etc/systemd/system/homelab-backup-v2.timer\"},\"InvocationID\":{\"rc\":0,\"value\":\"af3b714a30a541b0b91f4e1d2e2b5980\"},\"LoadState\":{\"rc\":0,\"value\":\"loaded\"},\"Result\":{\"rc\":0,\"value\":\"success\"},\"SubState\":{\"rc\":0,\"value\":\"waiting\"},\"UnitFileState\":{\"rc\":0,\"value\":\"enabled\"}}}\nAPP_FRESHNESS_PLAN_POST1045={\"lines\":[\"QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH\",\"QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH\"],\"rc\":0,\"stderr\":[]}\nVM9130_STATE={\"age_seconds\":108844,\"consecutive_failures\":0,\"exists\":true,\"failure_backoff_seconds\":0,\"generation\":\"v2-vm9130-20260919T210758Z-0aa987efac8c\",\"last_attempt_epoch\":1789853557,\"last_success_epoch\":1789853557,\"logical_id\":\"vm9130\",\"replica_count\":2,\"result\":\"SUCCESS\",\"sha256\":\"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db\"}\nV2_SERVICE_JOURNAL_1045_WINDOW={\"lines\":[\"-- No entries --\"],\"rc\":0,\"stderr\":[]}\nV2_SERVICE_ROOT_CAUSE_LINES=[]\nV2_WRAPPER_SOURCE={\"path\":\"/usr/local/sbin/homelab-backup-v2\",\"relevant_lines\":[{\"line\":4,\"text\":\"ACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b')\"}],\"sha256\":\"5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\"}\nVM_QUEUE_SOURCE={\"path\":\"/usr/local/sbin/homelab-vm-cloud-quorum-queue\",\"relevant_lines\":[{\"line\":5,\"text\":\"WORKER=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\"},{\"line\":18,\"text\":\" case \\\"$1\\\" in\"},{\"line\":30,\"text\":\" case \\\"$1\\\" in\"},{\"line\":36,\"text\":\"run_worker() {\"},{\"line\":43,\"text\":\" _run_worker_unlocked \\\"$@\\\"\"},{\"line\":46,\"text\":\"_run_worker_unlocked() {\"},{\"line\":52,\"text\":\" if [[ \\\"$action\\\" == \\\"verify\\\" ]]; then \\\"$WORKER\\\" verify \\\"$vmid\\\" \\\"$generation\\\"; else \\\"$WORKER\\\" \\\"$action\\\" \\\"$vmid\\\"; fi\"},{\"line\":56,\"text\":\" ssh \\\"${SSH_BASE[@]}\\\" \\\"$host\\\" \\\"$WORKER\\\" verify \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":58,\"text\":\" ssh \\\"${SSH_BASE[@]}\\\" \\\"$host\\\" \\\"$WORKER\\\" \\\"$action\\\" \\\"$vmid\\\"\"},{\"line\":85,\"text\":\" case \\\"$decision\\\" in\"},{\"line\":88,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_RESUME GENERATION=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\"\"},{\"line\":94,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_EPOCH_INVALID\\\\n' \\\"$vmid\\\"\"},{\"line\":99,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":101,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":103,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\" \\\"$DUE_SECONDS\\\" \\\"$FRESHNESS_SECONDS\\\"\"},{\"line\":105,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\\\\n' \\\"$vmid\\\" \\\"$generation\\\" \\\"$age\\\"\"},{\"line\":109,\"text\":\" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED\\\\n' \\\"$vmid\\\"\"},{\"line\":123,\"text\":\" decision=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\\\"\\\",$i);print $i}}' <<<\\\"$out\\\" | tail -1)\\\"\"},{\"line\":125,\"text\":\" printf 'QUEUE_FRESHNESS_PLAN VMID=%s DECISION=%s\\\\n' \\\"$vmid\\\" \\\"$decision\\\"\"},{\"line\":128,\"text\":\"backup_list_force(){\"},{\"line\":136,\"text\":\" out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":148,\"text\":\" [[ \\\"${HOMELAB_BACKUP_DISPATCHER:-}\\\" == 1 ]] || die \\\"DISPATCH_ONE_DIRECT_FORBIDDEN\\\" 64\"},{\"line\":163,\"text\":\" decision=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":164,\"text\":\" case \\\"$decision\\\" in\"},{\"line\":166,\"text\":\" generation=\\\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\\\"\\\",$i);print $i}}' <<<\\\"$freshness\\\" | tail -1)\\\"\"},{\"line\":174,\"text\":\" out=\\\"$(exec 8>&-; exec 9>&-; run_worker \\\"$vmid\\\" backup)\\\"\"},{\"line\":183,\"text\":\"case \\\"$ACTION\\\" in\"},{\"line\":188,\"text\":\" freshness-plan)\"},{\"line\":196,\"text\":\" dispatch-one)\"},{\"line\":202,\"text\":\" backup_list_force 110\"},{\"line\":206,\"text\":\" backup_list_force \\\"${MIGRATE_VMIDS[@]}\\\"\"},{\"line\":208,\"text\":\" all)\"},{\"line\":210,\"text\":\" [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die \\\"DISPATCHER_MISSING\\\" 69\"},{\"line\":211,\"text\":\" /usr/local/sbin/homelab-backup-dispatcher run VM_TIMER\"},{\"line\":215,\"text\":\" run_worker \\\"$2\\\" verify \\\"$3\\\"\"}],\"sha256\":\"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\"}\nDISPATCHER_SOURCE={\"path\":\"/usr/local/sbin/homelab-backup-dispatcher\",\"relevant_lines\":[{\"line\":4,\"text\":\"VMQ=/usr/local/sbin/homelab-vm-cloud-quorum-queue\"},{\"line\":5,\"text\":\"APPQ=/usr/local/sbin/homelab-app-cloud-quorum-queue\"},{\"line\":9,\"text\":\"rank(){ case \\\"$1\\\" in BACKUP_RESUME) printf 0;; BACKUP_UNVERIFIED) printf 1;; BACKUP_STALE) printf 2;; BACKUP_DUE) printf 3;; NOOP) printf 9;; *) return 1;; esac; }\"},{\"line\":11,\"text\":\" local kind=\\\"$1\\\" q=\\\"$2\\\" plan line decision target age pri best_target=\\\"\\\" best_decision=NOOP best_age=-1 best_pri=99\"},{\"line\":15,\"text\":\" if [[ \\\"$kind\\\" == VM ]]; then [[ \\\"$line\\\" == QUEUE_FRESHNESS\\\\ VMID=* ]] || continue; else [[ \\\"$line\\\" == QUEUE_FRESHNESS\\\\ APP=* ]] || continue; fi\"},{\"line\":16,\"text\":\" decision=\\\"\\\"; target=\\\"\\\"; age=-1\"},{\"line\":17,\"text\":\" for token in $line; do case \\\"$token\\\" in VMID=*) target=\\\"${token#VMID=}\\\";; APP=*) target=\\\"${token#APP=}\\\";; DECISION=*) decision=\\\"${token#DECISION=}\\\";; AGE_SECONDS=*) age=\\\"${token#AGE_SECONDS=}\\\";; esac; done\"},{\"line\":18,\"text\":\" [[ -n \\\"$target\\\" && -n \\\"$decision\\\" ]] || die \\\"SCAN_RECORD_INVALID_$kind\\\" 65\"},{\"line\":19,\"text\":\" case \\\"$decision\\\" in SKIP_FRESH) continue;; BACKUP_RESUME|BACKUP_UNVERIFIED) pri=\\\"$(rank \\\"$decision\\\")\\\";; BACKUP_STALE|BACKUP_DUE) pri=\\\"$(rank \\\"$decision\\\")\\\"; [[ \\\"$age\\\" =~ ^[0-9]+$ ]] || die \\\"AGE_INVALID_${kind}_${target}_${decision}\\\" 65;; *) die \\\"SCAN_DECISION_INVALID_${kind}_${target}_${decision}\\\" 65;; esac\"},{\"line\":20,\"text\":\" if [[ -z \\\"$best_target\\\" || \\\"$pri\\\" -lt \\\"$best_pri\\\" || ( \\\"$pri\\\" -eq \\\"$best_pri\\\" && ( \\\"$pri\\\" -eq 2 || \\\"$pri\\\" -eq 3 ) && \\\"$age\\\" -gt \\\"$best_age\\\" ) ]]; then best_target=\\\"$target\\\"; best_decision=\\\"$decision\\\"; best_age=\\\"$age\\\"; best_pri=\\\"$pri\\\"; fi\"},{\"line\":21,\"text\":\" done <<<\\\"$plan\\\"\"},{\"line\":22,\"text\":\" if [[ -z \\\"$best_target\\\" ]]; then printf 'CLASS_DISPATCH KIND=%s DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1\\\\n' \\\"$kind\\\"; else printf 'CLASS_DISPATCH KIND=%s TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$kind\\\" \\\"$best_target\\\" \\\"$best_decision\\\" \\\"$best_pri\\\" \\\"$best_age\\\"; fi\"},{\"line\":28,\"text\":\" for token in $vline; do case \\\"$token\\\" in TARGET=*) vt=\\\"${token#TARGET=}\\\";; DECISION=*) vd=\\\"${token#DECISION=}\\\";; PRIORITY=*) vp=\\\"${token#PRIORITY=}\\\";; AGE_SECONDS=*) va=\\\"${token#AGE_SECONDS=}\\\";; esac; done\"},{\"line\":29,\"text\":\" for token in $aline; do case \\\"$token\\\" in TARGET=*) at=\\\"${token#TARGET=}\\\";; DECISION=*) ad=\\\"${token#DECISION=}\\\";; PRIORITY=*) ap=\\\"${token#PRIORITY=}\\\";; AGE_SECONDS=*) aa=\\\"${token#AGE_SECONDS=}\\\";; esac; done\"},{\"line\":31,\"text\":\" if [[ \\\"$vd\\\" == NOOP ]]; then printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$at\\\" \\\"$ad\\\" \\\"$ap\\\" \\\"$aa\\\"; return 0; fi\"},{\"line\":32,\"text\":\" if [[ \\\"$ad\\\" == NOOP ]]; then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$vt\\\" \\\"$vd\\\" \\\"$vp\\\" \\\"$va\\\"; return 0; fi\"},{\"line\":33,\"text\":\" if (( vp < ap )); then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$vt\\\" \\\"$vd\\\" \\\"$vp\\\" \\\"$va\\\"; elif (( ap < vp )); then printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$at\\\" \\\"$ad\\\" \\\"$ap\\\" \\\"$aa\\\"; elif (( vp == ap && ( vp == 2 || vp == 3 ) && va > aa )); then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$vt\\\" \\\"$vd\\\" \\\"$vp\\\" \\\"$va\\\"; else printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\\\n' \\\"$at\\\" \\\"$ad\\\" \\\"$ap\\\" \\\"$aa\\\"; fi\"},{\"line\":37,\"text\":\" local trigger=\\\"$1\\\" i=0 actions=0 scan pick decision kind target out rc last_sealed=\\\"\\\"\"},{\"line\":42,\"text\":\" decision=\\\"\\\"; kind=\\\"\\\"; target=\\\"\\\"; for token in $pick; do case \\\"$token\\\" in KIND=*) kind=\\\"${token#KIND=}\\\";; TARGET=*) target=\\\"${token#TARGET=}\\\";; DECISION=*) decision=\\\"${token#DECISION=}\\\";; esac; done\"},{\"line\":43,\"text\":\" if [[ \\\"$decision\\\" == NOOP ]]; then printf 'DISPATCHER_DONE TRIGGER=%s ACTIONS=%s ITERATIONS=%s\\\\n' \\\"$trigger\\\" \\\"$actions\\\" \\\"$i\\\"; return 0; fi\"},{\"line\":44,\"text\":\" [[ \\\"$kind\\\" == VM || \\\"$kind\\\" == APP ]] || die GLOBAL_KIND_INVALID 65; [[ -n \\\"$target\\\" ]] || die GLOBAL_TARGET_MISSING 65\"},{\"line\":45,\"text\":\" if [[ \\\"$last_sealed\\\" == \\\"$kind:$target\\\" ]]; then die \\\"NO_PROGRESS_AFTER_SEALED_${kind}_${target}\\\" 65; fi\"},{\"line\":46,\"text\":\" if [[ \\\"$kind\\\" == VM ]]; then if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$VMQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; else if out=\\\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \\\"$APPQ\\\" dispatch-one \\\"$target\\\" 2>&1)\\\"; then rc=0; else rc=$?; fi; fi\"},{\"line\":48,\"text\":\" if grep -q '^QUEUE_DEFERRED ' <<<\\\"$out\\\"; then printf 'DISPATCHER_DEFERRED TRIGGER=%s KIND=%s TARGET=%s REASON=QUEUE_OR_HEAVY_BUSY\\\\n' \\\"$trigger\\\" \\\"$kind\\\" \\\"$target\\\"; return 0; fi\"},{\"line\":49,\"text\":\" if grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SEALED ' <<<\\\"$out\\\"; then last_sealed=\\\"$kind:$target\\\"; actions=$((actions+1)); elif grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SKIP_FRESH ' <<<\\\"$out\\\"; then last_sealed=\\\"\\\"; actions=$((actions+1)); else die \\\"DISPATCH_RESULT_UNCLASSIFIED_${kind}_${target}\\\" 65; fi\"},{\"line\":50,\"text\":\" done\"},{\"line\":53,\"text\":\"case \\\"${1:-}\\\" in plan) [[ \\\"$#\\\" == 1 ]] || die USAGE_PLAN 64; plan;; run) [[ \\\"$#\\\" == 2 ]] || die USAGE_RUN 64; run \\\"$2\\\";; *) die ACTION_INVALID 64;; esac\"}],\"sha256\":\"9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c\"}\nCORE_RUNTIME_ROLLBACK_VERIFIED=true\nAPP_CATCHUP_PRESERVED=true\nSOURCE_COMMIT_PRESERVED=true\nRETENTION_GC_RAN=false\nPOST1045_RCA_DECISION={\"app_catchup_preserved\":true,\"core_runtime_rollback_verified\":true,\"journal_has_root_cause_lines\":false,\"source_commit_preserved\":true,\"vm9130_needs_targeted_continuation\":true}\nNO_MUTATION=true\nNO_REPO_WRITE=true\nNO_GIT_FETCH=true\nNO_GIT_COMMIT=true\nNO_GIT_PUSH_UPDATE=true\nNO_SYSTEMD_ACTION=true\nNO_BACKUP_STARTED=true\nNO_RESTORE_STARTED=true\nNO_RETENTION_STARTED=true\nNO_DELETE=true\nNO_MAIL_SENT=true\nDURATION_SEC=1.467\nTASK_RESULT=PASS_HOMELAB_BACKUP_POST1045_V2_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\nWORKERS2_BACKUP_POST1045_V2_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index eb649786..dbd824cf 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,35 +1,74 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SIGNALBOT-260921-ORDER-NORMALIZATION-DURABLE-STAGE-052FIX2 -STATUS=FAIL -RC=76 +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=signalbot-order-normalization-durable-stage +MODE=read-only +COMPONENT=homelab-backup-post1045-v2-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=00d0ae6b43d4637e2c1d8387e2cd4211a08d98d93bdc0a3ee12cea754f2d79f2 +COMMAND_SHA256=e885553c8f307a102ca6681869f3474d99c8bd6c2674156eda9ad554a27a5741 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true -RESULT_CONTRACT_VALID=false -RESULT_CONTRACT_STATUS=UNKNOWN -RESULT_CONTRACT_ERROR=INVALID_CONTRACT:ValueError +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE COMMAND_RC=0 -CHANGES_MADE=unknown -ROLLBACK_STARTED=unknown +CHANGES_MADE=false +ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=UNKNOWN +MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=a87a39311ef0a510d26715190ede5d4c8e8f78ad2565d8047a48c25772926fde -SANITIZED_OUTPUT_SHA256=a87a39311ef0a510d26715190ede5d4c8e8f78ad2565d8047a48c25772926fde +RAW_EVIDENCE_SHA256=9bbb3ae7025709b275ab9507dcf0e7ce1dce96ea6f8d3b503011b80397be29af +SANITIZED_OUTPUT_SHA256=9bbb3ae7025709b275ab9507dcf0e7ce1dce96ea6f8d3b503011b80397be29af OUTPUT_BEGIN -FIX2={"phase":"rollback","error":"targeted: AssertionError: assert is \nE + where = ReconciliationWindow(window_id='1472770bf9bc5ff2323b035e21fec2a6f0f91161c2773593409d4ee46ce776f6', reconnect_event_seq... 13, tzinfo=datetime.timezone.utc), resolved_at=datetime.datetime(2026, 9, 20, 0, 0, 20, tzinfo=datetime.timezone.utc)).status\nE + and = ReconciliationStatus.RECONCILED\n\ntests/test_bitfinex_reconciliation.py:200: AssertionError\n=========================== short test summary info ============================\nFAILED tests/test_bitfinex_reconciliation.py::test_http_or_validation_failure_marks_window_unresolved\n1 failed in 0.17s\n","restored":true} -TASK_RESULT=HOLD_FIX2_ROLLED_BACK -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SIGNALBOT-260921-ORDER-NORMALIZATION-DURABLE-STAGE-052FIX2","status":"OK","changes_made":false,"rollback_started":true,"rollback_restored":true} +WORKERS2_BACKUP_POST1045_V2_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1 +MODE=read-only +COMPONENT=homelab-backup-post1045-v2-rca +HOST=pve01 +UID=0 +MUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL +WORKTREE_HEAD=47b9be72cf2487ba66295d7da998016bbeea57be +WORKTREE_HEAD_EXPECTED=true +WORKTREE_CLEAN=true +WORKTREE_STATUS_LINES=[] +SOURCE_COMMIT_SUMMARY={"lines":["47b9be7 CR-2026-1005: close V2 backup scheduling and retention gaps"," .../backup-closure-remediation-1045.md | 3 +"," changes/CR-2026-1005/runtime/SHA256SUMS | 4 +"," .../runtime/homelab-app-cloud-quorum-queue.service | 10 +"," .../CR-2026-1005/runtime/homelab-backup-audit-v2 | 200 +++++++++"," .../runtime/homelab-backup-retention-gc | 464 +++++++++++++++++++++"," .../runtime/homelab-backup-v2-app-scheduled | 27 ++"," 6 files changed, 708 insertions(+)"],"rc":0} +SOURCE_RUNTIME_TREE=[{"bytes":503,"name":"SHA256SUMS","sha256":"921dce0c32c2246bf5cf7cacd7e23f3d2677084165d79e05e303682dc14d35ed"},{"bytes":244,"name":"homelab-app-cloud-quorum-queue.service","sha256":"afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a"},{"bytes":11451,"name":"homelab-backup-audit-v2","sha256":"67fff795217e3e9c3a32c5a634a0401efa2b74c533b934f0a04b52a4cd9b6594"},{"bytes":32975,"name":"homelab-backup-retention-gc","sha256":"a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909"},{"bytes":1372,"name":"homelab-backup-v2-app-scheduled","sha256":"52951cd5c00a60b1a2f7d8c44d7e25feae39706820ad56964ef013e6451ca1ba"}] +POST1045_LIVE_RUNTIME={"app_scheduler":{"exists":false,"path":"/usr/local/sbin/homelab-backup-v2-app-scheduled","sha256":null},"app_service":{"baseline_match":true,"exists":true,"path":"/etc/systemd/system/homelab-app-cloud-quorum-queue.service","sha256":"70b78ba529c270ec61d26a123d656385a8b2e2e4206ccf623111fa6ed3854602"},"audit":{"baseline_match":true,"exists":true,"path":"/usr/local/sbin/homelab-backup-audit","sha256":"1474c9ce58bbc1b3ae7437261d1bdb923232926fb2943c7c830253f8b284724c"},"retention":{"baseline_match":true,"exists":true,"path":"/usr/local/sbin/homelab-backup-retention-gc","sha256":"8adb6c10f5911f39c53775b89eac3cd1823f1ed34b4caccade5279c213e998a3"}} +POST1045_UNIT_STATE={"homelab-app-cloud-quorum-queue.service":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainCode":{"rc":0,"value":"0"},"ExecMainStatus":{"rc":0,"value":"0"},"ExecStart":{"rc":0,"value":"{ path=/usr/local/sbin/homelab-app-cloud-quorum-queue ; argv[]=/usr/local/sbin/homelab-app-cloud-quorum-queue all ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }"},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-app-cloud-quorum-queue.service"},"InvocationID":{"rc":0,"value":""},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"static"}},"homelab-app-cloud-quorum-queue.timer":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainCode":{"rc":0,"value":""},"ExecMainStatus":{"rc":0,"value":""},"ExecStart":{"rc":0,"value":""},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-app-cloud-quorum-queue.timer"},"InvocationID":{"rc":0,"value":""},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"disabled"}},"homelab-backup-retention-gc.service":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainCode":{"rc":0,"value":"1"},"ExecMainStatus":{"rc":0,"value":"66"},"ExecStart":{"rc":0,"value":"{ path=/usr/local/sbin/homelab-backup-retention-gc ; argv[]=/usr/local/sbin/homelab-backup-retention-gc scheduled ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }"},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-backup-retention-gc.service"},"InvocationID":{"rc":0,"value":"426b1424d1f047e8adbb452ece2b3b80"},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"static"}},"homelab-backup-retention-gc.timer":{"ActiveState":{"rc":0,"value":"inactive"},"ExecMainCode":{"rc":0,"value":""},"ExecMainStatus":{"rc":0,"value":""},"ExecStart":{"rc":0,"value":""},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-backup-retention-gc.timer"},"InvocationID":{"rc":0,"value":""},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"dead"},"UnitFileState":{"rc":0,"value":"disabled"}},"homelab-backup-v2.service":{"ActiveState":{"rc":0,"value":"failed"},"ExecMainCode":{"rc":0,"value":"1"},"ExecMainStatus":{"rc":0,"value":"2"},"ExecStart":{"rc":0,"value":"{ path=/usr/local/sbin/homelab-backup-v2 ; argv[]=/usr/local/sbin/homelab-backup-v2 scheduled ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }"},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-backup-v2.service"},"InvocationID":{"rc":0,"value":"43c58a804cea48c2818a509fd2e75f48"},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"exit-code"},"SubState":{"rc":0,"value":"failed"},"UnitFileState":{"rc":0,"value":"static"}},"homelab-backup-v2.timer":{"ActiveState":{"rc":0,"value":"active"},"ExecMainCode":{"rc":0,"value":""},"ExecMainStatus":{"rc":0,"value":""},"ExecStart":{"rc":0,"value":""},"FragmentPath":{"rc":0,"value":"/etc/systemd/system/homelab-backup-v2.timer"},"InvocationID":{"rc":0,"value":"af3b714a30a541b0b91f4e1d2e2b5980"},"LoadState":{"rc":0,"value":"loaded"},"Result":{"rc":0,"value":"success"},"SubState":{"rc":0,"value":"waiting"},"UnitFileState":{"rc":0,"value":"enabled"}}} +APP_FRESHNESS_PLAN_POST1045={"lines":["QUEUE_FRESHNESS_PLAN APP=filebrowser DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=skladchik-reports-monitor DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=immich DECISION=SKIP_FRESH","QUEUE_FRESHNESS_PLAN APP=hbarhub DECISION=SKIP_FRESH"],"rc":0,"stderr":[]} +VM9130_STATE={"age_seconds":108844,"consecutive_failures":0,"exists":true,"failure_backoff_seconds":0,"generation":"v2-vm9130-20260919T210758Z-0aa987efac8c","last_attempt_epoch":1789853557,"last_success_epoch":1789853557,"logical_id":"vm9130","replica_count":2,"result":"SUCCESS","sha256":"6d9193e25bcf369a8800b1bf5fb6768934cf4786a9cb94e88f6b6e95645b92db"} +V2_SERVICE_JOURNAL_1045_WINDOW={"lines":["-- No entries --"],"rc":0,"stderr":[]} +V2_SERVICE_ROOT_CAUSE_LINES=[] +V2_WRAPPER_SOURCE={"path":"/usr/local/sbin/homelab-backup-v2","relevant_lines":[{"line":4,"text":"ACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b')"}],"sha256":"5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2"} +VM_QUEUE_SOURCE={"path":"/usr/local/sbin/homelab-vm-cloud-quorum-queue","relevant_lines":[{"line":5,"text":"WORKER=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker"},{"line":18,"text":" case \"$1\" in"},{"line":30,"text":" case \"$1\" in"},{"line":36,"text":"run_worker() {"},{"line":43,"text":" _run_worker_unlocked \"$@\""},{"line":46,"text":"_run_worker_unlocked() {"},{"line":52,"text":" if [[ \"$action\" == \"verify\" ]]; then \"$WORKER\" verify \"$vmid\" \"$generation\"; else \"$WORKER\" \"$action\" \"$vmid\"; fi"},{"line":56,"text":" ssh \"${SSH_BASE[@]}\" \"$host\" \"$WORKER\" verify \"$vmid\" \"$generation\""},{"line":58,"text":" ssh \"${SSH_BASE[@]}\" \"$host\" \"$WORKER\" \"$action\" \"$vmid\""},{"line":85,"text":" case \"$decision\" in"},{"line":88,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_RESUME GENERATION=%s\\n' \"$vmid\" \"$generation\""},{"line":94,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_EPOCH_INVALID\\n' \"$vmid\""},{"line":99,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$vmid\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\""},{"line":101,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$vmid\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\""},{"line":103,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$vmid\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\""},{"line":105,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\\n' \"$vmid\" \"$generation\" \"$age\""},{"line":109,"text":" printf 'QUEUE_FRESHNESS VMID=%s DECISION=BACKUP_UNVERIFIED\\n' \"$vmid\""},{"line":123,"text":" decision=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$out\" | tail -1)\""},{"line":125,"text":" printf 'QUEUE_FRESHNESS_PLAN VMID=%s DECISION=%s\\n' \"$vmid\" \"$decision\""},{"line":128,"text":"backup_list_force(){"},{"line":136,"text":" out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\""},{"line":148,"text":" [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64"},{"line":163,"text":" decision=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\""},{"line":164,"text":" case \"$decision\" in"},{"line":166,"text":" generation=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\""},{"line":174,"text":" out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\""},{"line":183,"text":"case \"$ACTION\" in"},{"line":188,"text":" freshness-plan)"},{"line":196,"text":" dispatch-one)"},{"line":202,"text":" backup_list_force 110"},{"line":206,"text":" backup_list_force \"${MIGRATE_VMIDS[@]}\""},{"line":208,"text":" all)"},{"line":210,"text":" [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die \"DISPATCHER_MISSING\" 69"},{"line":211,"text":" /usr/local/sbin/homelab-backup-dispatcher run VM_TIMER"},{"line":215,"text":" run_worker \"$2\" verify \"$3\""}],"sha256":"fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612"} +DISPATCHER_SOURCE={"path":"/usr/local/sbin/homelab-backup-dispatcher","relevant_lines":[{"line":4,"text":"VMQ=/usr/local/sbin/homelab-vm-cloud-quorum-queue"},{"line":5,"text":"APPQ=/usr/local/sbin/homelab-app-cloud-quorum-queue"},{"line":9,"text":"rank(){ case \"$1\" in BACKUP_RESUME) printf 0;; BACKUP_UNVERIFIED) printf 1;; BACKUP_STALE) printf 2;; BACKUP_DUE) printf 3;; NOOP) printf 9;; *) return 1;; esac; }"},{"line":11,"text":" local kind=\"$1\" q=\"$2\" plan line decision target age pri best_target=\"\" best_decision=NOOP best_age=-1 best_pri=99"},{"line":15,"text":" if [[ \"$kind\" == VM ]]; then [[ \"$line\" == QUEUE_FRESHNESS\\ VMID=* ]] || continue; else [[ \"$line\" == QUEUE_FRESHNESS\\ APP=* ]] || continue; fi"},{"line":16,"text":" decision=\"\"; target=\"\"; age=-1"},{"line":17,"text":" for token in $line; do case \"$token\" in VMID=*) target=\"${token#VMID=}\";; APP=*) target=\"${token#APP=}\";; DECISION=*) decision=\"${token#DECISION=}\";; AGE_SECONDS=*) age=\"${token#AGE_SECONDS=}\";; esac; done"},{"line":18,"text":" [[ -n \"$target\" && -n \"$decision\" ]] || die \"SCAN_RECORD_INVALID_$kind\" 65"},{"line":19,"text":" case \"$decision\" in SKIP_FRESH) continue;; BACKUP_RESUME|BACKUP_UNVERIFIED) pri=\"$(rank \"$decision\")\";; BACKUP_STALE|BACKUP_DUE) pri=\"$(rank \"$decision\")\"; [[ \"$age\" =~ ^[0-9]+$ ]] || die \"AGE_INVALID_${kind}_${target}_${decision}\" 65;; *) die \"SCAN_DECISION_INVALID_${kind}_${target}_${decision}\" 65;; esac"},{"line":20,"text":" if [[ -z \"$best_target\" || \"$pri\" -lt \"$best_pri\" || ( \"$pri\" -eq \"$best_pri\" && ( \"$pri\" -eq 2 || \"$pri\" -eq 3 ) && \"$age\" -gt \"$best_age\" ) ]]; then best_target=\"$target\"; best_decision=\"$decision\"; best_age=\"$age\"; best_pri=\"$pri\"; fi"},{"line":21,"text":" done <<<\"$plan\""},{"line":22,"text":" if [[ -z \"$best_target\" ]]; then printf 'CLASS_DISPATCH KIND=%s DECISION=NOOP PRIORITY=9 AGE_SECONDS=-1\\n' \"$kind\"; else printf 'CLASS_DISPATCH KIND=%s TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$kind\" \"$best_target\" \"$best_decision\" \"$best_pri\" \"$best_age\"; fi"},{"line":28,"text":" for token in $vline; do case \"$token\" in TARGET=*) vt=\"${token#TARGET=}\";; DECISION=*) vd=\"${token#DECISION=}\";; PRIORITY=*) vp=\"${token#PRIORITY=}\";; AGE_SECONDS=*) va=\"${token#AGE_SECONDS=}\";; esac; done"},{"line":29,"text":" for token in $aline; do case \"$token\" in TARGET=*) at=\"${token#TARGET=}\";; DECISION=*) ad=\"${token#DECISION=}\";; PRIORITY=*) ap=\"${token#PRIORITY=}\";; AGE_SECONDS=*) aa=\"${token#AGE_SECONDS=}\";; esac; done"},{"line":31,"text":" if [[ \"$vd\" == NOOP ]]; then printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$at\" \"$ad\" \"$ap\" \"$aa\"; return 0; fi"},{"line":32,"text":" if [[ \"$ad\" == NOOP ]]; then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$vt\" \"$vd\" \"$vp\" \"$va\"; return 0; fi"},{"line":33,"text":" if (( vp < ap )); then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$vt\" \"$vd\" \"$vp\" \"$va\"; elif (( ap < vp )); then printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$at\" \"$ad\" \"$ap\" \"$aa\"; elif (( vp == ap && ( vp == 2 || vp == 3 ) && va > aa )); then printf 'GLOBAL_DISPATCH KIND=VM TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$vt\" \"$vd\" \"$vp\" \"$va\"; else printf 'GLOBAL_DISPATCH KIND=APP TARGET=%s DECISION=%s PRIORITY=%s AGE_SECONDS=%s\\n' \"$at\" \"$ad\" \"$ap\" \"$aa\"; fi"},{"line":37,"text":" local trigger=\"$1\" i=0 actions=0 scan pick decision kind target out rc last_sealed=\"\""},{"line":42,"text":" decision=\"\"; kind=\"\"; target=\"\"; for token in $pick; do case \"$token\" in KIND=*) kind=\"${token#KIND=}\";; TARGET=*) target=\"${token#TARGET=}\";; DECISION=*) decision=\"${token#DECISION=}\";; esac; done"},{"line":43,"text":" if [[ \"$decision\" == NOOP ]]; then printf 'DISPATCHER_DONE TRIGGER=%s ACTIONS=%s ITERATIONS=%s\\n' \"$trigger\" \"$actions\" \"$i\"; return 0; fi"},{"line":44,"text":" [[ \"$kind\" == VM || \"$kind\" == APP ]] || die GLOBAL_KIND_INVALID 65; [[ -n \"$target\" ]] || die GLOBAL_TARGET_MISSING 65"},{"line":45,"text":" if [[ \"$last_sealed\" == \"$kind:$target\" ]]; then die \"NO_PROGRESS_AFTER_SEALED_${kind}_${target}\" 65; fi"},{"line":46,"text":" if [[ \"$kind\" == VM ]]; then if out=\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \"$VMQ\" dispatch-one \"$target\" 2>&1)\"; then rc=0; else rc=$?; fi; else if out=\"$(exec 7>&-; HOMELAB_BACKUP_DISPATCHER=1 \"$APPQ\" dispatch-one \"$target\" 2>&1)\"; then rc=0; else rc=$?; fi; fi"},{"line":48,"text":" if grep -q '^QUEUE_DEFERRED ' <<<\"$out\"; then printf 'DISPATCHER_DEFERRED TRIGGER=%s KIND=%s TARGET=%s REASON=QUEUE_OR_HEAVY_BUSY\\n' \"$trigger\" \"$kind\" \"$target\"; return 0; fi"},{"line":49,"text":" if grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SEALED ' <<<\"$out\"; then last_sealed=\"$kind:$target\"; actions=$((actions+1)); elif grep -q '^QUEUE_DISPATCH_RESULT .* RESULT=SKIP_FRESH ' <<<\"$out\"; then last_sealed=\"\"; actions=$((actions+1)); else die \"DISPATCH_RESULT_UNCLASSIFIED_${kind}_${target}\" 65; fi"},{"line":50,"text":" done"},{"line":53,"text":"case \"${1:-}\" in plan) [[ \"$#\" == 1 ]] || die USAGE_PLAN 64; plan;; run) [[ \"$#\" == 2 ]] || die USAGE_RUN 64; run \"$2\";; *) die ACTION_INVALID 64;; esac"}],"sha256":"9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c"} +CORE_RUNTIME_ROLLBACK_VERIFIED=true +APP_CATCHUP_PRESERVED=true +SOURCE_COMMIT_PRESERVED=true +RETENTION_GC_RAN=false +POST1045_RCA_DECISION={"app_catchup_preserved":true,"core_runtime_rollback_verified":true,"journal_has_root_cause_lines":false,"source_commit_preserved":true,"vm9130_needs_targeted_continuation":true} +NO_MUTATION=true +NO_REPO_WRITE=true +NO_GIT_FETCH=true +NO_GIT_COMMIT=true +NO_GIT_PUSH_UPDATE=true +NO_SYSTEMD_ACTION=true +NO_BACKUP_STARTED=true +NO_RESTORE_STARTED=true +NO_RETENTION_STARTED=true +NO_DELETE=true +NO_MAIL_SENT=true +DURATION_SEC=1.467 +TASK_RESULT=PASS_HOMELAB_BACKUP_POST1045_V2_RCA_READONLY +HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-POST1045-V2-RCA-READONLY-1046R1","rollback_restored":null,"rollback_started":false,"status":"OK","version":1} +WORKERS2_BACKUP_POST1045_V2_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END