From f730f95f97bfd38b8f8f8eb7d0ecb4c4c69f53e8 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Mon, 21 Sep 2026 06:26:05 +0000 Subject: [PATCH] runtime: publish SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1 --- ...-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1.json | 38 ++++ ...P-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1.txt | 55 +++++ runtime/latest.json | 39 +++- runtime/latest.txt | 203 ++++-------------- 4 files changed, 178 insertions(+), 157 deletions(-) create mode 100644 runtime/history/SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1.json create mode 100644 runtime/history/SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1.txt diff --git a/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1.json b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1.json new file mode 100644 index 00000000..011f7d0d --- /dev/null +++ b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-cloudb-preflight-rca", + "started_at_utc": "2026-09-21T06:26:01Z", + "finished_at_utc": "2026-09-21T06:26:05Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "b508f92361099e24264836513bd255c236078b8a0b80039c6c69016e3bce2085", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "a5e3e41807765a5f0bb7a13406d297e3deebf550a436821efe1df44356f9fa5c", + "sanitized_output_sha256": "a5e3e41807765a5f0bb7a13406d297e3deebf550a436821efe1df44356f9fa5c", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_CLOUDB_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1\nMODE=read-only\nCOMPONENT=homelab-backup-cloudb-preflight-rca\nMUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL\nSTATE_INVENTORY={\"latest_files\":[\"app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"app-skladchik-reports-monitor-pve01-20260916T113630Z.json\",\"app-skladchik-reports-monitor-pve01-20260915T223334Z.json\",\"app-skladchik-reports-monitor-pve01-20260915T093500Z.json\",\"app-skladchik-reports-monitor-pve01-20260914T193818Z.json\",\"app-skladchik-reports-monitor-pve01-20260914T063210Z.json\",\"app-skladchik-reports-monitor-pve01-20260913T174140Z.json\",\"app-skladchik-reports-monitor-pve01-20260913T033047Z.json\"],\"sealed\":57,\"total\":57}\nLATEST_SEALED_STATE={\"archive_sha256\":\"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140\",\"archive_size_bytes\":178434,\"cloud_a_verified\":true,\"cloud_b_verified\":true,\"cloud_quorum\":2,\"generation\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"local_cleanup\":\"OK\",\"local_payload_present\":false,\"logical_id\":\"app-skladchik-reports-monitor\",\"part_count\":1,\"part_hashes_verified\":true,\"parts_manifest_sha256\":\"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf\",\"payload_name\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst\",\"remote_a_base\":\"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"remote_b_base\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"schema\":\"homelab-cloud-quorum-payload-v1\",\"seal_written\":true,\"state_file\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"state_sha256\":\"41c4e42953c36022ea4f18b5451ec70ea251c86542502444e748545ffa799271\",\"status\":\"SEALED\",\"types\":{\"archive_sha256\":\"str\",\"archive_size_bytes\":\"int\",\"cloud_a_verified\":\"bool\",\"cloud_b_verified\":\"bool\",\"cloud_quorum\":\"int\",\"generation\":\"str\",\"local_cleanup\":\"str\",\"local_payload_present\":\"bool\",\"logical_id\":\"str\",\"part_count\":\"int\",\"part_hashes_verified\":\"bool\",\"parts_manifest_sha256\":\"str\",\"payload_name\":\"str\",\"remote_a_base\":\"str\",\"remote_b_base\":\"str\",\"schema\":\"str\",\"seal_written\":\"bool\",\"status\":\"str\"}}\nSTATE_RELEVANT_KEYS=[{\"path\":\"status\",\"type\":\"str\",\"value\":\"SEALED\"},{\"path\":\"generation\",\"type\":\"str\",\"value\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z\"},{\"path\":\"payload_path\",\"type\":\"str\",\"value\":\"/mnt/staging/homelab-app-cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst\"},{\"path\":\"payload_name\",\"type\":\"str\",\"value\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst\"},{\"path\":\"archive_size_bytes\",\"type\":\"int\",\"value\":178434},{\"path\":\"archive_sha256\",\"type\":\"str\",\"value\":\"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140\"},{\"path\":\"part_count\",\"type\":\"int\",\"value\":1},{\"path\":\"remote_a_root\",\"type\":\"str\",\"value\":\"pve01-mail-01-crypt:homelab-backups\"},{\"path\":\"remote_b_root\",\"type\":\"str\",\"value\":\"pve01-mail-02-crypt:homelab-backups\"},{\"path\":\"remote_a_base\",\"type\":\"str\",\"value\":\"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\"},{\"path\":\"remote_b_base\",\"type\":\"str\",\"value\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\"},{\"path\":\"cloud_a_verified\",\"type\":\"bool\",\"value\":true},{\"path\":\"cloud_b_verified\",\"type\":\"bool\",\"value\":true},{\"path\":\"cloud_a_reconstructed_sha256\",\"type\":\"str\",\"value\":\"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140\"},{\"path\":\"cloud_b_reconstructed_sha256\",\"type\":\"str\",\"value\":\"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140\"},{\"path\":\"cloud_a_verified_at_utc\",\"type\":\"str\",\"value\":\"2026-09-21T03:31:38Z\"},{\"path\":\"cloud_b_verified_at_utc\",\"type\":\"str\",\"value\":\"2026-09-21T03:31:43Z\"},{\"path\":\"cloud_quorum\",\"type\":\"int\",\"value\":2},{\"path\":\"part_hashes_verified\",\"type\":\"bool\",\"value\":true},{\"path\":\"parts_manifest_sha256\",\"type\":\"str\",\"value\":\"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf\"},{\"path\":\"local_payload_present\",\"type\":\"bool\",\"value\":false},{\"path\":\"cloud_a_seal_verified\",\"type\":\"bool\",\"value\":true},{\"path\":\"cloud_b_seal_verified\",\"type\":\"bool\",\"value\":true},{\"path\":\"cloud_a_seal_verified_at_utc\",\"type\":\"str\",\"value\":\"2026-09-21T03:31:48Z\"},{\"path\":\"cloud_b_seal_verified_at_utc\",\"type\":\"str\",\"value\":\"2026-09-21T03:31:48Z\"}]\nCLOUDB_STRING_CANDIDATES=[{\"path\":\"remote_b_root\",\"value\":\"pve01-mail-02-crypt:homelab-backups\"},{\"path\":\"remote_b_base\",\"value\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\"}]\nRESTORE_EVIDENCE={\"exists\":true,\"file\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"restore_source\":\"Cloud A\",\"result\":\"OK\"}\nCLOUDB_REMOTE_CHECKS=[{\"base\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"lsf_rc\":0,\"manifest_candidates\":[{\"bytes\":99,\"matches_state_count\":true,\"matches_state_manifest_sha\":true,\"matches_state_size\":true,\"path\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z/PARTS.tsv\",\"rc\":0,\"rows\":1,\"sha256\":\"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf\",\"total_size\":178434},{\"bytes\":0,\"path\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.parts/PARTS.tsv\",\"rc\":3},{\"bytes\":0,\"path\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z/parts/PARTS.tsv\",\"rc\":3}],\"source\":\"remote_b_base\",\"top_names\":[\"PARTS.tsv\",\"payload.parts/\",\"seal.json\"]},{\"base\":\"pve01-mail-02-crypt:homelab-backups\",\"lsf_rc\":0,\"manifest_candidates\":[{\"bytes\":0,\"path\":\"pve01-mail-02-crypt:homelab-backups/PARTS.tsv\",\"rc\":3},{\"bytes\":0,\"path\":\"pve01-mail-02-crypt:homelab-backups.parts/PARTS.tsv\",\"rc\":3},{\"bytes\":0,\"path\":\"pve01-mail-02-crypt:homelab-backups/parts/PARTS.tsv\",\"rc\":3}],\"source\":\"remote_b_root\",\"top_names\":[\"app-filebrowser/\",\"app-hbarhub/\",\"app-immich/\",\"app-memos/\",\"app-scrutiny/\",\"app-skladchik-moderator-assistant/\",\"app-skladchik-reports-monitor/\",\"control-bootstrap/\",\"control-bootstrap-v2/\",\"hb/\",\"pve01/\",\"retention-canary/\",\"vm110/\",\"vm111/\",\"vm112/\",\"vm113/\",\"vm130/\",\"vm150/\",\"vm160/\",\"vm170/\",\"vm171/\",\"vm180/\",\"vm190/\",\"vm200/\",\"vm211/\",\"vm9130/\"]}]\nRCA_CLASSIFICATION={\"fix\":\"REUSE_PROVEN_STATE_AND_MANIFEST_LAYOUT_WITH_CORRECTED_PREFLIGHT\",\"root_cause\":\"CLOUDB_PREFLIGHT_IMPLEMENTATION_BUG\"}\nTRANSPORT_SOURCE_FILES=[\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"/usr/local/sbin/homelab-control-backup\",\"/usr/local/sbin/homelab-rotating-boot-drill\",\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"/usr/local/libexec/homelab-app-cloud-quorum-worker\"]\nTRANSPORT_SOURCE_HITS=[{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"31: if not isinstance(obj.get(\\\"archive_size_bytes\\\"),int) or obj[\\\"archive_size_bytes\\\"] <= 0: return False\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"32: if not isinstance(obj.get(\\\"part_count\\\"),int) or obj[\\\"part_count\\\"] <= 0: return False\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"34: if not isinstance(obj.get(\\\"parts_manifest_sha256\\\"),str) or not HEX64.fullmatch(obj[\\\"parts_manifest_sha256\\\"]): return False\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"42: if obj.get(\\\"remote_a_base\\\") != expect_a or obj.get(\\\"remote_b_base\\\") != expect_b: return False\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"84: \\\"archive_size_bytes\\\":obj[\\\"archive_size_bytes\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"86: \\\"parts_manifest_sha256\\\":obj[\\\"parts_manifest_sha256\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"87: \\\"part_count\\\":obj[\\\"part_count\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"89: \\\"remote_b_base\\\":obj[\\\"remote_b_base\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-control-backup\",\"line\":\"72: expected_manifest=\\\"$(jq -er '.parts_manifest_sha256' \\\"$state\\\")\\\"; expected_archive=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\"\"},{\"file\":\"/usr/local/sbin/homelab-control-backup\",\"line\":\"73: manifest=\\\"${out}.PARTS.tsv\\\"; : >\\\"$out\\\"\"},{\"file\":\"/usr/local/sbin/homelab-control-backup\",\"line\":\"74: rclone --config \\\"$cfg\\\" cat \\\"${base%/}/PARTS.tsv\\\" >\\\"$manifest\\\" || die \\\"REMOTE_${label}_MANIFEST_READ_FAILED\\\" 74\"},{\"file\":\"/usr/local/sbin/homelab-control-backup\",\"line\":\"96: base_a=\\\"$(jq -er '.remote_a_base' \\\"$state\\\")\\\"; base_b=\\\"$(jq -er '.remote_b_base' \\\"$state\\\")\\\"; a=\\\"$work/A.tar.gz\\\"; b=\\\"$work/B.tar.gz\\\"\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"374: base=info[\\\"remote_a_base\\\"] if cloud==\\\"A\\\" else info[\\\"remote_b_base\\\"]\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"376: manifest_bytes=rclone_cat(base.rstrip(\\\"/\\\")+\\\"/PARTS.tsv\\\",60)\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"382: \\\"archive_size_bytes\\\":info[\\\"archive_size_bytes\\\"],\\\"archive_sha256\\\":info[\\\"archive_sha256\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"383: \\\"part_count\\\":info[\\\"part_count\\\"],\\\"parts_manifest_sha256\\\":info[\\\"parts_manifest_sha256\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"388: if hashlib.sha256(manifest_bytes).hexdigest()!=info[\\\"parts_manifest_sha256\\\"]: die(\\\"REMOTE_MANIFEST_SHA_MISMATCH_\\\"+info[\\\"vmid\\\"],66)\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"389: rows=parse_manifest(manifest_bytes,info[\\\"part_count\\\"],info[\\\"archive_size_bytes\\\"])\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"420: need_stage=qinfo[\\\"archive_size_bytes\\\"]+linfo[\\\"archive_size_bytes\\\"]+p[\\\"minimum_stage_headroom_bytes\\\"]\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"438: \\\"qemu_archive_bytes\\\":qi[\\\"archive_size_bytes\\\"],\\\"lxc_archive_bytes\\\":li[\\\"archive_size_bytes\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"583: if total!=info[\\\"archive_size_bytes\\\"] or h.hexdigest()!=info[\\\"archive_sha256\\\"]: die(\\\"ARCHIVE_VERIFY_FAILED_\\\"+info[\\\"vmid\\\"],70)\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"31: if not isinstance(obj.get(\\\"archive_size_bytes\\\"),int) or obj[\\\"archive_size_bytes\\\"] <= 0: return False\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"32: if not isinstance(obj.get(\\\"part_count\\\"),int) or obj[\\\"part_count\\\"] <= 0: return False\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"34: if not isinstance(obj.get(\\\"parts_manifest_sha256\\\"),str) or not HEX64.fullmatch(obj[\\\"parts_manifest_sha256\\\"]): return False\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"42: if obj.get(\\\"remote_a_base\\\") != expect_a or obj.get(\\\"remote_b_base\\\") != expect_b: return False\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"84: \\\"archive_size_bytes\\\":obj[\\\"archive_size_bytes\\\"],\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"86: \\\"parts_manifest_sha256\\\":obj[\\\"parts_manifest_sha256\\\"],\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"87: \\\"part_count\\\":obj[\\\"part_count\\\"],\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"89: \\\"remote_b_base\\\":obj[\\\"remote_b_base\\\"],\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"198: --arg remote_b_base \\\"$REMOTE_B_BASE\\\" \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"201: --argjson part_count \\\"$PART_COUNT\\\" \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"205: archive_size_bytes:$payload_size,archive_sha256:$payload_sha,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"206: chunk_size_bytes:$chunk_size,part_count:$part_count,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"208: remote_a_base:$remote_a_base,remote_b_base:$remote_b_base,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"212: cloud_quorum:0,part_hashes_verified:false,parts_manifest_sha256:null,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"230: --arg remote_b_base \\\"$REMOTE_B_BASE\\\" \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"232: --argjson part_count \\\"$PART_COUNT\\\" \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"235: .archive_size_bytes==$payload_size and\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"237: .part_count==$part_count and\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"239: .remote_b_base==$remote_b_base' \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"248: PAYLOAD_SIZE=\\\"$(jq -er '.archive_size_bytes | numbers | select(.>0 and floor==.)' \\\"$STATE_FILE\\\")\\\" || die \\\"STATE_PAYLOAD_SIZE_INVALID\\\" 65\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"249: PART_COUNT=\\\"$(jq -er '.part_count | numbers | select(.>0 and floor==.)' \\\"$STATE_FILE\\\")\\\" || die \\\"STATE_PART_COUNT_INVALID\\\" 65\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"452: .parts_manifest_sha256=$manifest_sha' \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"463: saved_manifest=\\\"$(jq -er '.parts_manifest_sha256 | strings | select(test(\\\"^[0-9a-f]{64}$\\\"))' \\\"$STATE_FILE\\\")\\\" || die \\\"STATE_PARTS_MANIFEST_SHA_INVALID\\\" 65\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"535: .parts_manifest_sha256==null\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"540: (.parts_manifest_sha256 |\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"560:PARTS_MANIFEST=\\\"$WORK/PARTS.tsv\\\"\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"636: printf '%s\\\\n' \\\"${base%/}/PARTS.tsv\\\"\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"687: PARTS_MANIFEST_SHA=\\\"$(jq -er '.parts_manifest_sha256 | strings | select(test(\\\"^[0-9a-f]{64}$\\\"))' \\\"$STATE_FILE\\\")\\\" || die \\\"STATE_PARTS_MANIFEST_SHA_INVALID\\\" 65\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"700: remote_manifest=\\\"$WORK/remote-${label}-PARTS.tsv\\\"\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"836: archive_size_bytes:.archive_size_bytes,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"839: part_count:.part_count,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"840: parts_manifest_sha256:.parts_manifest_sha256,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"856: (.parts_manifest_sha256|type==\\\"string\\\" and test(\\\"^[0-9a-f]{64}$\\\")) and\"},{\"file\":\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"line\":\"89: base=\\\"$(jq -er '.remote_a_base' \\\"$state\\\")\\\"; expected_msha=\\\"$(jq -er '.parts_manifest_sha256' \\\"$state\\\")\\\"; expected_count=\\\"$(jq -er '.part_count' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"file\":\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"line\":\"91: manifest=\\\"$work/PARTS.tsv\\\"; rclone cat \\\"${base%/}/PARTS.tsv\\\" >\\\"$manifest\\\"\"},{\"file\":\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"line\":\"121: payload_name=\\\"$(jq -er '.payload_name' \\\"$state\\\")\\\"; expected_sha=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"file\":\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"line\":\"124: base=\\\"$(prepare_cloud_a_manifest \\\"$vmid\\\" \\\"$gen\\\" \\\"$state\\\" \\\"$work\\\")\\\"; manifest=\\\"$work/PARTS.tsv\\\"; fifo=\\\"$work/archive.sha.fifo\\\"; sha_out=\\\"$work/archive.sha256\\\"; mkfifo -m 0600 \\\"$fifo\\\"\"},{\"file\":\"/usr/local/libexec/homelab-app-cloud-quorum-worker\",\"line\":\"352: base=\\\"$(jq -er '.remote_a_base' \\\"$state\\\")\\\"; expected_msha=\\\"$(jq -er '.parts_manifest_sha256' \\\"$state\\\")\\\"\"},{\"file\":\"/usr/local/libexec/homelab-app-cloud-quorum-worker\",\"line\":\"353: manifest=\\\"$(dirname \\\"$out\\\")/PARTS.tsv\\\"; rclone cat \\\"${base%/}/PARTS.tsv\\\" >\\\"$manifest\\\"\"}]\nNO_MUTATION=true\nNO_BACKUP_STARTED=true\nNO_RESTORE_STARTED=true\nNO_RETENTION_STARTED=true\nNO_DELETE=true\nNO_MAIL_SENT=true\nTASK_RESULT=PASS_HOMELAB_BACKUP_CLOUDB_PREFLIGHT_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_CLOUDB_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1.txt b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1.txt new file mode 100644 index 00000000..5fab162a --- /dev/null +++ b/runtime/history/SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1.txt @@ -0,0 +1,55 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-cloudb-preflight-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=b508f92361099e24264836513bd255c236078b8a0b80039c6c69016e3bce2085 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=a5e3e41807765a5f0bb7a13406d297e3deebf550a436821efe1df44356f9fa5c +SANITIZED_OUTPUT_SHA256=a5e3e41807765a5f0bb7a13406d297e3deebf550a436821efe1df44356f9fa5c +OUTPUT_BEGIN +WORKERS2_CLOUDB_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1 +MODE=read-only +COMPONENT=homelab-backup-cloudb-preflight-rca +MUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL +STATE_INVENTORY={"latest_files":["app-skladchik-reports-monitor-pve01-20260921T033135Z.json","app-skladchik-reports-monitor-pve01-20260916T113630Z.json","app-skladchik-reports-monitor-pve01-20260915T223334Z.json","app-skladchik-reports-monitor-pve01-20260915T093500Z.json","app-skladchik-reports-monitor-pve01-20260914T193818Z.json","app-skladchik-reports-monitor-pve01-20260914T063210Z.json","app-skladchik-reports-monitor-pve01-20260913T174140Z.json","app-skladchik-reports-monitor-pve01-20260913T033047Z.json"],"sealed":57,"total":57} +LATEST_SEALED_STATE={"archive_sha256":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140","archive_size_bytes":178434,"cloud_a_verified":true,"cloud_b_verified":true,"cloud_quorum":2,"generation":"app-skladchik-reports-monitor-pve01-20260921T033135Z","local_cleanup":"OK","local_payload_present":false,"logical_id":"app-skladchik-reports-monitor","part_count":1,"part_hashes_verified":true,"parts_manifest_sha256":"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf","payload_name":"app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst","remote_a_base":"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z","remote_b_base":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z","schema":"homelab-cloud-quorum-payload-v1","seal_written":true,"state_file":"app-skladchik-reports-monitor-pve01-20260921T033135Z.json","state_sha256":"41c4e42953c36022ea4f18b5451ec70ea251c86542502444e748545ffa799271","status":"SEALED","types":{"archive_sha256":"str","archive_size_bytes":"int","cloud_a_verified":"bool","cloud_b_verified":"bool","cloud_quorum":"int","generation":"str","local_cleanup":"str","local_payload_present":"bool","logical_id":"str","part_count":"int","part_hashes_verified":"bool","parts_manifest_sha256":"str","payload_name":"str","remote_a_base":"str","remote_b_base":"str","schema":"str","seal_written":"bool","status":"str"}} +STATE_RELEVANT_KEYS=[{"path":"status","type":"str","value":"SEALED"},{"path":"generation","type":"str","value":"app-skladchik-reports-monitor-pve01-20260921T033135Z"},{"path":"payload_path","type":"str","value":"/mnt/staging/homelab-app-cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst"},{"path":"payload_name","type":"str","value":"app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst"},{"path":"archive_size_bytes","type":"int","value":178434},{"path":"archive_sha256","type":"str","value":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140"},{"path":"part_count","type":"int","value":1},{"path":"remote_a_root","type":"str","value":"pve01-mail-01-crypt:homelab-backups"},{"path":"remote_b_root","type":"str","value":"pve01-mail-02-crypt:homelab-backups"},{"path":"remote_a_base","type":"str","value":"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z"},{"path":"remote_b_base","type":"str","value":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z"},{"path":"cloud_a_verified","type":"bool","value":true},{"path":"cloud_b_verified","type":"bool","value":true},{"path":"cloud_a_reconstructed_sha256","type":"str","value":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140"},{"path":"cloud_b_reconstructed_sha256","type":"str","value":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140"},{"path":"cloud_a_verified_at_utc","type":"str","value":"2026-09-21T03:31:38Z"},{"path":"cloud_b_verified_at_utc","type":"str","value":"2026-09-21T03:31:43Z"},{"path":"cloud_quorum","type":"int","value":2},{"path":"part_hashes_verified","type":"bool","value":true},{"path":"parts_manifest_sha256","type":"str","value":"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf"},{"path":"local_payload_present","type":"bool","value":false},{"path":"cloud_a_seal_verified","type":"bool","value":true},{"path":"cloud_b_seal_verified","type":"bool","value":true},{"path":"cloud_a_seal_verified_at_utc","type":"str","value":"2026-09-21T03:31:48Z"},{"path":"cloud_b_seal_verified_at_utc","type":"str","value":"2026-09-21T03:31:48Z"}] +CLOUDB_STRING_CANDIDATES=[{"path":"remote_b_root","value":"pve01-mail-02-crypt:homelab-backups"},{"path":"remote_b_base","value":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z"}] +RESTORE_EVIDENCE={"exists":true,"file":"app-skladchik-reports-monitor-pve01-20260921T033135Z.json","restore_source":"Cloud A","result":"OK"} +CLOUDB_REMOTE_CHECKS=[{"base":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z","lsf_rc":0,"manifest_candidates":[{"bytes":99,"matches_state_count":true,"matches_state_manifest_sha":true,"matches_state_size":true,"path":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z/PARTS.tsv","rc":0,"rows":1,"sha256":"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf","total_size":178434},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.parts/PARTS.tsv","rc":3},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z/parts/PARTS.tsv","rc":3}],"source":"remote_b_base","top_names":["PARTS.tsv","payload.parts/","seal.json"]},{"base":"pve01-mail-02-crypt:homelab-backups","lsf_rc":0,"manifest_candidates":[{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/PARTS.tsv","rc":3},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups.parts/PARTS.tsv","rc":3},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/parts/PARTS.tsv","rc":3}],"source":"remote_b_root","top_names":["app-filebrowser/","app-hbarhub/","app-immich/","app-memos/","app-scrutiny/","app-skladchik-moderator-assistant/","app-skladchik-reports-monitor/","control-bootstrap/","control-bootstrap-v2/","hb/","pve01/","retention-canary/","vm110/","vm111/","vm112/","vm113/","vm130/","vm150/","vm160/","vm170/","vm171/","vm180/","vm190/","vm200/","vm211/","vm9130/"]}] +RCA_CLASSIFICATION={"fix":"REUSE_PROVEN_STATE_AND_MANIFEST_LAYOUT_WITH_CORRECTED_PREFLIGHT","root_cause":"CLOUDB_PREFLIGHT_IMPLEMENTATION_BUG"} +TRANSPORT_SOURCE_FILES=["/usr/local/sbin/homelab-rotating-boot-drill-state-export","/usr/local/sbin/homelab-control-backup","/usr/local/sbin/homelab-rotating-boot-drill","/usr/local/libexec/homelab-rotating-boot-drill-state-export","/usr/local/libexec/homelab-cloud-quorum-payload","/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","/usr/local/libexec/homelab-app-cloud-quorum-worker"] +TRANSPORT_SOURCE_HITS=[{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"31: if not isinstance(obj.get(\"archive_size_bytes\"),int) or obj[\"archive_size_bytes\"] <= 0: return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"32: if not isinstance(obj.get(\"part_count\"),int) or obj[\"part_count\"] <= 0: return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"34: if not isinstance(obj.get(\"parts_manifest_sha256\"),str) or not HEX64.fullmatch(obj[\"parts_manifest_sha256\"]): return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"42: if obj.get(\"remote_a_base\") != expect_a or obj.get(\"remote_b_base\") != expect_b: return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"84: \"archive_size_bytes\":obj[\"archive_size_bytes\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"86: \"parts_manifest_sha256\":obj[\"parts_manifest_sha256\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"87: \"part_count\":obj[\"part_count\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"89: \"remote_b_base\":obj[\"remote_b_base\"],"},{"file":"/usr/local/sbin/homelab-control-backup","line":"72: expected_manifest=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"; expected_archive=\"$(jq -er '.archive_sha256' \"$state\")\""},{"file":"/usr/local/sbin/homelab-control-backup","line":"73: manifest=\"${out}.PARTS.tsv\"; : >\"$out\""},{"file":"/usr/local/sbin/homelab-control-backup","line":"74: rclone --config \"$cfg\" cat \"${base%/}/PARTS.tsv\" >\"$manifest\" || die \"REMOTE_${label}_MANIFEST_READ_FAILED\" 74"},{"file":"/usr/local/sbin/homelab-control-backup","line":"96: base_a=\"$(jq -er '.remote_a_base' \"$state\")\"; base_b=\"$(jq -er '.remote_b_base' \"$state\")\"; a=\"$work/A.tar.gz\"; b=\"$work/B.tar.gz\""},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"374: base=info[\"remote_a_base\"] if cloud==\"A\" else info[\"remote_b_base\"]"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"376: manifest_bytes=rclone_cat(base.rstrip(\"/\")+\"/PARTS.tsv\",60)"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"382: \"archive_size_bytes\":info[\"archive_size_bytes\"],\"archive_sha256\":info[\"archive_sha256\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"383: \"part_count\":info[\"part_count\"],\"parts_manifest_sha256\":info[\"parts_manifest_sha256\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"388: if hashlib.sha256(manifest_bytes).hexdigest()!=info[\"parts_manifest_sha256\"]: die(\"REMOTE_MANIFEST_SHA_MISMATCH_\"+info[\"vmid\"],66)"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"389: rows=parse_manifest(manifest_bytes,info[\"part_count\"],info[\"archive_size_bytes\"])"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"420: need_stage=qinfo[\"archive_size_bytes\"]+linfo[\"archive_size_bytes\"]+p[\"minimum_stage_headroom_bytes\"]"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"438: \"qemu_archive_bytes\":qi[\"archive_size_bytes\"],\"lxc_archive_bytes\":li[\"archive_size_bytes\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"583: if total!=info[\"archive_size_bytes\"] or h.hexdigest()!=info[\"archive_sha256\"]: die(\"ARCHIVE_VERIFY_FAILED_\"+info[\"vmid\"],70)"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"31: if not isinstance(obj.get(\"archive_size_bytes\"),int) or obj[\"archive_size_bytes\"] <= 0: return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"32: if not isinstance(obj.get(\"part_count\"),int) or obj[\"part_count\"] <= 0: return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"34: if not isinstance(obj.get(\"parts_manifest_sha256\"),str) or not HEX64.fullmatch(obj[\"parts_manifest_sha256\"]): return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"42: if obj.get(\"remote_a_base\") != expect_a or obj.get(\"remote_b_base\") != expect_b: return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"84: \"archive_size_bytes\":obj[\"archive_size_bytes\"],"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"86: \"parts_manifest_sha256\":obj[\"parts_manifest_sha256\"],"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"87: \"part_count\":obj[\"part_count\"],"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"89: \"remote_b_base\":obj[\"remote_b_base\"],"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"198: --arg remote_b_base \"$REMOTE_B_BASE\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"201: --argjson part_count \"$PART_COUNT\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"205: archive_size_bytes:$payload_size,archive_sha256:$payload_sha,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"206: chunk_size_bytes:$chunk_size,part_count:$part_count,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"208: remote_a_base:$remote_a_base,remote_b_base:$remote_b_base,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"212: cloud_quorum:0,part_hashes_verified:false,parts_manifest_sha256:null,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"230: --arg remote_b_base \"$REMOTE_B_BASE\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"232: --argjson part_count \"$PART_COUNT\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"235: .archive_size_bytes==$payload_size and"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"237: .part_count==$part_count and"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"239: .remote_b_base==$remote_b_base' \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"248: PAYLOAD_SIZE=\"$(jq -er '.archive_size_bytes | numbers | select(.>0 and floor==.)' \"$STATE_FILE\")\" || die \"STATE_PAYLOAD_SIZE_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"249: PART_COUNT=\"$(jq -er '.part_count | numbers | select(.>0 and floor==.)' \"$STATE_FILE\")\" || die \"STATE_PART_COUNT_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"452: .parts_manifest_sha256=$manifest_sha' \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"463: saved_manifest=\"$(jq -er '.parts_manifest_sha256 | strings | select(test(\"^[0-9a-f]{64}$\"))' \"$STATE_FILE\")\" || die \"STATE_PARTS_MANIFEST_SHA_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"535: .parts_manifest_sha256==null"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"540: (.parts_manifest_sha256 |"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"560:PARTS_MANIFEST=\"$WORK/PARTS.tsv\""},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"636: printf '%s\\n' \"${base%/}/PARTS.tsv\""},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"687: PARTS_MANIFEST_SHA=\"$(jq -er '.parts_manifest_sha256 | strings | select(test(\"^[0-9a-f]{64}$\"))' \"$STATE_FILE\")\" || die \"STATE_PARTS_MANIFEST_SHA_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"700: remote_manifest=\"$WORK/remote-${label}-PARTS.tsv\""},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"836: archive_size_bytes:.archive_size_bytes,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"839: part_count:.part_count,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"840: parts_manifest_sha256:.parts_manifest_sha256,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"856: (.parts_manifest_sha256|type==\"string\" and test(\"^[0-9a-f]{64}$\")) and"},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"89: base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"; expected_count=\"$(jq -er '.part_count' \"$state\")\"; expected_size=\"$(jq -er '.archive_size_bytes' \"$state\")\""},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"91: manifest=\"$work/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\""},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"121: payload_name=\"$(jq -er '.payload_name' \"$state\")\"; expected_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; expected_size=\"$(jq -er '.archive_size_bytes' \"$state\")\""},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"124: base=\"$(prepare_cloud_a_manifest \"$vmid\" \"$gen\" \"$state\" \"$work\")\"; manifest=\"$work/PARTS.tsv\"; fifo=\"$work/archive.sha.fifo\"; sha_out=\"$work/archive.sha256\"; mkfifo -m 0600 \"$fifo\""},{"file":"/usr/local/libexec/homelab-app-cloud-quorum-worker","line":"352: base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\""},{"file":"/usr/local/libexec/homelab-app-cloud-quorum-worker","line":"353: manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\""}] +NO_MUTATION=true +NO_BACKUP_STARTED=true +NO_RESTORE_STARTED=true +NO_RETENTION_STARTED=true +NO_DELETE=true +NO_MAIL_SENT=true +TASK_RESULT=PASS_HOMELAB_BACKUP_CLOUDB_PREFLIGHT_RCA_READONLY +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_CLOUDB_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index c84de1f5..011f7d0d 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1 +1,38 @@ -{"schema_version":1,"channel":"homelab-runtime","command_id":"CONTEXT-AUTO-20260921T061701Z","status":"OK","rc":0,"host":"pve01","mode":"read-only","component":"cluster-context","started_at_utc":"2026-09-21T06:17:01Z","finished_at_utc":"2026-09-21T06:17:02Z","reference_register_checked":true,"reference_sha256":"5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66","error_register_checked":true,"error_register_sha256":"3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0","changes_made":false,"sanitized":true,"secrets_included":false,"private_addresses_included":false,"output":"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK"} +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-cloudb-preflight-rca", + "started_at_utc": "2026-09-21T06:26:01Z", + "finished_at_utc": "2026-09-21T06:26:05Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "b508f92361099e24264836513bd255c236078b8a0b80039c6c69016e3bce2085", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "a5e3e41807765a5f0bb7a13406d297e3deebf550a436821efe1df44356f9fa5c", + "sanitized_output_sha256": "a5e3e41807765a5f0bb7a13406d297e3deebf550a436821efe1df44356f9fa5c", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_CLOUDB_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1\nMODE=read-only\nCOMPONENT=homelab-backup-cloudb-preflight-rca\nMUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL\nSTATE_INVENTORY={\"latest_files\":[\"app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"app-skladchik-reports-monitor-pve01-20260916T113630Z.json\",\"app-skladchik-reports-monitor-pve01-20260915T223334Z.json\",\"app-skladchik-reports-monitor-pve01-20260915T093500Z.json\",\"app-skladchik-reports-monitor-pve01-20260914T193818Z.json\",\"app-skladchik-reports-monitor-pve01-20260914T063210Z.json\",\"app-skladchik-reports-monitor-pve01-20260913T174140Z.json\",\"app-skladchik-reports-monitor-pve01-20260913T033047Z.json\"],\"sealed\":57,\"total\":57}\nLATEST_SEALED_STATE={\"archive_sha256\":\"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140\",\"archive_size_bytes\":178434,\"cloud_a_verified\":true,\"cloud_b_verified\":true,\"cloud_quorum\":2,\"generation\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"local_cleanup\":\"OK\",\"local_payload_present\":false,\"logical_id\":\"app-skladchik-reports-monitor\",\"part_count\":1,\"part_hashes_verified\":true,\"parts_manifest_sha256\":\"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf\",\"payload_name\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst\",\"remote_a_base\":\"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"remote_b_base\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"schema\":\"homelab-cloud-quorum-payload-v1\",\"seal_written\":true,\"state_file\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"state_sha256\":\"41c4e42953c36022ea4f18b5451ec70ea251c86542502444e748545ffa799271\",\"status\":\"SEALED\",\"types\":{\"archive_sha256\":\"str\",\"archive_size_bytes\":\"int\",\"cloud_a_verified\":\"bool\",\"cloud_b_verified\":\"bool\",\"cloud_quorum\":\"int\",\"generation\":\"str\",\"local_cleanup\":\"str\",\"local_payload_present\":\"bool\",\"logical_id\":\"str\",\"part_count\":\"int\",\"part_hashes_verified\":\"bool\",\"parts_manifest_sha256\":\"str\",\"payload_name\":\"str\",\"remote_a_base\":\"str\",\"remote_b_base\":\"str\",\"schema\":\"str\",\"seal_written\":\"bool\",\"status\":\"str\"}}\nSTATE_RELEVANT_KEYS=[{\"path\":\"status\",\"type\":\"str\",\"value\":\"SEALED\"},{\"path\":\"generation\",\"type\":\"str\",\"value\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z\"},{\"path\":\"payload_path\",\"type\":\"str\",\"value\":\"/mnt/staging/homelab-app-cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst\"},{\"path\":\"payload_name\",\"type\":\"str\",\"value\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst\"},{\"path\":\"archive_size_bytes\",\"type\":\"int\",\"value\":178434},{\"path\":\"archive_sha256\",\"type\":\"str\",\"value\":\"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140\"},{\"path\":\"part_count\",\"type\":\"int\",\"value\":1},{\"path\":\"remote_a_root\",\"type\":\"str\",\"value\":\"pve01-mail-01-crypt:homelab-backups\"},{\"path\":\"remote_b_root\",\"type\":\"str\",\"value\":\"pve01-mail-02-crypt:homelab-backups\"},{\"path\":\"remote_a_base\",\"type\":\"str\",\"value\":\"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\"},{\"path\":\"remote_b_base\",\"type\":\"str\",\"value\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\"},{\"path\":\"cloud_a_verified\",\"type\":\"bool\",\"value\":true},{\"path\":\"cloud_b_verified\",\"type\":\"bool\",\"value\":true},{\"path\":\"cloud_a_reconstructed_sha256\",\"type\":\"str\",\"value\":\"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140\"},{\"path\":\"cloud_b_reconstructed_sha256\",\"type\":\"str\",\"value\":\"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140\"},{\"path\":\"cloud_a_verified_at_utc\",\"type\":\"str\",\"value\":\"2026-09-21T03:31:38Z\"},{\"path\":\"cloud_b_verified_at_utc\",\"type\":\"str\",\"value\":\"2026-09-21T03:31:43Z\"},{\"path\":\"cloud_quorum\",\"type\":\"int\",\"value\":2},{\"path\":\"part_hashes_verified\",\"type\":\"bool\",\"value\":true},{\"path\":\"parts_manifest_sha256\",\"type\":\"str\",\"value\":\"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf\"},{\"path\":\"local_payload_present\",\"type\":\"bool\",\"value\":false},{\"path\":\"cloud_a_seal_verified\",\"type\":\"bool\",\"value\":true},{\"path\":\"cloud_b_seal_verified\",\"type\":\"bool\",\"value\":true},{\"path\":\"cloud_a_seal_verified_at_utc\",\"type\":\"str\",\"value\":\"2026-09-21T03:31:48Z\"},{\"path\":\"cloud_b_seal_verified_at_utc\",\"type\":\"str\",\"value\":\"2026-09-21T03:31:48Z\"}]\nCLOUDB_STRING_CANDIDATES=[{\"path\":\"remote_b_root\",\"value\":\"pve01-mail-02-crypt:homelab-backups\"},{\"path\":\"remote_b_base\",\"value\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\"}]\nRESTORE_EVIDENCE={\"exists\":true,\"file\":\"app-skladchik-reports-monitor-pve01-20260921T033135Z.json\",\"restore_source\":\"Cloud A\",\"result\":\"OK\"}\nCLOUDB_REMOTE_CHECKS=[{\"base\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z\",\"lsf_rc\":0,\"manifest_candidates\":[{\"bytes\":99,\"matches_state_count\":true,\"matches_state_manifest_sha\":true,\"matches_state_size\":true,\"path\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z/PARTS.tsv\",\"rc\":0,\"rows\":1,\"sha256\":\"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf\",\"total_size\":178434},{\"bytes\":0,\"path\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.parts/PARTS.tsv\",\"rc\":3},{\"bytes\":0,\"path\":\"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z/parts/PARTS.tsv\",\"rc\":3}],\"source\":\"remote_b_base\",\"top_names\":[\"PARTS.tsv\",\"payload.parts/\",\"seal.json\"]},{\"base\":\"pve01-mail-02-crypt:homelab-backups\",\"lsf_rc\":0,\"manifest_candidates\":[{\"bytes\":0,\"path\":\"pve01-mail-02-crypt:homelab-backups/PARTS.tsv\",\"rc\":3},{\"bytes\":0,\"path\":\"pve01-mail-02-crypt:homelab-backups.parts/PARTS.tsv\",\"rc\":3},{\"bytes\":0,\"path\":\"pve01-mail-02-crypt:homelab-backups/parts/PARTS.tsv\",\"rc\":3}],\"source\":\"remote_b_root\",\"top_names\":[\"app-filebrowser/\",\"app-hbarhub/\",\"app-immich/\",\"app-memos/\",\"app-scrutiny/\",\"app-skladchik-moderator-assistant/\",\"app-skladchik-reports-monitor/\",\"control-bootstrap/\",\"control-bootstrap-v2/\",\"hb/\",\"pve01/\",\"retention-canary/\",\"vm110/\",\"vm111/\",\"vm112/\",\"vm113/\",\"vm130/\",\"vm150/\",\"vm160/\",\"vm170/\",\"vm171/\",\"vm180/\",\"vm190/\",\"vm200/\",\"vm211/\",\"vm9130/\"]}]\nRCA_CLASSIFICATION={\"fix\":\"REUSE_PROVEN_STATE_AND_MANIFEST_LAYOUT_WITH_CORRECTED_PREFLIGHT\",\"root_cause\":\"CLOUDB_PREFLIGHT_IMPLEMENTATION_BUG\"}\nTRANSPORT_SOURCE_FILES=[\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"/usr/local/sbin/homelab-control-backup\",\"/usr/local/sbin/homelab-rotating-boot-drill\",\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"/usr/local/libexec/homelab-app-cloud-quorum-worker\"]\nTRANSPORT_SOURCE_HITS=[{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"31: if not isinstance(obj.get(\\\"archive_size_bytes\\\"),int) or obj[\\\"archive_size_bytes\\\"] <= 0: return False\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"32: if not isinstance(obj.get(\\\"part_count\\\"),int) or obj[\\\"part_count\\\"] <= 0: return False\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"34: if not isinstance(obj.get(\\\"parts_manifest_sha256\\\"),str) or not HEX64.fullmatch(obj[\\\"parts_manifest_sha256\\\"]): return False\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"42: if obj.get(\\\"remote_a_base\\\") != expect_a or obj.get(\\\"remote_b_base\\\") != expect_b: return False\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"84: \\\"archive_size_bytes\\\":obj[\\\"archive_size_bytes\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"86: \\\"parts_manifest_sha256\\\":obj[\\\"parts_manifest_sha256\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"87: \\\"part_count\\\":obj[\\\"part_count\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill-state-export\",\"line\":\"89: \\\"remote_b_base\\\":obj[\\\"remote_b_base\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-control-backup\",\"line\":\"72: expected_manifest=\\\"$(jq -er '.parts_manifest_sha256' \\\"$state\\\")\\\"; expected_archive=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\"\"},{\"file\":\"/usr/local/sbin/homelab-control-backup\",\"line\":\"73: manifest=\\\"${out}.PARTS.tsv\\\"; : >\\\"$out\\\"\"},{\"file\":\"/usr/local/sbin/homelab-control-backup\",\"line\":\"74: rclone --config \\\"$cfg\\\" cat \\\"${base%/}/PARTS.tsv\\\" >\\\"$manifest\\\" || die \\\"REMOTE_${label}_MANIFEST_READ_FAILED\\\" 74\"},{\"file\":\"/usr/local/sbin/homelab-control-backup\",\"line\":\"96: base_a=\\\"$(jq -er '.remote_a_base' \\\"$state\\\")\\\"; base_b=\\\"$(jq -er '.remote_b_base' \\\"$state\\\")\\\"; a=\\\"$work/A.tar.gz\\\"; b=\\\"$work/B.tar.gz\\\"\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"374: base=info[\\\"remote_a_base\\\"] if cloud==\\\"A\\\" else info[\\\"remote_b_base\\\"]\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"376: manifest_bytes=rclone_cat(base.rstrip(\\\"/\\\")+\\\"/PARTS.tsv\\\",60)\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"382: \\\"archive_size_bytes\\\":info[\\\"archive_size_bytes\\\"],\\\"archive_sha256\\\":info[\\\"archive_sha256\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"383: \\\"part_count\\\":info[\\\"part_count\\\"],\\\"parts_manifest_sha256\\\":info[\\\"parts_manifest_sha256\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"388: if hashlib.sha256(manifest_bytes).hexdigest()!=info[\\\"parts_manifest_sha256\\\"]: die(\\\"REMOTE_MANIFEST_SHA_MISMATCH_\\\"+info[\\\"vmid\\\"],66)\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"389: rows=parse_manifest(manifest_bytes,info[\\\"part_count\\\"],info[\\\"archive_size_bytes\\\"])\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"420: need_stage=qinfo[\\\"archive_size_bytes\\\"]+linfo[\\\"archive_size_bytes\\\"]+p[\\\"minimum_stage_headroom_bytes\\\"]\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"438: \\\"qemu_archive_bytes\\\":qi[\\\"archive_size_bytes\\\"],\\\"lxc_archive_bytes\\\":li[\\\"archive_size_bytes\\\"],\"},{\"file\":\"/usr/local/sbin/homelab-rotating-boot-drill\",\"line\":\"583: if total!=info[\\\"archive_size_bytes\\\"] or h.hexdigest()!=info[\\\"archive_sha256\\\"]: die(\\\"ARCHIVE_VERIFY_FAILED_\\\"+info[\\\"vmid\\\"],70)\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"31: if not isinstance(obj.get(\\\"archive_size_bytes\\\"),int) or obj[\\\"archive_size_bytes\\\"] <= 0: return False\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"32: if not isinstance(obj.get(\\\"part_count\\\"),int) or obj[\\\"part_count\\\"] <= 0: return False\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"34: if not isinstance(obj.get(\\\"parts_manifest_sha256\\\"),str) or not HEX64.fullmatch(obj[\\\"parts_manifest_sha256\\\"]): return False\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"42: if obj.get(\\\"remote_a_base\\\") != expect_a or obj.get(\\\"remote_b_base\\\") != expect_b: return False\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"84: \\\"archive_size_bytes\\\":obj[\\\"archive_size_bytes\\\"],\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"86: \\\"parts_manifest_sha256\\\":obj[\\\"parts_manifest_sha256\\\"],\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"87: \\\"part_count\\\":obj[\\\"part_count\\\"],\"},{\"file\":\"/usr/local/libexec/homelab-rotating-boot-drill-state-export\",\"line\":\"89: \\\"remote_b_base\\\":obj[\\\"remote_b_base\\\"],\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"198: --arg remote_b_base \\\"$REMOTE_B_BASE\\\" \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"201: --argjson part_count \\\"$PART_COUNT\\\" \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"205: archive_size_bytes:$payload_size,archive_sha256:$payload_sha,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"206: chunk_size_bytes:$chunk_size,part_count:$part_count,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"208: remote_a_base:$remote_a_base,remote_b_base:$remote_b_base,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"212: cloud_quorum:0,part_hashes_verified:false,parts_manifest_sha256:null,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"230: --arg remote_b_base \\\"$REMOTE_B_BASE\\\" \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"232: --argjson part_count \\\"$PART_COUNT\\\" \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"235: .archive_size_bytes==$payload_size and\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"237: .part_count==$part_count and\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"239: .remote_b_base==$remote_b_base' \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"248: PAYLOAD_SIZE=\\\"$(jq -er '.archive_size_bytes | numbers | select(.>0 and floor==.)' \\\"$STATE_FILE\\\")\\\" || die \\\"STATE_PAYLOAD_SIZE_INVALID\\\" 65\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"249: PART_COUNT=\\\"$(jq -er '.part_count | numbers | select(.>0 and floor==.)' \\\"$STATE_FILE\\\")\\\" || die \\\"STATE_PART_COUNT_INVALID\\\" 65\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"452: .parts_manifest_sha256=$manifest_sha' \\\\\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"463: saved_manifest=\\\"$(jq -er '.parts_manifest_sha256 | strings | select(test(\\\"^[0-9a-f]{64}$\\\"))' \\\"$STATE_FILE\\\")\\\" || die \\\"STATE_PARTS_MANIFEST_SHA_INVALID\\\" 65\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"535: .parts_manifest_sha256==null\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"540: (.parts_manifest_sha256 |\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"560:PARTS_MANIFEST=\\\"$WORK/PARTS.tsv\\\"\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"636: printf '%s\\\\n' \\\"${base%/}/PARTS.tsv\\\"\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"687: PARTS_MANIFEST_SHA=\\\"$(jq -er '.parts_manifest_sha256 | strings | select(test(\\\"^[0-9a-f]{64}$\\\"))' \\\"$STATE_FILE\\\")\\\" || die \\\"STATE_PARTS_MANIFEST_SHA_INVALID\\\" 65\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"700: remote_manifest=\\\"$WORK/remote-${label}-PARTS.tsv\\\"\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"836: archive_size_bytes:.archive_size_bytes,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"839: part_count:.part_count,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"840: parts_manifest_sha256:.parts_manifest_sha256,\"},{\"file\":\"/usr/local/libexec/homelab-cloud-quorum-payload\",\"line\":\"856: (.parts_manifest_sha256|type==\\\"string\\\" and test(\\\"^[0-9a-f]{64}$\\\")) and\"},{\"file\":\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"line\":\"89: base=\\\"$(jq -er '.remote_a_base' \\\"$state\\\")\\\"; expected_msha=\\\"$(jq -er '.parts_manifest_sha256' \\\"$state\\\")\\\"; expected_count=\\\"$(jq -er '.part_count' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"file\":\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"line\":\"91: manifest=\\\"$work/PARTS.tsv\\\"; rclone cat \\\"${base%/}/PARTS.tsv\\\" >\\\"$manifest\\\"\"},{\"file\":\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"line\":\"121: payload_name=\\\"$(jq -er '.payload_name' \\\"$state\\\")\\\"; expected_sha=\\\"$(jq -er '.archive_sha256' \\\"$state\\\")\\\"; expected_size=\\\"$(jq -er '.archive_size_bytes' \\\"$state\\\")\\\"\"},{\"file\":\"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\",\"line\":\"124: base=\\\"$(prepare_cloud_a_manifest \\\"$vmid\\\" \\\"$gen\\\" \\\"$state\\\" \\\"$work\\\")\\\"; manifest=\\\"$work/PARTS.tsv\\\"; fifo=\\\"$work/archive.sha.fifo\\\"; sha_out=\\\"$work/archive.sha256\\\"; mkfifo -m 0600 \\\"$fifo\\\"\"},{\"file\":\"/usr/local/libexec/homelab-app-cloud-quorum-worker\",\"line\":\"352: base=\\\"$(jq -er '.remote_a_base' \\\"$state\\\")\\\"; expected_msha=\\\"$(jq -er '.parts_manifest_sha256' \\\"$state\\\")\\\"\"},{\"file\":\"/usr/local/libexec/homelab-app-cloud-quorum-worker\",\"line\":\"353: manifest=\\\"$(dirname \\\"$out\\\")/PARTS.tsv\\\"; rclone cat \\\"${base%/}/PARTS.tsv\\\" >\\\"$manifest\\\"\"}]\nNO_MUTATION=true\nNO_BACKUP_STARTED=true\nNO_RESTORE_STARTED=true\nNO_RETENTION_STARTED=true\nNO_DELETE=true\nNO_MAIL_SENT=true\nTASK_RESULT=PASS_HOMELAB_BACKUP_CLOUDB_PREFLIGHT_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_CLOUDB_RCA_END=1\n" +} diff --git a/runtime/latest.txt b/runtime/latest.txt index 214cce10..5fab162a 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,164 +1,55 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=CONTEXT-AUTO-[PRIVATE_IP]701Z +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1 STATUS=OK RC=0 HOST=pve01 -COMPONENT=cluster-context -REFERENCE_SHA[PRIVATE_IP]e5154c41cb[PRIVATE_IP]aca68090be[PRIVATE_IP]bf[PRIVATE_IP]df[PRIVATE_IP] -ERROR_REGISTER_SHA[PRIVATE_IP]ec0f527ed3afeed[PRIVATE_IP]ee[PRIVATE_IP]bbfb[PRIVATE_IP]af4ba7ba0 +MODE=read-only +COMPONENT=homelab-backup-cloudb-preflight-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=b508f92361099e24264836513bd255c236078b8a0b80039c6c69016e3bce2085 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=a5e3e41807765a5f0bb7a13406d297e3deebf550a436821efe1df44356f9fa5c +SANITIZED_OUTPUT_SHA256=a5e3e41807765a5f0bb7a13406d297e3deebf550a436821efe1df44356f9fa5c OUTPUT_BEGIN -GENERATED_AT_UTC=[PRIVATE_IP]T06:17:01Z -Cluster information -------------------- -Name: homelab -Config Version: 3 -Transport: knet -Secure auth: on +WORKERS2_CLOUDB_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1 +MODE=read-only +COMPONENT=homelab-backup-cloudb-preflight-rca +MUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL +STATE_INVENTORY={"latest_files":["app-skladchik-reports-monitor-pve01-20260921T033135Z.json","app-skladchik-reports-monitor-pve01-20260916T113630Z.json","app-skladchik-reports-monitor-pve01-20260915T223334Z.json","app-skladchik-reports-monitor-pve01-20260915T093500Z.json","app-skladchik-reports-monitor-pve01-20260914T193818Z.json","app-skladchik-reports-monitor-pve01-20260914T063210Z.json","app-skladchik-reports-monitor-pve01-20260913T174140Z.json","app-skladchik-reports-monitor-pve01-20260913T033047Z.json"],"sealed":57,"total":57} +LATEST_SEALED_STATE={"archive_sha256":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140","archive_size_bytes":178434,"cloud_a_verified":true,"cloud_b_verified":true,"cloud_quorum":2,"generation":"app-skladchik-reports-monitor-pve01-20260921T033135Z","local_cleanup":"OK","local_payload_present":false,"logical_id":"app-skladchik-reports-monitor","part_count":1,"part_hashes_verified":true,"parts_manifest_sha256":"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf","payload_name":"app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst","remote_a_base":"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z","remote_b_base":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z","schema":"homelab-cloud-quorum-payload-v1","seal_written":true,"state_file":"app-skladchik-reports-monitor-pve01-20260921T033135Z.json","state_sha256":"41c4e42953c36022ea4f18b5451ec70ea251c86542502444e748545ffa799271","status":"SEALED","types":{"archive_sha256":"str","archive_size_bytes":"int","cloud_a_verified":"bool","cloud_b_verified":"bool","cloud_quorum":"int","generation":"str","local_cleanup":"str","local_payload_present":"bool","logical_id":"str","part_count":"int","part_hashes_verified":"bool","parts_manifest_sha256":"str","payload_name":"str","remote_a_base":"str","remote_b_base":"str","schema":"str","seal_written":"bool","status":"str"}} +STATE_RELEVANT_KEYS=[{"path":"status","type":"str","value":"SEALED"},{"path":"generation","type":"str","value":"app-skladchik-reports-monitor-pve01-20260921T033135Z"},{"path":"payload_path","type":"str","value":"/mnt/staging/homelab-app-cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst"},{"path":"payload_name","type":"str","value":"app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst"},{"path":"archive_size_bytes","type":"int","value":178434},{"path":"archive_sha256","type":"str","value":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140"},{"path":"part_count","type":"int","value":1},{"path":"remote_a_root","type":"str","value":"pve01-mail-01-crypt:homelab-backups"},{"path":"remote_b_root","type":"str","value":"pve01-mail-02-crypt:homelab-backups"},{"path":"remote_a_base","type":"str","value":"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z"},{"path":"remote_b_base","type":"str","value":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z"},{"path":"cloud_a_verified","type":"bool","value":true},{"path":"cloud_b_verified","type":"bool","value":true},{"path":"cloud_a_reconstructed_sha256","type":"str","value":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140"},{"path":"cloud_b_reconstructed_sha256","type":"str","value":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140"},{"path":"cloud_a_verified_at_utc","type":"str","value":"2026-09-21T03:31:38Z"},{"path":"cloud_b_verified_at_utc","type":"str","value":"2026-09-21T03:31:43Z"},{"path":"cloud_quorum","type":"int","value":2},{"path":"part_hashes_verified","type":"bool","value":true},{"path":"parts_manifest_sha256","type":"str","value":"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf"},{"path":"local_payload_present","type":"bool","value":false},{"path":"cloud_a_seal_verified","type":"bool","value":true},{"path":"cloud_b_seal_verified","type":"bool","value":true},{"path":"cloud_a_seal_verified_at_utc","type":"str","value":"2026-09-21T03:31:48Z"},{"path":"cloud_b_seal_verified_at_utc","type":"str","value":"2026-09-21T03:31:48Z"}] +CLOUDB_STRING_CANDIDATES=[{"path":"remote_b_root","value":"pve01-mail-02-crypt:homelab-backups"},{"path":"remote_b_base","value":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z"}] +RESTORE_EVIDENCE={"exists":true,"file":"app-skladchik-reports-monitor-pve01-20260921T033135Z.json","restore_source":"Cloud A","result":"OK"} +CLOUDB_REMOTE_CHECKS=[{"base":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z","lsf_rc":0,"manifest_candidates":[{"bytes":99,"matches_state_count":true,"matches_state_manifest_sha":true,"matches_state_size":true,"path":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z/PARTS.tsv","rc":0,"rows":1,"sha256":"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf","total_size":178434},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.parts/PARTS.tsv","rc":3},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z/parts/PARTS.tsv","rc":3}],"source":"remote_b_base","top_names":["PARTS.tsv","payload.parts/","seal.json"]},{"base":"pve01-mail-02-crypt:homelab-backups","lsf_rc":0,"manifest_candidates":[{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/PARTS.tsv","rc":3},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups.parts/PARTS.tsv","rc":3},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/parts/PARTS.tsv","rc":3}],"source":"remote_b_root","top_names":["app-filebrowser/","app-hbarhub/","app-immich/","app-memos/","app-scrutiny/","app-skladchik-moderator-assistant/","app-skladchik-reports-monitor/","control-bootstrap/","control-bootstrap-v2/","hb/","pve01/","retention-canary/","vm110/","vm111/","vm112/","vm113/","vm130/","vm150/","vm160/","vm170/","vm171/","vm180/","vm190/","vm200/","vm211/","vm9130/"]}] +RCA_CLASSIFICATION={"fix":"REUSE_PROVEN_STATE_AND_MANIFEST_LAYOUT_WITH_CORRECTED_PREFLIGHT","root_cause":"CLOUDB_PREFLIGHT_IMPLEMENTATION_BUG"} +TRANSPORT_SOURCE_FILES=["/usr/local/sbin/homelab-rotating-boot-drill-state-export","/usr/local/sbin/homelab-control-backup","/usr/local/sbin/homelab-rotating-boot-drill","/usr/local/libexec/homelab-rotating-boot-drill-state-export","/usr/local/libexec/homelab-cloud-quorum-payload","/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","/usr/local/libexec/homelab-app-cloud-quorum-worker"] +TRANSPORT_SOURCE_HITS=[{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"31: if not isinstance(obj.get(\"archive_size_bytes\"),int) or obj[\"archive_size_bytes\"] <= 0: return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"32: if not isinstance(obj.get(\"part_count\"),int) or obj[\"part_count\"] <= 0: return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"34: if not isinstance(obj.get(\"parts_manifest_sha256\"),str) or not HEX64.fullmatch(obj[\"parts_manifest_sha256\"]): return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"42: if obj.get(\"remote_a_base\") != expect_a or obj.get(\"remote_b_base\") != expect_b: return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"84: \"archive_size_bytes\":obj[\"archive_size_bytes\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"86: \"parts_manifest_sha256\":obj[\"parts_manifest_sha256\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"87: \"part_count\":obj[\"part_count\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"89: \"remote_b_base\":obj[\"remote_b_base\"],"},{"file":"/usr/local/sbin/homelab-control-backup","line":"72: expected_manifest=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"; expected_archive=\"$(jq -er '.archive_sha256' \"$state\")\""},{"file":"/usr/local/sbin/homelab-control-backup","line":"73: manifest=\"${out}.PARTS.tsv\"; : >\"$out\""},{"file":"/usr/local/sbin/homelab-control-backup","line":"74: rclone --config \"$cfg\" cat \"${base%/}/PARTS.tsv\" >\"$manifest\" || die \"REMOTE_${label}_MANIFEST_READ_FAILED\" 74"},{"file":"/usr/local/sbin/homelab-control-backup","line":"96: base_a=\"$(jq -er '.remote_a_base' \"$state\")\"; base_b=\"$(jq -er '.remote_b_base' \"$state\")\"; a=\"$work/A.tar.gz\"; b=\"$work/B.tar.gz\""},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"374: base=info[\"remote_a_base\"] if cloud==\"A\" else info[\"remote_b_base\"]"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"376: manifest_bytes=rclone_cat(base.rstrip(\"/\")+\"/PARTS.tsv\",60)"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"382: \"archive_size_bytes\":info[\"archive_size_bytes\"],\"archive_sha256\":info[\"archive_sha256\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"383: \"part_count\":info[\"part_count\"],\"parts_manifest_sha256\":info[\"parts_manifest_sha256\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"388: if hashlib.sha256(manifest_bytes).hexdigest()!=info[\"parts_manifest_sha256\"]: die(\"REMOTE_MANIFEST_SHA_MISMATCH_\"+info[\"vmid\"],66)"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"389: rows=parse_manifest(manifest_bytes,info[\"part_count\"],info[\"archive_size_bytes\"])"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"420: need_stage=qinfo[\"archive_size_bytes\"]+linfo[\"archive_size_bytes\"]+p[\"minimum_stage_headroom_bytes\"]"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"438: \"qemu_archive_bytes\":qi[\"archive_size_bytes\"],\"lxc_archive_bytes\":li[\"archive_size_bytes\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"583: if total!=info[\"archive_size_bytes\"] or h.hexdigest()!=info[\"archive_sha256\"]: die(\"ARCHIVE_VERIFY_FAILED_\"+info[\"vmid\"],70)"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"31: if not isinstance(obj.get(\"archive_size_bytes\"),int) or obj[\"archive_size_bytes\"] <= 0: return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"32: if not isinstance(obj.get(\"part_count\"),int) or obj[\"part_count\"] <= 0: return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"34: if not isinstance(obj.get(\"parts_manifest_sha256\"),str) or not HEX64.fullmatch(obj[\"parts_manifest_sha256\"]): return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"42: if obj.get(\"remote_a_base\") != expect_a or obj.get(\"remote_b_base\") != expect_b: return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"84: \"archive_size_bytes\":obj[\"archive_size_bytes\"],"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"86: \"parts_manifest_sha256\":obj[\"parts_manifest_sha256\"],"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"87: \"part_count\":obj[\"part_count\"],"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"89: \"remote_b_base\":obj[\"remote_b_base\"],"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"198: --arg remote_b_base \"$REMOTE_B_BASE\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"201: --argjson part_count \"$PART_COUNT\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"205: archive_size_bytes:$payload_size,archive_sha256:$payload_sha,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"206: chunk_size_bytes:$chunk_size,part_count:$part_count,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"208: remote_a_base:$remote_a_base,remote_b_base:$remote_b_base,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"212: cloud_quorum:0,part_hashes_verified:false,parts_manifest_sha256:null,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"230: --arg remote_b_base \"$REMOTE_B_BASE\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"232: --argjson part_count \"$PART_COUNT\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"235: .archive_size_bytes==$payload_size and"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"237: .part_count==$part_count and"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"239: .remote_b_base==$remote_b_base' \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"248: PAYLOAD_SIZE=\"$(jq -er '.archive_size_bytes | numbers | select(.>0 and floor==.)' \"$STATE_FILE\")\" || die \"STATE_PAYLOAD_SIZE_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"249: PART_COUNT=\"$(jq -er '.part_count | numbers | select(.>0 and floor==.)' \"$STATE_FILE\")\" || die \"STATE_PART_COUNT_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"452: .parts_manifest_sha256=$manifest_sha' \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"463: saved_manifest=\"$(jq -er '.parts_manifest_sha256 | strings | select(test(\"^[0-9a-f]{64}$\"))' \"$STATE_FILE\")\" || die \"STATE_PARTS_MANIFEST_SHA_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"535: .parts_manifest_sha256==null"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"540: (.parts_manifest_sha256 |"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"560:PARTS_MANIFEST=\"$WORK/PARTS.tsv\""},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"636: printf '%s\\n' \"${base%/}/PARTS.tsv\""},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"687: PARTS_MANIFEST_SHA=\"$(jq -er '.parts_manifest_sha256 | strings | select(test(\"^[0-9a-f]{64}$\"))' \"$STATE_FILE\")\" || die \"STATE_PARTS_MANIFEST_SHA_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"700: remote_manifest=\"$WORK/remote-${label}-PARTS.tsv\""},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"836: archive_size_bytes:.archive_size_bytes,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"839: part_count:.part_count,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"840: parts_manifest_sha256:.parts_manifest_sha256,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"856: (.parts_manifest_sha256|type==\"string\" and test(\"^[0-9a-f]{64}$\")) and"},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"89: base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"; expected_count=\"$(jq -er '.part_count' \"$state\")\"; expected_size=\"$(jq -er '.archive_size_bytes' \"$state\")\""},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"91: manifest=\"$work/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\""},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"121: payload_name=\"$(jq -er '.payload_name' \"$state\")\"; expected_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; expected_size=\"$(jq -er '.archive_size_bytes' \"$state\")\""},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"124: base=\"$(prepare_cloud_a_manifest \"$vmid\" \"$gen\" \"$state\" \"$work\")\"; manifest=\"$work/PARTS.tsv\"; fifo=\"$work/archive.sha.fifo\"; sha_out=\"$work/archive.sha256\"; mkfifo -m 0600 \"$fifo\""},{"file":"/usr/local/libexec/homelab-app-cloud-quorum-worker","line":"352: base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\""},{"file":"/usr/local/libexec/homelab-app-cloud-quorum-worker","line":"353: manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\""}] +NO_MUTATION=true +NO_BACKUP_STARTED=true +NO_RESTORE_STARTED=true +NO_RETENTION_STARTED=true +NO_DELETE=true +NO_MAIL_SENT=true +TASK_RESULT=PASS_HOMELAB_BACKUP_CLOUDB_PREFLIGHT_RCA_READONLY +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_CLOUDB_RCA_END=1 -Quorum information ------------------- -Date: Mon Sep [PRIVATE_IP] 2026 -Quorum provider: corosync_votequorum -Nodes: 3 -Node ID: [PRIVATE_IP] -Ring ID: 1.130 -Quorate: Yes - -Votequorum information ----------------------- -Expected votes: 3 -Highest expected: 3 -Total votes: 3 -Quorum: 2 -Flags: Quorate - -Membership information ----------------------- - Nodeid Votes Name -[PRIVATE_IP] [PRIVATE_IP].11 (local) -[PRIVATE_IP] [PRIVATE_IP].13 -[PRIVATE_IP] [PRIVATE_IP].12 - -Membership information ----------------------- - Nodeid Votes Name - 1 1 pve01 (local) - 2 1 pve03 - 3 1 pve02 -[{"cpu":[PRIVATE_IP]077022,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/100","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"pvepro-prod","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":105388,"vmid":100},{"cpu":0,"disk":0,"diskread":0,"diskwrite":0,"id":"qemu/101","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":0,"memhost":0,"name":"pvepro-staging-v1","netin":0,"netout":0,"node":"pve01","status":"stopped","template":0,"type":"qemu","uptime":0,"vmid":101},{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/110","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns1","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"lxc","uptime":[PRIVATE_IP],"vmid":110},{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/111","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns2","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"lxc","uptime":[PRIVATE_IP],"vmid":111},{"cpu":0,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/112","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound1","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"lxc","uptime":[PRIVATE_IP],"vmid":112},{"cpu":[PRIVATE_IP]312e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/113","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound2","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"lxc","uptime":[PRIVATE_IP],"vmid":113},{"cpu":[PRIVATE_IP]57329,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"edge-vm","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":130},{"cpu":[PRIVATE_IP]369448,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/150","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"snikket","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":150},{"cpu":[PRIVATE_IP]021165,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/160","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"forum-prod","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":160},{"cpu":[PRIVATE_IP]2688,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/170","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"core-apps","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":170},{"cpu":[PRIVATE_IP]78588,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/171","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"monitoring","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":171},{"cpu":[PRIVATE_IP]398092,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/190","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"homelab-ops-worker","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":190},{"cpu":[PRIVATE_IP]723235,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/210","maxcpu":2,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"newfi-prod","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":210},{"cpu":[PRIVATE_IP]595549,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/211","maxcpu":2,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"newfi-staging","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":[PRIVATE_IP],"vmid":211},{"cpu":0,"disk":0,"diskread":0,"diskwrite":0,"id":"qemu/9130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":0,"memhost":0,"name":"edge-cold-standby","netin":0,"netout":0,"node":"pve02","status":"stopped","template":0,"type":"qemu","uptime":0,"vmid":9130},{"cgroup-mode":2,"cpu":[PRIVATE_IP]14544,"disk":[PRIVATE_IP],"id":"node/pve02","level":"","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve02","status":"online","type":"node","uptime":[PRIVATE_IP]},{"cgroup-mode":2,"cpu":[PRIVATE_IP]38419,"disk":[PRIVATE_IP],"id":"node/pve03","level":"","maxcpu":8,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve03","status":"online","type":"node","uptime":[PRIVATE_IP]},{"cgroup-mode":2,"cpu":[PRIVATE_IP]2962,"disk":[PRIVATE_IP],"id":"node/pve01","level":"","maxcpu":24,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve01","status":"online","type":"node","uptime":[PRIVATE_IP]},{"content":"iso,import,backup,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve02/local","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"iso,import,backup,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve03/local","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"iso,import,backup,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve01/local","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve02/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve03/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve01/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"id":"network/pve02/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve02","status":"ok","type":"network"},{"id":"network/pve03/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve03","status":"ok","type":"network"},{"id":"network/pve01/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve01","status":"ok","type":"network"}] -Name Type Status Total (KiB) Used (KiB) Available (KiB) % -local dir active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 37.29% -local-lvm lvmthin active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 10.70% - UNIT LOAD ACTIVE SUB DESCRIPTION -● homelab-backup-audit.service loaded failed failed Homelab unified backup and DR audit -● homelab-vm-cloud-quorum-queue.service loaded failed failed Homelab canonical VM/CT cloud quorum backup queue -● netbird-peers-health.service loaded failed failed NetBird peers health check - -Legend: LOAD → Reflects whether the unit definition was properly loaded. - ACTIVE → The high-level unit activation state, i.e. generalization of SUB. - SUB → The low-level unit activation state, values depend on unit type. - -3 loaded units listed. -NEXT LEFT LAST PASSED UNIT ACTIVATES -Mon [PRIVATE_IP] 09:17:06 MSK 4s Mon [PRIVATE_IP] 09:16:02 MSK 1min 0s ago homelab-router-watchdog.timer homelab-router-watchdog.service -Mon [PRIVATE_IP] 09:17:16 MSK 13s Mon [PRIVATE_IP] 09:16:16 MSK 46s ago homelab-private-vpn-hosts-health.timer homelab-private-vpn-hosts-health.service -Mon [PRIVATE_IP] 09:18:16 MSK 1min 13s Mon [PRIVATE_IP] 09:13:16 MSK 3min 46s ago homelab-smartctl-textfile.timer homelab-smartctl-textfile.service -Mon [PRIVATE_IP] 09:18:16 MSK 1min 13s Mon [PRIVATE_IP] 09:13:16 MSK 3min 46s ago netbird-peers-health.timer netbird-peers-health.service -Mon [PRIVATE_IP] 09:18:16 MSK 1min 13s Mon [PRIVATE_IP] 09:08:16 MSK 8min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service -Mon [PRIVATE_IP] 09:19:58 MSK 2min 56s Mon [PRIVATE_IP] 09:04:58 MSK 12min ago prometheus-node-exporter-apt.timer prometheus-node-exporter-apt.service -Mon [PRIVATE_IP] 09:22:06 MSK 5min Mon [PRIVATE_IP] 09:16:48 MSK 13s ago homelab-health-metrics.timer homelab-health-metrics.service -Mon [PRIVATE_IP] 09:24:38 MSK 7min Mon [PRIVATE_IP] 09:09:32 MSK 7min ago homelab-alertmanager-backup-health.timer homelab-alertmanager-backup-health.service -Mon [PRIVATE_IP] 09:24:58 MSK 7min Mon [PRIVATE_IP] 09:09:58 MSK 7min ago homelab-disk-space-health.timer homelab-disk-space-health.service -Mon [PRIVATE_IP] 09:24:58 MSK 7min Mon [PRIVATE_IP] 09:09:58 MSK 7min ago prometheus-node-exporter-nvme.timer prometheus-node-exporter-nvme.service -Mon [PRIVATE_IP] 09:27:32 MSK 10min Mon [PRIVATE_IP] 09:12:32 MSK 4min 30s ago homelab-pve03-staging-capacity-health.timer homelab-pve03-staging-capacity-health.service -Mon [PRIVATE_IP] 09:30:00 MSK 12min Mon [PRIVATE_IP] 07:52:31 MSK 1h 24min ago homelab-app-cloud-quorum-queue.timer homelab-app-cloud-quorum-queue.service -Mon [PRIVATE_IP] 09:30:00 MSK 12min Mon [PRIVATE_IP] 09:15:14 MSK 1min 47s ago homelab-incident-journal.timer homelab-incident-journal.service -Mon [PRIVATE_IP] 09:30:07 MSK 13min Mon [PRIVATE_IP] 09:15:32 MSK 1min 30s ago homelab-external-probe-vps-health.timer homelab-external-probe-vps-health.service -Mon [PRIVATE_IP] 09:30:59 MSK 13min Mon [PRIVATE_IP] 09:15:32 MSK 1min 30s ago homelab-forum-snuffleupagus-health.timer homelab-forum-snuffleupagus-health.service -Mon [PRIVATE_IP] 09:30:59 MSK 13min Mon [PRIVATE_IP] 09:15:14 MSK 1min 47s ago homelab-duty-admin-v2.timer homelab-duty-admin-v2.service -Mon [PRIVATE_IP] 09:32:00 MSK 14min Mon [PRIVATE_IP] 09:00:58 MSK 16min ago homelab-mkdocs-auto-refresh.timer homelab-mkdocs-auto-refresh.service -Mon [PRIVATE_IP] 09:35:54 MSK 18min Mon [PRIVATE_IP] 09:05:54 MSK 11min ago homelab-backup-audit.timer homelab-backup-audit.service -Mon [PRIVATE_IP] 09:47:57 MSK 30min Mon [PRIVATE_IP] 09:15:52 MSK 1min 10s ago homelab-reference-refresh.timer homelab-reference-refresh.service -Mon [PRIVATE_IP] 09:56:15 MSK 39min Mon [PRIVATE_IP] 07:29:58 MSK 1h 47min ago homelab-mailru-trash-gc.timer homelab-mailru-trash-gc.service -Mon [PRIVATE_IP] 10:15:00 MSK 57min Sun [PRIVATE_IP] 10:15:12 MSK 23h ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service -Mon [PRIVATE_IP] 10:35:58 MSK 1h 18min Mon [PRIVATE_IP] 04:35:58 MSK 4h 41min ago homelab-vps-identity-audit.timer homelab-vps-identity-audit.service -Mon [PRIVATE_IP] 14:33:51 MSK 5h 16min Mon [PRIVATE_IP] 08:32:32 MSK 44min ago homelab-evidence-root.timer homelab-evidence-root.service -Mon [PRIVATE_IP] 16:47:58 MSK 7h Sun [PRIVATE_IP] 16:47:58 MSK 16h ago systemd-tmpfiles-clean.timer systemd-tmpfiles-clean.service -Mon [PRIVATE_IP] 21:05:47 MSK 11h Mon [PRIVATE_IP] 07:52:31 MSK 1h 24min ago apt-daily.timer apt-daily.service -Tue [PRIVATE_IP] 00:00:00 MSK 14h Mon [PRIVATE_IP] 00:00:14 MSK 9h ago dpkg-db-backup.timer dpkg-db-backup.service -Tue [PRIVATE_IP] 00:02:42 MSK 14h Mon [PRIVATE_IP] 00:07:58 MSK 9h ago homelab-docker-health.timer homelab-docker-health.service -Tue [PRIVATE_IP] 00:15:20 MSK 14h Mon [PRIVATE_IP] 00:16:22 MSK 9h ago homelab-storage-capacity.timer homelab-storage-capacity.service -Tue [PRIVATE_IP] 00:17:37 MSK 15h Mon [PRIVATE_IP] 00:17:23 MSK 8h ago homelab-quality-gate.timer homelab-quality-gate.service -Tue [PRIVATE_IP] 00:19:58 MSK 15h Mon [PRIVATE_IP] 00:07:44 MSK 9h ago homelab-evidence-catalog.timer homelab-evidence-catalog.service -Tue [PRIVATE_IP] 00:41:15 MSK 15h Mon [PRIVATE_IP] 00:23:58 MSK 8h ago logrotate.timer logrotate.service -Tue [PRIVATE_IP] 01:54:27 MSK 16h Mon [PRIVATE_IP] 02:49:58 MSK 6h ago pve-daily-update.timer pve-daily-update.service -Tue [PRIVATE_IP] 03:25:27 MSK 18h - - homelab-backup-retention-gc.timer homelab-backup-retention-gc.service -Tue [PRIVATE_IP] 06:49:12 MSK 21h Mon [PRIVATE_IP] 06:16:58 MSK 3h 0min ago apt-daily-upgrade.timer apt-daily-upgrade.service -Tue [PRIVATE_IP] 07:05:58 MSK 21h Mon [PRIVATE_IP] 07:06:58 MSK 2h 10min ago homelab-drift-check.timer homelab-drift-check.service -Tue [PRIVATE_IP] 07:38:49 MSK 22h Mon [PRIVATE_IP] 07:22:10 MSK 1h 54min ago homelab-service-registry-check.timer homelab-service-registry-check.service -Tue [PRIVATE_IP] 07:44:23 MSK 22h Mon [PRIVATE_IP] 07:47:53 MSK 1h 29min ago homelab-dependency-map-check.timer homelab-dependency-map-check.service -Tue [PRIVATE_IP] 07:45:44 MSK 22h Mon [PRIVATE_IP] 07:39:12 MSK 1h 37min ago homelab-alerting-health.timer homelab-alerting-health.service -Tue [PRIVATE_IP] 07:56:03 MSK 22h Mon [PRIVATE_IP] 08:02:58 MSK 1h 14min ago homelab-desired-state-sync.timer homelab-desired-state-sync.service -Tue [PRIVATE_IP] 08:00:13 MSK 22h Mon [PRIVATE_IP] 07:52:31 MSK 1h 24min ago homelab-runbook-generate.timer homelab-runbook-generate.service -Tue [PRIVATE_IP] 08:00:15 MSK 22h Mon [PRIVATE_IP] 08:12:41 MSK 1h 4min ago homelab-duty-admin-report.timer homelab-duty-admin-report.service -Tue [PRIVATE_IP] 08:19:06 MSK 23h Mon [PRIVATE_IP] 08:12:44 MSK 1h 4min ago homelab-overall-health.timer homelab-overall-health.service -Tue [PRIVATE_IP] 08:24:54 MSK 23h Mon [PRIVATE_IP] 08:18:45 MSK 58min ago homelab-kuma-monitor-policy.timer homelab-kuma-monitor-policy.service -Tue [PRIVATE_IP] 08:30:15 MSK 23h Mon [PRIVATE_IP] 08:27:07 MSK 49min ago homelab-final-readiness-gate.timer homelab-final-readiness-gate.service -Tue [PRIVATE_IP] 08:36:07 MSK 23h Mon [PRIVATE_IP] 08:36:28 MSK 40min ago homelab-cluster-passport.timer homelab-cluster-passport.service -Tue [PRIVATE_IP] 08:37:36 MSK 23h Mon [PRIVATE_IP] 08:38:16 MSK 38min ago homelab-secret-exposure-guard.timer homelab-secret-exposure-guard.service -Tue [PRIVATE_IP] 08:40:22 MSK 23h Mon [PRIVATE_IP] 08:33:58 MSK 43min ago homelab-capacity-risk.timer homelab-capacity-risk.service -Tue [PRIVATE_IP] 08:51:58 MSK 23h Mon [PRIVATE_IP] 08:48:05 MSK 28min ago homelab-golden-state-index.timer homelab-golden-state-index.service -Tue [PRIVATE_IP] 09:28:49 MSK 24h Mon [PRIVATE_IP] 06:34:34 MSK 2h 42min ago man-db.timer man-db.service -Sun [PRIVATE_IP] 03:10:36 MSK 5 days Sun [PRIVATE_IP] 03:10:58 MSK 1 day 6h ago xfs_scrub_all.timer xfs_scrub_all.service -Sun [PRIVATE_IP] 03:10:54 MSK 5 days Sun [PRIVATE_IP] 03:10:12 MSK 1 day 6h ago e2scrub_all.timer e2scrub_all.service -Sun [PRIVATE_IP] 07:32:40 MSK 5 days Sun [PRIVATE_IP] 07:40:22 MSK 1 day 1h ago homelab-rotating-boot-drill.timer homelab-rotating-boot-drill.service -Mon [PRIVATE_IP] 00:10:03 MSK 6 days Mon [PRIVATE_IP] 00:04:44 MSK 9h ago homelab-secret-sanity.timer homelab-secret-sanity.service -Mon [PRIVATE_IP] 01:16:12 MSK 6 days Mon [PRIVATE_IP] 00:09:14 MSK 9h ago fstrim.timer fstrim.service -- - Mon [PRIVATE_IP] 07:51:03 MSK 1h 25min ago homelab-backup-v2.timer homelab-backup-v2.service -- - - - prometheus-node-exporter-ipmitool-sensor.timer prometheus-node-exporter-ipmitool-sensor.service -- - - - prometheus-node-exporter-mellanox-hca-temp.timer prometheus-node-exporter-mellanox-hca-temp.service -- - - - prometheus-node-exporter-smartmon.timer prometheus-node-exporter-smartmon.service - -58 timers listed. -RUNTIME_MANIFEST_ID=df[PRIVATE_IP]ac9b5abd[PRIVATE_IP]b[PRIVATE_IP]d89496fb2bde[PRIVATE_IP] -EXPERIMENT_ID=5462ee[PRIVATE_IP]ebe[PRIVATE_IP]feb2ad[PRIVATE_IP]1ad87bbac0b8108 -COLLECTION_START_AT=[PRIVATE_IP]T[PRIVATE_IP]:00 -FIRST_COMPLETE_D1_CLOSE=[PRIVATE_IP]T[PRIVATE_IP]:00 -PRODUCTION_LAUNCH_ALLOWED=False -EXECUTION_AUTHORITY=False -RISK_AUTHORITY=False -STAGE_A_HEALTH=AWAITING_PRIMARY_FINALITY -FORENSIC_PREVIOUS_EPOCH_PRESERVED=YES - -## NEWFI_TASK6_AUTH_CHECKPOINT_V[PRIVATE_IP] -- Scope: historical accepted AUTH test, not a current SMTP connection test. -- Newfi approved SMTP account/sender: aleisaev@yandex.ru. -- Endpoint used by accepted test: smtp.yandex.ru:465 with certificate verification; single PLAIN initial response accepted with code 235. -- Evidence: incident ledger ### NEWFI_TASK6_YANDEX_AUTH_V2_OBSERVATION and ### NEWFI_TASK6_AUTH_PROOF_CLOSE_V1. -- No password file written, no application configuration changed, no mail sent by the accepted AUTH test. -- Persistent transport installation and application delivery are not proven by this checkpoint. Task6 is not complete. -- AUTH challenge and missing-file incidents closed; launch/session incident remains OPEN with original cause unproven. -- This update does not change VM211, VM160, routing, board state or Git. -- Other incidents and infrastructure health were not assessed. - - -## NEWFI_PRIVATE_STRUCTURE92_WRONG_MARKER_SCOPE_[PRIVATE_IP] CLOSURE -- Status: CLOSED -- RCA: NEWFI92 inspected the pve01 NEWFI90 attempt marker, which had not been finalized because the outer shell exited immediately after the guest returned RC3 following its successful rollback. The guest marker contained the true final rollback state. -- Correction: NEWFI93 required both the guest FINISHED/ROLLED_BACK record and a fresh exact demo-state readback before reconciling the outer NEWFI90 marker. NEWFI92 itself never reserved an attempt or mutated state. - - -## NEWFI_PRIVATE_NEWFI95_NESTED_TRIPLE_QUOTE_PARSE_[PRIVATE_IP] -- Status: OPEN -- Command: NEWFI95 -- Symptom: after the content-structure precheck passed, the pve03 Python wrapper failed to parse at the PHP line beginning with $root. No Portal discovery or application mutation was reached. -- RCA: an outer raw triple-single-quoted guest Python payload contained an inner raw triple-single-quoted PHP payload. The inner delimiter terminated the outer Python string, causing PHP source to be parsed as Python. -- Safety: NEWFI95 was read-only, created no attempt marker, and did not reach the intended nested guest execution. Do not replay NEWFI95. - - -## NEWFI_PRIVATE_NEWFI95_NESTED_TRIPLE_QUOTE_PARSE_[PRIVATE_IP] CLOSURE -- Status: CLOSED -- RCA: an inner triple-single-quoted PHP literal terminated the outer triple-single-quoted Python guest payload in NEWFI95. The wrapper therefore failed parsing before the intended Portal discovery executed. -- Correction: NEWFI96 removed the nested Python payload layer entirely and passed one Python program directly to VM211 through qm guest exec stdin. The read-only Portal contract completed with QGA exit 0 and the implementation worktree remained unchanged. OUTPUT_END CHAT_OUTPUT_END