CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-cloudb-preflight-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=b508f92361099e24264836513bd255c236078b8a0b80039c6c69016e3bce2085
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=a5e3e41807765a5f0bb7a13406d297e3deebf550a436821efe1df44356f9fa5c
SANITIZED_OUTPUT_SHA256=a5e3e41807765a5f0bb7a13406d297e3deebf550a436821efe1df44356f9fa5c
OUTPUT_BEGIN
WORKERS2_CLOUDB_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1
MODE=read-only
COMPONENT=homelab-backup-cloudb-preflight-rca
MUTATION_BOUNDARY=READ_ONLY_NO_REPO_WRITE_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_NO_DELETE_NO_MAIL
STATE_INVENTORY={"latest_files":["app-skladchik-reports-monitor-pve01-20260921T033135Z.json","app-skladchik-reports-monitor-pve01-20260916T113630Z.json","app-skladchik-reports-monitor-pve01-20260915T223334Z.json","app-skladchik-reports-monitor-pve01-20260915T093500Z.json","app-skladchik-reports-monitor-pve01-20260914T193818Z.json","app-skladchik-reports-monitor-pve01-20260914T063210Z.json","app-skladchik-reports-monitor-pve01-20260913T174140Z.json","app-skladchik-reports-monitor-pve01-20260913T033047Z.json"],"sealed":57,"total":57}
LATEST_SEALED_STATE={"archive_sha256":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140","archive_size_bytes":178434,"cloud_a_verified":true,"cloud_b_verified":true,"cloud_quorum":2,"generation":"app-skladchik-reports-monitor-pve01-20260921T033135Z","local_cleanup":"OK","local_payload_present":false,"logical_id":"app-skladchik-reports-monitor","part_count":1,"part_hashes_verified":true,"parts_manifest_sha256":"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf","payload_name":"app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst","remote_a_base":"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z","remote_b_base":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z","schema":"homelab-cloud-quorum-payload-v1","seal_written":true,"state_file":"app-skladchik-reports-monitor-pve01-20260921T033135Z.json","state_sha256":"41c4e42953c36022ea4f18b5451ec70ea251c86542502444e748545ffa799271","status":"SEALED","types":{"archive_sha256":"str","archive_size_bytes":"int","cloud_a_verified":"bool","cloud_b_verified":"bool","cloud_quorum":"int","generation":"str","local_cleanup":"str","local_payload_present":"bool","logical_id":"str","part_count":"int","part_hashes_verified":"bool","parts_manifest_sha256":"str","payload_name":"str","remote_a_base":"str","remote_b_base":"str","schema":"str","seal_written":"bool","status":"str"}}
STATE_RELEVANT_KEYS=[{"path":"status","type":"str","value":"SEALED"},{"path":"generation","type":"str","value":"app-skladchik-reports-monitor-pve01-20260921T033135Z"},{"path":"payload_path","type":"str","value":"/mnt/staging/homelab-app-cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst"},{"path":"payload_name","type":"str","value":"app-skladchik-reports-monitor-pve01-20260921T033135Z.tar.zst"},{"path":"archive_size_bytes","type":"int","value":178434},{"path":"archive_sha256","type":"str","value":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140"},{"path":"part_count","type":"int","value":1},{"path":"remote_a_root","type":"str","value":"pve01-mail-01-crypt:homelab-backups"},{"path":"remote_b_root","type":"str","value":"pve01-mail-02-crypt:homelab-backups"},{"path":"remote_a_base","type":"str","value":"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z"},{"path":"remote_b_base","type":"str","value":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z"},{"path":"cloud_a_verified","type":"bool","value":true},{"path":"cloud_b_verified","type":"bool","value":true},{"path":"cloud_a_reconstructed_sha256","type":"str","value":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140"},{"path":"cloud_b_reconstructed_sha256","type":"str","value":"c41f0f9f871a6d16ae9ba98c2e40b06f9107082d6f9985dcb2e0a19c34412140"},{"path":"cloud_a_verified_at_utc","type":"str","value":"2026-09-21T03:31:38Z"},{"path":"cloud_b_verified_at_utc","type":"str","value":"2026-09-21T03:31:43Z"},{"path":"cloud_quorum","type":"int","value":2},{"path":"part_hashes_verified","type":"bool","value":true},{"path":"parts_manifest_sha256","type":"str","value":"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf"},{"path":"local_payload_present","type":"bool","value":false},{"path":"cloud_a_seal_verified","type":"bool","value":true},{"path":"cloud_b_seal_verified","type":"bool","value":true},{"path":"cloud_a_seal_verified_at_utc","type":"str","value":"2026-09-21T03:31:48Z"},{"path":"cloud_b_seal_verified_at_utc","type":"str","value":"2026-09-21T03:31:48Z"}]
CLOUDB_STRING_CANDIDATES=[{"path":"remote_b_root","value":"pve01-mail-02-crypt:homelab-backups"},{"path":"remote_b_base","value":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z"}]
RESTORE_EVIDENCE={"exists":true,"file":"app-skladchik-reports-monitor-pve01-20260921T033135Z.json","restore_source":"Cloud A","result":"OK"}
CLOUDB_REMOTE_CHECKS=[{"base":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z","lsf_rc":0,"manifest_candidates":[{"bytes":99,"matches_state_count":true,"matches_state_manifest_sha":true,"matches_state_size":true,"path":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z/PARTS.tsv","rc":0,"rows":1,"sha256":"94b2185cab444b7835946dc3fd009adc7103ac09f1f607e93b9b328dbfeccedf","total_size":178434},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z.parts/PARTS.tsv","rc":3},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T033135Z/parts/PARTS.tsv","rc":3}],"source":"remote_b_base","top_names":["PARTS.tsv","payload.parts/","seal.json"]},{"base":"pve01-mail-02-crypt:homelab-backups","lsf_rc":0,"manifest_candidates":[{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/PARTS.tsv","rc":3},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups.parts/PARTS.tsv","rc":3},{"bytes":0,"path":"pve01-mail-02-crypt:homelab-backups/parts/PARTS.tsv","rc":3}],"source":"remote_b_root","top_names":["app-filebrowser/","app-hbarhub/","app-immich/","app-memos/","app-scrutiny/","app-skladchik-moderator-assistant/","app-skladchik-reports-monitor/","control-bootstrap/","control-bootstrap-v2/","hb/","pve01/","retention-canary/","vm110/","vm111/","vm112/","vm113/","vm130/","vm150/","vm160/","vm170/","vm171/","vm180/","vm190/","vm200/","vm211/","vm9130/"]}]
RCA_CLASSIFICATION={"fix":"REUSE_PROVEN_STATE_AND_MANIFEST_LAYOUT_WITH_CORRECTED_PREFLIGHT","root_cause":"CLOUDB_PREFLIGHT_IMPLEMENTATION_BUG"}
TRANSPORT_SOURCE_FILES=["/usr/local/sbin/homelab-rotating-boot-drill-state-export","/usr/local/sbin/homelab-control-backup","/usr/local/sbin/homelab-rotating-boot-drill","/usr/local/libexec/homelab-rotating-boot-drill-state-export","/usr/local/libexec/homelab-cloud-quorum-payload","/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","/usr/local/libexec/homelab-app-cloud-quorum-worker"]
TRANSPORT_SOURCE_HITS=[{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"31:    if not isinstance(obj.get(\"archive_size_bytes\"),int) or obj[\"archive_size_bytes\"] <= 0: return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"32:    if not isinstance(obj.get(\"part_count\"),int) or obj[\"part_count\"] <= 0: return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"34:    if not isinstance(obj.get(\"parts_manifest_sha256\"),str) or not HEX64.fullmatch(obj[\"parts_manifest_sha256\"]): return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"42:    if obj.get(\"remote_a_base\") != expect_a or obj.get(\"remote_b_base\") != expect_b: return False"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"84:      \"archive_size_bytes\":obj[\"archive_size_bytes\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"86:      \"parts_manifest_sha256\":obj[\"parts_manifest_sha256\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"87:      \"part_count\":obj[\"part_count\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill-state-export","line":"89:      \"remote_b_base\":obj[\"remote_b_base\"],"},{"file":"/usr/local/sbin/homelab-control-backup","line":"72:  expected_manifest=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"; expected_archive=\"$(jq -er '.archive_sha256' \"$state\")\""},{"file":"/usr/local/sbin/homelab-control-backup","line":"73:  manifest=\"${out}.PARTS.tsv\"; : >\"$out\""},{"file":"/usr/local/sbin/homelab-control-backup","line":"74:  rclone --config \"$cfg\" cat \"${base%/}/PARTS.tsv\" >\"$manifest\" || die \"REMOTE_${label}_MANIFEST_READ_FAILED\" 74"},{"file":"/usr/local/sbin/homelab-control-backup","line":"96:  base_a=\"$(jq -er '.remote_a_base' \"$state\")\"; base_b=\"$(jq -er '.remote_b_base' \"$state\")\"; a=\"$work/A.tar.gz\"; b=\"$work/B.tar.gz\""},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"374:    base=info[\"remote_a_base\"] if cloud==\"A\" else info[\"remote_b_base\"]"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"376:    manifest_bytes=rclone_cat(base.rstrip(\"/\")+\"/PARTS.tsv\",60)"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"382:      \"archive_size_bytes\":info[\"archive_size_bytes\"],\"archive_sha256\":info[\"archive_sha256\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"383:      \"part_count\":info[\"part_count\"],\"parts_manifest_sha256\":info[\"parts_manifest_sha256\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"388:    if hashlib.sha256(manifest_bytes).hexdigest()!=info[\"parts_manifest_sha256\"]: die(\"REMOTE_MANIFEST_SHA_MISMATCH_\"+info[\"vmid\"],66)"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"389:    rows=parse_manifest(manifest_bytes,info[\"part_count\"],info[\"archive_size_bytes\"])"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"420:    need_stage=qinfo[\"archive_size_bytes\"]+linfo[\"archive_size_bytes\"]+p[\"minimum_stage_headroom_bytes\"]"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"438:            \"qemu_archive_bytes\":qi[\"archive_size_bytes\"],\"lxc_archive_bytes\":li[\"archive_size_bytes\"],"},{"file":"/usr/local/sbin/homelab-rotating-boot-drill","line":"583:    if total!=info[\"archive_size_bytes\"] or h.hexdigest()!=info[\"archive_sha256\"]: die(\"ARCHIVE_VERIFY_FAILED_\"+info[\"vmid\"],70)"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"31:    if not isinstance(obj.get(\"archive_size_bytes\"),int) or obj[\"archive_size_bytes\"] <= 0: return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"32:    if not isinstance(obj.get(\"part_count\"),int) or obj[\"part_count\"] <= 0: return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"34:    if not isinstance(obj.get(\"parts_manifest_sha256\"),str) or not HEX64.fullmatch(obj[\"parts_manifest_sha256\"]): return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"42:    if obj.get(\"remote_a_base\") != expect_a or obj.get(\"remote_b_base\") != expect_b: return False"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"84:      \"archive_size_bytes\":obj[\"archive_size_bytes\"],"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"86:      \"parts_manifest_sha256\":obj[\"parts_manifest_sha256\"],"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"87:      \"part_count\":obj[\"part_count\"],"},{"file":"/usr/local/libexec/homelab-rotating-boot-drill-state-export","line":"89:      \"remote_b_base\":obj[\"remote_b_base\"],"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"198:    --arg remote_b_base \"$REMOTE_B_BASE\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"201:    --argjson part_count \"$PART_COUNT\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"205:      archive_size_bytes:$payload_size,archive_sha256:$payload_sha,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"206:      chunk_size_bytes:$chunk_size,part_count:$part_count,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"208:      remote_a_base:$remote_a_base,remote_b_base:$remote_b_base,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"212:      cloud_quorum:0,part_hashes_verified:false,parts_manifest_sha256:null,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"230:   --arg remote_b_base \"$REMOTE_B_BASE\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"232:    --argjson part_count \"$PART_COUNT\" \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"235:     .archive_size_bytes==$payload_size and"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"237:     .part_count==$part_count and"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"239:     .remote_b_base==$remote_b_base' \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"248:  PAYLOAD_SIZE=\"$(jq -er '.archive_size_bytes | numbers | select(.>0 and floor==.)' \"$STATE_FILE\")\" || die \"STATE_PAYLOAD_SIZE_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"249:  PART_COUNT=\"$(jq -er '.part_count | numbers | select(.>0 and floor==.)' \"$STATE_FILE\")\" || die \"STATE_PART_COUNT_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"452:     .parts_manifest_sha256=$manifest_sha' \\"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"463:  saved_manifest=\"$(jq -er '.parts_manifest_sha256 | strings | select(test(\"^[0-9a-f]{64}$\"))' \"$STATE_FILE\")\" || die \"STATE_PARTS_MANIFEST_SHA_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"535:      .parts_manifest_sha256==null"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"540:      (.parts_manifest_sha256 |"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"560:PARTS_MANIFEST=\"$WORK/PARTS.tsv\""},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"636:  printf '%s\\n' \"${base%/}/PARTS.tsv\""},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"687:  PARTS_MANIFEST_SHA=\"$(jq -er '.parts_manifest_sha256 | strings | select(test(\"^[0-9a-f]{64}$\"))' \"$STATE_FILE\")\" || die \"STATE_PARTS_MANIFEST_SHA_INVALID\" 65"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"700:  remote_manifest=\"$WORK/remote-${label}-PARTS.tsv\""},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"836:    archive_size_bytes:.archive_size_bytes,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"839:    part_count:.part_count,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"840:    parts_manifest_sha256:.parts_manifest_sha256,"},{"file":"/usr/local/libexec/homelab-cloud-quorum-payload","line":"856:    (.parts_manifest_sha256|type==\"string\" and test(\"^[0-9a-f]{64}$\")) and"},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"89:  base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"; expected_count=\"$(jq -er '.part_count' \"$state\")\"; expected_size=\"$(jq -er '.archive_size_bytes' \"$state\")\""},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"91:  manifest=\"$work/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\""},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"121:  payload_name=\"$(jq -er '.payload_name' \"$state\")\"; expected_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; expected_size=\"$(jq -er '.archive_size_bytes' \"$state\")\""},{"file":"/usr/local/libexec/homelab-vm-cloud-quorum-node-worker","line":"124:  base=\"$(prepare_cloud_a_manifest \"$vmid\" \"$gen\" \"$state\" \"$work\")\"; manifest=\"$work/PARTS.tsv\"; fifo=\"$work/archive.sha.fifo\"; sha_out=\"$work/archive.sha256\"; mkfifo -m 0600 \"$fifo\""},{"file":"/usr/local/libexec/homelab-app-cloud-quorum-worker","line":"352:  base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\""},{"file":"/usr/local/libexec/homelab-app-cloud-quorum-worker","line":"353:  manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\""}]
NO_MUTATION=true
NO_BACKUP_STARTED=true
NO_RESTORE_STARTED=true
NO_RETENTION_STARTED=true
NO_DELETE=true
NO_MAIL_SENT=true
TASK_RESULT=PASS_HOMELAB_BACKUP_CLOUDB_PREFLIGHT_RCA_READONLY
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-CLOUDB-PREFLIGHT-RCA-READONLY-1064R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
WORKERS2_CLOUDB_RCA_END=1

OUTPUT_END
CHAT_OUTPUT_END
