diff --git a/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1.json b/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1.json new file mode 100644 index 00000000..16739b13 --- /dev/null +++ b/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "infra-dr-backupv2-control-380-health-root-workspace-rca-fix-closure", + "started_at_utc": "2026-09-14T19:49:16Z", + "finished_at_utc": "2026-09-14T19:49:29Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "2d627054d90c1adc89eb735594479df46650a069d5bc3ee272945cfe3fa2e834", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "49037e8b0cca7de6424bf838d14a43d44ffe99e581bfd69d678bc1f40a2be554", + "sanitized_output_sha256": "49037e8b0cca7de6424bf838d14a43d44ffe99e581bfd69d678bc1f40a2be554", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1\nMODE=READ_ONLY_CONTROL_380_HEALTH_ROOT_WORKSPACE_RCA_FIX_CLOSURE\nANCHOR_379_COMMIT=96cfc84f65e5867d6085b03a5f71cf2170f6e4f3\nANCHOR_380_COMMIT=c52aeb801d354c97299c03cdb8cec9e83c340085\nEXPECTED_GENERATION=control-bootstrap-v2-pve01-20260914T080740Z\nNO_FORCE=YES\nNO_VERIFY=YES\nNO_BACKUP_DELETE=YES\nNO_RCLONE=YES\nNO_SYSTEMCTL_MUTATION=YES\nCURRENT_ID_GITEA_FRESHNESS=404_404_PASS\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nERROR_REGISTER_FRESH_READ=OK\nERROR_REGISTER_PROVENANCE_MATCH=YES\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nREFERENCE_FRESH_READ=OK\nREFERENCE_PROVENANCE_MATCH=YES\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nRUNNER_SHA_MATCH=YES\nCONTROL_ENGINE_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57\nCONTROL_ENGINE_SHA_MATCH=YES\nANCHOR_378_RUNTIME_IDENTITY=PASS\nANCHOR_379_RUNTIME_IDENTITY=PASS\nENGINE_VERIFY_OPERATIONAL_SUCCESS_ANCHORED=PASS\nENGINE_VERIFY_RECONSTRUCT_A_ANCHORED=PASS\nENGINE_VERIFY_RECONSTRUCT_B_ANCHORED=PASS\nENGINE_VERIFY_COMPARE_AB_ANCHORED=PASS\nENGINE_VERIFY_CLOUD_MUTATIONS_ANCHORED=ZERO\nANCHOR_380_RUNTIME_IDENTITY=PASS\nRCA_380_HEALTH_ROOT_OMISSION_ANCHORED=PASS\nSTATE_SCAN_ROOT_COUNT=3\nSTATE_SCAN_ROOT_1=/var/lib/homelab-backup\nSTATE_SCAN_ROOT_2=/var/lib/homelab-backup-v2\nSTATE_SCAN_ROOT_3=/var/lib/homelab-health\nSTATE_JSON_SCANNED=1562\nSTATE_JSON_PARSE_ERRORS=0\nGENERATION_MATCH_FILE_COUNT=3\nWINDOW_RELEVANT_FILE_COUNT=2\nEVIDENCE_IDENTITY_SUCCESS_FILE_COUNT=2\nHEALTH_BINDING_FILE_COUNT=1\nLEGACY_SCHEMA_MISMATCH_CONFIRMED_FILE_COUNT=1\nDOC_1_PATH=/var/lib/homelab-backup/control-restore-validation/control-bootstrap/control-bootstrap-v2-pve01-20260914T080740Z.json\nDOC_1_MTIME_UTC=2026-09-14T19:12:40.777301Z\nDOC_1_IN_379_WINDOW=YES\nDOC_1_HEALTHISH=NO\nDOC_1_RESTOREISH=YES\nDOC_1_LEGACY_TOP_STATUS_PRESENT=NO\nDOC_1_LEGACY_TOP_GENERATION_EXACT=YES\nDOC_1_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=NO\nDOC_1_GENERATION_MATCH_PATHS=[\"/generation\"]\nDOC_1_ARCHIVE_SHA_MATCH_PATHS=[\"/payload_sha256\"]\nDOC_1_SUCCESS_SEMANTIC_PATHS=[\"/cloud_a_semantic_restore_verified\",\"/cloud_b_byte_reconstruction_verified\",\"/control_manifest_verified\",\"/desired_git_bundle_verified\",\"/ops_git_bundle_verified\",\"/payload_checksums_verified\",\"/payload_sha256_verified\",\"/result\",\"/verified_at_utc\"]\nDOC_1_SCHEMA_LEAF_PATHS=[\"/clean_machine_bootstrap_proven\",\"/cloud_a_b_payload_equal\",\"/cloud_a_semantic_restore_verified\",\"/cloud_b_byte_reconstruction_verified\",\"/control_manifest_verified\",\"/control_scope\",\"/desired_git_bundle_verified\",\"/generation\",\"/logical_id\",\"/ops_git_bundle_verified\",\"/payload_checksums_verified\",\"/payload_sha256\",\"/payload_sha256_verified\",\"/restore_source\",\"/result\",\"/runtime_asset_count_verified\",\"/secret_escrow_separate\",\"/systemd_asset_count_verified\",\"/verified_at_utc\"]\nDOC_1_SCHEMA_PATHS_TRUNCATED=NO\nDOC_2_PATH=/var/lib/homelab-health/control-backup.json\nDOC_2_MTIME_UTC=2026-09-14T19:12:40.781301Z\nDOC_2_IN_379_WINDOW=YES\nDOC_2_HEALTHISH=YES\nDOC_2_RESTOREISH=YES\nDOC_2_LEGACY_TOP_STATUS_PRESENT=YES\nDOC_2_LEGACY_TOP_GENERATION_EXACT=YES\nDOC_2_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=YES\nDOC_2_GENERATION_MATCH_PATHS=[\"/generation\"]\nDOC_2_ARCHIVE_SHA_MATCH_PATHS=[\"/archive_sha256\"]\nDOC_2_SUCCESS_SEMANTIC_PATHS=[\"/status\"]\nDOC_2_SCHEMA_LEAF_PATHS=[\"/archive_sha256\",\"/cloud_quorum\",\"/control_freshness\",\"/full_site_dr_proven\",\"/generation\",\"/restore_evidence\",\"/schema\",\"/state_file\",\"/status\",\"/updated_at_utc\"]\nDOC_2_SCHEMA_PATHS_TRUNCATED=NO\nDOC_3_PATH=/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2/control-bootstrap-v2-pve01-20260914T080740Z.json\nDOC_3_MTIME_UTC=2026-09-14T08:07:54.154762Z\nDOC_3_IN_379_WINDOW=NO\nDOC_3_HEALTHISH=NO\nDOC_3_RESTOREISH=NO\nDOC_3_LEGACY_TOP_STATUS_PRESENT=YES\nDOC_3_LEGACY_TOP_GENERATION_EXACT=YES\nDOC_3_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=YES\nDOC_3_GENERATION_MATCH_PATHS=[\"/generation\"]\nDOC_3_ARCHIVE_SHA_MATCH_PATHS=[\"/archive_sha256\",\"/cloud_a_reconstructed_sha256\",\"/cloud_b_reconstructed_sha256\"]\nDOC_3_SUCCESS_SEMANTIC_PATHS=[\"/cloud_a_seal_verified\",\"/cloud_a_seal_verified_at_utc\",\"/cloud_a_verified\",\"/cloud_a_verified_at_utc\",\"/cloud_b_seal_verified\",\"/cloud_b_seal_verified_at_utc\",\"/cloud_b_verified\",\"/cloud_b_verified_at_utc\",\"/part_hashes_verified\",\"/status\"]\nDOC_3_SCHEMA_LEAF_PATHS=[\"/archive_sha256\",\"/archive_size_bytes\",\"/chunk_size_bytes\",\"/cloud_a_reconstructed_sha256\",\"/cloud_a_seal_verified\",\"/cloud_a_seal_verified_at_utc\",\"/cloud_a_verified\",\"/cloud_a_verified_at_utc\",\"/cloud_b_reconstructed_sha256\",\"/cloud_b_seal_verified\",\"/cloud_b_seal_verified_at_utc\",\"/cloud_b_verified\",\"/cloud_b_verified_at_utc\",\"/cloud_quorum\",\"/created_at_utc\",\"/delete_authorized_at_utc\",\"/generation\",\"/local_cleanup\",\"/local_delete_allowed\",\"/local_deleted_at_utc\",\"/local_payload_present\",\"/logical_id\",\"/part_count\",\"/part_hashes_verified\",\"/parts_manifest_sha256\",\"/payload_name\",\"/payload_path\",\"/remote_a_base\",\"/remote_a_root\",\"/remote_b_base\",\"/remote_b_root\",\"/schema\",\"/seal_sha256\",\"/seal_written\",\"/sealed_at_utc\",\"/status\",\"/updated_at_utc\"]\nDOC_3_SCHEMA_PATHS_TRUNCATED=NO\nVERIFY_WORKSPACE_DIRS_SCANNED=72\nVERIFY_WORKSPACE_RESIDUAL_COUNT=0\nRCA_379=WRAPPER_ASSUMED_TOP_LEVEL_STATUS_AND_ARCHIVE_SHA256_FIELDS_NOT_PRESENT_IN_ACTUAL_EVIDENCE_SCHEMA\nRCA_380=STATE_SCAN_OMITTED_VAR_LIB_HOMELAB_HEALTH_AND_WORKSPACE_SCANNER_USED_BROAD_NAME_MATCHES\nRCA_380_HEALTH_SCAN_ROOT_FIX=PASS\nRCA_380_VERIFY_WORKSPACE_SCOPE_FIX=PASS\nACTUAL_EVIDENCE_SCHEMA_IDENTITY_BINDING=PASS\nACTUAL_EVIDENCE_379_WRITE_WINDOW_BINDING=PASS\nACTUAL_HEALTH_379_WRITE_WINDOW_BINDING=PASS\nCONTROL_PRODUCTION_V2_GENERATION=PROVEN\nCONTROL_PRODUCTION_V2_RESTORE=PROVEN\nBACKUP_CONTROL_PLANE_PRODUCTION_V2_COVERAGE_COMPLETE=YES\nCONTROL_PLANE_FORMAL_CLOSURE_METHOD=READ_ONLY_ACCEPTANCE_OF_EXISTING_378_STATE_PLUS_379_SUCCESSFUL_VERIFY_EVIDENCE\nNEXT_GATE=BACKUPV2_FINAL_COVERAGE_REGISTRY_REAUDIT\nSTATUS=OK\nPHASE=COMPLETE\nREASON=NONE\nCHANGES_MADE=false\nMUTATION_OUTCOME=NO_MUTATION\nCLOUD_WRITES=0\nCLOUD_DELETIONS=0\nBACKUP_EXECUTIONS=0\nCONTROL_ENGINE_EXECUTIONS=0\nSYSTEMD_MUTATIONS=0\nSCHEDULER_CUTOVER=NO\nLEGACY_BACKUP_DISABLES=0\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1.txt b/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1.txt new file mode 100644 index 00000000..ab5c8dc3 --- /dev/null +++ b/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1.txt @@ -0,0 +1,139 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=infra-dr-backupv2-control-380-health-root-workspace-rca-fix-closure +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=2d627054d90c1adc89eb735594479df46650a069d5bc3ee272945cfe3fa2e834 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=49037e8b0cca7de6424bf838d14a43d44ffe99e581bfd69d678bc1f40a2be554 +SANITIZED_OUTPUT_SHA256=49037e8b0cca7de6424bf838d14a43d44ffe99e581bfd69d678bc1f40a2be554 +OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1 +MODE=READ_ONLY_CONTROL_380_HEALTH_ROOT_WORKSPACE_RCA_FIX_CLOSURE +ANCHOR_379_COMMIT=96cfc84f65e5867d6085b03a5f71cf2170f6e4f3 +ANCHOR_380_COMMIT=c52aeb801d354c97299c03cdb8cec9e83c340085 +EXPECTED_GENERATION=control-bootstrap-v2-pve01-20260914T080740Z +NO_FORCE=YES +NO_VERIFY=YES +NO_BACKUP_DELETE=YES +NO_RCLONE=YES +NO_SYSTEMCTL_MUTATION=YES +CURRENT_ID_GITEA_FRESHNESS=404_404_PASS +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +ERROR_REGISTER_FRESH_READ=OK +ERROR_REGISTER_PROVENANCE_MATCH=YES +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +REFERENCE_FRESH_READ=OK +REFERENCE_PROVENANCE_MATCH=YES +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +RUNNER_SHA_MATCH=YES +CONTROL_ENGINE_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57 +CONTROL_ENGINE_SHA_MATCH=YES +ANCHOR_378_RUNTIME_IDENTITY=PASS +ANCHOR_379_RUNTIME_IDENTITY=PASS +ENGINE_VERIFY_OPERATIONAL_SUCCESS_ANCHORED=PASS +ENGINE_VERIFY_RECONSTRUCT_A_ANCHORED=PASS +ENGINE_VERIFY_RECONSTRUCT_B_ANCHORED=PASS +ENGINE_VERIFY_COMPARE_AB_ANCHORED=PASS +ENGINE_VERIFY_CLOUD_MUTATIONS_ANCHORED=ZERO +ANCHOR_380_RUNTIME_IDENTITY=PASS +RCA_380_HEALTH_ROOT_OMISSION_ANCHORED=PASS +STATE_SCAN_ROOT_COUNT=3 +STATE_SCAN_ROOT_1=/var/lib/homelab-backup +STATE_SCAN_ROOT_2=/var/lib/homelab-backup-v2 +STATE_SCAN_ROOT_3=/var/lib/homelab-health +STATE_JSON_SCANNED=1562 +STATE_JSON_PARSE_ERRORS=0 +GENERATION_MATCH_FILE_COUNT=3 +WINDOW_RELEVANT_FILE_COUNT=2 +EVIDENCE_IDENTITY_SUCCESS_FILE_COUNT=2 +HEALTH_BINDING_FILE_COUNT=1 +LEGACY_SCHEMA_MISMATCH_CONFIRMED_FILE_COUNT=1 +DOC_1_PATH=/var/lib/homelab-backup/control-restore-validation/control-bootstrap/control-bootstrap-v2-pve01-20260914T080740Z.json +DOC_1_MTIME_UTC=2026-09-14T19:12:40.777301Z +DOC_1_IN_379_WINDOW=YES +DOC_1_HEALTHISH=NO +DOC_1_RESTOREISH=YES +DOC_1_LEGACY_TOP_STATUS_PRESENT=NO +DOC_1_LEGACY_TOP_GENERATION_EXACT=YES +DOC_1_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=NO +DOC_1_GENERATION_MATCH_PATHS=["/generation"] +DOC_1_ARCHIVE_SHA_MATCH_PATHS=["/payload_sha256"] +DOC_1_SUCCESS_SEMANTIC_PATHS=["/cloud_a_semantic_restore_verified","/cloud_b_byte_reconstruction_verified","/control_manifest_verified","/desired_git_bundle_verified","/ops_git_bundle_verified","/payload_checksums_verified","/payload_sha256_verified","/result","/verified_at_utc"] +DOC_1_SCHEMA_LEAF_PATHS=["/clean_machine_bootstrap_proven","/cloud_a_b_payload_equal","/cloud_a_semantic_restore_verified","/cloud_b_byte_reconstruction_verified","/control_manifest_verified","/control_scope","/desired_git_bundle_verified","/generation","/logical_id","/ops_git_bundle_verified","/payload_checksums_verified","/payload_sha256","/payload_sha256_verified","/restore_source","/result","/runtime_asset_count_verified","/secret_escrow_separate","/systemd_asset_count_verified","/verified_at_utc"] +DOC_1_SCHEMA_PATHS_TRUNCATED=NO +DOC_2_PATH=/var/lib/homelab-health/control-backup.json +DOC_2_MTIME_UTC=2026-09-14T19:12:40.781301Z +DOC_2_IN_379_WINDOW=YES +DOC_2_HEALTHISH=YES +DOC_2_RESTOREISH=YES +DOC_2_LEGACY_TOP_STATUS_PRESENT=YES +DOC_2_LEGACY_TOP_GENERATION_EXACT=YES +DOC_2_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=YES +DOC_2_GENERATION_MATCH_PATHS=["/generation"] +DOC_2_ARCHIVE_SHA_MATCH_PATHS=["/archive_sha256"] +DOC_2_SUCCESS_SEMANTIC_PATHS=["/status"] +DOC_2_SCHEMA_LEAF_PATHS=["/archive_sha256","/cloud_quorum","/control_freshness","/full_site_dr_proven","/generation","/restore_evidence","/schema","/state_file","/status","/updated_at_utc"] +DOC_2_SCHEMA_PATHS_TRUNCATED=NO +DOC_3_PATH=/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2/control-bootstrap-v2-pve01-20260914T080740Z.json +DOC_3_MTIME_UTC=2026-09-14T08:07:54.154762Z +DOC_3_IN_379_WINDOW=NO +DOC_3_HEALTHISH=NO +DOC_3_RESTOREISH=NO +DOC_3_LEGACY_TOP_STATUS_PRESENT=YES +DOC_3_LEGACY_TOP_GENERATION_EXACT=YES +DOC_3_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=YES +DOC_3_GENERATION_MATCH_PATHS=["/generation"] +DOC_3_ARCHIVE_SHA_MATCH_PATHS=["/archive_sha256","/cloud_a_reconstructed_sha256","/cloud_b_reconstructed_sha256"] +DOC_3_SUCCESS_SEMANTIC_PATHS=["/cloud_a_seal_verified","/cloud_a_seal_verified_at_utc","/cloud_a_verified","/cloud_a_verified_at_utc","/cloud_b_seal_verified","/cloud_b_seal_verified_at_utc","/cloud_b_verified","/cloud_b_verified_at_utc","/part_hashes_verified","/status"] +DOC_3_SCHEMA_LEAF_PATHS=["/archive_sha256","/archive_size_bytes","/chunk_size_bytes","/cloud_a_reconstructed_sha256","/cloud_a_seal_verified","/cloud_a_seal_verified_at_utc","/cloud_a_verified","/cloud_a_verified_at_utc","/cloud_b_reconstructed_sha256","/cloud_b_seal_verified","/cloud_b_seal_verified_at_utc","/cloud_b_verified","/cloud_b_verified_at_utc","/cloud_quorum","/created_at_utc","/delete_authorized_at_utc","/generation","/local_cleanup","/local_delete_allowed","/local_deleted_at_utc","/local_payload_present","/logical_id","/part_count","/part_hashes_verified","/parts_manifest_sha256","/payload_name","/payload_path","/remote_a_base","/remote_a_root","/remote_b_base","/remote_b_root","/schema","/seal_sha256","/seal_written","/sealed_at_utc","/status","/updated_at_utc"] +DOC_3_SCHEMA_PATHS_TRUNCATED=NO +VERIFY_WORKSPACE_DIRS_SCANNED=72 +VERIFY_WORKSPACE_RESIDUAL_COUNT=0 +RCA_379=WRAPPER_ASSUMED_TOP_LEVEL_STATUS_AND_ARCHIVE_SHA256_FIELDS_NOT_PRESENT_IN_ACTUAL_EVIDENCE_SCHEMA +RCA_380=STATE_SCAN_OMITTED_VAR_LIB_HOMELAB_HEALTH_AND_WORKSPACE_SCANNER_USED_BROAD_NAME_MATCHES +RCA_380_HEALTH_SCAN_ROOT_FIX=PASS +RCA_380_VERIFY_WORKSPACE_SCOPE_FIX=PASS +ACTUAL_EVIDENCE_SCHEMA_IDENTITY_BINDING=PASS +ACTUAL_EVIDENCE_379_WRITE_WINDOW_BINDING=PASS +ACTUAL_HEALTH_379_WRITE_WINDOW_BINDING=PASS +CONTROL_PRODUCTION_V2_GENERATION=PROVEN +CONTROL_PRODUCTION_V2_RESTORE=PROVEN +BACKUP_CONTROL_PLANE_PRODUCTION_V2_COVERAGE_COMPLETE=YES +CONTROL_PLANE_FORMAL_CLOSURE_METHOD=READ_ONLY_ACCEPTANCE_OF_EXISTING_378_STATE_PLUS_379_SUCCESSFUL_VERIFY_EVIDENCE +NEXT_GATE=BACKUPV2_FINAL_COVERAGE_REGISTRY_REAUDIT +STATUS=OK +PHASE=COMPLETE +REASON=NONE +CHANGES_MADE=false +MUTATION_OUTCOME=NO_MUTATION +CLOUD_WRITES=0 +CLOUD_DELETIONS=0 +BACKUP_EXECUTIONS=0 +CONTROL_ENGINE_EXECUTIONS=0 +SYSTEMD_MUTATIONS=0 +SCHEDULER_CUTOVER=NO +LEGACY_BACKUP_DISABLES=0 +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 11505953..16739b13 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-379-RESTORE-EVIDENCE-SCHEMA-RCA-380R1", - "status": "FAIL", - "rc": 2, + "command_id": "SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1", + "status": "OK", + "rc": 0, "host": "pve01", "mode": "read-only", - "component": "infra-dr-backupv2-control-379-restore-evidence-schema-rca", - "started_at_utc": "2026-09-14T19:40:43Z", - "finished_at_utc": "2026-09-14T19:40:51Z", + "component": "infra-dr-backupv2-control-380-health-root-workspace-rca-fix-closure", + "started_at_utc": "2026-09-14T19:49:16Z", + "finished_at_utc": "2026-09-14T19:49:29Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "906899503fcb26cfb7b293778f5f360a515d5dc03ffba1b5bf571f62031cab69", + "command_sha256": "2d627054d90c1adc89eb735594479df46650a069d5bc3ee272945cfe3fa2e834", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -21,7 +21,7 @@ "result_contract_valid": true, "result_contract_status": null, "result_contract_error": null, - "command_rc": 2, + "command_rc": 0, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "cc2acde743d0ab26b6aedc615503f3819410cf76dd5b2f121de38f8db1a730bf", - "sanitized_output_sha256": "cc2acde743d0ab26b6aedc615503f3819410cf76dd5b2f121de38f8db1a730bf", + "raw_evidence_sha256": "49037e8b0cca7de6424bf838d14a43d44ffe99e581bfd69d678bc1f40a2be554", + "sanitized_output_sha256": "49037e8b0cca7de6424bf838d14a43d44ffe99e581bfd69d678bc1f40a2be554", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-379-RESTORE-EVIDENCE-SCHEMA-RCA-380R1\nMODE=READ_ONLY_CONTROL_379_RESTORE_EVIDENCE_SCHEMA_RCA\nANCHOR_379_COMMIT=96cfc84f65e5867d6085b03a5f71cf2170f6e4f3\nEXPECTED_GENERATION=control-bootstrap-v2-pve01-20260914T080740Z\nNO_FORCE=YES\nNO_VERIFY=YES\nNO_BACKUP_DELETE=YES\nNO_RCLONE=YES\nNO_SYSTEMCTL_MUTATION=YES\nCURRENT_ID_GITEA_FRESHNESS=404_404_PASS\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nERROR_REGISTER_FRESH_READ=OK\nERROR_REGISTER_PROVENANCE_MATCH=YES\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nREFERENCE_FRESH_READ=OK\nREFERENCE_PROVENANCE_MATCH=YES\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nRUNNER_SHA_MATCH=YES\nCONTROL_ENGINE_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57\nCONTROL_ENGINE_SHA_MATCH=YES\nANCHOR_378_RUNTIME_IDENTITY=PASS\nANCHOR_379_RUNTIME_IDENTITY=PASS\nENGINE_VERIFY_OPERATIONAL_SUCCESS_ANCHORED=PASS\nENGINE_VERIFY_RECONSTRUCT_A_ANCHORED=PASS\nENGINE_VERIFY_RECONSTRUCT_B_ANCHORED=PASS\nENGINE_VERIFY_COMPARE_AB_ANCHORED=PASS\nENGINE_VERIFY_CLOUD_MUTATIONS_ANCHORED=ZERO\nSTATE_SCAN_ROOT_COUNT=2\nSTATE_SCAN_ROOT_1=/var/lib/homelab-backup\nSTATE_SCAN_ROOT_2=/var/lib/homelab-backup-v2\nSTATE_JSON_SCANNED=1540\nSTATE_JSON_PARSE_ERRORS=0\nGENERATION_MATCH_FILE_COUNT=2\nWINDOW_RELEVANT_FILE_COUNT=1\nEVIDENCE_IDENTITY_SUCCESS_FILE_COUNT=1\nHEALTH_BINDING_FILE_COUNT=0\nLEGACY_SCHEMA_MISMATCH_CONFIRMED_FILE_COUNT=1\nDOC_1_PATH=/var/lib/homelab-backup/control-restore-validation/control-bootstrap/control-bootstrap-v2-pve01-20260914T080740Z.json\nDOC_1_MTIME_UTC=2026-09-14T19:12:40.777301Z\nDOC_1_IN_379_WINDOW=YES\nDOC_1_HEALTHISH=NO\nDOC_1_RESTOREISH=YES\nDOC_1_LEGACY_TOP_STATUS_PRESENT=NO\nDOC_1_LEGACY_TOP_GENERATION_EXACT=YES\nDOC_1_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=NO\nDOC_1_GENERATION_MATCH_PATHS=[\"/generation\"]\nDOC_1_ARCHIVE_SHA_MATCH_PATHS=[\"/payload_sha256\"]\nDOC_1_SUCCESS_SEMANTIC_PATHS=[\"/cloud_a_semantic_restore_verified\",\"/cloud_b_byte_reconstruction_verified\",\"/control_manifest_verified\",\"/desired_git_bundle_verified\",\"/ops_git_bundle_verified\",\"/payload_checksums_verified\",\"/payload_sha256_verified\",\"/result\",\"/verified_at_utc\"]\nDOC_1_SCHEMA_LEAF_PATHS=[\"/clean_machine_bootstrap_proven\",\"/cloud_a_b_payload_equal\",\"/cloud_a_semantic_restore_verified\",\"/cloud_b_byte_reconstruction_verified\",\"/control_manifest_verified\",\"/control_scope\",\"/desired_git_bundle_verified\",\"/generation\",\"/logical_id\",\"/ops_git_bundle_verified\",\"/payload_checksums_verified\",\"/payload_sha256\",\"/payload_sha256_verified\",\"/restore_source\",\"/result\",\"/runtime_asset_count_verified\",\"/secret_escrow_separate\",\"/systemd_asset_count_verified\",\"/verified_at_utc\"]\nDOC_1_SCHEMA_PATHS_TRUNCATED=NO\nDOC_2_PATH=/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2/control-bootstrap-v2-pve01-20260914T080740Z.json\nDOC_2_MTIME_UTC=2026-09-14T08:07:54.154762Z\nDOC_2_IN_379_WINDOW=NO\nDOC_2_HEALTHISH=NO\nDOC_2_RESTOREISH=NO\nDOC_2_LEGACY_TOP_STATUS_PRESENT=YES\nDOC_2_LEGACY_TOP_GENERATION_EXACT=YES\nDOC_2_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=YES\nDOC_2_GENERATION_MATCH_PATHS=[\"/generation\"]\nDOC_2_ARCHIVE_SHA_MATCH_PATHS=[\"/archive_sha256\",\"/cloud_a_reconstructed_sha256\",\"/cloud_b_reconstructed_sha256\"]\nDOC_2_SUCCESS_SEMANTIC_PATHS=[\"/cloud_a_seal_verified\",\"/cloud_a_seal_verified_at_utc\",\"/cloud_a_verified\",\"/cloud_a_verified_at_utc\",\"/cloud_b_seal_verified\",\"/cloud_b_seal_verified_at_utc\",\"/cloud_b_verified\",\"/cloud_b_verified_at_utc\",\"/part_hashes_verified\",\"/status\"]\nDOC_2_SCHEMA_LEAF_PATHS=[\"/archive_sha256\",\"/archive_size_bytes\",\"/chunk_size_bytes\",\"/cloud_a_reconstructed_sha256\",\"/cloud_a_seal_verified\",\"/cloud_a_seal_verified_at_utc\",\"/cloud_a_verified\",\"/cloud_a_verified_at_utc\",\"/cloud_b_reconstructed_sha256\",\"/cloud_b_seal_verified\",\"/cloud_b_seal_verified_at_utc\",\"/cloud_b_verified\",\"/cloud_b_verified_at_utc\",\"/cloud_quorum\",\"/created_at_utc\",\"/delete_authorized_at_utc\",\"/generation\",\"/local_cleanup\",\"/local_delete_allowed\",\"/local_deleted_at_utc\",\"/local_payload_present\",\"/logical_id\",\"/part_count\",\"/part_hashes_verified\",\"/parts_manifest_sha256\",\"/payload_name\",\"/payload_path\",\"/remote_a_base\",\"/remote_a_root\",\"/remote_b_base\",\"/remote_b_root\",\"/schema\",\"/seal_sha256\",\"/seal_written\",\"/sealed_at_utc\",\"/status\",\"/updated_at_utc\"]\nDOC_2_SCHEMA_PATHS_TRUNCATED=NO\nVERIFY_WORKSPACE_DIRS_SCANNED=1518\nVERIFY_WORKSPACE_RESIDUAL_COUNT=7\nVERIFY_WORKSPACE_RESIDUAL_1=/mnt/staging/homelab-control-verify\nVERIFY_WORKSPACE_RESIDUAL_2=/tmp/homelab-mkdocs-site/docs/02-backup-and-restore\nVERIFY_WORKSPACE_RESIDUAL_3=/tmp/homelab-mkdocs-site/reports/backup-restore-dr-audit-20260622\nVERIFY_WORKSPACE_RESIDUAL_4=/tmp/homelab-mkdocs-site/reports/backup-restore-table-refresh-20260622\nVERIFY_WORKSPACE_RESIDUAL_5=/tmp/homelab-mkdocs-site/reports/netbox-backup-offhost-restore-patch-failed-recovered-20260622\nVERIFY_WORKSPACE_RESIDUAL_6=/tmp/homelab-mkdocs-site/reports/netbox-backup-offhost-restore-patch-v2-20260622\nVERIFY_WORKSPACE_RESIDUAL_7=/tmp/homelab-mkdocs-site/reports/netbox-backup-offhost-restore-proof-audit-20260622\nSTATUS=FAIL\nPHASE=HEALTH_STATE\nREASON=379_HEALTH_BINDING_NOT_PROVEN\nCHANGES_MADE=false\nMUTATION_OUTCOME=NO_MUTATION\nCLOUD_WRITES=0\nCLOUD_DELETIONS=0\nBACKUP_EXECUTIONS=0\nCONTROL_ENGINE_EXECUTIONS=0\nSYSTEMD_MUTATIONS=0\nSCHEDULER_CUTOVER=NO\nLEGACY_BACKUP_DISABLES=0\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-379-RESTORE-EVIDENCE-SCHEMA-RCA-380R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1\nMODE=READ_ONLY_CONTROL_380_HEALTH_ROOT_WORKSPACE_RCA_FIX_CLOSURE\nANCHOR_379_COMMIT=96cfc84f65e5867d6085b03a5f71cf2170f6e4f3\nANCHOR_380_COMMIT=c52aeb801d354c97299c03cdb8cec9e83c340085\nEXPECTED_GENERATION=control-bootstrap-v2-pve01-20260914T080740Z\nNO_FORCE=YES\nNO_VERIFY=YES\nNO_BACKUP_DELETE=YES\nNO_RCLONE=YES\nNO_SYSTEMCTL_MUTATION=YES\nCURRENT_ID_GITEA_FRESHNESS=404_404_PASS\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nERROR_REGISTER_FRESH_READ=OK\nERROR_REGISTER_PROVENANCE_MATCH=YES\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nREFERENCE_FRESH_READ=OK\nREFERENCE_PROVENANCE_MATCH=YES\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nRUNNER_SHA_MATCH=YES\nCONTROL_ENGINE_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57\nCONTROL_ENGINE_SHA_MATCH=YES\nANCHOR_378_RUNTIME_IDENTITY=PASS\nANCHOR_379_RUNTIME_IDENTITY=PASS\nENGINE_VERIFY_OPERATIONAL_SUCCESS_ANCHORED=PASS\nENGINE_VERIFY_RECONSTRUCT_A_ANCHORED=PASS\nENGINE_VERIFY_RECONSTRUCT_B_ANCHORED=PASS\nENGINE_VERIFY_COMPARE_AB_ANCHORED=PASS\nENGINE_VERIFY_CLOUD_MUTATIONS_ANCHORED=ZERO\nANCHOR_380_RUNTIME_IDENTITY=PASS\nRCA_380_HEALTH_ROOT_OMISSION_ANCHORED=PASS\nSTATE_SCAN_ROOT_COUNT=3\nSTATE_SCAN_ROOT_1=/var/lib/homelab-backup\nSTATE_SCAN_ROOT_2=/var/lib/homelab-backup-v2\nSTATE_SCAN_ROOT_3=/var/lib/homelab-health\nSTATE_JSON_SCANNED=1562\nSTATE_JSON_PARSE_ERRORS=0\nGENERATION_MATCH_FILE_COUNT=3\nWINDOW_RELEVANT_FILE_COUNT=2\nEVIDENCE_IDENTITY_SUCCESS_FILE_COUNT=2\nHEALTH_BINDING_FILE_COUNT=1\nLEGACY_SCHEMA_MISMATCH_CONFIRMED_FILE_COUNT=1\nDOC_1_PATH=/var/lib/homelab-backup/control-restore-validation/control-bootstrap/control-bootstrap-v2-pve01-20260914T080740Z.json\nDOC_1_MTIME_UTC=2026-09-14T19:12:40.777301Z\nDOC_1_IN_379_WINDOW=YES\nDOC_1_HEALTHISH=NO\nDOC_1_RESTOREISH=YES\nDOC_1_LEGACY_TOP_STATUS_PRESENT=NO\nDOC_1_LEGACY_TOP_GENERATION_EXACT=YES\nDOC_1_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=NO\nDOC_1_GENERATION_MATCH_PATHS=[\"/generation\"]\nDOC_1_ARCHIVE_SHA_MATCH_PATHS=[\"/payload_sha256\"]\nDOC_1_SUCCESS_SEMANTIC_PATHS=[\"/cloud_a_semantic_restore_verified\",\"/cloud_b_byte_reconstruction_verified\",\"/control_manifest_verified\",\"/desired_git_bundle_verified\",\"/ops_git_bundle_verified\",\"/payload_checksums_verified\",\"/payload_sha256_verified\",\"/result\",\"/verified_at_utc\"]\nDOC_1_SCHEMA_LEAF_PATHS=[\"/clean_machine_bootstrap_proven\",\"/cloud_a_b_payload_equal\",\"/cloud_a_semantic_restore_verified\",\"/cloud_b_byte_reconstruction_verified\",\"/control_manifest_verified\",\"/control_scope\",\"/desired_git_bundle_verified\",\"/generation\",\"/logical_id\",\"/ops_git_bundle_verified\",\"/payload_checksums_verified\",\"/payload_sha256\",\"/payload_sha256_verified\",\"/restore_source\",\"/result\",\"/runtime_asset_count_verified\",\"/secret_escrow_separate\",\"/systemd_asset_count_verified\",\"/verified_at_utc\"]\nDOC_1_SCHEMA_PATHS_TRUNCATED=NO\nDOC_2_PATH=/var/lib/homelab-health/control-backup.json\nDOC_2_MTIME_UTC=2026-09-14T19:12:40.781301Z\nDOC_2_IN_379_WINDOW=YES\nDOC_2_HEALTHISH=YES\nDOC_2_RESTOREISH=YES\nDOC_2_LEGACY_TOP_STATUS_PRESENT=YES\nDOC_2_LEGACY_TOP_GENERATION_EXACT=YES\nDOC_2_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=YES\nDOC_2_GENERATION_MATCH_PATHS=[\"/generation\"]\nDOC_2_ARCHIVE_SHA_MATCH_PATHS=[\"/archive_sha256\"]\nDOC_2_SUCCESS_SEMANTIC_PATHS=[\"/status\"]\nDOC_2_SCHEMA_LEAF_PATHS=[\"/archive_sha256\",\"/cloud_quorum\",\"/control_freshness\",\"/full_site_dr_proven\",\"/generation\",\"/restore_evidence\",\"/schema\",\"/state_file\",\"/status\",\"/updated_at_utc\"]\nDOC_2_SCHEMA_PATHS_TRUNCATED=NO\nDOC_3_PATH=/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2/control-bootstrap-v2-pve01-20260914T080740Z.json\nDOC_3_MTIME_UTC=2026-09-14T08:07:54.154762Z\nDOC_3_IN_379_WINDOW=NO\nDOC_3_HEALTHISH=NO\nDOC_3_RESTOREISH=NO\nDOC_3_LEGACY_TOP_STATUS_PRESENT=YES\nDOC_3_LEGACY_TOP_GENERATION_EXACT=YES\nDOC_3_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=YES\nDOC_3_GENERATION_MATCH_PATHS=[\"/generation\"]\nDOC_3_ARCHIVE_SHA_MATCH_PATHS=[\"/archive_sha256\",\"/cloud_a_reconstructed_sha256\",\"/cloud_b_reconstructed_sha256\"]\nDOC_3_SUCCESS_SEMANTIC_PATHS=[\"/cloud_a_seal_verified\",\"/cloud_a_seal_verified_at_utc\",\"/cloud_a_verified\",\"/cloud_a_verified_at_utc\",\"/cloud_b_seal_verified\",\"/cloud_b_seal_verified_at_utc\",\"/cloud_b_verified\",\"/cloud_b_verified_at_utc\",\"/part_hashes_verified\",\"/status\"]\nDOC_3_SCHEMA_LEAF_PATHS=[\"/archive_sha256\",\"/archive_size_bytes\",\"/chunk_size_bytes\",\"/cloud_a_reconstructed_sha256\",\"/cloud_a_seal_verified\",\"/cloud_a_seal_verified_at_utc\",\"/cloud_a_verified\",\"/cloud_a_verified_at_utc\",\"/cloud_b_reconstructed_sha256\",\"/cloud_b_seal_verified\",\"/cloud_b_seal_verified_at_utc\",\"/cloud_b_verified\",\"/cloud_b_verified_at_utc\",\"/cloud_quorum\",\"/created_at_utc\",\"/delete_authorized_at_utc\",\"/generation\",\"/local_cleanup\",\"/local_delete_allowed\",\"/local_deleted_at_utc\",\"/local_payload_present\",\"/logical_id\",\"/part_count\",\"/part_hashes_verified\",\"/parts_manifest_sha256\",\"/payload_name\",\"/payload_path\",\"/remote_a_base\",\"/remote_a_root\",\"/remote_b_base\",\"/remote_b_root\",\"/schema\",\"/seal_sha256\",\"/seal_written\",\"/sealed_at_utc\",\"/status\",\"/updated_at_utc\"]\nDOC_3_SCHEMA_PATHS_TRUNCATED=NO\nVERIFY_WORKSPACE_DIRS_SCANNED=72\nVERIFY_WORKSPACE_RESIDUAL_COUNT=0\nRCA_379=WRAPPER_ASSUMED_TOP_LEVEL_STATUS_AND_ARCHIVE_SHA256_FIELDS_NOT_PRESENT_IN_ACTUAL_EVIDENCE_SCHEMA\nRCA_380=STATE_SCAN_OMITTED_VAR_LIB_HOMELAB_HEALTH_AND_WORKSPACE_SCANNER_USED_BROAD_NAME_MATCHES\nRCA_380_HEALTH_SCAN_ROOT_FIX=PASS\nRCA_380_VERIFY_WORKSPACE_SCOPE_FIX=PASS\nACTUAL_EVIDENCE_SCHEMA_IDENTITY_BINDING=PASS\nACTUAL_EVIDENCE_379_WRITE_WINDOW_BINDING=PASS\nACTUAL_HEALTH_379_WRITE_WINDOW_BINDING=PASS\nCONTROL_PRODUCTION_V2_GENERATION=PROVEN\nCONTROL_PRODUCTION_V2_RESTORE=PROVEN\nBACKUP_CONTROL_PLANE_PRODUCTION_V2_COVERAGE_COMPLETE=YES\nCONTROL_PLANE_FORMAL_CLOSURE_METHOD=READ_ONLY_ACCEPTANCE_OF_EXISTING_378_STATE_PLUS_379_SUCCESSFUL_VERIFY_EVIDENCE\nNEXT_GATE=BACKUPV2_FINAL_COVERAGE_REGISTRY_REAUDIT\nSTATUS=OK\nPHASE=COMPLETE\nREASON=NONE\nCHANGES_MADE=false\nMUTATION_OUTCOME=NO_MUTATION\nCLOUD_WRITES=0\nCLOUD_DELETIONS=0\nBACKUP_EXECUTIONS=0\nCONTROL_ENGINE_EXECUTIONS=0\nSYSTEMD_MUTATIONS=0\nSCHEDULER_CUTOVER=NO\nLEGACY_BACKUP_DISABLES=0\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 8fe316d5..ab5c8dc3 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-379-RESTORE-EVIDENCE-SCHEMA-RCA-380R1 -STATUS=FAIL -RC=2 +COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1 +STATUS=OK +RC=0 HOST=pve01 MODE=read-only -COMPONENT=infra-dr-backupv2-control-379-restore-evidence-schema-rca +COMPONENT=infra-dr-backupv2-control-380-health-root-workspace-rca-fix-closure REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=906899503fcb26cfb7b293778f5f360a515d5dc03ffba1b5bf571f62031cab69 +COMMAND_SHA256=2d627054d90c1adc89eb735594479df46650a069d5bc3ee272945cfe3fa2e834 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=2 +COMMAND_RC=0 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,12 +24,13 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=cc2acde743d0ab26b6aedc615503f3819410cf76dd5b2f121de38f8db1a730bf -SANITIZED_OUTPUT_SHA256=cc2acde743d0ab26b6aedc615503f3819410cf76dd5b2f121de38f8db1a730bf +RAW_EVIDENCE_SHA256=49037e8b0cca7de6424bf838d14a43d44ffe99e581bfd69d678bc1f40a2be554 +SANITIZED_OUTPUT_SHA256=49037e8b0cca7de6424bf838d14a43d44ffe99e581bfd69d678bc1f40a2be554 OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-379-RESTORE-EVIDENCE-SCHEMA-RCA-380R1 -MODE=READ_ONLY_CONTROL_379_RESTORE_EVIDENCE_SCHEMA_RCA +COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1 +MODE=READ_ONLY_CONTROL_380_HEALTH_ROOT_WORKSPACE_RCA_FIX_CLOSURE ANCHOR_379_COMMIT=96cfc84f65e5867d6085b03a5f71cf2170f6e4f3 +ANCHOR_380_COMMIT=c52aeb801d354c97299c03cdb8cec9e83c340085 EXPECTED_GENERATION=control-bootstrap-v2-pve01-20260914T080740Z NO_FORCE=YES NO_VERIFY=YES @@ -54,15 +55,18 @@ ENGINE_VERIFY_RECONSTRUCT_A_ANCHORED=PASS ENGINE_VERIFY_RECONSTRUCT_B_ANCHORED=PASS ENGINE_VERIFY_COMPARE_AB_ANCHORED=PASS ENGINE_VERIFY_CLOUD_MUTATIONS_ANCHORED=ZERO -STATE_SCAN_ROOT_COUNT=2 +ANCHOR_380_RUNTIME_IDENTITY=PASS +RCA_380_HEALTH_ROOT_OMISSION_ANCHORED=PASS +STATE_SCAN_ROOT_COUNT=3 STATE_SCAN_ROOT_1=/var/lib/homelab-backup STATE_SCAN_ROOT_2=/var/lib/homelab-backup-v2 -STATE_JSON_SCANNED=1540 +STATE_SCAN_ROOT_3=/var/lib/homelab-health +STATE_JSON_SCANNED=1562 STATE_JSON_PARSE_ERRORS=0 -GENERATION_MATCH_FILE_COUNT=2 -WINDOW_RELEVANT_FILE_COUNT=1 -EVIDENCE_IDENTITY_SUCCESS_FILE_COUNT=1 -HEALTH_BINDING_FILE_COUNT=0 +GENERATION_MATCH_FILE_COUNT=3 +WINDOW_RELEVANT_FILE_COUNT=2 +EVIDENCE_IDENTITY_SUCCESS_FILE_COUNT=2 +HEALTH_BINDING_FILE_COUNT=1 LEGACY_SCHEMA_MISMATCH_CONFIRMED_FILE_COUNT=1 DOC_1_PATH=/var/lib/homelab-backup/control-restore-validation/control-bootstrap/control-bootstrap-v2-pve01-20260914T080740Z.json DOC_1_MTIME_UTC=2026-09-14T19:12:40.777301Z @@ -77,31 +81,49 @@ DOC_1_ARCHIVE_SHA_MATCH_PATHS=["/payload_sha256"] DOC_1_SUCCESS_SEMANTIC_PATHS=["/cloud_a_semantic_restore_verified","/cloud_b_byte_reconstruction_verified","/control_manifest_verified","/desired_git_bundle_verified","/ops_git_bundle_verified","/payload_checksums_verified","/payload_sha256_verified","/result","/verified_at_utc"] DOC_1_SCHEMA_LEAF_PATHS=["/clean_machine_bootstrap_proven","/cloud_a_b_payload_equal","/cloud_a_semantic_restore_verified","/cloud_b_byte_reconstruction_verified","/control_manifest_verified","/control_scope","/desired_git_bundle_verified","/generation","/logical_id","/ops_git_bundle_verified","/payload_checksums_verified","/payload_sha256","/payload_sha256_verified","/restore_source","/result","/runtime_asset_count_verified","/secret_escrow_separate","/systemd_asset_count_verified","/verified_at_utc"] DOC_1_SCHEMA_PATHS_TRUNCATED=NO -DOC_2_PATH=/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2/control-bootstrap-v2-pve01-20260914T080740Z.json -DOC_2_MTIME_UTC=2026-09-14T08:07:54.154762Z -DOC_2_IN_379_WINDOW=NO -DOC_2_HEALTHISH=NO -DOC_2_RESTOREISH=NO +DOC_2_PATH=/var/lib/homelab-health/control-backup.json +DOC_2_MTIME_UTC=2026-09-14T19:12:40.781301Z +DOC_2_IN_379_WINDOW=YES +DOC_2_HEALTHISH=YES +DOC_2_RESTOREISH=YES DOC_2_LEGACY_TOP_STATUS_PRESENT=YES DOC_2_LEGACY_TOP_GENERATION_EXACT=YES DOC_2_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=YES DOC_2_GENERATION_MATCH_PATHS=["/generation"] -DOC_2_ARCHIVE_SHA_MATCH_PATHS=["/archive_sha256","/cloud_a_reconstructed_sha256","/cloud_b_reconstructed_sha256"] -DOC_2_SUCCESS_SEMANTIC_PATHS=["/cloud_a_seal_verified","/cloud_a_seal_verified_at_utc","/cloud_a_verified","/cloud_a_verified_at_utc","/cloud_b_seal_verified","/cloud_b_seal_verified_at_utc","/cloud_b_verified","/cloud_b_verified_at_utc","/part_hashes_verified","/status"] -DOC_2_SCHEMA_LEAF_PATHS=["/archive_sha256","/archive_size_bytes","/chunk_size_bytes","/cloud_a_reconstructed_sha256","/cloud_a_seal_verified","/cloud_a_seal_verified_at_utc","/cloud_a_verified","/cloud_a_verified_at_utc","/cloud_b_reconstructed_sha256","/cloud_b_seal_verified","/cloud_b_seal_verified_at_utc","/cloud_b_verified","/cloud_b_verified_at_utc","/cloud_quorum","/created_at_utc","/delete_authorized_at_utc","/generation","/local_cleanup","/local_delete_allowed","/local_deleted_at_utc","/local_payload_present","/logical_id","/part_count","/part_hashes_verified","/parts_manifest_sha256","/payload_name","/payload_path","/remote_a_base","/remote_a_root","/remote_b_base","/remote_b_root","/schema","/seal_sha256","/seal_written","/sealed_at_utc","/status","/updated_at_utc"] +DOC_2_ARCHIVE_SHA_MATCH_PATHS=["/archive_sha256"] +DOC_2_SUCCESS_SEMANTIC_PATHS=["/status"] +DOC_2_SCHEMA_LEAF_PATHS=["/archive_sha256","/cloud_quorum","/control_freshness","/full_site_dr_proven","/generation","/restore_evidence","/schema","/state_file","/status","/updated_at_utc"] DOC_2_SCHEMA_PATHS_TRUNCATED=NO -VERIFY_WORKSPACE_DIRS_SCANNED=1518 -VERIFY_WORKSPACE_RESIDUAL_COUNT=7 -VERIFY_WORKSPACE_RESIDUAL_1=/mnt/staging/homelab-control-verify -VERIFY_WORKSPACE_RESIDUAL_2=/tmp/homelab-mkdocs-site/docs/02-backup-and-restore -VERIFY_WORKSPACE_RESIDUAL_3=/tmp/homelab-mkdocs-site/reports/backup-restore-dr-audit-20260622 -VERIFY_WORKSPACE_RESIDUAL_4=/tmp/homelab-mkdocs-site/reports/backup-restore-table-refresh-20260622 -VERIFY_WORKSPACE_RESIDUAL_5=/tmp/homelab-mkdocs-site/reports/netbox-backup-offhost-restore-patch-failed-recovered-20260622 -VERIFY_WORKSPACE_RESIDUAL_6=/tmp/homelab-mkdocs-site/reports/netbox-backup-offhost-restore-patch-v2-20260622 -VERIFY_WORKSPACE_RESIDUAL_7=/tmp/homelab-mkdocs-site/reports/netbox-backup-offhost-restore-proof-audit-20260622 -STATUS=FAIL -PHASE=HEALTH_STATE -REASON=379_HEALTH_BINDING_NOT_PROVEN +DOC_3_PATH=/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2/control-bootstrap-v2-pve01-20260914T080740Z.json +DOC_3_MTIME_UTC=2026-09-14T08:07:54.154762Z +DOC_3_IN_379_WINDOW=NO +DOC_3_HEALTHISH=NO +DOC_3_RESTOREISH=NO +DOC_3_LEGACY_TOP_STATUS_PRESENT=YES +DOC_3_LEGACY_TOP_GENERATION_EXACT=YES +DOC_3_LEGACY_TOP_ARCHIVE_SHA256_PRESENT=YES +DOC_3_GENERATION_MATCH_PATHS=["/generation"] +DOC_3_ARCHIVE_SHA_MATCH_PATHS=["/archive_sha256","/cloud_a_reconstructed_sha256","/cloud_b_reconstructed_sha256"] +DOC_3_SUCCESS_SEMANTIC_PATHS=["/cloud_a_seal_verified","/cloud_a_seal_verified_at_utc","/cloud_a_verified","/cloud_a_verified_at_utc","/cloud_b_seal_verified","/cloud_b_seal_verified_at_utc","/cloud_b_verified","/cloud_b_verified_at_utc","/part_hashes_verified","/status"] +DOC_3_SCHEMA_LEAF_PATHS=["/archive_sha256","/archive_size_bytes","/chunk_size_bytes","/cloud_a_reconstructed_sha256","/cloud_a_seal_verified","/cloud_a_seal_verified_at_utc","/cloud_a_verified","/cloud_a_verified_at_utc","/cloud_b_reconstructed_sha256","/cloud_b_seal_verified","/cloud_b_seal_verified_at_utc","/cloud_b_verified","/cloud_b_verified_at_utc","/cloud_quorum","/created_at_utc","/delete_authorized_at_utc","/generation","/local_cleanup","/local_delete_allowed","/local_deleted_at_utc","/local_payload_present","/logical_id","/part_count","/part_hashes_verified","/parts_manifest_sha256","/payload_name","/payload_path","/remote_a_base","/remote_a_root","/remote_b_base","/remote_b_root","/schema","/seal_sha256","/seal_written","/sealed_at_utc","/status","/updated_at_utc"] +DOC_3_SCHEMA_PATHS_TRUNCATED=NO +VERIFY_WORKSPACE_DIRS_SCANNED=72 +VERIFY_WORKSPACE_RESIDUAL_COUNT=0 +RCA_379=WRAPPER_ASSUMED_TOP_LEVEL_STATUS_AND_ARCHIVE_SHA256_FIELDS_NOT_PRESENT_IN_ACTUAL_EVIDENCE_SCHEMA +RCA_380=STATE_SCAN_OMITTED_VAR_LIB_HOMELAB_HEALTH_AND_WORKSPACE_SCANNER_USED_BROAD_NAME_MATCHES +RCA_380_HEALTH_SCAN_ROOT_FIX=PASS +RCA_380_VERIFY_WORKSPACE_SCOPE_FIX=PASS +ACTUAL_EVIDENCE_SCHEMA_IDENTITY_BINDING=PASS +ACTUAL_EVIDENCE_379_WRITE_WINDOW_BINDING=PASS +ACTUAL_HEALTH_379_WRITE_WINDOW_BINDING=PASS +CONTROL_PRODUCTION_V2_GENERATION=PROVEN +CONTROL_PRODUCTION_V2_RESTORE=PROVEN +BACKUP_CONTROL_PLANE_PRODUCTION_V2_COVERAGE_COMPLETE=YES +CONTROL_PLANE_FORMAL_CLOSURE_METHOD=READ_ONLY_ACCEPTANCE_OF_EXISTING_378_STATE_PLUS_379_SUCCESSFUL_VERIFY_EVIDENCE +NEXT_GATE=BACKUPV2_FINAL_COVERAGE_REGISTRY_REAUDIT +STATUS=OK +PHASE=COMPLETE +REASON=NONE CHANGES_MADE=false MUTATION_OUTCOME=NO_MUTATION CLOUD_WRITES=0 @@ -111,7 +133,7 @@ CONTROL_ENGINE_EXECUTIONS=0 SYSTEMD_MUTATIONS=0 SCHEDULER_CUTOVER=NO LEGACY_BACKUP_DISABLES=0 -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-379-RESTORE-EVIDENCE-SCHEMA-RCA-380R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260914-INFRA-DR-BACKUPV2-CONTROL-380-HEALTH-ROOT-WORKSPACE-RCA-FIX-CLOSURE-381R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END