diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1.json b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1.json new file mode 100644 index 00000000..574fa5a7 --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1", + "status": "FAIL", + "rc": 3, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-1111r2-metadata-rca", + "started_at_utc": "2026-09-22T05:26:41Z", + "finished_at_utc": "2026-09-22T05:26:42Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "7682fd29d69970b8027a5a07ca78e4c4b97a05bc2e5ef1dc75d70b230b1a669a", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 3, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411", + "sanitized_output_sha256": "745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_1111R2_METADATA_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1\nMODE=read-only\nCOMPONENT=homelab-backup-1111r2-metadata-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_SOURCE_AND_STATE_RECONCILE;NO_SYSTEMCTL_START_STOP_RESET;NO_BACKUP;NO_RESTORE;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nVMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nAST_ASSIGN_COUNT_BRANCH=1\nAST_ASSIGN_BRANCH_LINE=22\nAST_ASSIGN_BRANCH_EXPR=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_BRANCH_18=GFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nAUDIT_SOURCE_BRANCH_19=GFSP=\"/etc/homelab-backup/gfs-retention.json\"\nAUDIT_SOURCE_BRANCH_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_BRANCH_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_BRANCH_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_BRANCH_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_BRANCH_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_BRANCH_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nAUDIT_SOURCE_BRANCH_26=\nAST_ASSIGN_COUNT_CLOUDB_HEALTH=1\nAST_ASSIGN_CLOUDB_HEALTH_LINE=24\nAST_ASSIGN_CLOUDB_HEALTH_EXPR=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_CLOUDB_HEALTH_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_CLOUDB_HEALTH_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_CLOUDB_HEALTH_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_CLOUDB_HEALTH_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_CLOUDB_HEALTH_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_CLOUDB_HEALTH_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nAUDIT_SOURCE_CLOUDB_HEALTH_26=\nAUDIT_SOURCE_CLOUDB_HEALTH_27=def run(cmd,timeout=60,env=None):\nAUDIT_SOURCE_CLOUDB_HEALTH_28= try:\nAST_ASSIGN_COUNT_WT=1\nAST_ASSIGN_WT_LINE=21\nAST_ASSIGN_WT_EXPR=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_WT_17=APPQ=\"/usr/local/sbin/homelab-app-cloud-quorum-queue\"\nAUDIT_SOURCE_WT_18=GFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nAUDIT_SOURCE_WT_19=GFSP=\"/etc/homelab-backup/gfs-retention.json\"\nAUDIT_SOURCE_WT_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_WT_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_WT_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_WT_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_WT_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_WT_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nCLOUDB_HEALTH_CANDIDATE_COUNT=0\nCLOUDB_PRODUCER_COUNT=0\nCLOUDB_SERVICE_MATCH_COUNT=0\nSOURCE_WT=/root/homelab-ops-cr-2026-1005\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_HEAD_ALIGNMENT=UNAVAILABLE\nDECISION=HOLD_BACKUP_1112_METADATA_OR_IDENTITY_AMBIGUOUS\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1112_AMBIGUITY\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1111R2_METADATA_RCA\nCHANGES_MADE_BY_1112=false\nPRODUCT_MUTATION_BY_1112=false\nVM_MUTATION_BY_1112=false\nCLOUD_MUTATION_BY_1112=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1111R2_METADATA_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1.txt b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1.txt new file mode 100644 index 00000000..6aaa468b --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1.txt @@ -0,0 +1,101 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1 +STATUS=FAIL +RC=3 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-1111r2-metadata-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=7682fd29d69970b8027a5a07ca78e4c4b97a05bc2e5ef1dc75d70b230b1a669a +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=3 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411 +SANITIZED_OUTPUT_SHA256=745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411 +OUTPUT_BEGIN +WORKERS2_BACKUP_1111R2_METADATA_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1 +MODE=read-only +COMPONENT=homelab-backup-1111r2-metadata-rca +MUTATION_BOUNDARY=NONE;READ_ONLY_SOURCE_AND_STATE_RECONCILE;NO_SYSTEMCTL_START_STOP_RESET;NO_BACKUP;NO_RESTORE;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP +ERROR_REGISTER_CHECK=OK +REFERENCE_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 +AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 +VMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 +VM_QUEUE_CURRENT_STATE=success,inactive,dead +BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed +SCHEDULER_RC=0 +SELFTEST_REPLACEMENT486=PASS +SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET +AST_ASSIGN_COUNT_BRANCH=1 +AST_ASSIGN_BRANCH_LINE=22 +AST_ASSIGN_BRANCH_EXPR="cr-2026-1005-backup-v2-health-alerting-remediation" +AUDIT_SOURCE_BRANCH_18=GFSH="/usr/local/libexec/homelab-backup-gfs-policy" +AUDIT_SOURCE_BRANCH_19=GFSP="/etc/homelab-backup/gfs-retention.json" +AUDIT_SOURCE_BRANCH_20=RET="/usr/local/sbin/homelab-backup-retention-gc" +AUDIT_SOURCE_BRANCH_21=WT="/root/homelab-ops-cr-2026-1005" +AUDIT_SOURCE_BRANCH_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" +AUDIT_SOURCE_BRANCH_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") +AUDIT_SOURCE_BRANCH_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +AUDIT_SOURCE_BRANCH_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] +AUDIT_SOURCE_BRANCH_26= +AST_ASSIGN_COUNT_CLOUDB_HEALTH=1 +AST_ASSIGN_CLOUDB_HEALTH_LINE=24 +AST_ASSIGN_CLOUDB_HEALTH_EXPR=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +AUDIT_SOURCE_CLOUDB_HEALTH_20=RET="/usr/local/sbin/homelab-backup-retention-gc" +AUDIT_SOURCE_CLOUDB_HEALTH_21=WT="/root/homelab-ops-cr-2026-1005" +AUDIT_SOURCE_CLOUDB_HEALTH_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" +AUDIT_SOURCE_CLOUDB_HEALTH_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") +AUDIT_SOURCE_CLOUDB_HEALTH_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +AUDIT_SOURCE_CLOUDB_HEALTH_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] +AUDIT_SOURCE_CLOUDB_HEALTH_26= +AUDIT_SOURCE_CLOUDB_HEALTH_27=def run(cmd,timeout=60,env=None): +AUDIT_SOURCE_CLOUDB_HEALTH_28= try: +AST_ASSIGN_COUNT_WT=1 +AST_ASSIGN_WT_LINE=21 +AST_ASSIGN_WT_EXPR="/root/homelab-ops-cr-2026-1005" +AUDIT_SOURCE_WT_17=APPQ="/usr/local/sbin/homelab-app-cloud-quorum-queue" +AUDIT_SOURCE_WT_18=GFSH="/usr/local/libexec/homelab-backup-gfs-policy" +AUDIT_SOURCE_WT_19=GFSP="/etc/homelab-backup/gfs-retention.json" +AUDIT_SOURCE_WT_20=RET="/usr/local/sbin/homelab-backup-retention-gc" +AUDIT_SOURCE_WT_21=WT="/root/homelab-ops-cr-2026-1005" +AUDIT_SOURCE_WT_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" +AUDIT_SOURCE_WT_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") +AUDIT_SOURCE_WT_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +AUDIT_SOURCE_WT_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] +CLOUDB_HEALTH_CANDIDATE_COUNT=0 +CLOUDB_PRODUCER_COUNT=0 +CLOUDB_SERVICE_MATCH_COUNT=0 +SOURCE_WT=/root/homelab-ops-cr-2026-1005 +SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_HEAD_ALIGNMENT=UNAVAILABLE +DECISION=HOLD_BACKUP_1112_METADATA_OR_IDENTITY_AMBIGUOUS +NEXT_GATE=RECONCILE_ONLY_BACKUP_1112_AMBIGUITY +TASK_RESULT=HOLD_HOMELAB_BACKUP_1111R2_METADATA_RCA +CHANGES_MADE_BY_1112=false +PRODUCT_MUTATION_BY_1112=false +VM_MUTATION_BY_1112=false +CLOUD_MUTATION_BY_1112=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1111R2_METADATA_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 2a6a61a4..574fa5a7 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260922-HOMELAB-BACKUP-CLASSIFIED-BLOCKER-REPAIR-1111R2", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1", "status": "FAIL", - "rc": 76, + "rc": 3, "host": "pve01", - "mode": "change", - "component": "homelab-backup-classified-blocker-repair", - "started_at_utc": "2026-09-22T05:21:00Z", - "finished_at_utc": "2026-09-22T05:21:00Z", + "mode": "read-only", + "component": "homelab-backup-1111r2-metadata-rca", + "started_at_utc": "2026-09-22T05:26:41Z", + "finished_at_utc": "2026-09-22T05:26:42Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "855f589d68c17028701bd05db5ae3c74ce6f61dacfc825289682df6d06ddb0bb", + "command_sha256": "7682fd29d69970b8027a5a07ca78e4c4b97a05bc2e5ef1dc75d70b230b1a669a", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, - "result_contract_valid": false, + "change_declared": false, + "result_contract_valid": true, "result_contract_status": null, - "result_contract_error": "MISSING_CONTRACT", - "command_rc": 4, - "changes_made": null, - "rollback_started": null, + "result_contract_error": null, + "command_rc": 3, + "changes_made": false, + "rollback_started": false, "rollback_restored": null, - "mutation_outcome": "UNKNOWN", + "mutation_outcome": "NO_MUTATION", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "3ca024fa1baca32b44342ff9501f19a6fe6968b772077fbc096c8ba55284dad5", - "sanitized_output_sha256": "3ca024fa1baca32b44342ff9501f19a6fe6968b772077fbc096c8ba55284dad5", + "raw_evidence_sha256": "745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411", + "sanitized_output_sha256": "745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_BACKUP_CLASSIFIED_BLOCKER_REPAIR_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-CLASSIFIED-BLOCKER-REPAIR-1111R2\nMODE=change\nCOMPONENT=homelab-backup-classified-blocker-repair\nMUTATION_BOUNDARY=RESET_STALE_PRE1108_VMQUEUE_FAILED_STATE;RUN_EXISTING_CANONICAL_CLOUDB_RESTORE_DRILL;NO_NEW_BACKUP_GENERATION;NO_VM_CT_MUTATION;NO_TIMER_CHANGE;NO_SOURCE_PUSH;NO_CONFIG_REWRITE;POST_REPAIR_AUDIT_ONLY\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nVMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nSCHEDULER_BEFORE_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nVM_QUEUE_PRE_ACTIVE_STATE=failed\nVM_QUEUE_PRE_EXIT_EPOCH=1789579982\nPOLICY_FIX_1108_FINISH_EPOCH=1790030296\nVM_QUEUE_STALE_FAILED_STATE_RESET=true\nVM_QUEUE_POST_ACTIVE_STATE=inactive\nAUDIT_META_CLOUDB_HEALTH=\nAUDIT_META_WT=/root/homelab-ops-cr-2026-1005\nAUDIT_META_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nFAILURE=AUDIT_METADATA_PARSE_FAILED\nSOURCE_REPO_STATE=UNAVAILABLE\nDECISION=HOLD_BACKUP_1111_REPAIR_INCOMPLETE\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1111_FAILURE\nTASK_RESULT=HOLD_HOMELAB_BACKUP_CLASSIFIED_BLOCKER_REPAIR\nCHANGES_MADE_BY_1111R2=true\nPRODUCT_MUTATION_BY_1111R2=false\nVM_MUTATION_BY_1111R2=false\nBACKUP_GENERATION_BY_1111R2=false\nWORKERS2_BACKUP_CLASSIFIED_BLOCKER_REPAIR_END=1\n" + "output": "WORKERS2_BACKUP_1111R2_METADATA_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1\nMODE=read-only\nCOMPONENT=homelab-backup-1111r2-metadata-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_SOURCE_AND_STATE_RECONCILE;NO_SYSTEMCTL_START_STOP_RESET;NO_BACKUP;NO_RESTORE;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nVMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nAST_ASSIGN_COUNT_BRANCH=1\nAST_ASSIGN_BRANCH_LINE=22\nAST_ASSIGN_BRANCH_EXPR=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_BRANCH_18=GFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nAUDIT_SOURCE_BRANCH_19=GFSP=\"/etc/homelab-backup/gfs-retention.json\"\nAUDIT_SOURCE_BRANCH_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_BRANCH_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_BRANCH_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_BRANCH_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_BRANCH_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_BRANCH_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nAUDIT_SOURCE_BRANCH_26=\nAST_ASSIGN_COUNT_CLOUDB_HEALTH=1\nAST_ASSIGN_CLOUDB_HEALTH_LINE=24\nAST_ASSIGN_CLOUDB_HEALTH_EXPR=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_CLOUDB_HEALTH_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_CLOUDB_HEALTH_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_CLOUDB_HEALTH_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_CLOUDB_HEALTH_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_CLOUDB_HEALTH_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_CLOUDB_HEALTH_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nAUDIT_SOURCE_CLOUDB_HEALTH_26=\nAUDIT_SOURCE_CLOUDB_HEALTH_27=def run(cmd,timeout=60,env=None):\nAUDIT_SOURCE_CLOUDB_HEALTH_28= try:\nAST_ASSIGN_COUNT_WT=1\nAST_ASSIGN_WT_LINE=21\nAST_ASSIGN_WT_EXPR=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_WT_17=APPQ=\"/usr/local/sbin/homelab-app-cloud-quorum-queue\"\nAUDIT_SOURCE_WT_18=GFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nAUDIT_SOURCE_WT_19=GFSP=\"/etc/homelab-backup/gfs-retention.json\"\nAUDIT_SOURCE_WT_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_WT_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_WT_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_WT_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_WT_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_WT_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nCLOUDB_HEALTH_CANDIDATE_COUNT=0\nCLOUDB_PRODUCER_COUNT=0\nCLOUDB_SERVICE_MATCH_COUNT=0\nSOURCE_WT=/root/homelab-ops-cr-2026-1005\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_HEAD_ALIGNMENT=UNAVAILABLE\nDECISION=HOLD_BACKUP_1112_METADATA_OR_IDENTITY_AMBIGUOUS\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1112_AMBIGUITY\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1111R2_METADATA_RCA\nCHANGES_MADE_BY_1112=false\nPRODUCT_MUTATION_BY_1112=false\nVM_MUTATION_BY_1112=false\nCLOUD_MUTATION_BY_1112=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1111R2_METADATA_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 762e54bc..6aaa468b 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,37 +1,37 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-CLASSIFIED-BLOCKER-REPAIR-1111R2 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1 STATUS=FAIL -RC=76 +RC=3 HOST=pve01 -MODE=change -COMPONENT=homelab-backup-classified-blocker-repair +MODE=read-only +COMPONENT=homelab-backup-1111r2-metadata-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=855f589d68c17028701bd05db5ae3c74ce6f61dacfc825289682df6d06ddb0bb +COMMAND_SHA256=7682fd29d69970b8027a5a07ca78e4c4b97a05bc2e5ef1dc75d70b230b1a669a DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true -RESULT_CONTRACT_VALID=false -RESULT_CONTRACT_STATUS=UNKNOWN -RESULT_CONTRACT_ERROR=MISSING_CONTRACT -COMMAND_RC=4 -CHANGES_MADE=unknown -ROLLBACK_STARTED=unknown +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=3 +CHANGES_MADE=false +ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=UNKNOWN +MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=3ca024fa1baca32b44342ff9501f19a6fe6968b772077fbc096c8ba55284dad5 -SANITIZED_OUTPUT_SHA256=3ca024fa1baca32b44342ff9501f19a6fe6968b772077fbc096c8ba55284dad5 +RAW_EVIDENCE_SHA256=745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411 +SANITIZED_OUTPUT_SHA256=745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411 OUTPUT_BEGIN -WORKERS2_BACKUP_CLASSIFIED_BLOCKER_REPAIR_BEGIN=1 -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-CLASSIFIED-BLOCKER-REPAIR-1111R2 -MODE=change -COMPONENT=homelab-backup-classified-blocker-repair -MUTATION_BOUNDARY=RESET_STALE_PRE1108_VMQUEUE_FAILED_STATE;RUN_EXISTING_CANONICAL_CLOUDB_RESTORE_DRILL;NO_NEW_BACKUP_GENERATION;NO_VM_CT_MUTATION;NO_TIMER_CHANGE;NO_SOURCE_PUSH;NO_CONFIG_REWRITE;POST_REPAIR_AUDIT_ONLY +WORKERS2_BACKUP_1111R2_METADATA_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1 +MODE=read-only +COMPONENT=homelab-backup-1111r2-metadata-rca +MUTATION_BOUNDARY=NONE;READ_ONLY_SOURCE_AND_STATE_RECONCILE;NO_SYSTEMCTL_START_STOP_RESET;NO_BACKUP;NO_RESTORE;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP ERROR_REGISTER_CHECK=OK REFERENCE_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 @@ -40,27 +40,62 @@ RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 VMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 -SCHEDULER_BEFORE_RC=0 +VM_QUEUE_CURRENT_STATE=success,inactive,dead +BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed +SCHEDULER_RC=0 SELFTEST_REPLACEMENT486=PASS SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET -VM_QUEUE_PRE_ACTIVE_STATE=failed -VM_QUEUE_PRE_EXIT_EPOCH=1789579982 -POLICY_FIX_1108_FINISH_EPOCH=1790030296 -VM_QUEUE_STALE_FAILED_STATE_RESET=true -VM_QUEUE_POST_ACTIVE_STATE=inactive -AUDIT_META_CLOUDB_HEALTH= -AUDIT_META_WT=/root/homelab-ops-cr-2026-1005 -AUDIT_META_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation -FAILURE=AUDIT_METADATA_PARSE_FAILED -SOURCE_REPO_STATE=UNAVAILABLE -DECISION=HOLD_BACKUP_1111_REPAIR_INCOMPLETE -NEXT_GATE=RECONCILE_ONLY_BACKUP_1111_FAILURE -TASK_RESULT=HOLD_HOMELAB_BACKUP_CLASSIFIED_BLOCKER_REPAIR -CHANGES_MADE_BY_1111R2=true -PRODUCT_MUTATION_BY_1111R2=false -VM_MUTATION_BY_1111R2=false -BACKUP_GENERATION_BY_1111R2=false -WORKERS2_BACKUP_CLASSIFIED_BLOCKER_REPAIR_END=1 +AST_ASSIGN_COUNT_BRANCH=1 +AST_ASSIGN_BRANCH_LINE=22 +AST_ASSIGN_BRANCH_EXPR="cr-2026-1005-backup-v2-health-alerting-remediation" +AUDIT_SOURCE_BRANCH_18=GFSH="/usr/local/libexec/homelab-backup-gfs-policy" +AUDIT_SOURCE_BRANCH_19=GFSP="/etc/homelab-backup/gfs-retention.json" +AUDIT_SOURCE_BRANCH_20=RET="/usr/local/sbin/homelab-backup-retention-gc" +AUDIT_SOURCE_BRANCH_21=WT="/root/homelab-ops-cr-2026-1005" +AUDIT_SOURCE_BRANCH_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" +AUDIT_SOURCE_BRANCH_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") +AUDIT_SOURCE_BRANCH_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +AUDIT_SOURCE_BRANCH_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] +AUDIT_SOURCE_BRANCH_26= +AST_ASSIGN_COUNT_CLOUDB_HEALTH=1 +AST_ASSIGN_CLOUDB_HEALTH_LINE=24 +AST_ASSIGN_CLOUDB_HEALTH_EXPR=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +AUDIT_SOURCE_CLOUDB_HEALTH_20=RET="/usr/local/sbin/homelab-backup-retention-gc" +AUDIT_SOURCE_CLOUDB_HEALTH_21=WT="/root/homelab-ops-cr-2026-1005" +AUDIT_SOURCE_CLOUDB_HEALTH_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" +AUDIT_SOURCE_CLOUDB_HEALTH_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") +AUDIT_SOURCE_CLOUDB_HEALTH_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +AUDIT_SOURCE_CLOUDB_HEALTH_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] +AUDIT_SOURCE_CLOUDB_HEALTH_26= +AUDIT_SOURCE_CLOUDB_HEALTH_27=def run(cmd,timeout=60,env=None): +AUDIT_SOURCE_CLOUDB_HEALTH_28= try: +AST_ASSIGN_COUNT_WT=1 +AST_ASSIGN_WT_LINE=21 +AST_ASSIGN_WT_EXPR="/root/homelab-ops-cr-2026-1005" +AUDIT_SOURCE_WT_17=APPQ="/usr/local/sbin/homelab-app-cloud-quorum-queue" +AUDIT_SOURCE_WT_18=GFSH="/usr/local/libexec/homelab-backup-gfs-policy" +AUDIT_SOURCE_WT_19=GFSP="/etc/homelab-backup/gfs-retention.json" +AUDIT_SOURCE_WT_20=RET="/usr/local/sbin/homelab-backup-retention-gc" +AUDIT_SOURCE_WT_21=WT="/root/homelab-ops-cr-2026-1005" +AUDIT_SOURCE_WT_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" +AUDIT_SOURCE_WT_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") +AUDIT_SOURCE_WT_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +AUDIT_SOURCE_WT_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] +CLOUDB_HEALTH_CANDIDATE_COUNT=0 +CLOUDB_PRODUCER_COUNT=0 +CLOUDB_SERVICE_MATCH_COUNT=0 +SOURCE_WT=/root/homelab-ops-cr-2026-1005 +SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_HEAD_ALIGNMENT=UNAVAILABLE +DECISION=HOLD_BACKUP_1112_METADATA_OR_IDENTITY_AMBIGUOUS +NEXT_GATE=RECONCILE_ONLY_BACKUP_1112_AMBIGUITY +TASK_RESULT=HOLD_HOMELAB_BACKUP_1111R2_METADATA_RCA +CHANGES_MADE_BY_1112=false +PRODUCT_MUTATION_BY_1112=false +VM_MUTATION_BY_1112=false +CLOUD_MUTATION_BY_1112=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1111R2_METADATA_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END