runtime: publish SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1
This commit is contained in:
homelab-runtime-publisher
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"channel": "homelab-runtime",
|
||||
"command_id": "SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1",
|
||||
"status": "FAIL",
|
||||
"rc": 3,
|
||||
"host": "pve01",
|
||||
"mode": "read-only",
|
||||
"component": "homelab-backup-1111r2-metadata-rca",
|
||||
"started_at_utc": "2026-09-22T05:26:41Z",
|
||||
"finished_at_utc": "2026-09-22T05:26:42Z",
|
||||
"reference_register_checked": true,
|
||||
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
|
||||
"error_register_checked": true,
|
||||
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
|
||||
"command_sha256": "7682fd29d69970b8027a5a07ca78e4c4b97a05bc2e5ef1dc75d70b230b1a669a",
|
||||
"duplicate_failed_command_blocked": false,
|
||||
"block_reason": null,
|
||||
"execution_started": true,
|
||||
"change_declared": false,
|
||||
"result_contract_valid": true,
|
||||
"result_contract_status": null,
|
||||
"result_contract_error": null,
|
||||
"command_rc": 3,
|
||||
"changes_made": false,
|
||||
"rollback_started": false,
|
||||
"rollback_restored": null,
|
||||
"mutation_outcome": "NO_MUTATION",
|
||||
"sanitized": true,
|
||||
"secrets_included": false,
|
||||
"private_addresses_included": false,
|
||||
"raw_evidence_retained_locally": true,
|
||||
"raw_evidence_sha256": "745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411",
|
||||
"sanitized_output_sha256": "745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411",
|
||||
"output_truncated_in_json": false,
|
||||
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
||||
"output": "WORKERS2_BACKUP_1111R2_METADATA_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1\nMODE=read-only\nCOMPONENT=homelab-backup-1111r2-metadata-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_SOURCE_AND_STATE_RECONCILE;NO_SYSTEMCTL_START_STOP_RESET;NO_BACKUP;NO_RESTORE;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nVMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nAST_ASSIGN_COUNT_BRANCH=1\nAST_ASSIGN_BRANCH_LINE=22\nAST_ASSIGN_BRANCH_EXPR=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_BRANCH_18=GFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nAUDIT_SOURCE_BRANCH_19=GFSP=\"/etc/homelab-backup/gfs-retention.json\"\nAUDIT_SOURCE_BRANCH_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_BRANCH_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_BRANCH_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_BRANCH_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_BRANCH_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_BRANCH_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nAUDIT_SOURCE_BRANCH_26=\nAST_ASSIGN_COUNT_CLOUDB_HEALTH=1\nAST_ASSIGN_CLOUDB_HEALTH_LINE=24\nAST_ASSIGN_CLOUDB_HEALTH_EXPR=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_CLOUDB_HEALTH_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_CLOUDB_HEALTH_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_CLOUDB_HEALTH_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_CLOUDB_HEALTH_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_CLOUDB_HEALTH_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_CLOUDB_HEALTH_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nAUDIT_SOURCE_CLOUDB_HEALTH_26=\nAUDIT_SOURCE_CLOUDB_HEALTH_27=def run(cmd,timeout=60,env=None):\nAUDIT_SOURCE_CLOUDB_HEALTH_28= try:\nAST_ASSIGN_COUNT_WT=1\nAST_ASSIGN_WT_LINE=21\nAST_ASSIGN_WT_EXPR=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_WT_17=APPQ=\"/usr/local/sbin/homelab-app-cloud-quorum-queue\"\nAUDIT_SOURCE_WT_18=GFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nAUDIT_SOURCE_WT_19=GFSP=\"/etc/homelab-backup/gfs-retention.json\"\nAUDIT_SOURCE_WT_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_WT_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_WT_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_WT_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_WT_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_WT_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nCLOUDB_HEALTH_CANDIDATE_COUNT=0\nCLOUDB_PRODUCER_COUNT=0\nCLOUDB_SERVICE_MATCH_COUNT=0\nSOURCE_WT=/root/homelab-ops-cr-2026-1005\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_HEAD_ALIGNMENT=UNAVAILABLE\nDECISION=HOLD_BACKUP_1112_METADATA_OR_IDENTITY_AMBIGUOUS\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1112_AMBIGUITY\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1111R2_METADATA_RCA\nCHANGES_MADE_BY_1112=false\nPRODUCT_MUTATION_BY_1112=false\nVM_MUTATION_BY_1112=false\nCLOUD_MUTATION_BY_1112=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1111R2_METADATA_RCA_END=1\n"
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1
|
||||
STATUS=FAIL
|
||||
RC=3
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1111r2-metadata-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=7682fd29d69970b8027a5a07ca78e4c4b97a05bc2e5ef1dc75d70b230b1a669a
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=3
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411
|
||||
SANITIZED_OUTPUT_SHA256=745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_BACKUP_1111R2_METADATA_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1111r2-metadata-rca
|
||||
MUTATION_BOUNDARY=NONE;READ_ONLY_SOURCE_AND_STATE_RECONCILE;NO_SYSTEMCTL_START_STOP_RESET;NO_BACKUP;NO_RESTORE;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
REFERENCE_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
|
||||
AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
|
||||
VMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
|
||||
VM_QUEUE_CURRENT_STATE=success,inactive,dead
|
||||
BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed
|
||||
SCHEDULER_RC=0
|
||||
SELFTEST_REPLACEMENT486=PASS
|
||||
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
|
||||
AST_ASSIGN_COUNT_BRANCH=1
|
||||
AST_ASSIGN_BRANCH_LINE=22
|
||||
AST_ASSIGN_BRANCH_EXPR="cr-2026-1005-backup-v2-health-alerting-remediation"
|
||||
AUDIT_SOURCE_BRANCH_18=GFSH="/usr/local/libexec/homelab-backup-gfs-policy"
|
||||
AUDIT_SOURCE_BRANCH_19=GFSP="/etc/homelab-backup/gfs-retention.json"
|
||||
AUDIT_SOURCE_BRANCH_20=RET="/usr/local/sbin/homelab-backup-retention-gc"
|
||||
AUDIT_SOURCE_BRANCH_21=WT="/root/homelab-ops-cr-2026-1005"
|
||||
AUDIT_SOURCE_BRANCH_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation"
|
||||
AUDIT_SOURCE_BRANCH_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json")
|
||||
AUDIT_SOURCE_BRANCH_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
||||
AUDIT_SOURCE_BRANCH_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")]
|
||||
AUDIT_SOURCE_BRANCH_26=
|
||||
AST_ASSIGN_COUNT_CLOUDB_HEALTH=1
|
||||
AST_ASSIGN_CLOUDB_HEALTH_LINE=24
|
||||
AST_ASSIGN_CLOUDB_HEALTH_EXPR=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_20=RET="/usr/local/sbin/homelab-backup-retention-gc"
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_21=WT="/root/homelab-ops-cr-2026-1005"
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation"
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json")
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")]
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_26=
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_27=def run(cmd,timeout=60,env=None):
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_28= try:
|
||||
AST_ASSIGN_COUNT_WT=1
|
||||
AST_ASSIGN_WT_LINE=21
|
||||
AST_ASSIGN_WT_EXPR="/root/homelab-ops-cr-2026-1005"
|
||||
AUDIT_SOURCE_WT_17=APPQ="/usr/local/sbin/homelab-app-cloud-quorum-queue"
|
||||
AUDIT_SOURCE_WT_18=GFSH="/usr/local/libexec/homelab-backup-gfs-policy"
|
||||
AUDIT_SOURCE_WT_19=GFSP="/etc/homelab-backup/gfs-retention.json"
|
||||
AUDIT_SOURCE_WT_20=RET="/usr/local/sbin/homelab-backup-retention-gc"
|
||||
AUDIT_SOURCE_WT_21=WT="/root/homelab-ops-cr-2026-1005"
|
||||
AUDIT_SOURCE_WT_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation"
|
||||
AUDIT_SOURCE_WT_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json")
|
||||
AUDIT_SOURCE_WT_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
||||
AUDIT_SOURCE_WT_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")]
|
||||
CLOUDB_HEALTH_CANDIDATE_COUNT=0
|
||||
CLOUDB_PRODUCER_COUNT=0
|
||||
CLOUDB_SERVICE_MATCH_COUNT=0
|
||||
SOURCE_WT=/root/homelab-ops-cr-2026-1005
|
||||
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_HEAD_ALIGNMENT=UNAVAILABLE
|
||||
DECISION=HOLD_BACKUP_1112_METADATA_OR_IDENTITY_AMBIGUOUS
|
||||
NEXT_GATE=RECONCILE_ONLY_BACKUP_1112_AMBIGUITY
|
||||
TASK_RESULT=HOLD_HOMELAB_BACKUP_1111R2_METADATA_RCA
|
||||
CHANGES_MADE_BY_1112=false
|
||||
PRODUCT_MUTATION_BY_1112=false
|
||||
VM_MUTATION_BY_1112=false
|
||||
CLOUD_MUTATION_BY_1112=false
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_1111R2_METADATA_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+17
-17
@@ -1,38 +1,38 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"channel": "homelab-runtime",
|
||||
"command_id": "SUPPORT-260922-HOMELAB-BACKUP-CLASSIFIED-BLOCKER-REPAIR-1111R2",
|
||||
"command_id": "SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1",
|
||||
"status": "FAIL",
|
||||
"rc": 76,
|
||||
"rc": 3,
|
||||
"host": "pve01",
|
||||
"mode": "change",
|
||||
"component": "homelab-backup-classified-blocker-repair",
|
||||
"started_at_utc": "2026-09-22T05:21:00Z",
|
||||
"finished_at_utc": "2026-09-22T05:21:00Z",
|
||||
"mode": "read-only",
|
||||
"component": "homelab-backup-1111r2-metadata-rca",
|
||||
"started_at_utc": "2026-09-22T05:26:41Z",
|
||||
"finished_at_utc": "2026-09-22T05:26:42Z",
|
||||
"reference_register_checked": true,
|
||||
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
|
||||
"error_register_checked": true,
|
||||
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
|
||||
"command_sha256": "855f589d68c17028701bd05db5ae3c74ce6f61dacfc825289682df6d06ddb0bb",
|
||||
"command_sha256": "7682fd29d69970b8027a5a07ca78e4c4b97a05bc2e5ef1dc75d70b230b1a669a",
|
||||
"duplicate_failed_command_blocked": false,
|
||||
"block_reason": null,
|
||||
"execution_started": true,
|
||||
"change_declared": true,
|
||||
"result_contract_valid": false,
|
||||
"change_declared": false,
|
||||
"result_contract_valid": true,
|
||||
"result_contract_status": null,
|
||||
"result_contract_error": "MISSING_CONTRACT",
|
||||
"command_rc": 4,
|
||||
"changes_made": null,
|
||||
"rollback_started": null,
|
||||
"result_contract_error": null,
|
||||
"command_rc": 3,
|
||||
"changes_made": false,
|
||||
"rollback_started": false,
|
||||
"rollback_restored": null,
|
||||
"mutation_outcome": "UNKNOWN",
|
||||
"mutation_outcome": "NO_MUTATION",
|
||||
"sanitized": true,
|
||||
"secrets_included": false,
|
||||
"private_addresses_included": false,
|
||||
"raw_evidence_retained_locally": true,
|
||||
"raw_evidence_sha256": "3ca024fa1baca32b44342ff9501f19a6fe6968b772077fbc096c8ba55284dad5",
|
||||
"sanitized_output_sha256": "3ca024fa1baca32b44342ff9501f19a6fe6968b772077fbc096c8ba55284dad5",
|
||||
"raw_evidence_sha256": "745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411",
|
||||
"sanitized_output_sha256": "745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411",
|
||||
"output_truncated_in_json": false,
|
||||
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
||||
"output": "WORKERS2_BACKUP_CLASSIFIED_BLOCKER_REPAIR_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-CLASSIFIED-BLOCKER-REPAIR-1111R2\nMODE=change\nCOMPONENT=homelab-backup-classified-blocker-repair\nMUTATION_BOUNDARY=RESET_STALE_PRE1108_VMQUEUE_FAILED_STATE;RUN_EXISTING_CANONICAL_CLOUDB_RESTORE_DRILL;NO_NEW_BACKUP_GENERATION;NO_VM_CT_MUTATION;NO_TIMER_CHANGE;NO_SOURCE_PUSH;NO_CONFIG_REWRITE;POST_REPAIR_AUDIT_ONLY\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nVMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nSCHEDULER_BEFORE_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nVM_QUEUE_PRE_ACTIVE_STATE=failed\nVM_QUEUE_PRE_EXIT_EPOCH=1789579982\nPOLICY_FIX_1108_FINISH_EPOCH=1790030296\nVM_QUEUE_STALE_FAILED_STATE_RESET=true\nVM_QUEUE_POST_ACTIVE_STATE=inactive\nAUDIT_META_CLOUDB_HEALTH=\nAUDIT_META_WT=/root/homelab-ops-cr-2026-1005\nAUDIT_META_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nFAILURE=AUDIT_METADATA_PARSE_FAILED\nSOURCE_REPO_STATE=UNAVAILABLE\nDECISION=HOLD_BACKUP_1111_REPAIR_INCOMPLETE\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1111_FAILURE\nTASK_RESULT=HOLD_HOMELAB_BACKUP_CLASSIFIED_BLOCKER_REPAIR\nCHANGES_MADE_BY_1111R2=true\nPRODUCT_MUTATION_BY_1111R2=false\nVM_MUTATION_BY_1111R2=false\nBACKUP_GENERATION_BY_1111R2=false\nWORKERS2_BACKUP_CLASSIFIED_BLOCKER_REPAIR_END=1\n"
|
||||
"output": "WORKERS2_BACKUP_1111R2_METADATA_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1\nMODE=read-only\nCOMPONENT=homelab-backup-1111r2-metadata-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_SOURCE_AND_STATE_RECONCILE;NO_SYSTEMCTL_START_STOP_RESET;NO_BACKUP;NO_RESTORE;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nVMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nAST_ASSIGN_COUNT_BRANCH=1\nAST_ASSIGN_BRANCH_LINE=22\nAST_ASSIGN_BRANCH_EXPR=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_BRANCH_18=GFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nAUDIT_SOURCE_BRANCH_19=GFSP=\"/etc/homelab-backup/gfs-retention.json\"\nAUDIT_SOURCE_BRANCH_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_BRANCH_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_BRANCH_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_BRANCH_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_BRANCH_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_BRANCH_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nAUDIT_SOURCE_BRANCH_26=\nAST_ASSIGN_COUNT_CLOUDB_HEALTH=1\nAST_ASSIGN_CLOUDB_HEALTH_LINE=24\nAST_ASSIGN_CLOUDB_HEALTH_EXPR=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_CLOUDB_HEALTH_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_CLOUDB_HEALTH_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_CLOUDB_HEALTH_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_CLOUDB_HEALTH_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_CLOUDB_HEALTH_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_CLOUDB_HEALTH_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nAUDIT_SOURCE_CLOUDB_HEALTH_26=\nAUDIT_SOURCE_CLOUDB_HEALTH_27=def run(cmd,timeout=60,env=None):\nAUDIT_SOURCE_CLOUDB_HEALTH_28= try:\nAST_ASSIGN_COUNT_WT=1\nAST_ASSIGN_WT_LINE=21\nAST_ASSIGN_WT_EXPR=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_WT_17=APPQ=\"/usr/local/sbin/homelab-app-cloud-quorum-queue\"\nAUDIT_SOURCE_WT_18=GFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nAUDIT_SOURCE_WT_19=GFSP=\"/etc/homelab-backup/gfs-retention.json\"\nAUDIT_SOURCE_WT_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_WT_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_WT_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_WT_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_WT_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_WT_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nCLOUDB_HEALTH_CANDIDATE_COUNT=0\nCLOUDB_PRODUCER_COUNT=0\nCLOUDB_SERVICE_MATCH_COUNT=0\nSOURCE_WT=/root/homelab-ops-cr-2026-1005\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_HEAD_ALIGNMENT=UNAVAILABLE\nDECISION=HOLD_BACKUP_1112_METADATA_OR_IDENTITY_AMBIGUOUS\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1112_AMBIGUITY\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1111R2_METADATA_RCA\nCHANGES_MADE_BY_1112=false\nPRODUCT_MUTATION_BY_1112=false\nVM_MUTATION_BY_1112=false\nCLOUD_MUTATION_BY_1112=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1111R2_METADATA_RCA_END=1\n"
|
||||
}
|
||||
|
||||
+74
-39
@@ -1,37 +1,37 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-CLASSIFIED-BLOCKER-REPAIR-1111R2
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1
|
||||
STATUS=FAIL
|
||||
RC=76
|
||||
RC=3
|
||||
HOST=pve01
|
||||
MODE=change
|
||||
COMPONENT=homelab-backup-classified-blocker-repair
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1111r2-metadata-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=855f589d68c17028701bd05db5ae3c74ce6f61dacfc825289682df6d06ddb0bb
|
||||
COMMAND_SHA256=7682fd29d69970b8027a5a07ca78e4c4b97a05bc2e5ef1dc75d70b230b1a669a
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=true
|
||||
RESULT_CONTRACT_VALID=false
|
||||
RESULT_CONTRACT_STATUS=UNKNOWN
|
||||
RESULT_CONTRACT_ERROR=MISSING_CONTRACT
|
||||
COMMAND_RC=4
|
||||
CHANGES_MADE=unknown
|
||||
ROLLBACK_STARTED=unknown
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=3
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=UNKNOWN
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=3ca024fa1baca32b44342ff9501f19a6fe6968b772077fbc096c8ba55284dad5
|
||||
SANITIZED_OUTPUT_SHA256=3ca024fa1baca32b44342ff9501f19a6fe6968b772077fbc096c8ba55284dad5
|
||||
RAW_EVIDENCE_SHA256=745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411
|
||||
SANITIZED_OUTPUT_SHA256=745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_BACKUP_CLASSIFIED_BLOCKER_REPAIR_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-CLASSIFIED-BLOCKER-REPAIR-1111R2
|
||||
MODE=change
|
||||
COMPONENT=homelab-backup-classified-blocker-repair
|
||||
MUTATION_BOUNDARY=RESET_STALE_PRE1108_VMQUEUE_FAILED_STATE;RUN_EXISTING_CANONICAL_CLOUDB_RESTORE_DRILL;NO_NEW_BACKUP_GENERATION;NO_VM_CT_MUTATION;NO_TIMER_CHANGE;NO_SOURCE_PUSH;NO_CONFIG_REWRITE;POST_REPAIR_AUDIT_ONLY
|
||||
WORKERS2_BACKUP_1111R2_METADATA_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1111r2-metadata-rca
|
||||
MUTATION_BOUNDARY=NONE;READ_ONLY_SOURCE_AND_STATE_RECONCILE;NO_SYSTEMCTL_START_STOP_RESET;NO_BACKUP;NO_RESTORE;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
REFERENCE_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
@@ -40,27 +40,62 @@ RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
|
||||
AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
|
||||
VMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
|
||||
SCHEDULER_BEFORE_RC=0
|
||||
VM_QUEUE_CURRENT_STATE=success,inactive,dead
|
||||
BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed
|
||||
SCHEDULER_RC=0
|
||||
SELFTEST_REPLACEMENT486=PASS
|
||||
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
|
||||
VM_QUEUE_PRE_ACTIVE_STATE=failed
|
||||
VM_QUEUE_PRE_EXIT_EPOCH=1789579982
|
||||
POLICY_FIX_1108_FINISH_EPOCH=1790030296
|
||||
VM_QUEUE_STALE_FAILED_STATE_RESET=true
|
||||
VM_QUEUE_POST_ACTIVE_STATE=inactive
|
||||
AUDIT_META_CLOUDB_HEALTH=
|
||||
AUDIT_META_WT=/root/homelab-ops-cr-2026-1005
|
||||
AUDIT_META_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
FAILURE=AUDIT_METADATA_PARSE_FAILED
|
||||
SOURCE_REPO_STATE=UNAVAILABLE
|
||||
DECISION=HOLD_BACKUP_1111_REPAIR_INCOMPLETE
|
||||
NEXT_GATE=RECONCILE_ONLY_BACKUP_1111_FAILURE
|
||||
TASK_RESULT=HOLD_HOMELAB_BACKUP_CLASSIFIED_BLOCKER_REPAIR
|
||||
CHANGES_MADE_BY_1111R2=true
|
||||
PRODUCT_MUTATION_BY_1111R2=false
|
||||
VM_MUTATION_BY_1111R2=false
|
||||
BACKUP_GENERATION_BY_1111R2=false
|
||||
WORKERS2_BACKUP_CLASSIFIED_BLOCKER_REPAIR_END=1
|
||||
AST_ASSIGN_COUNT_BRANCH=1
|
||||
AST_ASSIGN_BRANCH_LINE=22
|
||||
AST_ASSIGN_BRANCH_EXPR="cr-2026-1005-backup-v2-health-alerting-remediation"
|
||||
AUDIT_SOURCE_BRANCH_18=GFSH="/usr/local/libexec/homelab-backup-gfs-policy"
|
||||
AUDIT_SOURCE_BRANCH_19=GFSP="/etc/homelab-backup/gfs-retention.json"
|
||||
AUDIT_SOURCE_BRANCH_20=RET="/usr/local/sbin/homelab-backup-retention-gc"
|
||||
AUDIT_SOURCE_BRANCH_21=WT="/root/homelab-ops-cr-2026-1005"
|
||||
AUDIT_SOURCE_BRANCH_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation"
|
||||
AUDIT_SOURCE_BRANCH_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json")
|
||||
AUDIT_SOURCE_BRANCH_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
||||
AUDIT_SOURCE_BRANCH_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")]
|
||||
AUDIT_SOURCE_BRANCH_26=
|
||||
AST_ASSIGN_COUNT_CLOUDB_HEALTH=1
|
||||
AST_ASSIGN_CLOUDB_HEALTH_LINE=24
|
||||
AST_ASSIGN_CLOUDB_HEALTH_EXPR=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_20=RET="/usr/local/sbin/homelab-backup-retention-gc"
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_21=WT="/root/homelab-ops-cr-2026-1005"
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation"
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json")
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")]
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_26=
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_27=def run(cmd,timeout=60,env=None):
|
||||
AUDIT_SOURCE_CLOUDB_HEALTH_28= try:
|
||||
AST_ASSIGN_COUNT_WT=1
|
||||
AST_ASSIGN_WT_LINE=21
|
||||
AST_ASSIGN_WT_EXPR="/root/homelab-ops-cr-2026-1005"
|
||||
AUDIT_SOURCE_WT_17=APPQ="/usr/local/sbin/homelab-app-cloud-quorum-queue"
|
||||
AUDIT_SOURCE_WT_18=GFSH="/usr/local/libexec/homelab-backup-gfs-policy"
|
||||
AUDIT_SOURCE_WT_19=GFSP="/etc/homelab-backup/gfs-retention.json"
|
||||
AUDIT_SOURCE_WT_20=RET="/usr/local/sbin/homelab-backup-retention-gc"
|
||||
AUDIT_SOURCE_WT_21=WT="/root/homelab-ops-cr-2026-1005"
|
||||
AUDIT_SOURCE_WT_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation"
|
||||
AUDIT_SOURCE_WT_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json")
|
||||
AUDIT_SOURCE_WT_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
||||
AUDIT_SOURCE_WT_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")]
|
||||
CLOUDB_HEALTH_CANDIDATE_COUNT=0
|
||||
CLOUDB_PRODUCER_COUNT=0
|
||||
CLOUDB_SERVICE_MATCH_COUNT=0
|
||||
SOURCE_WT=/root/homelab-ops-cr-2026-1005
|
||||
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_HEAD_ALIGNMENT=UNAVAILABLE
|
||||
DECISION=HOLD_BACKUP_1112_METADATA_OR_IDENTITY_AMBIGUOUS
|
||||
NEXT_GATE=RECONCILE_ONLY_BACKUP_1112_AMBIGUITY
|
||||
TASK_RESULT=HOLD_HOMELAB_BACKUP_1111R2_METADATA_RCA
|
||||
CHANGES_MADE_BY_1112=false
|
||||
PRODUCT_MUTATION_BY_1112=false
|
||||
VM_MUTATION_BY_1112=false
|
||||
CLOUD_MUTATION_BY_1112=false
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_1111R2_METADATA_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user