diff --git a/runtime/history/SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1.json b/runtime/history/SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1.json new file mode 100644 index 00000000..7c8fd430 --- /dev/null +++ b/runtime/history/SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "workers2-post99880-failed-service-rca", + "started_at_utc": "2026-10-08T10:42:02Z", + "finished_at_utc": "2026-10-08T10:42:03Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "11e698748c112ca3e4185b12b826760c35640fdff1c7e1061bd8aa3fbfc96895", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "7ca6ea7e3aa46859f0ca5b394b0bba7c5b8c165a64a97059bbc3edd8de17331e", + "sanitized_output_sha256": "7ca6ea7e3aa46859f0ca5b394b0bba7c5b8c165a64a97059bbc3edd8de17331e", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_POST99880_FAILED_SERVICE_RCA=V1\nMODE=read-only\nNO_PRODUCTION_BACKUP_CLOUD_OR_VM_MUTATION=true\nOOB_PINNED_SSH=true\nREGISTRY_SHA256=7f7b93b69e0a0ba262239c04ce14142c601734572afcf6995ab7b604fc9fd2f9\nREGISTRY_TARGET_COUNT=30\nREGISTRY_PVE_GUEST_COUNT=15\nVM100_TARGET_COUNT=1\nVM100_ENABLED=true\nVM100_LOGICAL_ID=vm100\nVM101_TARGET_COUNT=1\nVM101_ENABLED=true\nVM101_LOGICAL_ID=vm101\nVM110_TARGET_COUNT=0\nVM110_ENABLED=false\nVM110_LOGICAL_ID=NOT_UNIQUE\nVM112_TARGET_COUNT=0\nVM112_ENABLED=false\nVM112_LOGICAL_ID=NOT_UNIQUE\nVM150_TARGET_COUNT=0\nVM150_ENABLED=false\nVM150_LOGICAL_ID=NOT_UNIQUE\nVM170_TARGET_COUNT=0\nVM170_ENABLED=false\nVM170_LOGICAL_ID=NOT_UNIQUE\nVM171_TARGET_COUNT=0\nVM171_ENABLED=false\nVM171_LOGICAL_ID=NOT_UNIQUE\nVM210_TARGET_COUNT=0\nVM210_ENABLED=false\nVM210_LOGICAL_ID=NOT_UNIQUE\nREGISTRY_ALL_EIGHT_BY_VMID_ENABLED=false\nAUDIT_99880_LOGICAL_ID_AMBIGUITY_RESOLVED=false\nLOCAL_PVE_VMIDS_COUNT=8\nVM101_CLUSTER_STOPPED=true\nVM101_QM_STOPPED=true\nSERVICE_ACTIVE=failed\nSERVICE_RESULT=exit-code\nSERVICE_EXEC_MAIN_STATUS=1\nSERVICE_ENTRYPOINT_KNOWN=true\nSERVICE_UNIT_EXECSTART_SHA256=c37ae43bbb0ae3c4d4af85015710bf768a57329bac7df5f03b34d11cae66109a\nBACKUP_TIMER_ACTIVE=active\nJOURNAL_QUERY_RC=0\nJOURNAL_BYTES=1542\nJOURNAL_SHA256=4c06c00b80946a35099655b77c21c4afec6dbcf620bfa212437ad66a8873dbf3\nJOURNAL_LINE_COUNT=23\nTRACEBACK_COUNT=1\nSCHEDULED_TARGET_SAMPLES=0\nSCHEDULE_RESULT_SAMPLES=0\nEXCEPTION_SAMPLE_COUNT=1\nNATIVE_SAFE_ERROR_CODE_COUNT=0\nEXCEPTION_1_CLASS=BackupError\nEXCEPTION_1_CODE=TARGET_REGISTRY_COUNT\nTRACE_FRAME_SAMPLE_COUNT=5\nTRACE_FRAME_1_MODULE=homelab-backup-v2\nTRACE_FRAME_1_LINE=25\nTRACE_FRAME_1_FUNC=main\nTRACE_FRAME_2_MODULE=entry504.py\nTRACE_FRAME_2_LINE=123\nTRACE_FRAME_2_FUNC=main\nTRACE_FRAME_3_MODULE=entry504.py\nTRACE_FRAME_3_LINE=89\nTRACE_FRAME_3_FUNC=_load\nTRACE_FRAME_4_MODULE=wrapper.py\nTRACE_FRAME_4_LINE=70\nTRACE_FRAME_4_FUNC=selftest\nTRACE_FRAME_5_MODULE=wrapper.py\nTRACE_FRAME_5_LINE=23\nTRACE_FRAME_5_FUNC=canonical_targets\nSTAGING_FS_FREE_GIB=265\nOLD_QEMU_TMP_FS_FREE_GIB=47\nDECISION=PASS_REAL_CYCLE_EXCEPTION_CAPTURED_NO_WRITE\nNEXT_GATE=FIX_PRECISE_LATEST_BACKUP_EXCEPTION_THEN_RUN_REAL_BACKUP\nACTUAL_NEW_VM100_VM101_BACKUP_VERIFIED=false\nDR_RESTORE_VERIFIED=false\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nEND_MARKER=true\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1.txt b/runtime/history/SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1.txt new file mode 100644 index 00000000..676e08a1 --- /dev/null +++ b/runtime/history/SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1.txt @@ -0,0 +1,113 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=workers2-post99880-failed-service-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=11e698748c112ca3e4185b12b826760c35640fdff1c7e1061bd8aa3fbfc96895 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=7ca6ea7e3aa46859f0ca5b394b0bba7c5b8c165a64a97059bbc3edd8de17331e +SANITIZED_OUTPUT_SHA256=7ca6ea7e3aa46859f0ca5b394b0bba7c5b8c165a64a97059bbc3edd8de17331e +OUTPUT_BEGIN +WORKERS2_POST99880_FAILED_SERVICE_RCA=V1 +MODE=read-only +NO_PRODUCTION_BACKUP_CLOUD_OR_VM_MUTATION=true +OOB_PINNED_SSH=true +REGISTRY_SHA256=7f7b93b69e0a0ba262239c04ce14142c601734572afcf6995ab7b604fc9fd2f9 +REGISTRY_TARGET_COUNT=30 +REGISTRY_PVE_GUEST_COUNT=15 +VM100_TARGET_COUNT=1 +VM100_ENABLED=true +VM100_LOGICAL_ID=vm100 +VM101_TARGET_COUNT=1 +VM101_ENABLED=true +VM101_LOGICAL_ID=vm101 +VM110_TARGET_COUNT=0 +VM110_ENABLED=false +VM110_LOGICAL_ID=NOT_UNIQUE +VM112_TARGET_COUNT=0 +VM112_ENABLED=false +VM112_LOGICAL_ID=NOT_UNIQUE +VM150_TARGET_COUNT=0 +VM150_ENABLED=false +VM150_LOGICAL_ID=NOT_UNIQUE +VM170_TARGET_COUNT=0 +VM170_ENABLED=false +VM170_LOGICAL_ID=NOT_UNIQUE +VM171_TARGET_COUNT=0 +VM171_ENABLED=false +VM171_LOGICAL_ID=NOT_UNIQUE +VM210_TARGET_COUNT=0 +VM210_ENABLED=false +VM210_LOGICAL_ID=NOT_UNIQUE +REGISTRY_ALL_EIGHT_BY_VMID_ENABLED=false +AUDIT_99880_LOGICAL_ID_AMBIGUITY_RESOLVED=false +LOCAL_PVE_VMIDS_COUNT=8 +VM101_CLUSTER_STOPPED=true +VM101_QM_STOPPED=true +SERVICE_ACTIVE=failed +SERVICE_RESULT=exit-code +SERVICE_EXEC_MAIN_STATUS=1 +SERVICE_ENTRYPOINT_KNOWN=true +SERVICE_UNIT_EXECSTART_SHA256=c37ae43bbb0ae3c4d4af85015710bf768a57329bac7df5f03b34d11cae66109a +BACKUP_TIMER_ACTIVE=active +JOURNAL_QUERY_RC=0 +JOURNAL_BYTES=1542 +JOURNAL_SHA256=4c06c00b80946a35099655b77c21c4afec6dbcf620bfa212437ad66a8873dbf3 +JOURNAL_LINE_COUNT=23 +TRACEBACK_COUNT=1 +SCHEDULED_TARGET_SAMPLES=0 +SCHEDULE_RESULT_SAMPLES=0 +EXCEPTION_SAMPLE_COUNT=1 +NATIVE_SAFE_ERROR_CODE_COUNT=0 +EXCEPTION_1_CLASS=BackupError +EXCEPTION_1_CODE=TARGET_REGISTRY_COUNT +TRACE_FRAME_SAMPLE_COUNT=5 +TRACE_FRAME_1_MODULE=homelab-backup-v2 +TRACE_FRAME_1_LINE=25 +TRACE_FRAME_1_FUNC=main +TRACE_FRAME_2_MODULE=entry504.py +TRACE_FRAME_2_LINE=123 +TRACE_FRAME_2_FUNC=main +TRACE_FRAME_3_MODULE=entry504.py +TRACE_FRAME_3_LINE=89 +TRACE_FRAME_3_FUNC=_load +TRACE_FRAME_4_MODULE=wrapper.py +TRACE_FRAME_4_LINE=70 +TRACE_FRAME_4_FUNC=selftest +TRACE_FRAME_5_MODULE=wrapper.py +TRACE_FRAME_5_LINE=23 +TRACE_FRAME_5_FUNC=canonical_targets +STAGING_FS_FREE_GIB=265 +OLD_QEMU_TMP_FS_FREE_GIB=47 +DECISION=PASS_REAL_CYCLE_EXCEPTION_CAPTURED_NO_WRITE +NEXT_GATE=FIX_PRECISE_LATEST_BACKUP_EXCEPTION_THEN_RUN_REAL_BACKUP +ACTUAL_NEW_VM100_VM101_BACKUP_VERIFIED=false +DR_RESTORE_VERIFIED=false +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +END_MARKER=true +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 04709058..7c8fd430 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-261008-WORKERS2-POST99879-8GUEST-CLOUD-DR-AUDIT-99880R1", + "command_id": "SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "workers2-post99879-eight-cloud-dr-aggregate", - "started_at_utc": "2026-10-08T10:31:17Z", - "finished_at_utc": "2026-10-08T10:31:20Z", + "component": "workers2-post99880-failed-service-rca", + "started_at_utc": "2026-10-08T10:42:02Z", + "finished_at_utc": "2026-10-08T10:42:03Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "f2d9ab15fa835f48b7b709dd15337a8649a21f91363eddc863125c5952fd6812", + "command_sha256": "11e698748c112ca3e4185b12b826760c35640fdff1c7e1061bd8aa3fbfc96895", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "4a2d19b4cbe14e09505167f41b0b5deaf01a70baa262aa552d495a3725def9ee", - "sanitized_output_sha256": "4a2d19b4cbe14e09505167f41b0b5deaf01a70baa262aa552d495a3725def9ee", + "raw_evidence_sha256": "7ca6ea7e3aa46859f0ca5b394b0bba7c5b8c165a64a97059bbc3edd8de17331e", + "sanitized_output_sha256": "7ca6ea7e3aa46859f0ca5b394b0bba7c5b8c165a64a97059bbc3edd8de17331e", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_POST99879_AGGREGATED_AUDIT=V1\nMODE=read-only\nNO_PRODUCTION_BACKUP_VM_CLOUD_MUTATION=true\nPREVIOUS_99879_REGISTER_AND_STAGING_REDIRECT=PASS\nPINNED_HOSTKEY_CONFIRMED=true\nLOCAL_PVE_GUEST_COUNT=8\nEIGHT_GUEST_MEMBERSHIP_UNCHANGED=true\nVM101_CLUSTER_STATUS=stopped\nVM101_QM_STATUS_STOPPED=true\nREGISTRY_SHA256=7f7b93b69e0a0ba262239c04ce14142c601734572afcf6995ab7b604fc9fd2f9\nREGISTRY_TARGET_COUNT=30\nREGISTRY_PVE_GUEST_COUNT=15\nVM100_TARGET_COUNT=1\nVM100_TARGET_ENABLED=true\nVM100_TARGET_QEMU=true\nVM101_TARGET_COUNT=1\nVM101_TARGET_ENABLED=true\nVM101_TARGET_QEMU=true\nVM110_TARGET_COUNT=0\nVM110_TARGET_ENABLED=false\nVM112_TARGET_COUNT=0\nVM112_TARGET_ENABLED=false\nVM150_TARGET_COUNT=0\nVM150_TARGET_ENABLED=false\nVM170_TARGET_COUNT=0\nVM170_TARGET_ENABLED=false\nVM171_TARGET_COUNT=0\nVM171_TARGET_ENABLED=false\nVM210_TARGET_COUNT=0\nVM210_TARGET_ENABLED=false\nREGISTRY_EIGHT_TARGETS_ENABLED=false\nLEGACY_SENDMAIL_CONFIG_SHA_STABLE=true\nBACKUP_V2_SERVICE_ACTIVE=failed\nBACKUP_V2_SERVICE_RESULT=exit-code\nBACKUP_V2_SERVICE_EXITSTATUS=1\nBACKUP_V2_TIMER_ACTIVE=active\nBACKUP_V2_TIMER_RESULT=success\nBACKUP_V2_TIMER_EXITSTATUS=UNKNOWN\nBACKUP_AUDIT_SERVICE_ACTIVE=inactive\nBACKUP_AUDIT_SERVICE_RESULT=success\nBACKUP_AUDIT_SERVICE_EXITSTATUS=0\nBACKUP_AUDIT_TIMER_ACTIVE=active\nBACKUP_AUDIT_TIMER_RESULT=success\nBACKUP_AUDIT_TIMER_EXITSTATUS=UNKNOWN\nVM_CLOUD_QUEUE_ACTIVE=inactive\nVM_CLOUD_QUEUE_RESULT=success\nVM_CLOUD_QUEUE_EXITSTATUS=0\nRETENTION_TIMER_ACTIVE=active\nRETENTION_TIMER_RESULT=success\nRETENTION_TIMER_EXITSTATUS=UNKNOWN\nPVE_ACTIVE_TASK_COUNT=0\nPVE_ACTIVE_BACKUP_TASK_COUNT=0\nPVE_ACTIVE_VM101_BACKUP_TASK=false\nBACKUP_CYCLE_JOURNAL_QUERY_OK=true\nBACKUP_CYCLE_JOURNAL_SHA256=df824ebe98d5528819d1f46276b8a4f38cc775529c1cc49594954dd2c5915069\nBACKUP_SCHEDULED_TARGET_OBSERVATIONS=5\nVM100_SEEN_AS_SCHEDULED_TARGET=false\nVM101_SEEN_AS_SCHEDULED_TARGET=false\nBACKUP_SCHEDULE_SUCCESS_SIGNALS=4\nBACKUP_SCHEDULE_FAILURE_SIGNALS=1\nBACKUP_JOURNAL_TRACEBACK_PRESENT=true\nBACKUP_JOURNAL_NO_SPACE_SIGNAL=false\nBACKUP_AUDIT_V2_AVAILABLE=true\nBACKUP_AUDIT_V2_AGE_MIN=13\nBACKUP_AUDIT_LEGACY_AVAILABLE=true\nBACKUP_AUDIT_LEGACY_AGE_MIN=13\nSTATE_JSON_DISCOVERED_COUNT=31\nBACKUP_JSON_DISCOVERED_COUNT=1800\nHEALTH_JSON_DISCOVERED_COUNT=22\nJSON_METADATA_READABLE_COUNT=1853\nVM100_METADATA_CANDIDATES=0\nVM100_STATE_RESULT=NOT_FOUND\nVM100_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=false\nVM100_LOCAL_RESTORE_EVIDENCE=false\nVM100_CLOUD_RECEIPT_NEW_AFTER_99879=false\nVM101_METADATA_CANDIDATES=0\nVM101_STATE_RESULT=NOT_FOUND\nVM101_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=false\nVM101_LOCAL_RESTORE_EVIDENCE=false\nVM101_CLOUD_RECEIPT_NEW_AFTER_99879=false\nVM110_METADATA_CANDIDATES=99\nVM110_STATE_RESULT=DONE\nVM110_STATE_GENERATION_SHA256=9c50cdf9a3519bc384eb14663e67ab32b94b061034d5f46f6e8e980152856a5b\nVM110_LATEST_METADATA_AGE_MIN=23646\nVM110_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true\nVM110_LOCAL_RESTORE_EVIDENCE=false\nVM110_CLOUD_RECEIPT_NEW_AFTER_99879=false\nVM112_METADATA_CANDIDATES=97\nVM112_STATE_RESULT=DONE\nVM112_STATE_GENERATION_SHA256=e0e52cf0b0011bc8f8a60946a6c36783976d4c5b0277640aa5981b736c5f990f\nVM112_LATEST_METADATA_AGE_MIN=49561\nVM112_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true\nVM112_LOCAL_RESTORE_EVIDENCE=false\nVM112_CLOUD_RECEIPT_NEW_AFTER_99879=false\nVM150_METADATA_CANDIDATES=117\nVM150_STATE_RESULT=SUCCESS\nVM150_STATE_LAST_SUCCESS_EPOCH=1791432657\nVM150_NEW_SUCCESS_AFTER_99879=false\nVM150_STATE_GENERATION_SHA256=163289be7c598ce6e09581314bbc1a17082091ef3bcf7ded37890e683b6a61c3\nVM150_LATEST_METADATA_AGE_MIN=380\nVM150_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true\nVM150_LOCAL_RESTORE_EVIDENCE=false\nVM150_CLOUD_RECEIPT_NEW_AFTER_99879=false\nVM170_METADATA_CANDIDATES=114\nVM170_STATE_RESULT=SUCCESS\nVM170_STATE_LAST_SUCCESS_EPOCH=1791448273\nVM170_NEW_SUCCESS_AFTER_99879=false\nVM170_STATE_GENERATION_SHA256=ca3158fd0858945af825ba8662e5c149c8b11445540007e6df6034af11bfa4df\nVM170_LATEST_METADATA_AGE_MIN=120\nVM170_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true\nVM170_LOCAL_RESTORE_EVIDENCE=false\nVM170_CLOUD_RECEIPT_NEW_AFTER_99879=false\nVM171_METADATA_CANDIDATES=168\nVM171_STATE_RESULT=SUCCESS\nVM171_STATE_LAST_SUCCESS_EPOCH=1791450059\nVM171_NEW_SUCCESS_AFTER_99879=false\nVM171_STATE_GENERATION_SHA256=409af5b64e40b00f09653d8aff81d1ee07d58b18cb84de215e8d131e3fee918f\nVM171_LATEST_METADATA_AGE_MIN=90\nVM171_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true\nVM171_LOCAL_RESTORE_EVIDENCE=false\nVM171_CLOUD_RECEIPT_NEW_AFTER_99879=false\nVM210_METADATA_CANDIDATES=4\nVM210_STATE_RESULT=SUCCESS\nVM210_STATE_LAST_SUCCESS_EPOCH=1791452086\nVM210_NEW_SUCCESS_AFTER_99879=false\nVM210_STATE_GENERATION_SHA256=175c6e2afcb06e135ea429d133d12ec0e027b844ca9ea5858e352f73987dea35\nVM210_LATEST_METADATA_AGE_MIN=56\nVM210_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true\nVM210_LOCAL_RESTORE_EVIDENCE=false\nVM210_CLOUD_RECEIPT_NEW_AFTER_99879=false\nWORK_ROOT_FREE_GIB=265\nQEMU_OLD_TMP_FREE_GIB=47\nVM101_OLD_TMP_ARCHIVE_DIR_COUNT=3\nWORK_ROOT_TOPLEVEL_ENTRIES=168\nTARGETS_8_OF_8_REGISTERED=false\nVM100_NEW_SUCCESS_METADATA_AFTER_99879=false\nVM101_NEW_SUCCESS_METADATA_AFTER_99879=false\nINDIVIDUAL_LOCAL_CLOUD_SEAL_RECEIPT_COUNT=6\nINDIVIDUAL_LOCAL_RESTORE_RECEIPT_COUNT=0\nLIVE_REMOTE_CLOUD_READBACK_PERFORMED=false\nDR_RESTORE_EXECUTED_THIS_RUN=false\nDECISION=HOLD_REGISTRY_POSTCONDITION_DRIFT\nNEXT_GATE=TARGETED_REAL_BACKUP_OR_CLOUD_DR_EVIDENCE_FROM_AGGREGATE\nDIAGNOSTIC_COMPLETED=true\nCHANGES_MADE=false\nEND_MARKER=true\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261008-WORKERS2-POST99879-8GUEST-CLOUD-DR-AUDIT-99880R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "WORKERS2_POST99880_FAILED_SERVICE_RCA=V1\nMODE=read-only\nNO_PRODUCTION_BACKUP_CLOUD_OR_VM_MUTATION=true\nOOB_PINNED_SSH=true\nREGISTRY_SHA256=7f7b93b69e0a0ba262239c04ce14142c601734572afcf6995ab7b604fc9fd2f9\nREGISTRY_TARGET_COUNT=30\nREGISTRY_PVE_GUEST_COUNT=15\nVM100_TARGET_COUNT=1\nVM100_ENABLED=true\nVM100_LOGICAL_ID=vm100\nVM101_TARGET_COUNT=1\nVM101_ENABLED=true\nVM101_LOGICAL_ID=vm101\nVM110_TARGET_COUNT=0\nVM110_ENABLED=false\nVM110_LOGICAL_ID=NOT_UNIQUE\nVM112_TARGET_COUNT=0\nVM112_ENABLED=false\nVM112_LOGICAL_ID=NOT_UNIQUE\nVM150_TARGET_COUNT=0\nVM150_ENABLED=false\nVM150_LOGICAL_ID=NOT_UNIQUE\nVM170_TARGET_COUNT=0\nVM170_ENABLED=false\nVM170_LOGICAL_ID=NOT_UNIQUE\nVM171_TARGET_COUNT=0\nVM171_ENABLED=false\nVM171_LOGICAL_ID=NOT_UNIQUE\nVM210_TARGET_COUNT=0\nVM210_ENABLED=false\nVM210_LOGICAL_ID=NOT_UNIQUE\nREGISTRY_ALL_EIGHT_BY_VMID_ENABLED=false\nAUDIT_99880_LOGICAL_ID_AMBIGUITY_RESOLVED=false\nLOCAL_PVE_VMIDS_COUNT=8\nVM101_CLUSTER_STOPPED=true\nVM101_QM_STOPPED=true\nSERVICE_ACTIVE=failed\nSERVICE_RESULT=exit-code\nSERVICE_EXEC_MAIN_STATUS=1\nSERVICE_ENTRYPOINT_KNOWN=true\nSERVICE_UNIT_EXECSTART_SHA256=c37ae43bbb0ae3c4d4af85015710bf768a57329bac7df5f03b34d11cae66109a\nBACKUP_TIMER_ACTIVE=active\nJOURNAL_QUERY_RC=0\nJOURNAL_BYTES=1542\nJOURNAL_SHA256=4c06c00b80946a35099655b77c21c4afec6dbcf620bfa212437ad66a8873dbf3\nJOURNAL_LINE_COUNT=23\nTRACEBACK_COUNT=1\nSCHEDULED_TARGET_SAMPLES=0\nSCHEDULE_RESULT_SAMPLES=0\nEXCEPTION_SAMPLE_COUNT=1\nNATIVE_SAFE_ERROR_CODE_COUNT=0\nEXCEPTION_1_CLASS=BackupError\nEXCEPTION_1_CODE=TARGET_REGISTRY_COUNT\nTRACE_FRAME_SAMPLE_COUNT=5\nTRACE_FRAME_1_MODULE=homelab-backup-v2\nTRACE_FRAME_1_LINE=25\nTRACE_FRAME_1_FUNC=main\nTRACE_FRAME_2_MODULE=entry504.py\nTRACE_FRAME_2_LINE=123\nTRACE_FRAME_2_FUNC=main\nTRACE_FRAME_3_MODULE=entry504.py\nTRACE_FRAME_3_LINE=89\nTRACE_FRAME_3_FUNC=_load\nTRACE_FRAME_4_MODULE=wrapper.py\nTRACE_FRAME_4_LINE=70\nTRACE_FRAME_4_FUNC=selftest\nTRACE_FRAME_5_MODULE=wrapper.py\nTRACE_FRAME_5_LINE=23\nTRACE_FRAME_5_FUNC=canonical_targets\nSTAGING_FS_FREE_GIB=265\nOLD_QEMU_TMP_FS_FREE_GIB=47\nDECISION=PASS_REAL_CYCLE_EXCEPTION_CAPTURED_NO_WRITE\nNEXT_GATE=FIX_PRECISE_LATEST_BACKUP_EXCEPTION_THEN_RUN_REAL_BACKUP\nACTUAL_NEW_VM100_VM101_BACKUP_VERIFIED=false\nDR_RESTORE_VERIFIED=false\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nEND_MARKER=true\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 54b168fc..676e08a1 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-261008-WORKERS2-POST99879-8GUEST-CLOUD-DR-AUDIT-99880R1 +COMMAND_ID=SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=workers2-post99879-eight-cloud-dr-aggregate +COMPONENT=workers2-post99880-failed-service-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=f2d9ab15fa835f48b7b709dd15337a8649a21f91363eddc863125c5952fd6812 +COMMAND_SHA256=11e698748c112ca3e4185b12b826760c35640fdff1c7e1061bd8aa3fbfc96895 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,156 +24,90 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=4a2d19b4cbe14e09505167f41b0b5deaf01a70baa262aa552d495a3725def9ee -SANITIZED_OUTPUT_SHA256=4a2d19b4cbe14e09505167f41b0b5deaf01a70baa262aa552d495a3725def9ee +RAW_EVIDENCE_SHA256=7ca6ea7e3aa46859f0ca5b394b0bba7c5b8c165a64a97059bbc3edd8de17331e +SANITIZED_OUTPUT_SHA256=7ca6ea7e3aa46859f0ca5b394b0bba7c5b8c165a64a97059bbc3edd8de17331e OUTPUT_BEGIN -WORKERS2_POST99879_AGGREGATED_AUDIT=V1 +WORKERS2_POST99880_FAILED_SERVICE_RCA=V1 MODE=read-only -NO_PRODUCTION_BACKUP_VM_CLOUD_MUTATION=true -PREVIOUS_99879_REGISTER_AND_STAGING_REDIRECT=PASS -PINNED_HOSTKEY_CONFIRMED=true -LOCAL_PVE_GUEST_COUNT=8 -EIGHT_GUEST_MEMBERSHIP_UNCHANGED=true -VM101_CLUSTER_STATUS=stopped -VM101_QM_STATUS_STOPPED=true +NO_PRODUCTION_BACKUP_CLOUD_OR_VM_MUTATION=true +OOB_PINNED_SSH=true REGISTRY_SHA256=7f7b93b69e0a0ba262239c04ce14142c601734572afcf6995ab7b604fc9fd2f9 REGISTRY_TARGET_COUNT=30 REGISTRY_PVE_GUEST_COUNT=15 VM100_TARGET_COUNT=1 -VM100_TARGET_ENABLED=true -VM100_TARGET_QEMU=true +VM100_ENABLED=true +VM100_LOGICAL_ID=vm100 VM101_TARGET_COUNT=1 -VM101_TARGET_ENABLED=true -VM101_TARGET_QEMU=true +VM101_ENABLED=true +VM101_LOGICAL_ID=vm101 VM110_TARGET_COUNT=0 -VM110_TARGET_ENABLED=false +VM110_ENABLED=false +VM110_LOGICAL_ID=NOT_UNIQUE VM112_TARGET_COUNT=0 -VM112_TARGET_ENABLED=false +VM112_ENABLED=false +VM112_LOGICAL_ID=NOT_UNIQUE VM150_TARGET_COUNT=0 -VM150_TARGET_ENABLED=false +VM150_ENABLED=false +VM150_LOGICAL_ID=NOT_UNIQUE VM170_TARGET_COUNT=0 -VM170_TARGET_ENABLED=false +VM170_ENABLED=false +VM170_LOGICAL_ID=NOT_UNIQUE VM171_TARGET_COUNT=0 -VM171_TARGET_ENABLED=false +VM171_ENABLED=false +VM171_LOGICAL_ID=NOT_UNIQUE VM210_TARGET_COUNT=0 -VM210_TARGET_ENABLED=false -REGISTRY_EIGHT_TARGETS_ENABLED=false -LEGACY_SENDMAIL_CONFIG_SHA_STABLE=true -BACKUP_V2_SERVICE_ACTIVE=failed -BACKUP_V2_SERVICE_RESULT=exit-code -BACKUP_V2_SERVICE_EXITSTATUS=1 -BACKUP_V2_TIMER_ACTIVE=active -BACKUP_V2_TIMER_RESULT=success -BACKUP_V2_TIMER_EXITSTATUS=UNKNOWN -BACKUP_AUDIT_SERVICE_ACTIVE=inactive -BACKUP_AUDIT_SERVICE_RESULT=success -BACKUP_AUDIT_SERVICE_EXITSTATUS=0 -BACKUP_AUDIT_TIMER_ACTIVE=active -BACKUP_AUDIT_TIMER_RESULT=success -BACKUP_AUDIT_TIMER_EXITSTATUS=UNKNOWN -VM_CLOUD_QUEUE_ACTIVE=inactive -VM_CLOUD_QUEUE_RESULT=success -VM_CLOUD_QUEUE_EXITSTATUS=0 -RETENTION_TIMER_ACTIVE=active -RETENTION_TIMER_RESULT=success -RETENTION_TIMER_EXITSTATUS=UNKNOWN -PVE_ACTIVE_TASK_COUNT=0 -PVE_ACTIVE_BACKUP_TASK_COUNT=0 -PVE_ACTIVE_VM101_BACKUP_TASK=false -BACKUP_CYCLE_JOURNAL_QUERY_OK=true -BACKUP_CYCLE_JOURNAL_SHA256=df824ebe98d5528819d1f46276b8a4f38cc775529c1cc49594954dd2c5915069 -BACKUP_SCHEDULED_TARGET_OBSERVATIONS=5 -VM100_SEEN_AS_SCHEDULED_TARGET=false -VM101_SEEN_AS_SCHEDULED_TARGET=false -BACKUP_SCHEDULE_SUCCESS_SIGNALS=4 -BACKUP_SCHEDULE_FAILURE_SIGNALS=1 -BACKUP_JOURNAL_TRACEBACK_PRESENT=true -BACKUP_JOURNAL_NO_SPACE_SIGNAL=false -BACKUP_AUDIT_V2_AVAILABLE=true -BACKUP_AUDIT_V2_AGE_MIN=13 -BACKUP_AUDIT_LEGACY_AVAILABLE=true -BACKUP_AUDIT_LEGACY_AGE_MIN=13 -STATE_JSON_DISCOVERED_COUNT=31 -BACKUP_JSON_DISCOVERED_COUNT=1800 -HEALTH_JSON_DISCOVERED_COUNT=22 -JSON_METADATA_READABLE_COUNT=1853 -VM100_METADATA_CANDIDATES=0 -VM100_STATE_RESULT=NOT_FOUND -VM100_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=false -VM100_LOCAL_RESTORE_EVIDENCE=false -VM100_CLOUD_RECEIPT_NEW_AFTER_99879=false -VM101_METADATA_CANDIDATES=0 -VM101_STATE_RESULT=NOT_FOUND -VM101_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=false -VM101_LOCAL_RESTORE_EVIDENCE=false -VM101_CLOUD_RECEIPT_NEW_AFTER_99879=false -VM110_METADATA_CANDIDATES=99 -VM110_STATE_RESULT=DONE -VM110_STATE_GENERATION_SHA256=9c50cdf9a3519bc384eb14663e67ab32b94b061034d5f46f6e8e980152856a5b -VM110_LATEST_METADATA_AGE_MIN=23646 -VM110_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true -VM110_LOCAL_RESTORE_EVIDENCE=false -VM110_CLOUD_RECEIPT_NEW_AFTER_99879=false -VM112_METADATA_CANDIDATES=97 -VM112_STATE_RESULT=DONE -VM112_STATE_GENERATION_SHA256=e0e52cf0b0011bc8f8a60946a6c36783976d4c5b0277640aa5981b736c5f990f -VM112_LATEST_METADATA_AGE_MIN=49561 -VM112_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true -VM112_LOCAL_RESTORE_EVIDENCE=false -VM112_CLOUD_RECEIPT_NEW_AFTER_99879=false -VM150_METADATA_CANDIDATES=117 -VM150_STATE_RESULT=SUCCESS -VM150_STATE_LAST_SUCCESS_EPOCH=1791432657 -VM150_NEW_SUCCESS_AFTER_99879=false -VM150_STATE_GENERATION_SHA256=163289be7c598ce6e09581314bbc1a17082091ef3bcf7ded37890e683b6a61c3 -VM150_LATEST_METADATA_AGE_MIN=380 -VM150_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true -VM150_LOCAL_RESTORE_EVIDENCE=false -VM150_CLOUD_RECEIPT_NEW_AFTER_99879=false -VM170_METADATA_CANDIDATES=114 -VM170_STATE_RESULT=SUCCESS -VM170_STATE_LAST_SUCCESS_EPOCH=1791448273 -VM170_NEW_SUCCESS_AFTER_99879=false -VM170_STATE_GENERATION_SHA256=ca3158fd0858945af825ba8662e5c149c8b11445540007e6df6034af11bfa4df -VM170_LATEST_METADATA_AGE_MIN=120 -VM170_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true -VM170_LOCAL_RESTORE_EVIDENCE=false -VM170_CLOUD_RECEIPT_NEW_AFTER_99879=false -VM171_METADATA_CANDIDATES=168 -VM171_STATE_RESULT=SUCCESS -VM171_STATE_LAST_SUCCESS_EPOCH=1791450059 -VM171_NEW_SUCCESS_AFTER_99879=false -VM171_STATE_GENERATION_SHA256=409af5b64e40b00f09653d8aff81d1ee07d58b18cb84de215e8d131e3fee918f -VM171_LATEST_METADATA_AGE_MIN=90 -VM171_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true -VM171_LOCAL_RESTORE_EVIDENCE=false -VM171_CLOUD_RECEIPT_NEW_AFTER_99879=false -VM210_METADATA_CANDIDATES=4 -VM210_STATE_RESULT=SUCCESS -VM210_STATE_LAST_SUCCESS_EPOCH=1791452086 -VM210_NEW_SUCCESS_AFTER_99879=false -VM210_STATE_GENERATION_SHA256=175c6e2afcb06e135ea429d133d12ec0e027b844ca9ea5858e352f73987dea35 -VM210_LATEST_METADATA_AGE_MIN=56 -VM210_LOCAL_TWO_CLOUD_SEAL_EVIDENCE=true -VM210_LOCAL_RESTORE_EVIDENCE=false -VM210_CLOUD_RECEIPT_NEW_AFTER_99879=false -WORK_ROOT_FREE_GIB=265 -QEMU_OLD_TMP_FREE_GIB=47 -VM101_OLD_TMP_ARCHIVE_DIR_COUNT=3 -WORK_ROOT_TOPLEVEL_ENTRIES=168 -TARGETS_8_OF_8_REGISTERED=false -VM100_NEW_SUCCESS_METADATA_AFTER_99879=false -VM101_NEW_SUCCESS_METADATA_AFTER_99879=false -INDIVIDUAL_LOCAL_CLOUD_SEAL_RECEIPT_COUNT=6 -INDIVIDUAL_LOCAL_RESTORE_RECEIPT_COUNT=0 -LIVE_REMOTE_CLOUD_READBACK_PERFORMED=false -DR_RESTORE_EXECUTED_THIS_RUN=false -DECISION=HOLD_REGISTRY_POSTCONDITION_DRIFT -NEXT_GATE=TARGETED_REAL_BACKUP_OR_CLOUD_DR_EVIDENCE_FROM_AGGREGATE -DIAGNOSTIC_COMPLETED=true +VM210_ENABLED=false +VM210_LOGICAL_ID=NOT_UNIQUE +REGISTRY_ALL_EIGHT_BY_VMID_ENABLED=false +AUDIT_99880_LOGICAL_ID_AMBIGUITY_RESOLVED=false +LOCAL_PVE_VMIDS_COUNT=8 +VM101_CLUSTER_STOPPED=true +VM101_QM_STOPPED=true +SERVICE_ACTIVE=failed +SERVICE_RESULT=exit-code +SERVICE_EXEC_MAIN_STATUS=1 +SERVICE_ENTRYPOINT_KNOWN=true +SERVICE_UNIT_EXECSTART_SHA256=c37ae43bbb0ae3c4d4af85015710bf768a57329bac7df5f03b34d11cae66109a +BACKUP_TIMER_ACTIVE=active +JOURNAL_QUERY_RC=0 +JOURNAL_BYTES=1542 +JOURNAL_SHA256=4c06c00b80946a35099655b77c21c4afec6dbcf620bfa212437ad66a8873dbf3 +JOURNAL_LINE_COUNT=23 +TRACEBACK_COUNT=1 +SCHEDULED_TARGET_SAMPLES=0 +SCHEDULE_RESULT_SAMPLES=0 +EXCEPTION_SAMPLE_COUNT=1 +NATIVE_SAFE_ERROR_CODE_COUNT=0 +EXCEPTION_1_CLASS=BackupError +EXCEPTION_1_CODE=TARGET_REGISTRY_COUNT +TRACE_FRAME_SAMPLE_COUNT=5 +TRACE_FRAME_1_MODULE=homelab-backup-v2 +TRACE_FRAME_1_LINE=25 +TRACE_FRAME_1_FUNC=main +TRACE_FRAME_2_MODULE=entry504.py +TRACE_FRAME_2_LINE=123 +TRACE_FRAME_2_FUNC=main +TRACE_FRAME_3_MODULE=entry504.py +TRACE_FRAME_3_LINE=89 +TRACE_FRAME_3_FUNC=_load +TRACE_FRAME_4_MODULE=wrapper.py +TRACE_FRAME_4_LINE=70 +TRACE_FRAME_4_FUNC=selftest +TRACE_FRAME_5_MODULE=wrapper.py +TRACE_FRAME_5_LINE=23 +TRACE_FRAME_5_FUNC=canonical_targets +STAGING_FS_FREE_GIB=265 +OLD_QEMU_TMP_FS_FREE_GIB=47 +DECISION=PASS_REAL_CYCLE_EXCEPTION_CAPTURED_NO_WRITE +NEXT_GATE=FIX_PRECISE_LATEST_BACKUP_EXCEPTION_THEN_RUN_REAL_BACKUP +ACTUAL_NEW_VM100_VM101_BACKUP_VERIFIED=false +DR_RESTORE_VERIFIED=false CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION END_MARKER=true -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261008-WORKERS2-POST99879-8GUEST-CLOUD-DR-AUDIT-99880R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261008-WORKERS2-POST99880-FAILURE-RCA-99881R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END