diff --git a/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1.json b/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1.json new file mode 100644 index 00000000..4724d14e --- /dev/null +++ b/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1", + "status": "FAIL", + "rc": 2, + "host": "pve01", + "mode": "change", + "component": "infra-dr-backupv2-shadow-persistent-producers-stage-repeated-green", + "started_at_utc": "2026-09-14T21:27:56Z", + "finished_at_utc": "2026-09-14T22:12:27Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "aa76dfb18b34ce179de885ef646228fe8557024ca5264aa705dcb0688122a17d", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": true, + "result_contract_valid": true, + "result_contract_status": "FAIL", + "result_contract_error": null, + "command_rc": 2, + "changes_made": true, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "MUTATED", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "19b41a20cfc3738fe67d83e8ec14d34517d1b632b93c48aca9f0f870d04997a3", + "sanitized_output_sha256": "19b41a20cfc3738fe67d83e8ec14d34517d1b632b93c48aca9f0f870d04997a3", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1\nMODE=CHANGE_BACKUPV2_SHADOW_PERSISTENT_PRODUCERS_STAGE_REPEATED_GREEN\nANCHOR_388_COMMIT=bd85b2081fb599bc2d8b7122640f372dde20d881\nFAILED_389_COMMIT=fc1e012f6c552700bc670069672abb71ef180a16\nSIX_COMMAND_CAP_RUN_INDEX=4\nSIX_COMMAND_CAP_MAX=6\nSCHEDULER_CUTOVER=NO\nLEGACY_BACKUP_DISABLES=0\nNO_RETENTION_DELETE=YES\nCURRENT_ID_GITEA_FRESHNESS=404_404_PASS\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nERROR_REGISTER_FRESH_READ=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nREFERENCE_FRESH_READ=OK\nRUNNER_SHA_MATCH=YES\nANCHOR_388_RUNTIME_IDENTITY=PASS\nFAILED_389_NO_MUTATION_BINDING=PASS\nRCA_389=DUPLICATED_REPOSITORY_PATH_IN_GITEA_BASE_URL\nPROVEN_VERSIONED_SOURCE_IDENTITIES=PASS\nRCLONE_PHYSICAL_MAPPING_13_OF_13=PASS\nREQUIRED_LOCAL_EXECUTABLES=PASS\nHOMELAB_APP_CLOUD_QUORUM_QUEUE_TIMER_ENABLED=enabled\nHOMELAB_APP_CLOUD_QUORUM_QUEUE_TIMER_ACTIVE=active\nHOMELAB_VM_CLOUD_QUORUM_QUEUE_TIMER_ENABLED=enabled\nHOMELAB_VM_CLOUD_QUORUM_QUEUE_TIMER_ACTIVE=active\nHOMELAB_CONTROL_BACKUP_TIMER_ENABLED=enabled\nHOMELAB_CONTROL_BACKUP_TIMER_ACTIVE=active\nLEGACY_BACKUP_SERVICES_IDLE=PASS\nSTAGE_PATHS_ABSENT=PASS\nEMBEDDED_ARTIFACTS_PREFLIGHT=PASS\nENGINE_SHA256=f1524bf68b0495cc4a1f1036adb5ce293465547738ce35437a22b700f51cdd35\nTARGET_MANIFEST_SHA256=a723befc7fd42d70af9088406598f18156dc3b4c76d8b34c4bb725dfaa48dd38\nSERVICE_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565\nTIMER_SHA256=14eaa35cded62fb1d94a1d0d69a2864989cc81445635e1ca2ff90d6f0ffc2b63\nPERSISTENT_PRODUCER_STAGE=PASS\nTARGET_REGISTRY_COUNT=28\nREPEATED_GREEN_REPRESENTATIVE_TARGET_COUNT=9\nHOMELAB_BACKUP_V2_TIMER_ENABLED=disabled\nHOMELAB_BACKUP_V2_TIMER_ACTIVE=inactive\nVPS_OPERATOR_KEY_BOOTSTRAP={\"md-relay\":\"ADDED\",\"spb-ingress\":\"ADDED\",\"us-netbird\":\"ALREADY_US_OPERATOR_KEY\"}\nGREEN1_CT112=FAIL\nGREEN1_CT112_REASON=COMMAND_FAILED\nGREEN1_NODE_PVE01=PASS\nGREEN1_NODE_PVE01_GENERATION=v2-node-pve01-20260914T212832Z-160100a8e4f7\nGREEN1_NODE_PVE01_PAYLOAD_BYTES=14578\nGREEN1_NODE_PVE01_REPLICAS=2\nGREEN1_NODE_PVE01_ALLOCATOR_HEALTHY=13\nGREEN1_MD_RELAY=PASS\nGREEN1_MD_RELAY_GENERATION=v2-md-relay-20260914T212856Z-c252e096d17e\nGREEN1_MD_RELAY_PAYLOAD_BYTES=1317\nGREEN1_MD_RELAY_REPLICAS=2\nGREEN1_MD_RELAY_ALLOCATOR_HEALTHY=13\nGREEN1_SPB_INGRESS=PASS\nGREEN1_SPB_INGRESS_GENERATION=v2-spb-ingress-20260914T212920Z-64a0abc29cd6\nGREEN1_SPB_INGRESS_PAYLOAD_BYTES=11804\nGREEN1_SPB_INGRESS_REPLICAS=2\nGREEN1_SPB_INGRESS_ALLOCATOR_HEALTHY=13\nGREEN1_US_NETBIRD=PASS\nGREEN1_US_NETBIRD_GENERATION=v2-us-netbird-20260914T212940Z-684e3e368318\nGREEN1_US_NETBIRD_PAYLOAD_BYTES=33988010\nGREEN1_US_NETBIRD_REPLICAS=2\nGREEN1_US_NETBIRD_ALLOCATOR_HEALTHY=13\nGREEN1_ROUTER_NETCRAZE=FAIL\nGREEN1_ROUTER_NETCRAZE_REASON=ROUTER_PASSFILE_PARSE\nGREEN1_XF_CODEVIPE=PASS\nGREEN1_XF_CODEVIPE_GENERATION=v2-xf-codevipe-20260914T213022Z-f07e0edf5562\nGREEN1_XF_CODEVIPE_PAYLOAD_BYTES=39844622\nGREEN1_XF_CODEVIPE_REPLICAS=2\nGREEN1_XF_CODEVIPE_ALLOCATOR_HEALTHY=13\nGREEN1_XF_ZAKRUTIM=PASS\nGREEN1_XF_ZAKRUTIM_GENERATION=v2-xf-zakrutim-20260914T215009Z-6aef4f5b9eba\nGREEN1_XF_ZAKRUTIM_PAYLOAD_BYTES=44990465\nGREEN1_XF_ZAKRUTIM_REPLICAS=2\nGREEN1_XF_ZAKRUTIM_ALLOCATOR_HEALTHY=13\nGREEN1_CONTROL_BOOTSTRAP_V2=PASS\nGREEN1_CONTROL_BOOTSTRAP_V2_GENERATION=control-bootstrap-v2-pve01-20260914T210856Z\nGREEN1_CONTROL_BOOTSTRAP_V2_PAYLOAD_BYTES=0\nGREEN1_CONTROL_BOOTSTRAP_V2_REPLICAS=2\nGREEN1_TARGET_COUNT=9\nGREEN1_PASS_COUNT=7\nGREEN1_FAIL_COUNT=2\nGREEN1_CLOUD_BACKUP_TARGET_COUNT=8\nGREEN1_CONTROL_VERIFY_TARGET_COUNT=1\nENGINE_STATUS=FAIL\nENGINE_REASON=GREEN1_TARGET_FAILURES\nGREEN1_ENGINE_RC=2\nGREEN1_STDERR_BYTES=0\nGREEN1_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nGREEN1_BACKUP_EXECUTIONS_SUCCESS=6\nGREEN1_CONTROL_ENGINE_VERIFY_SUCCESS=1\nSTATUS=FAIL\nPHASE=GREEN1\nREASON=GREEN1_NOT_ALL_PASS\nCHANGES_MADE=true\nMUTATION_OUTCOME=MUTATED\nCLOUD_DELETIONS=0\nSCHEDULER_CUTOVER=NO\nLEGACY_BACKUP_DISABLES=0\nHOMELAB_RESULT_CONTRACT={\"changes_made\":true,\"command_id\":\"SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"FAIL\",\"version\":1}\n" +} diff --git a/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1.txt b/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1.txt new file mode 100644 index 00000000..f9cbe6a6 --- /dev/null +++ b/runtime/history/SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1.txt @@ -0,0 +1,131 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1 +STATUS=FAIL +RC=2 +HOST=pve01 +MODE=change +COMPONENT=infra-dr-backupv2-shadow-persistent-producers-stage-repeated-green +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=aa76dfb18b34ce179de885ef646228fe8557024ca5264aa705dcb0688122a17d +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=true +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=2 +CHANGES_MADE=true +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=MUTATED +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=19b41a20cfc3738fe67d83e8ec14d34517d1b632b93c48aca9f0f870d04997a3 +SANITIZED_OUTPUT_SHA256=19b41a20cfc3738fe67d83e8ec14d34517d1b632b93c48aca9f0f870d04997a3 +OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1 +MODE=CHANGE_BACKUPV2_SHADOW_PERSISTENT_PRODUCERS_STAGE_REPEATED_GREEN +ANCHOR_388_COMMIT=bd85b2081fb599bc2d8b7122640f372dde20d881 +FAILED_389_COMMIT=fc1e012f6c552700bc670069672abb71ef180a16 +SIX_COMMAND_CAP_RUN_INDEX=4 +SIX_COMMAND_CAP_MAX=6 +SCHEDULER_CUTOVER=NO +LEGACY_BACKUP_DISABLES=0 +NO_RETENTION_DELETE=YES +CURRENT_ID_GITEA_FRESHNESS=404_404_PASS +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +ERROR_REGISTER_FRESH_READ=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +REFERENCE_FRESH_READ=OK +RUNNER_SHA_MATCH=YES +ANCHOR_388_RUNTIME_IDENTITY=PASS +FAILED_389_NO_MUTATION_BINDING=PASS +RCA_389=DUPLICATED_REPOSITORY_PATH_IN_GITEA_BASE_URL +PROVEN_VERSIONED_SOURCE_IDENTITIES=PASS +RCLONE_PHYSICAL_MAPPING_13_OF_13=PASS +REQUIRED_LOCAL_EXECUTABLES=PASS +HOMELAB_APP_CLOUD_QUORUM_QUEUE_TIMER_ENABLED=enabled +HOMELAB_APP_CLOUD_QUORUM_QUEUE_TIMER_ACTIVE=active +HOMELAB_VM_CLOUD_QUORUM_QUEUE_TIMER_ENABLED=enabled +HOMELAB_VM_CLOUD_QUORUM_QUEUE_TIMER_ACTIVE=active +HOMELAB_CONTROL_BACKUP_TIMER_ENABLED=enabled +HOMELAB_CONTROL_BACKUP_TIMER_ACTIVE=active +LEGACY_BACKUP_SERVICES_IDLE=PASS +STAGE_PATHS_ABSENT=PASS +EMBEDDED_ARTIFACTS_PREFLIGHT=PASS +ENGINE_SHA256=f1524bf68b0495cc4a1f1036adb5ce293465547738ce35437a22b700f51cdd35 +TARGET_MANIFEST_SHA256=a723befc7fd42d70af9088406598f18156dc3b4c76d8b34c4bb725dfaa48dd38 +SERVICE_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565 +TIMER_SHA256=14eaa35cded62fb1d94a1d0d69a2864989cc81445635e1ca2ff90d6f0ffc2b63 +PERSISTENT_PRODUCER_STAGE=PASS +TARGET_REGISTRY_COUNT=28 +REPEATED_GREEN_REPRESENTATIVE_TARGET_COUNT=9 +HOMELAB_BACKUP_V2_TIMER_ENABLED=disabled +HOMELAB_BACKUP_V2_TIMER_ACTIVE=inactive +VPS_OPERATOR_KEY_BOOTSTRAP={"md-relay":"ADDED","spb-ingress":"ADDED","us-netbird":"ALREADY_US_OPERATOR_KEY"} +GREEN1_CT112=FAIL +GREEN1_CT112_REASON=COMMAND_FAILED +GREEN1_NODE_PVE01=PASS +GREEN1_NODE_PVE01_GENERATION=v2-node-pve01-20260914T212832Z-160100a8e4f7 +GREEN1_NODE_PVE01_PAYLOAD_BYTES=14578 +GREEN1_NODE_PVE01_REPLICAS=2 +GREEN1_NODE_PVE01_ALLOCATOR_HEALTHY=13 +GREEN1_MD_RELAY=PASS +GREEN1_MD_RELAY_GENERATION=v2-md-relay-20260914T212856Z-c252e096d17e +GREEN1_MD_RELAY_PAYLOAD_BYTES=1317 +GREEN1_MD_RELAY_REPLICAS=2 +GREEN1_MD_RELAY_ALLOCATOR_HEALTHY=13 +GREEN1_SPB_INGRESS=PASS +GREEN1_SPB_INGRESS_GENERATION=v2-spb-ingress-20260914T212920Z-64a0abc29cd6 +GREEN1_SPB_INGRESS_PAYLOAD_BYTES=11804 +GREEN1_SPB_INGRESS_REPLICAS=2 +GREEN1_SPB_INGRESS_ALLOCATOR_HEALTHY=13 +GREEN1_US_NETBIRD=PASS +GREEN1_US_NETBIRD_GENERATION=v2-us-netbird-20260914T212940Z-684e3e368318 +GREEN1_US_NETBIRD_PAYLOAD_BYTES=33988010 +GREEN1_US_NETBIRD_REPLICAS=2 +GREEN1_US_NETBIRD_ALLOCATOR_HEALTHY=13 +GREEN1_ROUTER_NETCRAZE=FAIL +GREEN1_ROUTER_NETCRAZE_REASON=ROUTER_PASSFILE_PARSE +GREEN1_XF_CODEVIPE=PASS +GREEN1_XF_CODEVIPE_GENERATION=v2-xf-codevipe-20260914T213022Z-f07e0edf5562 +GREEN1_XF_CODEVIPE_PAYLOAD_BYTES=39844622 +GREEN1_XF_CODEVIPE_REPLICAS=2 +GREEN1_XF_CODEVIPE_ALLOCATOR_HEALTHY=13 +GREEN1_XF_ZAKRUTIM=PASS +GREEN1_XF_ZAKRUTIM_GENERATION=v2-xf-zakrutim-20260914T215009Z-6aef4f5b9eba +GREEN1_XF_ZAKRUTIM_PAYLOAD_BYTES=44990465 +GREEN1_XF_ZAKRUTIM_REPLICAS=2 +GREEN1_XF_ZAKRUTIM_ALLOCATOR_HEALTHY=13 +GREEN1_CONTROL_BOOTSTRAP_V2=PASS +GREEN1_CONTROL_BOOTSTRAP_V2_GENERATION=control-bootstrap-v2-pve01-20260914T210856Z +GREEN1_CONTROL_BOOTSTRAP_V2_PAYLOAD_BYTES=0 +GREEN1_CONTROL_BOOTSTRAP_V2_REPLICAS=2 +GREEN1_TARGET_COUNT=9 +GREEN1_PASS_COUNT=7 +GREEN1_FAIL_COUNT=2 +GREEN1_CLOUD_BACKUP_TARGET_COUNT=8 +GREEN1_CONTROL_VERIFY_TARGET_COUNT=1 +ENGINE_STATUS=FAIL +ENGINE_REASON=GREEN1_TARGET_FAILURES +GREEN1_ENGINE_RC=2 +GREEN1_STDERR_BYTES=0 +GREEN1_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +GREEN1_BACKUP_EXECUTIONS_SUCCESS=6 +GREEN1_CONTROL_ENGINE_VERIFY_SUCCESS=1 +STATUS=FAIL +PHASE=GREEN1 +REASON=GREEN1_NOT_ALL_PASS +CHANGES_MADE=true +MUTATION_OUTCOME=MUTATED +CLOUD_DELETIONS=0 +SCHEDULER_CUTOVER=NO +LEGACY_BACKUP_DISABLES=0 +HOMELAB_RESULT_CONTRACT={"changes_made":true,"command_id":"SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1","rollback_restored":null,"rollback_started":false,"status":"FAIL","version":1} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 2436f9f0..4724d14e 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-GREEN1-389R1", + "command_id": "SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1", "status": "FAIL", "rc": 2, "host": "pve01", "mode": "change", - "component": "infra-dr-backupv2-shadow-persistent-producers-stage-green1", - "started_at_utc": "2026-09-14T21:22:57Z", - "finished_at_utc": "2026-09-14T21:22:57Z", + "component": "infra-dr-backupv2-shadow-persistent-producers-stage-repeated-green", + "started_at_utc": "2026-09-14T21:27:56Z", + "finished_at_utc": "2026-09-14T22:12:27Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "804f3b576b7fd39e42f30e4ac6f71c7a66a64e431a69ecbbcb4277e5ba865c6c", + "command_sha256": "aa76dfb18b34ce179de885ef646228fe8557024ca5264aa705dcb0688122a17d", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,17 +22,17 @@ "result_contract_status": "FAIL", "result_contract_error": null, "command_rc": 2, - "changes_made": false, + "changes_made": true, "rollback_started": false, "rollback_restored": null, - "mutation_outcome": "NO_MUTATION", + "mutation_outcome": "MUTATED", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "5ca62c7e5fb62c5c80b13013a2b2c94298663b8d58f9f529941688d80dcdfdb6", - "sanitized_output_sha256": "5ca62c7e5fb62c5c80b13013a2b2c94298663b8d58f9f529941688d80dcdfdb6", + "raw_evidence_sha256": "19b41a20cfc3738fe67d83e8ec14d34517d1b632b93c48aca9f0f870d04997a3", + "sanitized_output_sha256": "19b41a20cfc3738fe67d83e8ec14d34517d1b632b93c48aca9f0f870d04997a3", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-GREEN1-389R1\nMODE=CHANGE_BACKUPV2_SHADOW_PERSISTENT_PRODUCERS_STAGE_GREEN1\nANCHOR_388_COMMIT=bd85b2081fb599bc2d8b7122640f372dde20d881\nSIX_COMMAND_CAP_RUN_INDEX=3\nSIX_COMMAND_CAP_MAX=6\nSCHEDULER_CUTOVER=NO\nLEGACY_BACKUP_DISABLES=0\nNO_RETENTION_DELETE=YES\nCURRENT_ID_GITEA_FRESHNESS=404_404_PASS\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nERROR_REGISTER_FRESH_READ=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nREFERENCE_FRESH_READ=OK\nRUNNER_SHA_MATCH=YES\nSTATUS=FAIL\nPHASE=PREFLIGHT\nREASON=ANCHOR_388_FETCH\nCHANGES_MADE=false\nMUTATION_OUTCOME=NO_MUTATION\nCLOUD_DELETIONS=0\nSCHEDULER_CUTOVER=NO\nLEGACY_BACKUP_DISABLES=0\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-GREEN1-389R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"FAIL\",\"version\":1}\n" + "output": "COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1\nMODE=CHANGE_BACKUPV2_SHADOW_PERSISTENT_PRODUCERS_STAGE_REPEATED_GREEN\nANCHOR_388_COMMIT=bd85b2081fb599bc2d8b7122640f372dde20d881\nFAILED_389_COMMIT=fc1e012f6c552700bc670069672abb71ef180a16\nSIX_COMMAND_CAP_RUN_INDEX=4\nSIX_COMMAND_CAP_MAX=6\nSCHEDULER_CUTOVER=NO\nLEGACY_BACKUP_DISABLES=0\nNO_RETENTION_DELETE=YES\nCURRENT_ID_GITEA_FRESHNESS=404_404_PASS\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nERROR_REGISTER_FRESH_READ=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nREFERENCE_FRESH_READ=OK\nRUNNER_SHA_MATCH=YES\nANCHOR_388_RUNTIME_IDENTITY=PASS\nFAILED_389_NO_MUTATION_BINDING=PASS\nRCA_389=DUPLICATED_REPOSITORY_PATH_IN_GITEA_BASE_URL\nPROVEN_VERSIONED_SOURCE_IDENTITIES=PASS\nRCLONE_PHYSICAL_MAPPING_13_OF_13=PASS\nREQUIRED_LOCAL_EXECUTABLES=PASS\nHOMELAB_APP_CLOUD_QUORUM_QUEUE_TIMER_ENABLED=enabled\nHOMELAB_APP_CLOUD_QUORUM_QUEUE_TIMER_ACTIVE=active\nHOMELAB_VM_CLOUD_QUORUM_QUEUE_TIMER_ENABLED=enabled\nHOMELAB_VM_CLOUD_QUORUM_QUEUE_TIMER_ACTIVE=active\nHOMELAB_CONTROL_BACKUP_TIMER_ENABLED=enabled\nHOMELAB_CONTROL_BACKUP_TIMER_ACTIVE=active\nLEGACY_BACKUP_SERVICES_IDLE=PASS\nSTAGE_PATHS_ABSENT=PASS\nEMBEDDED_ARTIFACTS_PREFLIGHT=PASS\nENGINE_SHA256=f1524bf68b0495cc4a1f1036adb5ce293465547738ce35437a22b700f51cdd35\nTARGET_MANIFEST_SHA256=a723befc7fd42d70af9088406598f18156dc3b4c76d8b34c4bb725dfaa48dd38\nSERVICE_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565\nTIMER_SHA256=14eaa35cded62fb1d94a1d0d69a2864989cc81445635e1ca2ff90d6f0ffc2b63\nPERSISTENT_PRODUCER_STAGE=PASS\nTARGET_REGISTRY_COUNT=28\nREPEATED_GREEN_REPRESENTATIVE_TARGET_COUNT=9\nHOMELAB_BACKUP_V2_TIMER_ENABLED=disabled\nHOMELAB_BACKUP_V2_TIMER_ACTIVE=inactive\nVPS_OPERATOR_KEY_BOOTSTRAP={\"md-relay\":\"ADDED\",\"spb-ingress\":\"ADDED\",\"us-netbird\":\"ALREADY_US_OPERATOR_KEY\"}\nGREEN1_CT112=FAIL\nGREEN1_CT112_REASON=COMMAND_FAILED\nGREEN1_NODE_PVE01=PASS\nGREEN1_NODE_PVE01_GENERATION=v2-node-pve01-20260914T212832Z-160100a8e4f7\nGREEN1_NODE_PVE01_PAYLOAD_BYTES=14578\nGREEN1_NODE_PVE01_REPLICAS=2\nGREEN1_NODE_PVE01_ALLOCATOR_HEALTHY=13\nGREEN1_MD_RELAY=PASS\nGREEN1_MD_RELAY_GENERATION=v2-md-relay-20260914T212856Z-c252e096d17e\nGREEN1_MD_RELAY_PAYLOAD_BYTES=1317\nGREEN1_MD_RELAY_REPLICAS=2\nGREEN1_MD_RELAY_ALLOCATOR_HEALTHY=13\nGREEN1_SPB_INGRESS=PASS\nGREEN1_SPB_INGRESS_GENERATION=v2-spb-ingress-20260914T212920Z-64a0abc29cd6\nGREEN1_SPB_INGRESS_PAYLOAD_BYTES=11804\nGREEN1_SPB_INGRESS_REPLICAS=2\nGREEN1_SPB_INGRESS_ALLOCATOR_HEALTHY=13\nGREEN1_US_NETBIRD=PASS\nGREEN1_US_NETBIRD_GENERATION=v2-us-netbird-20260914T212940Z-684e3e368318\nGREEN1_US_NETBIRD_PAYLOAD_BYTES=33988010\nGREEN1_US_NETBIRD_REPLICAS=2\nGREEN1_US_NETBIRD_ALLOCATOR_HEALTHY=13\nGREEN1_ROUTER_NETCRAZE=FAIL\nGREEN1_ROUTER_NETCRAZE_REASON=ROUTER_PASSFILE_PARSE\nGREEN1_XF_CODEVIPE=PASS\nGREEN1_XF_CODEVIPE_GENERATION=v2-xf-codevipe-20260914T213022Z-f07e0edf5562\nGREEN1_XF_CODEVIPE_PAYLOAD_BYTES=39844622\nGREEN1_XF_CODEVIPE_REPLICAS=2\nGREEN1_XF_CODEVIPE_ALLOCATOR_HEALTHY=13\nGREEN1_XF_ZAKRUTIM=PASS\nGREEN1_XF_ZAKRUTIM_GENERATION=v2-xf-zakrutim-20260914T215009Z-6aef4f5b9eba\nGREEN1_XF_ZAKRUTIM_PAYLOAD_BYTES=44990465\nGREEN1_XF_ZAKRUTIM_REPLICAS=2\nGREEN1_XF_ZAKRUTIM_ALLOCATOR_HEALTHY=13\nGREEN1_CONTROL_BOOTSTRAP_V2=PASS\nGREEN1_CONTROL_BOOTSTRAP_V2_GENERATION=control-bootstrap-v2-pve01-20260914T210856Z\nGREEN1_CONTROL_BOOTSTRAP_V2_PAYLOAD_BYTES=0\nGREEN1_CONTROL_BOOTSTRAP_V2_REPLICAS=2\nGREEN1_TARGET_COUNT=9\nGREEN1_PASS_COUNT=7\nGREEN1_FAIL_COUNT=2\nGREEN1_CLOUD_BACKUP_TARGET_COUNT=8\nGREEN1_CONTROL_VERIFY_TARGET_COUNT=1\nENGINE_STATUS=FAIL\nENGINE_REASON=GREEN1_TARGET_FAILURES\nGREEN1_ENGINE_RC=2\nGREEN1_STDERR_BYTES=0\nGREEN1_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nGREEN1_BACKUP_EXECUTIONS_SUCCESS=6\nGREEN1_CONTROL_ENGINE_VERIFY_SUCCESS=1\nSTATUS=FAIL\nPHASE=GREEN1\nREASON=GREEN1_NOT_ALL_PASS\nCHANGES_MADE=true\nMUTATION_OUTCOME=MUTATED\nCLOUD_DELETIONS=0\nSCHEDULER_CUTOVER=NO\nLEGACY_BACKUP_DISABLES=0\nHOMELAB_RESULT_CONTRACT={\"changes_made\":true,\"command_id\":\"SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"FAIL\",\"version\":1}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 0bbdab8d..f9cbe6a6 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-GREEN1-389R1 +COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1 STATUS=FAIL RC=2 HOST=pve01 MODE=change -COMPONENT=infra-dr-backupv2-shadow-persistent-producers-stage-green1 +COMPONENT=infra-dr-backupv2-shadow-persistent-producers-stage-repeated-green REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=804f3b576b7fd39e42f30e4ac6f71c7a66a64e431a69ecbbcb4277e5ba865c6c +COMMAND_SHA256=aa76dfb18b34ce179de885ef646228fe8557024ca5264aa705dcb0688122a17d DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=true @@ -17,20 +17,21 @@ RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=FAIL RESULT_CONTRACT_ERROR=NONE COMMAND_RC=2 -CHANGES_MADE=false +CHANGES_MADE=true ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=NO_MUTATION +MUTATION_OUTCOME=MUTATED SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=5ca62c7e5fb62c5c80b13013a2b2c94298663b8d58f9f529941688d80dcdfdb6 -SANITIZED_OUTPUT_SHA256=5ca62c7e5fb62c5c80b13013a2b2c94298663b8d58f9f529941688d80dcdfdb6 +RAW_EVIDENCE_SHA256=19b41a20cfc3738fe67d83e8ec14d34517d1b632b93c48aca9f0f870d04997a3 +SANITIZED_OUTPUT_SHA256=19b41a20cfc3738fe67d83e8ec14d34517d1b632b93c48aca9f0f870d04997a3 OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-GREEN1-389R1 -MODE=CHANGE_BACKUPV2_SHADOW_PERSISTENT_PRODUCERS_STAGE_GREEN1 +COMMAND_ID=SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1 +MODE=CHANGE_BACKUPV2_SHADOW_PERSISTENT_PRODUCERS_STAGE_REPEATED_GREEN ANCHOR_388_COMMIT=bd85b2081fb599bc2d8b7122640f372dde20d881 -SIX_COMMAND_CAP_RUN_INDEX=3 +FAILED_389_COMMIT=fc1e012f6c552700bc670069672abb71ef180a16 +SIX_COMMAND_CAP_RUN_INDEX=4 SIX_COMMAND_CAP_MAX=6 SCHEDULER_CUTOVER=NO LEGACY_BACKUP_DISABLES=0 @@ -41,15 +42,90 @@ ERROR_REGISTER_FRESH_READ=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 REFERENCE_FRESH_READ=OK RUNNER_SHA_MATCH=YES +ANCHOR_388_RUNTIME_IDENTITY=PASS +FAILED_389_NO_MUTATION_BINDING=PASS +RCA_389=DUPLICATED_REPOSITORY_PATH_IN_GITEA_BASE_URL +PROVEN_VERSIONED_SOURCE_IDENTITIES=PASS +RCLONE_PHYSICAL_MAPPING_13_OF_13=PASS +REQUIRED_LOCAL_EXECUTABLES=PASS +HOMELAB_APP_CLOUD_QUORUM_QUEUE_TIMER_ENABLED=enabled +HOMELAB_APP_CLOUD_QUORUM_QUEUE_TIMER_ACTIVE=active +HOMELAB_VM_CLOUD_QUORUM_QUEUE_TIMER_ENABLED=enabled +HOMELAB_VM_CLOUD_QUORUM_QUEUE_TIMER_ACTIVE=active +HOMELAB_CONTROL_BACKUP_TIMER_ENABLED=enabled +HOMELAB_CONTROL_BACKUP_TIMER_ACTIVE=active +LEGACY_BACKUP_SERVICES_IDLE=PASS +STAGE_PATHS_ABSENT=PASS +EMBEDDED_ARTIFACTS_PREFLIGHT=PASS +ENGINE_SHA256=f1524bf68b0495cc4a1f1036adb5ce293465547738ce35437a22b700f51cdd35 +TARGET_MANIFEST_SHA256=a723befc7fd42d70af9088406598f18156dc3b4c76d8b34c4bb725dfaa48dd38 +SERVICE_SHA256=1ef3e6bc660767a1f29a6a0bfa8d1b7a2419370b2250764ca181c34c93188565 +TIMER_SHA256=14eaa35cded62fb1d94a1d0d69a2864989cc81445635e1ca2ff90d6f0ffc2b63 +PERSISTENT_PRODUCER_STAGE=PASS +TARGET_REGISTRY_COUNT=28 +REPEATED_GREEN_REPRESENTATIVE_TARGET_COUNT=9 +HOMELAB_BACKUP_V2_TIMER_ENABLED=disabled +HOMELAB_BACKUP_V2_TIMER_ACTIVE=inactive +VPS_OPERATOR_KEY_BOOTSTRAP={"md-relay":"ADDED","spb-ingress":"ADDED","us-netbird":"ALREADY_US_OPERATOR_KEY"} +GREEN1_CT112=FAIL +GREEN1_CT112_REASON=COMMAND_FAILED +GREEN1_NODE_PVE01=PASS +GREEN1_NODE_PVE01_GENERATION=v2-node-pve01-20260914T212832Z-160100a8e4f7 +GREEN1_NODE_PVE01_PAYLOAD_BYTES=14578 +GREEN1_NODE_PVE01_REPLICAS=2 +GREEN1_NODE_PVE01_ALLOCATOR_HEALTHY=13 +GREEN1_MD_RELAY=PASS +GREEN1_MD_RELAY_GENERATION=v2-md-relay-20260914T212856Z-c252e096d17e +GREEN1_MD_RELAY_PAYLOAD_BYTES=1317 +GREEN1_MD_RELAY_REPLICAS=2 +GREEN1_MD_RELAY_ALLOCATOR_HEALTHY=13 +GREEN1_SPB_INGRESS=PASS +GREEN1_SPB_INGRESS_GENERATION=v2-spb-ingress-20260914T212920Z-64a0abc29cd6 +GREEN1_SPB_INGRESS_PAYLOAD_BYTES=11804 +GREEN1_SPB_INGRESS_REPLICAS=2 +GREEN1_SPB_INGRESS_ALLOCATOR_HEALTHY=13 +GREEN1_US_NETBIRD=PASS +GREEN1_US_NETBIRD_GENERATION=v2-us-netbird-20260914T212940Z-684e3e368318 +GREEN1_US_NETBIRD_PAYLOAD_BYTES=33988010 +GREEN1_US_NETBIRD_REPLICAS=2 +GREEN1_US_NETBIRD_ALLOCATOR_HEALTHY=13 +GREEN1_ROUTER_NETCRAZE=FAIL +GREEN1_ROUTER_NETCRAZE_REASON=ROUTER_PASSFILE_PARSE +GREEN1_XF_CODEVIPE=PASS +GREEN1_XF_CODEVIPE_GENERATION=v2-xf-codevipe-20260914T213022Z-f07e0edf5562 +GREEN1_XF_CODEVIPE_PAYLOAD_BYTES=39844622 +GREEN1_XF_CODEVIPE_REPLICAS=2 +GREEN1_XF_CODEVIPE_ALLOCATOR_HEALTHY=13 +GREEN1_XF_ZAKRUTIM=PASS +GREEN1_XF_ZAKRUTIM_GENERATION=v2-xf-zakrutim-20260914T215009Z-6aef4f5b9eba +GREEN1_XF_ZAKRUTIM_PAYLOAD_BYTES=44990465 +GREEN1_XF_ZAKRUTIM_REPLICAS=2 +GREEN1_XF_ZAKRUTIM_ALLOCATOR_HEALTHY=13 +GREEN1_CONTROL_BOOTSTRAP_V2=PASS +GREEN1_CONTROL_BOOTSTRAP_V2_GENERATION=control-bootstrap-v2-pve01-20260914T210856Z +GREEN1_CONTROL_BOOTSTRAP_V2_PAYLOAD_BYTES=0 +GREEN1_CONTROL_BOOTSTRAP_V2_REPLICAS=2 +GREEN1_TARGET_COUNT=9 +GREEN1_PASS_COUNT=7 +GREEN1_FAIL_COUNT=2 +GREEN1_CLOUD_BACKUP_TARGET_COUNT=8 +GREEN1_CONTROL_VERIFY_TARGET_COUNT=1 +ENGINE_STATUS=FAIL +ENGINE_REASON=GREEN1_TARGET_FAILURES +GREEN1_ENGINE_RC=2 +GREEN1_STDERR_BYTES=0 +GREEN1_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +GREEN1_BACKUP_EXECUTIONS_SUCCESS=6 +GREEN1_CONTROL_ENGINE_VERIFY_SUCCESS=1 STATUS=FAIL -PHASE=PREFLIGHT -REASON=ANCHOR_388_FETCH -CHANGES_MADE=false -MUTATION_OUTCOME=NO_MUTATION +PHASE=GREEN1 +REASON=GREEN1_NOT_ALL_PASS +CHANGES_MADE=true +MUTATION_OUTCOME=MUTATED CLOUD_DELETIONS=0 SCHEDULER_CUTOVER=NO LEGACY_BACKUP_DISABLES=0 -HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-GREEN1-389R1","rollback_restored":null,"rollback_started":false,"status":"FAIL","version":1} +HOMELAB_RESULT_CONTRACT={"changes_made":true,"command_id":"SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-REPEATED-GREEN-390R1","rollback_restored":null,"rollback_started":false,"status":"FAIL","version":1} OUTPUT_END CHAT_OUTPUT_END