diff --git a/runtime/history/SUPPORT-260910-INFRA-DR-VPS-IDENTITY-166R1.json b/runtime/history/SUPPORT-260910-INFRA-DR-VPS-IDENTITY-166R1.json new file mode 100644 index 00000000..30efa796 --- /dev/null +++ b/runtime/history/SUPPORT-260910-INFRA-DR-VPS-IDENTITY-166R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260910-INFRA-DR-VPS-IDENTITY-166R1", + "status": "FAIL", + "rc": 3, + "host": "pve01", + "mode": "read-only", + "component": "infra-dr-vps-identity-pve01", + "started_at_utc": "2026-09-10T20:52:53Z", + "finished_at_utc": "2026-09-10T20:52:53Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "02256138e22b01f922957c04d8afcdee2246502c879cf772d5cd7c8a8968e848", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 3, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "6d69aec080cb88d265cb8cf04d22caf8928b3dbc2306a29eb53a99fd71fe9914", + "sanitized_output_sha256": "6d69aec080cb88d265cb8cf04d22caf8928b3dbc2306a29eb53a99fd71fe9914", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "INFRA166_ERROR=UNEXPECTED_KeyError\nINFRA166_DECISION=HOLD_VPS_IDENTITY_EXCEPTION_NO_MUTATION\n" +} diff --git a/runtime/history/SUPPORT-260910-INFRA-DR-VPS-IDENTITY-166R1.txt b/runtime/history/SUPPORT-260910-INFRA-DR-VPS-IDENTITY-166R1.txt new file mode 100644 index 00000000..3eff9199 --- /dev/null +++ b/runtime/history/SUPPORT-260910-INFRA-DR-VPS-IDENTITY-166R1.txt @@ -0,0 +1,34 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260910-INFRA-DR-VPS-IDENTITY-166R1 +STATUS=FAIL +RC=3 +HOST=pve01 +MODE=read-only +COMPONENT=infra-dr-vps-identity-pve01 +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=02256138e22b01f922957c04d8afcdee2246502c879cf772d5cd7c8a8968e848 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=3 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=6d69aec080cb88d265cb8cf04d22caf8928b3dbc2306a29eb53a99fd71fe9914 +SANITIZED_OUTPUT_SHA256=6d69aec080cb88d265cb8cf04d22caf8928b3dbc2306a29eb53a99fd71fe9914 +OUTPUT_BEGIN +INFRA166_ERROR=UNEXPECTED_KeyError +INFRA166_DECISION=HOLD_VPS_IDENTITY_EXCEPTION_NO_MUTATION + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index c5416c5d..30efa796 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260910-INFRA-DR-VPS-ROUTER-CLOSURE-165R1", - "status": "OK", - "rc": 0, + "command_id": "SUPPORT-260910-INFRA-DR-VPS-IDENTITY-166R1", + "status": "FAIL", + "rc": 3, "host": "pve01", "mode": "read-only", - "component": "infra-dr-vps-router-closure-pve01", - "started_at_utc": "2026-09-10T20:36:36Z", - "finished_at_utc": "2026-09-10T20:36:39Z", + "component": "infra-dr-vps-identity-pve01", + "started_at_utc": "2026-09-10T20:52:53Z", + "finished_at_utc": "2026-09-10T20:52:53Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "6c6a394dfc56ea904590c2f1e5bc42c1e765fe7a46bb656d304ff5037003c997", + "command_sha256": "02256138e22b01f922957c04d8afcdee2246502c879cf772d5cd7c8a8968e848", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -21,7 +21,7 @@ "result_contract_valid": true, "result_contract_status": null, "result_contract_error": null, - "command_rc": 0, + "command_rc": 3, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "bf2c2002b34c12ffd6d3aaa3b006273efc57b011e8099c24d677f437311f8ae2", - "sanitized_output_sha256": "bf2c2002b34c12ffd6d3aaa3b006273efc57b011e8099c24d677f437311f8ae2", + "raw_evidence_sha256": "6d69aec080cb88d265cb8cf04d22caf8928b3dbc2306a29eb53a99fd71fe9914", + "sanitized_output_sha256": "6d69aec080cb88d265cb8cf04d22caf8928b3dbc2306a29eb53a99fd71fe9914", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "INFRA165_NETBIRD_DETAIL={\"count\":10,\"state\":\"OK\",\"with_ip_hash\":9}\nINFRA165_NETBIRD_DETAIL_001={\"ip_hashes\":[\"0f157abc3c70d23478139db34a670fc2088d80aadffa68984f36ee74edd298bc\",\"b275aed314fd679867c9b200e6c342666a821fc1a41209371204472d07048065\"],\"name\":\"comp01.netbird.selfhosted\",\"os_hints\":[],\"statuses\":[\"Connecting\"]}\nINFRA165_NETBIRD_DETAIL_002={\"ip_hashes\":[\"3f25a12c4a4a1b528acb70748aebdaa912740cb68341ffeaa23c1654cfac1f37\",\"abd1bbea8d823ca14db62a8498fe7db416a44fd556922de63862cd1e06ebc738\"],\"name\":\"desktop-server.netbird.selfhosted\",\"os_hints\":[],\"statuses\":[\"Connecting\"]}\nINFRA165_NETBIRD_DETAIL_003={\"ip_hashes\":[\"cb4c52b8c38c6512aa6733c5d0ea510c76a499fbcb8c1c129d331908e88f2528\",\"cbcf39fece4356231e47ae8708d84f7c3e2a6effd8e5c71451548c094453c0e5\"],\"name\":\"e3qxxx-183-106.netbird.selfhosted\",\"os_hints\":[],\"statuses\":[\"Connecting\"]}\nINFRA165_NETBIRD_DETAIL_004={\"ip_hashes\":[\"8c7543543fdd2a16f1408d0c1df50ed2d8bcb79d5e09b63ae1fdf202e1fbb4b3\",\"97466ca985f4a2f69684aee45965bd2d11a92507761287e0fef56038c1a5dbb9\"],\"name\":\"e3qxxx.netbird.selfhosted\",\"os_hints\":[],\"statuses\":[\"Connecting\"]}\nINFRA165_NETBIRD_DETAIL_005={\"ip_hashes\":[\"3bab8fa15fda36b5b2ac9140d093076e6f90c78861b0cdf02cc22b0e3b5d26be\",\"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440\"],\"name\":\"edge-vm.netbird.selfhosted\",\"os_hints\":[],\"statuses\":[\"Connected\"]}\nINFRA165_NETBIRD_DETAIL_006={\"ip_hashes\":[\"950bcfe548181e4a19ff0236f84126d24165986627fa97a7c45c9ee4b5f95c44\",\"b980955f01148169b9b74541d4d8ece284de73daa9b31a6aaba2a56c9d531179\"],\"name\":\"mail.netbird.selfhosted\",\"os_hints\":[],\"statuses\":[\"Connecting\"]}\nINFRA165_NETBIRD_DETAIL_007={\"ip_hashes\":[],\"name\":\"pve01.netbird.selfhosted\",\"os_hints\":[],\"statuses\":[]}\nINFRA165_NETBIRD_DETAIL_008={\"ip_hashes\":[\"cd36ccf2c54b184a3912e3ec361ef72777776d417360c45839a969319c4465b3\",\"d4b86c5df6a85dc3e63384204c995fb44e5b6055f4265ff691d6f5d2ba20574c\"],\"name\":\"pve02.netbird.selfhosted\",\"os_hints\":[],\"statuses\":[\"Connected\"]}\nINFRA165_NETBIRD_DETAIL_009={\"ip_hashes\":[\"66380a583df95a875aa6325e57dda92f97ebe4e283e04c598485310d62c8c93f\",\"72c55530da4a09d1e32871d19bbe48339cfdd367b777d66e682986cd84bd7159\"],\"name\":\"pve03.netbird.selfhosted\",\"os_hints\":[],\"statuses\":[\"Connected\"]}\nINFRA165_NETBIRD_DETAIL_010={\"ip_hashes\":[\"0f61feb58699a097fdfae741ce53e84cd9eae29e8c8d893a0822f20af9231d4c\",\"d7ae4b440c214f6114944cb388d09b4fa11a0b8ede45358684225e4094485afe\"],\"name\":\"relay.netbird.selfhosted\",\"os_hints\":[],\"statuses\":[\"Connected\"]}\nINFRA165_VPS_SOURCE_001={\"ipv4_hashes\":[\"4f9da1283166a1e9f4e6bc7dd240c76aef85772177f717f92abb6e8e2bb8de90\"],\"netbird_names\":[],\"path\":\"/usr/local/sbin/homelab-router-running-config-mail-cloud\",\"peer_names\":[],\"terms\":[\"backup\",\"mail\",\"router\"]}\nINFRA165_VPS_SOURCE_002={\"ipv4_hashes\":[\"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440\",\"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620\"],\"netbird_names\":[],\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"peer_names\":[\"edge-vm.netbird.selfhosted\"],\"terms\":[\"backup\",\"edge\",\"netbird\",\"vps\"]}\nINFRA165_VPS_SOURCE_003={\"ipv4_hashes\":[\"01df4a7e1c20cbeb1656e21eb725b486a6b35e31bad29c61a4f63d6d4fb5660b\",\"fb345731d21c612ed2cb13088c10fd26bac959c79a7634cb55e23ee6bfb428d9\"],\"netbird_names\":[],\"path\":\"/usr/local/sbin/homelab-router-hard-power-cycle\",\"peer_names\":[],\"terms\":[\"router\"]}\nINFRA165_VPS_SOURCE_004={\"ipv4_hashes\":[\"f1412386aa8db2579aff2636cb9511cacc5fd9880ecab60c048508fbe26ee4d9\"],\"netbird_names\":[],\"path\":\"/usr/local/sbin/homelab-router-watchdog\",\"peer_names\":[],\"terms\":[\"backup\",\"router\"]}\nINFRA165_VPS_SOURCE_005={\"ipv4_hashes\":[\"12ca17b49af2289436f303e0166030a21e525d266e209267433801a8fd4071a0\",\"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440\",\"bc517b4af3298846906234d3d697d3820b0b652ade1880c937c9d71cd802c834\"],\"netbird_names\":[],\"path\":\"/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z\",\"peer_names\":[\"edge-vm.netbird.selfhosted\"],\"terms\":[\"edge\"]}\nINFRA165_VPS_SOURCE_006={\"ipv4_hashes\":[\"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620\"],\"netbird_names\":[],\"path\":\"/usr/local/sbin/homelab-vps-identity-audit\",\"peer_names\":[],\"terms\":[\"backup\",\"netbird\",\"vps\"]}\nINFRA165_VPS_SOURCE_007={\"ipv4_hashes\":[\"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440\",\"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620\",\"fb345731d21c612ed2cb13088c10fd26bac959c79a7634cb55e23ee6bfb428d9\"],\"netbird_names\":[],\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"peer_names\":[\"edge-vm.netbird.selfhosted\"],\"terms\":[\"backup\",\"edge\",\"netbird\",\"vps\"]}\nINFRA165_VPS_SOURCE_008={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/homelab-external-probe-vps-health.timer\",\"peer_names\":[],\"terms\":[\"vps\"]}\nINFRA165_VPS_SOURCE_009={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/homelab-router-watchdog.service\",\"peer_names\":[],\"terms\":[\"router\"]}\nINFRA165_VPS_SOURCE_010={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/homelab-external-probe-vps-health.service\",\"peer_names\":[],\"terms\":[\"vps\"]}\nINFRA165_VPS_SOURCE_011={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/homelab-router-watchdog.timer\",\"peer_names\":[],\"terms\":[\"router\"]}\nINFRA165_VPS_SOURCE_012={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/netbird.service\",\"peer_names\":[],\"terms\":[\"netbird\"]}\nINFRA165_VPS_SOURCE_013={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/homelab-router-running-config-mail-cloud.service\",\"peer_names\":[],\"terms\":[\"backup\",\"mail\",\"router\"]}\nINFRA165_VPS_SOURCE_014={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/homelab-vps-identity-audit.timer\",\"peer_names\":[],\"terms\":[\"vps\"]}\nINFRA165_VPS_SOURCE_015={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/netbird-peers-health.service\",\"peer_names\":[],\"terms\":[\"netbird\"]}\nINFRA165_VPS_SOURCE_016={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/homelab-router-running-config-mail-cloud.timer\",\"peer_names\":[],\"terms\":[\"backup\",\"mail\",\"router\"]}\nINFRA165_VPS_SOURCE_017={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/netbird-peers-health.timer\",\"peer_names\":[],\"terms\":[\"netbird\"]}\nINFRA165_VPS_SOURCE_018={\"ipv4_hashes\":[],\"netbird_names\":[],\"path\":\"/etc/systemd/system/homelab-vps-identity-audit.service\",\"peer_names\":[],\"terms\":[\"vps\"]}\nINFRA165_VPS_SOURCE_019={\"ipv4_hashes\":[\"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16\"],\"netbird_names\":[],\"path\":\"/srv/homelab-ops/kb/runbooks/04_NETBIRD_USA.md\",\"peer_names\":[],\"terms\":[\"netbird\"]}\nINFRA165_VPS_CANDIDATE_001={\"candidate\":false,\"excluded\":[\"CLUSTER_IDENTITY\"],\"ip_hashes\":[\"3bab8fa15fda36b5b2ac9140d093076e6f90c78861b0cdf02cc22b0e3b5d26be\",\"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440\"],\"name\":\"edge-vm.netbird.selfhosted\",\"os_hints\":[],\"roles\":[\"VPS_SOURCE_PATH\"],\"score\":7,\"source_count\":3,\"sources\":[\"/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z\",\"/usr/local/sbin/homelab-external-probe-vps\",\"/usr/local/sbin/homelab-external-probe-vps-health\"],\"statuses\":[\"Connected\"]}\nINFRA165_VPS_CANDIDATE_002={\"candidate\":false,\"excluded\":[],\"ip_hashes\":[\"0f157abc3c70d23478139db34a670fc2088d80aadffa68984f36ee74edd298bc\",\"b275aed314fd679867c9b200e6c342666a821fc1a41209371204472d07048065\"],\"name\":\"comp01.netbird.selfhosted\",\"os_hints\":[],\"roles\":[],\"score\":0,\"source_count\":0,\"sources\":[],\"statuses\":[\"Connecting\"]}\nINFRA165_VPS_CANDIDATE_003={\"candidate\":false,\"excluded\":[],\"ip_hashes\":[\"3f25a12c4a4a1b528acb70748aebdaa912740cb68341ffeaa23c1654cfac1f37\",\"abd1bbea8d823ca14db62a8498fe7db416a44fd556922de63862cd1e06ebc738\"],\"name\":\"desktop-server.netbird.selfhosted\",\"os_hints\":[],\"roles\":[],\"score\":0,\"source_count\":0,\"sources\":[],\"statuses\":[\"Connecting\"]}\nINFRA165_VPS_CANDIDATE_004={\"candidate\":false,\"excluded\":[],\"ip_hashes\":[\"cb4c52b8c38c6512aa6733c5d0ea510c76a499fbcb8c1c129d331908e88f2528\",\"cbcf39fece4356231e47ae8708d84f7c3e2a6effd8e5c71451548c094453c0e5\"],\"name\":\"e3qxxx-183-106.netbird.selfhosted\",\"os_hints\":[],\"roles\":[],\"score\":0,\"source_count\":0,\"sources\":[],\"statuses\":[\"Connecting\"]}\nINFRA165_VPS_CANDIDATE_005={\"candidate\":false,\"excluded\":[],\"ip_hashes\":[\"8c7543543fdd2a16f1408d0c1df50ed2d8bcb79d5e09b63ae1fdf202e1fbb4b3\",\"97466ca985f4a2f69684aee45965bd2d11a92507761287e0fef56038c1a5dbb9\"],\"name\":\"e3qxxx.netbird.selfhosted\",\"os_hints\":[],\"roles\":[],\"score\":0,\"source_count\":0,\"sources\":[],\"statuses\":[\"Connecting\"]}\nINFRA165_VPS_CANDIDATE_006={\"candidate\":false,\"excluded\":[],\"ip_hashes\":[\"950bcfe548181e4a19ff0236f84126d24165986627fa97a7c45c9ee4b5f95c44\",\"b980955f01148169b9b74541d4d8ece284de73daa9b31a6aaba2a56c9d531179\"],\"name\":\"mail.netbird.selfhosted\",\"os_hints\":[],\"roles\":[],\"score\":0,\"source_count\":0,\"sources\":[],\"statuses\":[\"Connecting\"]}\nINFRA165_VPS_CANDIDATE_007={\"candidate\":false,\"excluded\":[\"CLUSTER_IDENTITY\"],\"ip_hashes\":[],\"name\":\"pve01.netbird.selfhosted\",\"os_hints\":[],\"roles\":[],\"score\":0,\"source_count\":0,\"sources\":[],\"statuses\":[]}\nINFRA165_VPS_CANDIDATE_008={\"candidate\":false,\"excluded\":[\"CLUSTER_IDENTITY\"],\"ip_hashes\":[\"cd36ccf2c54b184a3912e3ec361ef72777776d417360c45839a969319c4465b3\",\"d4b86c5df6a85dc3e63384204c995fb44e5b6055f4265ff691d6f5d2ba20574c\"],\"name\":\"pve02.netbird.selfhosted\",\"os_hints\":[],\"roles\":[],\"score\":0,\"source_count\":0,\"sources\":[],\"statuses\":[\"Connected\"]}\nINFRA165_VPS_CANDIDATE_009={\"candidate\":false,\"excluded\":[\"CLUSTER_IDENTITY\"],\"ip_hashes\":[\"66380a583df95a875aa6325e57dda92f97ebe4e283e04c598485310d62c8c93f\",\"72c55530da4a09d1e32871d19bbe48339cfdd367b777d66e682986cd84bd7159\"],\"name\":\"pve03.netbird.selfhosted\",\"os_hints\":[],\"roles\":[],\"score\":0,\"source_count\":0,\"sources\":[],\"statuses\":[\"Connected\"]}\nINFRA165_VPS_CANDIDATE_010={\"candidate\":false,\"excluded\":[\"NETBIRD_RELAY_SERVICE\"],\"ip_hashes\":[\"0f61feb58699a097fdfae741ce53e84cd9eae29e8c8d893a0822f20af9231d4c\",\"d7ae4b440c214f6114944cb388d09b4fa11a0b8ede45358684225e4094485afe\"],\"name\":\"relay.netbird.selfhosted\",\"os_hints\":[],\"roles\":[],\"score\":0,\"source_count\":0,\"sources\":[],\"statuses\":[\"Connected\"]}\nINFRA165_ROUTER_TCP=[{\"banner\":null,\"open\":false,\"port\":22},{\"banner\":null,\"open\":false,\"port\":23},{\"banner\":null,\"open\":true,\"port\":53},{\"banner\":null,\"open\":false,\"port\":80},{\"banner\":null,\"open\":false,\"port\":81},{\"banner\":null,\"open\":false,\"port\":443},{\"banner\":null,\"open\":false,\"port\":8080},{\"banner\":null,\"open\":false,\"port\":8291},{\"banner\":null,\"open\":false,\"port\":8443},{\"banner\":null,\"open\":false,\"port\":8728},{\"banner\":null,\"open\":false,\"port\":8729}]\nINFRA165_ROUTER_IDENTITY={\"gateway_sha256\":\"4f9da1283166a1e9f4e6bc7dd240c76aef85772177f717f92abb6e8e2bb8de90\",\"http\":[],\"identity\":\"UNKNOWN\",\"state\":\"COMPLETE\"}\nINFRA165_ROUTER_SOURCE_001={\"path\":\"/usr/local/sbin/homelab-router-running-config-mail-cloud\",\"terms\":[\"backup\",\"mail\",\"router\"]}\nINFRA165_ROUTER_SOURCE_002={\"path\":\"/usr/local/sbin/homelab-router-hard-power-cycle\",\"terms\":[\"router\"]}\nINFRA165_ROUTER_SOURCE_003={\"path\":\"/usr/local/sbin/homelab-router-watchdog\",\"terms\":[\"backup\",\"router\"]}\nINFRA165_ROUTER_SOURCE_004={\"path\":\"/etc/systemd/system/homelab-router-watchdog.service\",\"terms\":[\"router\"]}\nINFRA165_ROUTER_SOURCE_005={\"path\":\"/etc/systemd/system/homelab-router-watchdog.timer\",\"terms\":[\"router\"]}\nINFRA165_ROUTER_SOURCE_006={\"path\":\"/etc/systemd/system/homelab-router-running-config-mail-cloud.service\",\"terms\":[\"backup\",\"mail\",\"router\"]}\nINFRA165_ROUTER_SOURCE_007={\"path\":\"/etc/systemd/system/homelab-router-running-config-mail-cloud.timer\",\"terms\":[\"backup\",\"mail\",\"router\"]}\nINFRA165_EXTERNAL_SSH_ATTEMPTS=0\nINFRA165_ROUTER_AUTH_ATTEMPTS=0\nINFRA165_MUTATIONS=0\nINFRA165_DECISION=VPS_ROUTER_CLOSURE_CAPTURED_NO_MUTATION\n" + "output": "INFRA166_ERROR=UNEXPECTED_KeyError\nINFRA166_DECISION=HOLD_VPS_IDENTITY_EXCEPTION_NO_MUTATION\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 7ea84e49..3eff9199 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260910-INFRA-DR-VPS-ROUTER-CLOSURE-165R1 -STATUS=OK -RC=0 +COMMAND_ID=SUPPORT-260910-INFRA-DR-VPS-IDENTITY-166R1 +STATUS=FAIL +RC=3 HOST=pve01 MODE=read-only -COMPONENT=infra-dr-vps-router-closure-pve01 +COMPONENT=infra-dr-vps-identity-pve01 REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=6c6a394dfc56ea904590c2f1e5bc42c1e765fe7a46bb656d304ff5037003c997 +COMMAND_SHA256=02256138e22b01f922957c04d8afcdee2246502c879cf772d5cd7c8a8968e848 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=0 +COMMAND_RC=3 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,62 +24,11 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=bf2c2002b34c12ffd6d3aaa3b006273efc57b011e8099c24d677f437311f8ae2 -SANITIZED_OUTPUT_SHA256=bf2c2002b34c12ffd6d3aaa3b006273efc57b011e8099c24d677f437311f8ae2 +RAW_EVIDENCE_SHA256=6d69aec080cb88d265cb8cf04d22caf8928b3dbc2306a29eb53a99fd71fe9914 +SANITIZED_OUTPUT_SHA256=6d69aec080cb88d265cb8cf04d22caf8928b3dbc2306a29eb53a99fd71fe9914 OUTPUT_BEGIN -INFRA165_NETBIRD_DETAIL={"count":10,"state":"OK","with_ip_hash":9} -INFRA165_NETBIRD_DETAIL_001={"ip_hashes":["0f157abc3c70d23478139db34a670fc2088d80aadffa68984f36ee74edd298bc","b275aed314fd679867c9b200e6c342666a821fc1a41209371204472d07048065"],"name":"comp01.netbird.selfhosted","os_hints":[],"statuses":["Connecting"]} -INFRA165_NETBIRD_DETAIL_002={"ip_hashes":["3f25a12c4a4a1b528acb70748aebdaa912740cb68341ffeaa23c1654cfac1f37","abd1bbea8d823ca14db62a8498fe7db416a44fd556922de63862cd1e06ebc738"],"name":"desktop-server.netbird.selfhosted","os_hints":[],"statuses":["Connecting"]} -INFRA165_NETBIRD_DETAIL_003={"ip_hashes":["cb4c52b8c38c6512aa6733c5d0ea510c76a499fbcb8c1c129d331908e88f2528","cbcf39fece4356231e47ae8708d84f7c3e2a6effd8e5c71451548c094453c0e5"],"name":"e3qxxx-183-106.netbird.selfhosted","os_hints":[],"statuses":["Connecting"]} -INFRA165_NETBIRD_DETAIL_004={"ip_hashes":["8c7543543fdd2a16f1408d0c1df50ed2d8bcb79d5e09b63ae1fdf202e1fbb4b3","97466ca985f4a2f69684aee45965bd2d11a92507761287e0fef56038c1a5dbb9"],"name":"e3qxxx.netbird.selfhosted","os_hints":[],"statuses":["Connecting"]} -INFRA165_NETBIRD_DETAIL_005={"ip_hashes":["3bab8fa15fda36b5b2ac9140d093076e6f90c78861b0cdf02cc22b0e3b5d26be","5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440"],"name":"edge-vm.netbird.selfhosted","os_hints":[],"statuses":["Connected"]} -INFRA165_NETBIRD_DETAIL_006={"ip_hashes":["950bcfe548181e4a19ff0236f84126d24165986627fa97a7c45c9ee4b5f95c44","b980955f01148169b9b74541d4d8ece284de73daa9b31a6aaba2a56c9d531179"],"name":"mail.netbird.selfhosted","os_hints":[],"statuses":["Connecting"]} -INFRA165_NETBIRD_DETAIL_007={"ip_hashes":[],"name":"pve01.netbird.selfhosted","os_hints":[],"statuses":[]} -INFRA165_NETBIRD_DETAIL_008={"ip_hashes":["cd36ccf2c54b184a3912e3ec361ef72777776d417360c45839a969319c4465b3","d4b86c5df6a85dc3e63384204c995fb44e5b6055f4265ff691d6f5d2ba20574c"],"name":"pve02.netbird.selfhosted","os_hints":[],"statuses":["Connected"]} -INFRA165_NETBIRD_DETAIL_009={"ip_hashes":["66380a583df95a875aa6325e57dda92f97ebe4e283e04c598485310d62c8c93f","72c55530da4a09d1e32871d19bbe48339cfdd367b777d66e682986cd84bd7159"],"name":"pve03.netbird.selfhosted","os_hints":[],"statuses":["Connected"]} -INFRA165_NETBIRD_DETAIL_010={"ip_hashes":["0f61feb58699a097fdfae741ce53e84cd9eae29e8c8d893a0822f20af9231d4c","d7ae4b440c214f6114944cb388d09b4fa11a0b8ede45358684225e4094485afe"],"name":"relay.netbird.selfhosted","os_hints":[],"statuses":["Connected"]} -INFRA165_VPS_SOURCE_001={"ipv4_hashes":["4f9da1283166a1e9f4e6bc7dd240c76aef85772177f717f92abb6e8e2bb8de90"],"netbird_names":[],"path":"/usr/local/sbin/homelab-router-running-config-mail-cloud","peer_names":[],"terms":["backup","mail","router"]} -INFRA165_VPS_SOURCE_002={"ipv4_hashes":["5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440","f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620"],"netbird_names":[],"path":"/usr/local/sbin/homelab-external-probe-vps","peer_names":["edge-vm.netbird.selfhosted"],"terms":["backup","edge","netbird","vps"]} -INFRA165_VPS_SOURCE_003={"ipv4_hashes":["01df4a7e1c20cbeb1656e21eb725b486a6b35e31bad29c61a4f63d6d4fb5660b","fb345731d21c612ed2cb13088c10fd26bac959c79a7634cb55e23ee6bfb428d9"],"netbird_names":[],"path":"/usr/local/sbin/homelab-router-hard-power-cycle","peer_names":[],"terms":["router"]} -INFRA165_VPS_SOURCE_004={"ipv4_hashes":["f1412386aa8db2579aff2636cb9511cacc5fd9880ecab60c048508fbe26ee4d9"],"netbird_names":[],"path":"/usr/local/sbin/homelab-router-watchdog","peer_names":[],"terms":["backup","router"]} -INFRA165_VPS_SOURCE_005={"ipv4_hashes":["12ca17b49af2289436f303e0166030a21e525d266e209267433801a8fd4071a0","5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440","bc517b4af3298846906234d3d697d3820b0b652ade1880c937c9d71cd802c834"],"netbird_names":[],"path":"/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z","peer_names":["edge-vm.netbird.selfhosted"],"terms":["edge"]} -INFRA165_VPS_SOURCE_006={"ipv4_hashes":["f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620"],"netbird_names":[],"path":"/usr/local/sbin/homelab-vps-identity-audit","peer_names":[],"terms":["backup","netbird","vps"]} -INFRA165_VPS_SOURCE_007={"ipv4_hashes":["5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440","f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620","fb345731d21c612ed2cb13088c10fd26bac959c79a7634cb55e23ee6bfb428d9"],"netbird_names":[],"path":"/usr/local/sbin/homelab-external-probe-vps-health","peer_names":["edge-vm.netbird.selfhosted"],"terms":["backup","edge","netbird","vps"]} -INFRA165_VPS_SOURCE_008={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/homelab-external-probe-vps-health.timer","peer_names":[],"terms":["vps"]} -INFRA165_VPS_SOURCE_009={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/homelab-router-watchdog.service","peer_names":[],"terms":["router"]} -INFRA165_VPS_SOURCE_010={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/homelab-external-probe-vps-health.service","peer_names":[],"terms":["vps"]} -INFRA165_VPS_SOURCE_011={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/homelab-router-watchdog.timer","peer_names":[],"terms":["router"]} -INFRA165_VPS_SOURCE_012={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/netbird.service","peer_names":[],"terms":["netbird"]} -INFRA165_VPS_SOURCE_013={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/homelab-router-running-config-mail-cloud.service","peer_names":[],"terms":["backup","mail","router"]} -INFRA165_VPS_SOURCE_014={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/homelab-vps-identity-audit.timer","peer_names":[],"terms":["vps"]} -INFRA165_VPS_SOURCE_015={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/netbird-peers-health.service","peer_names":[],"terms":["netbird"]} -INFRA165_VPS_SOURCE_016={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/homelab-router-running-config-mail-cloud.timer","peer_names":[],"terms":["backup","mail","router"]} -INFRA165_VPS_SOURCE_017={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/netbird-peers-health.timer","peer_names":[],"terms":["netbird"]} -INFRA165_VPS_SOURCE_018={"ipv4_hashes":[],"netbird_names":[],"path":"/etc/systemd/system/homelab-vps-identity-audit.service","peer_names":[],"terms":["vps"]} -INFRA165_VPS_SOURCE_019={"ipv4_hashes":["1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16"],"netbird_names":[],"path":"/srv/homelab-ops/kb/runbooks/04_NETBIRD_USA.md","peer_names":[],"terms":["netbird"]} -INFRA165_VPS_CANDIDATE_001={"candidate":false,"excluded":["CLUSTER_IDENTITY"],"ip_hashes":["3bab8fa15fda36b5b2ac9140d093076e6f90c78861b0cdf02cc22b0e3b5d26be","5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440"],"name":"edge-vm.netbird.selfhosted","os_hints":[],"roles":["VPS_SOURCE_PATH"],"score":7,"source_count":3,"sources":["/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z","/usr/local/sbin/homelab-external-probe-vps","/usr/local/sbin/homelab-external-probe-vps-health"],"statuses":["Connected"]} -INFRA165_VPS_CANDIDATE_002={"candidate":false,"excluded":[],"ip_hashes":["0f157abc3c70d23478139db34a670fc2088d80aadffa68984f36ee74edd298bc","b275aed314fd679867c9b200e6c342666a821fc1a41209371204472d07048065"],"name":"comp01.netbird.selfhosted","os_hints":[],"roles":[],"score":0,"source_count":0,"sources":[],"statuses":["Connecting"]} -INFRA165_VPS_CANDIDATE_003={"candidate":false,"excluded":[],"ip_hashes":["3f25a12c4a4a1b528acb70748aebdaa912740cb68341ffeaa23c1654cfac1f37","abd1bbea8d823ca14db62a8498fe7db416a44fd556922de63862cd1e06ebc738"],"name":"desktop-server.netbird.selfhosted","os_hints":[],"roles":[],"score":0,"source_count":0,"sources":[],"statuses":["Connecting"]} -INFRA165_VPS_CANDIDATE_004={"candidate":false,"excluded":[],"ip_hashes":["cb4c52b8c38c6512aa6733c5d0ea510c76a499fbcb8c1c129d331908e88f2528","cbcf39fece4356231e47ae8708d84f7c3e2a6effd8e5c71451548c094453c0e5"],"name":"e3qxxx-183-106.netbird.selfhosted","os_hints":[],"roles":[],"score":0,"source_count":0,"sources":[],"statuses":["Connecting"]} -INFRA165_VPS_CANDIDATE_005={"candidate":false,"excluded":[],"ip_hashes":["8c7543543fdd2a16f1408d0c1df50ed2d8bcb79d5e09b63ae1fdf202e1fbb4b3","97466ca985f4a2f69684aee45965bd2d11a92507761287e0fef56038c1a5dbb9"],"name":"e3qxxx.netbird.selfhosted","os_hints":[],"roles":[],"score":0,"source_count":0,"sources":[],"statuses":["Connecting"]} -INFRA165_VPS_CANDIDATE_006={"candidate":false,"excluded":[],"ip_hashes":["950bcfe548181e4a19ff0236f84126d24165986627fa97a7c45c9ee4b5f95c44","b980955f01148169b9b74541d4d8ece284de73daa9b31a6aaba2a56c9d531179"],"name":"mail.netbird.selfhosted","os_hints":[],"roles":[],"score":0,"source_count":0,"sources":[],"statuses":["Connecting"]} -INFRA165_VPS_CANDIDATE_007={"candidate":false,"excluded":["CLUSTER_IDENTITY"],"ip_hashes":[],"name":"pve01.netbird.selfhosted","os_hints":[],"roles":[],"score":0,"source_count":0,"sources":[],"statuses":[]} -INFRA165_VPS_CANDIDATE_008={"candidate":false,"excluded":["CLUSTER_IDENTITY"],"ip_hashes":["cd36ccf2c54b184a3912e3ec361ef72777776d417360c45839a969319c4465b3","d4b86c5df6a85dc3e63384204c995fb44e5b6055f4265ff691d6f5d2ba20574c"],"name":"pve02.netbird.selfhosted","os_hints":[],"roles":[],"score":0,"source_count":0,"sources":[],"statuses":["Connected"]} -INFRA165_VPS_CANDIDATE_009={"candidate":false,"excluded":["CLUSTER_IDENTITY"],"ip_hashes":["66380a583df95a875aa6325e57dda92f97ebe4e283e04c598485310d62c8c93f","72c55530da4a09d1e32871d19bbe48339cfdd367b777d66e682986cd84bd7159"],"name":"pve03.netbird.selfhosted","os_hints":[],"roles":[],"score":0,"source_count":0,"sources":[],"statuses":["Connected"]} -INFRA165_VPS_CANDIDATE_010={"candidate":false,"excluded":["NETBIRD_RELAY_SERVICE"],"ip_hashes":["0f61feb58699a097fdfae741ce53e84cd9eae29e8c8d893a0822f20af9231d4c","d7ae4b440c214f6114944cb388d09b4fa11a0b8ede45358684225e4094485afe"],"name":"relay.netbird.selfhosted","os_hints":[],"roles":[],"score":0,"source_count":0,"sources":[],"statuses":["Connected"]} -INFRA165_ROUTER_TCP=[{"banner":null,"open":false,"port":22},{"banner":null,"open":false,"port":23},{"banner":null,"open":true,"port":53},{"banner":null,"open":false,"port":80},{"banner":null,"open":false,"port":81},{"banner":null,"open":false,"port":443},{"banner":null,"open":false,"port":8080},{"banner":null,"open":false,"port":8291},{"banner":null,"open":false,"port":8443},{"banner":null,"open":false,"port":8728},{"banner":null,"open":false,"port":8729}] -INFRA165_ROUTER_IDENTITY={"gateway_sha256":"4f9da1283166a1e9f4e6bc7dd240c76aef85772177f717f92abb6e8e2bb8de90","http":[],"identity":"UNKNOWN","state":"COMPLETE"} -INFRA165_ROUTER_SOURCE_001={"path":"/usr/local/sbin/homelab-router-running-config-mail-cloud","terms":["backup","mail","router"]} -INFRA165_ROUTER_SOURCE_002={"path":"/usr/local/sbin/homelab-router-hard-power-cycle","terms":["router"]} -INFRA165_ROUTER_SOURCE_003={"path":"/usr/local/sbin/homelab-router-watchdog","terms":["backup","router"]} -INFRA165_ROUTER_SOURCE_004={"path":"/etc/systemd/system/homelab-router-watchdog.service","terms":["router"]} -INFRA165_ROUTER_SOURCE_005={"path":"/etc/systemd/system/homelab-router-watchdog.timer","terms":["router"]} -INFRA165_ROUTER_SOURCE_006={"path":"/etc/systemd/system/homelab-router-running-config-mail-cloud.service","terms":["backup","mail","router"]} -INFRA165_ROUTER_SOURCE_007={"path":"/etc/systemd/system/homelab-router-running-config-mail-cloud.timer","terms":["backup","mail","router"]} -INFRA165_EXTERNAL_SSH_ATTEMPTS=0 -INFRA165_ROUTER_AUTH_ATTEMPTS=0 -INFRA165_MUTATIONS=0 -INFRA165_DECISION=VPS_ROUTER_CLOSURE_CAPTURED_NO_MUTATION +INFRA166_ERROR=UNEXPECTED_KeyError +INFRA166_DECISION=HOLD_VPS_IDENTITY_EXCEPTION_NO_MUTATION OUTPUT_END CHAT_OUTPUT_END