diff --git a/runtime/history/SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1.json b/runtime/history/SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1.json new file mode 100644 index 00000000..e0160a21 --- /dev/null +++ b/runtime/history/SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "infra-dr-vps-unit-trace-pve01", + "started_at_utc": "2026-09-11T03:49:23Z", + "finished_at_utc": "2026-09-11T03:49:23Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "59ac720637b5a262dac71928edd44bbbf906f7be1b225a371a55f1074eb0a4a4", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "5e8e4e3d538b446a7c0c0deb37ca79eba873606c075765fd974bc6b0cbf7936e", + "sanitized_output_sha256": "5e8e4e3d538b446a7c0c0deb37ca79eba873606c075765fd974bc6b0cbf7936e", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "INFRA171_BEGIN\nCOMMAND_ID=SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1\nMODE=READ_ONLY\nPRIOR_170_COMMIT=5c9001893ac691f04b4b8551c801a41c2ddd507c\nINFRA171_ROLE_DNS={\"MD_RELAY\":{\"hostname\":\"nb.pvepro.ru\",\"ip_refs\":[{\"address_class\":\"PUBLIC\",\"ip_sha256\":\"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16\"}],\"rc\":0},\"SPB_INGRESS\":{\"hostname\":\"newfi.ru\",\"ip_refs\":[{\"address_class\":\"PUBLIC\",\"identity_marker\":\"HISTORICAL_SPB_INGRESS\",\"ip_sha256\":\"6d6149f42864ce03daf22828cf236bf64d5d9a004040746ddbb051d71c9fcbc5\"}],\"rc\":0},\"US_NETBIRD\":{\"hostname\":\"nb.gram1.ru\",\"ip_refs\":[{\"address_class\":\"PUBLIC\",\"ip_sha256\":\"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16\"}],\"rc\":0}}\nINFRA171_UNITS={\"count\":2,\"rows\":[{\"directives\":[{\"key\":\"User\",\"line\":9,\"value\":{\"kind\":\"SANITIZED\",\"text\":\"root\"}},{\"key\":\"Group\",\"line\":10,\"value\":{\"kind\":\"SANITIZED\",\"text\":\"root\"}},{\"key\":\"ExecStart\",\"line\":11,\"value\":{\"kind\":\"SANITIZED\",\"text\":\"/usr/local/sbin/homelab-ingress-standby-refresh\"}}],\"file\":{\"basename\":\"homelab-ingress-standby-refresh.service\",\"path_sha256\":\"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e\",\"scope\":\"ETC\"}},{\"directives\":[],\"file\":{\"basename\":\"homelab-ingress-standby-refresh.timer\",\"path_sha256\":\"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1\",\"scope\":\"ETC\"}}]}\nINFRA171_FILES={\"bytes_read\":21032,\"count\":6,\"files\":[{\"basename\":\"homelab-ingress-standby-refresh.service\",\"path_sha256\":\"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e\",\"scope\":\"ETC\"},{\"basename\":\"homelab-ingress-standby-refresh.timer\",\"path_sha256\":\"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1\",\"scope\":\"ETC\"},{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},{\"basename\":\"homelab-ingress-failover\",\"path\":\"/usr/local/sbin/homelab-ingress-failover\",\"scope\":\"USR_LOCAL\"},{\"basename\":\"homelab-ingress-standby-refresh\",\"path\":\"/usr/local/sbin/homelab-ingress-standby-refresh\",\"scope\":\"USR_LOCAL\"}]}\nINFRA171_SAFE_ASSIGNMENTS={\"count\":19,\"rows\":[{\"key\":\"User\",\"kind\":\"USER\",\"line\":9,\"source\":{\"basename\":\"homelab-ingress-standby-refresh.service\",\"path_sha256\":\"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e\",\"scope\":\"ETC\"},\"value\":\"root\"},{\"key\":\"ExecStart\",\"kind\":\"RELEVANT_KEY\",\"line\":11,\"source\":{\"basename\":\"homelab-ingress-standby-refresh.service\",\"path_sha256\":\"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e\",\"scope\":\"ETC\"},\"value_sha256\":\"13d114d64c8f3496ac6b3a8eee4a94ae9e583182842310e10dc40f15561e400c\"},{\"key\":\"REPORT\",\"kind\":\"RELEVANT_KEY\",\"line\":5,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"16de6e2523ad6290564a03adb219172bc0031e08d824a49be9838f00eb297f61\"},{\"key\":\"vps_trust\",\"kind\":\"RELEVANT_KEY\",\"line\":37,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"33003123cbc00728f9934444716841ec1ff9b529c997839691bb14c20d436e53\"},{\"key\":\"trusted_netbird_identity\",\"kind\":\"RELEVANT_KEY\",\"line\":38,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"ebb24cf2960c2243b155740605cba8127cedb6e016de2d44b914019d3fdbe085\"},{\"key\":\"trusted_netbird_ip\",\"kind\":\"RELEVANT_KEY\",\"line\":39,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440\"},{\"key\":\"public_vps_ip\",\"kind\":\"HOST\",\"line\":40,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value\":{\"address_class\":\"PUBLIC\",\"ip_sha256\":\"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620\"}},{\"key\":\"public_vps_decision\",\"kind\":\"RELEVANT_KEY\",\"line\":41,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"348e0553be62997b4c48bd45853803cba5a55da3b958f8e9b9419fa3c734b19d\"},{\"key\":\"backup_hostkeys\",\"kind\":\"RELEVANT_KEY\",\"line\":42,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"2526f72dab61a237d504a652be0ab5e869e292ba1290fe76f9d34ea5578d980d\"},{\"key\":\"REPORT\",\"kind\":\"RELEVANT_KEY\",\"line\":5,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"16de6e2523ad6290564a03adb219172bc0031e08d824a49be9838f00eb297f61\"},{\"key\":\"vps_trust\",\"kind\":\"RELEVANT_KEY\",\"line\":36,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"33003123cbc00728f9934444716841ec1ff9b529c997839691bb14c20d436e53\"},{\"key\":\"trusted_netbird_identity\",\"kind\":\"RELEVANT_KEY\",\"line\":37,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"ebb24cf2960c2243b155740605cba8127cedb6e016de2d44b914019d3fdbe085\"},{\"key\":\"trusted_netbird_ip\",\"kind\":\"RELEVANT_KEY\",\"line\":38,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440\"},{\"key\":\"public_vps_ip\",\"kind\":\"HOST\",\"line\":39,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value\":{\"address_class\":\"PUBLIC\",\"ip_sha256\":\"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620\"}},{\"key\":\"public_vps_decision\",\"kind\":\"RELEVANT_KEY\",\"line\":40,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"348e0553be62997b4c48bd45853803cba5a55da3b958f8e9b9419fa3c734b19d\"},{\"key\":\"backup_hostkeys\",\"kind\":\"RELEVANT_KEY\",\"line\":41,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"2526f72dab61a237d504a652be0ab5e869e292ba1290fe76f9d34ea5578d980d\"},{\"key\":\"DRILL_REPORT\",\"kind\":\"RELEVANT_KEY\",\"line\":6,\"source\":{\"basename\":\"homelab-ingress-failover\",\"path\":\"/usr/local/sbin/homelab-ingress-failover\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"242de3f3a1197026b997b7565bcd2a7c61a69229904a4e133f1988e206575baf\"},{\"key\":\"DRILL_REPORT\",\"kind\":\"RELEVANT_KEY\",\"line\":6,\"source\":{\"basename\":\"homelab-ingress-standby-refresh\",\"path\":\"/usr/local/sbin/homelab-ingress-standby-refresh\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"242de3f3a1197026b997b7565bcd2a7c61a69229904a4e133f1988e206575baf\"},{\"key\":\"DEST\",\"kind\":\"HOST_EXPR\",\"line\":120,\"source\":{\"basename\":\"homelab-ingress-standby-refresh\",\"path\":\"/usr/local/sbin/homelab-ingress-standby-refresh\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"fc4d602aa16170b89bda4c70d75d89e6183b14f994516aa7b3e032c3f18cbb4e\"}]}\nINFRA171_FILE_EVIDENCE={\"count\":6,\"rows\":[{\"file\":{\"basename\":\"homelab-ingress-standby-refresh.service\",\"path_sha256\":\"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e\",\"scope\":\"ETC\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"Description=Create current ingress recovery artifact on cold standby node\"},\"line\":2},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"After=network-online.target pve-cluster.service\"},\"line\":3},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"Wants=network-online.target\"},\"line\":4},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"User=root\"},\"line\":9},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\"},\"line\":11}]},{\"file\":{\"basename\":\"homelab-ingress-standby-refresh.timer\",\"path_sha256\":\"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1\",\"scope\":\"ETC\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"Description=Daily ingress cold standby recovery artifact refresh\"},\"line\":2},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"Unit=homelab-ingress-standby-refresh.service\"},\"line\":8},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"WantedBy=timers.target\"},\"line\":11}]},{\"file\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ROOT=\\\"/var/lib/homelab-external-probe-vps\\\"\"},\"line\":3},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"HEALTH=\\\"/var/lib/homelab-health/external-probe-vps.txt\\\"\"},\"line\":4},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"REPORT=\\\"$ROOT/latest.md\\\"\"},\"line\":5},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"^STATUS=OK\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":9},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"^STATUS=OK\\\" /var/lib/homelab-health/netbird-vps-trust-closure.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":10},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"TRUSTED_NETBIRD_IDENTITY=edge-vm\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":12},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":13},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"cat > \\\"$REPORT\\\" <\"},\"line\":39},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"public_vps_ip=\"},\"line\":40},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"public_vps_decision=REJECTED_HOSTKEY_MISMATCH\"},\"line\":41},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"backup_hostkeys=not_used_public_vps_rejected\"},\"line\":42},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\"},\"line\":45},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\"},\"line\":46},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"closed_tail=no_known_hosts_change\"},\"line\":48},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=external-probe-vps BAD=$BAD WARN=$WARN MODE=STRATEGY_ONLY_NO_INDEPENDENT_RUNNER DEPLOYED=NO DEPLOY_DECISION=NO_DEPLOY VPS_TRUST=edge_vm_trusted_public_vps_rejected BACKUP_HOSTKEYS=not_used_public_vps_rejected PVE01_PUBLIC_SMOKE_BAD=$PVE01_PUBLIC_SMOKE_BAD INDEPENDENT_RUNNER=missing INDEPENDENT_RUNNER_AVAILABLE=no TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP= PUBLIC_VPS_IP= PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH REPORT=$REPORT\\\" > \\\"$HEALTH\\\"\"},\"line\":51}]},{\"file\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ROOT=\\\"/var/lib/homelab-external-probe-vps\\\"\"},\"line\":3},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"HEALTH=\\\"/var/lib/homelab-health/external-probe-vps.txt\\\"\"},\"line\":4},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"REPORT=\\\"$ROOT/latest.md\\\"\"},\"line\":5},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"for url in $URLS; do safe=\\\"$(echo \\\"$url\\\" | sed \\\"s#[^A-Za-z0-9]#_#g\\\")\\\"; body=\\\"/tmp/external-probe-vps-health-${safe}.body\\\"; code=\\\"$(curl -k -sS -o \\\"$body\\\" -w \\\"%{http_code}\\\" --connect-timeout 8 --max-time 18 \\\"$url\\\" 2>/tmp/external-probe-vps-health.err)\\\"; first=\\\"$(head -n1 \\\"$body\\\" 2>/dev/null | tr -cd \\\"[:print:]\\\")\\\"; bytes=\\\"$(wc -c < \\\"$body\\\" 2>/dev/null || echo 0)\\\"; printf \\\"%s\\\\t%s\\\\t%s\\\\t%s\\\\n\\\" \\\"$url\\\" \\\"$code\\\" \\\"$bytes\\\" \\\"$first\\\" >> \\\"$SMOKE_TSV\\\"; case \\\"$code:$first\\\" in 200:STATUS=*) : ;; *) PVE01_PUBLIC_SMOKE_BAD=$((PVE01_PUBLIC_SMOKE_BAD+1));; esac; done\"},\"line\":12},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"^STATUS=OK\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":13},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"^STATUS=OK\\\" /var/lib/homelab-health/netbird-vps-trust-closure.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":14},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"TRUSTED_NETBIRD_IDENTITY=edge-vm\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":16},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":17},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"cat > \\\"$REPORT\\\" <\"},\"line\":38},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"public_vps_ip=\"},\"line\":39},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"public_vps_decision=REJECTED_HOSTKEY_MISMATCH\"},\"line\":40},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"backup_hostkeys=not_used_public_vps_rejected\"},\"line\":41},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\"},\"line\":44},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\"},\"line\":45},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"closed_tail=no_known_hosts_change\"},\"line\":47},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=external-probe-vps BAD=$BAD WARN=$WARN DEPLOYED=NO DEPLOY_DECISION=NO_DEPLOY VPS_TRUST=edge_vm_trusted_public_vps_rejected BACKUP_HOSTKEYS=not_used_public_vps_rejected PVE01_PUBLIC_SMOKE_BAD=$PVE01_PUBLIC_SMOKE_BAD INDEPENDENT_RUNNER=missing INDEPENDENT_RUNNER_AVAILABLE=no TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP= PUBLIC_VPS_IP= PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH REPORT=$REPORT\\\" > \\\"$HEALTH\\\"\"},\"line\":54}]},{\"file\":{\"basename\":\"homelab-ingress-failover\",\"path\":\"/usr/local/sbin/homelab-ingress-failover\",\"scope\":\"USR_LOCAL\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"export LC_ALL=C\"},\"line\":3},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"DRILL_REPORT=\\\"/var/lib/homelab-chat/CR0023-cold-standby-drill.txt\\\"\"},\"line\":6},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"STATE_DIR=\\\"/var/lib/homelab-health/ingress-failover\\\"\"},\"line\":7},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"LOCK_FILE=\\\"/run/lock/homelab-ingress-failover.lock\\\"\"},\"line\":9},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"echo \\\"INGRESS_FAILOVER_LOCK=BUSY\\\"\"},\"line\":15},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"test -f \\\"$DRILL_REPORT\\\"\"},\"line\":19},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"sed -n 's/^STANDBY_VMID=//p' \\\"$DRILL_REPORT\\\" |\"},\"line\":22},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"sed -n 's/^STANDBY_NODE=//p' \\\"$DRILL_REPORT\\\" |\"},\"line\":27},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"import json\"},\"line\":48},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"import sys\"},\"line\":49},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"if [ \\\"$node\\\" = \\\"$(hostname -s)\\\" ] ||\"},\"line\":71},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"[ \\\"$node\\\" = \\\"$(hostname)\\\" ]\"},\"line\":72},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ssh \\\\\"},\"line\":76},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"export LC_ALL=C\"},\"line\":196},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -Rqs \\\"server_name[[:space:]]*git\\\\.gram1\\\\.ru\\\" \\\\\"},\"line\":213},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"echo \\\"ACTIVE_INGRESS=$active\\\"\"},\"line\":241}]},{\"file\":{\"basename\":\"homelab-ingress-standby-refresh\",\"path\":\"/usr/local/sbin/homelab-ingress-standby-refresh\",\"scope\":\"USR_LOCAL\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"export LC_ALL=C\"},\"line\":3},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"LOCK=\\\"/run/lock/homelab-ingress-standby-refresh.lock\\\"\"},\"line\":5},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"DRILL_REPORT=\\\"/var/lib/homelab-chat/CR0023-cold-standby-drill.txt\\\"\"},\"line\":6},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"STATE_DIR=\\\"/var/lib/homelab-ingress-standby\\\"\"},\"line\":7},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"RELAY=\\\"$(mktemp)\\\"\"},\"line\":12},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"rm -f \\\"$TMP_STATE\\\" \\\"$RELAY\\\" \\\"$RESOURCES\\\"\"},\"line\":16},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"test -f \\\"$DRILL_REPORT\\\"\"},\"line\":26},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q '^CR0023_COLD_STANDBY_DRILL_STATUS=SUCCESS$' \\\"$DRILL_REPORT\\\"\"},\"line\":27},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"STANDBY_VMID=\\\"$(sed -n 's/^STANDBY_VMID=//p' \\\"$DRILL_REPORT\\\" | head -n 1)\\\"\"},\"line\":29},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"STANDBY_NODE=\\\"$(sed -n 's/^STANDBY_NODE=//p' \\\"$DRILL_REPORT\\\" | head -n 1)\\\"\"},\"line\":30},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"import json, sys\"},\"line\":39},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"import json, sys\"},\"line\":47},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"if [ \\\"$node\\\" = \\\"$(hostname -s)\\\" ] || [ \\\"$node\\\" = \\\"$(hostname)\\\" ]; then\"},\"line\":60},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\\\"$node\\\" \\\"$@\\\"\"},\"line\":63},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"if [ \\\"$SOURCE_NODE\\\" = \\\"$(hostname -s)\\\" ] || [ \\\"$SOURCE_NODE\\\" = \\\"$(hostname)\\\" ]; then\"},\"line\":91},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"cat \\\"$SOURCE_BACKUP\\\" >\\\"$RELAY\\\"\"},\"line\":92},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\\\"$SOURCE_NODE\\\" \\\"cat '$SOURCE_BACKUP'\\\" >\\\"$RELAY\\\"\"},\"line\":94},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"test \\\"$(stat -c %s \\\"$RELAY\\\")\\\" = \\\"$SOURCE_SIZE\\\"\"},\"line\":97},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"test \\\"$(sha256sum \\\"$RELAY\\\" | awk '{print $1}')\\\" = \\\"$SOURCE_SHA256\\\"\"},\"line\":98},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"DEST_DIR=\\\"/var/backups/ingress-standby-source\\\"\"},\"line\":100},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"if [ \\\"$STANDBY_NODE\\\" = \\\"$(hostname -s)\\\" ] || [ \\\"$STANDBY_NODE\\\" = \\\"$(hostname)\\\" ]; then\"},\"line\":108},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"cat \\\"$RELAY\\\" >\\\"$DEST_TMP\\\"\"},\"line\":109},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\\\"$STANDBY_NODE\\\" \\\"cat >'$DEST_TMP'\\\" <\\\"$RELAY\\\"\"},\"line\":111},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\\\"$SOURCE_NODE\\\" \\\"$STANDBY_VMID\\\" \\\"$KEEP\\\" <<'REMOTE'\"},\"line\":116},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"REMOTE\"},\"line\":171}]}]}\nINFRA171_ROUTE_CANDIDATES={\"count\":0,\"rows\":[]}\nINFRA171_ROLE_ROUTE_SUMMARY={\"MD_RELAY\":{\"route_candidates\":[],\"trusted_user_route_proven\":false,\"trusted_user_routes\":[]},\"SPB_INGRESS\":{\"route_candidates\":[],\"trusted_user_route_proven\":false,\"trusted_user_routes\":[]},\"US_NETBIRD\":{\"route_candidates\":[],\"trusted_user_route_proven\":false,\"trusted_user_routes\":[]}}\nINFRA171_SUMMARY={\"decision\":\"HOLD_EXTERNAL_SSH_UNIT_TRACE_INCOMPLETE\",\"holds\":[\"US_NETBIRD:TRUSTED_USER_ROUTE_NOT_PROVEN\",\"MD_RELAY:TRUSTED_USER_ROUTE_NOT_PROVEN\",\"SPB_INGRESS:TRUSTED_USER_ROUTE_NOT_PROVEN\"],\"roles_hold\":[\"US_NETBIRD\",\"MD_RELAY\",\"SPB_INGRESS\"],\"roles_ready\":[]}\nEXTERNAL_SSH_ATTEMPTS=0\nSSH_KEYSCAN_ATTEMPTS=0\nKNOWN_HOSTS_WRITES=0\nSSH_CONFIG_WRITES=0\nCONFIGURATION_WRITES=0\nFILE_WRITES_BY_PROBE=0\nSECRETS_PRINTED=NO\nINFRA171_END\n" +} diff --git a/runtime/history/SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1.txt b/runtime/history/SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1.txt new file mode 100644 index 00000000..6d6b3e5c --- /dev/null +++ b/runtime/history/SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1.txt @@ -0,0 +1,52 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=infra-dr-vps-unit-trace-pve01 +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=59ac720637b5a262dac71928edd44bbbf906f7be1b225a371a55f1074eb0a4a4 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=5e8e4e3d538b446a7c0c0deb37ca79eba873606c075765fd974bc6b0cbf7936e +SANITIZED_OUTPUT_SHA256=5e8e4e3d538b446a7c0c0deb37ca79eba873606c075765fd974bc6b0cbf7936e +OUTPUT_BEGIN +INFRA171_BEGIN +COMMAND_ID=SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1 +MODE=READ_ONLY +PRIOR_170_COMMIT=5c9001893ac691f04b4b8551c801a41c2ddd507c +INFRA171_ROLE_DNS={"MD_RELAY":{"hostname":"nb.pvepro.ru","ip_refs":[{"address_class":"PUBLIC","ip_sha256":"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16"}],"rc":0},"SPB_INGRESS":{"hostname":"newfi.ru","ip_refs":[{"address_class":"PUBLIC","identity_marker":"HISTORICAL_SPB_INGRESS","ip_sha256":"6d6149f42864ce03daf22828cf236bf64d5d9a004040746ddbb051d71c9fcbc5"}],"rc":0},"US_NETBIRD":{"hostname":"nb.gram1.ru","ip_refs":[{"address_class":"PUBLIC","ip_sha256":"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16"}],"rc":0}} +INFRA171_UNITS={"count":2,"rows":[{"directives":[{"key":"User","line":9,"value":{"kind":"SANITIZED","text":"root"}},{"key":"Group","line":10,"value":{"kind":"SANITIZED","text":"root"}},{"key":"ExecStart","line":11,"value":{"kind":"SANITIZED","text":"/usr/local/sbin/homelab-ingress-standby-refresh"}}],"file":{"basename":"homelab-ingress-standby-refresh.service","path_sha256":"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e","scope":"ETC"}},{"directives":[],"file":{"basename":"homelab-ingress-standby-refresh.timer","path_sha256":"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1","scope":"ETC"}}]} +INFRA171_FILES={"bytes_read":21032,"count":6,"files":[{"basename":"homelab-ingress-standby-refresh.service","path_sha256":"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e","scope":"ETC"},{"basename":"homelab-ingress-standby-refresh.timer","path_sha256":"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1","scope":"ETC"},{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},{"basename":"homelab-ingress-failover","path":"/usr/local/sbin/homelab-ingress-failover","scope":"USR_LOCAL"},{"basename":"homelab-ingress-standby-refresh","path":"/usr/local/sbin/homelab-ingress-standby-refresh","scope":"USR_LOCAL"}]} +INFRA171_SAFE_ASSIGNMENTS={"count":19,"rows":[{"key":"User","kind":"USER","line":9,"source":{"basename":"homelab-ingress-standby-refresh.service","path_sha256":"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e","scope":"ETC"},"value":"root"},{"key":"ExecStart","kind":"RELEVANT_KEY","line":11,"source":{"basename":"homelab-ingress-standby-refresh.service","path_sha256":"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e","scope":"ETC"},"value_sha256":"13d114d64c8f3496ac6b3a8eee4a94ae9e583182842310e10dc40f15561e400c"},{"key":"REPORT","kind":"RELEVANT_KEY","line":5,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"16de6e2523ad6290564a03adb219172bc0031e08d824a49be9838f00eb297f61"},{"key":"vps_trust","kind":"RELEVANT_KEY","line":37,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"33003123cbc00728f9934444716841ec1ff9b529c997839691bb14c20d436e53"},{"key":"trusted_netbird_identity","kind":"RELEVANT_KEY","line":38,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"ebb24cf2960c2243b155740605cba8127cedb6e016de2d44b914019d3fdbe085"},{"key":"trusted_netbird_ip","kind":"RELEVANT_KEY","line":39,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440"},{"key":"public_vps_ip","kind":"HOST","line":40,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value":{"address_class":"PUBLIC","ip_sha256":"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620"}},{"key":"public_vps_decision","kind":"RELEVANT_KEY","line":41,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"348e0553be62997b4c48bd45853803cba5a55da3b958f8e9b9419fa3c734b19d"},{"key":"backup_hostkeys","kind":"RELEVANT_KEY","line":42,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"2526f72dab61a237d504a652be0ab5e869e292ba1290fe76f9d34ea5578d980d"},{"key":"REPORT","kind":"RELEVANT_KEY","line":5,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"16de6e2523ad6290564a03adb219172bc0031e08d824a49be9838f00eb297f61"},{"key":"vps_trust","kind":"RELEVANT_KEY","line":36,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"33003123cbc00728f9934444716841ec1ff9b529c997839691bb14c20d436e53"},{"key":"trusted_netbird_identity","kind":"RELEVANT_KEY","line":37,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"ebb24cf2960c2243b155740605cba8127cedb6e016de2d44b914019d3fdbe085"},{"key":"trusted_netbird_ip","kind":"RELEVANT_KEY","line":38,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440"},{"key":"public_vps_ip","kind":"HOST","line":39,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value":{"address_class":"PUBLIC","ip_sha256":"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620"}},{"key":"public_vps_decision","kind":"RELEVANT_KEY","line":40,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"348e0553be62997b4c48bd45853803cba5a55da3b958f8e9b9419fa3c734b19d"},{"key":"backup_hostkeys","kind":"RELEVANT_KEY","line":41,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"2526f72dab61a237d504a652be0ab5e869e292ba1290fe76f9d34ea5578d980d"},{"key":"DRILL_REPORT","kind":"RELEVANT_KEY","line":6,"source":{"basename":"homelab-ingress-failover","path":"/usr/local/sbin/homelab-ingress-failover","scope":"USR_LOCAL"},"value_sha256":"242de3f3a1197026b997b7565bcd2a7c61a69229904a4e133f1988e206575baf"},{"key":"DRILL_REPORT","kind":"RELEVANT_KEY","line":6,"source":{"basename":"homelab-ingress-standby-refresh","path":"/usr/local/sbin/homelab-ingress-standby-refresh","scope":"USR_LOCAL"},"value_sha256":"242de3f3a1197026b997b7565bcd2a7c61a69229904a4e133f1988e206575baf"},{"key":"DEST","kind":"HOST_EXPR","line":120,"source":{"basename":"homelab-ingress-standby-refresh","path":"/usr/local/sbin/homelab-ingress-standby-refresh","scope":"USR_LOCAL"},"value_sha256":"fc4d602aa16170b89bda4c70d75d89e6183b14f994516aa7b3e032c3f18cbb4e"}]} +INFRA171_FILE_EVIDENCE={"count":6,"rows":[{"file":{"basename":"homelab-ingress-standby-refresh.service","path_sha256":"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e","scope":"ETC"},"rows":[{"evidence":{"kind":"SANITIZED","text":"Description=Create current ingress recovery artifact on cold standby node"},"line":2},{"evidence":{"kind":"SANITIZED","text":"After=network-online.target pve-cluster.service"},"line":3},{"evidence":{"kind":"SANITIZED","text":"Wants=network-online.target"},"line":4},{"evidence":{"kind":"SANITIZED","text":"User=root"},"line":9},{"evidence":{"kind":"SANITIZED","text":""},"line":11}]},{"file":{"basename":"homelab-ingress-standby-refresh.timer","path_sha256":"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1","scope":"ETC"},"rows":[{"evidence":{"kind":"SANITIZED","text":"Description=Daily ingress cold standby recovery artifact refresh"},"line":2},{"evidence":{"kind":"SANITIZED","text":"Unit=homelab-ingress-standby-refresh.service"},"line":8},{"evidence":{"kind":"SANITIZED","text":"WantedBy=timers.target"},"line":11}]},{"file":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"rows":[{"evidence":{"kind":"SANITIZED","text":"ROOT=\"/var/lib/homelab-external-probe-vps\""},"line":3},{"evidence":{"kind":"SANITIZED","text":"HEALTH=\"/var/lib/homelab-health/external-probe-vps.txt\""},"line":4},{"evidence":{"kind":"SANITIZED","text":"REPORT=\"$ROOT/latest.md\""},"line":5},{"evidence":{"kind":"SANITIZED","text":"grep -q \"^STATUS=OK\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":9},{"evidence":{"kind":"SANITIZED","text":"grep -q \"^STATUS=OK\" /var/lib/homelab-health/netbird-vps-trust-closure.txt 2>/dev/null || BAD=$((BAD+1))"},"line":10},{"evidence":{"kind":"SANITIZED","text":"grep -q \"TRUSTED_NETBIRD_IDENTITY=edge-vm\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":12},{"evidence":{"kind":"SANITIZED","text":"grep -q \"PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":13},{"evidence":{"kind":"SANITIZED","text":"cat > \"$REPORT\" <"},"line":39},{"evidence":{"kind":"SANITIZED","text":"public_vps_ip="},"line":40},{"evidence":{"kind":"SANITIZED","text":"public_vps_decision=REJECTED_HOSTKEY_MISMATCH"},"line":41},{"evidence":{"kind":"SANITIZED","text":"backup_hostkeys=not_used_public_vps_rejected"},"line":42},{"evidence":{"kind":"SANITIZED","text":""},"line":45},{"evidence":{"kind":"SANITIZED","text":""},"line":46},{"evidence":{"kind":"SANITIZED","text":"closed_tail=no_known_hosts_change"},"line":48},{"evidence":{"kind":"SANITIZED","text":"echo \"STATUS=$STATUS TS=$TS TYPE=external-probe-vps BAD=$BAD WARN=$WARN MODE=STRATEGY_ONLY_NO_INDEPENDENT_RUNNER DEPLOYED=NO DEPLOY_DECISION=NO_DEPLOY VPS_TRUST=edge_vm_trusted_public_vps_rejected BACKUP_HOSTKEYS=not_used_public_vps_rejected PVE01_PUBLIC_SMOKE_BAD=$PVE01_PUBLIC_SMOKE_BAD INDEPENDENT_RUNNER=missing INDEPENDENT_RUNNER_AVAILABLE=no TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP= PUBLIC_VPS_IP= PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH REPORT=$REPORT\" > \"$HEALTH\""},"line":51}]},{"file":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"rows":[{"evidence":{"kind":"SANITIZED","text":"ROOT=\"/var/lib/homelab-external-probe-vps\""},"line":3},{"evidence":{"kind":"SANITIZED","text":"HEALTH=\"/var/lib/homelab-health/external-probe-vps.txt\""},"line":4},{"evidence":{"kind":"SANITIZED","text":"REPORT=\"$ROOT/latest.md\""},"line":5},{"evidence":{"kind":"SANITIZED","text":"for url in $URLS; do safe=\"$(echo \"$url\" | sed \"s#[^A-Za-z0-9]#_#g\")\"; body=\"/tmp/external-probe-vps-health-${safe}.body\"; code=\"$(curl -k -sS -o \"$body\" -w \"%{http_code}\" --connect-timeout 8 --max-time 18 \"$url\" 2>/tmp/external-probe-vps-health.err)\"; first=\"$(head -n1 \"$body\" 2>/dev/null | tr -cd \"[:print:]\")\"; bytes=\"$(wc -c < \"$body\" 2>/dev/null || echo 0)\"; printf \"%s\\t%s\\t%s\\t%s\\n\" \"$url\" \"$code\" \"$bytes\" \"$first\" >> \"$SMOKE_TSV\"; case \"$code:$first\" in 200:STATUS=*) : ;; *) PVE01_PUBLIC_SMOKE_BAD=$((PVE01_PUBLIC_SMOKE_BAD+1));; esac; done"},"line":12},{"evidence":{"kind":"SANITIZED","text":"grep -q \"^STATUS=OK\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":13},{"evidence":{"kind":"SANITIZED","text":"grep -q \"^STATUS=OK\" /var/lib/homelab-health/netbird-vps-trust-closure.txt 2>/dev/null || BAD=$((BAD+1))"},"line":14},{"evidence":{"kind":"SANITIZED","text":"grep -q \"TRUSTED_NETBIRD_IDENTITY=edge-vm\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":16},{"evidence":{"kind":"SANITIZED","text":"grep -q \"PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":17},{"evidence":{"kind":"SANITIZED","text":"cat > \"$REPORT\" <"},"line":38},{"evidence":{"kind":"SANITIZED","text":"public_vps_ip="},"line":39},{"evidence":{"kind":"SANITIZED","text":"public_vps_decision=REJECTED_HOSTKEY_MISMATCH"},"line":40},{"evidence":{"kind":"SANITIZED","text":"backup_hostkeys=not_used_public_vps_rejected"},"line":41},{"evidence":{"kind":"SANITIZED","text":""},"line":44},{"evidence":{"kind":"SANITIZED","text":""},"line":45},{"evidence":{"kind":"SANITIZED","text":"closed_tail=no_known_hosts_change"},"line":47},{"evidence":{"kind":"SANITIZED","text":"echo \"STATUS=$STATUS TS=$TS TYPE=external-probe-vps BAD=$BAD WARN=$WARN DEPLOYED=NO DEPLOY_DECISION=NO_DEPLOY VPS_TRUST=edge_vm_trusted_public_vps_rejected BACKUP_HOSTKEYS=not_used_public_vps_rejected PVE01_PUBLIC_SMOKE_BAD=$PVE01_PUBLIC_SMOKE_BAD INDEPENDENT_RUNNER=missing INDEPENDENT_RUNNER_AVAILABLE=no TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP= PUBLIC_VPS_IP= PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH REPORT=$REPORT\" > \"$HEALTH\""},"line":54}]},{"file":{"basename":"homelab-ingress-failover","path":"/usr/local/sbin/homelab-ingress-failover","scope":"USR_LOCAL"},"rows":[{"evidence":{"kind":"SANITIZED","text":"export LC_ALL=C"},"line":3},{"evidence":{"kind":"SANITIZED","text":"DRILL_REPORT=\"/var/lib/homelab-chat/CR0023-cold-standby-drill.txt\""},"line":6},{"evidence":{"kind":"SANITIZED","text":"STATE_DIR=\"/var/lib/homelab-health/ingress-failover\""},"line":7},{"evidence":{"kind":"SANITIZED","text":"LOCK_FILE=\"/run/lock/homelab-ingress-failover.lock\""},"line":9},{"evidence":{"kind":"SANITIZED","text":"echo \"INGRESS_FAILOVER_LOCK=BUSY\""},"line":15},{"evidence":{"kind":"SANITIZED","text":"test -f \"$DRILL_REPORT\""},"line":19},{"evidence":{"kind":"SANITIZED","text":"sed -n 's/^STANDBY_VMID=//p' \"$DRILL_REPORT\" |"},"line":22},{"evidence":{"kind":"SANITIZED","text":"sed -n 's/^STANDBY_NODE=//p' \"$DRILL_REPORT\" |"},"line":27},{"evidence":{"kind":"SANITIZED","text":"import json"},"line":48},{"evidence":{"kind":"SANITIZED","text":"import sys"},"line":49},{"evidence":{"kind":"SANITIZED","text":"if [ \"$node\" = \"$(hostname -s)\" ] ||"},"line":71},{"evidence":{"kind":"SANITIZED","text":"[ \"$node\" = \"$(hostname)\" ]"},"line":72},{"evidence":{"kind":"SANITIZED","text":"ssh \\"},"line":76},{"evidence":{"kind":"SANITIZED","text":"export LC_ALL=C"},"line":196},{"evidence":{"kind":"SANITIZED","text":"grep -Rqs \"server_name[[:space:]]*git\\.gram1\\.ru\" \\"},"line":213},{"evidence":{"kind":"SANITIZED","text":"echo \"ACTIVE_INGRESS=$active\""},"line":241}]},{"file":{"basename":"homelab-ingress-standby-refresh","path":"/usr/local/sbin/homelab-ingress-standby-refresh","scope":"USR_LOCAL"},"rows":[{"evidence":{"kind":"SANITIZED","text":"export LC_ALL=C"},"line":3},{"evidence":{"kind":"SANITIZED","text":"LOCK=\"/run/lock/homelab-ingress-standby-refresh.lock\""},"line":5},{"evidence":{"kind":"SANITIZED","text":"DRILL_REPORT=\"/var/lib/homelab-chat/CR0023-cold-standby-drill.txt\""},"line":6},{"evidence":{"kind":"SANITIZED","text":"STATE_DIR=\"/var/lib/homelab-ingress-standby\""},"line":7},{"evidence":{"kind":"SANITIZED","text":"RELAY=\"$(mktemp)\""},"line":12},{"evidence":{"kind":"SANITIZED","text":"rm -f \"$TMP_STATE\" \"$RELAY\" \"$RESOURCES\""},"line":16},{"evidence":{"kind":"SANITIZED","text":"test -f \"$DRILL_REPORT\""},"line":26},{"evidence":{"kind":"SANITIZED","text":"grep -q '^CR0023_COLD_STANDBY_DRILL_STATUS=SUCCESS$' \"$DRILL_REPORT\""},"line":27},{"evidence":{"kind":"SANITIZED","text":"STANDBY_VMID=\"$(sed -n 's/^STANDBY_VMID=//p' \"$DRILL_REPORT\" | head -n 1)\""},"line":29},{"evidence":{"kind":"SANITIZED","text":"STANDBY_NODE=\"$(sed -n 's/^STANDBY_NODE=//p' \"$DRILL_REPORT\" | head -n 1)\""},"line":30},{"evidence":{"kind":"SANITIZED","text":"import json, sys"},"line":39},{"evidence":{"kind":"SANITIZED","text":"import json, sys"},"line":47},{"evidence":{"kind":"SANITIZED","text":"if [ \"$node\" = \"$(hostname -s)\" ] || [ \"$node\" = \"$(hostname)\" ]; then"},"line":60},{"evidence":{"kind":"SANITIZED","text":"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$node\" \"$@\""},"line":63},{"evidence":{"kind":"SANITIZED","text":"if [ \"$SOURCE_NODE\" = \"$(hostname -s)\" ] || [ \"$SOURCE_NODE\" = \"$(hostname)\" ]; then"},"line":91},{"evidence":{"kind":"SANITIZED","text":"cat \"$SOURCE_BACKUP\" >\"$RELAY\""},"line":92},{"evidence":{"kind":"SANITIZED","text":"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$SOURCE_NODE\" \"cat '$SOURCE_BACKUP'\" >\"$RELAY\""},"line":94},{"evidence":{"kind":"SANITIZED","text":"test \"$(stat -c %s \"$RELAY\")\" = \"$SOURCE_SIZE\""},"line":97},{"evidence":{"kind":"SANITIZED","text":"test \"$(sha256sum \"$RELAY\" | awk '{print $1}')\" = \"$SOURCE_SHA256\""},"line":98},{"evidence":{"kind":"SANITIZED","text":"DEST_DIR=\"/var/backups/ingress-standby-source\""},"line":100},{"evidence":{"kind":"SANITIZED","text":"if [ \"$STANDBY_NODE\" = \"$(hostname -s)\" ] || [ \"$STANDBY_NODE\" = \"$(hostname)\" ]; then"},"line":108},{"evidence":{"kind":"SANITIZED","text":"cat \"$RELAY\" >\"$DEST_TMP\""},"line":109},{"evidence":{"kind":"SANITIZED","text":"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$STANDBY_NODE\" \"cat >'$DEST_TMP'\" <\"$RELAY\""},"line":111},{"evidence":{"kind":"SANITIZED","text":"\"$SOURCE_NODE\" \"$STANDBY_VMID\" \"$KEEP\" <<'REMOTE'"},"line":116},{"evidence":{"kind":"SANITIZED","text":"REMOTE"},"line":171}]}]} +INFRA171_ROUTE_CANDIDATES={"count":0,"rows":[]} +INFRA171_ROLE_ROUTE_SUMMARY={"MD_RELAY":{"route_candidates":[],"trusted_user_route_proven":false,"trusted_user_routes":[]},"SPB_INGRESS":{"route_candidates":[],"trusted_user_route_proven":false,"trusted_user_routes":[]},"US_NETBIRD":{"route_candidates":[],"trusted_user_route_proven":false,"trusted_user_routes":[]}} +INFRA171_SUMMARY={"decision":"HOLD_EXTERNAL_SSH_UNIT_TRACE_INCOMPLETE","holds":["US_NETBIRD:TRUSTED_USER_ROUTE_NOT_PROVEN","MD_RELAY:TRUSTED_USER_ROUTE_NOT_PROVEN","SPB_INGRESS:TRUSTED_USER_ROUTE_NOT_PROVEN"],"roles_hold":["US_NETBIRD","MD_RELAY","SPB_INGRESS"],"roles_ready":[]} +EXTERNAL_SSH_ATTEMPTS=0 +SSH_KEYSCAN_ATTEMPTS=0 +KNOWN_HOSTS_WRITES=0 +SSH_CONFIG_WRITES=0 +CONFIGURATION_WRITES=0 +FILE_WRITES_BY_PROBE=0 +SECRETS_PRINTED=NO +INFRA171_END + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index d589ae8b..e0160a21 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260911-INFRA-DR-VPS-ROUTE-AUTH-170R1", + "command_id": "SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "infra-dr-vps-route-auth-pve01", - "started_at_utc": "2026-09-11T03:42:05Z", - "finished_at_utc": "2026-09-11T03:42:05Z", + "component": "infra-dr-vps-unit-trace-pve01", + "started_at_utc": "2026-09-11T03:49:23Z", + "finished_at_utc": "2026-09-11T03:49:23Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "396759f2f07e402daf35047c1bde8557e62601310a1987f75906bec86336363a", + "command_sha256": "59ac720637b5a262dac71928edd44bbbf906f7be1b225a371a55f1074eb0a4a4", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "5fb23cd7f83ef292be65ad46e52f7c1e5378ce4834aa194a549a99cf6e615fd1", - "sanitized_output_sha256": "5fb23cd7f83ef292be65ad46e52f7c1e5378ce4834aa194a549a99cf6e615fd1", + "raw_evidence_sha256": "5e8e4e3d538b446a7c0c0deb37ca79eba873606c075765fd974bc6b0cbf7936e", + "sanitized_output_sha256": "5e8e4e3d538b446a7c0c0deb37ca79eba873606c075765fd974bc6b0cbf7936e", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "INFRA170_BEGIN\nCOMMAND_ID=SUPPORT-260911-INFRA-DR-VPS-ROUTE-AUTH-170R1\nMODE=READ_ONLY\nPRIOR_169_COMMIT=c934e939d51c24b6fabc5be1b72ce82effa05a5e\nINFRA170_ROLE_DNS={\"MD_RELAY\":{\"hostname\":\"nb.pvepro.ru\",\"ip_refs\":[{\"address_class\":\"PUBLIC\",\"ip_sha256\":\"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16\"}],\"rc\":0},\"SPB_INGRESS\":{\"hostname\":\"newfi.ru\",\"ip_refs\":[{\"address_class\":\"PUBLIC\",\"identity_marker\":\"HISTORICAL_SPB_INGRESS\",\"ip_sha256\":\"6d6149f42864ce03daf22828cf236bf64d5d9a004040746ddbb051d71c9fcbc5\"}],\"rc\":0},\"US_NETBIRD\":{\"hostname\":\"nb.gram1.ru\",\"ip_refs\":[{\"address_class\":\"PUBLIC\",\"ip_sha256\":\"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16\"}],\"rc\":0}}\nINFRA170_UNIT_CHAIN={\"bytes_read\":15144,\"file_count\":4,\"files_scanned\":[\"/etc/systemd/system/homelab-ingress-standby-refresh.service\",\"/usr/local/sbin/homelab-ingress-standby-refresh\",\"/etc/systemd/system/homelab-ingress-standby-refresh.timer\",\"/usr/local/sbin/homelab-ingress-failover\"],\"unit_count\":2,\"units\":[{\"path\":\"/etc/systemd/system/homelab-ingress-standby-refresh.service\",\"safe_directives\":[{\"key\":\"User\",\"line\":9,\"value\":\"root\"},{\"key\":\"Group\",\"line\":10,\"value\":\"root\"},{\"key\":\"ExecStart\",\"line\":11,\"value_kind\":\"EXEC\"}]},{\"path\":\"/etc/systemd/system/homelab-ingress-standby-refresh.timer\",\"safe_directives\":[]}]}\nINFRA170_SAFE_ASSIGNMENTS={\"count\":0,\"rows\":[]}\nINFRA170_ROUTE_CANDIDATES={\"count\":0,\"rows\":[]}\nINFRA170_SAME_FILE_BUNDLES={\"count\":0,\"rows\":[]}\nINFRA170_ROLE_ROUTE_SUMMARY={\"MD_RELAY\":{\"route_candidates\":[],\"trusted_user_route_proven\":false,\"trusted_user_routes\":[]},\"SPB_INGRESS\":{\"route_candidates\":[],\"trusted_user_route_proven\":false,\"trusted_user_routes\":[]},\"US_NETBIRD\":{\"route_candidates\":[],\"trusted_user_route_proven\":false,\"trusted_user_routes\":[]}}\nINFRA170_SUMMARY={\"decision\":\"HOLD_EXTERNAL_SSH_ROUTE_AUTH_INCOMPLETE\",\"holds\":[\"US_NETBIRD:TRUSTED_USER_ROUTE_NOT_PROVEN\",\"MD_RELAY:TRUSTED_USER_ROUTE_NOT_PROVEN\",\"SPB_INGRESS:TRUSTED_USER_ROUTE_NOT_PROVEN\"],\"roles_hold\":[\"US_NETBIRD\",\"MD_RELAY\",\"SPB_INGRESS\"],\"roles_ready\":[]}\nEXTERNAL_SSH_ATTEMPTS=0\nSSH_KEYSCAN_ATTEMPTS=0\nKNOWN_HOSTS_WRITES=0\nSSH_CONFIG_WRITES=0\nCONFIGURATION_WRITES=0\nFILE_WRITES_BY_PROBE=0\nSECRETS_PRINTED=NO\nINFRA170_END\n" + "output": "INFRA171_BEGIN\nCOMMAND_ID=SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1\nMODE=READ_ONLY\nPRIOR_170_COMMIT=5c9001893ac691f04b4b8551c801a41c2ddd507c\nINFRA171_ROLE_DNS={\"MD_RELAY\":{\"hostname\":\"nb.pvepro.ru\",\"ip_refs\":[{\"address_class\":\"PUBLIC\",\"ip_sha256\":\"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16\"}],\"rc\":0},\"SPB_INGRESS\":{\"hostname\":\"newfi.ru\",\"ip_refs\":[{\"address_class\":\"PUBLIC\",\"identity_marker\":\"HISTORICAL_SPB_INGRESS\",\"ip_sha256\":\"6d6149f42864ce03daf22828cf236bf64d5d9a004040746ddbb051d71c9fcbc5\"}],\"rc\":0},\"US_NETBIRD\":{\"hostname\":\"nb.gram1.ru\",\"ip_refs\":[{\"address_class\":\"PUBLIC\",\"ip_sha256\":\"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16\"}],\"rc\":0}}\nINFRA171_UNITS={\"count\":2,\"rows\":[{\"directives\":[{\"key\":\"User\",\"line\":9,\"value\":{\"kind\":\"SANITIZED\",\"text\":\"root\"}},{\"key\":\"Group\",\"line\":10,\"value\":{\"kind\":\"SANITIZED\",\"text\":\"root\"}},{\"key\":\"ExecStart\",\"line\":11,\"value\":{\"kind\":\"SANITIZED\",\"text\":\"/usr/local/sbin/homelab-ingress-standby-refresh\"}}],\"file\":{\"basename\":\"homelab-ingress-standby-refresh.service\",\"path_sha256\":\"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e\",\"scope\":\"ETC\"}},{\"directives\":[],\"file\":{\"basename\":\"homelab-ingress-standby-refresh.timer\",\"path_sha256\":\"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1\",\"scope\":\"ETC\"}}]}\nINFRA171_FILES={\"bytes_read\":21032,\"count\":6,\"files\":[{\"basename\":\"homelab-ingress-standby-refresh.service\",\"path_sha256\":\"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e\",\"scope\":\"ETC\"},{\"basename\":\"homelab-ingress-standby-refresh.timer\",\"path_sha256\":\"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1\",\"scope\":\"ETC\"},{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},{\"basename\":\"homelab-ingress-failover\",\"path\":\"/usr/local/sbin/homelab-ingress-failover\",\"scope\":\"USR_LOCAL\"},{\"basename\":\"homelab-ingress-standby-refresh\",\"path\":\"/usr/local/sbin/homelab-ingress-standby-refresh\",\"scope\":\"USR_LOCAL\"}]}\nINFRA171_SAFE_ASSIGNMENTS={\"count\":19,\"rows\":[{\"key\":\"User\",\"kind\":\"USER\",\"line\":9,\"source\":{\"basename\":\"homelab-ingress-standby-refresh.service\",\"path_sha256\":\"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e\",\"scope\":\"ETC\"},\"value\":\"root\"},{\"key\":\"ExecStart\",\"kind\":\"RELEVANT_KEY\",\"line\":11,\"source\":{\"basename\":\"homelab-ingress-standby-refresh.service\",\"path_sha256\":\"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e\",\"scope\":\"ETC\"},\"value_sha256\":\"13d114d64c8f3496ac6b3a8eee4a94ae9e583182842310e10dc40f15561e400c\"},{\"key\":\"REPORT\",\"kind\":\"RELEVANT_KEY\",\"line\":5,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"16de6e2523ad6290564a03adb219172bc0031e08d824a49be9838f00eb297f61\"},{\"key\":\"vps_trust\",\"kind\":\"RELEVANT_KEY\",\"line\":37,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"33003123cbc00728f9934444716841ec1ff9b529c997839691bb14c20d436e53\"},{\"key\":\"trusted_netbird_identity\",\"kind\":\"RELEVANT_KEY\",\"line\":38,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"ebb24cf2960c2243b155740605cba8127cedb6e016de2d44b914019d3fdbe085\"},{\"key\":\"trusted_netbird_ip\",\"kind\":\"RELEVANT_KEY\",\"line\":39,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440\"},{\"key\":\"public_vps_ip\",\"kind\":\"HOST\",\"line\":40,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value\":{\"address_class\":\"PUBLIC\",\"ip_sha256\":\"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620\"}},{\"key\":\"public_vps_decision\",\"kind\":\"RELEVANT_KEY\",\"line\":41,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"348e0553be62997b4c48bd45853803cba5a55da3b958f8e9b9419fa3c734b19d\"},{\"key\":\"backup_hostkeys\",\"kind\":\"RELEVANT_KEY\",\"line\":42,\"source\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"2526f72dab61a237d504a652be0ab5e869e292ba1290fe76f9d34ea5578d980d\"},{\"key\":\"REPORT\",\"kind\":\"RELEVANT_KEY\",\"line\":5,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"16de6e2523ad6290564a03adb219172bc0031e08d824a49be9838f00eb297f61\"},{\"key\":\"vps_trust\",\"kind\":\"RELEVANT_KEY\",\"line\":36,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"33003123cbc00728f9934444716841ec1ff9b529c997839691bb14c20d436e53\"},{\"key\":\"trusted_netbird_identity\",\"kind\":\"RELEVANT_KEY\",\"line\":37,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"ebb24cf2960c2243b155740605cba8127cedb6e016de2d44b914019d3fdbe085\"},{\"key\":\"trusted_netbird_ip\",\"kind\":\"RELEVANT_KEY\",\"line\":38,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440\"},{\"key\":\"public_vps_ip\",\"kind\":\"HOST\",\"line\":39,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value\":{\"address_class\":\"PUBLIC\",\"ip_sha256\":\"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620\"}},{\"key\":\"public_vps_decision\",\"kind\":\"RELEVANT_KEY\",\"line\":40,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"348e0553be62997b4c48bd45853803cba5a55da3b958f8e9b9419fa3c734b19d\"},{\"key\":\"backup_hostkeys\",\"kind\":\"RELEVANT_KEY\",\"line\":41,\"source\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"2526f72dab61a237d504a652be0ab5e869e292ba1290fe76f9d34ea5578d980d\"},{\"key\":\"DRILL_REPORT\",\"kind\":\"RELEVANT_KEY\",\"line\":6,\"source\":{\"basename\":\"homelab-ingress-failover\",\"path\":\"/usr/local/sbin/homelab-ingress-failover\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"242de3f3a1197026b997b7565bcd2a7c61a69229904a4e133f1988e206575baf\"},{\"key\":\"DRILL_REPORT\",\"kind\":\"RELEVANT_KEY\",\"line\":6,\"source\":{\"basename\":\"homelab-ingress-standby-refresh\",\"path\":\"/usr/local/sbin/homelab-ingress-standby-refresh\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"242de3f3a1197026b997b7565bcd2a7c61a69229904a4e133f1988e206575baf\"},{\"key\":\"DEST\",\"kind\":\"HOST_EXPR\",\"line\":120,\"source\":{\"basename\":\"homelab-ingress-standby-refresh\",\"path\":\"/usr/local/sbin/homelab-ingress-standby-refresh\",\"scope\":\"USR_LOCAL\"},\"value_sha256\":\"fc4d602aa16170b89bda4c70d75d89e6183b14f994516aa7b3e032c3f18cbb4e\"}]}\nINFRA171_FILE_EVIDENCE={\"count\":6,\"rows\":[{\"file\":{\"basename\":\"homelab-ingress-standby-refresh.service\",\"path_sha256\":\"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e\",\"scope\":\"ETC\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"Description=Create current ingress recovery artifact on cold standby node\"},\"line\":2},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"After=network-online.target pve-cluster.service\"},\"line\":3},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"Wants=network-online.target\"},\"line\":4},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"User=root\"},\"line\":9},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\"},\"line\":11}]},{\"file\":{\"basename\":\"homelab-ingress-standby-refresh.timer\",\"path_sha256\":\"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1\",\"scope\":\"ETC\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"Description=Daily ingress cold standby recovery artifact refresh\"},\"line\":2},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"Unit=homelab-ingress-standby-refresh.service\"},\"line\":8},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"WantedBy=timers.target\"},\"line\":11}]},{\"file\":{\"basename\":\"homelab-external-probe-vps\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps\",\"scope\":\"USR_LOCAL\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ROOT=\\\"/var/lib/homelab-external-probe-vps\\\"\"},\"line\":3},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"HEALTH=\\\"/var/lib/homelab-health/external-probe-vps.txt\\\"\"},\"line\":4},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"REPORT=\\\"$ROOT/latest.md\\\"\"},\"line\":5},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"^STATUS=OK\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":9},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"^STATUS=OK\\\" /var/lib/homelab-health/netbird-vps-trust-closure.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":10},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"TRUSTED_NETBIRD_IDENTITY=edge-vm\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":12},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":13},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"cat > \\\"$REPORT\\\" <\"},\"line\":39},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"public_vps_ip=\"},\"line\":40},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"public_vps_decision=REJECTED_HOSTKEY_MISMATCH\"},\"line\":41},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"backup_hostkeys=not_used_public_vps_rejected\"},\"line\":42},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\"},\"line\":45},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\"},\"line\":46},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"closed_tail=no_known_hosts_change\"},\"line\":48},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=external-probe-vps BAD=$BAD WARN=$WARN MODE=STRATEGY_ONLY_NO_INDEPENDENT_RUNNER DEPLOYED=NO DEPLOY_DECISION=NO_DEPLOY VPS_TRUST=edge_vm_trusted_public_vps_rejected BACKUP_HOSTKEYS=not_used_public_vps_rejected PVE01_PUBLIC_SMOKE_BAD=$PVE01_PUBLIC_SMOKE_BAD INDEPENDENT_RUNNER=missing INDEPENDENT_RUNNER_AVAILABLE=no TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP= PUBLIC_VPS_IP= PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH REPORT=$REPORT\\\" > \\\"$HEALTH\\\"\"},\"line\":51}]},{\"file\":{\"basename\":\"homelab-external-probe-vps-health\",\"path\":\"/usr/local/sbin/homelab-external-probe-vps-health\",\"scope\":\"USR_LOCAL\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ROOT=\\\"/var/lib/homelab-external-probe-vps\\\"\"},\"line\":3},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"HEALTH=\\\"/var/lib/homelab-health/external-probe-vps.txt\\\"\"},\"line\":4},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"REPORT=\\\"$ROOT/latest.md\\\"\"},\"line\":5},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"for url in $URLS; do safe=\\\"$(echo \\\"$url\\\" | sed \\\"s#[^A-Za-z0-9]#_#g\\\")\\\"; body=\\\"/tmp/external-probe-vps-health-${safe}.body\\\"; code=\\\"$(curl -k -sS -o \\\"$body\\\" -w \\\"%{http_code}\\\" --connect-timeout 8 --max-time 18 \\\"$url\\\" 2>/tmp/external-probe-vps-health.err)\\\"; first=\\\"$(head -n1 \\\"$body\\\" 2>/dev/null | tr -cd \\\"[:print:]\\\")\\\"; bytes=\\\"$(wc -c < \\\"$body\\\" 2>/dev/null || echo 0)\\\"; printf \\\"%s\\\\t%s\\\\t%s\\\\t%s\\\\n\\\" \\\"$url\\\" \\\"$code\\\" \\\"$bytes\\\" \\\"$first\\\" >> \\\"$SMOKE_TSV\\\"; case \\\"$code:$first\\\" in 200:STATUS=*) : ;; *) PVE01_PUBLIC_SMOKE_BAD=$((PVE01_PUBLIC_SMOKE_BAD+1));; esac; done\"},\"line\":12},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"^STATUS=OK\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":13},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"^STATUS=OK\\\" /var/lib/homelab-health/netbird-vps-trust-closure.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":14},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"TRUSTED_NETBIRD_IDENTITY=edge-vm\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":16},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q \\\"PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH\\\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))\"},\"line\":17},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"cat > \\\"$REPORT\\\" <\"},\"line\":38},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"public_vps_ip=\"},\"line\":39},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"public_vps_decision=REJECTED_HOSTKEY_MISMATCH\"},\"line\":40},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"backup_hostkeys=not_used_public_vps_rejected\"},\"line\":41},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\"},\"line\":44},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\"},\"line\":45},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"closed_tail=no_known_hosts_change\"},\"line\":47},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=external-probe-vps BAD=$BAD WARN=$WARN DEPLOYED=NO DEPLOY_DECISION=NO_DEPLOY VPS_TRUST=edge_vm_trusted_public_vps_rejected BACKUP_HOSTKEYS=not_used_public_vps_rejected PVE01_PUBLIC_SMOKE_BAD=$PVE01_PUBLIC_SMOKE_BAD INDEPENDENT_RUNNER=missing INDEPENDENT_RUNNER_AVAILABLE=no TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP= PUBLIC_VPS_IP= PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH REPORT=$REPORT\\\" > \\\"$HEALTH\\\"\"},\"line\":54}]},{\"file\":{\"basename\":\"homelab-ingress-failover\",\"path\":\"/usr/local/sbin/homelab-ingress-failover\",\"scope\":\"USR_LOCAL\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"export LC_ALL=C\"},\"line\":3},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"DRILL_REPORT=\\\"/var/lib/homelab-chat/CR0023-cold-standby-drill.txt\\\"\"},\"line\":6},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"STATE_DIR=\\\"/var/lib/homelab-health/ingress-failover\\\"\"},\"line\":7},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"LOCK_FILE=\\\"/run/lock/homelab-ingress-failover.lock\\\"\"},\"line\":9},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"echo \\\"INGRESS_FAILOVER_LOCK=BUSY\\\"\"},\"line\":15},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"test -f \\\"$DRILL_REPORT\\\"\"},\"line\":19},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"sed -n 's/^STANDBY_VMID=//p' \\\"$DRILL_REPORT\\\" |\"},\"line\":22},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"sed -n 's/^STANDBY_NODE=//p' \\\"$DRILL_REPORT\\\" |\"},\"line\":27},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"import json\"},\"line\":48},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"import sys\"},\"line\":49},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"if [ \\\"$node\\\" = \\\"$(hostname -s)\\\" ] ||\"},\"line\":71},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"[ \\\"$node\\\" = \\\"$(hostname)\\\" ]\"},\"line\":72},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ssh \\\\\"},\"line\":76},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"export LC_ALL=C\"},\"line\":196},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -Rqs \\\"server_name[[:space:]]*git\\\\.gram1\\\\.ru\\\" \\\\\"},\"line\":213},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"echo \\\"ACTIVE_INGRESS=$active\\\"\"},\"line\":241}]},{\"file\":{\"basename\":\"homelab-ingress-standby-refresh\",\"path\":\"/usr/local/sbin/homelab-ingress-standby-refresh\",\"scope\":\"USR_LOCAL\"},\"rows\":[{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"export LC_ALL=C\"},\"line\":3},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"LOCK=\\\"/run/lock/homelab-ingress-standby-refresh.lock\\\"\"},\"line\":5},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"DRILL_REPORT=\\\"/var/lib/homelab-chat/CR0023-cold-standby-drill.txt\\\"\"},\"line\":6},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"STATE_DIR=\\\"/var/lib/homelab-ingress-standby\\\"\"},\"line\":7},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"RELAY=\\\"$(mktemp)\\\"\"},\"line\":12},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"rm -f \\\"$TMP_STATE\\\" \\\"$RELAY\\\" \\\"$RESOURCES\\\"\"},\"line\":16},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"test -f \\\"$DRILL_REPORT\\\"\"},\"line\":26},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"grep -q '^CR0023_COLD_STANDBY_DRILL_STATUS=SUCCESS$' \\\"$DRILL_REPORT\\\"\"},\"line\":27},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"STANDBY_VMID=\\\"$(sed -n 's/^STANDBY_VMID=//p' \\\"$DRILL_REPORT\\\" | head -n 1)\\\"\"},\"line\":29},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"STANDBY_NODE=\\\"$(sed -n 's/^STANDBY_NODE=//p' \\\"$DRILL_REPORT\\\" | head -n 1)\\\"\"},\"line\":30},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"import json, sys\"},\"line\":39},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"import json, sys\"},\"line\":47},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"if [ \\\"$node\\\" = \\\"$(hostname -s)\\\" ] || [ \\\"$node\\\" = \\\"$(hostname)\\\" ]; then\"},\"line\":60},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\\\"$node\\\" \\\"$@\\\"\"},\"line\":63},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"if [ \\\"$SOURCE_NODE\\\" = \\\"$(hostname -s)\\\" ] || [ \\\"$SOURCE_NODE\\\" = \\\"$(hostname)\\\" ]; then\"},\"line\":91},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"cat \\\"$SOURCE_BACKUP\\\" >\\\"$RELAY\\\"\"},\"line\":92},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\\\"$SOURCE_NODE\\\" \\\"cat '$SOURCE_BACKUP'\\\" >\\\"$RELAY\\\"\"},\"line\":94},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"test \\\"$(stat -c %s \\\"$RELAY\\\")\\\" = \\\"$SOURCE_SIZE\\\"\"},\"line\":97},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"test \\\"$(sha256sum \\\"$RELAY\\\" | awk '{print $1}')\\\" = \\\"$SOURCE_SHA256\\\"\"},\"line\":98},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"DEST_DIR=\\\"/var/backups/ingress-standby-source\\\"\"},\"line\":100},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"if [ \\\"$STANDBY_NODE\\\" = \\\"$(hostname -s)\\\" ] || [ \\\"$STANDBY_NODE\\\" = \\\"$(hostname)\\\" ]; then\"},\"line\":108},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"cat \\\"$RELAY\\\" >\\\"$DEST_TMP\\\"\"},\"line\":109},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\\\"$STANDBY_NODE\\\" \\\"cat >'$DEST_TMP'\\\" <\\\"$RELAY\\\"\"},\"line\":111},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"\\\"$SOURCE_NODE\\\" \\\"$STANDBY_VMID\\\" \\\"$KEEP\\\" <<'REMOTE'\"},\"line\":116},{\"evidence\":{\"kind\":\"SANITIZED\",\"text\":\"REMOTE\"},\"line\":171}]}]}\nINFRA171_ROUTE_CANDIDATES={\"count\":0,\"rows\":[]}\nINFRA171_ROLE_ROUTE_SUMMARY={\"MD_RELAY\":{\"route_candidates\":[],\"trusted_user_route_proven\":false,\"trusted_user_routes\":[]},\"SPB_INGRESS\":{\"route_candidates\":[],\"trusted_user_route_proven\":false,\"trusted_user_routes\":[]},\"US_NETBIRD\":{\"route_candidates\":[],\"trusted_user_route_proven\":false,\"trusted_user_routes\":[]}}\nINFRA171_SUMMARY={\"decision\":\"HOLD_EXTERNAL_SSH_UNIT_TRACE_INCOMPLETE\",\"holds\":[\"US_NETBIRD:TRUSTED_USER_ROUTE_NOT_PROVEN\",\"MD_RELAY:TRUSTED_USER_ROUTE_NOT_PROVEN\",\"SPB_INGRESS:TRUSTED_USER_ROUTE_NOT_PROVEN\"],\"roles_hold\":[\"US_NETBIRD\",\"MD_RELAY\",\"SPB_INGRESS\"],\"roles_ready\":[]}\nEXTERNAL_SSH_ATTEMPTS=0\nSSH_KEYSCAN_ATTEMPTS=0\nKNOWN_HOSTS_WRITES=0\nSSH_CONFIG_WRITES=0\nCONFIGURATION_WRITES=0\nFILE_WRITES_BY_PROBE=0\nSECRETS_PRINTED=NO\nINFRA171_END\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 16d6dc77..6d6b3e5c 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260911-INFRA-DR-VPS-ROUTE-AUTH-170R1 +COMMAND_ID=SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=infra-dr-vps-route-auth-pve01 +COMPONENT=infra-dr-vps-unit-trace-pve01 REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=396759f2f07e402daf35047c1bde8557e62601310a1987f75906bec86336363a +COMMAND_SHA256=59ac720637b5a262dac71928edd44bbbf906f7be1b225a371a55f1074eb0a4a4 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,20 +24,21 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=5fb23cd7f83ef292be65ad46e52f7c1e5378ce4834aa194a549a99cf6e615fd1 -SANITIZED_OUTPUT_SHA256=5fb23cd7f83ef292be65ad46e52f7c1e5378ce4834aa194a549a99cf6e615fd1 +RAW_EVIDENCE_SHA256=5e8e4e3d538b446a7c0c0deb37ca79eba873606c075765fd974bc6b0cbf7936e +SANITIZED_OUTPUT_SHA256=5e8e4e3d538b446a7c0c0deb37ca79eba873606c075765fd974bc6b0cbf7936e OUTPUT_BEGIN -INFRA170_BEGIN -COMMAND_ID=SUPPORT-260911-INFRA-DR-VPS-ROUTE-AUTH-170R1 +INFRA171_BEGIN +COMMAND_ID=SUPPORT-260911-INFRA-DR-VPS-UNIT-TRACE-171R1 MODE=READ_ONLY -PRIOR_169_COMMIT=c934e939d51c24b6fabc5be1b72ce82effa05a5e -INFRA170_ROLE_DNS={"MD_RELAY":{"hostname":"nb.pvepro.ru","ip_refs":[{"address_class":"PUBLIC","ip_sha256":"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16"}],"rc":0},"SPB_INGRESS":{"hostname":"newfi.ru","ip_refs":[{"address_class":"PUBLIC","identity_marker":"HISTORICAL_SPB_INGRESS","ip_sha256":"6d6149f42864ce03daf22828cf236bf64d5d9a004040746ddbb051d71c9fcbc5"}],"rc":0},"US_NETBIRD":{"hostname":"nb.gram1.ru","ip_refs":[{"address_class":"PUBLIC","ip_sha256":"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16"}],"rc":0}} -INFRA170_UNIT_CHAIN={"bytes_read":15144,"file_count":4,"files_scanned":["/etc/systemd/system/homelab-ingress-standby-refresh.service","/usr/local/sbin/homelab-ingress-standby-refresh","/etc/systemd/system/homelab-ingress-standby-refresh.timer","/usr/local/sbin/homelab-ingress-failover"],"unit_count":2,"units":[{"path":"/etc/systemd/system/homelab-ingress-standby-refresh.service","safe_directives":[{"key":"User","line":9,"value":"root"},{"key":"Group","line":10,"value":"root"},{"key":"ExecStart","line":11,"value_kind":"EXEC"}]},{"path":"/etc/systemd/system/homelab-ingress-standby-refresh.timer","safe_directives":[]}]} -INFRA170_SAFE_ASSIGNMENTS={"count":0,"rows":[]} -INFRA170_ROUTE_CANDIDATES={"count":0,"rows":[]} -INFRA170_SAME_FILE_BUNDLES={"count":0,"rows":[]} -INFRA170_ROLE_ROUTE_SUMMARY={"MD_RELAY":{"route_candidates":[],"trusted_user_route_proven":false,"trusted_user_routes":[]},"SPB_INGRESS":{"route_candidates":[],"trusted_user_route_proven":false,"trusted_user_routes":[]},"US_NETBIRD":{"route_candidates":[],"trusted_user_route_proven":false,"trusted_user_routes":[]}} -INFRA170_SUMMARY={"decision":"HOLD_EXTERNAL_SSH_ROUTE_AUTH_INCOMPLETE","holds":["US_NETBIRD:TRUSTED_USER_ROUTE_NOT_PROVEN","MD_RELAY:TRUSTED_USER_ROUTE_NOT_PROVEN","SPB_INGRESS:TRUSTED_USER_ROUTE_NOT_PROVEN"],"roles_hold":["US_NETBIRD","MD_RELAY","SPB_INGRESS"],"roles_ready":[]} +PRIOR_170_COMMIT=5c9001893ac691f04b4b8551c801a41c2ddd507c +INFRA171_ROLE_DNS={"MD_RELAY":{"hostname":"nb.pvepro.ru","ip_refs":[{"address_class":"PUBLIC","ip_sha256":"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16"}],"rc":0},"SPB_INGRESS":{"hostname":"newfi.ru","ip_refs":[{"address_class":"PUBLIC","identity_marker":"HISTORICAL_SPB_INGRESS","ip_sha256":"6d6149f42864ce03daf22828cf236bf64d5d9a004040746ddbb051d71c9fcbc5"}],"rc":0},"US_NETBIRD":{"hostname":"nb.gram1.ru","ip_refs":[{"address_class":"PUBLIC","ip_sha256":"1fc73136d6d7ddbd0e67ae9c4ddbe6798056fba742305ef773f1015df2712c16"}],"rc":0}} +INFRA171_UNITS={"count":2,"rows":[{"directives":[{"key":"User","line":9,"value":{"kind":"SANITIZED","text":"root"}},{"key":"Group","line":10,"value":{"kind":"SANITIZED","text":"root"}},{"key":"ExecStart","line":11,"value":{"kind":"SANITIZED","text":"/usr/local/sbin/homelab-ingress-standby-refresh"}}],"file":{"basename":"homelab-ingress-standby-refresh.service","path_sha256":"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e","scope":"ETC"}},{"directives":[],"file":{"basename":"homelab-ingress-standby-refresh.timer","path_sha256":"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1","scope":"ETC"}}]} +INFRA171_FILES={"bytes_read":21032,"count":6,"files":[{"basename":"homelab-ingress-standby-refresh.service","path_sha256":"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e","scope":"ETC"},{"basename":"homelab-ingress-standby-refresh.timer","path_sha256":"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1","scope":"ETC"},{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},{"basename":"homelab-ingress-failover","path":"/usr/local/sbin/homelab-ingress-failover","scope":"USR_LOCAL"},{"basename":"homelab-ingress-standby-refresh","path":"/usr/local/sbin/homelab-ingress-standby-refresh","scope":"USR_LOCAL"}]} +INFRA171_SAFE_ASSIGNMENTS={"count":19,"rows":[{"key":"User","kind":"USER","line":9,"source":{"basename":"homelab-ingress-standby-refresh.service","path_sha256":"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e","scope":"ETC"},"value":"root"},{"key":"ExecStart","kind":"RELEVANT_KEY","line":11,"source":{"basename":"homelab-ingress-standby-refresh.service","path_sha256":"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e","scope":"ETC"},"value_sha256":"13d114d64c8f3496ac6b3a8eee4a94ae9e583182842310e10dc40f15561e400c"},{"key":"REPORT","kind":"RELEVANT_KEY","line":5,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"16de6e2523ad6290564a03adb219172bc0031e08d824a49be9838f00eb297f61"},{"key":"vps_trust","kind":"RELEVANT_KEY","line":37,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"33003123cbc00728f9934444716841ec1ff9b529c997839691bb14c20d436e53"},{"key":"trusted_netbird_identity","kind":"RELEVANT_KEY","line":38,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"ebb24cf2960c2243b155740605cba8127cedb6e016de2d44b914019d3fdbe085"},{"key":"trusted_netbird_ip","kind":"RELEVANT_KEY","line":39,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440"},{"key":"public_vps_ip","kind":"HOST","line":40,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value":{"address_class":"PUBLIC","ip_sha256":"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620"}},{"key":"public_vps_decision","kind":"RELEVANT_KEY","line":41,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"348e0553be62997b4c48bd45853803cba5a55da3b958f8e9b9419fa3c734b19d"},{"key":"backup_hostkeys","kind":"RELEVANT_KEY","line":42,"source":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"value_sha256":"2526f72dab61a237d504a652be0ab5e869e292ba1290fe76f9d34ea5578d980d"},{"key":"REPORT","kind":"RELEVANT_KEY","line":5,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"16de6e2523ad6290564a03adb219172bc0031e08d824a49be9838f00eb297f61"},{"key":"vps_trust","kind":"RELEVANT_KEY","line":36,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"33003123cbc00728f9934444716841ec1ff9b529c997839691bb14c20d436e53"},{"key":"trusted_netbird_identity","kind":"RELEVANT_KEY","line":37,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"ebb24cf2960c2243b155740605cba8127cedb6e016de2d44b914019d3fdbe085"},{"key":"trusted_netbird_ip","kind":"RELEVANT_KEY","line":38,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"5649415146501fa3dc7b2e08f469ef80b6b04e46b86079acd21229eb875e9440"},{"key":"public_vps_ip","kind":"HOST","line":39,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value":{"address_class":"PUBLIC","ip_sha256":"f743a81d85de402239bdb782892adc0886327b57d99fdb0b889aaf4768e59620"}},{"key":"public_vps_decision","kind":"RELEVANT_KEY","line":40,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"348e0553be62997b4c48bd45853803cba5a55da3b958f8e9b9419fa3c734b19d"},{"key":"backup_hostkeys","kind":"RELEVANT_KEY","line":41,"source":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"value_sha256":"2526f72dab61a237d504a652be0ab5e869e292ba1290fe76f9d34ea5578d980d"},{"key":"DRILL_REPORT","kind":"RELEVANT_KEY","line":6,"source":{"basename":"homelab-ingress-failover","path":"/usr/local/sbin/homelab-ingress-failover","scope":"USR_LOCAL"},"value_sha256":"242de3f3a1197026b997b7565bcd2a7c61a69229904a4e133f1988e206575baf"},{"key":"DRILL_REPORT","kind":"RELEVANT_KEY","line":6,"source":{"basename":"homelab-ingress-standby-refresh","path":"/usr/local/sbin/homelab-ingress-standby-refresh","scope":"USR_LOCAL"},"value_sha256":"242de3f3a1197026b997b7565bcd2a7c61a69229904a4e133f1988e206575baf"},{"key":"DEST","kind":"HOST_EXPR","line":120,"source":{"basename":"homelab-ingress-standby-refresh","path":"/usr/local/sbin/homelab-ingress-standby-refresh","scope":"USR_LOCAL"},"value_sha256":"fc4d602aa16170b89bda4c70d75d89e6183b14f994516aa7b3e032c3f18cbb4e"}]} +INFRA171_FILE_EVIDENCE={"count":6,"rows":[{"file":{"basename":"homelab-ingress-standby-refresh.service","path_sha256":"d267675a7c11436bf83d33ba637a28d00f08e91013f12af00a28d138eddc531e","scope":"ETC"},"rows":[{"evidence":{"kind":"SANITIZED","text":"Description=Create current ingress recovery artifact on cold standby node"},"line":2},{"evidence":{"kind":"SANITIZED","text":"After=network-online.target pve-cluster.service"},"line":3},{"evidence":{"kind":"SANITIZED","text":"Wants=network-online.target"},"line":4},{"evidence":{"kind":"SANITIZED","text":"User=root"},"line":9},{"evidence":{"kind":"SANITIZED","text":""},"line":11}]},{"file":{"basename":"homelab-ingress-standby-refresh.timer","path_sha256":"96feda8920362dd961046ef5f1375e8ccb4182400fe02408c76604e4aa4832c1","scope":"ETC"},"rows":[{"evidence":{"kind":"SANITIZED","text":"Description=Daily ingress cold standby recovery artifact refresh"},"line":2},{"evidence":{"kind":"SANITIZED","text":"Unit=homelab-ingress-standby-refresh.service"},"line":8},{"evidence":{"kind":"SANITIZED","text":"WantedBy=timers.target"},"line":11}]},{"file":{"basename":"homelab-external-probe-vps","path":"/usr/local/sbin/homelab-external-probe-vps","scope":"USR_LOCAL"},"rows":[{"evidence":{"kind":"SANITIZED","text":"ROOT=\"/var/lib/homelab-external-probe-vps\""},"line":3},{"evidence":{"kind":"SANITIZED","text":"HEALTH=\"/var/lib/homelab-health/external-probe-vps.txt\""},"line":4},{"evidence":{"kind":"SANITIZED","text":"REPORT=\"$ROOT/latest.md\""},"line":5},{"evidence":{"kind":"SANITIZED","text":"grep -q \"^STATUS=OK\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":9},{"evidence":{"kind":"SANITIZED","text":"grep -q \"^STATUS=OK\" /var/lib/homelab-health/netbird-vps-trust-closure.txt 2>/dev/null || BAD=$((BAD+1))"},"line":10},{"evidence":{"kind":"SANITIZED","text":"grep -q \"TRUSTED_NETBIRD_IDENTITY=edge-vm\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":12},{"evidence":{"kind":"SANITIZED","text":"grep -q \"PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":13},{"evidence":{"kind":"SANITIZED","text":"cat > \"$REPORT\" <"},"line":39},{"evidence":{"kind":"SANITIZED","text":"public_vps_ip="},"line":40},{"evidence":{"kind":"SANITIZED","text":"public_vps_decision=REJECTED_HOSTKEY_MISMATCH"},"line":41},{"evidence":{"kind":"SANITIZED","text":"backup_hostkeys=not_used_public_vps_rejected"},"line":42},{"evidence":{"kind":"SANITIZED","text":""},"line":45},{"evidence":{"kind":"SANITIZED","text":""},"line":46},{"evidence":{"kind":"SANITIZED","text":"closed_tail=no_known_hosts_change"},"line":48},{"evidence":{"kind":"SANITIZED","text":"echo \"STATUS=$STATUS TS=$TS TYPE=external-probe-vps BAD=$BAD WARN=$WARN MODE=STRATEGY_ONLY_NO_INDEPENDENT_RUNNER DEPLOYED=NO DEPLOY_DECISION=NO_DEPLOY VPS_TRUST=edge_vm_trusted_public_vps_rejected BACKUP_HOSTKEYS=not_used_public_vps_rejected PVE01_PUBLIC_SMOKE_BAD=$PVE01_PUBLIC_SMOKE_BAD INDEPENDENT_RUNNER=missing INDEPENDENT_RUNNER_AVAILABLE=no TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP= PUBLIC_VPS_IP= PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH REPORT=$REPORT\" > \"$HEALTH\""},"line":51}]},{"file":{"basename":"homelab-external-probe-vps-health","path":"/usr/local/sbin/homelab-external-probe-vps-health","scope":"USR_LOCAL"},"rows":[{"evidence":{"kind":"SANITIZED","text":"ROOT=\"/var/lib/homelab-external-probe-vps\""},"line":3},{"evidence":{"kind":"SANITIZED","text":"HEALTH=\"/var/lib/homelab-health/external-probe-vps.txt\""},"line":4},{"evidence":{"kind":"SANITIZED","text":"REPORT=\"$ROOT/latest.md\""},"line":5},{"evidence":{"kind":"SANITIZED","text":"for url in $URLS; do safe=\"$(echo \"$url\" | sed \"s#[^A-Za-z0-9]#_#g\")\"; body=\"/tmp/external-probe-vps-health-${safe}.body\"; code=\"$(curl -k -sS -o \"$body\" -w \"%{http_code}\" --connect-timeout 8 --max-time 18 \"$url\" 2>/tmp/external-probe-vps-health.err)\"; first=\"$(head -n1 \"$body\" 2>/dev/null | tr -cd \"[:print:]\")\"; bytes=\"$(wc -c < \"$body\" 2>/dev/null || echo 0)\"; printf \"%s\\t%s\\t%s\\t%s\\n\" \"$url\" \"$code\" \"$bytes\" \"$first\" >> \"$SMOKE_TSV\"; case \"$code:$first\" in 200:STATUS=*) : ;; *) PVE01_PUBLIC_SMOKE_BAD=$((PVE01_PUBLIC_SMOKE_BAD+1));; esac; done"},"line":12},{"evidence":{"kind":"SANITIZED","text":"grep -q \"^STATUS=OK\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":13},{"evidence":{"kind":"SANITIZED","text":"grep -q \"^STATUS=OK\" /var/lib/homelab-health/netbird-vps-trust-closure.txt 2>/dev/null || BAD=$((BAD+1))"},"line":14},{"evidence":{"kind":"SANITIZED","text":"grep -q \"TRUSTED_NETBIRD_IDENTITY=edge-vm\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":16},{"evidence":{"kind":"SANITIZED","text":"grep -q \"PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH\" /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt 2>/dev/null || BAD=$((BAD+1))"},"line":17},{"evidence":{"kind":"SANITIZED","text":"cat > \"$REPORT\" <"},"line":38},{"evidence":{"kind":"SANITIZED","text":"public_vps_ip="},"line":39},{"evidence":{"kind":"SANITIZED","text":"public_vps_decision=REJECTED_HOSTKEY_MISMATCH"},"line":40},{"evidence":{"kind":"SANITIZED","text":"backup_hostkeys=not_used_public_vps_rejected"},"line":41},{"evidence":{"kind":"SANITIZED","text":""},"line":44},{"evidence":{"kind":"SANITIZED","text":""},"line":45},{"evidence":{"kind":"SANITIZED","text":"closed_tail=no_known_hosts_change"},"line":47},{"evidence":{"kind":"SANITIZED","text":"echo \"STATUS=$STATUS TS=$TS TYPE=external-probe-vps BAD=$BAD WARN=$WARN DEPLOYED=NO DEPLOY_DECISION=NO_DEPLOY VPS_TRUST=edge_vm_trusted_public_vps_rejected BACKUP_HOSTKEYS=not_used_public_vps_rejected PVE01_PUBLIC_SMOKE_BAD=$PVE01_PUBLIC_SMOKE_BAD INDEPENDENT_RUNNER=missing INDEPENDENT_RUNNER_AVAILABLE=no TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP= PUBLIC_VPS_IP= PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH REPORT=$REPORT\" > \"$HEALTH\""},"line":54}]},{"file":{"basename":"homelab-ingress-failover","path":"/usr/local/sbin/homelab-ingress-failover","scope":"USR_LOCAL"},"rows":[{"evidence":{"kind":"SANITIZED","text":"export LC_ALL=C"},"line":3},{"evidence":{"kind":"SANITIZED","text":"DRILL_REPORT=\"/var/lib/homelab-chat/CR0023-cold-standby-drill.txt\""},"line":6},{"evidence":{"kind":"SANITIZED","text":"STATE_DIR=\"/var/lib/homelab-health/ingress-failover\""},"line":7},{"evidence":{"kind":"SANITIZED","text":"LOCK_FILE=\"/run/lock/homelab-ingress-failover.lock\""},"line":9},{"evidence":{"kind":"SANITIZED","text":"echo \"INGRESS_FAILOVER_LOCK=BUSY\""},"line":15},{"evidence":{"kind":"SANITIZED","text":"test -f \"$DRILL_REPORT\""},"line":19},{"evidence":{"kind":"SANITIZED","text":"sed -n 's/^STANDBY_VMID=//p' \"$DRILL_REPORT\" |"},"line":22},{"evidence":{"kind":"SANITIZED","text":"sed -n 's/^STANDBY_NODE=//p' \"$DRILL_REPORT\" |"},"line":27},{"evidence":{"kind":"SANITIZED","text":"import json"},"line":48},{"evidence":{"kind":"SANITIZED","text":"import sys"},"line":49},{"evidence":{"kind":"SANITIZED","text":"if [ \"$node\" = \"$(hostname -s)\" ] ||"},"line":71},{"evidence":{"kind":"SANITIZED","text":"[ \"$node\" = \"$(hostname)\" ]"},"line":72},{"evidence":{"kind":"SANITIZED","text":"ssh \\"},"line":76},{"evidence":{"kind":"SANITIZED","text":"export LC_ALL=C"},"line":196},{"evidence":{"kind":"SANITIZED","text":"grep -Rqs \"server_name[[:space:]]*git\\.gram1\\.ru\" \\"},"line":213},{"evidence":{"kind":"SANITIZED","text":"echo \"ACTIVE_INGRESS=$active\""},"line":241}]},{"file":{"basename":"homelab-ingress-standby-refresh","path":"/usr/local/sbin/homelab-ingress-standby-refresh","scope":"USR_LOCAL"},"rows":[{"evidence":{"kind":"SANITIZED","text":"export LC_ALL=C"},"line":3},{"evidence":{"kind":"SANITIZED","text":"LOCK=\"/run/lock/homelab-ingress-standby-refresh.lock\""},"line":5},{"evidence":{"kind":"SANITIZED","text":"DRILL_REPORT=\"/var/lib/homelab-chat/CR0023-cold-standby-drill.txt\""},"line":6},{"evidence":{"kind":"SANITIZED","text":"STATE_DIR=\"/var/lib/homelab-ingress-standby\""},"line":7},{"evidence":{"kind":"SANITIZED","text":"RELAY=\"$(mktemp)\""},"line":12},{"evidence":{"kind":"SANITIZED","text":"rm -f \"$TMP_STATE\" \"$RELAY\" \"$RESOURCES\""},"line":16},{"evidence":{"kind":"SANITIZED","text":"test -f \"$DRILL_REPORT\""},"line":26},{"evidence":{"kind":"SANITIZED","text":"grep -q '^CR0023_COLD_STANDBY_DRILL_STATUS=SUCCESS$' \"$DRILL_REPORT\""},"line":27},{"evidence":{"kind":"SANITIZED","text":"STANDBY_VMID=\"$(sed -n 's/^STANDBY_VMID=//p' \"$DRILL_REPORT\" | head -n 1)\""},"line":29},{"evidence":{"kind":"SANITIZED","text":"STANDBY_NODE=\"$(sed -n 's/^STANDBY_NODE=//p' \"$DRILL_REPORT\" | head -n 1)\""},"line":30},{"evidence":{"kind":"SANITIZED","text":"import json, sys"},"line":39},{"evidence":{"kind":"SANITIZED","text":"import json, sys"},"line":47},{"evidence":{"kind":"SANITIZED","text":"if [ \"$node\" = \"$(hostname -s)\" ] || [ \"$node\" = \"$(hostname)\" ]; then"},"line":60},{"evidence":{"kind":"SANITIZED","text":"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$node\" \"$@\""},"line":63},{"evidence":{"kind":"SANITIZED","text":"if [ \"$SOURCE_NODE\" = \"$(hostname -s)\" ] || [ \"$SOURCE_NODE\" = \"$(hostname)\" ]; then"},"line":91},{"evidence":{"kind":"SANITIZED","text":"cat \"$SOURCE_BACKUP\" >\"$RELAY\""},"line":92},{"evidence":{"kind":"SANITIZED","text":"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$SOURCE_NODE\" \"cat '$SOURCE_BACKUP'\" >\"$RELAY\""},"line":94},{"evidence":{"kind":"SANITIZED","text":"test \"$(stat -c %s \"$RELAY\")\" = \"$SOURCE_SIZE\""},"line":97},{"evidence":{"kind":"SANITIZED","text":"test \"$(sha256sum \"$RELAY\" | awk '{print $1}')\" = \"$SOURCE_SHA256\""},"line":98},{"evidence":{"kind":"SANITIZED","text":"DEST_DIR=\"/var/backups/ingress-standby-source\""},"line":100},{"evidence":{"kind":"SANITIZED","text":"if [ \"$STANDBY_NODE\" = \"$(hostname -s)\" ] || [ \"$STANDBY_NODE\" = \"$(hostname)\" ]; then"},"line":108},{"evidence":{"kind":"SANITIZED","text":"cat \"$RELAY\" >\"$DEST_TMP\""},"line":109},{"evidence":{"kind":"SANITIZED","text":"ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$STANDBY_NODE\" \"cat >'$DEST_TMP'\" <\"$RELAY\""},"line":111},{"evidence":{"kind":"SANITIZED","text":"\"$SOURCE_NODE\" \"$STANDBY_VMID\" \"$KEEP\" <<'REMOTE'"},"line":116},{"evidence":{"kind":"SANITIZED","text":"REMOTE"},"line":171}]}]} +INFRA171_ROUTE_CANDIDATES={"count":0,"rows":[]} +INFRA171_ROLE_ROUTE_SUMMARY={"MD_RELAY":{"route_candidates":[],"trusted_user_route_proven":false,"trusted_user_routes":[]},"SPB_INGRESS":{"route_candidates":[],"trusted_user_route_proven":false,"trusted_user_routes":[]},"US_NETBIRD":{"route_candidates":[],"trusted_user_route_proven":false,"trusted_user_routes":[]}} +INFRA171_SUMMARY={"decision":"HOLD_EXTERNAL_SSH_UNIT_TRACE_INCOMPLETE","holds":["US_NETBIRD:TRUSTED_USER_ROUTE_NOT_PROVEN","MD_RELAY:TRUSTED_USER_ROUTE_NOT_PROVEN","SPB_INGRESS:TRUSTED_USER_ROUTE_NOT_PROVEN"],"roles_hold":["US_NETBIRD","MD_RELAY","SPB_INGRESS"],"roles_ready":[]} EXTERNAL_SSH_ATTEMPTS=0 SSH_KEYSCAN_ATTEMPTS=0 KNOWN_HOSTS_WRITES=0 @@ -45,7 +46,7 @@ SSH_CONFIG_WRITES=0 CONFIGURATION_WRITES=0 FILE_WRITES_BY_PROBE=0 SECRETS_PRINTED=NO -INFRA170_END +INFRA171_END OUTPUT_END CHAT_OUTPUT_END