diff --git a/runtime/history/SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1.json b/runtime/history/SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1.json new file mode 100644 index 00000000..0d889b61 --- /dev/null +++ b/runtime/history/SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "skladchik-usa-migration-authority-miniread", + "started_at_utc": "2026-09-25T18:31:29Z", + "finished_at_utc": "2026-09-25T18:31:29Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "50e4c7308ff3d4c751750cc63eee7536aa985f884c94b84ffcc1180c5eb60b05", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "f9810e721990dfc12c47863f23a39ad78582eaaf2cf212ac359cfbff17121390", + "sanitized_output_sha256": "f9810e721990dfc12c47863f23a39ad78582eaaf2cf212ac359cfbff17121390", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "RCA_VERSION=73276_V1\nMODE=read-only\nMIGRATION_DOC_EXISTS=true\nMIGRATION_DOC_SHA256=12fe079849346352315aac76fae82d00cbd17f88a3553270ca2c5f2749840552\nRUNBOOK_EXISTS=true\nRUNBOOK_SHA256=725e2068337b379ebdf47822d5871aa5f4d5ec5956546674b2cf55b3250fb7a3\nMIGRATION_DOC_SHA_MATCH=true\nRUNBOOK_SHA_MATCH=true\nFILE_BEGIN=20_NETBIRD_USA_MIGRATION_2026-08-18.md\n1:# NETBIRD USA MIGRATION — FINAL CURRENT RECORD\n5:Old USA mail/NetBird VPS:\n11:Host-level NetBird client left running for rollback/peer observation.\n54:- [TOKEN_OR_KEY] startup\n56:- Moldova synthetic check passes\n57:- edge peer works\n61:- `:9000/health` 503 in combined relay/server mode\n70:- narrow TrustedPeers\n71:- fix Traefik Compose network warning\nFILE_END=20_NETBIRD_USA_MIGRATION_2026-08-18.md\nFILE_BEGIN=04_NETBIRD_USA.md\n1:# Runbook: NetBird USA\n5:Do not use `/api/health` or combined `:9000/health` as generic readiness gates for this exact deployment; they were 404/503 and caused a false rollback. Use repeated Dashboard 200 + OIDC 200 + expected listeners + real client/peer connectivity.\n14:7. retain migration key only until old USA is no longer needed.\nFILE_END=04_NETBIRD_USA.md\nFILE_BEGIN=771z_after_manual_netbird_egress_capi.sh\n3:if ! test -s [TOKEN_OR_KEY].md || ! test -s [TOKEN_OR_KEY].md; then echo [TOKEN_OR_KEY]; exit 1; fi\n8:PROOF=\"[TOKEN_OR_KEY]${TS}_PROOF.txt\"\n14: echo \"[TOKEN_OR_KEY]\"\n17: echo \"[TOKEN_OR_KEY]\"\n18: echo \"TARGETS=edge-vm:[IP] relay-moldova:[IP] mail-usa:[IP]\"\n67: networks:\n71:networks:\n94: networks:\n98:networks:\n180: raise SystemExit(\"api.server block not found\")\n201:echo \"PEERS_RELAY_MAIL_EDGE\"\n209: for peer in data.get(\"peers\",{}).get(\"details\",[]):\n210: fqdn=peer.get(\"fqdn\",\"\")\n211: ip=peer.get(\"netbirdIp\",\"\")\n212: if ip in {\"[IP]\",\"[IP]\",\"[IP]\"} or any(x in fqdn for x in [\"relay\",\"mail\",\"edge-vm\"]):\n213: print(\"PEER\", fqdn, ip, \"status=\"+str(peer.get(\"status\")), \"handshake=\"+str(peer.get(\"lastWireguardHandshake\")), \"conn=\"+str(peer.get(\"connectionType\")))\n216:echo \"RELAY_MAIL_CONNECTIVITY\"\n219: ip route get \"$ip\" 2>&1 || true\n224:echo \"WG_ALLOWED_IPS_RELAY_MAIL\"\n226: wg show wt0 allowed-ips 2>/dev/null | sed -E 's/^[A-Za-z0-9+\\/=]{20,}/PEERKEY_REDACTED/' | grep -E '100\\.100\\.19\\.1|100\\.100\\.147\\.204|0\\.0\\.0\\.0/0|100\\.100\\.0\\.0/16' || echo [TOKEN_OR_KEY]\n229:echo \"[TOKEN_OR_KEY]\"\n239: wait_lapi \"CAPI_ROUTE_NOT_READY_SAFE\" || true\n240: write_health \"[TOKEN_OR_KEY]\" \"After manual NetBird setup, edge still cannot reach api.crowdsec.net; CrowdSec remains LAPI-only no-capi\"\n241: echo \"[TOKEN_OR_KEY]\"\n243: exit 70\n246:echo \"[TOKEN_OR_KEY]\"\n247:cd \"$STACK\" || exit 71\n248:mkdir -p \"manual-backups/771z-$TS\"\n249:test -f config/config.yaml && cp -a config/config.yaml \"manual-backups/771z-$TS/config.yaml.before\" || true\n250:test -f config/user.yaml && cp -a config/user.yaml \"manual-backups/771z-$TS/user.yaml.before\" || true\n251:test -f [TOKEN_OR_KEY].yaml && mv [TOKEN_OR_KEY].yaml \"manual-backups/771z-$TS/online_api_credentials.yaml.before\" || true\n257: write_health \"[TOKEN_OR_KEY]\" \"Direct CAPI route works, but compose CAPI enable failed\"\n258: exit 72\n262:wait_lapi \"REGISTER_MODE\" || exit 73\n284: write_health \"[TOKEN_OR_KEY]\" \"Direct CAPI route works, but cscli capi register failed; restored no-capi\"\n287: exit 74\n290:stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' [TOKEN_OR_KEY].yaml || true\n294:enable_capi_compose || exit 75\n315: write_health \"[TOKEN_OR_KEY]\" \"CrowdSec CAPI registered and stable after manual NetBird egress via relay/mail\"\n318: exit 0\n326:exit 76\n329: echo \"POST_ROUTE_SMOKE\"\n337: echo [TOKEN_OR_KEY]\nFILE_END=771z_after_manual_netbird_egress_capi.sh\nAUTHORITY_CANONICAL_READY=true\nDECISION=PASS_USA_MIGRATION_AUTHORITY_COLLECTED\nNEXT_GATE=BUILD_USA_EGRESS_AND_LOCAL_STACK_REPAIR\nTASK_RESULT=PASS_USA_MIGRATION_AUTHORITY_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1.txt b/runtime/history/SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1.txt new file mode 100644 index 00000000..a7ed8603 --- /dev/null +++ b/runtime/history/SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1.txt @@ -0,0 +1,106 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=skladchik-usa-migration-authority-miniread +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=50e4c7308ff3d4c751750cc63eee7536aa985f884c94b84ffcc1180c5eb60b05 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=f9810e721990dfc12c47863f23a39ad78582eaaf2cf212ac359cfbff17121390 +SANITIZED_OUTPUT_SHA256=f9810e721990dfc12c47863f23a39ad78582eaaf2cf212ac359cfbff17121390 +OUTPUT_BEGIN +RCA_VERSION=73276_V1 +MODE=read-only +MIGRATION_DOC_EXISTS=true +MIGRATION_DOC_SHA256=12fe079849346352315aac76fae82d00cbd17f88a3553270ca2c5f2749840552 +RUNBOOK_EXISTS=true +RUNBOOK_SHA256=725e2068337b379ebdf47822d5871aa5f4d5ec5956546674b2cf55b3250fb7a3 +MIGRATION_DOC_SHA_MATCH=true +RUNBOOK_SHA_MATCH=true +FILE_BEGIN=20_NETBIRD_USA_MIGRATION_2026-08-18.md +1:# NETBIRD USA MIGRATION — FINAL CURRENT RECORD +5:Old USA mail/NetBird VPS: +11:Host-level NetBird client left running for rollback/peer observation. +54:- [TOKEN_OR_KEY] startup +56:- Moldova synthetic check passes +57:- edge peer works +61:- `:9000/health` 503 in combined relay/server mode +70:- narrow TrustedPeers +71:- fix Traefik Compose network warning +FILE_END=20_NETBIRD_USA_MIGRATION_2026-08-18.md +FILE_BEGIN=04_NETBIRD_USA.md +1:# Runbook: NetBird USA +5:Do not use `/api/health` or combined `:9000/health` as generic readiness gates for this exact deployment; they were 404/503 and caused a false rollback. Use repeated Dashboard 200 + OIDC 200 + expected listeners + real client/peer connectivity. +14:7. retain migration key only until old USA is no longer needed. +FILE_END=04_NETBIRD_USA.md +FILE_BEGIN=771z_after_manual_netbird_egress_capi.sh +3:if ! test -s [TOKEN_OR_KEY].md || ! test -s [TOKEN_OR_KEY].md; then echo [TOKEN_OR_KEY]; exit 1; fi +8:PROOF="[TOKEN_OR_KEY]${TS}_PROOF.txt" +14: echo "[TOKEN_OR_KEY]" +17: echo "[TOKEN_OR_KEY]" +18: echo "TARGETS=edge-vm:[IP] relay-moldova:[IP] mail-usa:[IP]" +67: networks: +71:networks: +94: networks: +98:networks: +180: raise SystemExit("api.server block not found") +201:echo "PEERS_RELAY_MAIL_EDGE" +209: for peer in data.get("peers",{}).get("details",[]): +210: fqdn=peer.get("fqdn","") +211: ip=peer.get("netbirdIp","") +212: if ip in {"[IP]","[IP]","[IP]"} or any(x in fqdn for x in ["relay","mail","edge-vm"]): +213: print("PEER", fqdn, ip, "status="+str(peer.get("status")), "handshake="+str(peer.get("lastWireguardHandshake")), "conn="+str(peer.get("connectionType"))) +216:echo "RELAY_MAIL_CONNECTIVITY" +219: ip route get "$ip" 2>&1 || true +224:echo "WG_ALLOWED_IPS_RELAY_MAIL" +226: wg show wt0 allowed-ips 2>/dev/null | sed -E 's/^[A-Za-z0-9+\/=]{20,}/PEERKEY_REDACTED/' | grep -E '100\.100\.19\.1|100\.100\.147\.204|0\.0\.0\.0/0|100\.100\.0\.0/16' || echo [TOKEN_OR_KEY] +229:echo "[TOKEN_OR_KEY]" +239: wait_lapi "CAPI_ROUTE_NOT_READY_SAFE" || true +240: write_health "[TOKEN_OR_KEY]" "After manual NetBird setup, edge still cannot reach api.crowdsec.net; CrowdSec remains LAPI-only no-capi" +241: echo "[TOKEN_OR_KEY]" +243: exit 70 +246:echo "[TOKEN_OR_KEY]" +247:cd "$STACK" || exit 71 +248:mkdir -p "manual-backups/771z-$TS" +249:test -f config/config.yaml && cp -a config/config.yaml "manual-backups/771z-$TS/config.yaml.before" || true +250:test -f config/user.yaml && cp -a config/user.yaml "manual-backups/771z-$TS/user.yaml.before" || true +251:test -f [TOKEN_OR_KEY].yaml && mv [TOKEN_OR_KEY].yaml "manual-backups/771z-$TS/online_api_credentials.yaml.before" || true +257: write_health "[TOKEN_OR_KEY]" "Direct CAPI route works, but compose CAPI enable failed" +258: exit 72 +262:wait_lapi "REGISTER_MODE" || exit 73 +284: write_health "[TOKEN_OR_KEY]" "Direct CAPI route works, but cscli capi register failed; restored no-capi" +287: exit 74 +290:stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' [TOKEN_OR_KEY].yaml || true +294:enable_capi_compose || exit 75 +315: write_health "[TOKEN_OR_KEY]" "CrowdSec CAPI registered and stable after manual NetBird egress via relay/mail" +318: exit 0 +326:exit 76 +329: echo "POST_ROUTE_SMOKE" +337: echo [TOKEN_OR_KEY] +FILE_END=771z_after_manual_netbird_egress_capi.sh +AUTHORITY_CANONICAL_READY=true +DECISION=PASS_USA_MIGRATION_AUTHORITY_COLLECTED +NEXT_GATE=BUILD_USA_EGRESS_AND_LOCAL_STACK_REPAIR +TASK_RESULT=PASS_USA_MIGRATION_AUTHORITY_RCA +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index d5937f87..0d889b61 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260925-BEGET-DNS-FULL-APEX-SEAL-9229R1", + "command_id": "SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "beget-dns-d02-reconcile", - "started_at_utc": "2026-09-25T18:23:30Z", - "finished_at_utc": "2026-09-25T18:23:34Z", + "component": "skladchik-usa-migration-authority-miniread", + "started_at_utc": "2026-09-25T18:31:29Z", + "finished_at_utc": "2026-09-25T18:31:29Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "347741fe7efdfe04d15bc78278dd532d9ec4fee834bdada4a81a9efcfc621923", + "command_sha256": "50e4c7308ff3d4c751750cc63eee7536aa985f884c94b84ffcc1180c5eb60b05", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "f3a17098a4c883e192b9b1e8c2d3d1425c03d1a68dde3692bf21c0a783dd66db", - "sanitized_output_sha256": "f3a17098a4c883e192b9b1e8c2d3d1425c03d1a68dde3692bf21c0a783dd66db", + "raw_evidence_sha256": "f9810e721990dfc12c47863f23a39ad78582eaaf2cf212ac359cfbff17121390", + "sanitized_output_sha256": "f9810e721990dfc12c47863f23a39ad78582eaaf2cf212ac359cfbff17121390", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "DNS9229_BEGIN=true\nAUTH_PRESTATE_RECORD_TOTAL=49\nPUBLIC_NS_ERROR_COUNT=0\nPUBLIC_NS_DISAGREEMENT_COUNT=0\nPUBLIC_NS_STATE_SHA256=b79d69d4a96265c31b26d4bb1ea42a9c14aa8b98eb9fa561600354359d791bce\nAUTH_RESOLUTION_EMPTY_COUNT=0\nAUTH_RESOLUTION_PARTIAL_COUNT=0\nAUTH_RESOLUTION_DISAGREEMENT_COUNT=0\nAUTH_ENDPOINT_MAP_SHA256=92f1d82025fb93cef34454acfb694a9b959ecb434423dbad2642142cbe07e26c\nAUTH_ENDPOINT_COUNT=4\nSNAPSHOT1_ENDPOINT_COUNT=4\nSNAPSHOT1_QUERY_ERROR_COUNT=0\nSNAPSHOT1_CONSENSUS_ERROR_COUNT=0\nSNAPSHOT1_BASELINE_OWNER_COUNT=12\nSNAPSHOT1_BASELINE_OWNER_SHA256=375317c1f25a2452a2abb316efdd06c67102cf473086a28534ca321aab27b9d8\nSNAPSHOT1_MISMATCH_OWNER_COUNT=0\nSNAPSHOT1_MISMATCH_OWNER_SHA256=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945\nSNAPSHOT2_ENDPOINT_COUNT=4\nSNAPSHOT2_QUERY_ERROR_COUNT=0\nSNAPSHOT2_CONSENSUS_ERROR_COUNT=0\nSNAPSHOT2_BASELINE_OWNER_COUNT=12\nSNAPSHOT2_BASELINE_OWNER_SHA256=375317c1f25a2452a2abb316efdd06c67102cf473086a28534ca321aab27b9d8\nSNAPSHOT2_MISMATCH_OWNER_COUNT=0\nSNAPSHOT2_MISMATCH_OWNER_SHA256=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945\nSNAPSHOT_STABLE=true\nTASK_COMPLETE=true\nTASK_RESULT=PASS_9229_ALL12_APEX_AUTHORITATIVE_SEALED\nDECISION=NEXT_SHORT_NORMAL_OWNER_GHOST_AUDIT\nDNS_MUTATION_BY_9229=0\nPUBLIC_DNS_CHANGED=0\nPUBLIC_NS_CHANGED=0\nPVEPRO_MUTATED=0\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260925-BEGET-DNS-FULL-APEX-SEAL-9229R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nDNS9229_END=true\n" + "output": "RCA_VERSION=73276_V1\nMODE=read-only\nMIGRATION_DOC_EXISTS=true\nMIGRATION_DOC_SHA256=12fe079849346352315aac76fae82d00cbd17f88a3553270ca2c5f2749840552\nRUNBOOK_EXISTS=true\nRUNBOOK_SHA256=725e2068337b379ebdf47822d5871aa5f4d5ec5956546674b2cf55b3250fb7a3\nMIGRATION_DOC_SHA_MATCH=true\nRUNBOOK_SHA_MATCH=true\nFILE_BEGIN=20_NETBIRD_USA_MIGRATION_2026-08-18.md\n1:# NETBIRD USA MIGRATION — FINAL CURRENT RECORD\n5:Old USA mail/NetBird VPS:\n11:Host-level NetBird client left running for rollback/peer observation.\n54:- [TOKEN_OR_KEY] startup\n56:- Moldova synthetic check passes\n57:- edge peer works\n61:- `:9000/health` 503 in combined relay/server mode\n70:- narrow TrustedPeers\n71:- fix Traefik Compose network warning\nFILE_END=20_NETBIRD_USA_MIGRATION_2026-08-18.md\nFILE_BEGIN=04_NETBIRD_USA.md\n1:# Runbook: NetBird USA\n5:Do not use `/api/health` or combined `:9000/health` as generic readiness gates for this exact deployment; they were 404/503 and caused a false rollback. Use repeated Dashboard 200 + OIDC 200 + expected listeners + real client/peer connectivity.\n14:7. retain migration key only until old USA is no longer needed.\nFILE_END=04_NETBIRD_USA.md\nFILE_BEGIN=771z_after_manual_netbird_egress_capi.sh\n3:if ! test -s [TOKEN_OR_KEY].md || ! test -s [TOKEN_OR_KEY].md; then echo [TOKEN_OR_KEY]; exit 1; fi\n8:PROOF=\"[TOKEN_OR_KEY]${TS}_PROOF.txt\"\n14: echo \"[TOKEN_OR_KEY]\"\n17: echo \"[TOKEN_OR_KEY]\"\n18: echo \"TARGETS=edge-vm:[IP] relay-moldova:[IP] mail-usa:[IP]\"\n67: networks:\n71:networks:\n94: networks:\n98:networks:\n180: raise SystemExit(\"api.server block not found\")\n201:echo \"PEERS_RELAY_MAIL_EDGE\"\n209: for peer in data.get(\"peers\",{}).get(\"details\",[]):\n210: fqdn=peer.get(\"fqdn\",\"\")\n211: ip=peer.get(\"netbirdIp\",\"\")\n212: if ip in {\"[IP]\",\"[IP]\",\"[IP]\"} or any(x in fqdn for x in [\"relay\",\"mail\",\"edge-vm\"]):\n213: print(\"PEER\", fqdn, ip, \"status=\"+str(peer.get(\"status\")), \"handshake=\"+str(peer.get(\"lastWireguardHandshake\")), \"conn=\"+str(peer.get(\"connectionType\")))\n216:echo \"RELAY_MAIL_CONNECTIVITY\"\n219: ip route get \"$ip\" 2>&1 || true\n224:echo \"WG_ALLOWED_IPS_RELAY_MAIL\"\n226: wg show wt0 allowed-ips 2>/dev/null | sed -E 's/^[A-Za-z0-9+\\/=]{20,}/PEERKEY_REDACTED/' | grep -E '100\\.100\\.19\\.1|100\\.100\\.147\\.204|0\\.0\\.0\\.0/0|100\\.100\\.0\\.0/16' || echo [TOKEN_OR_KEY]\n229:echo \"[TOKEN_OR_KEY]\"\n239: wait_lapi \"CAPI_ROUTE_NOT_READY_SAFE\" || true\n240: write_health \"[TOKEN_OR_KEY]\" \"After manual NetBird setup, edge still cannot reach api.crowdsec.net; CrowdSec remains LAPI-only no-capi\"\n241: echo \"[TOKEN_OR_KEY]\"\n243: exit 70\n246:echo \"[TOKEN_OR_KEY]\"\n247:cd \"$STACK\" || exit 71\n248:mkdir -p \"manual-backups/771z-$TS\"\n249:test -f config/config.yaml && cp -a config/config.yaml \"manual-backups/771z-$TS/config.yaml.before\" || true\n250:test -f config/user.yaml && cp -a config/user.yaml \"manual-backups/771z-$TS/user.yaml.before\" || true\n251:test -f [TOKEN_OR_KEY].yaml && mv [TOKEN_OR_KEY].yaml \"manual-backups/771z-$TS/online_api_credentials.yaml.before\" || true\n257: write_health \"[TOKEN_OR_KEY]\" \"Direct CAPI route works, but compose CAPI enable failed\"\n258: exit 72\n262:wait_lapi \"REGISTER_MODE\" || exit 73\n284: write_health \"[TOKEN_OR_KEY]\" \"Direct CAPI route works, but cscli capi register failed; restored no-capi\"\n287: exit 74\n290:stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' [TOKEN_OR_KEY].yaml || true\n294:enable_capi_compose || exit 75\n315: write_health \"[TOKEN_OR_KEY]\" \"CrowdSec CAPI registered and stable after manual NetBird egress via relay/mail\"\n318: exit 0\n326:exit 76\n329: echo \"POST_ROUTE_SMOKE\"\n337: echo [TOKEN_OR_KEY]\nFILE_END=771z_after_manual_netbird_egress_capi.sh\nAUTHORITY_CANONICAL_READY=true\nDECISION=PASS_USA_MIGRATION_AUTHORITY_COLLECTED\nNEXT_GATE=BUILD_USA_EGRESS_AND_LOCAL_STACK_REPAIR\nTASK_RESULT=PASS_USA_MIGRATION_AUTHORITY_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 26609926..a7ed8603 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260925-BEGET-DNS-FULL-APEX-SEAL-9229R1 +COMMAND_ID=SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=beget-dns-d02-reconcile +COMPONENT=skladchik-usa-migration-authority-miniread REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=347741fe7efdfe04d15bc78278dd532d9ec4fee834bdada4a81a9efcfc621923 +COMMAND_SHA256=50e4c7308ff3d4c751750cc63eee7536aa985f884c94b84ffcc1180c5eb60b05 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,43 +24,83 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=f3a17098a4c883e192b9b1e8c2d3d1425c03d1a68dde3692bf21c0a783dd66db -SANITIZED_OUTPUT_SHA256=f3a17098a4c883e192b9b1e8c2d3d1425c03d1a68dde3692bf21c0a783dd66db +RAW_EVIDENCE_SHA256=f9810e721990dfc12c47863f23a39ad78582eaaf2cf212ac359cfbff17121390 +SANITIZED_OUTPUT_SHA256=f9810e721990dfc12c47863f23a39ad78582eaaf2cf212ac359cfbff17121390 OUTPUT_BEGIN -DNS9229_BEGIN=true -AUTH_PRESTATE_RECORD_TOTAL=49 -PUBLIC_NS_ERROR_COUNT=0 -PUBLIC_NS_DISAGREEMENT_COUNT=0 -PUBLIC_NS_STATE_SHA256=b79d69d4a96265c31b26d4bb1ea42a9c14aa8b98eb9fa561600354359d791bce -AUTH_RESOLUTION_EMPTY_COUNT=0 -AUTH_RESOLUTION_PARTIAL_COUNT=0 -AUTH_RESOLUTION_DISAGREEMENT_COUNT=0 -AUTH_ENDPOINT_MAP_SHA256=92f1d82025fb93cef34454acfb694a9b959ecb434423dbad2642142cbe07e26c -AUTH_ENDPOINT_COUNT=4 -SNAPSHOT1_ENDPOINT_COUNT=4 -SNAPSHOT1_QUERY_ERROR_COUNT=0 -SNAPSHOT1_CONSENSUS_ERROR_COUNT=0 -SNAPSHOT1_BASELINE_OWNER_COUNT=12 -SNAPSHOT1_BASELINE_OWNER_SHA256=375317c1f25a2452a2abb316efdd06c67102cf473086a28534ca321aab27b9d8 -SNAPSHOT1_MISMATCH_OWNER_COUNT=0 -SNAPSHOT1_MISMATCH_OWNER_SHA256=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 -SNAPSHOT2_ENDPOINT_COUNT=4 -SNAPSHOT2_QUERY_ERROR_COUNT=0 -SNAPSHOT2_CONSENSUS_ERROR_COUNT=0 -SNAPSHOT2_BASELINE_OWNER_COUNT=12 -SNAPSHOT2_BASELINE_OWNER_SHA256=375317c1f25a2452a2abb316efdd06c67102cf473086a28534ca321aab27b9d8 -SNAPSHOT2_MISMATCH_OWNER_COUNT=0 -SNAPSHOT2_MISMATCH_OWNER_SHA256=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 -SNAPSHOT_STABLE=true -TASK_COMPLETE=true -TASK_RESULT=PASS_9229_ALL12_APEX_AUTHORITATIVE_SEALED -DECISION=NEXT_SHORT_NORMAL_OWNER_GHOST_AUDIT -DNS_MUTATION_BY_9229=0 -PUBLIC_DNS_CHANGED=0 -PUBLIC_NS_CHANGED=0 -PVEPRO_MUTATED=0 -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-BEGET-DNS-FULL-APEX-SEAL-9229R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} -DNS9229_END=true +RCA_VERSION=73276_V1 +MODE=read-only +MIGRATION_DOC_EXISTS=true +MIGRATION_DOC_SHA256=12fe079849346352315aac76fae82d00cbd17f88a3553270ca2c5f2749840552 +RUNBOOK_EXISTS=true +RUNBOOK_SHA256=725e2068337b379ebdf47822d5871aa5f4d5ec5956546674b2cf55b3250fb7a3 +MIGRATION_DOC_SHA_MATCH=true +RUNBOOK_SHA_MATCH=true +FILE_BEGIN=20_NETBIRD_USA_MIGRATION_2026-08-18.md +1:# NETBIRD USA MIGRATION — FINAL CURRENT RECORD +5:Old USA mail/NetBird VPS: +11:Host-level NetBird client left running for rollback/peer observation. +54:- [TOKEN_OR_KEY] startup +56:- Moldova synthetic check passes +57:- edge peer works +61:- `:9000/health` 503 in combined relay/server mode +70:- narrow TrustedPeers +71:- fix Traefik Compose network warning +FILE_END=20_NETBIRD_USA_MIGRATION_2026-08-18.md +FILE_BEGIN=04_NETBIRD_USA.md +1:# Runbook: NetBird USA +5:Do not use `/api/health` or combined `:9000/health` as generic readiness gates for this exact deployment; they were 404/503 and caused a false rollback. Use repeated Dashboard 200 + OIDC 200 + expected listeners + real client/peer connectivity. +14:7. retain migration key only until old USA is no longer needed. +FILE_END=04_NETBIRD_USA.md +FILE_BEGIN=771z_after_manual_netbird_egress_capi.sh +3:if ! test -s [TOKEN_OR_KEY].md || ! test -s [TOKEN_OR_KEY].md; then echo [TOKEN_OR_KEY]; exit 1; fi +8:PROOF="[TOKEN_OR_KEY]${TS}_PROOF.txt" +14: echo "[TOKEN_OR_KEY]" +17: echo "[TOKEN_OR_KEY]" +18: echo "TARGETS=edge-vm:[IP] relay-moldova:[IP] mail-usa:[IP]" +67: networks: +71:networks: +94: networks: +98:networks: +180: raise SystemExit("api.server block not found") +201:echo "PEERS_RELAY_MAIL_EDGE" +209: for peer in data.get("peers",{}).get("details",[]): +210: fqdn=peer.get("fqdn","") +211: ip=peer.get("netbirdIp","") +212: if ip in {"[IP]","[IP]","[IP]"} or any(x in fqdn for x in ["relay","mail","edge-vm"]): +213: print("PEER", fqdn, ip, "status="+str(peer.get("status")), "handshake="+str(peer.get("lastWireguardHandshake")), "conn="+str(peer.get("connectionType"))) +216:echo "RELAY_MAIL_CONNECTIVITY" +219: ip route get "$ip" 2>&1 || true +224:echo "WG_ALLOWED_IPS_RELAY_MAIL" +226: wg show wt0 allowed-ips 2>/dev/null | sed -E 's/^[A-Za-z0-9+\/=]{20,}/PEERKEY_REDACTED/' | grep -E '100\.100\.19\.1|100\.100\.147\.204|0\.0\.0\.0/0|100\.100\.0\.0/16' || echo [TOKEN_OR_KEY] +229:echo "[TOKEN_OR_KEY]" +239: wait_lapi "CAPI_ROUTE_NOT_READY_SAFE" || true +240: write_health "[TOKEN_OR_KEY]" "After manual NetBird setup, edge still cannot reach api.crowdsec.net; CrowdSec remains LAPI-only no-capi" +241: echo "[TOKEN_OR_KEY]" +243: exit 70 +246:echo "[TOKEN_OR_KEY]" +247:cd "$STACK" || exit 71 +248:mkdir -p "manual-backups/771z-$TS" +249:test -f config/config.yaml && cp -a config/config.yaml "manual-backups/771z-$TS/config.yaml.before" || true +250:test -f config/user.yaml && cp -a config/user.yaml "manual-backups/771z-$TS/user.yaml.before" || true +251:test -f [TOKEN_OR_KEY].yaml && mv [TOKEN_OR_KEY].yaml "manual-backups/771z-$TS/online_api_credentials.yaml.before" || true +257: write_health "[TOKEN_OR_KEY]" "Direct CAPI route works, but compose CAPI enable failed" +258: exit 72 +262:wait_lapi "REGISTER_MODE" || exit 73 +284: write_health "[TOKEN_OR_KEY]" "Direct CAPI route works, but cscli capi register failed; restored no-capi" +287: exit 74 +290:stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' [TOKEN_OR_KEY].yaml || true +294:enable_capi_compose || exit 75 +315: write_health "[TOKEN_OR_KEY]" "CrowdSec CAPI registered and stable after manual NetBird egress via relay/mail" +318: exit 0 +326:exit 76 +329: echo "POST_ROUTE_SMOKE" +337: echo [TOKEN_OR_KEY] +FILE_END=771z_after_manual_netbird_egress_capi.sh +AUTHORITY_CANONICAL_READY=true +DECISION=PASS_USA_MIGRATION_AUTHORITY_COLLECTED +NEXT_GATE=BUILD_USA_EGRESS_AND_LOCAL_STACK_REPAIR +TASK_RESULT=PASS_USA_MIGRATION_AUTHORITY_RCA +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-SKLADCHIK-USA-MIGRATION-AUTHORITY-MINIREAD-73276R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END