runtime: publish SUPPORT-260922-HOMELAB-BACKUP-MOLDOVA-CONTROL-RCA-1123R1
This commit is contained in:
homelab-runtime-publisher
File diff suppressed because one or more lines are too long
@@ -0,0 +1,998 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-MOLDOVA-CONTROL-RCA-1123R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-moldova-control-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=0988f0ccf22795188f482b9de7d6e5bbd0042f96af78f459db72983c90a6fc99
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=0
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=e6275a6d7a9af5aa0706e1fc384f1c4cd5d89bdba7691222cc0ad8bbf480bf25
|
||||
SANITIZED_OUTPUT_SHA256=e6275a6d7a9af5aa0706e1fc384f1c4cd5d89bdba7691222cc0ad8bbf480bf25
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_BACKUP_MOLDOVA_CONTROL_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-MOLDOVA-CONTROL-RCA-1123R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-moldova-control-rca
|
||||
MUTATION_BOUNDARY=NONE;STATIC_CONTROL_PATH_CLASSIFICATION_ONLY;NO_PROVIDER_CALL;NO_NETWORK_MUTATION;NO_TARGET_SSH;NO_BACKUP;NO_SYSTEMD_ACTION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
|
||||
AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
|
||||
LEGACY_SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7
|
||||
SCHEDULER_RC=0
|
||||
SELFTEST_REPLACEMENT486=PASS
|
||||
SCHEDULER_LIVE_SELECTION={"due_seconds":86400,"enabled":true,"logical_id":"xf-newfi"}
|
||||
TARGETS_CONFIG_SHA256=aa4a75455bbfe92afaf666e8d404b35c5b41e70bc014e770c9301865ee84e221
|
||||
{"logical_id":"us-netbird","type":"vps_us","scope":"vps","user":"root","port":22,"enabled":true,"due_seconds":86400}
|
||||
CONTROL_PATH_CLASSIFICATION_BEGIN=1
|
||||
{"bytes":20163,"executable":true,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771r_edge_netbird_egress_capi_20260702T165517Z.sh","sha256":"9c6f911768869c984ec41016b3134be75620100fac8c60a8b21ef6bde58c6868"}
|
||||
{"bytes":20006,"executable":true,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771s_edge_capi_netbird_20260702T170251Z.sh","sha256":"c4783c14a3a132616af5db6a065270ccd39a7690f9fd38d0ef27fe5de6bd07e5"}
|
||||
{"bytes":8977,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":true,"path":"/root/771p_netbird_vps_peer_repair.sh","sha256":"f37aac25cac5b1c3983392070ddaff09e28497efa28884b03d23efc09a825747"}
|
||||
{"bytes":6970,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":false,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771ad_final_crowdsec_capi_netbird_closure.sh","sha256":"c99d08385f5d7c0a266c1c8002b3c7b054e1bb561eb48b1f8a227f37fe430ab6"}
|
||||
{"bytes":28134,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771l_crowdsec_capi_netbird_vps_egress.sh","sha256":"401396a25d4a5cfa487f67b355b45b27140ed1ac6b49b41e342306cb618dfdb5"}
|
||||
{"bytes":24918,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771l_crowdsec_capi_netbird_vps_egress_remote_20260702T151045Z.sh","sha256":"6a6df3cc085363c2bbbb92b8c3083123fb0e583e14e242c583a321bdb5612002"}
|
||||
{"bytes":31306,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771m_purge_warp_fix_netbird_egress.sh","sha256":"0ae3b0e1e8016da55cf1756e868c51a23fe12328fae6ef4baa61c50b9e48e715"}
|
||||
{"bytes":27881,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771m_purge_warp_fix_netbird_remote_20260702T151637Z.sh","sha256":"941f683d148a01d9af9ef6ce938c8ee6874d7af1ac1ac5ee96c7d9ca7cc262a4"}
|
||||
{"bytes":12586,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771n_netbird_usa_moldova_egress_only.sh","sha256":"420d835318960ae2644dafc93e0427584505acaffa9786927623aeb59ba6ac0e"}
|
||||
{"bytes":10125,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771n_netbird_usa_moldova_egress_remote_20260702T152139Z.sh","sha256":"1c25cbc13f524d0f6974a71c255c5634c69380818cb1b5ad4f8f6ea9bf8c6a01"}
|
||||
{"bytes":7440,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":false,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771o_netbird_management_acl_discovery.sh","sha256":"50526afeaeb99fa9a8018374abe82731dcb9e93eac4be64934d0c6a8610be322"}
|
||||
{"bytes":25955,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771q_netbird_public_vps_repair_then_capi.sh","sha256":"0b6e42e5b4013f7ee2191b140c5c507877fb05e2a4fdd7b39bea7b7c6b108071"}
|
||||
{"bytes":26359,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771r_netbird_public_vps_fixed_then_capi.sh","sha256":"dc8119b6815d60e08f442e77faba6ce6e9899c4120d78490886e222b8510efa7"}
|
||||
{"bytes":28279,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771s_vps_identity_repair_netbird_capi.sh","sha256":"16121a810320b1517291830ba128baf6c68e7b3e6f78786481dedb78a1f061b9"}
|
||||
{"bytes":25344,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771t_temp_hostkey_netbird_identity_then_capi.sh","sha256":"0c3d628b23b67849d52112322ab2e1ee3779012c10578a0b8cece0d8ddf639da"}
|
||||
{"bytes":27834,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771u_find_key_fix_netbird_egress_capi.sh","sha256":"6b6511ec3d614793e3542777ccfcb5e8c5f09cff77ee2d57624b3ac980787bcb"}
|
||||
{"bytes":30154,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771v_deep_key_backup_netbird_capi.sh","sha256":"92191ce6124981ee0468cc8f95c749c47bca5fc61f9aecaeedeef91cdd434170"}
|
||||
{"bytes":13072,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/root/771z_after_manual_netbird_egress_capi.sh","sha256":"f198ed621fd726ed6f15c4a0dd49fad307befd01e7a58ebbda1ddc379f5b49ee"}
|
||||
{"bytes":9197,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/01_CURRENT_AUTHORITATIVE_STATE.md","sha256":"844a17f9af701211699b078f5a5e8ff28bb401b377acc10fe364aa78aa3bfc9b"}
|
||||
{"bytes":38895,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":false,"path":"/srv/homelab-ops/.git/index","sha256":"93bd029aaadbac051ea3e9c7266ae0d0643ec8d0eff24a02ccb44805dc9764d2"}
|
||||
{"bytes":38895,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":false,"path":"/srv/homelab-ops/.git/worktrees/homelab-ops-cr-2026-0095/index","sha256":"6327b174188ff5b751c03ba3e6b14d884a1d61dda42df86d8cf2e2151d74efe1"}
|
||||
{"bytes":53159,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":false,"path":"/srv/homelab-ops/.git/worktrees/homelab-ops-cr-2026-0096/index","sha256":"0f244e69df78feba15d92d1c45c7d456e62726609ff45a6000cc5f11cc6b8e64"}
|
||||
{"bytes":75082,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":false,"path":"/srv/homelab-ops/.git/worktrees/homelab-ops-cr-2026-1005/index","sha256":"d7c61f503b6b7494e9aa4dde883d5a42493d81c346ba30938fe86386002dc388"}
|
||||
{"bytes":31037,"executable":false,"has_http":true,"has_netbird":false,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/docs/superpowers/plans/2026-08-18-gitea-edge-publisher-v3.md","sha256":"f42b0e0be98c7f6d9f0e06d5565ec1cfc277e4221cbc164f21c157fe4121ad72"}
|
||||
{"bytes":3587,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":false,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/AI_CONTEXT.md","sha256":"5fc43e34a1f0714ddacf7c00ed5c4bbaf788df3b8407dcfebd1333a23a623acc"}
|
||||
{"bytes":1322,"executable":false,"has_http":false,"has_netbird":false,"has_power_action":false,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":false,"path":"/srv/homelab-ops/kb/current/03_ACCESS_AND_SECRETS.md","sha256":"e750284217dc41b5809268a0bbc54ed0948bdd14680fe2f282f307640937496c"}
|
||||
{"bytes":805,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/current/04_P0_NEXT.md","sha256":"0f6eb8b23382becc1868e8a22318d5b7629184568205bad47c3c38967a346b8f"}
|
||||
{"bytes":3063,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":false,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/README_FIRST.md","sha256":"02f7db5a75d4371a430a0034dc3e74a61ea355064fe44a1a221e8e175422a008"}
|
||||
{"bytes":33401,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":false,"has_provider_hint":true,"has_secret_reference":false,"has_ssh":false,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/SHA256SUMS","sha256":"0939f8b9ab4de481c6e68a6abd16cb14ae5348a94e3af0d0e3f107815d0d3169"}
|
||||
{"bytes":2726,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/00_START_HERE.md","sha256":"4c4e4c85805aa00bd5f94f64fc9c2c985c44bf467058546e3cbe9d8d166d044f"}
|
||||
{"bytes":3259,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":false,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/04_NETWORK_DNS_INGRESS.md","sha256":"e3edd29ea8754ce340929f531c0ff8c1ebda4fbbf7d20a53445a665e014fbc85"}
|
||||
{"bytes":3828,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/11_OPEN_ISSUES_AND_FUTURE_WORK.md","sha256":"8621c33341c93f6104e58821daeda1e534353290c685a273620b884014ddd647"}
|
||||
{"bytes":2056,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":false,"has_ssh":false,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/12_NEW_CHAT_FIRST_MESSAGE.txt","sha256":"380315d5793449c258356145e7cff7dee7493d2c462dff3ca6f6d9eabfb673e0"}
|
||||
{"bytes":1744,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":false,"has_provider_hint":true,"has_secret_reference":false,"has_ssh":false,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/13_PROVENANCE_AND_FRESHNESS.md","sha256":"804a09f8ae5fe307c3fb96243246ef4817bb98695068a80f181978933ee70089"}
|
||||
{"bytes":1740,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":false,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":false,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/15_CRITICAL_KEEP_RETIRE_DECISIONS.md","sha256":"ad3ea70345063e8a13608a83acb4d6a1118533eb6acab22929e0bf1ac20c06af"}
|
||||
{"bytes":3638,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/18_P0_CLOSEOUT_MASTER_PLAN_2026-08-19.md","sha256":"0ce52a34278e57ff3dba122e2e65c57251e75484025ba13588971d0979d78ec6"}
|
||||
{"bytes":1792,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":false,"has_provider_hint":true,"has_secret_reference":false,"has_ssh":false,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/20_NETBIRD_USA_MIGRATION_2026-08-18.md","sha256":"12fe079849346352315aac76fae82d00cbd17f88a3553270ca2c5f2749840552"}
|
||||
{"bytes":2265,"executable":false,"has_http":true,"has_netbird":true,"has_power_action":false,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":false,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/22_MOLDOVA_HEALTHCHECK_GOTIFY_2026-08-18.md","sha256":"fe0f5ca63837626583d150e8c4b9ce9c795832ae4f59af25369a00fa7825fdb5"}
|
||||
{"bytes":3828,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":true,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/24_GLOBAL_BACKLOG_MASTER_2026-08-19.md","sha256":"8621c33341c93f6104e58821daeda1e534353290c685a273620b884014ddd647"}
|
||||
{"bytes":1733,"executable":false,"has_http":false,"has_netbird":false,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/25_OPERATOR_CHAT_RULES_2026-08-19.md","sha256":"69bec1f0893c32c2ea67100355f8702f20bb6671177c6c1da69e1d2407c71232"}
|
||||
{"bytes":1744,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":false,"has_provider_hint":true,"has_secret_reference":false,"has_ssh":false,"path":"/srv/homelab-ops/kb/private/archive/2026-08-19/docs/28_PROVENANCE_AND_FRESHNESS_2026-08-19.md","sha256":"804a09f8ae5fe307c3fb96243246ef4817bb98695068a80f181978933ee70089"}
|
||||
{"bytes":700,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":false,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/runbooks/04_NETBIRD_USA.md","sha256":"725e2068337b379ebdf47822d5871aa5f4d5ec5956546674b2cf55b3250fb7a3"}
|
||||
{"bytes":4057,"executable":false,"has_http":false,"has_netbird":true,"has_power_action":true,"has_provider_hint":false,"has_secret_reference":true,"has_ssh":true,"path":"/srv/homelab-ops/kb/scripts/command_guard.py","sha256":"33c59d92a2b9c380d9208a0e2fd1353029e6edc4878053df95dbd7b9dd987d0b"}
|
||||
CONTROL_PATH_CLASSIFICATION_END=1
|
||||
CONTROL_TOTAL_ROWS=43
|
||||
CONTROL_EXEC_ROWS=2
|
||||
CONTROL_EXTERNAL_PROVIDER_ACTION_CANDIDATES=0
|
||||
CONTROL_SSH_DEPENDENT_CANDIDATES=2
|
||||
CONTROL_NETBIRD_ONLY_CANDIDATES=0
|
||||
KNOWN_771_SAFE_STATIC_BEGIN=1
|
||||
KNOWN_SCRIPT=/root/771n_netbird_usa_moldova_egress_only.sh SHA256=420d835318960ae2644dafc93e0427584505acaffa9786927623aeb59ba6ac0e EXECUTABLE=false
|
||||
20:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
22:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
||||
41: echo "CHECK=crowdsec-capi"
|
||||
53: DISABLE_ONLINE_API: "true"
|
||||
54: ARGS: "-no-capi"
|
||||
93: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
||||
95:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
||||
102:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
||||
103:ExecStart=/bin/sh -lc 'exec /usr/bin/ssh -N -D "${SOCKS_BIND}:${SOCKS_PORT}" -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${SSH_HOST}"'
|
||||
104:Restart=always
|
||||
105:RestartSec=5
|
||||
111: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
||||
115:base=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
123:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
124:systemctl restart privoxy 2>/dev/null || true
|
||||
126:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
||||
128: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
||||
131: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
||||
132: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
134: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
||||
146: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
||||
149: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
||||
160:out.append("# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
||||
163:out.append("# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_END")
|
||||
168: systemctl restart privoxy
|
||||
189: if timeout 12 ssh -n -o BatchMode=yes -o ConnectTimeout=7 -o StrictHostKeyChecking=accept-new "$user@$ip" "echo SSH_OK; hostname; uname -a | cut -c1-120" 2>&1 | redact; then
|
||||
192: if ssh -n -fN -M -S "$ctl" -o BatchMode=yes -o ConnectTimeout=8 -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new -D "[PRIVATE_IP]:${port}" "$user@$ip" 2>/tmp/771n_tunnel_${profile}.err; then
|
||||
194: code="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771n_capi_${profile}.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
195: trace="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 https://www.cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -E 'ip=|colo=' | sed -n '1,5p' || true)"
|
||||
196: echo "SOCKS_CAPI_HTTP=$code"
|
||||
200: ssh -S "$ctl" -O exit "$user@$ip" >/dev/null 2>&1 || true
|
||||
232:docker exec crowdsec cscli lapi status 2>&1 | redact || true
|
||||
233:docker exec crowdsec cscli capi status 2>&1 | redact || true
|
||||
236:netbird status 2>&1 | redact || true
|
||||
239:systemctl restart netbird 2>/dev/null || true
|
||||
241:netbird status 2>&1 | redact || true
|
||||
244:USA_IP="$(awk '$1 ~ /^e3qxxx\.netbird\.selfhosted$/ {print $2}' < <(netbird status 2>/dev/null || true) | head -1)"
|
||||
245:MOLDOVA_IP="$(awk '$1 ~ /^e3qxxx-183-106\.netbird\.selfhosted$/ {print $2}' < <(netbird status 2>/dev/null || true) | head -1)"
|
||||
261: write_health "REVIEW_USA_MOLDOVA_NETBIRD_PEERS_NOT_REACHABLE" "WARP purged; USA/Moldova NetBird VPS peers are not reachable or do not allow SSH from edge"
|
||||
262: echo "NEEDED_ON_VPS=NetBird peer online, SSH reachable over NetBird, and outbound TLS to api.crowdsec.net working"
|
||||
276:proxy_code="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 -o /tmp/771n_active_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
277:echo "ACTIVE_HTTP_PROXY_CAPI_HTTP=$proxy_code"
|
||||
280: write_health "REVIEW_NETBIRD_EGRESS_PROXY_FAILED" "NetBird VPS SSH SOCKS profile exists but HTTP proxy did not reach CrowdSec CAPI"
|
||||
286:write_health "OK_NETBIRD_EGRESS_PROFILE_READY_CAPI_REGISTRATION_NEXT" "NetBird VPS egress profile=$profile is ready; next command can register CrowdSec CAPI through proxy http://${b}:${HTTP_PROXY_PORT}"
|
||||
287:echo "READY_PROXY=http://${b}:${HTTP_PROXY_PORT}"
|
||||
288:echo "READY_SWITCH=homelab-crowdsec-capi-egress-switch $profile"
|
||||
290:echo "REMOTE_STATUS=OK_NETBIRD_EGRESS_PROFILE_READY_CAPI_REGISTRATION_NEXT"
|
||||
306: scp -q "$LOCAL_REMOTE_SCRIPT" "debian@$EDGE:$REMOTE_SCRIPT"
|
||||
315: ssh "debian@$EDGE" "sudo bash '$REMOTE_SCRIPT' '$TS'"
|
||||
KNOWN_SCRIPT=/root/771o_netbird_management_acl_discovery.sh SHA256=50526afeaeb99fa9a8018374abe82731dcb9e93eac4be64934d0c6a8610be322 EXECUTABLE=false
|
||||
25: ssh debian@$EDGE "sudo bash -lc '
|
||||
35: netbird status --json >/tmp/771o_netbird_status.json 2>/tmp/771o_netbird_status_json.err || true
|
||||
109: ssh -o BatchMode=yes -o ConnectTimeout=8 "$user@$host" "sudo bash -lc '
|
||||
KNOWN_SCRIPT=/root/771p_netbird_vps_peer_repair.sh SHA256=f37aac25cac5b1c3983392070ddaff09e28497efa28884b03d23efc09a825747 EXECUTABLE=false
|
||||
19: echo "RULE=NO_WARP_FIX_NETBIRD_USA_MOLDOVA_PEERS_FOR_CROWDSEC_CAPI"
|
||||
23: echo "CONFIG_CHANGE=YES_RESTART_NETBIRD_ON_VPS_IF_PUBLIC_SSH_FOUND"
|
||||
36: ssh debian@$EDGE "sudo bash -lc '
|
||||
46: netbird status --json >/tmp/771p_edge_nb.json 2>/tmp/771p_edge_nb.err || true
|
||||
80: grep -nEi 'e3qxxx|183-106|moldova|молдов|usa|america|vps|netbird|alexhost|aeza|hetzner|public ip|external ip|ssh' /etc/pve/31_HOMELAB_REFERENCE.md 2>/dev/null | sed -n '1,260p' || true
|
||||
138: echo "TEST_PUBLIC_SSH_AND_RESTART_NETBIRD_ON_VPS_IF_FOUND"
|
||||
158: ssh debian@$EDGE "sudo bash -lc '
|
||||
159: systemctl restart netbird 2>/dev/null || true
|
||||
174: ssh debian@$EDGE "sudo bash -lc '
|
||||
182: echo "STATUS=OK_771P_VPS_PUBLIC_ACCESS_FOUND_RESTARTED_NETBIRD_REVIEW_RECHECK"
|
||||
KNOWN_SCRIPT=/root/771q_netbird_public_vps_repair_then_capi.sh SHA256=0b6e42e5b4013f7ee2191b140c5c507877fb05e2a4fdd7b39bea7b7c6b108071 EXECUTABLE=false
|
||||
8:PROOF="/root/evidence/771Q_NETBIRD_PUBLIC_VPS_REPAIR_THEN_CAPI_${TS}_PROOF.txt"
|
||||
16: echo "STEP=771Q_NETBIRD_PUBLIC_VPS_REPAIR_THEN_CAPI"
|
||||
19: echo "RULE=NO_WARP_REPAIR_NETBIRD_USA_MOLDOVA_THEN_CROWDSEC_CAPI"
|
||||
23: echo "CONFIG_CHANGE=YES_RESTART_PUBLIC_VPS_NETBIRD_AND_CAPI_IF_ROUTE_READY"
|
||||
33: if timeout 18 ssh -n \
|
||||
38: "echo SSH_OK; echo HOSTNAME=\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || echo NETBIRD_CLI_MISSING; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771q_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
||||
53: ssh debian@$EDGE "sudo bash -lc 'command -v warp-cli >/dev/null 2>&1 && echo WARP_CLI_STILL_PRESENT || echo WARP_CLI_ABSENT=YES; dpkg -l 2>/dev/null | grep -E \"^ii[[:space:]]+cloudflare-warp\" || echo CLOUDFLARE_WARP_PACKAGE_ABSENT=YES; ss -ltnp | grep -E \":(40000|40001)\\b\" || echo WARP_PROXY_PORTS_ABSENT=YES'"
|
||||
74: echo "RESTART_NETBIRD_ON_REAL_PUBLIC_VPS_ONLY"
|
||||
75: : >/tmp/771q_restarted.tsv
|
||||
80: echo "RESTART_ATTEMPT profile=$profile user=$user host=$host"
|
||||
81: out="/tmp/771q_restart_${profile}_${user}_$(echo "$host" | tr -c A-Za-z0-9 _).out"
|
||||
82: if timeout 45 ssh -n \
|
||||
87: "echo BEFORE_HOSTNAME=\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || true; (sudo systemctl restart netbird 2>/dev/null || systemctl restart netbird 2>/dev/null || true); sleep 15; echo AFTER; command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || true; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771q_capi_after.body -w 'DIRECT_CAPI_HTTP_AFTER=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
||||
91: printf '%s\t%s\t%s\n' "$profile" "$user" "$host" >>/tmp/771q_restarted.tsv
|
||||
94: echo "RESTART_FAILED profile=$profile user=$user host=$host"
|
||||
103: echo "PUBLIC_VPS_RESTARTED"
|
||||
104: cat /tmp/771q_restarted.tsv || true
|
||||
106: echo "RECHECK_EDGE_USA_MOLDOVA_AFTER_PUBLIC_RESTART"
|
||||
107: ssh debian@$EDGE "sudo bash -lc '
|
||||
109: systemctl restart netbird 2>/dev/null || true
|
||||
112: netbird status --json >/tmp/771q_edge_nb.json 2>/tmp/771q_edge_nb.err || true
|
||||
139: echo "EDGE_BUILD_NETBIRD_EGRESS_AND_REGISTER_CAPI_IF_REACHABLE"
|
||||
140: ssh debian@$EDGE "sudo bash -s" <<'EDGE_SCRIPT'
|
||||
146:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
148:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
||||
167: echo "CHECK=crowdsec-capi"
|
||||
173:force_no_capi_stable() {
|
||||
179: DISABLE_ONLINE_API: "true"
|
||||
180: ARGS: "-no-capi"
|
||||
201:wait_lapi() {
|
||||
203: echo "WAIT_LAPI=$label"
|
||||
206: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771q_health.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
208: rc="$(printf '%s\n' "$out" | awk -F= '/^LAPI_RC=/{print $2}' | tail -1)"
|
||||
209: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA}"
|
||||
271: raise SystemExit("api.server block not found")
|
||||
274: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
||||
310: grep -nE 'DISABLE_ONLINE_API|ARGS|GODEBUG|HTTP_PROXY|HTTPS_PROXY|NO_PROXY|socket-proxy|published:|target:' /tmp/771q_compose.yml || true
|
||||
311: ! grep -qE 'DISABLE_ONLINE_API|ARGS:.*no-capi|-no-capi' /tmp/771q_compose.yml
|
||||
321: if timeout 12 ssh -n -o BatchMode=yes -o ConnectTimeout=7 -o StrictHostKeyChecking=accept-new "$user@$ip" "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771q_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" 2>&1 | redact; then
|
||||
324: if ssh -n -fN -M -S "$ctl" -o BatchMode=yes -o ConnectTimeout=8 -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new -D "[PRIVATE_IP]:${port}" "$user@$ip" 2>/tmp/771q_tunnel.err; then
|
||||
326: code="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771q_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
327: trace="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 https://www.cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -E 'ip=|colo=' | sed -n '1,5p' || true)"
|
||||
328: echo "SOCKS_CAPI_HTTP=$code"
|
||||
332: ssh -S "$ctl" -O exit "$user@$ip" >/dev/null 2>&1 || true
|
||||
360: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
||||
362:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
||||
369:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
||||
370:ExecStart=/bin/sh -lc 'exec /usr/bin/ssh -N -D "${SOCKS_BIND}:${SOCKS_PORT}" -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${SSH_HOST}"'
|
||||
371:Restart=always
|
||||
372:RestartSec=5
|
||||
378: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
||||
382:base=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
390:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
391:systemctl restart privoxy 2>/dev/null || true
|
||||
393:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
||||
395: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
||||
398: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
||||
399: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
401: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
||||
415: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
||||
417: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
||||
427:out.append("# HOMELAB_771Q_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
||||
430:out.append("# HOMELAB_771Q_CROWDSEC_CAPI_NETBIRD_END")
|
||||
435: systemctl restart privoxy
|
||||
440: code="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 -o /tmp/771q_http_proxy_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
441: echo "HTTP_PROXY_CAPI_HTTP=$code"
|
||||
445:register_capi() {
|
||||
447: proxy_url="http://${b}:${HTTP_PROXY_PORT}"
|
||||
448: echo "REGISTER_CAPI proxy=$proxy_url"
|
||||
450: force_no_capi_stable
|
||||
451: wait_lapi "BEFORE_CAPI_REGISTER" || return 10
|
||||
455: if test -f config/online_api_credentials.yaml; then
|
||||
456: mkdir -p /opt/stacks/crowdsec/manual-backups/771q-old-online-creds-"$TS"
|
||||
457: mv config/online_api_credentials.yaml /opt/stacks/crowdsec/manual-backups/771q-old-online-creds-"$TS"/online_api_credentials.yaml.before-register
|
||||
462: wait_lapi "REGISTER_MODE" || return 13
|
||||
467: if cscli capi register --help 2>&1 | grep -q -- "-y"; then
|
||||
468: timeout 240 cscli capi register -y >/tmp/771q_register.out 2>&1
|
||||
470: timeout 240 sh -c "yes | cscli capi register" >/tmp/771q_register.out 2>&1
|
||||
483: if ! test -f config/online_api_credentials.yaml; then
|
||||
487: stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' config/online_api_credentials.yaml || true
|
||||
KNOWN_SCRIPT=/root/771r_edge_netbird_egress_capi_20260702T165517Z.sh SHA256=9c6f911768869c984ec41016b3134be75620100fac8c60a8b21ef6bde58c6868 EXECUTABLE=true
|
||||
7:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
9:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
||||
29: echo "CHECK=crowdsec-capi"
|
||||
35:force_no_capi_stable() {
|
||||
41: DISABLE_ONLINE_API: "true"
|
||||
42: ARGS: "-no-capi"
|
||||
65:wait_lapi() {
|
||||
68: echo "WAIT_LAPI=$label"
|
||||
71: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
73: rc="$(printf '%s\n' "$out" | awk -F= '/^LAPI_RC=/{print $2}' | tail -1)"
|
||||
74: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
||||
75: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
||||
149: raise SystemExit("api.server block not found")
|
||||
153: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
||||
192: grep -nE 'DISABLE_ONLINE_API|ARGS|GODEBUG|HTTP_PROXY|HTTPS_PROXY|NO_PROXY|socket-proxy|published:|target:' /tmp/771r_compose.yml || true
|
||||
193: ! grep -qE 'DISABLE_ONLINE_API|ARGS:.*no-capi|-no-capi' /tmp/771r_compose.yml
|
||||
207: timeout 12 ssh -n \
|
||||
212: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
||||
224: if ssh -n -fN -M -S "$ctl" \
|
||||
234: code="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771r_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
235: trace="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 https://www.cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -E 'ip=|colo=' | sed -n '1,5p' || true)"
|
||||
236: echo "SOCKS_CAPI_HTTP=$code"
|
||||
240: ssh -S "$ctl" -O exit "$user@$ip" >/dev/null 2>&1 || true
|
||||
273: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
||||
275:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
||||
282:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
||||
283:ExecStart=/bin/sh -lc 'exec /usr/bin/ssh -N -D "${SOCKS_BIND}:${SOCKS_PORT}" -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${SSH_HOST}"'
|
||||
284:Restart=always
|
||||
285:RestartSec=5
|
||||
291: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
||||
295:base=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
303:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
304:systemctl restart privoxy 2>/dev/null || true
|
||||
306:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
||||
308: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
||||
311: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
||||
312: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
314: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
||||
328: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
||||
330: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
||||
340:out.append("# HOMELAB_771R_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
||||
343:out.append("# HOMELAB_771R_CROWDSEC_CAPI_NETBIRD_END")
|
||||
348: systemctl restart privoxy
|
||||
353: code="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 -o /tmp/771r_http_proxy_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
354: echo "HTTP_PROXY_CAPI_HTTP=$code"
|
||||
358:register_capi() {
|
||||
359: local b proxy_url reg_out reg_rc ready n health lapi_rc capi_out capi_rc fatal envbad rc_before rc_after health_after lapi_after capi_after fatal_after env_after
|
||||
361: proxy_url="http://${b}:${HTTP_PROXY_PORT}"
|
||||
362: echo "REGISTER_CAPI proxy=$proxy_url"
|
||||
364: force_no_capi_stable
|
||||
365: wait_lapi "BEFORE_CAPI_REGISTER" || return 10
|
||||
369: if test -f config/online_api_credentials.yaml; then
|
||||
370: mkdir -p /opt/stacks/crowdsec/manual-backups/771r-old-online-creds-"$TS"
|
||||
371: mv config/online_api_credentials.yaml /opt/stacks/crowdsec/manual-backups/771r-old-online-creds-"$TS"/online_api_credentials.yaml.before-register
|
||||
376: wait_lapi "REGISTER_MODE" || return 13
|
||||
381: if cscli capi register --help 2>&1 | grep -q -- "-y"; then
|
||||
382: timeout 240 cscli capi register -y >/tmp/771r_register.out 2>&1
|
||||
384: timeout 240 sh -c "yes | cscli capi register" >/tmp/771r_register.out 2>&1
|
||||
397: if ! test -f config/online_api_credentials.yaml; then
|
||||
401: stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' config/online_api_credentials.yaml || true
|
||||
402: awk -F: '/^[A-Za-z0-9_ -]+:/ {gsub(/^[ \t]+|[ \t]+$/, "", $1); print $1 ": REDACTED"}' config/online_api_credentials.yaml | sed -n '1,40p'
|
||||
404: grep -Eq '^[[:space:]]*login:' config/online_api_credentials.yaml || return 22
|
||||
411: echo "VALIDATE_CAPI"
|
||||
415: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health2.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
416: lapi_rc="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771r_lapi2.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
418: capi_rc="$(printf '%s\n' "$capi_out" | awk -F= '/^CAPI_RC=/{print $2}' | tail -1)"
|
||||
419: fatal="$(docker logs --since 60s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
420: envbad="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
421: echo "VALIDATE_TRY=$n HEALTH=$health LAPI_RC=${lapi_rc:-NA} CAPI_RC=${capi_rc:-NA} FATAL=$fatal ENV_BAD=${envbad:-NONE}"
|
||||
422: printf '%s\n' "$capi_out"
|
||||
423: if test "$health" = "200" && test "${lapi_rc:-NA}" = "0" && test "${capi_rc:-NA}" = "0" && test "$fatal" = "0" && test -z "$envbad"; then
|
||||
430: rc_before="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
432: rc_after="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
433: health_after="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health_after.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
434: lapi_after="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771r_lapi_after.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
435: capi_after="$(docker exec crowdsec sh -lc 'cscli capi status >/tmp/771r_capi_after.out 2>&1; echo CAPI_RC=$?' 2>/dev/null | awk -F= '/^CAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
436: fatal_after="$(docker logs --since 240s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
437: env_after="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
439: echo "FINAL_STABILITY RC_BEFORE=$rc_before RC_AFTER=$rc_after HEALTH_AFTER=$health_after LAPI_AFTER=${lapi_after:-NA} CAPI_AFTER=${capi_after:-NA} FATAL_AFTER=$fatal_after ENV_BAD_AFTER=${env_after:-NONE}"
|
||||
441: test "$rc_before" = "$rc_after" && test "$health_after" = "200" && test "${lapi_after:-NA}" = "0" && test "${capi_after:-NA}" = "0" && test "$fatal_after" = "0" && test -z "$env_after"
|
||||
KNOWN_SCRIPT=/root/771r_netbird_public_vps_fixed_then_capi.sh SHA256=dc8119b6815d60e08f442e77faba6ce6e9899c4120d78490886e222b8510efa7 EXECUTABLE=false
|
||||
8:PROOF="/root/evidence/771R_NETBIRD_PUBLIC_VPS_FIXED_THEN_CAPI_${TS}_PROOF.txt"
|
||||
10:EDGE_REMOTE="/tmp/771r_edge_netbird_egress_capi_${TS}.sh"
|
||||
11:EDGE_LOCAL="/root/771r_edge_netbird_egress_capi_${TS}.sh"
|
||||
26: timeout 20 ssh -n \
|
||||
31: "echo SSH_OK; echo USER=\$(id -un); echo HOSTNAME=\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || echo NETBIRD_CLI_MISSING; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
||||
51:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
53:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
||||
73: echo "CHECK=crowdsec-capi"
|
||||
79:force_no_capi_stable() {
|
||||
85: DISABLE_ONLINE_API: "true"
|
||||
86: ARGS: "-no-capi"
|
||||
109:wait_lapi() {
|
||||
112: echo "WAIT_LAPI=$label"
|
||||
115: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
117: rc="$(printf '%s\n' "$out" | awk -F= '/^LAPI_RC=/{print $2}' | tail -1)"
|
||||
118: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
||||
119: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
||||
193: raise SystemExit("api.server block not found")
|
||||
197: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
||||
236: grep -nE 'DISABLE_ONLINE_API|ARGS|GODEBUG|HTTP_PROXY|HTTPS_PROXY|NO_PROXY|socket-proxy|published:|target:' /tmp/771r_compose.yml || true
|
||||
237: ! grep -qE 'DISABLE_ONLINE_API|ARGS:.*no-capi|-no-capi' /tmp/771r_compose.yml
|
||||
251: timeout 12 ssh -n \
|
||||
256: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
||||
268: if ssh -n -fN -M -S "$ctl" \
|
||||
278: code="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771r_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
279: trace="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 https://www.cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -E 'ip=|colo=' | sed -n '1,5p' || true)"
|
||||
280: echo "SOCKS_CAPI_HTTP=$code"
|
||||
284: ssh -S "$ctl" -O exit "$user@$ip" >/dev/null 2>&1 || true
|
||||
317: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
||||
319:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
||||
326:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
||||
327:ExecStart=/bin/sh -lc 'exec /usr/bin/ssh -N -D "${SOCKS_BIND}:${SOCKS_PORT}" -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${SSH_HOST}"'
|
||||
328:Restart=always
|
||||
329:RestartSec=5
|
||||
335: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
||||
339:base=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
347:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
348:systemctl restart privoxy 2>/dev/null || true
|
||||
350:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
||||
352: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
||||
355: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
||||
356: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
358: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
||||
372: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
||||
374: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
||||
384:out.append("# HOMELAB_771R_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
||||
387:out.append("# HOMELAB_771R_CROWDSEC_CAPI_NETBIRD_END")
|
||||
392: systemctl restart privoxy
|
||||
397: code="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 -o /tmp/771r_http_proxy_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
398: echo "HTTP_PROXY_CAPI_HTTP=$code"
|
||||
402:register_capi() {
|
||||
403: local b proxy_url reg_out reg_rc ready n health lapi_rc capi_out capi_rc fatal envbad rc_before rc_after health_after lapi_after capi_after fatal_after env_after
|
||||
405: proxy_url="http://${b}:${HTTP_PROXY_PORT}"
|
||||
406: echo "REGISTER_CAPI proxy=$proxy_url"
|
||||
408: force_no_capi_stable
|
||||
409: wait_lapi "BEFORE_CAPI_REGISTER" || return 10
|
||||
413: if test -f config/online_api_credentials.yaml; then
|
||||
414: mkdir -p /opt/stacks/crowdsec/manual-backups/771r-old-online-creds-"$TS"
|
||||
415: mv config/online_api_credentials.yaml /opt/stacks/crowdsec/manual-backups/771r-old-online-creds-"$TS"/online_api_credentials.yaml.before-register
|
||||
420: wait_lapi "REGISTER_MODE" || return 13
|
||||
425: if cscli capi register --help 2>&1 | grep -q -- "-y"; then
|
||||
426: timeout 240 cscli capi register -y >/tmp/771r_register.out 2>&1
|
||||
428: timeout 240 sh -c "yes | cscli capi register" >/tmp/771r_register.out 2>&1
|
||||
441: if ! test -f config/online_api_credentials.yaml; then
|
||||
445: stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' config/online_api_credentials.yaml || true
|
||||
446: awk -F: '/^[A-Za-z0-9_ -]+:/ {gsub(/^[ \t]+|[ \t]+$/, "", $1); print $1 ": REDACTED"}' config/online_api_credentials.yaml | sed -n '1,40p'
|
||||
448: grep -Eq '^[[:space:]]*login:' config/online_api_credentials.yaml || return 22
|
||||
455: echo "VALIDATE_CAPI"
|
||||
459: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health2.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
460: lapi_rc="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771r_lapi2.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
462: capi_rc="$(printf '%s\n' "$capi_out" | awk -F= '/^CAPI_RC=/{print $2}' | tail -1)"
|
||||
463: fatal="$(docker logs --since 60s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
464: envbad="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
465: echo "VALIDATE_TRY=$n HEALTH=$health LAPI_RC=${lapi_rc:-NA} CAPI_RC=${capi_rc:-NA} FATAL=$fatal ENV_BAD=${envbad:-NONE}"
|
||||
466: printf '%s\n' "$capi_out"
|
||||
467: if test "$health" = "200" && test "${lapi_rc:-NA}" = "0" && test "${capi_rc:-NA}" = "0" && test "$fatal" = "0" && test -z "$envbad"; then
|
||||
474: rc_before="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
476: rc_after="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
477: health_after="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health_after.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
478: lapi_after="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771r_lapi_after.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
KNOWN_SCRIPT=/root/771s_edge_capi_netbird_20260702T170251Z.sh SHA256=c4783c14a3a132616af5db6a065270ccd39a7690f9fd38d0ef27fe5de6bd07e5 EXECUTABLE=true
|
||||
9:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
11:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
||||
30: echo "CHECK=crowdsec-capi"
|
||||
36:force_no_capi_stable() {
|
||||
42: DISABLE_ONLINE_API: "true"
|
||||
43: ARGS: "-no-capi"
|
||||
65:wait_lapi() {
|
||||
67: echo "WAIT_LAPI=$label"
|
||||
70: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
72: rc="$(printf '%s\n' "$out" | awk -F= '/^LAPI_RC=/{print $2}' | tail -1)"
|
||||
73: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
||||
74: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
||||
149: raise SystemExit("api.server block not found")
|
||||
153: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
||||
191: grep -nE 'DISABLE_ONLINE_API|ARGS|GODEBUG|HTTP_PROXY|HTTPS_PROXY|NO_PROXY|socket-proxy|published:|target:' /tmp/771s_compose.yml || true
|
||||
192: ! grep -qE 'DISABLE_ONLINE_API|ARGS:.*no-capi|-no-capi' /tmp/771s_compose.yml
|
||||
204: timeout 12 ssh -n \
|
||||
209: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771s_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
||||
221: if ssh -n -fN -M -S "$ctl" \
|
||||
231: code="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771s_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
232: trace="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 https://www.cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -E 'ip=|colo=' | sed -n '1,5p' || true)"
|
||||
233: echo "SOCKS_CAPI_HTTP=$code"
|
||||
237: ssh -S "$ctl" -O exit "$user@$ip" >/dev/null 2>&1 || true
|
||||
269: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
||||
271:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
||||
278:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
||||
279:ExecStart=/bin/sh -lc 'exec /usr/bin/ssh -N -D "${SOCKS_BIND}:${SOCKS_PORT}" -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${SSH_HOST}"'
|
||||
280:Restart=always
|
||||
281:RestartSec=5
|
||||
287: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
||||
291:base=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
299:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
300:systemctl restart privoxy 2>/dev/null || true
|
||||
302:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
||||
304: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
||||
307: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
||||
308: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
310: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
||||
325: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
||||
328: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
||||
339:out.append("# HOMELAB_771S_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
||||
342:out.append("# HOMELAB_771S_CROWDSEC_CAPI_NETBIRD_END")
|
||||
347: systemctl restart privoxy
|
||||
352: code="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 -o /tmp/771s_http_proxy_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
353: echo "HTTP_PROXY_CAPI_HTTP=$code"
|
||||
357:register_capi() {
|
||||
359: proxy_url="http://${b}:${HTTP_PROXY_PORT}"
|
||||
360: echo "REGISTER_CAPI proxy=$proxy_url"
|
||||
362: force_no_capi_stable
|
||||
363: wait_lapi "BEFORE_CAPI_REGISTER" || return 10
|
||||
367: if test -f config/online_api_credentials.yaml; then
|
||||
368: mkdir -p /opt/stacks/crowdsec/manual-backups/771s-old-online-creds-"$TS"
|
||||
369: mv config/online_api_credentials.yaml /opt/stacks/crowdsec/manual-backups/771s-old-online-creds-"$TS"/online_api_credentials.yaml.before-register
|
||||
374: wait_lapi "REGISTER_MODE" || return 13
|
||||
379: if cscli capi register --help 2>&1 | grep -q -- "-y"; then
|
||||
380: timeout 240 cscli capi register -y >/tmp/771s_register.out 2>&1
|
||||
382: timeout 240 sh -c "yes | cscli capi register" >/tmp/771s_register.out 2>&1
|
||||
395: if ! test -f config/online_api_credentials.yaml; then
|
||||
399: stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' config/online_api_credentials.yaml || true
|
||||
400: awk -F: '/^[A-Za-z0-9_ -]+:/ {gsub(/^[ \t]+|[ \t]+$/, "", $1); print $1 ": REDACTED"}' config/online_api_credentials.yaml | sed -n '1,40p'
|
||||
402: grep -Eq '^[[:space:]]*login:' config/online_api_credentials.yaml || return 22
|
||||
409: echo "VALIDATE_CAPI"
|
||||
413: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health2.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
414: lapi_rc="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771s_lapi2.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
416: capi_rc="$(printf '%s\n' "$capi_out" | awk -F= '/^CAPI_RC=/{print $2}' | tail -1)"
|
||||
417: fatal="$(docker logs --since 60s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
418: envbad="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
419: echo "VALIDATE_TRY=$n HEALTH=$health LAPI_RC=${lapi_rc:-NA} CAPI_RC=${capi_rc:-NA} FATAL=$fatal ENV_BAD=${envbad:-NONE}"
|
||||
420: printf '%s\n' "$capi_out"
|
||||
421: if test "$health" = "200" && test "${lapi_rc:-NA}" = "0" && test "${capi_rc:-NA}" = "0" && test "$fatal" = "0" && test -z "$envbad"; then
|
||||
428: rc_before="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
430: rc_after="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
431: health_after="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health_after.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
432: lapi_after="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771s_lapi_after.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
433: capi_after="$(docker exec crowdsec sh -lc 'cscli capi status >/tmp/771s_capi_after.out 2>&1; echo CAPI_RC=$?' 2>/dev/null | awk -F= '/^CAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
434: fatal_after="$(docker logs --since 240s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
435: env_after="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
437: echo "FINAL_STABILITY RC_BEFORE=$rc_before RC_AFTER=$rc_after HEALTH_AFTER=$health_after LAPI_AFTER=${lapi_after:-NA} CAPI_AFTER=${capi_after:-NA} FATAL_AFTER=$fatal_after ENV_BAD_AFTER=${env_after:-NONE}"
|
||||
439: test "$rc_before" = "$rc_after" && test "$health_after" = "200" && test "${lapi_after:-NA}" = "0" && test "${capi_after:-NA}" = "0" && test "$fatal_after" = "0" && test -z "$env_after"
|
||||
443:echo "STEP=771S_EDGE_NETBIRD_EGRESS_CAPI"
|
||||
KNOWN_SCRIPT=/root/771s_vps_identity_repair_netbird_capi.sh SHA256=16121a810320b1517291830ba128baf6c68e7b3e6f78786481dedb78a1f061b9 EXECUTABLE=false
|
||||
8:PROOF="/root/evidence/771S_VPS_IDENTITY_REPAIR_NETBIRD_CAPI_${TS}_PROOF.txt"
|
||||
13:EDGE_REMOTE="/tmp/771s_edge_capi_netbird_${TS}.sh"
|
||||
14:EDGE_LOCAL="/root/771s_edge_capi_netbird_${TS}.sh"
|
||||
31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
||||
52: echo "CHECK=crowdsec-capi"
|
||||
58:force_no_capi_stable() {
|
||||
64: DISABLE_ONLINE_API: "true"
|
||||
65: ARGS: "-no-capi"
|
||||
87:wait_lapi() {
|
||||
89: echo "WAIT_LAPI=$label"
|
||||
92: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
94: rc="$(printf '%s\n' "$out" | awk -F= '/^LAPI_RC=/{print $2}' | tail -1)"
|
||||
95: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
||||
96: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
||||
171: raise SystemExit("api.server block not found")
|
||||
175: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
||||
213: grep -nE 'DISABLE_ONLINE_API|ARGS|GODEBUG|HTTP_PROXY|HTTPS_PROXY|NO_PROXY|socket-proxy|published:|target:' /tmp/771s_compose.yml || true
|
||||
214: ! grep -qE 'DISABLE_ONLINE_API|ARGS:.*no-capi|-no-capi' /tmp/771s_compose.yml
|
||||
226: timeout 12 ssh -n \
|
||||
231: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771s_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
||||
243: if ssh -n -fN -M -S "$ctl" \
|
||||
253: code="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771s_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
254: trace="$(curl -sk --proxy "socks5h://[PRIVATE_IP]:${port}" --connect-timeout 12 --max-time 45 https://www.cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -E 'ip=|colo=' | sed -n '1,5p' || true)"
|
||||
255: echo "SOCKS_CAPI_HTTP=$code"
|
||||
259: ssh -S "$ctl" -O exit "$user@$ip" >/dev/null 2>&1 || true
|
||||
291: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
||||
293:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
||||
300:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
||||
301:ExecStart=/bin/sh -lc 'exec /usr/bin/ssh -N -D "${SOCKS_BIND}:${SOCKS_PORT}" -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${SSH_HOST}"'
|
||||
302:Restart=always
|
||||
303:RestartSec=5
|
||||
309: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
||||
313:base=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
321:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
322:systemctl restart privoxy 2>/dev/null || true
|
||||
324:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
||||
326: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
||||
329: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
||||
330: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
332: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
||||
347: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
||||
350: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
||||
361:out.append("# HOMELAB_771S_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
||||
364:out.append("# HOMELAB_771S_CROWDSEC_CAPI_NETBIRD_END")
|
||||
369: systemctl restart privoxy
|
||||
374: code="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 -o /tmp/771s_http_proxy_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
375: echo "HTTP_PROXY_CAPI_HTTP=$code"
|
||||
379:register_capi() {
|
||||
381: proxy_url="http://${b}:${HTTP_PROXY_PORT}"
|
||||
382: echo "REGISTER_CAPI proxy=$proxy_url"
|
||||
384: force_no_capi_stable
|
||||
385: wait_lapi "BEFORE_CAPI_REGISTER" || return 10
|
||||
389: if test -f config/online_api_credentials.yaml; then
|
||||
390: mkdir -p /opt/stacks/crowdsec/manual-backups/771s-old-online-creds-"$TS"
|
||||
391: mv config/online_api_credentials.yaml /opt/stacks/crowdsec/manual-backups/771s-old-online-creds-"$TS"/online_api_credentials.yaml.before-register
|
||||
396: wait_lapi "REGISTER_MODE" || return 13
|
||||
401: if cscli capi register --help 2>&1 | grep -q -- "-y"; then
|
||||
402: timeout 240 cscli capi register -y >/tmp/771s_register.out 2>&1
|
||||
404: timeout 240 sh -c "yes | cscli capi register" >/tmp/771s_register.out 2>&1
|
||||
417: if ! test -f config/online_api_credentials.yaml; then
|
||||
421: stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' config/online_api_credentials.yaml || true
|
||||
422: awk -F: '/^[A-Za-z0-9_ -]+:/ {gsub(/^[ \t]+|[ \t]+$/, "", $1); print $1 ": REDACTED"}' config/online_api_credentials.yaml | sed -n '1,40p'
|
||||
424: grep -Eq '^[[:space:]]*login:' config/online_api_credentials.yaml || return 22
|
||||
431: echo "VALIDATE_CAPI"
|
||||
435: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health2.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
436: lapi_rc="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771s_lapi2.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
438: capi_rc="$(printf '%s\n' "$capi_out" | awk -F= '/^CAPI_RC=/{print $2}' | tail -1)"
|
||||
439: fatal="$(docker logs --since 60s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
440: envbad="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
441: echo "VALIDATE_TRY=$n HEALTH=$health LAPI_RC=${lapi_rc:-NA} CAPI_RC=${capi_rc:-NA} FATAL=$fatal ENV_BAD=${envbad:-NONE}"
|
||||
442: printf '%s\n' "$capi_out"
|
||||
443: if test "$health" = "200" && test "${lapi_rc:-NA}" = "0" && test "${capi_rc:-NA}" = "0" && test "$fatal" = "0" && test -z "$envbad"; then
|
||||
450: rc_before="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
452: rc_after="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
453: health_after="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health_after.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
454: lapi_after="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771s_lapi_after.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
455: capi_after="$(docker exec crowdsec sh -lc 'cscli capi status >/tmp/771s_capi_after.out 2>&1; echo CAPI_RC=$?' 2>/dev/null | awk -F= '/^CAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
456: fatal_after="$(docker logs --since 240s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
457: env_after="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
KNOWN_SCRIPT=/root/771t_temp_hostkey_netbird_identity_then_capi.sh SHA256=0c3d628b23b67849d52112322ab2e1ee3779012c10578a0b8cece0d8ddf639da EXECUTABLE=false
|
||||
8:PROOF="/root/evidence/771T_TEMP_HOSTKEY_NETBIRD_IDENTITY_THEN_CAPI_${TS}_PROOF.txt"
|
||||
11:EDGE_SCRIPT_LOCAL="/root/771t_edge_capi_${TS}.sh"
|
||||
12:EDGE_SCRIPT_REMOTE="/tmp/771t_edge_capi_${TS}.sh"
|
||||
31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
||||
50: echo "CHECK=crowdsec-capi"
|
||||
56:force_no_capi_stable() {
|
||||
62: DISABLE_ONLINE_API: "true"
|
||||
63: ARGS: "-no-capi"
|
||||
85:wait_lapi() {
|
||||
87: echo "WAIT_LAPI=$label"
|
||||
90: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771t_health.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
92: rc="$(printf '%s\n' "$out" | awk -F= '/^LAPI_RC=/{print $2}' | tail -1)"
|
||||
93: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
||||
94: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
||||
166: raise SystemExit("api.server block not found")
|
||||
169: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
||||
207: grep -nE 'DISABLE_ONLINE_API|ARGS|GODEBUG|HTTP_PROXY|HTTPS_PROXY|NO_PROXY|socket-proxy|published:|target:' /tmp/771t_compose.yml || true
|
||||
208: ! grep -qE 'DISABLE_ONLINE_API|ARGS:.*no-capi|-no-capi' /tmp/771t_compose.yml
|
||||
217: timeout 15 ssh -n -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "$TARGET_USER@$TARGET_NB" \
|
||||
218: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771t_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" >"$out" 2>&1
|
||||
236: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
||||
238:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
||||
245:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
||||
246:ExecStart=/bin/sh -lc 'exec /usr/bin/ssh -N -D "${SOCKS_BIND}:${SOCKS_PORT}" -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${SSH_HOST}"'
|
||||
247:Restart=always
|
||||
248:RestartSec=5
|
||||
254: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
||||
258:base=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
266:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
267:systemctl restart privoxy 2>/dev/null || true
|
||||
269:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
||||
271: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
||||
274: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
||||
275: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
277: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
||||
293: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
||||
296: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
||||
307:out.append("# HOMELAB_771T_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
||||
310:out.append("# HOMELAB_771T_CROWDSEC_CAPI_NETBIRD_END")
|
||||
315: systemctl restart privoxy
|
||||
320: code="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 -o /tmp/771t_http_proxy_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
321: trace="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 https://www.cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -E 'ip=|colo=' | sed -n '1,6p' || true)"
|
||||
322: echo "HTTP_PROXY_CAPI_HTTP=$code"
|
||||
329:register_capi() {
|
||||
331: proxy_url="http://${b}:${HTTP_PROXY_PORT}"
|
||||
332: echo "REGISTER_CAPI proxy=$proxy_url"
|
||||
334: force_no_capi_stable
|
||||
335: wait_lapi "BEFORE_CAPI_REGISTER" || return 10
|
||||
339: if test -f config/online_api_credentials.yaml; then
|
||||
340: mkdir -p /opt/stacks/crowdsec/manual-backups/771t-old-online-creds-"$TS"
|
||||
341: mv config/online_api_credentials.yaml /opt/stacks/crowdsec/manual-backups/771t-old-online-creds-"$TS"/online_api_credentials.yaml.before-register
|
||||
346: wait_lapi "REGISTER_MODE" || return 13
|
||||
351: if cscli capi register --help 2>&1 | grep -q -- "-y"; then
|
||||
352: timeout 240 cscli capi register -y >/tmp/771t_register.out 2>&1
|
||||
354: timeout 240 sh -c "yes | cscli capi register" >/tmp/771t_register.out 2>&1
|
||||
367: if ! test -f config/online_api_credentials.yaml; then
|
||||
371: stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' config/online_api_credentials.yaml || true
|
||||
372: awk -F: '/^[A-Za-z0-9_ -]+:/ {gsub(/^[ \t]+|[ \t]+$/, "", $1); print $1 ": REDACTED"}' config/online_api_credentials.yaml | sed -n '1,40p'
|
||||
374: grep -Eq '^[[:space:]]*login:' config/online_api_credentials.yaml || return 22
|
||||
381: echo "VALIDATE_CAPI"
|
||||
385: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771t_health2.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
386: lapi_rc="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771t_lapi2.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
388: capi_rc="$(printf '%s\n' "$capi_out" | awk -F= '/^CAPI_RC=/{print $2}' | tail -1)"
|
||||
389: fatal="$(docker logs --since 60s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
390: envbad="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
391: echo "VALIDATE_TRY=$n HEALTH=$health LAPI_RC=${lapi_rc:-NA} CAPI_RC=${capi_rc:-NA} FATAL=$fatal ENV_BAD=${envbad:-NONE}"
|
||||
392: printf '%s\n' "$capi_out"
|
||||
393: if test "$health" = "200" && test "${lapi_rc:-NA}" = "0" && test "${capi_rc:-NA}" = "0" && test "$fatal" = "0" && test -z "$envbad"; then
|
||||
400: rc_before="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
402: rc_after="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
403: health_after="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771t_health_after.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
404: lapi_after="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771t_lapi_after.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
405: capi_after="$(docker exec crowdsec sh -lc 'cscli capi status >/tmp/771t_capi_after.out 2>&1; echo CAPI_RC=$?' 2>/dev/null | awk -F= '/^CAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
406: fatal_after="$(docker logs --since 240s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
407: env_after="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
409: echo "FINAL_STABILITY RC_BEFORE=$rc_before RC_AFTER=$rc_after HEALTH_AFTER=$health_after LAPI_AFTER=${lapi_after:-NA} CAPI_AFTER=${capi_after:-NA} FATAL_AFTER=$fatal_after ENV_BAD_AFTER=${env_after:-NONE}"
|
||||
411: test "$rc_before" = "$rc_after" && test "$health_after" = "200" && test "${lapi_after:-NA}" = "0" && test "${capi_after:-NA}" = "0" && test "$fatal_after" = "0" && test -z "$env_after"
|
||||
415:echo "STEP=771T_EDGE_CAPI_VIA_VERIFIED_NETBIRD_PEER"
|
||||
418:systemctl restart netbird 2>/dev/null || true
|
||||
KNOWN_SCRIPT=/root/771u_find_key_fix_netbird_egress_capi.sh SHA256=6b6511ec3d614793e3542777ccfcb5e8c5f09cff77ee2d57624b3ac980787bcb EXECUTABLE=false
|
||||
8:PROOF="/root/evidence/771U_FIND_KEY_FIX_NETBIRD_EGRESS_CAPI_${TS}_PROOF.txt"
|
||||
11:EDGE_REMOTE="/tmp/771u_edge_register_capi_${TS}.sh"
|
||||
12:EDGE_LOCAL="/root/771u_edge_register_capi_${TS}.sh"
|
||||
31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
||||
52: echo "CHECK=crowdsec-capi"
|
||||
58:force_no_capi_stable() {
|
||||
64: DISABLE_ONLINE_API: "true"
|
||||
65: ARGS: "-no-capi"
|
||||
87:wait_lapi() {
|
||||
89: echo "WAIT_LAPI=$label"
|
||||
92: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771u_health.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
94: rc="$(printf '%s\n' "$out" | awk -F= '/^LAPI_RC=/{print $2}' | tail -1)"
|
||||
95: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
||||
96: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
||||
167: raise SystemExit("api.server block not found")
|
||||
170: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
||||
207: grep -nE 'DISABLE_ONLINE_API|ARGS|GODEBUG|HTTP_PROXY|HTTPS_PROXY|NO_PROXY|socket-proxy|published:|target:' /tmp/771u_compose.yml || true
|
||||
208: ! grep -qE 'DISABLE_ONLINE_API|ARGS:.*no-capi|-no-capi' /tmp/771u_compose.yml
|
||||
213: ssh-keygen -q -t ed25519 -N "" -C "homelab-crowdsec-capi-netbird-egress-edge" -f "$SSH_KEY"
|
||||
227: timeout 15 ssh -n \
|
||||
234: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771u_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" >"$out" 2>&1
|
||||
253: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
||||
255:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
||||
262:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
||||
263:ExecStart=/bin/sh -lc 'exec /usr/bin/ssh -N -i "${SSH_KEY}" -D "${SOCKS_BIND}:${SOCKS_PORT}" -o IdentitiesOnly=yes -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${SSH_HOST}"'
|
||||
264:Restart=always
|
||||
265:RestartSec=5
|
||||
271: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
||||
275:base=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
283:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
284:systemctl restart privoxy 2>/dev/null || true
|
||||
286:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
||||
288: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
||||
291: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
||||
292: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
294: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
||||
310: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
||||
313: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
||||
324:out.append("# HOMELAB_771U_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
||||
327:out.append("# HOMELAB_771U_CROWDSEC_CAPI_NETBIRD_END")
|
||||
332: systemctl restart privoxy
|
||||
337: code="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 -o /tmp/771u_http_proxy_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
338: trace="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 https://www.cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -E 'ip=|colo=' | sed -n '1,6p' || true)"
|
||||
339: echo "HTTP_PROXY_CAPI_HTTP=$code"
|
||||
346:register_capi() {
|
||||
348: proxy_url="http://${b}:${HTTP_PROXY_PORT}"
|
||||
349: echo "REGISTER_CAPI proxy=$proxy_url"
|
||||
351: force_no_capi_stable
|
||||
352: wait_lapi "BEFORE_CAPI_REGISTER" || return 10
|
||||
356: if test -f config/online_api_credentials.yaml; then
|
||||
357: mkdir -p /opt/stacks/crowdsec/manual-backups/771u-old-online-creds-"$TS"
|
||||
358: mv config/online_api_credentials.yaml /opt/stacks/crowdsec/manual-backups/771u-old-online-creds-"$TS"/online_api_credentials.yaml.before-register
|
||||
363: wait_lapi "REGISTER_MODE" || return 13
|
||||
368: if cscli capi register --help 2>&1 | grep -q -- "-y"; then
|
||||
369: timeout 240 cscli capi register -y >/tmp/771u_register.out 2>&1
|
||||
371: timeout 240 sh -c "yes | cscli capi register" >/tmp/771u_register.out 2>&1
|
||||
384: if ! test -f config/online_api_credentials.yaml; then
|
||||
388: stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' config/online_api_credentials.yaml || true
|
||||
389: awk -F: '/^[A-Za-z0-9_ -]+:/ {gsub(/^[ \t]+|[ \t]+$/, "", $1); print $1 ": REDACTED"}' config/online_api_credentials.yaml | sed -n '1,40p'
|
||||
391: grep -Eq '^[[:space:]]*login:' config/online_api_credentials.yaml || return 22
|
||||
398: echo "VALIDATE_CAPI"
|
||||
402: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771u_health2.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
403: lapi_rc="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771u_lapi2.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
405: capi_rc="$(printf '%s\n' "$capi_out" | awk -F= '/^CAPI_RC=/{print $2}' | tail -1)"
|
||||
406: fatal="$(docker logs --since 60s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
407: envbad="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
408: echo "VALIDATE_TRY=$n HEALTH=$health LAPI_RC=${lapi_rc:-NA} CAPI_RC=${capi_rc:-NA} FATAL=$fatal ENV_BAD=${envbad:-NONE}"
|
||||
409: printf '%s\n' "$capi_out"
|
||||
410: if test "$health" = "200" && test "${lapi_rc:-NA}" = "0" && test "${capi_rc:-NA}" = "0" && test "$fatal" = "0" && test -z "$envbad"; then
|
||||
417: rc_before="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
419: rc_after="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
420: health_after="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771u_health_after.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
421: lapi_after="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771u_lapi_after.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
422: capi_after="$(docker exec crowdsec sh -lc 'cscli capi status >/tmp/771u_capi_after.out 2>&1; echo CAPI_RC=$?' 2>/dev/null | awk -F= '/^CAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
423: fatal_after="$(docker logs --since 240s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
424: env_after="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
426: echo "FINAL_STABILITY RC_BEFORE=$rc_before RC_AFTER=$rc_after HEALTH_AFTER=$health_after LAPI_AFTER=${lapi_after:-NA} CAPI_AFTER=${capi_after:-NA} FATAL_AFTER=$fatal_after ENV_BAD_AFTER=${env_after:-NONE}"
|
||||
428: test "$rc_before" = "$rc_after" && test "$health_after" = "200" && test "${lapi_after:-NA}" = "0" && test "${capi_after:-NA}" = "0" && test "$fatal_after" = "0" && test -z "$env_after"
|
||||
432:echo "STEP=771U_EDGE_EGRESS_CAPI"
|
||||
KNOWN_SCRIPT=/root/771v_deep_key_backup_netbird_capi.sh SHA256=92191ce6124981ee0468cc8f95c749c47bca5fc61f9aecaeedeef91cdd434170 EXECUTABLE=false
|
||||
8:PROOF="/root/evidence/771V_DEEP_KEY_BACKUP_NETBIRD_CAPI_${TS}_PROOF.txt"
|
||||
11:EDGE_LOCAL="/root/771v_edge_netbird_capi_${TS}.sh"
|
||||
12:EDGE_REMOTE="/tmp/771v_edge_netbird_capi_${TS}.sh"
|
||||
31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
||||
52: echo "CHECK=crowdsec-capi"
|
||||
58:force_no_capi_stable() {
|
||||
64: DISABLE_ONLINE_API: "true"
|
||||
65: ARGS: "-no-capi"
|
||||
87:wait_lapi() {
|
||||
89: echo "WAIT_LAPI=$label"
|
||||
92: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771v_health.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
94: rc="$(printf '%s\n' "$out" | awk -F= '/^LAPI_RC=/{print $2}' | tail -1)"
|
||||
95: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
||||
96: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
||||
168: raise SystemExit("api.server block not found")
|
||||
172: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
||||
210: grep -nE 'DISABLE_ONLINE_API|ARGS|GODEBUG|HTTP_PROXY|HTTPS_PROXY|NO_PROXY|socket-proxy|published:|target:' /tmp/771v_compose.yml || true
|
||||
211: ! grep -qE 'DISABLE_ONLINE_API|ARGS:.*no-capi|-no-capi' /tmp/771v_compose.yml
|
||||
216: ssh-keygen -q -t ed25519 -N "" -C "homelab-crowdsec-capi-netbird-egress-edge" -f "$SSH_KEY"
|
||||
230: timeout 15 ssh -n \
|
||||
237: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771v_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" >"$out" 2>&1
|
||||
256: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
||||
258:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
||||
265:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
||||
266:ExecStart=/bin/sh -lc 'exec /usr/bin/ssh -N -i "${SSH_KEY}" -D "${SOCKS_BIND}:${SOCKS_PORT}" -o IdentitiesOnly=yes -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${SSH_HOST}"'
|
||||
267:Restart=always
|
||||
268:RestartSec=5
|
||||
274: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
||||
278:base=/etc/homelab-crowdsec-capi-netbird-egress
|
||||
286:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
287:systemctl restart privoxy 2>/dev/null || true
|
||||
289:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
||||
291: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
||||
294: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
||||
295: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
||||
297: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
||||
313: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
||||
316: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
||||
327:out.append("# HOMELAB_771V_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
||||
330:out.append("# HOMELAB_771V_CROWDSEC_CAPI_NETBIRD_END")
|
||||
335: systemctl restart privoxy
|
||||
340: code="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 -o /tmp/771v_http_proxy_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
341: trace="$(curl -sk --proxy "http://${b}:${HTTP_PROXY_PORT}" --connect-timeout 12 --max-time 45 https://www.cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -E 'ip=|colo=' | sed -n '1,6p' || true)"
|
||||
342: echo "HTTP_PROXY_CAPI_HTTP=$code"
|
||||
349:register_capi() {
|
||||
351: proxy_url="http://${b}:${HTTP_PROXY_PORT}"
|
||||
352: echo "REGISTER_CAPI proxy=$proxy_url"
|
||||
354: force_no_capi_stable
|
||||
355: wait_lapi "BEFORE_CAPI_REGISTER" || return 10
|
||||
359: if test -f config/online_api_credentials.yaml; then
|
||||
360: mkdir -p /opt/stacks/crowdsec/manual-backups/771v-old-online-creds-"$TS"
|
||||
361: mv config/online_api_credentials.yaml /opt/stacks/crowdsec/manual-backups/771v-old-online-creds-"$TS"/online_api_credentials.yaml.before-register
|
||||
366: wait_lapi "REGISTER_MODE" || return 13
|
||||
371: if cscli capi register --help 2>&1 | grep -q -- "-y"; then
|
||||
372: timeout 240 cscli capi register -y >/tmp/771v_register.out 2>&1
|
||||
374: timeout 240 sh -c "yes | cscli capi register" >/tmp/771v_register.out 2>&1
|
||||
387: if ! test -f config/online_api_credentials.yaml; then
|
||||
391: stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' config/online_api_credentials.yaml || true
|
||||
392: awk -F: '/^[A-Za-z0-9_ -]+:/ {gsub(/^[ \t]+|[ \t]+$/, "", $1); print $1 ": REDACTED"}' config/online_api_credentials.yaml | sed -n '1,40p'
|
||||
394: grep -Eq '^[[:space:]]*login:' config/online_api_credentials.yaml || return 22
|
||||
401: echo "VALIDATE_CAPI"
|
||||
405: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771v_health2.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
406: lapi_rc="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771v_lapi2.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
408: capi_rc="$(printf '%s\n' "$capi_out" | awk -F= '/^CAPI_RC=/{print $2}' | tail -1)"
|
||||
409: fatal="$(docker logs --since 60s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
410: envbad="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
411: echo "VALIDATE_TRY=$n HEALTH=$health LAPI_RC=${lapi_rc:-NA} CAPI_RC=${capi_rc:-NA} FATAL=$fatal ENV_BAD=${envbad:-NONE}"
|
||||
412: printf '%s\n' "$capi_out"
|
||||
413: if test "$health" = "200" && test "${lapi_rc:-NA}" = "0" && test "${capi_rc:-NA}" = "0" && test "$fatal" = "0" && test -z "$envbad"; then
|
||||
420: rc_before="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
422: rc_after="$(docker inspect crowdsec --format '{{.RestartCount}}' 2>/dev/null || echo NA)"
|
||||
423: health_after="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771v_health_after.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
424: lapi_after="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771v_lapi_after.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
425: capi_after="$(docker exec crowdsec sh -lc 'cscli capi status >/tmp/771v_capi_after.out 2>&1; echo CAPI_RC=$?' 2>/dev/null | awk -F= '/^CAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
426: fatal_after="$(docker logs --since 240s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml|proxyconnect tcp' || true)"
|
||||
427: env_after="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
429: echo "FINAL_STABILITY RC_BEFORE=$rc_before RC_AFTER=$rc_after HEALTH_AFTER=$health_after LAPI_AFTER=${lapi_after:-NA} CAPI_AFTER=${capi_after:-NA} FATAL_AFTER=$fatal_after ENV_BAD_AFTER=${env_after:-NONE}"
|
||||
431: test "$rc_before" = "$rc_after" && test "$health_after" = "200" && test "${lapi_after:-NA}" = "0" && test "${capi_after:-NA}" = "0" && test "$fatal_after" = "0" && test -z "$env_after"
|
||||
435:echo "STEP=771V_EDGE_NETBIRD_EGRESS_CAPI"
|
||||
KNOWN_SCRIPT=/root/771z_after_manual_netbird_egress_capi.sh SHA256=f198ed621fd726ed6f15c4a0dd49fad307befd01e7a58ebbda1ddc379f5b49ee EXECUTABLE=false
|
||||
8:PROOF="/root/evidence/771Z_AFTER_MANUAL_NETBIRD_EGRESS_CAPI_${TS}_PROOF.txt"
|
||||
14: echo "STEP=771Z_AFTER_MANUAL_NETBIRD_EGRESS_CAPI"
|
||||
17: echo "RULE=VERIFY_RELAY_MAIL_EGRESS_AND_REGISTER_CROWDSEC_CAPI"
|
||||
21: ssh debian@$EDGE "sudo bash -s" <<'EDGE'
|
||||
27:HEALTH=/var/lib/homelab-health/crowdsec-capi.txt
|
||||
38: echo "CHECK=crowdsec-capi"
|
||||
44:wait_lapi() {
|
||||
46: echo "WAIT_LAPI=$label"
|
||||
49: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771z_health.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
51: rc="$(printf '%s\n' "$out" | awk -F= '/^LAPI_RC=/{print $2}' | tail -1)"
|
||||
52: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA}"
|
||||
59:force_no_capi() {
|
||||
65: DISABLE_ONLINE_API: "true"
|
||||
66: ARGS: "-no-capi"
|
||||
87:enable_capi_compose() {
|
||||
112: grep -nE 'DISABLE_ONLINE_API|ARGS|GODEBUG|socket-proxy|published:|target:' /tmp/771z_compose.yml || true
|
||||
113: ! grep -qE 'DISABLE_ONLINE_API|ARGS:.*no-capi|-no-capi' /tmp/771z_compose.yml
|
||||
180: raise SystemExit("api.server block not found")
|
||||
183: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
||||
194:echo "NETBIRD_RESTART"
|
||||
195:systemctl restart netbird 2>/dev/null || true
|
||||
199:netbird status 2>&1 | redact || true
|
||||
202:netbird status --json >/tmp/771z_nb.json 2>/tmp/771z_nb.err || true
|
||||
229:echo "CAPI_DIRECT_TEST_AFTER_MANUAL_NETBIRD"
|
||||
230:capi_code="$(curl -4 -sk --http1.1 --connect-timeout 12 --max-time 45 -o /tmp/771z_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
||||
231:trace="$(curl -4 -sk --connect-timeout 12 --max-time 30 https://www.cloudflare.com/cdn-cgi/trace 2>/dev/null | grep -E 'ip=|colo=' | sed -n '1,6p' || true)"
|
||||
232:echo "CAPI_DIRECT_HTTP=$capi_code"
|
||||
237:if test "$capi_code" = "000"; then
|
||||
238: force_no_capi
|
||||
239: wait_lapi "CAPI_ROUTE_NOT_READY_SAFE" || true
|
||||
240: write_health "REVIEW_MANUAL_NETBIRD_EGRESS_NOT_READY" "After manual NetBird setup, edge still cannot reach api.crowdsec.net; CrowdSec remains LAPI-only no-capi"
|
||||
241: echo "EDGE_STATUS=REVIEW_MANUAL_NETBIRD_EGRESS_NOT_READY_CAPI_NOT_DONE"
|
||||
246:echo "CAPI_ROUTE_READY_REGISTER_NOW"
|
||||
248:mkdir -p "manual-backups/771z-$TS"
|
||||
249:test -f config/config.yaml && cp -a config/config.yaml "manual-backups/771z-$TS/config.yaml.before" || true
|
||||
250:test -f config/user.yaml && cp -a config/user.yaml "manual-backups/771z-$TS/user.yaml.before" || true
|
||||
251:test -f config/online_api_credentials.yaml && mv config/online_api_credentials.yaml "manual-backups/771z-$TS/online_api_credentials.yaml.before" || true
|
||||
254:enable_capi_compose || {
|
||||
255: force_no_capi
|
||||
256: wait_lapi "COMPOSE_FAIL_SAFE" || true
|
||||
257: write_health "REVIEW_CAPI_COMPOSE_ENABLE_FAILED" "Direct CAPI route works, but compose CAPI enable failed"
|
||||
262:wait_lapi "REGISTER_MODE" || exit 73
|
||||
266: if cscli capi register --help 2>&1 | grep -q -- "-y"; then
|
||||
267: timeout 240 cscli capi register -y >/tmp/771z_register.out 2>&1
|
||||
269: timeout 240 sh -c "yes | cscli capi register" >/tmp/771z_register.out 2>&1
|
||||
281:if test "${reg_rc:-NA}" != "0" || ! test -f config/online_api_credentials.yaml; then
|
||||
282: force_no_capi
|
||||
283: wait_lapi "REGISTER_FAIL_SAFE" || true
|
||||
284: write_health "REVIEW_CAPI_ROUTE_READY_BUT_REGISTER_FAILED" "Direct CAPI route works, but cscli capi register failed; restored no-capi"
|
||||
285: echo "EDGE_STATUS=REVIEW_CAPI_REGISTER_FAILED_NOT_DONE"
|
||||
290:stat -c 'ONLINE_CREDS path=%n mode=%a owner=%U group=%G size=%s' config/online_api_credentials.yaml || true
|
||||
291:awk -F: '/^[A-Za-z0-9_ -]+:/ {gsub(/^[ \t]+|[ \t]+$/, "", $1); print $1 ": REDACTED"}' config/online_api_credentials.yaml | sed -n '1,40p'
|
||||
294:enable_capi_compose || exit 75
|
||||
300: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771z_health2.html -w '%{http_code}' http://[PRIVATE_IP]:8088/health || true)"
|
||||
301: lapi_rc="$(docker exec crowdsec sh -lc 'cscli lapi status >/tmp/771z_lapi2.out 2>&1; echo LAPI_RC=$?' 2>/dev/null | awk -F= '/^LAPI_RC=/{print $2}' | tail -1 || true)"
|
||||
303: capi_rc="$(printf '%s\n' "$capi_out" | awk -F= '/^CAPI_RC=/{print $2}' | tail -1)"
|
||||
304: fatal="$(docker logs --since 60s crowdsec 2>&1 | grep -Eic 'fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|open /etc/crowdsec/online_api_credentials.yaml' || true)"
|
||||
305: envbad="$(docker inspect crowdsec --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep -E 'DISABLE_ONLINE_API|ARGS=.*no-capi|-no-capi' || true)"
|
||||
306: echo "VALIDATE_TRY=$n HEALTH=$health LAPI_RC=${lapi_rc:-NA} CAPI_RC=${capi_rc:-NA} FATAL=$fatal ENV_BAD=${envbad:-NONE}"
|
||||
307: printf '%s\n' "$capi_out"
|
||||
308: if test "$health" = "200" && test "${lapi_rc:-NA}" = "0" && test "${capi_rc:-NA}" = "0" && test "$fatal" = "0" && test -z "$envbad"; then
|
||||
315: write_health "OK_CAPI_REGISTERED_ENABLED_STABLE_MANUAL_NETBIRD_EGRESS" "CrowdSec CAPI registered and stable after manual NetBird egress via relay/mail"
|
||||
316: echo "EDGE_STATUS=OK_CROWDSEC_CAPI_100_PERCENT_REGISTERED_ENABLED_STABLE"
|
||||
321:force_no_capi
|
||||
322:wait_lapi "VALIDATION_FAIL_SAFE" || true
|
||||
323:write_health "REVIEW_CAPI_REGISTERED_BUT_NOT_STABLE_RESTORED_NO_CAPI" "CAPI registration happened but stability validation failed; restored no-capi"
|
||||
324:echo "EDGE_STATUS=REVIEW_CAPI_NOT_STABLE_RESTORED_NO_CAPI"
|
||||
331: code=$(curl -sk --connect-timeout 8 --max-time 20 --resolve "$h:443:$EDGE" -o "/tmp/771z_$h.html" -w "%{http_code}" "https://$h/" || true)
|
||||
337: echo STATUS=OK_771Z_AFTER_MANUAL_NETBIRD_EGRESS_CAPI_DONE
|
||||
KNOWN_771_SAFE_STATIC_END=1
|
||||
POLICY_DOCS_BEGIN=1
|
||||
POLICY_DOC=/root/_2/workers2-cr0108-final-of79_umj/repo/kb/private/archive/2026-08-19/docs/22_MOLDOVA_HEALTHCHECK_GOTIFY_2026-08-18.md SHA256=fe0f5ca63837626583d150e8c4b9ce9c795832ae4f59af25369a00fa7825fdb5
|
||||
1:# MOLDOVA HEALTHCHECK V3 / GOTIFY NOTIFIER — CURRENT RECORD
|
||||
9:NetBird:
|
||||
14:Old checker referenced retired Mailcow/old NetBird IP and caused false failures.
|
||||
16:Current:
|
||||
17:`/usr/local/sbin/pvepro-relay-healthcheck`
|
||||
27:- new NetBird TCP 80/443
|
||||
37:`PASS_RELAY_HEALTHCHECK_OK`
|
||||
40:enabled/active.
|
||||
48:Old USA observer had been converted to Gotify-only before retirement.
|
||||
50:Moldova direct public DNS for `gotify.gram1.ru` is not relied upon.
|
||||
78:`/etc/systemd/system/pvepro-relay-healthcheck.service.d/20-gotify-notifier.conf`
|
||||
86:- timer active
|
||||
89:Rollback backup:
|
||||
92:Old USA observer then:
|
||||
93:- backup created
|
||||
95:- service inactive
|
||||
96:- 80-second freeze proved no further health-file updates
|
||||
98:- old NetBird server still stopped
|
||||
99:- old host NetBird client still connected
|
||||
100:- Moldova peer reachable
|
||||
102:Old observer backup:
|
||||
103:`/root/retired-homelab-dr-observer-20260818T221046Z`
|
||||
POLICY_DOC=/root/_2/workers2-cr0108-final-of79_umj/repo/kb/private/archive/2026-08-19/docs/20_NETBIRD_USA_MIGRATION_2026-08-18.md SHA256=12fe079849346352315aac76fae82d00cbd17f88a3553270ca2c5f2749840552
|
||||
1:# NETBIRD USA MIGRATION — FINAL CURRENT RECORD
|
||||
5:Old USA mail/NetBird VPS:
|
||||
7:- secondary NetBird IPv4 `[PRIVATE_IP]`
|
||||
10:Old NetBird server stack stopped after successful migration.
|
||||
11:Host-level NetBird client left running for rollback/peer observation.
|
||||
33:NetBird:
|
||||
36:Compose bind changed only from old NetBird secondary public IP to new `[PRIVATE_IP]`.
|
||||
56:- Moldova synthetic check passes
|
||||
59:Important health discovery:
|
||||
60:- `/api/health` 404 on exact deployed version
|
||||
61:- `:9000/health` 503 in combined relay/server mode
|
||||
62:- first rollback was triggered by incorrectly assuming this endpoint must be healthy
|
||||
65:Backups on new server include migration/cutover snapshots such as:
|
||||
66:`/root/netbird-cutover-20260818T145807Z`
|
||||
69:- upgrade NetBird
|
||||
POLICY_DOC=/root/_2/workers2-cr0108-final-of79_umj/repo/kb/private/archive/2026-08-19/docs/15_CRITICAL_KEEP_RETIRE_DECISIONS.md SHA256=ad3ea70345063e8a13608a83acb4d6a1118533eb6acab22929e0bf1ac20c06af
|
||||
1:# CRITICAL KEEP / RETIRE DECISIONS
|
||||
3:This is a convenience matrix distilled from the historical handbook plus current state. A RETIRE label is not authorization to delete without fresh proof.
|
||||
11:| VM9130 edge-cold-standby | KEEP UNTIL DR PROOF | do not delete before timed VM130 restore |
|
||||
12:| VM150 Snikket | KEEP/CLOSED | do not reopen destructive rebuild without new defect |
|
||||
16:| VM180 cluster-admin | historical future RETIRE candidate | only after dependency/backup/monitoring cleanup |
|
||||
18:| CT200 skladchik-mod | historical future RETIRE candidate | preserve required data/monitoring first |
|
||||
19:| public edge01 | KEEP/CRITICAL | current git-read V3 and public edge duties |
|
||||
20:| Moldova relay | KEEP | independent relay/external health |
|
||||
21:| USA mail/NetBird | KEEP | infra mail/monitoring/no-PII |
|
||||
24:| pve01 18788 | KEEP NOW | reader-v3, usage proof before retirement |
|
||||
27:| Cloud.ru prepared bucket | KEEP PREPARED | real backup workload not yet active |
|
||||
POLICY_DOC=/root/_2/workers2-cr0108-final-of79_umj/FINAL_HANDOFF/repo/kb/private/archive/2026-08-19/docs/22_MOLDOVA_HEALTHCHECK_GOTIFY_2026-08-18.md SHA256=fe0f5ca63837626583d150e8c4b9ce9c795832ae4f59af25369a00fa7825fdb5
|
||||
1:# MOLDOVA HEALTHCHECK V3 / GOTIFY NOTIFIER — CURRENT RECORD
|
||||
9:NetBird:
|
||||
14:Old checker referenced retired Mailcow/old NetBird IP and caused false failures.
|
||||
16:Current:
|
||||
17:`/usr/local/sbin/pvepro-relay-healthcheck`
|
||||
27:- new NetBird TCP 80/443
|
||||
37:`PASS_RELAY_HEALTHCHECK_OK`
|
||||
40:enabled/active.
|
||||
48:Old USA observer had been converted to Gotify-only before retirement.
|
||||
50:Moldova direct public DNS for `gotify.gram1.ru` is not relied upon.
|
||||
78:`/etc/systemd/system/pvepro-relay-healthcheck.service.d/20-gotify-notifier.conf`
|
||||
86:- timer active
|
||||
89:Rollback backup:
|
||||
92:Old USA observer then:
|
||||
93:- backup created
|
||||
95:- service inactive
|
||||
96:- 80-second freeze proved no further health-file updates
|
||||
98:- old NetBird server still stopped
|
||||
99:- old host NetBird client still connected
|
||||
100:- Moldova peer reachable
|
||||
102:Old observer backup:
|
||||
103:`/root/retired-homelab-dr-observer-20260818T221046Z`
|
||||
POLICY_DOC=/root/_2/workers2-cr0108-final-of79_umj/FINAL_HANDOFF/repo/kb/private/archive/2026-08-19/docs/20_NETBIRD_USA_MIGRATION_2026-08-18.md SHA256=12fe079849346352315aac76fae82d00cbd17f88a3553270ca2c5f2749840552
|
||||
1:# NETBIRD USA MIGRATION — FINAL CURRENT RECORD
|
||||
5:Old USA mail/NetBird VPS:
|
||||
7:- secondary NetBird IPv4 `[PRIVATE_IP]`
|
||||
10:Old NetBird server stack stopped after successful migration.
|
||||
11:Host-level NetBird client left running for rollback/peer observation.
|
||||
33:NetBird:
|
||||
36:Compose bind changed only from old NetBird secondary public IP to new `[PRIVATE_IP]`.
|
||||
56:- Moldova synthetic check passes
|
||||
59:Important health discovery:
|
||||
60:- `/api/health` 404 on exact deployed version
|
||||
61:- `:9000/health` 503 in combined relay/server mode
|
||||
62:- first rollback was triggered by incorrectly assuming this endpoint must be healthy
|
||||
65:Backups on new server include migration/cutover snapshots such as:
|
||||
66:`/root/netbird-cutover-20260818T145807Z`
|
||||
69:- upgrade NetBird
|
||||
POLICY_DOC=/root/_2/workers2-cr0108-final-of79_umj/FINAL_HANDOFF/repo/kb/private/archive/2026-08-19/docs/15_CRITICAL_KEEP_RETIRE_DECISIONS.md SHA256=ad3ea70345063e8a13608a83acb4d6a1118533eb6acab22929e0bf1ac20c06af
|
||||
1:# CRITICAL KEEP / RETIRE DECISIONS
|
||||
3:This is a convenience matrix distilled from the historical handbook plus current state. A RETIRE label is not authorization to delete without fresh proof.
|
||||
11:| VM9130 edge-cold-standby | KEEP UNTIL DR PROOF | do not delete before timed VM130 restore |
|
||||
12:| VM150 Snikket | KEEP/CLOSED | do not reopen destructive rebuild without new defect |
|
||||
16:| VM180 cluster-admin | historical future RETIRE candidate | only after dependency/backup/monitoring cleanup |
|
||||
18:| CT200 skladchik-mod | historical future RETIRE candidate | preserve required data/monitoring first |
|
||||
19:| public edge01 | KEEP/CRITICAL | current git-read V3 and public edge duties |
|
||||
20:| Moldova relay | KEEP | independent relay/external health |
|
||||
21:| USA mail/NetBird | KEEP | infra mail/monitoring/no-PII |
|
||||
24:| pve01 18788 | KEEP NOW | reader-v3, usage proof before retirement |
|
||||
27:| Cloud.ru prepared bucket | KEEP PREPARED | real backup workload not yet active |
|
||||
POLICY_DOCS_END=1
|
||||
CONTROL_UNIT_REFERENCES_BEGIN=1
|
||||
CONTROL_UNIT_REFERENCES_END=1
|
||||
DECISION=PASS_BACKUP_1123_NO_EXTERNAL_PROVIDER_CONTROL_PATH_PROVEN
|
||||
NEXT_GATE=RETIRE_US_NETBIRD_TARGET_FROM_ACTIVE_BACKUPV2_POLICY_THEN_COMBINED_REPAIR
|
||||
TASK_RESULT=PASS_HOMELAB_BACKUP_MOLDOVA_CONTROL_RCA
|
||||
CHANGES_MADE_BY_1123=false
|
||||
PRODUCT_MUTATION_BY_1123=false
|
||||
VM_MUTATION_BY_1123=false
|
||||
CLOUD_MUTATION_BY_1123=false
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-MOLDOVA-CONTROL-RCA-1123R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_MOLDOVA_CONTROL_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+8
-8
File diff suppressed because one or more lines are too long
+960
-321
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user