From f76a9a3b6de7f9137e4d60ff0f1e03aa4c4d11ab Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 6 Oct 2026 08:45:21 +0000 Subject: [PATCH] runtime: publish SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1 --- ...61006-PVE01-GFS-VPN-FINAL-RCA-99696R1.json | 38 +++ ...261006-PVE01-GFS-VPN-FINAL-RCA-99696R1.txt | 216 +++++++++++++ runtime/latest.json | 16 +- runtime/latest.txt | 288 ++++++++++-------- 4 files changed, 429 insertions(+), 129 deletions(-) create mode 100644 runtime/history/SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1.json create mode 100644 runtime/history/SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1.txt diff --git a/runtime/history/SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1.json b/runtime/history/SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1.json new file mode 100644 index 00000000..2e25ece0 --- /dev/null +++ b/runtime/history/SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "pve01-gfs-vpn-final-rca", + "started_at_utc": "2026-10-06T08:45:16Z", + "finished_at_utc": "2026-10-06T08:45:20Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "9c86990b05592de487ff89f09077586a7f5c5c5c54fe3be43817e81dc4df0276", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "5de6a14be4a76e65a8cd5b1d048ed7f51c51c4d57732588bb9207e7de92f4d28", + "sanitized_output_sha256": "5de6a14be4a76e65a8cd5b1d048ed7f51c51c4d57732588bb9207e7de92f4d28", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "PVE01_GFS_VPN_FINAL_RCA=V1\nMODE=read-only\nNO_REBOOT=true\nNO_APT_UPDATE=true\nNO_PACKAGE_MUTATION=true\nNO_SERVICE_START_STOP_RESTART=true\nNO_RESET_FAILED=true\nNO_CONFIG_WRITE=true\nNO_FILE_DELETE=true\nNO_GUEST_MUTATION=true\nNO_BACKUP_EXECUTION=true\nNO_NETWORK_MUTATION=true\nNODE_HOSTNAME=pve01\nREC|FILE|BASENAME=targets.json|PRESENT=true|SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49\nREC|FILE|BASENAME=homelab-backup-gfs-policy|PRESENT=true|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94\nREC|FILE|BASENAME=gfs-retention.json|PRESENT=true|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d\nREC|FILE|BASENAME=homelab-app-cloud-quorum-queue|PRESENT=true|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\nREC|FILE|BASENAME=homelab-backup-retention-gc|PRESENT=true|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909\nREC|FILE|BASENAME=homelab-backup-target-registry|PRESENT=true|SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a\nREC|FILE|BASENAME=homelab-check-private-vpn-hosts|PRESENT=true|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82\nGFS_VALIDATE_CORRECT_ENV_RC=66\nREC|GFS_CORRECT_ERR|GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm160\nREC|GFS_SOURCE|LINE=1|#!PATH_REDACTED python3\nREC|GFS_SOURCE|LINE=2|import datetime as dt, json, os, pathlib, sys\nREC|GFS_SOURCE|LINE=3|CFG=pathlib.Path(os.environ.get(\"HOMELAB_BACKUP_GFS_POLICY\",\"PATH_REDACTED\nREC|GFS_SOURCE|LINE=4|REG=pathlib.Path(os.environ.get(\"HOMELAB_BACKUP_TARGET_REGISTRY\",\"PATH_REDACTED\nREC|GFS_SOURCE|LINE=5|def die(msg,rc=65):\nREC|GFS_SOURCE|LINE=6| print(f\"GFS_POLICY_ERROR={msg}\",file=sys.stderr); raise SystemExit(rc)\nREC|GFS_SOURCE|LINE=7|def load():\nREC|GFS_SOURCE|LINE=8| try: c=json.loads(CFG.read_text()); r=json.loads(REG.read_text())\nREC|GFS_SOURCE|LINE=9| except Exception: die(\"READ_OR_JSON_INVALID\",66)\nREC|GFS_SOURCE|LINE=10| if c.get(\"schema\")!=\"homelab-backup-gfs-policy-v1\" or c.get(\"policy_version\")!=1: die(\"SCHEMA_INVALID\",66)\nREC|GFS_SOURCE|LINE=11| if c.get(\"completed_periods_only\") is not True or c.get(\"weekly_buckets\")!=4 or c.get(\"monthly_buckets\")!=6 or c.get(\"capacity_guard_max_free_fraction_milli\")!=500 or c.get(\"capacity_guard_target_max_age_seconds\")!=172800 or c.get(\"capacity_guard_cloud_max_age_seconds\")!=14400: die(\"POLICY_INVALID\",66)\nREC|GFS_SOURCE|LINE=12| crit=c.get(\"critical_vm_logicals\")\nREC|GFS_SOURCE|LINE=13| if not isinstance(crit,list) or not crit or len(set(crit))!=len(crit): die(\"CRITICAL_SET_INVALID\",66)\nREC|GFS_SOURCE|LINE=14| rows={x.get(\"logical_id\"):x for x in r.get(\"targets\",[])}\nREC|GFS_SOURCE|LINE=15| for logical in crit:\nREC|GFS_SOURCE|LINE=16| rec=rows.get(logical)\nREC|GFS_SOURCE|LINE=17| if not rec or rec.get(\"kind\")!=\"VM\" or rec.get(\"lifecycle\")!=\"ACTIVE\": die(f\"CRITICAL_NOT_ACTIVE_VM_{logical}\",66)\nREC|GFS_SOURCE|LINE=18| return c,r\nREC|GFS_SOURCE|LINE=19|def parse_utc(s):\nREC|GFS_SOURCE|LINE=20| try:\nREC|GFS_SOURCE|LINE=21| if s.endswith('Z'): s=s[:-1]+'+00:00'\nREC|GFS_SOURCE|LINE=22| x=dt.datetime.fromisoformat(s)\nREC|GFS_SOURCE|LINE=23| if x.tzinfo is None: x=x.replace(tzinfo=dt.timezone.utc)\nREC|GFS_SOURCE|LINE=24| return x.astimezone(dt.timezone.utc)\nREC|GFS_SOURCE|LINE=25| except Exception: die(\"SEALED_TIME_INVALID\",66)\nREC|GFS_SOURCE|LINE=26|def now_utc():\nREC|GFS_SOURCE|LINE=27| v=os.environ.get(\"HOMELAB_GFS_NOW_UTC\")\nREC|GFS_SOURCE|LINE=28| return parse_utc(v) if v else dt.datetime.now(dt.timezone.utc)\nREC|GFS_SOURCE|LINE=29|def select(logical,c):\nREC|GFS_SOURCE|LINE=30| if logical not in c[\"critical_vm_logicals\"]: return\nREC|GFS_SOURCE|LINE=31| now=now_utc(); cur_iso=now.isocalendar(); cur_week=(cur_iso.year,cur_iso.week); cur_month=(now.year,now.month)\nREC|GFS_SOURCE|LINE=32| weekly=[]; monthly=[]; seen_w=set(); seen_m=set()\nREC|GFS_SOURCE|LINE=33| for raw in sys.stdin:\nREC|GFS_SOURCE|LINE=34| raw=raw.rstrip(\"\\n\")\nREC|GFS_SOURCE|LINE=35| if not raw: continue\nREC|GFS_SOURCE|LINE=36| parts=raw.split(\"\\t\")\nREC|GFS_SOURCE|LINE=37| if len(parts)!=2: die(\"SELECT_INPUT_INVALID\",66)\nREC|GFS_SOURCE|LINE=38| gen,sealed=parts; when=parse_utc(sealed); iso=when.isocalendar(); wb=(iso.year,iso.week); mb=(when.year,when.month)\nREC|GFS_SOURCE|LINE=39| if wb!=cur_week and len(weekly)=c[\"weekly_buckets\"] and len(monthly)>=c[\"monthly_buckets\"]: break\nREC|GFS_SOURCE|LINE=44| for gen,b in weekly: print(f\"WEEKLY\\t{gen}\\t{b}\")\nREC|GFS_SOURCE|LINE=45| for gen,b in monthly: print(f\"MONTHLY\\t{gen}\\t{b}\")\nREC|GFS_SOURCE|LINE=46|def main():\nREC|GFS_SOURCE|LINE=47| c,_=load(); args=sys.argv[1:]\nREC|GFS_SOURCE|LINE=48| if args==[\"validate\"]: print(\"GFS_POLICY=OK\"); return\nREC|GFS_SOURCE|LINE=49| if args==[\"critical-logicals\"]:\nREC|GFS_SOURCE|LINE=50| print(\"\\n\".join(c[\"critical_vm_logicals\"])); return\nREC|GFS_SOURCE|LINE=51| if args==[\"weekly-buckets\"]: print(c[\"weekly_buckets\"]); return\nREC|GFS_SOURCE|LINE=52| if args==[\"monthly-buckets\"]: print(c[\"monthly_buckets\"]); return\nREC|GFS_SOURCE|LINE=53| if args==[\"capacity-guard-milli\"]: print(c[\"capacity_guard_max_free_fraction_milli\"]); return\nREC|GFS_SOURCE|LINE=54| if len(args)==2 and args[0]==\"is-critical\": raise SystemExit(0 if args[1] in c[\"critical_vm_logicals\"] else 1)\nREC|GFS_SOURCE|LINE=55| if len(args)==2 and args[0]==\"select\": select(args[1],c); return\nREC|GFS_SOURCE|LINE=56| die(\"USAGE\",64)\nREC|GFS_SOURCE|LINE=57|if __name__==\"__main__\": main()\nGFS_POLICY_SCHEMA=homelab-backup-gfs-policy-v1\nGFS_POLICY_VERSION=1\nGFS_POLICY_KEYS=capacity_guard_cloud_max_age_seconds,capacity_guard_max_free_fraction_milli,capacity_guard_target_max_age_seconds,completed_periods_only,critical_vm_logicals,monthly_buckets,policy_version,schema,weekly_buckets\nREC|GFS_POLICY_SCALAR|KEY=capacity_guard_cloud_max_age_seconds|VALUE=14400\nREC|GFS_POLICY_SCALAR|KEY=capacity_guard_max_free_fraction_milli|VALUE=500\nREC|GFS_POLICY_SCALAR|KEY=capacity_guard_target_max_age_seconds|VALUE=172800\nREC|GFS_POLICY_SCALAR|KEY=completed_periods_only|VALUE=True\nREC|GFS_POLICY_LIST|KEY=critical_vm_logicals|COUNT=8|VALUES=vm110,vm111,vm112,vm113,vm130,vm150,vm160,vm170\nREC|GFS_POLICY_SCALAR|KEY=monthly_buckets|VALUE=6\nREC|GFS_POLICY_SCALAR|KEY=policy_version|VALUE=1\nREC|GFS_POLICY_SCALAR|KEY=schema|VALUE=homelab-backup-gfs-policy-v1\nREC|GFS_POLICY_SCALAR|KEY=weekly_buckets|VALUE=4\nUNIFIED_TOTAL=21\nUNIFIED_VM_ACTIVE=10\nUNIFIED_VM_RETIRED_PRESERVE=3\nUNIFIED_APP_ACTIVE=4\nUNIFIED_APP_RETIRED_PRESERVE=3\nUNIFIED_CONTROL_ACTIVE=1\nUNIFIED_CONTROL_RETIRED_PRESERVE=0\nUNIFIED_LOGICAL_IDS=app-filebrowser,app-hbarhub,app-immich,app-memos,app-scrutiny,app-skladchik-moderator-assistant,app-skladchik-reports-monitor,control-bootstrap-v2,vm110,vm111,vm112,vm113,vm130,vm150,vm160,vm170,vm171,vm180,vm190,vm200,vm9130\nAPPQ_DIRECT_RC=65\nREC|APPQ_ERR|APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\nRET_DIRECT_RC=65\nREC|RET_ERR|GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\nVPN_LITERAL_NAME_COUNT=5\nREC|VPN_NAME|IDX=1|NAME_SHA=1588bd492f97c578a05fad877bb1c11e6738df68992b121042a13fb2f17329c0\nREC|VPN_NAME|IDX=2|NAME_SHA=bc1585521e17de78e3a7bd903ca2427f5941f3bc1276f324fafe794ea8eabdbc\nREC|VPN_NAME|IDX=3|NAME_SHA=968bf918c9b60ba4ae098281e24580e629b4ac33446a76981ed53c671db1e8c8\nREC|VPN_NAME|IDX=4|NAME_SHA=9f64956d61ca126d479e1ca923eed5c6994916944111962c6e2f3ad1d2538dcd\nREC|VPN_NAME|IDX=5|NAME_SHA=95f56744bb158973029a5113cf2904b44a928103143dfab662552f9beb28b9d9\nREC|VPN_SOURCE_ALL|LINE=1|#!PATH_REDACTED bash\nREC|VPN_SOURCE_ALL|LINE=2|set -u\nREC|VPN_SOURCE_ALL|LINE=4|H=\"PATH_REDACTED\nREC|VPN_SOURCE_ALL|LINE=5|HOSTS=\"\nREC|VPN_SOURCE_ALL|LINE=6|VPN_HOST_REDACTED\nREC|VPN_SOURCE_ALL|LINE=7|VPN_HOST_REDACTED\nREC|VPN_SOURCE_ALL|LINE=8|VPN_HOST_REDACTED\nREC|VPN_SOURCE_ALL|LINE=9|VPN_HOST_REDACTED\nREC|VPN_SOURCE_ALL|LINE=10|VPN_HOST_REDACTED\nREC|VPN_SOURCE_ALL|LINE=11|\"\nREC|VPN_SOURCE_ALL|LINE=13|mkdir -p PATH_REDACTED\nREC|VPN_SOURCE_ALL|LINE=14|tmp=\"$(mktemp)\"\nREC|VPN_SOURCE_ALL|LINE=15|trap 'rm -f \"$tmp\"' EXIT\nREC|VPN_SOURCE_ALL|LINE=17|bad=0\nREC|VPN_SOURCE_ALL|LINE=18|for h in $HOSTS; do\nREC|VPN_SOURCE_ALL|LINE=19|ip=\"$(getent hosts \"$h\" 2>PATH_REDACTED | awk '{print $1}' | head -n1 || true)\"\nREC|VPN_SOURCE_ALL|LINE=20|http=\"000error\"\nREC|VPN_SOURCE_ALL|LINE=21|https=\"000error\"\nREC|VPN_SOURCE_ALL|LINE=22|location=\"none\"\nREC|VPN_SOURCE_ALL|LINE=23|cert_ok=\"yes\"\nREC|VPN_SOURCE_ALL|LINE=25|if [ -n \"$ip\" ]; then\nREC|VPN_SOURCE_ALL|LINE=26|http_code=\"$(curl -4 -ksS -o PATH_REDACTED -D PATH_REDACTED -w '%{http_code}' --max-time 10 \"http:PATH_REDACTED 2>PATH_REDACTED || true)\"\nREC|VPN_SOURCE_ALL|LINE=27|https_code=\"$(curl -4 -ksS -o PATH_REDACTED -D PATH_REDACTED -w '%{http_code}' --max-time 10 \"https:PATH_REDACTED 2>PATH_REDACTED || true)\"\nREC|VPN_SOURCE_ALL|LINE=28|[ -n \"$http_code\" ] && http=\"$http_code\"\nREC|VPN_SOURCE_ALL|LINE=29|[ -n \"$https_code\" ] && https=\"$https_code\"\nREC|VPN_SOURCE_ALL|LINE=30|location=\"$(awk 'tolower($1)==\"location:\"{print $2; exit}' PATH_REDACTED 2>PATH_REDACTED | tr -d '\\r' || true)\"\nREC|VPN_SOURCE_ALL|LINE=31|[ -n \"$location\" ] || location=\"none\"\nREC|VPN_SOURCE_ALL|LINE=32|fi\nREC|VPN_SOURCE_ALL|LINE=34|case \"$https\" in\nREC|VPN_SOURCE_ALL|LINE=35|200|301|302|401|403) ;;\nREC|VPN_SOURCE_ALL|LINE=36|*) bad=$((bad+1)) ;;\nREC|VPN_SOURCE_ALL|LINE=37|esac\nREC|VPN_SOURCE_ALL|LINE=39|printf 'HOST|%s|fqdn=%s|dns=%s|http=%s|https=%s|location=%s|cert_ok=%s\\n' \\\nREC|VPN_SOURCE_ALL|LINE=40|\"${h%%.*}\" \"$h\" \"${ip:-NONE}\" \"$http\" \"$https\" \"$location\" \"$cert_ok\" >> \"$tmp\"\nREC|VPN_SOURCE_ALL|LINE=41|done\nREC|VPN_SOURCE_ALL|LINE=43|if [ \"$bad\" = \"0\" ]; then\nREC|VPN_SOURCE_ALL|LINE=44|status=OK\nREC|VPN_SOURCE_ALL|LINE=45|else\nREC|VPN_SOURCE_ALL|LINE=46|status=ERROR\nREC|VPN_SOURCE_ALL|LINE=47|fi\nREC|VPN_SOURCE_ALL|LINE=49|{\nREC|VPN_SOURCE_ALL|LINE=50|cat \"$tmp\"\nREC|VPN_SOURCE_ALL|LINE=51|printf 'STATUS=%s\\n' \"$status\"\nREC|VPN_SOURCE_ALL|LINE=52|} > \"$H\"\nREC|VPN_SOURCE_ALL|LINE=54|cat \"$H\"\nREC|VPN_SOURCE_ALL|LINE=55|[ \"$status\" = \"OK\" ]\nREC|VPN_RESOLUTION|IDX=1|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true\nREC|VPN_RESOLUTION|IDX=2|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true\nREC|VPN_RESOLUTION|IDX=3|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true\nREC|VPN_RESOLUTION|IDX=4|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true\nREC|VPN_RESOLUTION|IDX=5|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true\nREC|RESOLVER|HOST=pve01|RESOLV_IS_SYMLINK=false\nREC|RESOLVER|HOST=pve01|RESOLV_LINK_BASENAME=NONE\nREC|RESOLVER|HOST=pve01|RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74\nREC|RESOLVER|HOST=pve01|NAMESERVER_COUNT=1\nREC|RESOLVER|HOST=pve01|SEARCH_LINE_COUNT=1\nREC|RESOLVER|HOST=pve01|REC|RESOLVER_NS_SHA|a6f12182e30cba92f9e91dc59ae899c758ecc34e1c3b9b06404eb741a56916ac\nREC|RESOLVER|HOST=pve02|RESOLV_IS_SYMLINK=false\nREC|RESOLVER|HOST=pve02|RESOLV_LINK_BASENAME=NONE\nREC|RESOLVER|HOST=pve02|RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f\nREC|RESOLVER|HOST=pve02|NAMESERVER_COUNT=1\nREC|RESOLVER|HOST=pve02|SEARCH_LINE_COUNT=1\nREC|RESOLVER|HOST=pve02|REC|RESOLVER_NS_SHA|3e731e83b523045ce83835503795f722459caf77bb2f8c0afcfaf75971b2bd78\nREC|PVE01_NS_CONFIG_REF|BASENAME=interfaces|PARENT=network|MATCH_COUNT=0|SHA=8558349d08d267b08a8abb490a591b5d4b7d539d3cc515b08f3165f3af0b8898\nFAILED_UNIT_COUNT_NOW=2\nREC|FAILED_UNIT_NOW|homelab-backup-audit.service\nREC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service\nDECISION=PASS_PVE01_GFS_VPN_FINAL_RCA\nNEXT_GATE=PVE01_HEALTH_HELPERS_AND_VPN_REPAIR\nTASK_RESULT=PVE01_GFS_VPN_FINAL_RCA\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1.txt b/runtime/history/SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1.txt new file mode 100644 index 00000000..05005c18 --- /dev/null +++ b/runtime/history/SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1.txt @@ -0,0 +1,216 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=pve01-gfs-vpn-final-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=9c86990b05592de487ff89f09077586a7f5c5c5c54fe3be43817e81dc4df0276 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=5de6a14be4a76e65a8cd5b1d048ed7f51c51c4d57732588bb9207e7de92f4d28 +SANITIZED_OUTPUT_SHA256=5de6a14be4a76e65a8cd5b1d048ed7f51c51c4d57732588bb9207e7de92f4d28 +OUTPUT_BEGIN +PVE01_GFS_VPN_FINAL_RCA=V1 +MODE=read-only +NO_REBOOT=true +NO_APT_UPDATE=true +NO_PACKAGE_MUTATION=true +NO_SERVICE_START_STOP_RESTART=true +NO_RESET_FAILED=true +NO_CONFIG_WRITE=true +NO_FILE_DELETE=true +NO_GUEST_MUTATION=true +NO_BACKUP_EXECUTION=true +NO_NETWORK_MUTATION=true +NODE_HOSTNAME=pve01 +REC|FILE|BASENAME=targets.json|PRESENT=true|SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49 +REC|FILE|BASENAME=homelab-backup-gfs-policy|PRESENT=true|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94 +REC|FILE|BASENAME=gfs-retention.json|PRESENT=true|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d +REC|FILE|BASENAME=homelab-app-cloud-quorum-queue|PRESENT=true|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650 +REC|FILE|BASENAME=homelab-backup-retention-gc|PRESENT=true|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909 +REC|FILE|BASENAME=homelab-backup-target-registry|PRESENT=true|SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a +REC|FILE|BASENAME=homelab-check-private-vpn-hosts|PRESENT=true|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82 +GFS_VALIDATE_CORRECT_ENV_RC=66 +REC|GFS_CORRECT_ERR|GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm160 +REC|GFS_SOURCE|LINE=1|#!PATH_REDACTED python3 +REC|GFS_SOURCE|LINE=2|import datetime as dt, json, os, pathlib, sys +REC|GFS_SOURCE|LINE=3|CFG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_GFS_POLICY","PATH_REDACTED +REC|GFS_SOURCE|LINE=4|REG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_TARGET_REGISTRY","PATH_REDACTED +REC|GFS_SOURCE|LINE=5|def die(msg,rc=65): +REC|GFS_SOURCE|LINE=6| print(f"GFS_POLICY_ERROR={msg}",file=sys.stderr); raise SystemExit(rc) +REC|GFS_SOURCE|LINE=7|def load(): +REC|GFS_SOURCE|LINE=8| try: c=json.loads(CFG.read_text()); r=json.loads(REG.read_text()) +REC|GFS_SOURCE|LINE=9| except Exception: die("READ_OR_JSON_INVALID",66) +REC|GFS_SOURCE|LINE=10| if c.get("schema")!="homelab-backup-gfs-policy-v1" or c.get("policy_version")!=1: die("SCHEMA_INVALID",66) +REC|GFS_SOURCE|LINE=11| if c.get("completed_periods_only") is not True or c.get("weekly_buckets")!=4 or c.get("monthly_buckets")!=6 or c.get("capacity_guard_max_free_fraction_milli")!=500 or c.get("capacity_guard_target_max_age_seconds")!=172800 or c.get("capacity_guard_cloud_max_age_seconds")!=14400: die("POLICY_INVALID",66) +REC|GFS_SOURCE|LINE=12| crit=c.get("critical_vm_logicals") +REC|GFS_SOURCE|LINE=13| if not isinstance(crit,list) or not crit or len(set(crit))!=len(crit): die("CRITICAL_SET_INVALID",66) +REC|GFS_SOURCE|LINE=14| rows={x.get("logical_id"):x for x in r.get("targets",[])} +REC|GFS_SOURCE|LINE=15| for logical in crit: +REC|GFS_SOURCE|LINE=16| rec=rows.get(logical) +REC|GFS_SOURCE|LINE=17| if not rec or rec.get("kind")!="VM" or rec.get("lifecycle")!="ACTIVE": die(f"CRITICAL_NOT_ACTIVE_VM_{logical}",66) +REC|GFS_SOURCE|LINE=18| return c,r +REC|GFS_SOURCE|LINE=19|def parse_utc(s): +REC|GFS_SOURCE|LINE=20| try: +REC|GFS_SOURCE|LINE=21| if s.endswith('Z'): s=s[:-1]+'+00:00' +REC|GFS_SOURCE|LINE=22| x=dt.datetime.fromisoformat(s) +REC|GFS_SOURCE|LINE=23| if x.tzinfo is None: x=x.replace(tzinfo=dt.timezone.utc) +REC|GFS_SOURCE|LINE=24| return x.astimezone(dt.timezone.utc) +REC|GFS_SOURCE|LINE=25| except Exception: die("SEALED_TIME_INVALID",66) +REC|GFS_SOURCE|LINE=26|def now_utc(): +REC|GFS_SOURCE|LINE=27| v=os.environ.get("HOMELAB_GFS_NOW_UTC") +REC|GFS_SOURCE|LINE=28| return parse_utc(v) if v else dt.datetime.now(dt.timezone.utc) +REC|GFS_SOURCE|LINE=29|def select(logical,c): +REC|GFS_SOURCE|LINE=30| if logical not in c["critical_vm_logicals"]: return +REC|GFS_SOURCE|LINE=31| now=now_utc(); cur_iso=now.isocalendar(); cur_week=(cur_iso.year,cur_iso.week); cur_month=(now.year,now.month) +REC|GFS_SOURCE|LINE=32| weekly=[]; monthly=[]; seen_w=set(); seen_m=set() +REC|GFS_SOURCE|LINE=33| for raw in sys.stdin: +REC|GFS_SOURCE|LINE=34| raw=raw.rstrip("\n") +REC|GFS_SOURCE|LINE=35| if not raw: continue +REC|GFS_SOURCE|LINE=36| parts=raw.split("\t") +REC|GFS_SOURCE|LINE=37| if len(parts)!=2: die("SELECT_INPUT_INVALID",66) +REC|GFS_SOURCE|LINE=38| gen,sealed=parts; when=parse_utc(sealed); iso=when.isocalendar(); wb=(iso.year,iso.week); mb=(when.year,when.month) +REC|GFS_SOURCE|LINE=39| if wb!=cur_week and len(weekly)=c["weekly_buckets"] and len(monthly)>=c["monthly_buckets"]: break +REC|GFS_SOURCE|LINE=44| for gen,b in weekly: print(f"WEEKLY\t{gen}\t{b}") +REC|GFS_SOURCE|LINE=45| for gen,b in monthly: print(f"MONTHLY\t{gen}\t{b}") +REC|GFS_SOURCE|LINE=46|def main(): +REC|GFS_SOURCE|LINE=47| c,_=load(); args=sys.argv[1:] +REC|GFS_SOURCE|LINE=48| if args==["validate"]: print("GFS_POLICY=OK"); return +REC|GFS_SOURCE|LINE=49| if args==["critical-logicals"]: +REC|GFS_SOURCE|LINE=50| print("\n".join(c["critical_vm_logicals"])); return +REC|GFS_SOURCE|LINE=51| if args==["weekly-buckets"]: print(c["weekly_buckets"]); return +REC|GFS_SOURCE|LINE=52| if args==["monthly-buckets"]: print(c["monthly_buckets"]); return +REC|GFS_SOURCE|LINE=53| if args==["capacity-guard-milli"]: print(c["capacity_guard_max_free_fraction_milli"]); return +REC|GFS_SOURCE|LINE=54| if len(args)==2 and args[0]=="is-critical": raise SystemExit(0 if args[1] in c["critical_vm_logicals"] else 1) +REC|GFS_SOURCE|LINE=55| if len(args)==2 and args[0]=="select": select(args[1],c); return +REC|GFS_SOURCE|LINE=56| die("USAGE",64) +REC|GFS_SOURCE|LINE=57|if __name__=="__main__": main() +GFS_POLICY_SCHEMA=homelab-backup-gfs-policy-v1 +GFS_POLICY_VERSION=1 +GFS_POLICY_KEYS=capacity_guard_cloud_max_age_seconds,capacity_guard_max_free_fraction_milli,capacity_guard_target_max_age_seconds,completed_periods_only,critical_vm_logicals,monthly_buckets,policy_version,schema,weekly_buckets +REC|GFS_POLICY_SCALAR|KEY=capacity_guard_cloud_max_age_seconds|VALUE=14400 +REC|GFS_POLICY_SCALAR|KEY=capacity_guard_max_free_fraction_milli|VALUE=500 +REC|GFS_POLICY_SCALAR|KEY=capacity_guard_target_max_age_seconds|VALUE=172800 +REC|GFS_POLICY_SCALAR|KEY=completed_periods_only|VALUE=True +REC|GFS_POLICY_LIST|KEY=critical_vm_logicals|COUNT=8|VALUES=vm110,vm111,vm112,vm113,vm130,vm150,vm160,vm170 +REC|GFS_POLICY_SCALAR|KEY=monthly_buckets|VALUE=6 +REC|GFS_POLICY_SCALAR|KEY=policy_version|VALUE=1 +REC|GFS_POLICY_SCALAR|KEY=schema|VALUE=homelab-backup-gfs-policy-v1 +REC|GFS_POLICY_SCALAR|KEY=weekly_buckets|VALUE=4 +UNIFIED_TOTAL=21 +UNIFIED_VM_ACTIVE=10 +UNIFIED_VM_RETIRED_PRESERVE=3 +UNIFIED_APP_ACTIVE=4 +UNIFIED_APP_RETIRED_PRESERVE=3 +UNIFIED_CONTROL_ACTIVE=1 +UNIFIED_CONTROL_RETIRED_PRESERVE=0 +UNIFIED_LOGICAL_IDS=app-filebrowser,app-hbarhub,app-immich,app-memos,app-scrutiny,app-skladchik-moderator-assistant,app-skladchik-reports-monitor,control-bootstrap-v2,vm110,vm111,vm112,vm113,vm130,vm150,vm160,vm170,vm171,vm180,vm190,vm200,vm9130 +APPQ_DIRECT_RC=65 +REC|APPQ_ERR|APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT +RET_DIRECT_RC=65 +REC|RET_ERR|GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT +VPN_LITERAL_NAME_COUNT=5 +REC|VPN_NAME|IDX=1|NAME_SHA=1588bd492f97c578a05fad877bb1c11e6738df68992b121042a13fb2f17329c0 +REC|VPN_NAME|IDX=2|NAME_SHA=bc1585521e17de78e3a7bd903ca2427f5941f3bc1276f324fafe794ea8eabdbc +REC|VPN_NAME|IDX=3|NAME_SHA=968bf918c9b60ba4ae098281e24580e629b4ac33446a76981ed53c671db1e8c8 +REC|VPN_NAME|IDX=4|NAME_SHA=9f64956d61ca126d479e1ca923eed5c6994916944111962c6e2f3ad1d2538dcd +REC|VPN_NAME|IDX=5|NAME_SHA=95f56744bb158973029a5113cf2904b44a928103143dfab662552f9beb28b9d9 +REC|VPN_SOURCE_ALL|LINE=1|#!PATH_REDACTED bash +REC|VPN_SOURCE_ALL|LINE=2|set -u +REC|VPN_SOURCE_ALL|LINE=4|H="PATH_REDACTED +REC|VPN_SOURCE_ALL|LINE=5|HOSTS=" +REC|VPN_SOURCE_ALL|LINE=6|VPN_HOST_REDACTED +REC|VPN_SOURCE_ALL|LINE=7|VPN_HOST_REDACTED +REC|VPN_SOURCE_ALL|LINE=8|VPN_HOST_REDACTED +REC|VPN_SOURCE_ALL|LINE=9|VPN_HOST_REDACTED +REC|VPN_SOURCE_ALL|LINE=10|VPN_HOST_REDACTED +REC|VPN_SOURCE_ALL|LINE=11|" +REC|VPN_SOURCE_ALL|LINE=13|mkdir -p PATH_REDACTED +REC|VPN_SOURCE_ALL|LINE=14|tmp="$(mktemp)" +REC|VPN_SOURCE_ALL|LINE=15|trap 'rm -f "$tmp"' EXIT +REC|VPN_SOURCE_ALL|LINE=17|bad=0 +REC|VPN_SOURCE_ALL|LINE=18|for h in $HOSTS; do +REC|VPN_SOURCE_ALL|LINE=19|ip="$(getent hosts "$h" 2>PATH_REDACTED | awk '{print $1}' | head -n1 || true)" +REC|VPN_SOURCE_ALL|LINE=20|http="000error" +REC|VPN_SOURCE_ALL|LINE=21|https="000error" +REC|VPN_SOURCE_ALL|LINE=22|location="none" +REC|VPN_SOURCE_ALL|LINE=23|cert_ok="yes" +REC|VPN_SOURCE_ALL|LINE=25|if [ -n "$ip" ]; then +REC|VPN_SOURCE_ALL|LINE=26|http_code="$(curl -4 -ksS -o PATH_REDACTED -D PATH_REDACTED -w '%{http_code}' --max-time 10 "http:PATH_REDACTED 2>PATH_REDACTED || true)" +REC|VPN_SOURCE_ALL|LINE=27|https_code="$(curl -4 -ksS -o PATH_REDACTED -D PATH_REDACTED -w '%{http_code}' --max-time 10 "https:PATH_REDACTED 2>PATH_REDACTED || true)" +REC|VPN_SOURCE_ALL|LINE=28|[ -n "$http_code" ] && http="$http_code" +REC|VPN_SOURCE_ALL|LINE=29|[ -n "$https_code" ] && https="$https_code" +REC|VPN_SOURCE_ALL|LINE=30|location="$(awk 'tolower($1)=="location:"{print $2; exit}' PATH_REDACTED 2>PATH_REDACTED | tr -d '\r' || true)" +REC|VPN_SOURCE_ALL|LINE=31|[ -n "$location" ] || location="none" +REC|VPN_SOURCE_ALL|LINE=32|fi +REC|VPN_SOURCE_ALL|LINE=34|case "$https" in +REC|VPN_SOURCE_ALL|LINE=35|200|301|302|401|403) ;; +REC|VPN_SOURCE_ALL|LINE=36|*) bad=$((bad+1)) ;; +REC|VPN_SOURCE_ALL|LINE=37|esac +REC|VPN_SOURCE_ALL|LINE=39|printf 'HOST|%s|fqdn=%s|dns=%s|http=%s|https=%s|location=%s|cert_ok=%s\n' \ +REC|VPN_SOURCE_ALL|LINE=40|"${h%%.*}" "$h" "${ip:-NONE}" "$http" "$https" "$location" "$cert_ok" >> "$tmp" +REC|VPN_SOURCE_ALL|LINE=41|done +REC|VPN_SOURCE_ALL|LINE=43|if [ "$bad" = "0" ]; then +REC|VPN_SOURCE_ALL|LINE=44|status=OK +REC|VPN_SOURCE_ALL|LINE=45|else +REC|VPN_SOURCE_ALL|LINE=46|status=ERROR +REC|VPN_SOURCE_ALL|LINE=47|fi +REC|VPN_SOURCE_ALL|LINE=49|{ +REC|VPN_SOURCE_ALL|LINE=50|cat "$tmp" +REC|VPN_SOURCE_ALL|LINE=51|printf 'STATUS=%s\n' "$status" +REC|VPN_SOURCE_ALL|LINE=52|} > "$H" +REC|VPN_SOURCE_ALL|LINE=54|cat "$H" +REC|VPN_SOURCE_ALL|LINE=55|[ "$status" = "OK" ] +REC|VPN_RESOLUTION|IDX=1|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true +REC|VPN_RESOLUTION|IDX=2|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true +REC|VPN_RESOLUTION|IDX=3|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true +REC|VPN_RESOLUTION|IDX=4|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true +REC|VPN_RESOLUTION|IDX=5|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true +REC|RESOLVER|HOST=pve01|RESOLV_IS_SYMLINK=false +REC|RESOLVER|HOST=pve01|RESOLV_LINK_BASENAME=NONE +REC|RESOLVER|HOST=pve01|RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74 +REC|RESOLVER|HOST=pve01|NAMESERVER_COUNT=1 +REC|RESOLVER|HOST=pve01|SEARCH_LINE_COUNT=1 +REC|RESOLVER|HOST=pve01|REC|RESOLVER_NS_SHA|a6f12182e30cba92f9e91dc59ae899c758ecc34e1c3b9b06404eb741a56916ac +REC|RESOLVER|HOST=pve02|RESOLV_IS_SYMLINK=false +REC|RESOLVER|HOST=pve02|RESOLV_LINK_BASENAME=NONE +REC|RESOLVER|HOST=pve02|RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f +REC|RESOLVER|HOST=pve02|NAMESERVER_COUNT=1 +REC|RESOLVER|HOST=pve02|SEARCH_LINE_COUNT=1 +REC|RESOLVER|HOST=pve02|REC|RESOLVER_NS_SHA|3e731e83b523045ce83835503795f722459caf77bb2f8c0afcfaf75971b2bd78 +REC|PVE01_NS_CONFIG_REF|BASENAME=interfaces|PARENT=network|MATCH_COUNT=0|SHA=8558349d08d267b08a8abb490a591b5d4b7d539d3cc515b08f3165f3af0b8898 +FAILED_UNIT_COUNT_NOW=2 +REC|FAILED_UNIT_NOW|homelab-backup-audit.service +REC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service +DECISION=PASS_PVE01_GFS_VPN_FINAL_RCA +NEXT_GATE=PVE01_HEALTH_HELPERS_AND_VPN_REPAIR +TASK_RESULT=PVE01_GFS_VPN_FINAL_RCA +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 878c0bdb..2e25ece0 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1", + "command_id": "SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "pve01-audit-helpers-vpn-exact-rca", - "started_at_utc": "2026-10-06T08:40:50Z", - "finished_at_utc": "2026-10-06T08:40:50Z", + "component": "pve01-gfs-vpn-final-rca", + "started_at_utc": "2026-10-06T08:45:16Z", + "finished_at_utc": "2026-10-06T08:45:20Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "5cf24ab1bb98675b9bfe49e8bdc859479f976f0b83c5d2d1a01f49508b38be64", + "command_sha256": "9c86990b05592de487ff89f09077586a7f5c5c5c54fe3be43817e81dc4df0276", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd", - "sanitized_output_sha256": "abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd", + "raw_evidence_sha256": "5de6a14be4a76e65a8cd5b1d048ed7f51c51c4d57732588bb9207e7de92f4d28", + "sanitized_output_sha256": "5de6a14be4a76e65a8cd5b1d048ed7f51c51c4d57732588bb9207e7de92f4d28", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "PVE01_AUDIT_HELPERS_VPN_EXACT_RCA=V1\nMODE=read-only\nNO_REBOOT=true\nNO_APT_UPDATE=true\nNO_PACKAGE_MUTATION=true\nNO_SERVICE_START_STOP_RESTART=true\nNO_RESET_FAILED=true\nNO_CONFIG_WRITE=true\nNO_FILE_DELETE=true\nNO_GUEST_MUTATION=true\nNO_BACKUP_EXECUTION=true\nNO_NETWORK_MUTATION=true\nNODE_HOSTNAME=pve01\nAUDIT_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3\nREGISTRY_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49\nHELPER_SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a\nAUDIT_CONST_APPQ_PRESENT=true\nAUDIT_CONST_APPQ_BASENAME=homelab-app-cloud-quorum-queue\nAUDIT_CONST_APPQ_PARENT=sbin\nAUDIT_CONST_APPQ_EXISTS=true\nAUDIT_CONST_APPQ_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\nAUDIT_CONST_APPQ_PATH_SHA=75315ca46bfa3b06e14300fe6c03852aaf085c4d2843247dd39fd7e8b53f7800\nAUDIT_CONST_GFSH_PRESENT=true\nAUDIT_CONST_GFSH_BASENAME=homelab-backup-gfs-policy\nAUDIT_CONST_GFSH_PARENT=libexec\nAUDIT_CONST_GFSH_EXISTS=true\nAUDIT_CONST_GFSH_SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94\nAUDIT_CONST_GFSH_PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9\nAUDIT_CONST_GFSP_PRESENT=true\nAUDIT_CONST_GFSP_BASENAME=gfs-retention.json\nAUDIT_CONST_GFSP_PARENT=homelab-backup\nAUDIT_CONST_GFSP_EXISTS=true\nAUDIT_CONST_GFSP_SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d\nAUDIT_CONST_GFSP_PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d\nAUDIT_CONST_REG_PRESENT=false\nAUDIT_CONST_RET_PRESENT=true\nAUDIT_CONST_RET_BASENAME=homelab-backup-retention-gc\nAUDIT_CONST_RET_PARENT=sbin\nAUDIT_CONST_RET_EXISTS=true\nAUDIT_CONST_RET_SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909\nAUDIT_CONST_RET_PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a\nREC|AUDIT_HELPER_FILE|LABEL=APPQ|BASENAME=homelab-app-cloud-quorum-queue|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=6|REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=7|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=8|REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=11|[[ -x \"$REGISTRY_HELPER\" && ! -L \"$REGISTRY_HELPER\" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_MISSING\\n' >&2; exit 69; }\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=12|[[ \"$(sha256sum \"$REGISTRY_HELPER\"|awk '{print $1}')\" == \"$REGISTRY_HELPER_SHA_EXPECTED\" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=19|[[ \"${#ALL_APPS[@]}\" == 4 && \"${#MIGRATE_APPS[@]}\" == 4 ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_COUNT_INVALID\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_FILE|LABEL=GFSH|BASENAME=homelab-backup-gfs-policy|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=3|CFG=pathlib.Path(os.environ.get(\"HOMELAB_BACKUP_GFS_POLICY\",\"PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=4|REG=pathlib.Path(os.environ.get(\"HOMELAB_BACKUP_TARGET_REGISTRY\",\"PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=6|print(f\"GFS_POLICY_ERROR={msg}\",file=sys.stderr); raise SystemExit(rc)\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=10|if c.get(\"schema\")!=\"homelab-backup-gfs-policy-v1\" or c.get(\"policy_version\")!=1: die(\"SCHEMA_INVALID\",66)\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=14|rows={x.get(\"logical_id\"):x for x in r.get(\"targets\",[])}\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=48|if args==[\"validate\"]: print(\"GFS_POLICY=OK\"); return\nREC|AUDIT_HELPER_FILE|LABEL=GFSP|BASENAME=gfs-retention.json|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSP|LINE=18|\"schema\": \"homelab-backup-gfs-policy-v1\",\nREC|AUDIT_HELPER_FILE|LABEL=RET|BASENAME=homelab-backup-retention-gc|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=8|RETENTION_ROOT=PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=21|REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=22|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=23|GFS_HELPER=\"${HOMELAB_BACKUP_GFS_POLICY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=25|GFS_POLICY=\"${HOMELAB_BACKUP_GFS_POLICY:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=27|[[ -x \"$REGISTRY_HELPER\" && ! -L \"$REGISTRY_HELPER\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_MISSING\\n' >&2; exit 69; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=28|[[ \"$(sha256sum \"$REGISTRY_HELPER\"|awk '{print $1}')\" == \"$REGISTRY_HELPER_SHA_EXPECTED\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=29|\"$REGISTRY_HELPER\" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_INVALID\\n' >&2; exit 66; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=30|[[ -x \"$GFS_HELPER\" && ! -L \"$GFS_HELPER\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_MISSING\\n' >&2; exit 69; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=31|[[ \"$(sha256sum \"$GFS_HELPER\"|awk '{print $1}')\" == \"$GFS_HELPER_SHA_EXPECTED\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=32|HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID\\n' >&2; exit 66; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=33|GFS_WEEKLY=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" weekly-buckets)\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=34|GFS_MONTHLY=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" monthly-buckets)\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=35|GFS_CRITICAL_COUNT=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" critical-logicals | wc -l)\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=74|jq -e --arg l \"$logical\" --arg g \"$generation\" '.schema==\"homelab-cloud-quorum-payload-v1\" and .logical_id==$l and .generation==$g and .status==\"SEALED\" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_delete_allowed==true and .local_payload_present==false and .local_cleanup==\"OK\"' <<<\"$json\" >PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=106|f=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=111|f=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=117|[[ -d \"$RETENTION_ROOT/transactions\" ]] || { printf '0\\n'; return 0; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=126|done < <(find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -print | LC_ALL=C sort)\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=131|[[ -n \"${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}\" || -n \"${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}\" ]]\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=134|local f=\"${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}\" expected=\"${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}\" actual line logical generation owner state_file state_sha count=0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=160|jq -e --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" '.schema==\"homelab-backup-retention-transaction-v1\" and .task_id==$task and .change_id==$change and (.logical_id|type)==\"string\" and (.generation|type)==\"string\" and (.state_sha256|type)==\"string\" and (.status==\"PLANNED\" or .status==\"B_DELETED\" or .status==\"DONE\")' \"$f\" >PATH_REDACTED || die \"PINNED_TX_CONTENT_INVALID_$f\" 66\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=165|expected_path=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=170|local root=\"$RETENTION_ROOT/transactions\" f\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=184|txf=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=188|grep -Fqx \"$(pinned_tx_identity \"$txf\")\" \"$HOMELAB_RETENTION_PINNED_DELETE_FILE\" || die \"PINNED_PLAN_PROTECT_NOT_PINNED_${logical}_${generation}\" 66\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=199|if [[ -d \"$RETENTION_ROOT/transactions\" ]]; then\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=200|while IFS= read -r f; do pinned_tx_identity \"$f\" >>\"$actual\"; done < <(find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -print | LC_ALL=C sort)\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=203|LC_ALL=C sort \"$HOMELAB_RETENTION_PINNED_DELETE_FILE\" >\"$expected\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=208|printf 'PINNED_DELETE_SCOPE=OK|stage=%s|count=%s|sha256=%s\\n' \"$stage\" \"$count_expected\" \"$HOMELAB_RETENTION_PINNED_DELETE_SHA256\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=218|if ! HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" is-critical \"$logical\"; then return 0; fi\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=226|out=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" select \"$logical\" <\"$TMPROOT/$logical.gfs-eligible\")\" || die \"GFS_SELECT_FAILED_$logical\" 66\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=240|local now logical node keep lifecycle open_count file json schema lid gen status sha sealed_at epoch age idx tx gfs_bucket candidates=0 protected=0 kept=0 gfs_kept=0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=252|schema=\"$(jq -r '.schema/PATH_REDACTED <<<\"$json\")\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=256|if [[ \"$schema\" != homelab-cloud-quorum-payload-v1 || \"$lid\" != \"$logical\" ]] || ! safe_generation \"$logical\" \"$gen\"; then\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=321|path=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=324|jq -n --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" --arg l \"$logical\" --arg g \"$generation\" --arg s \"$state_sha\" --arg status \"$status\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --argjson aa \"$a_absent\" --argjson ba \"$b_absent\" '{schema:\"homelab-backup-retention-transaction-v1\",task_id:$task,change_id:$change,logical_id:$l,generation:$g,state_sha256:$s,status:$status,updated_at_utc:$at,remote_a_absent:$aa,remote_b_absent:$ba}' >\"$tmp\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=357|[[ -d \"$RETENTION_ROOT/transactions\" ]] || return 0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=381|done < <(find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -print | LC_ALL=C sort)\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=415|if find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -exec jq -e '.status==\"DONE\" and .remote_a_absent==true and .remote_b_absent==true' {} \\; -print 2>\"$TMPROOT/tx-find.err\" | grep -q .; then hist=true; fi\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=418|jq -n --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --argjson canary \"$canary_proven\" --argjson hist \"$hist\" --argjson candidates \"${candidates:-0}\" --argjson protected \"${protected:-0}\" --argjson kept \"${kept:-0}\" --argjson gfs_kept \"${gfs_kept:-0}\" --argjson deleted \"$deleted\" --argjson gfs_critical \"$GFS_CRITICAL_COUNT\" --argjson gfs_weekly \"$GFS_WEEKLY\" --argjson gfs_monthly \"$GFS_MONTHLY\" '{schema:\"homelab-backup-retention-v1\",status:\"OK\",task_id:\nAPPQ_FRESHNESS_PLAN_RC=65\nREC|APPQ_ERR|APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\nGFS_VALIDATE_RC=66\nREC|GFS_ERR|GFS_POLICY_ERROR=READ_OR_JSON_INVALID\nRETENTION_CONTRACT_RC=65\nREC|RET_ERR|GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\nVPN_EXECSTART_COUNT=1\nREC|VPN_EXEC_PATH|IDX=1|BASENAME=bash|PARENT=bin|SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f|EXEC=true\nREC|VPN_EXEC_PATH|IDX=2|BASENAME=homelab-check-private-vpn-hosts|PARENT=sbin|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82|EXEC=true\nREC|VPN_EXEC_PATH|IDX=3|BASENAME=private-vpn-hosts.txt|PARENT=homelab-health|SHA=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d|EXEC=false\nVPN_SCRIPT_FOUND=true\nVPN_SCRIPT_BASENAME=homelab-check-private-vpn-hosts\nVPN_SCRIPT_SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82\nVPN_SCRIPT_MUTATION_PATTERN_COUNT=1\nVPN_LITERAL_NAME_COUNT=5\nREC|VPN_SOURCE|LINE=6|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=7|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=8|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=9|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=10|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=19|ip=\"$(getent hosts \"$h\" 2>/dev/null | awk '{print $1}' | head -n1 || true)\"\nVPN_SCRIPT_DIRECT_SKIPPED_MUTATION_SCAN=true\nPVE01_RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74\nPVE01_NAMESERVER_COUNT=1\nPVE01_VPN_HOSTS_LINE_COUNT=0\nPVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f\nPVE2_NAMESERVER_COUNT=1\nPVE2_VPN_HOSTS_LINE_COUNT=0\nFAILED_UNIT_COUNT_NOW=2\nREC|FAILED_UNIT_NOW|homelab-backup-audit.service\nREC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service\nDECISION=PASS_PVE01_AUDIT_HELPERS_VPN_EXACT_RCA\nNEXT_GATE=PVE01_AUDIT_HELPERS_VPN_REPAIR\nTASK_RESULT=PVE01_AUDIT_HELPERS_VPN_EXACT_RCA\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "PVE01_GFS_VPN_FINAL_RCA=V1\nMODE=read-only\nNO_REBOOT=true\nNO_APT_UPDATE=true\nNO_PACKAGE_MUTATION=true\nNO_SERVICE_START_STOP_RESTART=true\nNO_RESET_FAILED=true\nNO_CONFIG_WRITE=true\nNO_FILE_DELETE=true\nNO_GUEST_MUTATION=true\nNO_BACKUP_EXECUTION=true\nNO_NETWORK_MUTATION=true\nNODE_HOSTNAME=pve01\nREC|FILE|BASENAME=targets.json|PRESENT=true|SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49\nREC|FILE|BASENAME=homelab-backup-gfs-policy|PRESENT=true|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94\nREC|FILE|BASENAME=gfs-retention.json|PRESENT=true|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d\nREC|FILE|BASENAME=homelab-app-cloud-quorum-queue|PRESENT=true|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\nREC|FILE|BASENAME=homelab-backup-retention-gc|PRESENT=true|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909\nREC|FILE|BASENAME=homelab-backup-target-registry|PRESENT=true|SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a\nREC|FILE|BASENAME=homelab-check-private-vpn-hosts|PRESENT=true|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82\nGFS_VALIDATE_CORRECT_ENV_RC=66\nREC|GFS_CORRECT_ERR|GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm160\nREC|GFS_SOURCE|LINE=1|#!PATH_REDACTED python3\nREC|GFS_SOURCE|LINE=2|import datetime as dt, json, os, pathlib, sys\nREC|GFS_SOURCE|LINE=3|CFG=pathlib.Path(os.environ.get(\"HOMELAB_BACKUP_GFS_POLICY\",\"PATH_REDACTED\nREC|GFS_SOURCE|LINE=4|REG=pathlib.Path(os.environ.get(\"HOMELAB_BACKUP_TARGET_REGISTRY\",\"PATH_REDACTED\nREC|GFS_SOURCE|LINE=5|def die(msg,rc=65):\nREC|GFS_SOURCE|LINE=6| print(f\"GFS_POLICY_ERROR={msg}\",file=sys.stderr); raise SystemExit(rc)\nREC|GFS_SOURCE|LINE=7|def load():\nREC|GFS_SOURCE|LINE=8| try: c=json.loads(CFG.read_text()); r=json.loads(REG.read_text())\nREC|GFS_SOURCE|LINE=9| except Exception: die(\"READ_OR_JSON_INVALID\",66)\nREC|GFS_SOURCE|LINE=10| if c.get(\"schema\")!=\"homelab-backup-gfs-policy-v1\" or c.get(\"policy_version\")!=1: die(\"SCHEMA_INVALID\",66)\nREC|GFS_SOURCE|LINE=11| if c.get(\"completed_periods_only\") is not True or c.get(\"weekly_buckets\")!=4 or c.get(\"monthly_buckets\")!=6 or c.get(\"capacity_guard_max_free_fraction_milli\")!=500 or c.get(\"capacity_guard_target_max_age_seconds\")!=172800 or c.get(\"capacity_guard_cloud_max_age_seconds\")!=14400: die(\"POLICY_INVALID\",66)\nREC|GFS_SOURCE|LINE=12| crit=c.get(\"critical_vm_logicals\")\nREC|GFS_SOURCE|LINE=13| if not isinstance(crit,list) or not crit or len(set(crit))!=len(crit): die(\"CRITICAL_SET_INVALID\",66)\nREC|GFS_SOURCE|LINE=14| rows={x.get(\"logical_id\"):x for x in r.get(\"targets\",[])}\nREC|GFS_SOURCE|LINE=15| for logical in crit:\nREC|GFS_SOURCE|LINE=16| rec=rows.get(logical)\nREC|GFS_SOURCE|LINE=17| if not rec or rec.get(\"kind\")!=\"VM\" or rec.get(\"lifecycle\")!=\"ACTIVE\": die(f\"CRITICAL_NOT_ACTIVE_VM_{logical}\",66)\nREC|GFS_SOURCE|LINE=18| return c,r\nREC|GFS_SOURCE|LINE=19|def parse_utc(s):\nREC|GFS_SOURCE|LINE=20| try:\nREC|GFS_SOURCE|LINE=21| if s.endswith('Z'): s=s[:-1]+'+00:00'\nREC|GFS_SOURCE|LINE=22| x=dt.datetime.fromisoformat(s)\nREC|GFS_SOURCE|LINE=23| if x.tzinfo is None: x=x.replace(tzinfo=dt.timezone.utc)\nREC|GFS_SOURCE|LINE=24| return x.astimezone(dt.timezone.utc)\nREC|GFS_SOURCE|LINE=25| except Exception: die(\"SEALED_TIME_INVALID\",66)\nREC|GFS_SOURCE|LINE=26|def now_utc():\nREC|GFS_SOURCE|LINE=27| v=os.environ.get(\"HOMELAB_GFS_NOW_UTC\")\nREC|GFS_SOURCE|LINE=28| return parse_utc(v) if v else dt.datetime.now(dt.timezone.utc)\nREC|GFS_SOURCE|LINE=29|def select(logical,c):\nREC|GFS_SOURCE|LINE=30| if logical not in c[\"critical_vm_logicals\"]: return\nREC|GFS_SOURCE|LINE=31| now=now_utc(); cur_iso=now.isocalendar(); cur_week=(cur_iso.year,cur_iso.week); cur_month=(now.year,now.month)\nREC|GFS_SOURCE|LINE=32| weekly=[]; monthly=[]; seen_w=set(); seen_m=set()\nREC|GFS_SOURCE|LINE=33| for raw in sys.stdin:\nREC|GFS_SOURCE|LINE=34| raw=raw.rstrip(\"\\n\")\nREC|GFS_SOURCE|LINE=35| if not raw: continue\nREC|GFS_SOURCE|LINE=36| parts=raw.split(\"\\t\")\nREC|GFS_SOURCE|LINE=37| if len(parts)!=2: die(\"SELECT_INPUT_INVALID\",66)\nREC|GFS_SOURCE|LINE=38| gen,sealed=parts; when=parse_utc(sealed); iso=when.isocalendar(); wb=(iso.year,iso.week); mb=(when.year,when.month)\nREC|GFS_SOURCE|LINE=39| if wb!=cur_week and len(weekly)=c[\"weekly_buckets\"] and len(monthly)>=c[\"monthly_buckets\"]: break\nREC|GFS_SOURCE|LINE=44| for gen,b in weekly: print(f\"WEEKLY\\t{gen}\\t{b}\")\nREC|GFS_SOURCE|LINE=45| for gen,b in monthly: print(f\"MONTHLY\\t{gen}\\t{b}\")\nREC|GFS_SOURCE|LINE=46|def main():\nREC|GFS_SOURCE|LINE=47| c,_=load(); args=sys.argv[1:]\nREC|GFS_SOURCE|LINE=48| if args==[\"validate\"]: print(\"GFS_POLICY=OK\"); return\nREC|GFS_SOURCE|LINE=49| if args==[\"critical-logicals\"]:\nREC|GFS_SOURCE|LINE=50| print(\"\\n\".join(c[\"critical_vm_logicals\"])); return\nREC|GFS_SOURCE|LINE=51| if args==[\"weekly-buckets\"]: print(c[\"weekly_buckets\"]); return\nREC|GFS_SOURCE|LINE=52| if args==[\"monthly-buckets\"]: print(c[\"monthly_buckets\"]); return\nREC|GFS_SOURCE|LINE=53| if args==[\"capacity-guard-milli\"]: print(c[\"capacity_guard_max_free_fraction_milli\"]); return\nREC|GFS_SOURCE|LINE=54| if len(args)==2 and args[0]==\"is-critical\": raise SystemExit(0 if args[1] in c[\"critical_vm_logicals\"] else 1)\nREC|GFS_SOURCE|LINE=55| if len(args)==2 and args[0]==\"select\": select(args[1],c); return\nREC|GFS_SOURCE|LINE=56| die(\"USAGE\",64)\nREC|GFS_SOURCE|LINE=57|if __name__==\"__main__\": main()\nGFS_POLICY_SCHEMA=homelab-backup-gfs-policy-v1\nGFS_POLICY_VERSION=1\nGFS_POLICY_KEYS=capacity_guard_cloud_max_age_seconds,capacity_guard_max_free_fraction_milli,capacity_guard_target_max_age_seconds,completed_periods_only,critical_vm_logicals,monthly_buckets,policy_version,schema,weekly_buckets\nREC|GFS_POLICY_SCALAR|KEY=capacity_guard_cloud_max_age_seconds|VALUE=14400\nREC|GFS_POLICY_SCALAR|KEY=capacity_guard_max_free_fraction_milli|VALUE=500\nREC|GFS_POLICY_SCALAR|KEY=capacity_guard_target_max_age_seconds|VALUE=172800\nREC|GFS_POLICY_SCALAR|KEY=completed_periods_only|VALUE=True\nREC|GFS_POLICY_LIST|KEY=critical_vm_logicals|COUNT=8|VALUES=vm110,vm111,vm112,vm113,vm130,vm150,vm160,vm170\nREC|GFS_POLICY_SCALAR|KEY=monthly_buckets|VALUE=6\nREC|GFS_POLICY_SCALAR|KEY=policy_version|VALUE=1\nREC|GFS_POLICY_SCALAR|KEY=schema|VALUE=homelab-backup-gfs-policy-v1\nREC|GFS_POLICY_SCALAR|KEY=weekly_buckets|VALUE=4\nUNIFIED_TOTAL=21\nUNIFIED_VM_ACTIVE=10\nUNIFIED_VM_RETIRED_PRESERVE=3\nUNIFIED_APP_ACTIVE=4\nUNIFIED_APP_RETIRED_PRESERVE=3\nUNIFIED_CONTROL_ACTIVE=1\nUNIFIED_CONTROL_RETIRED_PRESERVE=0\nUNIFIED_LOGICAL_IDS=app-filebrowser,app-hbarhub,app-immich,app-memos,app-scrutiny,app-skladchik-moderator-assistant,app-skladchik-reports-monitor,control-bootstrap-v2,vm110,vm111,vm112,vm113,vm130,vm150,vm160,vm170,vm171,vm180,vm190,vm200,vm9130\nAPPQ_DIRECT_RC=65\nREC|APPQ_ERR|APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\nRET_DIRECT_RC=65\nREC|RET_ERR|GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\nVPN_LITERAL_NAME_COUNT=5\nREC|VPN_NAME|IDX=1|NAME_SHA=1588bd492f97c578a05fad877bb1c11e6738df68992b121042a13fb2f17329c0\nREC|VPN_NAME|IDX=2|NAME_SHA=bc1585521e17de78e3a7bd903ca2427f5941f3bc1276f324fafe794ea8eabdbc\nREC|VPN_NAME|IDX=3|NAME_SHA=968bf918c9b60ba4ae098281e24580e629b4ac33446a76981ed53c671db1e8c8\nREC|VPN_NAME|IDX=4|NAME_SHA=9f64956d61ca126d479e1ca923eed5c6994916944111962c6e2f3ad1d2538dcd\nREC|VPN_NAME|IDX=5|NAME_SHA=95f56744bb158973029a5113cf2904b44a928103143dfab662552f9beb28b9d9\nREC|VPN_SOURCE_ALL|LINE=1|#!PATH_REDACTED bash\nREC|VPN_SOURCE_ALL|LINE=2|set -u\nREC|VPN_SOURCE_ALL|LINE=4|H=\"PATH_REDACTED\nREC|VPN_SOURCE_ALL|LINE=5|HOSTS=\"\nREC|VPN_SOURCE_ALL|LINE=6|VPN_HOST_REDACTED\nREC|VPN_SOURCE_ALL|LINE=7|VPN_HOST_REDACTED\nREC|VPN_SOURCE_ALL|LINE=8|VPN_HOST_REDACTED\nREC|VPN_SOURCE_ALL|LINE=9|VPN_HOST_REDACTED\nREC|VPN_SOURCE_ALL|LINE=10|VPN_HOST_REDACTED\nREC|VPN_SOURCE_ALL|LINE=11|\"\nREC|VPN_SOURCE_ALL|LINE=13|mkdir -p PATH_REDACTED\nREC|VPN_SOURCE_ALL|LINE=14|tmp=\"$(mktemp)\"\nREC|VPN_SOURCE_ALL|LINE=15|trap 'rm -f \"$tmp\"' EXIT\nREC|VPN_SOURCE_ALL|LINE=17|bad=0\nREC|VPN_SOURCE_ALL|LINE=18|for h in $HOSTS; do\nREC|VPN_SOURCE_ALL|LINE=19|ip=\"$(getent hosts \"$h\" 2>PATH_REDACTED | awk '{print $1}' | head -n1 || true)\"\nREC|VPN_SOURCE_ALL|LINE=20|http=\"000error\"\nREC|VPN_SOURCE_ALL|LINE=21|https=\"000error\"\nREC|VPN_SOURCE_ALL|LINE=22|location=\"none\"\nREC|VPN_SOURCE_ALL|LINE=23|cert_ok=\"yes\"\nREC|VPN_SOURCE_ALL|LINE=25|if [ -n \"$ip\" ]; then\nREC|VPN_SOURCE_ALL|LINE=26|http_code=\"$(curl -4 -ksS -o PATH_REDACTED -D PATH_REDACTED -w '%{http_code}' --max-time 10 \"http:PATH_REDACTED 2>PATH_REDACTED || true)\"\nREC|VPN_SOURCE_ALL|LINE=27|https_code=\"$(curl -4 -ksS -o PATH_REDACTED -D PATH_REDACTED -w '%{http_code}' --max-time 10 \"https:PATH_REDACTED 2>PATH_REDACTED || true)\"\nREC|VPN_SOURCE_ALL|LINE=28|[ -n \"$http_code\" ] && http=\"$http_code\"\nREC|VPN_SOURCE_ALL|LINE=29|[ -n \"$https_code\" ] && https=\"$https_code\"\nREC|VPN_SOURCE_ALL|LINE=30|location=\"$(awk 'tolower($1)==\"location:\"{print $2; exit}' PATH_REDACTED 2>PATH_REDACTED | tr -d '\\r' || true)\"\nREC|VPN_SOURCE_ALL|LINE=31|[ -n \"$location\" ] || location=\"none\"\nREC|VPN_SOURCE_ALL|LINE=32|fi\nREC|VPN_SOURCE_ALL|LINE=34|case \"$https\" in\nREC|VPN_SOURCE_ALL|LINE=35|200|301|302|401|403) ;;\nREC|VPN_SOURCE_ALL|LINE=36|*) bad=$((bad+1)) ;;\nREC|VPN_SOURCE_ALL|LINE=37|esac\nREC|VPN_SOURCE_ALL|LINE=39|printf 'HOST|%s|fqdn=%s|dns=%s|http=%s|https=%s|location=%s|cert_ok=%s\\n' \\\nREC|VPN_SOURCE_ALL|LINE=40|\"${h%%.*}\" \"$h\" \"${ip:-NONE}\" \"$http\" \"$https\" \"$location\" \"$cert_ok\" >> \"$tmp\"\nREC|VPN_SOURCE_ALL|LINE=41|done\nREC|VPN_SOURCE_ALL|LINE=43|if [ \"$bad\" = \"0\" ]; then\nREC|VPN_SOURCE_ALL|LINE=44|status=OK\nREC|VPN_SOURCE_ALL|LINE=45|else\nREC|VPN_SOURCE_ALL|LINE=46|status=ERROR\nREC|VPN_SOURCE_ALL|LINE=47|fi\nREC|VPN_SOURCE_ALL|LINE=49|{\nREC|VPN_SOURCE_ALL|LINE=50|cat \"$tmp\"\nREC|VPN_SOURCE_ALL|LINE=51|printf 'STATUS=%s\\n' \"$status\"\nREC|VPN_SOURCE_ALL|LINE=52|} > \"$H\"\nREC|VPN_SOURCE_ALL|LINE=54|cat \"$H\"\nREC|VPN_SOURCE_ALL|LINE=55|[ \"$status\" = \"OK\" ]\nREC|VPN_RESOLUTION|IDX=1|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true\nREC|VPN_RESOLUTION|IDX=2|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true\nREC|VPN_RESOLUTION|IDX=3|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true\nREC|VPN_RESOLUTION|IDX=4|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true\nREC|VPN_RESOLUTION|IDX=5|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true\nREC|RESOLVER|HOST=pve01|RESOLV_IS_SYMLINK=false\nREC|RESOLVER|HOST=pve01|RESOLV_LINK_BASENAME=NONE\nREC|RESOLVER|HOST=pve01|RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74\nREC|RESOLVER|HOST=pve01|NAMESERVER_COUNT=1\nREC|RESOLVER|HOST=pve01|SEARCH_LINE_COUNT=1\nREC|RESOLVER|HOST=pve01|REC|RESOLVER_NS_SHA|a6f12182e30cba92f9e91dc59ae899c758ecc34e1c3b9b06404eb741a56916ac\nREC|RESOLVER|HOST=pve02|RESOLV_IS_SYMLINK=false\nREC|RESOLVER|HOST=pve02|RESOLV_LINK_BASENAME=NONE\nREC|RESOLVER|HOST=pve02|RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f\nREC|RESOLVER|HOST=pve02|NAMESERVER_COUNT=1\nREC|RESOLVER|HOST=pve02|SEARCH_LINE_COUNT=1\nREC|RESOLVER|HOST=pve02|REC|RESOLVER_NS_SHA|3e731e83b523045ce83835503795f722459caf77bb2f8c0afcfaf75971b2bd78\nREC|PVE01_NS_CONFIG_REF|BASENAME=interfaces|PARENT=network|MATCH_COUNT=0|SHA=8558349d08d267b08a8abb490a591b5d4b7d539d3cc515b08f3165f3af0b8898\nFAILED_UNIT_COUNT_NOW=2\nREC|FAILED_UNIT_NOW|homelab-backup-audit.service\nREC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service\nDECISION=PASS_PVE01_GFS_VPN_FINAL_RCA\nNEXT_GATE=PVE01_HEALTH_HELPERS_AND_VPN_REPAIR\nTASK_RESULT=PVE01_GFS_VPN_FINAL_RCA\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 7585e107..05005c18 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1 +COMMAND_ID=SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=pve01-audit-helpers-vpn-exact-rca +COMPONENT=pve01-gfs-vpn-final-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=5cf24ab1bb98675b9bfe49e8bdc859479f976f0b83c5d2d1a01f49508b38be64 +COMMAND_SHA256=9c86990b05592de487ff89f09077586a7f5c5c5c54fe3be43817e81dc4df0276 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,10 +24,10 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd -SANITIZED_OUTPUT_SHA256=abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd +RAW_EVIDENCE_SHA256=5de6a14be4a76e65a8cd5b1d048ed7f51c51c4d57732588bb9207e7de92f4d28 +SANITIZED_OUTPUT_SHA256=5de6a14be4a76e65a8cd5b1d048ed7f51c51c4d57732588bb9207e7de92f4d28 OUTPUT_BEGIN -PVE01_AUDIT_HELPERS_VPN_EXACT_RCA=V1 +PVE01_GFS_VPN_FINAL_RCA=V1 MODE=read-only NO_REBOOT=true NO_APT_UPDATE=true @@ -40,131 +40,177 @@ NO_GUEST_MUTATION=true NO_BACKUP_EXECUTION=true NO_NETWORK_MUTATION=true NODE_HOSTNAME=pve01 -AUDIT_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3 -REGISTRY_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49 -HELPER_SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a -AUDIT_CONST_APPQ_PRESENT=true -AUDIT_CONST_APPQ_BASENAME=homelab-app-cloud-quorum-queue -AUDIT_CONST_APPQ_PARENT=sbin -AUDIT_CONST_APPQ_EXISTS=true -AUDIT_CONST_APPQ_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650 -AUDIT_CONST_APPQ_PATH_SHA=75315ca46bfa3b06e14300fe6c03852aaf085c4d2843247dd39fd7e8b53f7800 -AUDIT_CONST_GFSH_PRESENT=true -AUDIT_CONST_GFSH_BASENAME=homelab-backup-gfs-policy -AUDIT_CONST_GFSH_PARENT=libexec -AUDIT_CONST_GFSH_EXISTS=true -AUDIT_CONST_GFSH_SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94 -AUDIT_CONST_GFSH_PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9 -AUDIT_CONST_GFSP_PRESENT=true -AUDIT_CONST_GFSP_BASENAME=gfs-retention.json -AUDIT_CONST_GFSP_PARENT=homelab-backup -AUDIT_CONST_GFSP_EXISTS=true -AUDIT_CONST_GFSP_SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d -AUDIT_CONST_GFSP_PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d -AUDIT_CONST_REG_PRESENT=false -AUDIT_CONST_RET_PRESENT=true -AUDIT_CONST_RET_BASENAME=homelab-backup-retention-gc -AUDIT_CONST_RET_PARENT=sbin -AUDIT_CONST_RET_EXISTS=true -AUDIT_CONST_RET_SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909 -AUDIT_CONST_RET_PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a -REC|AUDIT_HELPER_FILE|LABEL=APPQ|BASENAME=homelab-app-cloud-quorum-queue|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0 -REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=6|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED -REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=7|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4 -REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=8|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED -REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=11|[[ -x "$REGISTRY_HELPER" && ! -L "$REGISTRY_HELPER" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_MISSING\n' >&2; exit 69; } -REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=12|[[ "$(sha256sum "$REGISTRY_HELPER"|awk '{print $1}')" == "$REGISTRY_HELPER_SHA_EXPECTED" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\n' >&2; exit 65; } -REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=19|[[ "${#ALL_APPS[@]}" == 4 && "${#MIGRATE_APPS[@]}" == 4 ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_COUNT_INVALID\n' >&2; exit 65; } -REC|AUDIT_HELPER_FILE|LABEL=GFSH|BASENAME=homelab-backup-gfs-policy|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0 -REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=3|CFG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_GFS_POLICY","PATH_REDACTED -REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=4|REG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_TARGET_REGISTRY","PATH_REDACTED -REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=6|print(f"GFS_POLICY_ERROR={msg}",file=sys.stderr); raise SystemExit(rc) -REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=10|if c.get("schema")!="homelab-backup-gfs-policy-v1" or c.get("policy_version")!=1: die("SCHEMA_INVALID",66) -REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=14|rows={x.get("logical_id"):x for x in r.get("targets",[])} -REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=48|if args==["validate"]: print("GFS_POLICY=OK"); return -REC|AUDIT_HELPER_FILE|LABEL=GFSP|BASENAME=gfs-retention.json|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0 -REC|AUDIT_HELPER_SOURCE|LABEL=GFSP|LINE=18|"schema": "homelab-backup-gfs-policy-v1", -REC|AUDIT_HELPER_FILE|LABEL=RET|BASENAME=homelab-backup-retention-gc|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0 -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=8|RETENTION_ROOT=PATH_REDACTED -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=21|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=22|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4 -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=23|GFS_HELPER="${HOMELAB_BACKUP_GFS_POLICY_HELPER:-PATH_REDACTED -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=25|GFS_POLICY="${HOMELAB_BACKUP_GFS_POLICY:-PATH_REDACTED -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=27|[[ -x "$REGISTRY_HELPER" && ! -L "$REGISTRY_HELPER" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_MISSING\n' >&2; exit 69; } -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=28|[[ "$(sha256sum "$REGISTRY_HELPER"|awk '{print $1}')" == "$REGISTRY_HELPER_SHA_EXPECTED" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\n' >&2; exit 65; } -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=29|"$REGISTRY_HELPER" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_INVALID\n' >&2; exit 66; } -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=30|[[ -x "$GFS_HELPER" && ! -L "$GFS_HELPER" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_MISSING\n' >&2; exit 69; } -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=31|[[ "$(sha256sum "$GFS_HELPER"|awk '{print $1}')" == "$GFS_HELPER_SHA_EXPECTED" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\n' >&2; exit 65; } -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=32|HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID\n' >&2; exit 66; } -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=33|GFS_WEEKLY="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" weekly-buckets)" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=34|GFS_MONTHLY="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" monthly-buckets)" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=35|GFS_CRITICAL_COUNT="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" critical-logicals | wc -l)" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=74|jq -e --arg l "$logical" --arg g "$generation" '.schema=="homelab-cloud-quorum-payload-v1" and .logical_id==$l and .generation==$g and .status=="SEALED" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_delete_allowed==true and .local_payload_present==false and .local_cleanup=="OK"' <<<"$json" >PATH_REDACTED -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=106|f="$RETENTION_ROOT/transactions/$logical/$generation.json" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=111|f="$RETENTION_ROOT/transactions/$logical/$generation.json" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=117|[[ -d "$RETENTION_ROOT/transactions" ]] || { printf '0\n'; return 0; } -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=126|done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort) -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=131|[[ -n "${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}" || -n "${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}" ]] -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=134|local f="${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}" expected="${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}" actual line logical generation owner state_file state_sha count=0 -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=160|jq -e --arg task "$TASK_ID" --arg change "$CHANGE_ID" '.schema=="homelab-backup-retention-transaction-v1" and .task_id==$task and .change_id==$change and (.logical_id|type)=="string" and (.generation|type)=="string" and (.state_sha256|type)=="string" and (.status=="PLANNED" or .status=="B_DELETED" or .status=="DONE")' "$f" >PATH_REDACTED || die "PINNED_TX_CONTENT_INVALID_$f" 66 -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=165|expected_path="$RETENTION_ROOT/transactions/$logical/$generation.json" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=170|local root="$RETENTION_ROOT/transactions" f -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=184|txf="$RETENTION_ROOT/transactions/$logical/$generation.json" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=188|grep -Fqx "$(pinned_tx_identity "$txf")" "$HOMELAB_RETENTION_PINNED_DELETE_FILE" || die "PINNED_PLAN_PROTECT_NOT_PINNED_${logical}_${generation}" 66 -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=199|if [[ -d "$RETENTION_ROOT/transactions" ]]; then -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=200|while IFS= read -r f; do pinned_tx_identity "$f" >>"$actual"; done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort) -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=203|LC_ALL=C sort "$HOMELAB_RETENTION_PINNED_DELETE_FILE" >"$expected" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=208|printf 'PINNED_DELETE_SCOPE=OK|stage=%s|count=%s|sha256=%s\n' "$stage" "$count_expected" "$HOMELAB_RETENTION_PINNED_DELETE_SHA256" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=218|if ! HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" is-critical "$logical"; then return 0; fi -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=226|out="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" select "$logical" <"$TMPROOT/$logical.gfs-eligible")" || die "GFS_SELECT_FAILED_$logical" 66 -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=240|local now logical node keep lifecycle open_count file json schema lid gen status sha sealed_at epoch age idx tx gfs_bucket candidates=0 protected=0 kept=0 gfs_kept=0 -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=252|schema="$(jq -r '.schema/PATH_REDACTED <<<"$json")" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=256|if [[ "$schema" != homelab-cloud-quorum-payload-v1 || "$lid" != "$logical" ]] || ! safe_generation "$logical" "$gen"; then -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=321|path="$RETENTION_ROOT/transactions/$logical/$generation.json" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=324|jq -n --arg task "$TASK_ID" --arg change "$CHANGE_ID" --arg l "$logical" --arg g "$generation" --arg s "$state_sha" --arg status "$status" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson aa "$a_absent" --argjson ba "$b_absent" '{schema:"homelab-backup-retention-transaction-v1",task_id:$task,change_id:$change,logical_id:$l,generation:$g,state_sha256:$s,status:$status,updated_at_utc:$at,remote_a_absent:$aa,remote_b_absent:$ba}' >"$tmp" -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=357|[[ -d "$RETENTION_ROOT/transactions" ]] || return 0 -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=381|done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort) -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=415|if find "$RETENTION_ROOT/transactions" -type f -name '*.json' -exec jq -e '.status=="DONE" and .remote_a_absent==true and .remote_b_absent==true' {} \; -print 2>"$TMPROOT/tx-find.err" | grep -q .; then hist=true; fi -REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=418|jq -n --arg task "$TASK_ID" --arg change "$CHANGE_ID" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson canary "$canary_proven" --argjson hist "$hist" --argjson candidates "${candidates:-0}" --argjson protected "${protected:-0}" --argjson kept "${kept:-0}" --argjson gfs_kept "${gfs_kept:-0}" --argjson deleted "$deleted" --argjson gfs_critical "$GFS_CRITICAL_COUNT" --argjson gfs_weekly "$GFS_WEEKLY" --argjson gfs_monthly "$GFS_MONTHLY" '{schema:"homelab-backup-retention-v1",status:"OK",task_id: -APPQ_FRESHNESS_PLAN_RC=65 +REC|FILE|BASENAME=targets.json|PRESENT=true|SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49 +REC|FILE|BASENAME=homelab-backup-gfs-policy|PRESENT=true|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94 +REC|FILE|BASENAME=gfs-retention.json|PRESENT=true|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d +REC|FILE|BASENAME=homelab-app-cloud-quorum-queue|PRESENT=true|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650 +REC|FILE|BASENAME=homelab-backup-retention-gc|PRESENT=true|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909 +REC|FILE|BASENAME=homelab-backup-target-registry|PRESENT=true|SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a +REC|FILE|BASENAME=homelab-check-private-vpn-hosts|PRESENT=true|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82 +GFS_VALIDATE_CORRECT_ENV_RC=66 +REC|GFS_CORRECT_ERR|GFS_POLICY_ERROR=CRITICAL_NOT_ACTIVE_VM_vm160 +REC|GFS_SOURCE|LINE=1|#!PATH_REDACTED python3 +REC|GFS_SOURCE|LINE=2|import datetime as dt, json, os, pathlib, sys +REC|GFS_SOURCE|LINE=3|CFG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_GFS_POLICY","PATH_REDACTED +REC|GFS_SOURCE|LINE=4|REG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_TARGET_REGISTRY","PATH_REDACTED +REC|GFS_SOURCE|LINE=5|def die(msg,rc=65): +REC|GFS_SOURCE|LINE=6| print(f"GFS_POLICY_ERROR={msg}",file=sys.stderr); raise SystemExit(rc) +REC|GFS_SOURCE|LINE=7|def load(): +REC|GFS_SOURCE|LINE=8| try: c=json.loads(CFG.read_text()); r=json.loads(REG.read_text()) +REC|GFS_SOURCE|LINE=9| except Exception: die("READ_OR_JSON_INVALID",66) +REC|GFS_SOURCE|LINE=10| if c.get("schema")!="homelab-backup-gfs-policy-v1" or c.get("policy_version")!=1: die("SCHEMA_INVALID",66) +REC|GFS_SOURCE|LINE=11| if c.get("completed_periods_only") is not True or c.get("weekly_buckets")!=4 or c.get("monthly_buckets")!=6 or c.get("capacity_guard_max_free_fraction_milli")!=500 or c.get("capacity_guard_target_max_age_seconds")!=172800 or c.get("capacity_guard_cloud_max_age_seconds")!=14400: die("POLICY_INVALID",66) +REC|GFS_SOURCE|LINE=12| crit=c.get("critical_vm_logicals") +REC|GFS_SOURCE|LINE=13| if not isinstance(crit,list) or not crit or len(set(crit))!=len(crit): die("CRITICAL_SET_INVALID",66) +REC|GFS_SOURCE|LINE=14| rows={x.get("logical_id"):x for x in r.get("targets",[])} +REC|GFS_SOURCE|LINE=15| for logical in crit: +REC|GFS_SOURCE|LINE=16| rec=rows.get(logical) +REC|GFS_SOURCE|LINE=17| if not rec or rec.get("kind")!="VM" or rec.get("lifecycle")!="ACTIVE": die(f"CRITICAL_NOT_ACTIVE_VM_{logical}",66) +REC|GFS_SOURCE|LINE=18| return c,r +REC|GFS_SOURCE|LINE=19|def parse_utc(s): +REC|GFS_SOURCE|LINE=20| try: +REC|GFS_SOURCE|LINE=21| if s.endswith('Z'): s=s[:-1]+'+00:00' +REC|GFS_SOURCE|LINE=22| x=dt.datetime.fromisoformat(s) +REC|GFS_SOURCE|LINE=23| if x.tzinfo is None: x=x.replace(tzinfo=dt.timezone.utc) +REC|GFS_SOURCE|LINE=24| return x.astimezone(dt.timezone.utc) +REC|GFS_SOURCE|LINE=25| except Exception: die("SEALED_TIME_INVALID",66) +REC|GFS_SOURCE|LINE=26|def now_utc(): +REC|GFS_SOURCE|LINE=27| v=os.environ.get("HOMELAB_GFS_NOW_UTC") +REC|GFS_SOURCE|LINE=28| return parse_utc(v) if v else dt.datetime.now(dt.timezone.utc) +REC|GFS_SOURCE|LINE=29|def select(logical,c): +REC|GFS_SOURCE|LINE=30| if logical not in c["critical_vm_logicals"]: return +REC|GFS_SOURCE|LINE=31| now=now_utc(); cur_iso=now.isocalendar(); cur_week=(cur_iso.year,cur_iso.week); cur_month=(now.year,now.month) +REC|GFS_SOURCE|LINE=32| weekly=[]; monthly=[]; seen_w=set(); seen_m=set() +REC|GFS_SOURCE|LINE=33| for raw in sys.stdin: +REC|GFS_SOURCE|LINE=34| raw=raw.rstrip("\n") +REC|GFS_SOURCE|LINE=35| if not raw: continue +REC|GFS_SOURCE|LINE=36| parts=raw.split("\t") +REC|GFS_SOURCE|LINE=37| if len(parts)!=2: die("SELECT_INPUT_INVALID",66) +REC|GFS_SOURCE|LINE=38| gen,sealed=parts; when=parse_utc(sealed); iso=when.isocalendar(); wb=(iso.year,iso.week); mb=(when.year,when.month) +REC|GFS_SOURCE|LINE=39| if wb!=cur_week and len(weekly)=c["weekly_buckets"] and len(monthly)>=c["monthly_buckets"]: break +REC|GFS_SOURCE|LINE=44| for gen,b in weekly: print(f"WEEKLY\t{gen}\t{b}") +REC|GFS_SOURCE|LINE=45| for gen,b in monthly: print(f"MONTHLY\t{gen}\t{b}") +REC|GFS_SOURCE|LINE=46|def main(): +REC|GFS_SOURCE|LINE=47| c,_=load(); args=sys.argv[1:] +REC|GFS_SOURCE|LINE=48| if args==["validate"]: print("GFS_POLICY=OK"); return +REC|GFS_SOURCE|LINE=49| if args==["critical-logicals"]: +REC|GFS_SOURCE|LINE=50| print("\n".join(c["critical_vm_logicals"])); return +REC|GFS_SOURCE|LINE=51| if args==["weekly-buckets"]: print(c["weekly_buckets"]); return +REC|GFS_SOURCE|LINE=52| if args==["monthly-buckets"]: print(c["monthly_buckets"]); return +REC|GFS_SOURCE|LINE=53| if args==["capacity-guard-milli"]: print(c["capacity_guard_max_free_fraction_milli"]); return +REC|GFS_SOURCE|LINE=54| if len(args)==2 and args[0]=="is-critical": raise SystemExit(0 if args[1] in c["critical_vm_logicals"] else 1) +REC|GFS_SOURCE|LINE=55| if len(args)==2 and args[0]=="select": select(args[1],c); return +REC|GFS_SOURCE|LINE=56| die("USAGE",64) +REC|GFS_SOURCE|LINE=57|if __name__=="__main__": main() +GFS_POLICY_SCHEMA=homelab-backup-gfs-policy-v1 +GFS_POLICY_VERSION=1 +GFS_POLICY_KEYS=capacity_guard_cloud_max_age_seconds,capacity_guard_max_free_fraction_milli,capacity_guard_target_max_age_seconds,completed_periods_only,critical_vm_logicals,monthly_buckets,policy_version,schema,weekly_buckets +REC|GFS_POLICY_SCALAR|KEY=capacity_guard_cloud_max_age_seconds|VALUE=14400 +REC|GFS_POLICY_SCALAR|KEY=capacity_guard_max_free_fraction_milli|VALUE=500 +REC|GFS_POLICY_SCALAR|KEY=capacity_guard_target_max_age_seconds|VALUE=172800 +REC|GFS_POLICY_SCALAR|KEY=completed_periods_only|VALUE=True +REC|GFS_POLICY_LIST|KEY=critical_vm_logicals|COUNT=8|VALUES=vm110,vm111,vm112,vm113,vm130,vm150,vm160,vm170 +REC|GFS_POLICY_SCALAR|KEY=monthly_buckets|VALUE=6 +REC|GFS_POLICY_SCALAR|KEY=policy_version|VALUE=1 +REC|GFS_POLICY_SCALAR|KEY=schema|VALUE=homelab-backup-gfs-policy-v1 +REC|GFS_POLICY_SCALAR|KEY=weekly_buckets|VALUE=4 +UNIFIED_TOTAL=21 +UNIFIED_VM_ACTIVE=10 +UNIFIED_VM_RETIRED_PRESERVE=3 +UNIFIED_APP_ACTIVE=4 +UNIFIED_APP_RETIRED_PRESERVE=3 +UNIFIED_CONTROL_ACTIVE=1 +UNIFIED_CONTROL_RETIRED_PRESERVE=0 +UNIFIED_LOGICAL_IDS=app-filebrowser,app-hbarhub,app-immich,app-memos,app-scrutiny,app-skladchik-moderator-assistant,app-skladchik-reports-monitor,control-bootstrap-v2,vm110,vm111,vm112,vm113,vm130,vm150,vm160,vm170,vm171,vm180,vm190,vm200,vm9130 +APPQ_DIRECT_RC=65 REC|APPQ_ERR|APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT -GFS_VALIDATE_RC=66 -REC|GFS_ERR|GFS_POLICY_ERROR=READ_OR_JSON_INVALID -RETENTION_CONTRACT_RC=65 +RET_DIRECT_RC=65 REC|RET_ERR|GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT -VPN_EXECSTART_COUNT=1 -REC|VPN_EXEC_PATH|IDX=1|BASENAME=bash|PARENT=bin|SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f|EXEC=true -REC|VPN_EXEC_PATH|IDX=2|BASENAME=homelab-check-private-vpn-hosts|PARENT=sbin|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82|EXEC=true -REC|VPN_EXEC_PATH|IDX=3|BASENAME=private-vpn-hosts.txt|PARENT=homelab-health|SHA=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d|EXEC=false -VPN_SCRIPT_FOUND=true -VPN_SCRIPT_BASENAME=homelab-check-private-vpn-hosts -VPN_SCRIPT_SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82 -VPN_SCRIPT_MUTATION_PATTERN_COUNT=1 VPN_LITERAL_NAME_COUNT=5 -REC|VPN_SOURCE|LINE=6|VPN_HOST_REDACTED -REC|VPN_SOURCE|LINE=7|VPN_HOST_REDACTED -REC|VPN_SOURCE|LINE=8|VPN_HOST_REDACTED -REC|VPN_SOURCE|LINE=9|VPN_HOST_REDACTED -REC|VPN_SOURCE|LINE=10|VPN_HOST_REDACTED -REC|VPN_SOURCE|LINE=19|ip="$(getent hosts "$h" 2>/dev/null | awk '{print $1}' | head -n1 || true)" -VPN_SCRIPT_DIRECT_SKIPPED_MUTATION_SCAN=true -PVE01_RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74 -PVE01_NAMESERVER_COUNT=1 -PVE01_VPN_HOSTS_LINE_COUNT=0 -PVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f -PVE2_NAMESERVER_COUNT=1 -PVE2_VPN_HOSTS_LINE_COUNT=0 +REC|VPN_NAME|IDX=1|NAME_SHA=1588bd492f97c578a05fad877bb1c11e6738df68992b121042a13fb2f17329c0 +REC|VPN_NAME|IDX=2|NAME_SHA=bc1585521e17de78e3a7bd903ca2427f5941f3bc1276f324fafe794ea8eabdbc +REC|VPN_NAME|IDX=3|NAME_SHA=968bf918c9b60ba4ae098281e24580e629b4ac33446a76981ed53c671db1e8c8 +REC|VPN_NAME|IDX=4|NAME_SHA=9f64956d61ca126d479e1ca923eed5c6994916944111962c6e2f3ad1d2538dcd +REC|VPN_NAME|IDX=5|NAME_SHA=95f56744bb158973029a5113cf2904b44a928103143dfab662552f9beb28b9d9 +REC|VPN_SOURCE_ALL|LINE=1|#!PATH_REDACTED bash +REC|VPN_SOURCE_ALL|LINE=2|set -u +REC|VPN_SOURCE_ALL|LINE=4|H="PATH_REDACTED +REC|VPN_SOURCE_ALL|LINE=5|HOSTS=" +REC|VPN_SOURCE_ALL|LINE=6|VPN_HOST_REDACTED +REC|VPN_SOURCE_ALL|LINE=7|VPN_HOST_REDACTED +REC|VPN_SOURCE_ALL|LINE=8|VPN_HOST_REDACTED +REC|VPN_SOURCE_ALL|LINE=9|VPN_HOST_REDACTED +REC|VPN_SOURCE_ALL|LINE=10|VPN_HOST_REDACTED +REC|VPN_SOURCE_ALL|LINE=11|" +REC|VPN_SOURCE_ALL|LINE=13|mkdir -p PATH_REDACTED +REC|VPN_SOURCE_ALL|LINE=14|tmp="$(mktemp)" +REC|VPN_SOURCE_ALL|LINE=15|trap 'rm -f "$tmp"' EXIT +REC|VPN_SOURCE_ALL|LINE=17|bad=0 +REC|VPN_SOURCE_ALL|LINE=18|for h in $HOSTS; do +REC|VPN_SOURCE_ALL|LINE=19|ip="$(getent hosts "$h" 2>PATH_REDACTED | awk '{print $1}' | head -n1 || true)" +REC|VPN_SOURCE_ALL|LINE=20|http="000error" +REC|VPN_SOURCE_ALL|LINE=21|https="000error" +REC|VPN_SOURCE_ALL|LINE=22|location="none" +REC|VPN_SOURCE_ALL|LINE=23|cert_ok="yes" +REC|VPN_SOURCE_ALL|LINE=25|if [ -n "$ip" ]; then +REC|VPN_SOURCE_ALL|LINE=26|http_code="$(curl -4 -ksS -o PATH_REDACTED -D PATH_REDACTED -w '%{http_code}' --max-time 10 "http:PATH_REDACTED 2>PATH_REDACTED || true)" +REC|VPN_SOURCE_ALL|LINE=27|https_code="$(curl -4 -ksS -o PATH_REDACTED -D PATH_REDACTED -w '%{http_code}' --max-time 10 "https:PATH_REDACTED 2>PATH_REDACTED || true)" +REC|VPN_SOURCE_ALL|LINE=28|[ -n "$http_code" ] && http="$http_code" +REC|VPN_SOURCE_ALL|LINE=29|[ -n "$https_code" ] && https="$https_code" +REC|VPN_SOURCE_ALL|LINE=30|location="$(awk 'tolower($1)=="location:"{print $2; exit}' PATH_REDACTED 2>PATH_REDACTED | tr -d '\r' || true)" +REC|VPN_SOURCE_ALL|LINE=31|[ -n "$location" ] || location="none" +REC|VPN_SOURCE_ALL|LINE=32|fi +REC|VPN_SOURCE_ALL|LINE=34|case "$https" in +REC|VPN_SOURCE_ALL|LINE=35|200|301|302|401|403) ;; +REC|VPN_SOURCE_ALL|LINE=36|*) bad=$((bad+1)) ;; +REC|VPN_SOURCE_ALL|LINE=37|esac +REC|VPN_SOURCE_ALL|LINE=39|printf 'HOST|%s|fqdn=%s|dns=%s|http=%s|https=%s|location=%s|cert_ok=%s\n' \ +REC|VPN_SOURCE_ALL|LINE=40|"${h%%.*}" "$h" "${ip:-NONE}" "$http" "$https" "$location" "$cert_ok" >> "$tmp" +REC|VPN_SOURCE_ALL|LINE=41|done +REC|VPN_SOURCE_ALL|LINE=43|if [ "$bad" = "0" ]; then +REC|VPN_SOURCE_ALL|LINE=44|status=OK +REC|VPN_SOURCE_ALL|LINE=45|else +REC|VPN_SOURCE_ALL|LINE=46|status=ERROR +REC|VPN_SOURCE_ALL|LINE=47|fi +REC|VPN_SOURCE_ALL|LINE=49|{ +REC|VPN_SOURCE_ALL|LINE=50|cat "$tmp" +REC|VPN_SOURCE_ALL|LINE=51|printf 'STATUS=%s\n' "$status" +REC|VPN_SOURCE_ALL|LINE=52|} > "$H" +REC|VPN_SOURCE_ALL|LINE=54|cat "$H" +REC|VPN_SOURCE_ALL|LINE=55|[ "$status" = "OK" ] +REC|VPN_RESOLUTION|IDX=1|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true +REC|VPN_RESOLUTION|IDX=2|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true +REC|VPN_RESOLUTION|IDX=3|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true +REC|VPN_RESOLUTION|IDX=4|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true +REC|VPN_RESOLUTION|IDX=5|PVE01_COUNT=0|PVE01_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PVE02_COUNT=0|PVE02_SET_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|SETS_EQUAL=true +REC|RESOLVER|HOST=pve01|RESOLV_IS_SYMLINK=false +REC|RESOLVER|HOST=pve01|RESOLV_LINK_BASENAME=NONE +REC|RESOLVER|HOST=pve01|RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74 +REC|RESOLVER|HOST=pve01|NAMESERVER_COUNT=1 +REC|RESOLVER|HOST=pve01|SEARCH_LINE_COUNT=1 +REC|RESOLVER|HOST=pve01|REC|RESOLVER_NS_SHA|a6f12182e30cba92f9e91dc59ae899c758ecc34e1c3b9b06404eb741a56916ac +REC|RESOLVER|HOST=pve02|RESOLV_IS_SYMLINK=false +REC|RESOLVER|HOST=pve02|RESOLV_LINK_BASENAME=NONE +REC|RESOLVER|HOST=pve02|RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f +REC|RESOLVER|HOST=pve02|NAMESERVER_COUNT=1 +REC|RESOLVER|HOST=pve02|SEARCH_LINE_COUNT=1 +REC|RESOLVER|HOST=pve02|REC|RESOLVER_NS_SHA|3e731e83b523045ce83835503795f722459caf77bb2f8c0afcfaf75971b2bd78 +REC|PVE01_NS_CONFIG_REF|BASENAME=interfaces|PARENT=network|MATCH_COUNT=0|SHA=8558349d08d267b08a8abb490a591b5d4b7d539d3cc515b08f3165f3af0b8898 FAILED_UNIT_COUNT_NOW=2 REC|FAILED_UNIT_NOW|homelab-backup-audit.service REC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service -DECISION=PASS_PVE01_AUDIT_HELPERS_VPN_EXACT_RCA -NEXT_GATE=PVE01_AUDIT_HELPERS_VPN_REPAIR -TASK_RESULT=PVE01_AUDIT_HELPERS_VPN_EXACT_RCA +DECISION=PASS_PVE01_GFS_VPN_FINAL_RCA +NEXT_GATE=PVE01_HEALTH_HELPERS_AND_VPN_REPAIR +TASK_RESULT=PVE01_GFS_VPN_FINAL_RCA CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null MUTATION_OUTCOME=NO_MUTATION -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-GFS-VPN-FINAL-RCA-99696R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END