1
0
Files
homelab-public-context/runtime/history/SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1.txt
T
2026-10-06 08:40:51 +00:00

171 lines
14 KiB
Plaintext

CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=pve01-audit-helpers-vpn-exact-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=5cf24ab1bb98675b9bfe49e8bdc859479f976f0b83c5d2d1a01f49508b38be64
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd
SANITIZED_OUTPUT_SHA256=abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd
OUTPUT_BEGIN
PVE01_AUDIT_HELPERS_VPN_EXACT_RCA=V1
MODE=read-only
NO_REBOOT=true
NO_APT_UPDATE=true
NO_PACKAGE_MUTATION=true
NO_SERVICE_START_STOP_RESTART=true
NO_RESET_FAILED=true
NO_CONFIG_WRITE=true
NO_FILE_DELETE=true
NO_GUEST_MUTATION=true
NO_BACKUP_EXECUTION=true
NO_NETWORK_MUTATION=true
NODE_HOSTNAME=pve01
AUDIT_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3
REGISTRY_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49
HELPER_SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a
AUDIT_CONST_APPQ_PRESENT=true
AUDIT_CONST_APPQ_BASENAME=homelab-app-cloud-quorum-queue
AUDIT_CONST_APPQ_PARENT=sbin
AUDIT_CONST_APPQ_EXISTS=true
AUDIT_CONST_APPQ_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650
AUDIT_CONST_APPQ_PATH_SHA=75315ca46bfa3b06e14300fe6c03852aaf085c4d2843247dd39fd7e8b53f7800
AUDIT_CONST_GFSH_PRESENT=true
AUDIT_CONST_GFSH_BASENAME=homelab-backup-gfs-policy
AUDIT_CONST_GFSH_PARENT=libexec
AUDIT_CONST_GFSH_EXISTS=true
AUDIT_CONST_GFSH_SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94
AUDIT_CONST_GFSH_PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9
AUDIT_CONST_GFSP_PRESENT=true
AUDIT_CONST_GFSP_BASENAME=gfs-retention.json
AUDIT_CONST_GFSP_PARENT=homelab-backup
AUDIT_CONST_GFSP_EXISTS=true
AUDIT_CONST_GFSP_SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d
AUDIT_CONST_GFSP_PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d
AUDIT_CONST_REG_PRESENT=false
AUDIT_CONST_RET_PRESENT=true
AUDIT_CONST_RET_BASENAME=homelab-backup-retention-gc
AUDIT_CONST_RET_PARENT=sbin
AUDIT_CONST_RET_EXISTS=true
AUDIT_CONST_RET_SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909
AUDIT_CONST_RET_PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a
REC|AUDIT_HELPER_FILE|LABEL=APPQ|BASENAME=homelab-app-cloud-quorum-queue|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0
REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=6|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED
REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=7|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4
REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=8|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED
REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=11|[[ -x "$REGISTRY_HELPER" && ! -L "$REGISTRY_HELPER" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_MISSING\n' >&2; exit 69; }
REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=12|[[ "$(sha256sum "$REGISTRY_HELPER"|awk '{print $1}')" == "$REGISTRY_HELPER_SHA_EXPECTED" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\n' >&2; exit 65; }
REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=19|[[ "${#ALL_APPS[@]}" == 4 && "${#MIGRATE_APPS[@]}" == 4 ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_COUNT_INVALID\n' >&2; exit 65; }
REC|AUDIT_HELPER_FILE|LABEL=GFSH|BASENAME=homelab-backup-gfs-policy|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0
REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=3|CFG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_GFS_POLICY","PATH_REDACTED
REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=4|REG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_TARGET_REGISTRY","PATH_REDACTED
REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=6|print(f"GFS_POLICY_ERROR={msg}",file=sys.stderr); raise SystemExit(rc)
REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=10|if c.get("schema")!="homelab-backup-gfs-policy-v1" or c.get("policy_version")!=1: die("SCHEMA_INVALID",66)
REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=14|rows={x.get("logical_id"):x for x in r.get("targets",[])}
REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=48|if args==["validate"]: print("GFS_POLICY=OK"); return
REC|AUDIT_HELPER_FILE|LABEL=GFSP|BASENAME=gfs-retention.json|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0
REC|AUDIT_HELPER_SOURCE|LABEL=GFSP|LINE=18|"schema": "homelab-backup-gfs-policy-v1",
REC|AUDIT_HELPER_FILE|LABEL=RET|BASENAME=homelab-backup-retention-gc|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=8|RETENTION_ROOT=PATH_REDACTED
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=21|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=22|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=23|GFS_HELPER="${HOMELAB_BACKUP_GFS_POLICY_HELPER:-PATH_REDACTED
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=25|GFS_POLICY="${HOMELAB_BACKUP_GFS_POLICY:-PATH_REDACTED
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=27|[[ -x "$REGISTRY_HELPER" && ! -L "$REGISTRY_HELPER" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_MISSING\n' >&2; exit 69; }
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=28|[[ "$(sha256sum "$REGISTRY_HELPER"|awk '{print $1}')" == "$REGISTRY_HELPER_SHA_EXPECTED" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\n' >&2; exit 65; }
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=29|"$REGISTRY_HELPER" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_INVALID\n' >&2; exit 66; }
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=30|[[ -x "$GFS_HELPER" && ! -L "$GFS_HELPER" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_MISSING\n' >&2; exit 69; }
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=31|[[ "$(sha256sum "$GFS_HELPER"|awk '{print $1}')" == "$GFS_HELPER_SHA_EXPECTED" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\n' >&2; exit 65; }
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=32|HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID\n' >&2; exit 66; }
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=33|GFS_WEEKLY="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" weekly-buckets)"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=34|GFS_MONTHLY="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" monthly-buckets)"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=35|GFS_CRITICAL_COUNT="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" critical-logicals | wc -l)"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=74|jq -e --arg l "$logical" --arg g "$generation" '.schema=="homelab-cloud-quorum-payload-v1" and .logical_id==$l and .generation==$g and .status=="SEALED" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_delete_allowed==true and .local_payload_present==false and .local_cleanup=="OK"' <<<"$json" >PATH_REDACTED
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=106|f="$RETENTION_ROOT/transactions/$logical/$generation.json"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=111|f="$RETENTION_ROOT/transactions/$logical/$generation.json"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=117|[[ -d "$RETENTION_ROOT/transactions" ]] || { printf '0\n'; return 0; }
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=126|done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort)
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=131|[[ -n "${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}" || -n "${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}" ]]
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=134|local f="${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}" expected="${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}" actual line logical generation owner state_file state_sha count=0
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=160|jq -e --arg task "$TASK_ID" --arg change "$CHANGE_ID" '.schema=="homelab-backup-retention-transaction-v1" and .task_id==$task and .change_id==$change and (.logical_id|type)=="string" and (.generation|type)=="string" and (.state_sha256|type)=="string" and (.status=="PLANNED" or .status=="B_DELETED" or .status=="DONE")' "$f" >PATH_REDACTED || die "PINNED_TX_CONTENT_INVALID_$f" 66
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=165|expected_path="$RETENTION_ROOT/transactions/$logical/$generation.json"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=170|local root="$RETENTION_ROOT/transactions" f
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=184|txf="$RETENTION_ROOT/transactions/$logical/$generation.json"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=188|grep -Fqx "$(pinned_tx_identity "$txf")" "$HOMELAB_RETENTION_PINNED_DELETE_FILE" || die "PINNED_PLAN_PROTECT_NOT_PINNED_${logical}_${generation}" 66
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=199|if [[ -d "$RETENTION_ROOT/transactions" ]]; then
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=200|while IFS= read -r f; do pinned_tx_identity "$f" >>"$actual"; done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort)
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=203|LC_ALL=C sort "$HOMELAB_RETENTION_PINNED_DELETE_FILE" >"$expected"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=208|printf 'PINNED_DELETE_SCOPE=OK|stage=%s|count=%s|sha256=%s\n' "$stage" "$count_expected" "$HOMELAB_RETENTION_PINNED_DELETE_SHA256"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=218|if ! HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" is-critical "$logical"; then return 0; fi
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=226|out="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" select "$logical" <"$TMPROOT/$logical.gfs-eligible")" || die "GFS_SELECT_FAILED_$logical" 66
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=240|local now logical node keep lifecycle open_count file json schema lid gen status sha sealed_at epoch age idx tx gfs_bucket candidates=0 protected=0 kept=0 gfs_kept=0
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=252|schema="$(jq -r '.schema/PATH_REDACTED <<<"$json")"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=256|if [[ "$schema" != homelab-cloud-quorum-payload-v1 || "$lid" != "$logical" ]] || ! safe_generation "$logical" "$gen"; then
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=321|path="$RETENTION_ROOT/transactions/$logical/$generation.json"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=324|jq -n --arg task "$TASK_ID" --arg change "$CHANGE_ID" --arg l "$logical" --arg g "$generation" --arg s "$state_sha" --arg status "$status" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson aa "$a_absent" --argjson ba "$b_absent" '{schema:"homelab-backup-retention-transaction-v1",task_id:$task,change_id:$change,logical_id:$l,generation:$g,state_sha256:$s,status:$status,updated_at_utc:$at,remote_a_absent:$aa,remote_b_absent:$ba}' >"$tmp"
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=357|[[ -d "$RETENTION_ROOT/transactions" ]] || return 0
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=381|done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort)
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=415|if find "$RETENTION_ROOT/transactions" -type f -name '*.json' -exec jq -e '.status=="DONE" and .remote_a_absent==true and .remote_b_absent==true' {} \; -print 2>"$TMPROOT/tx-find.err" | grep -q .; then hist=true; fi
REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=418|jq -n --arg task "$TASK_ID" --arg change "$CHANGE_ID" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson canary "$canary_proven" --argjson hist "$hist" --argjson candidates "${candidates:-0}" --argjson protected "${protected:-0}" --argjson kept "${kept:-0}" --argjson gfs_kept "${gfs_kept:-0}" --argjson deleted "$deleted" --argjson gfs_critical "$GFS_CRITICAL_COUNT" --argjson gfs_weekly "$GFS_WEEKLY" --argjson gfs_monthly "$GFS_MONTHLY" '{schema:"homelab-backup-retention-v1",status:"OK",task_id:
APPQ_FRESHNESS_PLAN_RC=65
REC|APPQ_ERR|APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT
GFS_VALIDATE_RC=66
REC|GFS_ERR|GFS_POLICY_ERROR=READ_OR_JSON_INVALID
RETENTION_CONTRACT_RC=65
REC|RET_ERR|GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT
VPN_EXECSTART_COUNT=1
REC|VPN_EXEC_PATH|IDX=1|BASENAME=bash|PARENT=bin|SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f|EXEC=true
REC|VPN_EXEC_PATH|IDX=2|BASENAME=homelab-check-private-vpn-hosts|PARENT=sbin|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82|EXEC=true
REC|VPN_EXEC_PATH|IDX=3|BASENAME=private-vpn-hosts.txt|PARENT=homelab-health|SHA=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d|EXEC=false
VPN_SCRIPT_FOUND=true
VPN_SCRIPT_BASENAME=homelab-check-private-vpn-hosts
VPN_SCRIPT_SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82
VPN_SCRIPT_MUTATION_PATTERN_COUNT=1
VPN_LITERAL_NAME_COUNT=5
REC|VPN_SOURCE|LINE=6|VPN_HOST_REDACTED
REC|VPN_SOURCE|LINE=7|VPN_HOST_REDACTED
REC|VPN_SOURCE|LINE=8|VPN_HOST_REDACTED
REC|VPN_SOURCE|LINE=9|VPN_HOST_REDACTED
REC|VPN_SOURCE|LINE=10|VPN_HOST_REDACTED
REC|VPN_SOURCE|LINE=19|ip="$(getent hosts "$h" 2>/dev/null | awk '{print $1}' | head -n1 || true)"
VPN_SCRIPT_DIRECT_SKIPPED_MUTATION_SCAN=true
PVE01_RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74
PVE01_NAMESERVER_COUNT=1
PVE01_VPN_HOSTS_LINE_COUNT=0
PVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f
PVE2_NAMESERVER_COUNT=1
PVE2_VPN_HOSTS_LINE_COUNT=0
FAILED_UNIT_COUNT_NOW=2
REC|FAILED_UNIT_NOW|homelab-backup-audit.service
REC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service
DECISION=PASS_PVE01_AUDIT_HELPERS_VPN_EXACT_RCA
NEXT_GATE=PVE01_AUDIT_HELPERS_VPN_REPAIR
TASK_RESULT=PVE01_AUDIT_HELPERS_VPN_EXACT_RCA
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END