diff --git a/runtime/history/SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1.json b/runtime/history/SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1.json new file mode 100644 index 00000000..878c0bdb --- /dev/null +++ b/runtime/history/SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "pve01-audit-helpers-vpn-exact-rca", + "started_at_utc": "2026-10-06T08:40:50Z", + "finished_at_utc": "2026-10-06T08:40:50Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "5cf24ab1bb98675b9bfe49e8bdc859479f976f0b83c5d2d1a01f49508b38be64", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd", + "sanitized_output_sha256": "abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "PVE01_AUDIT_HELPERS_VPN_EXACT_RCA=V1\nMODE=read-only\nNO_REBOOT=true\nNO_APT_UPDATE=true\nNO_PACKAGE_MUTATION=true\nNO_SERVICE_START_STOP_RESTART=true\nNO_RESET_FAILED=true\nNO_CONFIG_WRITE=true\nNO_FILE_DELETE=true\nNO_GUEST_MUTATION=true\nNO_BACKUP_EXECUTION=true\nNO_NETWORK_MUTATION=true\nNODE_HOSTNAME=pve01\nAUDIT_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3\nREGISTRY_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49\nHELPER_SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a\nAUDIT_CONST_APPQ_PRESENT=true\nAUDIT_CONST_APPQ_BASENAME=homelab-app-cloud-quorum-queue\nAUDIT_CONST_APPQ_PARENT=sbin\nAUDIT_CONST_APPQ_EXISTS=true\nAUDIT_CONST_APPQ_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\nAUDIT_CONST_APPQ_PATH_SHA=75315ca46bfa3b06e14300fe6c03852aaf085c4d2843247dd39fd7e8b53f7800\nAUDIT_CONST_GFSH_PRESENT=true\nAUDIT_CONST_GFSH_BASENAME=homelab-backup-gfs-policy\nAUDIT_CONST_GFSH_PARENT=libexec\nAUDIT_CONST_GFSH_EXISTS=true\nAUDIT_CONST_GFSH_SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94\nAUDIT_CONST_GFSH_PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9\nAUDIT_CONST_GFSP_PRESENT=true\nAUDIT_CONST_GFSP_BASENAME=gfs-retention.json\nAUDIT_CONST_GFSP_PARENT=homelab-backup\nAUDIT_CONST_GFSP_EXISTS=true\nAUDIT_CONST_GFSP_SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d\nAUDIT_CONST_GFSP_PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d\nAUDIT_CONST_REG_PRESENT=false\nAUDIT_CONST_RET_PRESENT=true\nAUDIT_CONST_RET_BASENAME=homelab-backup-retention-gc\nAUDIT_CONST_RET_PARENT=sbin\nAUDIT_CONST_RET_EXISTS=true\nAUDIT_CONST_RET_SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909\nAUDIT_CONST_RET_PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a\nREC|AUDIT_HELPER_FILE|LABEL=APPQ|BASENAME=homelab-app-cloud-quorum-queue|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=6|REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=7|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=8|REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=11|[[ -x \"$REGISTRY_HELPER\" && ! -L \"$REGISTRY_HELPER\" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_MISSING\\n' >&2; exit 69; }\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=12|[[ \"$(sha256sum \"$REGISTRY_HELPER\"|awk '{print $1}')\" == \"$REGISTRY_HELPER_SHA_EXPECTED\" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=19|[[ \"${#ALL_APPS[@]}\" == 4 && \"${#MIGRATE_APPS[@]}\" == 4 ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_COUNT_INVALID\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_FILE|LABEL=GFSH|BASENAME=homelab-backup-gfs-policy|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=3|CFG=pathlib.Path(os.environ.get(\"HOMELAB_BACKUP_GFS_POLICY\",\"PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=4|REG=pathlib.Path(os.environ.get(\"HOMELAB_BACKUP_TARGET_REGISTRY\",\"PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=6|print(f\"GFS_POLICY_ERROR={msg}\",file=sys.stderr); raise SystemExit(rc)\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=10|if c.get(\"schema\")!=\"homelab-backup-gfs-policy-v1\" or c.get(\"policy_version\")!=1: die(\"SCHEMA_INVALID\",66)\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=14|rows={x.get(\"logical_id\"):x for x in r.get(\"targets\",[])}\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=48|if args==[\"validate\"]: print(\"GFS_POLICY=OK\"); return\nREC|AUDIT_HELPER_FILE|LABEL=GFSP|BASENAME=gfs-retention.json|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSP|LINE=18|\"schema\": \"homelab-backup-gfs-policy-v1\",\nREC|AUDIT_HELPER_FILE|LABEL=RET|BASENAME=homelab-backup-retention-gc|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=8|RETENTION_ROOT=PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=21|REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=22|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=23|GFS_HELPER=\"${HOMELAB_BACKUP_GFS_POLICY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=25|GFS_POLICY=\"${HOMELAB_BACKUP_GFS_POLICY:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=27|[[ -x \"$REGISTRY_HELPER\" && ! -L \"$REGISTRY_HELPER\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_MISSING\\n' >&2; exit 69; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=28|[[ \"$(sha256sum \"$REGISTRY_HELPER\"|awk '{print $1}')\" == \"$REGISTRY_HELPER_SHA_EXPECTED\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=29|\"$REGISTRY_HELPER\" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_INVALID\\n' >&2; exit 66; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=30|[[ -x \"$GFS_HELPER\" && ! -L \"$GFS_HELPER\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_MISSING\\n' >&2; exit 69; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=31|[[ \"$(sha256sum \"$GFS_HELPER\"|awk '{print $1}')\" == \"$GFS_HELPER_SHA_EXPECTED\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=32|HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID\\n' >&2; exit 66; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=33|GFS_WEEKLY=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" weekly-buckets)\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=34|GFS_MONTHLY=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" monthly-buckets)\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=35|GFS_CRITICAL_COUNT=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" critical-logicals | wc -l)\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=74|jq -e --arg l \"$logical\" --arg g \"$generation\" '.schema==\"homelab-cloud-quorum-payload-v1\" and .logical_id==$l and .generation==$g and .status==\"SEALED\" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_delete_allowed==true and .local_payload_present==false and .local_cleanup==\"OK\"' <<<\"$json\" >PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=106|f=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=111|f=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=117|[[ -d \"$RETENTION_ROOT/transactions\" ]] || { printf '0\\n'; return 0; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=126|done < <(find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -print | LC_ALL=C sort)\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=131|[[ -n \"${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}\" || -n \"${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}\" ]]\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=134|local f=\"${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}\" expected=\"${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}\" actual line logical generation owner state_file state_sha count=0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=160|jq -e --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" '.schema==\"homelab-backup-retention-transaction-v1\" and .task_id==$task and .change_id==$change and (.logical_id|type)==\"string\" and (.generation|type)==\"string\" and (.state_sha256|type)==\"string\" and (.status==\"PLANNED\" or .status==\"B_DELETED\" or .status==\"DONE\")' \"$f\" >PATH_REDACTED || die \"PINNED_TX_CONTENT_INVALID_$f\" 66\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=165|expected_path=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=170|local root=\"$RETENTION_ROOT/transactions\" f\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=184|txf=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=188|grep -Fqx \"$(pinned_tx_identity \"$txf\")\" \"$HOMELAB_RETENTION_PINNED_DELETE_FILE\" || die \"PINNED_PLAN_PROTECT_NOT_PINNED_${logical}_${generation}\" 66\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=199|if [[ -d \"$RETENTION_ROOT/transactions\" ]]; then\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=200|while IFS= read -r f; do pinned_tx_identity \"$f\" >>\"$actual\"; done < <(find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -print | LC_ALL=C sort)\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=203|LC_ALL=C sort \"$HOMELAB_RETENTION_PINNED_DELETE_FILE\" >\"$expected\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=208|printf 'PINNED_DELETE_SCOPE=OK|stage=%s|count=%s|sha256=%s\\n' \"$stage\" \"$count_expected\" \"$HOMELAB_RETENTION_PINNED_DELETE_SHA256\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=218|if ! HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" is-critical \"$logical\"; then return 0; fi\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=226|out=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" select \"$logical\" <\"$TMPROOT/$logical.gfs-eligible\")\" || die \"GFS_SELECT_FAILED_$logical\" 66\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=240|local now logical node keep lifecycle open_count file json schema lid gen status sha sealed_at epoch age idx tx gfs_bucket candidates=0 protected=0 kept=0 gfs_kept=0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=252|schema=\"$(jq -r '.schema/PATH_REDACTED <<<\"$json\")\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=256|if [[ \"$schema\" != homelab-cloud-quorum-payload-v1 || \"$lid\" != \"$logical\" ]] || ! safe_generation \"$logical\" \"$gen\"; then\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=321|path=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=324|jq -n --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" --arg l \"$logical\" --arg g \"$generation\" --arg s \"$state_sha\" --arg status \"$status\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --argjson aa \"$a_absent\" --argjson ba \"$b_absent\" '{schema:\"homelab-backup-retention-transaction-v1\",task_id:$task,change_id:$change,logical_id:$l,generation:$g,state_sha256:$s,status:$status,updated_at_utc:$at,remote_a_absent:$aa,remote_b_absent:$ba}' >\"$tmp\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=357|[[ -d \"$RETENTION_ROOT/transactions\" ]] || return 0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=381|done < <(find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -print | LC_ALL=C sort)\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=415|if find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -exec jq -e '.status==\"DONE\" and .remote_a_absent==true and .remote_b_absent==true' {} \\; -print 2>\"$TMPROOT/tx-find.err\" | grep -q .; then hist=true; fi\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=418|jq -n --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --argjson canary \"$canary_proven\" --argjson hist \"$hist\" --argjson candidates \"${candidates:-0}\" --argjson protected \"${protected:-0}\" --argjson kept \"${kept:-0}\" --argjson gfs_kept \"${gfs_kept:-0}\" --argjson deleted \"$deleted\" --argjson gfs_critical \"$GFS_CRITICAL_COUNT\" --argjson gfs_weekly \"$GFS_WEEKLY\" --argjson gfs_monthly \"$GFS_MONTHLY\" '{schema:\"homelab-backup-retention-v1\",status:\"OK\",task_id:\nAPPQ_FRESHNESS_PLAN_RC=65\nREC|APPQ_ERR|APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\nGFS_VALIDATE_RC=66\nREC|GFS_ERR|GFS_POLICY_ERROR=READ_OR_JSON_INVALID\nRETENTION_CONTRACT_RC=65\nREC|RET_ERR|GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\nVPN_EXECSTART_COUNT=1\nREC|VPN_EXEC_PATH|IDX=1|BASENAME=bash|PARENT=bin|SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f|EXEC=true\nREC|VPN_EXEC_PATH|IDX=2|BASENAME=homelab-check-private-vpn-hosts|PARENT=sbin|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82|EXEC=true\nREC|VPN_EXEC_PATH|IDX=3|BASENAME=private-vpn-hosts.txt|PARENT=homelab-health|SHA=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d|EXEC=false\nVPN_SCRIPT_FOUND=true\nVPN_SCRIPT_BASENAME=homelab-check-private-vpn-hosts\nVPN_SCRIPT_SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82\nVPN_SCRIPT_MUTATION_PATTERN_COUNT=1\nVPN_LITERAL_NAME_COUNT=5\nREC|VPN_SOURCE|LINE=6|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=7|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=8|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=9|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=10|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=19|ip=\"$(getent hosts \"$h\" 2>/dev/null | awk '{print $1}' | head -n1 || true)\"\nVPN_SCRIPT_DIRECT_SKIPPED_MUTATION_SCAN=true\nPVE01_RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74\nPVE01_NAMESERVER_COUNT=1\nPVE01_VPN_HOSTS_LINE_COUNT=0\nPVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f\nPVE2_NAMESERVER_COUNT=1\nPVE2_VPN_HOSTS_LINE_COUNT=0\nFAILED_UNIT_COUNT_NOW=2\nREC|FAILED_UNIT_NOW|homelab-backup-audit.service\nREC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service\nDECISION=PASS_PVE01_AUDIT_HELPERS_VPN_EXACT_RCA\nNEXT_GATE=PVE01_AUDIT_HELPERS_VPN_REPAIR\nTASK_RESULT=PVE01_AUDIT_HELPERS_VPN_EXACT_RCA\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1.txt b/runtime/history/SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1.txt new file mode 100644 index 00000000..7585e107 --- /dev/null +++ b/runtime/history/SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1.txt @@ -0,0 +1,170 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=pve01-audit-helpers-vpn-exact-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=5cf24ab1bb98675b9bfe49e8bdc859479f976f0b83c5d2d1a01f49508b38be64 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd +SANITIZED_OUTPUT_SHA256=abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd +OUTPUT_BEGIN +PVE01_AUDIT_HELPERS_VPN_EXACT_RCA=V1 +MODE=read-only +NO_REBOOT=true +NO_APT_UPDATE=true +NO_PACKAGE_MUTATION=true +NO_SERVICE_START_STOP_RESTART=true +NO_RESET_FAILED=true +NO_CONFIG_WRITE=true +NO_FILE_DELETE=true +NO_GUEST_MUTATION=true +NO_BACKUP_EXECUTION=true +NO_NETWORK_MUTATION=true +NODE_HOSTNAME=pve01 +AUDIT_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3 +REGISTRY_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49 +HELPER_SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a +AUDIT_CONST_APPQ_PRESENT=true +AUDIT_CONST_APPQ_BASENAME=homelab-app-cloud-quorum-queue +AUDIT_CONST_APPQ_PARENT=sbin +AUDIT_CONST_APPQ_EXISTS=true +AUDIT_CONST_APPQ_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650 +AUDIT_CONST_APPQ_PATH_SHA=75315ca46bfa3b06e14300fe6c03852aaf085c4d2843247dd39fd7e8b53f7800 +AUDIT_CONST_GFSH_PRESENT=true +AUDIT_CONST_GFSH_BASENAME=homelab-backup-gfs-policy +AUDIT_CONST_GFSH_PARENT=libexec +AUDIT_CONST_GFSH_EXISTS=true +AUDIT_CONST_GFSH_SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94 +AUDIT_CONST_GFSH_PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9 +AUDIT_CONST_GFSP_PRESENT=true +AUDIT_CONST_GFSP_BASENAME=gfs-retention.json +AUDIT_CONST_GFSP_PARENT=homelab-backup +AUDIT_CONST_GFSP_EXISTS=true +AUDIT_CONST_GFSP_SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d +AUDIT_CONST_GFSP_PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d +AUDIT_CONST_REG_PRESENT=false +AUDIT_CONST_RET_PRESENT=true +AUDIT_CONST_RET_BASENAME=homelab-backup-retention-gc +AUDIT_CONST_RET_PARENT=sbin +AUDIT_CONST_RET_EXISTS=true +AUDIT_CONST_RET_SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909 +AUDIT_CONST_RET_PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a +REC|AUDIT_HELPER_FILE|LABEL=APPQ|BASENAME=homelab-app-cloud-quorum-queue|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0 +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=6|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=7|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4 +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=8|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=11|[[ -x "$REGISTRY_HELPER" && ! -L "$REGISTRY_HELPER" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_MISSING\n' >&2; exit 69; } +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=12|[[ "$(sha256sum "$REGISTRY_HELPER"|awk '{print $1}')" == "$REGISTRY_HELPER_SHA_EXPECTED" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\n' >&2; exit 65; } +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=19|[[ "${#ALL_APPS[@]}" == 4 && "${#MIGRATE_APPS[@]}" == 4 ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_COUNT_INVALID\n' >&2; exit 65; } +REC|AUDIT_HELPER_FILE|LABEL=GFSH|BASENAME=homelab-backup-gfs-policy|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0 +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=3|CFG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_GFS_POLICY","PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=4|REG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_TARGET_REGISTRY","PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=6|print(f"GFS_POLICY_ERROR={msg}",file=sys.stderr); raise SystemExit(rc) +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=10|if c.get("schema")!="homelab-backup-gfs-policy-v1" or c.get("policy_version")!=1: die("SCHEMA_INVALID",66) +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=14|rows={x.get("logical_id"):x for x in r.get("targets",[])} +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=48|if args==["validate"]: print("GFS_POLICY=OK"); return +REC|AUDIT_HELPER_FILE|LABEL=GFSP|BASENAME=gfs-retention.json|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0 +REC|AUDIT_HELPER_SOURCE|LABEL=GFSP|LINE=18|"schema": "homelab-backup-gfs-policy-v1", +REC|AUDIT_HELPER_FILE|LABEL=RET|BASENAME=homelab-backup-retention-gc|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=8|RETENTION_ROOT=PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=21|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=22|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=23|GFS_HELPER="${HOMELAB_BACKUP_GFS_POLICY_HELPER:-PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=25|GFS_POLICY="${HOMELAB_BACKUP_GFS_POLICY:-PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=27|[[ -x "$REGISTRY_HELPER" && ! -L "$REGISTRY_HELPER" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_MISSING\n' >&2; exit 69; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=28|[[ "$(sha256sum "$REGISTRY_HELPER"|awk '{print $1}')" == "$REGISTRY_HELPER_SHA_EXPECTED" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\n' >&2; exit 65; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=29|"$REGISTRY_HELPER" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_INVALID\n' >&2; exit 66; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=30|[[ -x "$GFS_HELPER" && ! -L "$GFS_HELPER" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_MISSING\n' >&2; exit 69; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=31|[[ "$(sha256sum "$GFS_HELPER"|awk '{print $1}')" == "$GFS_HELPER_SHA_EXPECTED" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\n' >&2; exit 65; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=32|HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID\n' >&2; exit 66; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=33|GFS_WEEKLY="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" weekly-buckets)" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=34|GFS_MONTHLY="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" monthly-buckets)" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=35|GFS_CRITICAL_COUNT="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" critical-logicals | wc -l)" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=74|jq -e --arg l "$logical" --arg g "$generation" '.schema=="homelab-cloud-quorum-payload-v1" and .logical_id==$l and .generation==$g and .status=="SEALED" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_delete_allowed==true and .local_payload_present==false and .local_cleanup=="OK"' <<<"$json" >PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=106|f="$RETENTION_ROOT/transactions/$logical/$generation.json" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=111|f="$RETENTION_ROOT/transactions/$logical/$generation.json" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=117|[[ -d "$RETENTION_ROOT/transactions" ]] || { printf '0\n'; return 0; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=126|done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort) +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=131|[[ -n "${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}" || -n "${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}" ]] +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=134|local f="${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}" expected="${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}" actual line logical generation owner state_file state_sha count=0 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=160|jq -e --arg task "$TASK_ID" --arg change "$CHANGE_ID" '.schema=="homelab-backup-retention-transaction-v1" and .task_id==$task and .change_id==$change and (.logical_id|type)=="string" and (.generation|type)=="string" and (.state_sha256|type)=="string" and (.status=="PLANNED" or .status=="B_DELETED" or .status=="DONE")' "$f" >PATH_REDACTED || die "PINNED_TX_CONTENT_INVALID_$f" 66 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=165|expected_path="$RETENTION_ROOT/transactions/$logical/$generation.json" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=170|local root="$RETENTION_ROOT/transactions" f +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=184|txf="$RETENTION_ROOT/transactions/$logical/$generation.json" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=188|grep -Fqx "$(pinned_tx_identity "$txf")" "$HOMELAB_RETENTION_PINNED_DELETE_FILE" || die "PINNED_PLAN_PROTECT_NOT_PINNED_${logical}_${generation}" 66 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=199|if [[ -d "$RETENTION_ROOT/transactions" ]]; then +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=200|while IFS= read -r f; do pinned_tx_identity "$f" >>"$actual"; done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort) +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=203|LC_ALL=C sort "$HOMELAB_RETENTION_PINNED_DELETE_FILE" >"$expected" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=208|printf 'PINNED_DELETE_SCOPE=OK|stage=%s|count=%s|sha256=%s\n' "$stage" "$count_expected" "$HOMELAB_RETENTION_PINNED_DELETE_SHA256" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=218|if ! HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" is-critical "$logical"; then return 0; fi +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=226|out="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" select "$logical" <"$TMPROOT/$logical.gfs-eligible")" || die "GFS_SELECT_FAILED_$logical" 66 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=240|local now logical node keep lifecycle open_count file json schema lid gen status sha sealed_at epoch age idx tx gfs_bucket candidates=0 protected=0 kept=0 gfs_kept=0 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=252|schema="$(jq -r '.schema/PATH_REDACTED <<<"$json")" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=256|if [[ "$schema" != homelab-cloud-quorum-payload-v1 || "$lid" != "$logical" ]] || ! safe_generation "$logical" "$gen"; then +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=321|path="$RETENTION_ROOT/transactions/$logical/$generation.json" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=324|jq -n --arg task "$TASK_ID" --arg change "$CHANGE_ID" --arg l "$logical" --arg g "$generation" --arg s "$state_sha" --arg status "$status" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson aa "$a_absent" --argjson ba "$b_absent" '{schema:"homelab-backup-retention-transaction-v1",task_id:$task,change_id:$change,logical_id:$l,generation:$g,state_sha256:$s,status:$status,updated_at_utc:$at,remote_a_absent:$aa,remote_b_absent:$ba}' >"$tmp" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=357|[[ -d "$RETENTION_ROOT/transactions" ]] || return 0 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=381|done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort) +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=415|if find "$RETENTION_ROOT/transactions" -type f -name '*.json' -exec jq -e '.status=="DONE" and .remote_a_absent==true and .remote_b_absent==true' {} \; -print 2>"$TMPROOT/tx-find.err" | grep -q .; then hist=true; fi +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=418|jq -n --arg task "$TASK_ID" --arg change "$CHANGE_ID" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson canary "$canary_proven" --argjson hist "$hist" --argjson candidates "${candidates:-0}" --argjson protected "${protected:-0}" --argjson kept "${kept:-0}" --argjson gfs_kept "${gfs_kept:-0}" --argjson deleted "$deleted" --argjson gfs_critical "$GFS_CRITICAL_COUNT" --argjson gfs_weekly "$GFS_WEEKLY" --argjson gfs_monthly "$GFS_MONTHLY" '{schema:"homelab-backup-retention-v1",status:"OK",task_id: +APPQ_FRESHNESS_PLAN_RC=65 +REC|APPQ_ERR|APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT +GFS_VALIDATE_RC=66 +REC|GFS_ERR|GFS_POLICY_ERROR=READ_OR_JSON_INVALID +RETENTION_CONTRACT_RC=65 +REC|RET_ERR|GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT +VPN_EXECSTART_COUNT=1 +REC|VPN_EXEC_PATH|IDX=1|BASENAME=bash|PARENT=bin|SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f|EXEC=true +REC|VPN_EXEC_PATH|IDX=2|BASENAME=homelab-check-private-vpn-hosts|PARENT=sbin|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82|EXEC=true +REC|VPN_EXEC_PATH|IDX=3|BASENAME=private-vpn-hosts.txt|PARENT=homelab-health|SHA=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d|EXEC=false +VPN_SCRIPT_FOUND=true +VPN_SCRIPT_BASENAME=homelab-check-private-vpn-hosts +VPN_SCRIPT_SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82 +VPN_SCRIPT_MUTATION_PATTERN_COUNT=1 +VPN_LITERAL_NAME_COUNT=5 +REC|VPN_SOURCE|LINE=6|VPN_HOST_REDACTED +REC|VPN_SOURCE|LINE=7|VPN_HOST_REDACTED +REC|VPN_SOURCE|LINE=8|VPN_HOST_REDACTED +REC|VPN_SOURCE|LINE=9|VPN_HOST_REDACTED +REC|VPN_SOURCE|LINE=10|VPN_HOST_REDACTED +REC|VPN_SOURCE|LINE=19|ip="$(getent hosts "$h" 2>/dev/null | awk '{print $1}' | head -n1 || true)" +VPN_SCRIPT_DIRECT_SKIPPED_MUTATION_SCAN=true +PVE01_RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74 +PVE01_NAMESERVER_COUNT=1 +PVE01_VPN_HOSTS_LINE_COUNT=0 +PVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f +PVE2_NAMESERVER_COUNT=1 +PVE2_VPN_HOSTS_LINE_COUNT=0 +FAILED_UNIT_COUNT_NOW=2 +REC|FAILED_UNIT_NOW|homelab-backup-audit.service +REC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service +DECISION=PASS_PVE01_AUDIT_HELPERS_VPN_EXACT_RCA +NEXT_GATE=PVE01_AUDIT_HELPERS_VPN_REPAIR +TASK_RESULT=PVE01_AUDIT_HELPERS_VPN_EXACT_RCA +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 29238b9c..878c0bdb 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1", + "command_id": "SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "pve01-remaining-failed-units-rca", - "started_at_utc": "2026-10-06T08:36:35Z", - "finished_at_utc": "2026-10-06T08:36:41Z", + "component": "pve01-audit-helpers-vpn-exact-rca", + "started_at_utc": "2026-10-06T08:40:50Z", + "finished_at_utc": "2026-10-06T08:40:50Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "18e8f9dc2347e65c7d6310e27f48bdd48e3a1978f434af5fee6ab8ab48185dac", + "command_sha256": "5cf24ab1bb98675b9bfe49e8bdc859479f976f0b83c5d2d1a01f49508b38be64", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "4228b00b8b8cab02f55eb05e3b5c238c8acdf45fcc4c59e66c5978d7c6ee6fd1", - "sanitized_output_sha256": "4228b00b8b8cab02f55eb05e3b5c238c8acdf45fcc4c59e66c5978d7c6ee6fd1", + "raw_evidence_sha256": "abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd", + "sanitized_output_sha256": "abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "PVE01_REMAINING_FAILED_UNITS_RCA=V1\nMODE=read-only\nNO_REBOOT=true\nNO_APT_UPDATE=true\nNO_PACKAGE_MUTATION=true\nNO_SERVICE_START_STOP_RESTART=true\nNO_RESET_FAILED=true\nNO_CONFIG_WRITE=true\nNO_FILE_DELETE=true\nNO_GUEST_MUTATION=true\nNO_BACKUP_EXECUTION=true\nNO_NETWORK_MUTATION=true\nNODE_HOSTNAME=pve01\nUNIFIED_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49\nV2_SHA=7f413e4a3bc276be668ad0d7a877a0f2b2bd64bf1dead9cb929d966b25a44e25\nHELPER_SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a\nPROBE_SHA=44ea82521afb37e862ec7fb7f0e3eeb2f0f582b784506d8dec23f73ccea86050\nWORKER_SHA=a86bc027eefc37bbbcb9ad9b18c1c2c2a4f4ef9172a9cd9918a00f1d94c9e19a\nHELPER_VALIDATE_RC=0\nCANONICAL_PROBE_RC=0\nCANONICAL_PROBE_SEALED_PROVEN=true\nBACKUP_AUDIT_EXEC_PRESENT=true\nBACKUP_AUDIT_EXEC_BASENAME=homelab-backup-audit\nBACKUP_AUDIT_EXEC_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3\nBACKUP_AUDIT_RESULT_NOW=exit-code\nBACKUP_AUDIT_EXEC_STATUS_NOW=1\nBACKUP_AUDIT_FRESH_LINES=48\nREC|AUDIT_FRESH|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|legacy_age_seconds=14441|generation=app-filebrowser-pve01-20261006T043010Z\nREC|AUDIT_FRESH|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|legacy_age_seconds=14279|generation=app-hbarhub-pve01-20261006T043148Z\nREC|AUDIT_FRESH|TARGET|logical_id=app-immich|kind=APP|state=FRESH|legacy_age_seconds=14385|generation=app-immich-pve01-20261006T043031Z\nREC|AUDIT_FRESH|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|legacy_age_seconds=43247|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z\nREC|AUDIT_FRESH|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=370|generation=control-bootstrap-v2-pve01-20260916T120057Z\nREC|AUDIT_FRESH|TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=3875|legacy_age_seconds=2775090|replica_count=2|state_id=ct110|generation=v2-ct110-20261006T072242Z-655b3dc1c042\nREC|AUDIT_FRESH|TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=1522|replica_count=2|state_id=ct111|generation=v2-ct111-20261006T080259Z-417f9545ef1b\nREC|AUDIT_FRESH|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=1|legacy_age_seconds=2813585|replica_count=2|state_id=ct112|generation=v2-ct112-20261006T082950Z-2538063ab2f8\nREC|AUDIT_FRESH|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=67118|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7\nREC|AUDIT_FRESH|TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=5102|replica_count=2|state_id=vm130|generation=v2-vm130-20261006T064416Z-6541a84254d2\nREC|AUDIT_FRESH|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=65014|legacy_age_seconds=83432|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0\nREC|AUDIT_FRESH|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51585|legacy_age_seconds=1719827|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94\nREC|AUDIT_FRESH|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=47820|legacy_age_seconds=1749359|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d\nREC|AUDIT_FRESH|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=62823|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9\nREC|AUDIT_FRESH|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=30496|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9\nREC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65\nREC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED\nREC|AUDIT_FRESH|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65\nREC|AUDIT_FRESH|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nREC|AUDIT_FRESH|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|legacy_age_seconds=14739|generation=app-filebrowser-pve01-20261006T043010Z\nREC|AUDIT_FRESH|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|legacy_age_seconds=14577|generation=app-hbarhub-pve01-20261006T043148Z\nREC|AUDIT_FRESH|TARGET|logical_id=app-immich|kind=APP|state=FRESH|legacy_age_seconds=14683|generation=app-immich-pve01-20261006T043031Z\nREC|AUDIT_FRESH|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|legacy_age_seconds=43545|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z\nREC|AUDIT_FRESH|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=668|generation=control-bootstrap-v2-pve01-20260916T120057Z\nREC|AUDIT_FRESH|TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=4173|legacy_age_seconds=2775388|replica_count=2|state_id=ct110|generation=v2-ct110-20261006T072242Z-655b3dc1c042\nREC|AUDIT_FRESH|TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=1820|replica_count=2|state_id=ct111|generation=v2-ct111-20261006T080259Z-417f9545ef1b\nREC|AUDIT_FRESH|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=299|legacy_age_seconds=2813883|replica_count=2|state_id=ct112|generation=v2-ct112-20261006T082950Z-2538063ab2f8\nREC|AUDIT_FRESH|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=67416|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7\nREC|AUDIT_FRESH|TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=5400|replica_count=2|state_id=vm130|generation=v2-vm130-20261006T064416Z-6541a84254d2\nREC|AUDIT_FRESH|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=65312|legacy_age_seconds=83730|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0\nREC|AUDIT_FRESH|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51883|legacy_age_seconds=1720125|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94\nREC|AUDIT_FRESH|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=48118|legacy_age_seconds=1749657|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d\nREC|AUDIT_FRESH|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=63121|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9\nREC|AUDIT_FRESH|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=30794|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9\nREC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65\nREC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED\nREC|AUDIT_FRESH|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65\nREC|AUDIT_FRESH|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nAUDIT_SOURCE_APP_FRESHNESS_PLAN_FAILED_HIT_COUNT=1\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=112|resources={}\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=113|if rrc==0:\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=114|try:\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=115|for x in json.loads(rout): resources[str(x.get(\"vmid\"))]=x\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=116|except Exception: pass\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=117|\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=118|qrc,qout,qerr=run([APPQ,\"freshness-plan\"],180)\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=119|app_dec=parse_app_plan(qout if qrc==0 else \"\")\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=120|if qrc!=0: issues.append({\"severity\":\"CRITICAL\",\"code\":\"APP_FRESHNESS_PLAN_FAILED\",\"logical_id\":\"-\",\"root_cause\":\"APP_QUEUE_RC_\"+str(qrc)})\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=121|\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=122|for lid,r in sorted(active.items()):\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=123|kind=r[\"kind\"]; ident=r[\"id\"]; rec={\"logical_id\":lid,\"kind\":kind,\"state\":\"UNVERIFIED\"}\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=124|if kind==\"VM\":\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=125|typ=str(resources.get(ident,{}).get(\"type\",\"qemu\"))\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=126|sid=(\"ct\"+ident) if typ==\"lxc\" else (\"vm\"+ident)\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=127|v2=None\nREC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=128|try: v2=json.loads((STATE/(sid+\".json\")).read_text(encoding=\"utf-8\"))\nAUDIT_SOURCE_GFS_INVALID_HIT_COUNT=1\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=207|if lock==\"PATH_REDACTED and expected_v2_heavy_lock_owner():\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=208|pass\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=209|else:\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=210|issues.append({\"severity\":\"HIGH\",\"code\":\"LOCK_BUSY\",\"logical_id\":lock,\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\"})\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=211|\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=212|env=os.environ.copy(); env[\"HOMELAB_BACKUP_GFS_POLICY\"]=GFSP; env[\"HOMELAB_BACKUP_TARGET_REGISTRY\"]=str(REG)\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=213|grc,gout,_=run([GFSH,\"validate\"],20,env)\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=214|if grc!=0 or \"GFS_POLICY=OK\" not in gout:\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=215|issues.append({\"severity\":\"CRITICAL\",\"code\":\"GFS_INVALID\",\"logical_id\":\"-\",\"root_cause\":\"GFS_VALIDATE_FAILED\"})\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=216|rr,_,_=run([RET,\"contract\"],120)\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=217|if rr!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_CONTRACT_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_CONTRACT_RC_\"+str(rr)})\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=218|rhp=pathlib.Path(\"PATH_REDACTED\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=219|if not rhp.is_file():\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=220|issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_FILE_MISSING\"})\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=221|else:\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=222|try:\nREC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=223|rh=json.loads(rhp.read_text(encoding=\"utf-8\"))\nAUDIT_SOURCE_RETENTION_CONTRACT_BAD_HIT_COUNT=1\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=209|else:\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=210|issues.append({\"severity\":\"HIGH\",\"code\":\"LOCK_BUSY\",\"logical_id\":lock,\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\"})\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=211|\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=212|env=os.environ.copy(); env[\"HOMELAB_BACKUP_GFS_POLICY\"]=GFSP; env[\"HOMELAB_BACKUP_TARGET_REGISTRY\"]=str(REG)\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=213|grc,gout,_=run([GFSH,\"validate\"],20,env)\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=214|if grc!=0 or \"GFS_POLICY=OK\" not in gout:\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=215|issues.append({\"severity\":\"CRITICAL\",\"code\":\"GFS_INVALID\",\"logical_id\":\"-\",\"root_cause\":\"GFS_VALIDATE_FAILED\"})\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=216|rr,_,_=run([RET,\"contract\"],120)\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=217|if rr!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_CONTRACT_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_CONTRACT_RC_\"+str(rr)})\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=218|rhp=pathlib.Path(\"PATH_REDACTED\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=219|if not rhp.is_file():\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=220|issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_FILE_MISSING\"})\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=221|else:\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=222|try:\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=223|rh=json.loads(rhp.read_text(encoding=\"utf-8\"))\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=224|if rh.get(\"status\")!=\"OK\": issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_NOT_OK\"})\nREC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=225|except Exception: issues.append({\"severity\":\"HIGH\",\"code\":\"RETENTION_HEALTH_BAD\",\"logical_id\":\"-\",\"root_cause\":\"RETENTION_HEALTH_UNREADABLE\"})\nREC|AUDIT_PATH_REF|BASENAME=targets.json|PARENT=homelab-backup|TYPE=file|SHA256=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49|PATH_SHA=2f97838a94d9466163297d47d72c9c2e903a12cfb344abc58ff33fcbbae05528\nREC|AUDIT_PATH_REF|BASENAME=state|PARENT=homelab-backup-v2|TYPE=dir|SHA256=NONE|PATH_SHA=6b3144b23b95313197d83e8d55533ff96cc06062af4ae20c3d7a635cdc56145e\nREC|AUDIT_PATH_REF|BASENAME=cloud-quorum|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=8757ada57c0d5e9d3c7d43ff9f4463f9cd391a8f19c6952b0c872e10a305630c\nREC|AUDIT_PATH_REF|BASENAME=vm-restore-validation|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=c56996e4ea53994a88601ceeed1e5291c3e7ea79661386090cf582c3036616f8\nREC|AUDIT_PATH_REF|BASENAME=app-restore-validation|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=c70a6d9e128e0f6fc7e29453ab78510eb2ff57124d19767bd88f87659063751c\nREC|AUDIT_PATH_REF|BASENAME=homelab-backup-target-registry|PARENT=libexec|TYPE=file|SHA256=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a|PATH_SHA=cd6e91b27939aee60ecf8f93a42b3c793926bfd8930593f3ae3bb95f72692162\nREC|AUDIT_PATH_REF|BASENAME=homelab-backup-gfs-policy|PARENT=libexec|TYPE=file|SHA256=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9\nREC|AUDIT_PATH_REF|BASENAME=gfs-retention.json|PARENT=homelab-backup|TYPE=file|SHA256=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d\nREC|AUDIT_PATH_REF|BASENAME=homelab-backup-retention-gc|PARENT=sbin|TYPE=file|SHA256=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a\nREC|AUDIT_PATH_REF|BASENAME=restore-check-health.txt|PARENT=skladchik-reports-monitor-backup|TYPE=file|SHA256=08a290bea388e4ca0e1160aab1c909ff0cef9e98326b6dd32abe9f5541125ce8|PATH_SHA=c7b01e883c50df0f85e999374fa9d383932f47207b7a088028bbbd3fae148509\nREC|AUDIT_PATH_REF|BASENAME=backup-audit-v2.json|PARENT=homelab-health|TYPE=file|SHA256=2ece5343c5d732cbaa65d16feafaade791a2eb2eae465d8a6aba4e644c12e49a|PATH_SHA=e520cdf4c0a133ac68165ccf96f4bec1031f4d7a3e7bddccf5c84aea2a26cad7\nREC|AUDIT_PATH_REF|BASENAME=backup-audit.json|PARENT=homelab-health|TYPE=file|SHA256=2ece5343c5d732cbaa65d16feafaade791a2eb2eae465d8a6aba4e644c12e49a|PATH_SHA=091a828db6ee335d0272a370019f274a88f3a0fa4c9fd92d268358a3674f0b1d\nREC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93\nREC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93\nREC|AUDIT_PATH_REF|BASENAME=homelab-backup-dispatcher.lock|PARENT=lock|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=448d0e8fe73ac64db817c9df04aede92c9cea524652755a9fcdf60fbad00c1aa\nREC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93\nREC|AUDIT_PATH_REF|BASENAME=backup-retention-safe-gc.json|PARENT=homelab-health|TYPE=file|SHA256=4260fcbc5797375b9281e9c64fec35b203fcd35110395e10b94a602a2a67b48c|PATH_SHA=9427dedc6832eded4f955d7a1b6251430298b78c17ed2970664df3370485c719\nREC|AUDIT_PATH_REF|BASENAME=app-skladchik-reports-monitor|PARENT=cloud-quorum|TYPE=dir|SHA256=NONE|PATH_SHA=dc50a937c87b10fe436df9da05826c17b7a3d17d316d87035e5571051c8358a3\nREC|AUDIT_PATH_REF|BASENAME=app-skladchik-reports-monitor|PARENT=app-restore-validation|TYPE=dir|SHA256=NONE|PATH_SHA=c75d5957fa9c11e592017a8399ae56adcdb992ade172a54162a8177c1291f45a\nBACKUP_AUDIT_DIRECT_RC=1\nREC|AUDIT_DIRECT|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65\nREC|AUDIT_DIRECT|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED\nREC|AUDIT_DIRECT|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65\nREC|AUDIT_DIRECT|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nBACKUP_AUDIT_DIRECT_STDERR_LINES=0\nVPN_EXEC_PRESENT=true\nVPN_EXEC_BASENAME=bash\nVPN_EXEC_SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f\nVPN_RESULT_NOW=exit-code\nVPN_EXEC_STATUS_NOW=1\nREC|VPN_JOURNAL|1|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...\nREC|VPN_JOURNAL|2|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE\nREC|VPN_JOURNAL|3|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.\nREC|VPN_JOURNAL|4|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.\nREC|VPN_JOURNAL|5|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...\nREC|VPN_JOURNAL|6|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE\nREC|VPN_JOURNAL|7|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.\nREC|VPN_JOURNAL|8|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.\nREC|VPN_JOURNAL|9|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...\nREC|VPN_JOURNAL|10|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE\nREC|VPN_JOURNAL|11|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.\nREC|VPN_JOURNAL|12|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.\nREC|VPN_JOURNAL|13|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...\nREC|VPN_JOURNAL|14|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE\nREC|VPN_JOURNAL|15|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.\nREC|VPN_JOURNAL|16|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.\nREC|VPN_JOURNAL|17|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...\nREC|VPN_JOURNAL|18|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE\nREC|VPN_JOURNAL|19|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.\nREC|VPN_JOURNAL|20|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.\nREC|VPN_JOURNAL|21|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...\nREC|VPN_JOURNAL|22|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE\nREC|VPN_JOURNAL|23|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.\nREC|VPN_JOURNAL|24|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.\nREC|VPN_JOURNAL|25|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...\nREC|VPN_JOURNAL|26|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE\nREC|VPN_JOURNAL|27|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.\nREC|VPN_JOURNAL|28|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.\nREC|VPN_JOURNAL|29|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...\nREC|VPN_JOURNAL|30|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE\nREC|VPN_JOURNAL|31|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.\nREC|VPN_JOURNAL|32|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.\nREC|VPN_JOURNAL|33|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...\nREC|VPN_JOURNAL|34|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE\nREC|VPN_JOURNAL|35|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.\nREC|VPN_JOURNAL|36|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.\nREC|VPN_JOURNAL|37|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...\nREC|VPN_JOURNAL|38|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE\nREC|VPN_JOURNAL|39|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.\nREC|VPN_JOURNAL|40|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.\nVPN_EXPECTED_NAME_COUNT=0\nVPN_PVE01_RESOLV_CONF_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74\nVPN_PVE01_RESOLV_NAMESERVER_COUNT=1\nVPN_DIRECT_RC=0\nREC|VPN_DIRECT_OUT|1|PVE2_VPN_UNIT_EXISTS=false\nREC|VPN_DIRECT_OUT|2|PVE2_VPN_RESULT=success\nREC|VPN_DIRECT_OUT|3|PVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f\nREC|VPN_DIRECT_OUT|4|PVE2_RESOLV_NAMESERVER_COUNT=1\nREC|VPN_DIRECT_OUT|5|FAILED_UNIT_COUNT_NOW=2\nREC|VPN_DIRECT_OUT|6|REC|FAILED_UNIT_NOW|homelab-backup-audit.service\nREC|VPN_DIRECT_OUT|7|REC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service\nREC|VPN_DIRECT_OUT|8|APT_FULL_UPGRADE_SIM_RC=0\nREC|VPN_DIRECT_OUT|9|APT_FULL_UPGRADE_SIM_UPGRADE_COUNT=184\nREC|VPN_DIRECT_OUT|10|APT_FULL_UPGRADE_SIM_REMOVE_COUNT=0\nREC|VPN_DIRECT_OUT|11|DECISION=PASS_PVE01_REMAINING_FAILED_UNITS_RCA\nREC|VPN_DIRECT_OUT|12|NEXT_GATE=PVE01_REMAINING_FAILED_UNITS_REPAIR\nREC|VPN_DIRECT_OUT|13|TASK_RESULT=PVE01_REMAINING_FAILED_UNITS_RCA\nREC|VPN_DIRECT_OUT|14|CHANGES_MADE=false\nREC|VPN_DIRECT_OUT|15|ROLLBACK_STARTED=false\nREC|VPN_DIRECT_OUT|16|ROLLBACK_RESTORED=null\nREC|VPN_DIRECT_OUT|17|MUTATION_OUTCOME=NO_MUTATION\nREC|VPN_DIRECT_OUT|18|HOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "PVE01_AUDIT_HELPERS_VPN_EXACT_RCA=V1\nMODE=read-only\nNO_REBOOT=true\nNO_APT_UPDATE=true\nNO_PACKAGE_MUTATION=true\nNO_SERVICE_START_STOP_RESTART=true\nNO_RESET_FAILED=true\nNO_CONFIG_WRITE=true\nNO_FILE_DELETE=true\nNO_GUEST_MUTATION=true\nNO_BACKUP_EXECUTION=true\nNO_NETWORK_MUTATION=true\nNODE_HOSTNAME=pve01\nAUDIT_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3\nREGISTRY_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49\nHELPER_SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a\nAUDIT_CONST_APPQ_PRESENT=true\nAUDIT_CONST_APPQ_BASENAME=homelab-app-cloud-quorum-queue\nAUDIT_CONST_APPQ_PARENT=sbin\nAUDIT_CONST_APPQ_EXISTS=true\nAUDIT_CONST_APPQ_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\nAUDIT_CONST_APPQ_PATH_SHA=75315ca46bfa3b06e14300fe6c03852aaf085c4d2843247dd39fd7e8b53f7800\nAUDIT_CONST_GFSH_PRESENT=true\nAUDIT_CONST_GFSH_BASENAME=homelab-backup-gfs-policy\nAUDIT_CONST_GFSH_PARENT=libexec\nAUDIT_CONST_GFSH_EXISTS=true\nAUDIT_CONST_GFSH_SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94\nAUDIT_CONST_GFSH_PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9\nAUDIT_CONST_GFSP_PRESENT=true\nAUDIT_CONST_GFSP_BASENAME=gfs-retention.json\nAUDIT_CONST_GFSP_PARENT=homelab-backup\nAUDIT_CONST_GFSP_EXISTS=true\nAUDIT_CONST_GFSP_SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d\nAUDIT_CONST_GFSP_PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d\nAUDIT_CONST_REG_PRESENT=false\nAUDIT_CONST_RET_PRESENT=true\nAUDIT_CONST_RET_BASENAME=homelab-backup-retention-gc\nAUDIT_CONST_RET_PARENT=sbin\nAUDIT_CONST_RET_EXISTS=true\nAUDIT_CONST_RET_SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909\nAUDIT_CONST_RET_PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a\nREC|AUDIT_HELPER_FILE|LABEL=APPQ|BASENAME=homelab-app-cloud-quorum-queue|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=6|REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=7|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=8|REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=11|[[ -x \"$REGISTRY_HELPER\" && ! -L \"$REGISTRY_HELPER\" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_MISSING\\n' >&2; exit 69; }\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=12|[[ \"$(sha256sum \"$REGISTRY_HELPER\"|awk '{print $1}')\" == \"$REGISTRY_HELPER_SHA_EXPECTED\" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=19|[[ \"${#ALL_APPS[@]}\" == 4 && \"${#MIGRATE_APPS[@]}\" == 4 ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_COUNT_INVALID\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_FILE|LABEL=GFSH|BASENAME=homelab-backup-gfs-policy|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=3|CFG=pathlib.Path(os.environ.get(\"HOMELAB_BACKUP_GFS_POLICY\",\"PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=4|REG=pathlib.Path(os.environ.get(\"HOMELAB_BACKUP_TARGET_REGISTRY\",\"PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=6|print(f\"GFS_POLICY_ERROR={msg}\",file=sys.stderr); raise SystemExit(rc)\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=10|if c.get(\"schema\")!=\"homelab-backup-gfs-policy-v1\" or c.get(\"policy_version\")!=1: die(\"SCHEMA_INVALID\",66)\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=14|rows={x.get(\"logical_id\"):x for x in r.get(\"targets\",[])}\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=48|if args==[\"validate\"]: print(\"GFS_POLICY=OK\"); return\nREC|AUDIT_HELPER_FILE|LABEL=GFSP|BASENAME=gfs-retention.json|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=GFSP|LINE=18|\"schema\": \"homelab-backup-gfs-policy-v1\",\nREC|AUDIT_HELPER_FILE|LABEL=RET|BASENAME=homelab-backup-retention-gc|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=8|RETENTION_ROOT=PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=21|REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=22|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=23|GFS_HELPER=\"${HOMELAB_BACKUP_GFS_POLICY_HELPER:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=25|GFS_POLICY=\"${HOMELAB_BACKUP_GFS_POLICY:-PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=27|[[ -x \"$REGISTRY_HELPER\" && ! -L \"$REGISTRY_HELPER\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_MISSING\\n' >&2; exit 69; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=28|[[ \"$(sha256sum \"$REGISTRY_HELPER\"|awk '{print $1}')\" == \"$REGISTRY_HELPER_SHA_EXPECTED\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=29|\"$REGISTRY_HELPER\" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_INVALID\\n' >&2; exit 66; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=30|[[ -x \"$GFS_HELPER\" && ! -L \"$GFS_HELPER\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_MISSING\\n' >&2; exit 69; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=31|[[ \"$(sha256sum \"$GFS_HELPER\"|awk '{print $1}')\" == \"$GFS_HELPER_SHA_EXPECTED\" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\\n' >&2; exit 65; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=32|HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID\\n' >&2; exit 66; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=33|GFS_WEEKLY=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" weekly-buckets)\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=34|GFS_MONTHLY=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" monthly-buckets)\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=35|GFS_CRITICAL_COUNT=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" critical-logicals | wc -l)\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=74|jq -e --arg l \"$logical\" --arg g \"$generation\" '.schema==\"homelab-cloud-quorum-payload-v1\" and .logical_id==$l and .generation==$g and .status==\"SEALED\" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_delete_allowed==true and .local_payload_present==false and .local_cleanup==\"OK\"' <<<\"$json\" >PATH_REDACTED\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=106|f=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=111|f=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=117|[[ -d \"$RETENTION_ROOT/transactions\" ]] || { printf '0\\n'; return 0; }\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=126|done < <(find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -print | LC_ALL=C sort)\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=131|[[ -n \"${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}\" || -n \"${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}\" ]]\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=134|local f=\"${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}\" expected=\"${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}\" actual line logical generation owner state_file state_sha count=0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=160|jq -e --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" '.schema==\"homelab-backup-retention-transaction-v1\" and .task_id==$task and .change_id==$change and (.logical_id|type)==\"string\" and (.generation|type)==\"string\" and (.state_sha256|type)==\"string\" and (.status==\"PLANNED\" or .status==\"B_DELETED\" or .status==\"DONE\")' \"$f\" >PATH_REDACTED || die \"PINNED_TX_CONTENT_INVALID_$f\" 66\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=165|expected_path=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=170|local root=\"$RETENTION_ROOT/transactions\" f\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=184|txf=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=188|grep -Fqx \"$(pinned_tx_identity \"$txf\")\" \"$HOMELAB_RETENTION_PINNED_DELETE_FILE\" || die \"PINNED_PLAN_PROTECT_NOT_PINNED_${logical}_${generation}\" 66\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=199|if [[ -d \"$RETENTION_ROOT/transactions\" ]]; then\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=200|while IFS= read -r f; do pinned_tx_identity \"$f\" >>\"$actual\"; done < <(find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -print | LC_ALL=C sort)\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=203|LC_ALL=C sort \"$HOMELAB_RETENTION_PINNED_DELETE_FILE\" >\"$expected\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=208|printf 'PINNED_DELETE_SCOPE=OK|stage=%s|count=%s|sha256=%s\\n' \"$stage\" \"$count_expected\" \"$HOMELAB_RETENTION_PINNED_DELETE_SHA256\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=218|if ! HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" is-critical \"$logical\"; then return 0; fi\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=226|out=\"$(HOMELAB_BACKUP_GFS_POLICY=\"$GFS_POLICY\" \"$GFS_HELPER\" select \"$logical\" <\"$TMPROOT/$logical.gfs-eligible\")\" || die \"GFS_SELECT_FAILED_$logical\" 66\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=240|local now logical node keep lifecycle open_count file json schema lid gen status sha sealed_at epoch age idx tx gfs_bucket candidates=0 protected=0 kept=0 gfs_kept=0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=252|schema=\"$(jq -r '.schema/PATH_REDACTED <<<\"$json\")\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=256|if [[ \"$schema\" != homelab-cloud-quorum-payload-v1 || \"$lid\" != \"$logical\" ]] || ! safe_generation \"$logical\" \"$gen\"; then\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=321|path=\"$RETENTION_ROOT/transactions/$logical/$generation.json\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=324|jq -n --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" --arg l \"$logical\" --arg g \"$generation\" --arg s \"$state_sha\" --arg status \"$status\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --argjson aa \"$a_absent\" --argjson ba \"$b_absent\" '{schema:\"homelab-backup-retention-transaction-v1\",task_id:$task,change_id:$change,logical_id:$l,generation:$g,state_sha256:$s,status:$status,updated_at_utc:$at,remote_a_absent:$aa,remote_b_absent:$ba}' >\"$tmp\"\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=357|[[ -d \"$RETENTION_ROOT/transactions\" ]] || return 0\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=381|done < <(find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -print | LC_ALL=C sort)\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=415|if find \"$RETENTION_ROOT/transactions\" -type f -name '*.json' -exec jq -e '.status==\"DONE\" and .remote_a_absent==true and .remote_b_absent==true' {} \\; -print 2>\"$TMPROOT/tx-find.err\" | grep -q .; then hist=true; fi\nREC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=418|jq -n --arg task \"$TASK_ID\" --arg change \"$CHANGE_ID\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --argjson canary \"$canary_proven\" --argjson hist \"$hist\" --argjson candidates \"${candidates:-0}\" --argjson protected \"${protected:-0}\" --argjson kept \"${kept:-0}\" --argjson gfs_kept \"${gfs_kept:-0}\" --argjson deleted \"$deleted\" --argjson gfs_critical \"$GFS_CRITICAL_COUNT\" --argjson gfs_weekly \"$GFS_WEEKLY\" --argjson gfs_monthly \"$GFS_MONTHLY\" '{schema:\"homelab-backup-retention-v1\",status:\"OK\",task_id:\nAPPQ_FRESHNESS_PLAN_RC=65\nREC|APPQ_ERR|APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\nGFS_VALIDATE_RC=66\nREC|GFS_ERR|GFS_POLICY_ERROR=READ_OR_JSON_INVALID\nRETENTION_CONTRACT_RC=65\nREC|RET_ERR|GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\nVPN_EXECSTART_COUNT=1\nREC|VPN_EXEC_PATH|IDX=1|BASENAME=bash|PARENT=bin|SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f|EXEC=true\nREC|VPN_EXEC_PATH|IDX=2|BASENAME=homelab-check-private-vpn-hosts|PARENT=sbin|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82|EXEC=true\nREC|VPN_EXEC_PATH|IDX=3|BASENAME=private-vpn-hosts.txt|PARENT=homelab-health|SHA=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d|EXEC=false\nVPN_SCRIPT_FOUND=true\nVPN_SCRIPT_BASENAME=homelab-check-private-vpn-hosts\nVPN_SCRIPT_SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82\nVPN_SCRIPT_MUTATION_PATTERN_COUNT=1\nVPN_LITERAL_NAME_COUNT=5\nREC|VPN_SOURCE|LINE=6|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=7|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=8|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=9|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=10|VPN_HOST_REDACTED\nREC|VPN_SOURCE|LINE=19|ip=\"$(getent hosts \"$h\" 2>/dev/null | awk '{print $1}' | head -n1 || true)\"\nVPN_SCRIPT_DIRECT_SKIPPED_MUTATION_SCAN=true\nPVE01_RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74\nPVE01_NAMESERVER_COUNT=1\nPVE01_VPN_HOSTS_LINE_COUNT=0\nPVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f\nPVE2_NAMESERVER_COUNT=1\nPVE2_VPN_HOSTS_LINE_COUNT=0\nFAILED_UNIT_COUNT_NOW=2\nREC|FAILED_UNIT_NOW|homelab-backup-audit.service\nREC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service\nDECISION=PASS_PVE01_AUDIT_HELPERS_VPN_EXACT_RCA\nNEXT_GATE=PVE01_AUDIT_HELPERS_VPN_REPAIR\nTASK_RESULT=PVE01_AUDIT_HELPERS_VPN_EXACT_RCA\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index d42227d3..7585e107 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1 +COMMAND_ID=SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=pve01-remaining-failed-units-rca +COMPONENT=pve01-audit-helpers-vpn-exact-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=18e8f9dc2347e65c7d6310e27f48bdd48e3a1978f434af5fee6ab8ab48185dac +COMMAND_SHA256=5cf24ab1bb98675b9bfe49e8bdc859479f976f0b83c5d2d1a01f49508b38be64 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,10 +24,10 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=4228b00b8b8cab02f55eb05e3b5c238c8acdf45fcc4c59e66c5978d7c6ee6fd1 -SANITIZED_OUTPUT_SHA256=4228b00b8b8cab02f55eb05e3b5c238c8acdf45fcc4c59e66c5978d7c6ee6fd1 +RAW_EVIDENCE_SHA256=abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd +SANITIZED_OUTPUT_SHA256=abdcbd61eee87c42f8ece8c9fbaa28bec75cf09c6122e8e11ea16f1e7f5b6dbd OUTPUT_BEGIN -PVE01_REMAINING_FAILED_UNITS_RCA=V1 +PVE01_AUDIT_HELPERS_VPN_EXACT_RCA=V1 MODE=read-only NO_REBOOT=true NO_APT_UPDATE=true @@ -40,204 +40,131 @@ NO_GUEST_MUTATION=true NO_BACKUP_EXECUTION=true NO_NETWORK_MUTATION=true NODE_HOSTNAME=pve01 -UNIFIED_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49 -V2_SHA=7f413e4a3bc276be668ad0d7a877a0f2b2bd64bf1dead9cb929d966b25a44e25 +AUDIT_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3 +REGISTRY_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49 HELPER_SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a -PROBE_SHA=44ea82521afb37e862ec7fb7f0e3eeb2f0f582b784506d8dec23f73ccea86050 -WORKER_SHA=a86bc027eefc37bbbcb9ad9b18c1c2c2a4f4ef9172a9cd9918a00f1d94c9e19a -HELPER_VALIDATE_RC=0 -CANONICAL_PROBE_RC=0 -CANONICAL_PROBE_SEALED_PROVEN=true -BACKUP_AUDIT_EXEC_PRESENT=true -BACKUP_AUDIT_EXEC_BASENAME=homelab-backup-audit -BACKUP_AUDIT_EXEC_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3 -BACKUP_AUDIT_RESULT_NOW=exit-code -BACKUP_AUDIT_EXEC_STATUS_NOW=1 -BACKUP_AUDIT_FRESH_LINES=48 -REC|AUDIT_FRESH|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|legacy_age_seconds=14441|generation=app-filebrowser-pve01-20261006T043010Z -REC|AUDIT_FRESH|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|legacy_age_seconds=14279|generation=app-hbarhub-pve01-20261006T043148Z -REC|AUDIT_FRESH|TARGET|logical_id=app-immich|kind=APP|state=FRESH|legacy_age_seconds=14385|generation=app-immich-pve01-20261006T043031Z -REC|AUDIT_FRESH|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|legacy_age_seconds=43247|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z -REC|AUDIT_FRESH|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=370|generation=control-bootstrap-v2-pve01-20260916T120057Z -REC|AUDIT_FRESH|TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=3875|legacy_age_seconds=2775090|replica_count=2|state_id=ct110|generation=v2-ct110-20261006T072242Z-655b3dc1c042 -REC|AUDIT_FRESH|TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=1522|replica_count=2|state_id=ct111|generation=v2-ct111-20261006T080259Z-417f9545ef1b -REC|AUDIT_FRESH|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=1|legacy_age_seconds=2813585|replica_count=2|state_id=ct112|generation=v2-ct112-20261006T082950Z-2538063ab2f8 -REC|AUDIT_FRESH|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=67118|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7 -REC|AUDIT_FRESH|TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=5102|replica_count=2|state_id=vm130|generation=v2-vm130-20261006T064416Z-6541a84254d2 -REC|AUDIT_FRESH|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=65014|legacy_age_seconds=83432|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0 -REC|AUDIT_FRESH|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51585|legacy_age_seconds=1719827|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94 -REC|AUDIT_FRESH|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=47820|legacy_age_seconds=1749359|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d -REC|AUDIT_FRESH|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=62823|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9 -REC|AUDIT_FRESH|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=30496|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9 -REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65 -REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED -REC|AUDIT_FRESH|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65 -REC|AUDIT_FRESH|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -REC|AUDIT_FRESH|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|legacy_age_seconds=14739|generation=app-filebrowser-pve01-20261006T043010Z -REC|AUDIT_FRESH|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|legacy_age_seconds=14577|generation=app-hbarhub-pve01-20261006T043148Z -REC|AUDIT_FRESH|TARGET|logical_id=app-immich|kind=APP|state=FRESH|legacy_age_seconds=14683|generation=app-immich-pve01-20261006T043031Z -REC|AUDIT_FRESH|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|legacy_age_seconds=43545|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z -REC|AUDIT_FRESH|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=668|generation=control-bootstrap-v2-pve01-20260916T120057Z -REC|AUDIT_FRESH|TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=4173|legacy_age_seconds=2775388|replica_count=2|state_id=ct110|generation=v2-ct110-20261006T072242Z-655b3dc1c042 -REC|AUDIT_FRESH|TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=1820|replica_count=2|state_id=ct111|generation=v2-ct111-20261006T080259Z-417f9545ef1b -REC|AUDIT_FRESH|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=299|legacy_age_seconds=2813883|replica_count=2|state_id=ct112|generation=v2-ct112-20261006T082950Z-2538063ab2f8 -REC|AUDIT_FRESH|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=67416|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7 -REC|AUDIT_FRESH|TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=5400|replica_count=2|state_id=vm130|generation=v2-vm130-20261006T064416Z-6541a84254d2 -REC|AUDIT_FRESH|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=65312|legacy_age_seconds=83730|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0 -REC|AUDIT_FRESH|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51883|legacy_age_seconds=1720125|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94 -REC|AUDIT_FRESH|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=48118|legacy_age_seconds=1749657|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d -REC|AUDIT_FRESH|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=63121|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9 -REC|AUDIT_FRESH|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=30794|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9 -REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65 -REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED -REC|AUDIT_FRESH|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65 -REC|AUDIT_FRESH|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -AUDIT_SOURCE_APP_FRESHNESS_PLAN_FAILED_HIT_COUNT=1 -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=112|resources={} -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=113|if rrc==0: -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=114|try: -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=115|for x in json.loads(rout): resources[str(x.get("vmid"))]=x -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=116|except Exception: pass -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=117| -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=118|qrc,qout,qerr=run([APPQ,"freshness-plan"],180) -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=119|app_dec=parse_app_plan(qout if qrc==0 else "") -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=120|if qrc!=0: issues.append({"severity":"CRITICAL","code":"APP_FRESHNESS_PLAN_FAILED","logical_id":"-","root_cause":"APP_QUEUE_RC_"+str(qrc)}) -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=121| -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=122|for lid,r in sorted(active.items()): -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=123|kind=r["kind"]; ident=r["id"]; rec={"logical_id":lid,"kind":kind,"state":"UNVERIFIED"} -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=124|if kind=="VM": -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=125|typ=str(resources.get(ident,{}).get("type","qemu")) -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=126|sid=("ct"+ident) if typ=="lxc" else ("vm"+ident) -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=127|v2=None -REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=128|try: v2=json.loads((STATE/(sid+".json")).read_text(encoding="utf-8")) -AUDIT_SOURCE_GFS_INVALID_HIT_COUNT=1 -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=207|if lock=="PATH_REDACTED and expected_v2_heavy_lock_owner(): -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=208|pass -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=209|else: -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=210|issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"}) -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=211| -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=212|env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG) -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=213|grc,gout,_=run([GFSH,"validate"],20,env) -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=214|if grc!=0 or "GFS_POLICY=OK" not in gout: -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=215|issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"}) -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=216|rr,_,_=run([RET,"contract"],120) -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=217|if rr!=0: issues.append({"severity":"HIGH","code":"RETENTION_CONTRACT_BAD","logical_id":"-","root_cause":"RETENTION_CONTRACT_RC_"+str(rr)}) -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=218|rhp=pathlib.Path("PATH_REDACTED -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=219|if not rhp.is_file(): -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=220|issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_MISSING","logical_id":"-","root_cause":"RETENTION_HEALTH_FILE_MISSING"}) -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=221|else: -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=222|try: -REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=223|rh=json.loads(rhp.read_text(encoding="utf-8")) -AUDIT_SOURCE_RETENTION_CONTRACT_BAD_HIT_COUNT=1 -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=209|else: -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=210|issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"}) -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=211| -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=212|env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG) -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=213|grc,gout,_=run([GFSH,"validate"],20,env) -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=214|if grc!=0 or "GFS_POLICY=OK" not in gout: -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=215|issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"}) -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=216|rr,_,_=run([RET,"contract"],120) -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=217|if rr!=0: issues.append({"severity":"HIGH","code":"RETENTION_CONTRACT_BAD","logical_id":"-","root_cause":"RETENTION_CONTRACT_RC_"+str(rr)}) -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=218|rhp=pathlib.Path("PATH_REDACTED -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=219|if not rhp.is_file(): -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=220|issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_MISSING","logical_id":"-","root_cause":"RETENTION_HEALTH_FILE_MISSING"}) -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=221|else: -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=222|try: -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=223|rh=json.loads(rhp.read_text(encoding="utf-8")) -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=224|if rh.get("status")!="OK": issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_NOT_OK"}) -REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=225|except Exception: issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_UNREADABLE"}) -REC|AUDIT_PATH_REF|BASENAME=targets.json|PARENT=homelab-backup|TYPE=file|SHA256=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49|PATH_SHA=2f97838a94d9466163297d47d72c9c2e903a12cfb344abc58ff33fcbbae05528 -REC|AUDIT_PATH_REF|BASENAME=state|PARENT=homelab-backup-v2|TYPE=dir|SHA256=NONE|PATH_SHA=6b3144b23b95313197d83e8d55533ff96cc06062af4ae20c3d7a635cdc56145e -REC|AUDIT_PATH_REF|BASENAME=cloud-quorum|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=8757ada57c0d5e9d3c7d43ff9f4463f9cd391a8f19c6952b0c872e10a305630c -REC|AUDIT_PATH_REF|BASENAME=vm-restore-validation|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=c56996e4ea53994a88601ceeed1e5291c3e7ea79661386090cf582c3036616f8 -REC|AUDIT_PATH_REF|BASENAME=app-restore-validation|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=c70a6d9e128e0f6fc7e29453ab78510eb2ff57124d19767bd88f87659063751c -REC|AUDIT_PATH_REF|BASENAME=homelab-backup-target-registry|PARENT=libexec|TYPE=file|SHA256=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a|PATH_SHA=cd6e91b27939aee60ecf8f93a42b3c793926bfd8930593f3ae3bb95f72692162 -REC|AUDIT_PATH_REF|BASENAME=homelab-backup-gfs-policy|PARENT=libexec|TYPE=file|SHA256=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9 -REC|AUDIT_PATH_REF|BASENAME=gfs-retention.json|PARENT=homelab-backup|TYPE=file|SHA256=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d -REC|AUDIT_PATH_REF|BASENAME=homelab-backup-retention-gc|PARENT=sbin|TYPE=file|SHA256=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a -REC|AUDIT_PATH_REF|BASENAME=restore-check-health.txt|PARENT=skladchik-reports-monitor-backup|TYPE=file|SHA256=08a290bea388e4ca0e1160aab1c909ff0cef9e98326b6dd32abe9f5541125ce8|PATH_SHA=c7b01e883c50df0f85e999374fa9d383932f47207b7a088028bbbd3fae148509 -REC|AUDIT_PATH_REF|BASENAME=backup-audit-v2.json|PARENT=homelab-health|TYPE=file|SHA256=2ece5343c5d732cbaa65d16feafaade791a2eb2eae465d8a6aba4e644c12e49a|PATH_SHA=e520cdf4c0a133ac68165ccf96f4bec1031f4d7a3e7bddccf5c84aea2a26cad7 -REC|AUDIT_PATH_REF|BASENAME=backup-audit.json|PARENT=homelab-health|TYPE=file|SHA256=2ece5343c5d732cbaa65d16feafaade791a2eb2eae465d8a6aba4e644c12e49a|PATH_SHA=091a828db6ee335d0272a370019f274a88f3a0fa4c9fd92d268358a3674f0b1d -REC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93 -REC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93 -REC|AUDIT_PATH_REF|BASENAME=homelab-backup-dispatcher.lock|PARENT=lock|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=448d0e8fe73ac64db817c9df04aede92c9cea524652755a9fcdf60fbad00c1aa -REC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93 -REC|AUDIT_PATH_REF|BASENAME=backup-retention-safe-gc.json|PARENT=homelab-health|TYPE=file|SHA256=4260fcbc5797375b9281e9c64fec35b203fcd35110395e10b94a602a2a67b48c|PATH_SHA=9427dedc6832eded4f955d7a1b6251430298b78c17ed2970664df3370485c719 -REC|AUDIT_PATH_REF|BASENAME=app-skladchik-reports-monitor|PARENT=cloud-quorum|TYPE=dir|SHA256=NONE|PATH_SHA=dc50a937c87b10fe436df9da05826c17b7a3d17d316d87035e5571051c8358a3 -REC|AUDIT_PATH_REF|BASENAME=app-skladchik-reports-monitor|PARENT=app-restore-validation|TYPE=dir|SHA256=NONE|PATH_SHA=c75d5957fa9c11e592017a8399ae56adcdb992ade172a54162a8177c1291f45a -BACKUP_AUDIT_DIRECT_RC=1 -REC|AUDIT_DIRECT|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65 -REC|AUDIT_DIRECT|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED -REC|AUDIT_DIRECT|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65 -REC|AUDIT_DIRECT|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -BACKUP_AUDIT_DIRECT_STDERR_LINES=0 -VPN_EXEC_PRESENT=true -VPN_EXEC_BASENAME=bash -VPN_EXEC_SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f -VPN_RESULT_NOW=exit-code -VPN_EXEC_STATUS_NOW=1 -REC|VPN_JOURNAL|1|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check... -REC|VPN_JOURNAL|2|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE -REC|VPN_JOURNAL|3|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'. -REC|VPN_JOURNAL|4|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check. -REC|VPN_JOURNAL|5|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check... -REC|VPN_JOURNAL|6|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE -REC|VPN_JOURNAL|7|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'. -REC|VPN_JOURNAL|8|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check. -REC|VPN_JOURNAL|9|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check... -REC|VPN_JOURNAL|10|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE -REC|VPN_JOURNAL|11|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'. -REC|VPN_JOURNAL|12|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check. -REC|VPN_JOURNAL|13|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check... -REC|VPN_JOURNAL|14|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE -REC|VPN_JOURNAL|15|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'. -REC|VPN_JOURNAL|16|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check. -REC|VPN_JOURNAL|17|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check... -REC|VPN_JOURNAL|18|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE -REC|VPN_JOURNAL|19|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'. -REC|VPN_JOURNAL|20|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check. -REC|VPN_JOURNAL|21|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check... -REC|VPN_JOURNAL|22|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE -REC|VPN_JOURNAL|23|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'. -REC|VPN_JOURNAL|24|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check. -REC|VPN_JOURNAL|25|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check... -REC|VPN_JOURNAL|26|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE -REC|VPN_JOURNAL|27|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'. -REC|VPN_JOURNAL|28|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check. -REC|VPN_JOURNAL|29|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check... -REC|VPN_JOURNAL|30|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE -REC|VPN_JOURNAL|31|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'. -REC|VPN_JOURNAL|32|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check. -REC|VPN_JOURNAL|33|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check... -REC|VPN_JOURNAL|34|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE -REC|VPN_JOURNAL|35|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'. -REC|VPN_JOURNAL|36|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check. -REC|VPN_JOURNAL|37|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check... -REC|VPN_JOURNAL|38|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE -REC|VPN_JOURNAL|39|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'. -REC|VPN_JOURNAL|40|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check. -VPN_EXPECTED_NAME_COUNT=0 -VPN_PVE01_RESOLV_CONF_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74 -VPN_PVE01_RESOLV_NAMESERVER_COUNT=1 -VPN_DIRECT_RC=0 -REC|VPN_DIRECT_OUT|1|PVE2_VPN_UNIT_EXISTS=false -REC|VPN_DIRECT_OUT|2|PVE2_VPN_RESULT=success -REC|VPN_DIRECT_OUT|3|PVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f -REC|VPN_DIRECT_OUT|4|PVE2_RESOLV_NAMESERVER_COUNT=1 -REC|VPN_DIRECT_OUT|5|FAILED_UNIT_COUNT_NOW=2 -REC|VPN_DIRECT_OUT|6|REC|FAILED_UNIT_NOW|homelab-backup-audit.service -REC|VPN_DIRECT_OUT|7|REC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service -REC|VPN_DIRECT_OUT|8|APT_FULL_UPGRADE_SIM_RC=0 -REC|VPN_DIRECT_OUT|9|APT_FULL_UPGRADE_SIM_UPGRADE_COUNT=184 -REC|VPN_DIRECT_OUT|10|APT_FULL_UPGRADE_SIM_REMOVE_COUNT=0 -REC|VPN_DIRECT_OUT|11|DECISION=PASS_PVE01_REMAINING_FAILED_UNITS_RCA -REC|VPN_DIRECT_OUT|12|NEXT_GATE=PVE01_REMAINING_FAILED_UNITS_REPAIR -REC|VPN_DIRECT_OUT|13|TASK_RESULT=PVE01_REMAINING_FAILED_UNITS_RCA -REC|VPN_DIRECT_OUT|14|CHANGES_MADE=false -REC|VPN_DIRECT_OUT|15|ROLLBACK_STARTED=false -REC|VPN_DIRECT_OUT|16|ROLLBACK_RESTORED=null -REC|VPN_DIRECT_OUT|17|MUTATION_OUTCOME=NO_MUTATION -REC|VPN_DIRECT_OUT|18|HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +AUDIT_CONST_APPQ_PRESENT=true +AUDIT_CONST_APPQ_BASENAME=homelab-app-cloud-quorum-queue +AUDIT_CONST_APPQ_PARENT=sbin +AUDIT_CONST_APPQ_EXISTS=true +AUDIT_CONST_APPQ_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650 +AUDIT_CONST_APPQ_PATH_SHA=75315ca46bfa3b06e14300fe6c03852aaf085c4d2843247dd39fd7e8b53f7800 +AUDIT_CONST_GFSH_PRESENT=true +AUDIT_CONST_GFSH_BASENAME=homelab-backup-gfs-policy +AUDIT_CONST_GFSH_PARENT=libexec +AUDIT_CONST_GFSH_EXISTS=true +AUDIT_CONST_GFSH_SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94 +AUDIT_CONST_GFSH_PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9 +AUDIT_CONST_GFSP_PRESENT=true +AUDIT_CONST_GFSP_BASENAME=gfs-retention.json +AUDIT_CONST_GFSP_PARENT=homelab-backup +AUDIT_CONST_GFSP_EXISTS=true +AUDIT_CONST_GFSP_SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d +AUDIT_CONST_GFSP_PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d +AUDIT_CONST_REG_PRESENT=false +AUDIT_CONST_RET_PRESENT=true +AUDIT_CONST_RET_BASENAME=homelab-backup-retention-gc +AUDIT_CONST_RET_PARENT=sbin +AUDIT_CONST_RET_EXISTS=true +AUDIT_CONST_RET_SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909 +AUDIT_CONST_RET_PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a +REC|AUDIT_HELPER_FILE|LABEL=APPQ|BASENAME=homelab-app-cloud-quorum-queue|SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0 +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=6|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=7|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4 +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=8|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=11|[[ -x "$REGISTRY_HELPER" && ! -L "$REGISTRY_HELPER" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_MISSING\n' >&2; exit 69; } +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=12|[[ "$(sha256sum "$REGISTRY_HELPER"|awk '{print $1}')" == "$REGISTRY_HELPER_SHA_EXPECTED" ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT\n' >&2; exit 65; } +REC|AUDIT_HELPER_SOURCE|LABEL=APPQ|LINE=19|[[ "${#ALL_APPS[@]}" == 4 && "${#MIGRATE_APPS[@]}" == 4 ]] || { printf 'APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_COUNT_INVALID\n' >&2; exit 65; } +REC|AUDIT_HELPER_FILE|LABEL=GFSH|BASENAME=homelab-backup-gfs-policy|SHA=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0 +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=3|CFG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_GFS_POLICY","PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=4|REG=pathlib.Path(os.environ.get("HOMELAB_BACKUP_TARGET_REGISTRY","PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=6|print(f"GFS_POLICY_ERROR={msg}",file=sys.stderr); raise SystemExit(rc) +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=10|if c.get("schema")!="homelab-backup-gfs-policy-v1" or c.get("policy_version")!=1: die("SCHEMA_INVALID",66) +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=14|rows={x.get("logical_id"):x for x in r.get("targets",[])} +REC|AUDIT_HELPER_SOURCE|LABEL=GFSH|LINE=48|if args==["validate"]: print("GFS_POLICY=OK"); return +REC|AUDIT_HELPER_FILE|LABEL=GFSP|BASENAME=gfs-retention.json|SHA=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|OLD_HELPER_SHA_COUNT=0|NEW_HELPER_SHA_COUNT=0 +REC|AUDIT_HELPER_SOURCE|LABEL=GFSP|LINE=18|"schema": "homelab-backup-gfs-policy-v1", +REC|AUDIT_HELPER_FILE|LABEL=RET|BASENAME=homelab-backup-retention-gc|SHA=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|OLD_HELPER_SHA_COUNT=1|NEW_HELPER_SHA_COUNT=0 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=8|RETENTION_ROOT=PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=21|REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=22|REGISTRY_HELPER_SHA_EXPECTED=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=23|GFS_HELPER="${HOMELAB_BACKUP_GFS_POLICY_HELPER:-PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=25|GFS_POLICY="${HOMELAB_BACKUP_GFS_POLICY:-PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=27|[[ -x "$REGISTRY_HELPER" && ! -L "$REGISTRY_HELPER" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_MISSING\n' >&2; exit 69; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=28|[[ "$(sha256sum "$REGISTRY_HELPER"|awk '{print $1}')" == "$REGISTRY_HELPER_SHA_EXPECTED" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT\n' >&2; exit 65; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=29|"$REGISTRY_HELPER" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_INVALID\n' >&2; exit 66; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=30|[[ -x "$GFS_HELPER" && ! -L "$GFS_HELPER" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_MISSING\n' >&2; exit 69; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=31|[[ "$(sha256sum "$GFS_HELPER"|awk '{print $1}')" == "$GFS_HELPER_SHA_EXPECTED" ]] || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\n' >&2; exit 65; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=32|HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" validate >PATH_REDACTED || { printf 'GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_INVALID\n' >&2; exit 66; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=33|GFS_WEEKLY="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" weekly-buckets)" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=34|GFS_MONTHLY="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" monthly-buckets)" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=35|GFS_CRITICAL_COUNT="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" critical-logicals | wc -l)" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=74|jq -e --arg l "$logical" --arg g "$generation" '.schema=="homelab-cloud-quorum-payload-v1" and .logical_id==$l and .generation==$g and .status=="SEALED" and .cloud_quorum==2 and .cloud_a_verified==true and .cloud_b_verified==true and .part_hashes_verified==true and .seal_written==true and .cloud_a_seal_verified==true and .cloud_b_seal_verified==true and .local_delete_allowed==true and .local_payload_present==false and .local_cleanup=="OK"' <<<"$json" >PATH_REDACTED +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=106|f="$RETENTION_ROOT/transactions/$logical/$generation.json" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=111|f="$RETENTION_ROOT/transactions/$logical/$generation.json" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=117|[[ -d "$RETENTION_ROOT/transactions" ]] || { printf '0\n'; return 0; } +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=126|done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort) +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=131|[[ -n "${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}" || -n "${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}" ]] +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=134|local f="${HOMELAB_RETENTION_PINNED_DELETE_FILE:-}" expected="${HOMELAB_RETENTION_PINNED_DELETE_SHA256:-}" actual line logical generation owner state_file state_sha count=0 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=160|jq -e --arg task "$TASK_ID" --arg change "$CHANGE_ID" '.schema=="homelab-backup-retention-transaction-v1" and .task_id==$task and .change_id==$change and (.logical_id|type)=="string" and (.generation|type)=="string" and (.state_sha256|type)=="string" and (.status=="PLANNED" or .status=="B_DELETED" or .status=="DONE")' "$f" >PATH_REDACTED || die "PINNED_TX_CONTENT_INVALID_$f" 66 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=165|expected_path="$RETENTION_ROOT/transactions/$logical/$generation.json" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=170|local root="$RETENTION_ROOT/transactions" f +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=184|txf="$RETENTION_ROOT/transactions/$logical/$generation.json" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=188|grep -Fqx "$(pinned_tx_identity "$txf")" "$HOMELAB_RETENTION_PINNED_DELETE_FILE" || die "PINNED_PLAN_PROTECT_NOT_PINNED_${logical}_${generation}" 66 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=199|if [[ -d "$RETENTION_ROOT/transactions" ]]; then +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=200|while IFS= read -r f; do pinned_tx_identity "$f" >>"$actual"; done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort) +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=203|LC_ALL=C sort "$HOMELAB_RETENTION_PINNED_DELETE_FILE" >"$expected" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=208|printf 'PINNED_DELETE_SCOPE=OK|stage=%s|count=%s|sha256=%s\n' "$stage" "$count_expected" "$HOMELAB_RETENTION_PINNED_DELETE_SHA256" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=218|if ! HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" is-critical "$logical"; then return 0; fi +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=226|out="$(HOMELAB_BACKUP_GFS_POLICY="$GFS_POLICY" "$GFS_HELPER" select "$logical" <"$TMPROOT/$logical.gfs-eligible")" || die "GFS_SELECT_FAILED_$logical" 66 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=240|local now logical node keep lifecycle open_count file json schema lid gen status sha sealed_at epoch age idx tx gfs_bucket candidates=0 protected=0 kept=0 gfs_kept=0 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=252|schema="$(jq -r '.schema/PATH_REDACTED <<<"$json")" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=256|if [[ "$schema" != homelab-cloud-quorum-payload-v1 || "$lid" != "$logical" ]] || ! safe_generation "$logical" "$gen"; then +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=321|path="$RETENTION_ROOT/transactions/$logical/$generation.json" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=324|jq -n --arg task "$TASK_ID" --arg change "$CHANGE_ID" --arg l "$logical" --arg g "$generation" --arg s "$state_sha" --arg status "$status" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson aa "$a_absent" --argjson ba "$b_absent" '{schema:"homelab-backup-retention-transaction-v1",task_id:$task,change_id:$change,logical_id:$l,generation:$g,state_sha256:$s,status:$status,updated_at_utc:$at,remote_a_absent:$aa,remote_b_absent:$ba}' >"$tmp" +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=357|[[ -d "$RETENTION_ROOT/transactions" ]] || return 0 +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=381|done < <(find "$RETENTION_ROOT/transactions" -type f -name '*.json' -print | LC_ALL=C sort) +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=415|if find "$RETENTION_ROOT/transactions" -type f -name '*.json' -exec jq -e '.status=="DONE" and .remote_a_absent==true and .remote_b_absent==true' {} \; -print 2>"$TMPROOT/tx-find.err" | grep -q .; then hist=true; fi +REC|AUDIT_HELPER_SOURCE|LABEL=RET|LINE=418|jq -n --arg task "$TASK_ID" --arg change "$CHANGE_ID" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson canary "$canary_proven" --argjson hist "$hist" --argjson candidates "${candidates:-0}" --argjson protected "${protected:-0}" --argjson kept "${kept:-0}" --argjson gfs_kept "${gfs_kept:-0}" --argjson deleted "$deleted" --argjson gfs_critical "$GFS_CRITICAL_COUNT" --argjson gfs_weekly "$GFS_WEEKLY" --argjson gfs_monthly "$GFS_MONTHLY" '{schema:"homelab-backup-retention-v1",status:"OK",task_id: +APPQ_FRESHNESS_PLAN_RC=65 +REC|APPQ_ERR|APP_QUORUM_QUEUE_ERROR=TARGET_REGISTRY_HELPER_SHA_DRIFT +GFS_VALIDATE_RC=66 +REC|GFS_ERR|GFS_POLICY_ERROR=READ_OR_JSON_INVALID +RETENTION_CONTRACT_RC=65 +REC|RET_ERR|GC_STATUS=ERROR|ROOT_CAUSE=TARGET_REGISTRY_HELPER_SHA_DRIFT +VPN_EXECSTART_COUNT=1 +REC|VPN_EXEC_PATH|IDX=1|BASENAME=bash|PARENT=bin|SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f|EXEC=true +REC|VPN_EXEC_PATH|IDX=2|BASENAME=homelab-check-private-vpn-hosts|PARENT=sbin|SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82|EXEC=true +REC|VPN_EXEC_PATH|IDX=3|BASENAME=private-vpn-hosts.txt|PARENT=homelab-health|SHA=09a974d2739c76c5ebfcba8ec9c43b757a63e91570a87535c67537bf073c273d|EXEC=false +VPN_SCRIPT_FOUND=true +VPN_SCRIPT_BASENAME=homelab-check-private-vpn-hosts +VPN_SCRIPT_SHA=591217150ef742117f677c4db12578528b1eaa0ec69ff31122003505a4fc7a82 +VPN_SCRIPT_MUTATION_PATTERN_COUNT=1 +VPN_LITERAL_NAME_COUNT=5 +REC|VPN_SOURCE|LINE=6|VPN_HOST_REDACTED +REC|VPN_SOURCE|LINE=7|VPN_HOST_REDACTED +REC|VPN_SOURCE|LINE=8|VPN_HOST_REDACTED +REC|VPN_SOURCE|LINE=9|VPN_HOST_REDACTED +REC|VPN_SOURCE|LINE=10|VPN_HOST_REDACTED +REC|VPN_SOURCE|LINE=19|ip="$(getent hosts "$h" 2>/dev/null | awk '{print $1}' | head -n1 || true)" +VPN_SCRIPT_DIRECT_SKIPPED_MUTATION_SCAN=true +PVE01_RESOLV_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74 +PVE01_NAMESERVER_COUNT=1 +PVE01_VPN_HOSTS_LINE_COUNT=0 +PVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f +PVE2_NAMESERVER_COUNT=1 +PVE2_VPN_HOSTS_LINE_COUNT=0 +FAILED_UNIT_COUNT_NOW=2 +REC|FAILED_UNIT_NOW|homelab-backup-audit.service +REC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service +DECISION=PASS_PVE01_AUDIT_HELPERS_VPN_EXACT_RCA +NEXT_GATE=PVE01_AUDIT_HELPERS_VPN_REPAIR +TASK_RESULT=PVE01_AUDIT_HELPERS_VPN_EXACT_RCA +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-AUDIT-HELPERS-VPN-EXACT-RCA-99695R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END