diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1.json b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1.json new file mode 100644 index 00000000..4144ca12 --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1", + "status": "FAIL", + "rc": 3, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-1112-ambiguity-rca", + "started_at_utc": "2026-09-22T05:32:23Z", + "finished_at_utc": "2026-09-22T05:32:23Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "d1e970ab78309057995c785598ad19f335660b3bdad9d91f82f8a58aae349f74", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 3, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "87b985744c8a04978f232aba11354a2e66388e428b0eeec0a763adafc74e8fa5", + "sanitized_output_sha256": "2beac0e910df9a79ef0fdc3a96ecd450e8b0a17ff922f22c44187d8b2e629d3e", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_1112_AMBIGUITY_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1\nMODE=read-only\nCOMPONENT=homelab-backup-1112-ambiguity-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_HEALTH_PRODUCER_AND_SOURCE_RECONCILE;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nVMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nEXPECTED_CLOUDB_HEALTH_PATH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\nCLOUDB_HEALTH_EXISTS=true\nCLOUDB_HEALTH_BYTES=71\nCLOUDB_HEALTH_MTIME_EPOCH=1789966497\nCLOUDB_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6\nCLOUDB_HEALTH_MARKERS_BEGIN=1\nSTATUS=FAIL\nCLOUDB_HEALTH_MARKERS_END=1\nHEALTH_PATH_SOURCE_MATCH_COUNT=8\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 BYTES=15391\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/homelab-backup-audit\n19-GFSP=\"/etc/homelab-backup/gfs-retention.json\"\n20-RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\n21-WT=\"/root/homelab-ops-cr-2026-1005\"\n22-BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\n23-DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\n24:CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\n25-HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\n26-\n27-def run(cmd,timeout=60,env=None):\n28- try:\n29- p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)\n--\n231- txt=CLOUDB_HEALTH.read_text(encoding=\"utf-8\",errors=\"replace\")\n232- kv={}\n233- for line in txt.splitlines():\n234- if \"=\" in line:\n235- k,v=line.split(\"=\",1); kv[k]=v\n236: cloudb_ok=(kv.get(\"STATUS\")==\"OK\" and kv.get(\"CLOUD_2_STATUS\")==\"OK\" and kv.get(\"SHA256_OK\")==\"yes\" and kv.get(\"TAR_OK\")==\"yes\" and kv.get(\"REQUIRED_FILES_OK\")==\"yes\")\n237-if not cloudb_ok:\n238- issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"LATEST_CLOUD_B_DRILL_NOT_OK\"})\n239-\n240-source={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\n241-hrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/homelab-backup-audit\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc SHA256=f9c9a17c09c542a653df544b74321c797e89cd2dbd726b52287bfd6097b0998a BYTES=29834\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc SHA256=1cff2c1761a8487dd0c7d6e2a0d6b4a6a45710b23e7afdaad1059526b96f396c BYTES=24854\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc SHA256=8acaa25f3d259e2d00d0bccf86a64c616d2fe447bbab492f9e12c4b996334582 BYTES=23194\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 BYTES=5610\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup\n1-#!/usr/bin/env bash\n2-set -Eeuo pipefail\n3-umask 077\n4:CONF=/etc/skladchik-reports-monitor-backup.conf\n5-[ -r \"$CONF\" ] && . \"$CONF\"\n6-: \"${EDGE_HOST:=[PRIVATE_IP]}\"\n7-: \"${EDGE_USER:=debian}\"\n8-: \"${SERVICE_NAME:=skladchik-reports-monitor}\"\n9:: \"${LOCAL_ROOT:=/mnt/staging/skladchik-reports-monitor-backup}\"\n10:: \"${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}\"\n11-: \"${CLOUD_1:=pve01-mail-01-crypt:}\"\n12-: \"${CLOUD_2:=pve01-mail-02-crypt:}\"\n13-: \"${KEEP_DAILY:=30}\"\n14-: \"${CONNECT_TIMEOUT:=20}\"\n15-SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=\"${CONNECT_TIMEOUT}\" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\n16-RUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n17-DAY=\"$(date -u +%Y%m%d)\"\n18-OUTDIR=\"${LOCAL_ROOT}/daily/${RUN_ID}\"\n19:LOGDIR=/var/log/skladchik-reports-monitor-backup\n20:HEALTH=/var/lib/skladchik-reports-monitor-backup/health.txt\n21:LOCK=/run/skladchik-reports-monitor-backup.lock\n22-mkdir -p \"$OUTDIR\" \"$LOGDIR\" \"$(dirname \"$HEALTH\")\"\n23-exec 9>\"$LOCK\"\n24-if ! flock -n 9; then echo \"STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$HEALTH\"; exit 0; fi\n25-log(){ printf '%s %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/backup.log\"; }\n26-fail(){ rc=$?; printf 'STATUS=FAIL\\nTS=%s\\nRUN_ID=%s\\nerror_rc=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$rc\" > \"$HEALTH\"; log \"FAIL rc=$rc\"; exit \"$rc\"; }\n--\n30-log \"START run_id=$RUN_ID edge=${EDGE_USER}@${EDGE_HOST}\"\n31-ARCHIVE=\"${OUTDIR}/skladchik-reports-monitor-${RUN_ID}.tar.gz\"\n32-ssh \"${SSH_OPTS[@]}\" \"${EDGE_USER}@${EDGE_HOST}\" sudo bash -s \"$RUN_ID\" > \"$ARCHIVE\" <<'REMOTE_SCRIPT'\n33-set -Eeuo pipefail\n34-RUN_ID=\"$1\"\n35:TMP=\"$(mktemp -d /tmp/skladchik-reports-monitor-backup.XXXXXX)\"\n36-cleanup(){ rm -rf \"$TMP\"; }\n37-trap cleanup EXIT\n38-mkdir -p \"$TMP/payload/var-lib-skladchik-reports-monitor\" \"$TMP/payload/systemd\"\n39-cp -a /etc/skladchik-reports-monitor \"$TMP/payload/etc-skladchik-reports-monitor\" 2>/dev/null || true\n40-cp -a /opt/skladchik-reports-monitor \"$TMP/payload/opt-skladchik-reports-monitor\" 2>/dev/null || true\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 BYTES=6851\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\n1-#!/usr/bin/env bash\n2-set -Eeuo pipefail\n3-umask 077\n4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n5-[ -r \"$CONF\" ] && . \"$CONF\"\n6:: \"${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}\"\n7-: \"${CLOUD_1:=pve01-mail-01-crypt:}\"\n8-: \"${CLOUD_2:=pve01-mail-02-crypt:}\"\n9:: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n10-: \"${KEEP_LOCAL:=10}\"\n11:LOGDIR=/var/log/skladchik-reports-monitor-backup\n12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n14-RUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n15-WORK=\"${WORK_ROOT}/${RUN_ID}\"\n16-mkdir -p \"$WORK\" \"$LOGDIR\" \"$(dirname \"$HEALTH\")\"\n17-exec 9>\"$LOCK\"\n18-if ! flock -n 9; then echo \"STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$HEALTH\"; exit 0; fi\n--\n88- py_status=skipped\n89- if command -v python3 >/dev/null; then\n90- python3 -m py_compile \"$extract/opt-skladchik-reports-monitor/monitor.py\" \"$extract/opt-skladchik-reports-monitor/health_alert.py\" \"$extract/opt-skladchik-reports-monitor/operational_digest.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-runner.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-core\"\n91- py_status=ok\n92- fi\n93: printf '%s_STATUS=OK\\n%s_RUN=%s\\n%s_ARCHIVE_BYTES=%s\\n%s_SHA256_OK=yes\\n%s_TAR_OK=yes\\n%s_REQUIRED_FILES_OK=yes\\n%s_COOKIE_JAR_OK=yes\\n%s_EXTENSION_HEALTH_FILES_OK=yes\\n%s_EXTENSION_SCRIPTS_OK=yes\\n%s_EXTENSION_UNITS_OK=yes\\n%s_SQLITE=%s\\n%s_PY_COMPILE=%s\\n' \"$label\" \"$label\" \"$latest\" \"$label\" \"$(stat -c %s \"$archive\")\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$sqlite_status\" \"$label\" \"$py_status\" >> \"$WORK/result.env\"\n94- log \"${label} OK run=${latest} bytes=$(stat -c %s \"$archive\") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}\"\n95-}\n96-: > \"$WORK/result.env\"\n97-check_one_remote CLOUD_1 \"$CLOUD_1\"\n98-check_one_remote CLOUD_2 \"$CLOUD_2\"\n99-. \"$WORK/result.env\"\n100:printf 'STATUS=OK\\nTS=%s\\nRUN_ID=%s\\nREMOTE_BASE=%s\\nCLOUD_1_STATUS=%s\\nCLOUD_1_RUN=%s\\nCLOUD_2_STATUS=%s\\nCLOUD_2_RUN=%s\\nSHA256_OK=yes\\nTAR_OK=yes\\nREQUIRED_FILES_OK=yes\\nCOOKIE_JAR_OK=yes\\nEXTENSION_HEALTH_FILES_OK=yes\\nEXTENSION_SCRIPTS_OK=yes\\nEXTENSION_UNITS_OK=yes\\nSQLITE_OK=yes\\nPY_COMPILE_OK=yes\\nRESTORE_CHECK_VERSION=4\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$REMOTE_BASE\" \"${CLOUD_1_STATUS:-unknown}\" \"${CLOUD_1_RUN:-}\" \"${CLOUD_2_STATUS:-unknown}\" \"${CLOUD_2_RUN:-}\" > \"$HEALTH\"\n101-find \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\n102-log \"OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}\"\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67 BYTES=753\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status\n1-#!/usr/bin/env bash\n2-set +e\n3-printf 'RESTORE_CHECK_HEALTH_BEGIN\\n'\n4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true\n5-printf 'RESTORE_CHECK_HEALTH_END\\n'\n6-printf 'RESTORE_CHECK_TIMER_BEGIN\\n'\n7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true\n8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true\n9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true\n10-printf 'RESTORE_CHECK_TIMER_END\\n'\n11-printf 'RESTORE_CHECK_LOG_TAIL_BEGIN\\n'\n12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true\n13-printf 'RESTORE_CHECK_LOG_TAIL_END\\n'\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0 BYTES=606\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-status\n1-#!/usr/bin/env bash\n2-set +e\n3-printf 'HEALTH_BEGIN\\n'\n4:cat /var/lib/skladchik-reports-monitor-backup/health.txt 2>/dev/null || true\n5-printf 'HEALTH_END\\n'\n6-printf 'TIMER_BEGIN\\n'\n7:systemctl is-enabled skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/enabled=/' || true\n8:systemctl is-active skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/active=/' || true\n9:systemctl list-timers --all | grep skladchik-reports-monitor-backup || true\n10-printf 'TIMER_END\\n'\n11-printf 'LOG_TAIL_BEGIN\\n'\n12:tail -n 60 /var/log/skladchik-reports-monitor-backup/backup.log 2>/dev/null || true\n13-printf 'LOG_TAIL_END\\n'\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-status\nHEALTH_EXECUTABLE_MATCH_COUNT=5\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0\nHEALTH_SERVICE_MATCH_COUNT=1\nHEALTH_SERVICE_FILE=/etc/systemd/system/homelab-backup-audit.service SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa\nHEALTH_SERVICE_UNIT=homelab-backup-audit.service\nDescription=Homelab unified backup and DR audit\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-audit scheduled\nUser=root\nGroup=root\nResult=exit-code,ExecMainStatus=1,ActiveState=failed,SubState=failed,\nSKLADCHIK_BACKUP_STATE_FILES_BEGIN=1\nMTIME_FLOAT|71|/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\nMTIME_FLOAT|484|/var/lib/skladchik-reports-monitor-backup/health.txt\nSKLADCHIK_BACKUP_STATE_FILES_END=1\nSOURCE_WORKTREE_VALID=true\nSOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614\nSOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9\nSOURCE_PUSHURL_CLASS=DISABLED\nSOURCE_REMOTE_HEAD=UNKNOWN\nSOURCE_HEAD_ALIGNMENT=UNKNOWN\nDECISION=HOLD_BACKUP_1113_PRODUCER_OR_SOURCE_STILL_AMBIGUOUS\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1113_AMBIGUITY\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1112_AMBIGUITY_RCA\nCHANGES_MADE_BY_1113=false\nPRODUCT_MUTATION_BY_1113=false\nVM_MUTATION_BY_1113=false\nCLOUD_MUTATION_BY_1113=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1112_AMBIGUITY_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1.txt b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1.txt new file mode 100644 index 00000000..f2759749 --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1.txt @@ -0,0 +1,240 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1 +STATUS=FAIL +RC=3 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-1112-ambiguity-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=d1e970ab78309057995c785598ad19f335660b3bdad9d91f82f8a58aae349f74 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=3 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=87b985744c8a04978f232aba11354a2e66388e428b0eeec0a763adafc74e8fa5 +SANITIZED_OUTPUT_SHA256=2beac0e910df9a79ef0fdc3a96ecd450e8b0a17ff922f22c44187d8b2e629d3e +OUTPUT_BEGIN +WORKERS2_BACKUP_1112_AMBIGUITY_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1 +MODE=read-only +COMPONENT=homelab-backup-1112-ambiguity-rca +MUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_HEALTH_PRODUCER_AND_SOURCE_RECONCILE;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP +ERROR_REGISTER_CHECK=OK +REFERENCE_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 +AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 +VMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 +VM_QUEUE_CURRENT_STATE=success,inactive,dead +BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed +SCHEDULER_RC=0 +SELFTEST_REPLACEMENT486=PASS +SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET +EXPECTED_CLOUDB_HEALTH_PATH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +CLOUDB_HEALTH_EXISTS=true +CLOUDB_HEALTH_BYTES=71 +CLOUDB_HEALTH_MTIME_EPOCH=1789966497 +CLOUDB_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6 +CLOUDB_HEALTH_MARKERS_BEGIN=1 +STATUS=FAIL +CLOUDB_HEALTH_MARKERS_END=1 +HEALTH_PATH_SOURCE_MATCH_COUNT=8 +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 BYTES=15391 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/homelab-backup-audit +19-GFSP="/etc/homelab-backup/gfs-retention.json" +20-RET="/usr/local/sbin/homelab-backup-retention-gc" +21-WT="/root/homelab-ops-cr-2026-1005" +22-BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" +23-DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") +24:CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +25-HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] +26- +27-def run(cmd,timeout=60,env=None): +28- try: +29- p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env) +-- +231- txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace") +232- kv={} +233- for line in txt.splitlines(): +234- if "=" in line: +235- k,v=line.split("=",1); kv[k]=v +236: cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes") +237-if not cloudb_ok: +238- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"}) +239- +240-source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +241-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/homelab-backup-audit +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc SHA256=f9c9a17c09c542a653df544b74321c797e89cd2dbd726b52287bfd6097b0998a BYTES=29834 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc SHA256=1cff2c1761a8487dd0c7d6e2a0d6b4a6a45710b23e7afdaad1059526b96f396c BYTES=24854 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc SHA256=8acaa25f3d259e2d00d0bccf86a64c616d2fe447bbab492f9e12c4b996334582 BYTES=23194 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 BYTES=5610 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup +1-#!/usr/bin/env bash +2-set -Eeuo pipefail +3-umask 077 +4:CONF=/etc/skladchik-reports-monitor-backup.conf +5-[ -r "$CONF" ] && . "$CONF" +6-: "${EDGE_HOST:=[PRIVATE_IP]}" +7-: "${EDGE_USER:=debian}" +8-: "${SERVICE_NAME:=skladchik-reports-monitor}" +9:: "${LOCAL_ROOT:=/mnt/staging/skladchik-reports-monitor-backup}" +10:: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}" +11-: "${CLOUD_1:=pve01-mail-01-crypt:}" +12-: "${CLOUD_2:=pve01-mail-02-crypt:}" +13-: "${KEEP_DAILY:=30}" +14-: "${CONNECT_TIMEOUT:=20}" +15-SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout="${CONNECT_TIMEOUT}" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] +16-RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +17-DAY="$(date -u +%Y%m%d)" +18-OUTDIR="${LOCAL_ROOT}/daily/${RUN_ID}" +19:LOGDIR=/var/log/skladchik-reports-monitor-backup +20:HEALTH=/var/lib/skladchik-reports-monitor-backup/health.txt +21:LOCK=/run/skladchik-reports-monitor-backup.lock +22-mkdir -p "$OUTDIR" "$LOGDIR" "$(dirname "$HEALTH")" +23-exec 9>"$LOCK" +24-if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi +25-log(){ printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/backup.log"; } +26-fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; } +-- +30-log "START run_id=$RUN_ID edge=${EDGE_USER}@${EDGE_HOST}" +31-ARCHIVE="${OUTDIR}/skladchik-reports-monitor-${RUN_ID}.tar.gz" +32-ssh "${SSH_OPTS[@]}" "${EDGE_USER}@${EDGE_HOST}" sudo bash -s "$RUN_ID" > "$ARCHIVE" <<'REMOTE_SCRIPT' +33-set -Eeuo pipefail +34-RUN_ID="$1" +35:TMP="$(mktemp -d /tmp/skladchik-reports-monitor-backup.XXXXXX)" +36-cleanup(){ rm -rf "$TMP"; } +37-trap cleanup EXIT +38-mkdir -p "$TMP/payload/var-lib-skladchik-reports-monitor" "$TMP/payload/systemd" +39-cp -a /etc/skladchik-reports-monitor "$TMP/payload/etc-skladchik-reports-monitor" 2>/dev/null || true +40-cp -a /opt/skladchik-reports-monitor "$TMP/payload/opt-skladchik-reports-monitor" 2>/dev/null || true +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 BYTES=6851 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check +1-#!/usr/bin/env bash +2-set -Eeuo pipefail +3-umask 077 +4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf +5-[ -r "$CONF" ] && . "$CONF" +6:: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}" +7-: "${CLOUD_1:=pve01-mail-01-crypt:}" +8-: "${CLOUD_2:=pve01-mail-02-crypt:}" +9:: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}" +10-: "${KEEP_LOCAL:=10}" +11:LOGDIR=/var/log/skladchik-reports-monitor-backup +12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock +14-RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +15-WORK="${WORK_ROOT}/${RUN_ID}" +16-mkdir -p "$WORK" "$LOGDIR" "$(dirname "$HEALTH")" +17-exec 9>"$LOCK" +18-if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi +-- +88- py_status=skipped +89- if command -v python3 >/dev/null; then +90- python3 -m py_compile "$extract/opt-skladchik-reports-monitor/monitor.py" "$extract/opt-skladchik-reports-monitor/health_alert.py" "$extract/opt-skladchik-reports-monitor/operational_digest.py" "$extract/opt-skladchik-reports-monitor/event-selftest-runner.py" "$extract/opt-skladchik-reports-monitor/event-selftest-core" +91- py_status=ok +92- fi +93: printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env" +94- log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}" +95-} +96-: > "$WORK/result.env" +97-check_one_remote CLOUD_1 "$CLOUD_1" +98-check_one_remote CLOUD_2 "$CLOUD_2" +99-. "$WORK/result.env" +100:printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH" +101-find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done +102-log "OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}" +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67 BYTES=753 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status +1-#!/usr/bin/env bash +2-set +e +3-printf 'RESTORE_CHECK_HEALTH_BEGIN\n' +4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true +5-printf 'RESTORE_CHECK_HEALTH_END\n' +6-printf 'RESTORE_CHECK_TIMER_BEGIN\n' +7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true +8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true +9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true +10-printf 'RESTORE_CHECK_TIMER_END\n' +11-printf 'RESTORE_CHECK_LOG_TAIL_BEGIN\n' +12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true +13-printf 'RESTORE_CHECK_LOG_TAIL_END\n' +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0 BYTES=606 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-status +1-#!/usr/bin/env bash +2-set +e +3-printf 'HEALTH_BEGIN\n' +4:cat /var/lib/skladchik-reports-monitor-backup/health.txt 2>/dev/null || true +5-printf 'HEALTH_END\n' +6-printf 'TIMER_BEGIN\n' +7:systemctl is-enabled skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/enabled=/' || true +8:systemctl is-active skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/active=/' || true +9:systemctl list-timers --all | grep skladchik-reports-monitor-backup || true +10-printf 'TIMER_END\n' +11-printf 'LOG_TAIL_BEGIN\n' +12:tail -n 60 /var/log/skladchik-reports-monitor-backup/backup.log 2>/dev/null || true +13-printf 'LOG_TAIL_END\n' +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-status +HEALTH_EXECUTABLE_MATCH_COUNT=5 +HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 +HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 +HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 +HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67 +HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0 +HEALTH_SERVICE_MATCH_COUNT=1 +HEALTH_SERVICE_FILE=/etc/systemd/system/homelab-backup-audit.service SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa +HEALTH_SERVICE_UNIT=homelab-backup-audit.service +Description=Homelab unified backup and DR audit +Type=oneshot +ExecStart=/usr/local/sbin/homelab-backup-audit scheduled +User=root +Group=root +Result=exit-code,ExecMainStatus=1,ActiveState=failed,SubState=failed, +SKLADCHIK_BACKUP_STATE_FILES_BEGIN=1 +MTIME_FLOAT|71|/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +MTIME_FLOAT|484|/var/lib/skladchik-reports-monitor-backup/health.txt +SKLADCHIK_BACKUP_STATE_FILES_END=1 +SOURCE_WORKTREE_VALID=true +SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614 +SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9 +SOURCE_PUSHURL_CLASS=DISABLED +SOURCE_REMOTE_HEAD=UNKNOWN +SOURCE_HEAD_ALIGNMENT=UNKNOWN +DECISION=HOLD_BACKUP_1113_PRODUCER_OR_SOURCE_STILL_AMBIGUOUS +NEXT_GATE=RECONCILE_ONLY_BACKUP_1113_AMBIGUITY +TASK_RESULT=HOLD_HOMELAB_BACKUP_1112_AMBIGUITY_RCA +CHANGES_MADE_BY_1113=false +PRODUCT_MUTATION_BY_1113=false +VM_MUTATION_BY_1113=false +CLOUD_MUTATION_BY_1113=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1112_AMBIGUITY_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 574fa5a7..4144ca12 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1", "status": "FAIL", "rc": 3, "host": "pve01", "mode": "read-only", - "component": "homelab-backup-1111r2-metadata-rca", - "started_at_utc": "2026-09-22T05:26:41Z", - "finished_at_utc": "2026-09-22T05:26:42Z", + "component": "homelab-backup-1112-ambiguity-rca", + "started_at_utc": "2026-09-22T05:32:23Z", + "finished_at_utc": "2026-09-22T05:32:23Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "7682fd29d69970b8027a5a07ca78e4c4b97a05bc2e5ef1dc75d70b230b1a669a", + "command_sha256": "d1e970ab78309057995c785598ad19f335660b3bdad9d91f82f8a58aae349f74", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411", - "sanitized_output_sha256": "745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411", + "raw_evidence_sha256": "87b985744c8a04978f232aba11354a2e66388e428b0eeec0a763adafc74e8fa5", + "sanitized_output_sha256": "2beac0e910df9a79ef0fdc3a96ecd450e8b0a17ff922f22c44187d8b2e629d3e", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_BACKUP_1111R2_METADATA_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1\nMODE=read-only\nCOMPONENT=homelab-backup-1111r2-metadata-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_SOURCE_AND_STATE_RECONCILE;NO_SYSTEMCTL_START_STOP_RESET;NO_BACKUP;NO_RESTORE;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nVMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nAST_ASSIGN_COUNT_BRANCH=1\nAST_ASSIGN_BRANCH_LINE=22\nAST_ASSIGN_BRANCH_EXPR=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_BRANCH_18=GFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nAUDIT_SOURCE_BRANCH_19=GFSP=\"/etc/homelab-backup/gfs-retention.json\"\nAUDIT_SOURCE_BRANCH_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_BRANCH_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_BRANCH_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_BRANCH_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_BRANCH_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_BRANCH_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nAUDIT_SOURCE_BRANCH_26=\nAST_ASSIGN_COUNT_CLOUDB_HEALTH=1\nAST_ASSIGN_CLOUDB_HEALTH_LINE=24\nAST_ASSIGN_CLOUDB_HEALTH_EXPR=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_CLOUDB_HEALTH_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_CLOUDB_HEALTH_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_CLOUDB_HEALTH_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_CLOUDB_HEALTH_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_CLOUDB_HEALTH_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_CLOUDB_HEALTH_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nAUDIT_SOURCE_CLOUDB_HEALTH_26=\nAUDIT_SOURCE_CLOUDB_HEALTH_27=def run(cmd,timeout=60,env=None):\nAUDIT_SOURCE_CLOUDB_HEALTH_28= try:\nAST_ASSIGN_COUNT_WT=1\nAST_ASSIGN_WT_LINE=21\nAST_ASSIGN_WT_EXPR=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_WT_17=APPQ=\"/usr/local/sbin/homelab-app-cloud-quorum-queue\"\nAUDIT_SOURCE_WT_18=GFSH=\"/usr/local/libexec/homelab-backup-gfs-policy\"\nAUDIT_SOURCE_WT_19=GFSP=\"/etc/homelab-backup/gfs-retention.json\"\nAUDIT_SOURCE_WT_20=RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\nAUDIT_SOURCE_WT_21=WT=\"/root/homelab-ops-cr-2026-1005\"\nAUDIT_SOURCE_WT_22=BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\nAUDIT_SOURCE_WT_23=DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\nAUDIT_SOURCE_WT_24=CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nAUDIT_SOURCE_WT_25=HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\nCLOUDB_HEALTH_CANDIDATE_COUNT=0\nCLOUDB_PRODUCER_COUNT=0\nCLOUDB_SERVICE_MATCH_COUNT=0\nSOURCE_WT=/root/homelab-ops-cr-2026-1005\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_HEAD_ALIGNMENT=UNAVAILABLE\nDECISION=HOLD_BACKUP_1112_METADATA_OR_IDENTITY_AMBIGUOUS\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1112_AMBIGUITY\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1111R2_METADATA_RCA\nCHANGES_MADE_BY_1112=false\nPRODUCT_MUTATION_BY_1112=false\nVM_MUTATION_BY_1112=false\nCLOUD_MUTATION_BY_1112=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1111R2_METADATA_RCA_END=1\n" + "output": "WORKERS2_BACKUP_1112_AMBIGUITY_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1\nMODE=read-only\nCOMPONENT=homelab-backup-1112-ambiguity-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_HEALTH_PRODUCER_AND_SOURCE_RECONCILE;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nVMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nEXPECTED_CLOUDB_HEALTH_PATH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\nCLOUDB_HEALTH_EXISTS=true\nCLOUDB_HEALTH_BYTES=71\nCLOUDB_HEALTH_MTIME_EPOCH=1789966497\nCLOUDB_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6\nCLOUDB_HEALTH_MARKERS_BEGIN=1\nSTATUS=FAIL\nCLOUDB_HEALTH_MARKERS_END=1\nHEALTH_PATH_SOURCE_MATCH_COUNT=8\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 BYTES=15391\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/homelab-backup-audit\n19-GFSP=\"/etc/homelab-backup/gfs-retention.json\"\n20-RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\n21-WT=\"/root/homelab-ops-cr-2026-1005\"\n22-BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\n23-DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\n24:CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\n25-HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\n26-\n27-def run(cmd,timeout=60,env=None):\n28- try:\n29- p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)\n--\n231- txt=CLOUDB_HEALTH.read_text(encoding=\"utf-8\",errors=\"replace\")\n232- kv={}\n233- for line in txt.splitlines():\n234- if \"=\" in line:\n235- k,v=line.split(\"=\",1); kv[k]=v\n236: cloudb_ok=(kv.get(\"STATUS\")==\"OK\" and kv.get(\"CLOUD_2_STATUS\")==\"OK\" and kv.get(\"SHA256_OK\")==\"yes\" and kv.get(\"TAR_OK\")==\"yes\" and kv.get(\"REQUIRED_FILES_OK\")==\"yes\")\n237-if not cloudb_ok:\n238- issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"LATEST_CLOUD_B_DRILL_NOT_OK\"})\n239-\n240-source={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\n241-hrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/homelab-backup-audit\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc SHA256=f9c9a17c09c542a653df544b74321c797e89cd2dbd726b52287bfd6097b0998a BYTES=29834\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc SHA256=1cff2c1761a8487dd0c7d6e2a0d6b4a6a45710b23e7afdaad1059526b96f396c BYTES=24854\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc SHA256=8acaa25f3d259e2d00d0bccf86a64c616d2fe447bbab492f9e12c4b996334582 BYTES=23194\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 BYTES=5610\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup\n1-#!/usr/bin/env bash\n2-set -Eeuo pipefail\n3-umask 077\n4:CONF=/etc/skladchik-reports-monitor-backup.conf\n5-[ -r \"$CONF\" ] && . \"$CONF\"\n6-: \"${EDGE_HOST:=[PRIVATE_IP]}\"\n7-: \"${EDGE_USER:=debian}\"\n8-: \"${SERVICE_NAME:=skladchik-reports-monitor}\"\n9:: \"${LOCAL_ROOT:=/mnt/staging/skladchik-reports-monitor-backup}\"\n10:: \"${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}\"\n11-: \"${CLOUD_1:=pve01-mail-01-crypt:}\"\n12-: \"${CLOUD_2:=pve01-mail-02-crypt:}\"\n13-: \"${KEEP_DAILY:=30}\"\n14-: \"${CONNECT_TIMEOUT:=20}\"\n15-SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=\"${CONNECT_TIMEOUT}\" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\n16-RUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n17-DAY=\"$(date -u +%Y%m%d)\"\n18-OUTDIR=\"${LOCAL_ROOT}/daily/${RUN_ID}\"\n19:LOGDIR=/var/log/skladchik-reports-monitor-backup\n20:HEALTH=/var/lib/skladchik-reports-monitor-backup/health.txt\n21:LOCK=/run/skladchik-reports-monitor-backup.lock\n22-mkdir -p \"$OUTDIR\" \"$LOGDIR\" \"$(dirname \"$HEALTH\")\"\n23-exec 9>\"$LOCK\"\n24-if ! flock -n 9; then echo \"STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$HEALTH\"; exit 0; fi\n25-log(){ printf '%s %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/backup.log\"; }\n26-fail(){ rc=$?; printf 'STATUS=FAIL\\nTS=%s\\nRUN_ID=%s\\nerror_rc=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$rc\" > \"$HEALTH\"; log \"FAIL rc=$rc\"; exit \"$rc\"; }\n--\n30-log \"START run_id=$RUN_ID edge=${EDGE_USER}@${EDGE_HOST}\"\n31-ARCHIVE=\"${OUTDIR}/skladchik-reports-monitor-${RUN_ID}.tar.gz\"\n32-ssh \"${SSH_OPTS[@]}\" \"${EDGE_USER}@${EDGE_HOST}\" sudo bash -s \"$RUN_ID\" > \"$ARCHIVE\" <<'REMOTE_SCRIPT'\n33-set -Eeuo pipefail\n34-RUN_ID=\"$1\"\n35:TMP=\"$(mktemp -d /tmp/skladchik-reports-monitor-backup.XXXXXX)\"\n36-cleanup(){ rm -rf \"$TMP\"; }\n37-trap cleanup EXIT\n38-mkdir -p \"$TMP/payload/var-lib-skladchik-reports-monitor\" \"$TMP/payload/systemd\"\n39-cp -a /etc/skladchik-reports-monitor \"$TMP/payload/etc-skladchik-reports-monitor\" 2>/dev/null || true\n40-cp -a /opt/skladchik-reports-monitor \"$TMP/payload/opt-skladchik-reports-monitor\" 2>/dev/null || true\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 BYTES=6851\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\n1-#!/usr/bin/env bash\n2-set -Eeuo pipefail\n3-umask 077\n4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n5-[ -r \"$CONF\" ] && . \"$CONF\"\n6:: \"${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}\"\n7-: \"${CLOUD_1:=pve01-mail-01-crypt:}\"\n8-: \"${CLOUD_2:=pve01-mail-02-crypt:}\"\n9:: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n10-: \"${KEEP_LOCAL:=10}\"\n11:LOGDIR=/var/log/skladchik-reports-monitor-backup\n12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n14-RUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n15-WORK=\"${WORK_ROOT}/${RUN_ID}\"\n16-mkdir -p \"$WORK\" \"$LOGDIR\" \"$(dirname \"$HEALTH\")\"\n17-exec 9>\"$LOCK\"\n18-if ! flock -n 9; then echo \"STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$HEALTH\"; exit 0; fi\n--\n88- py_status=skipped\n89- if command -v python3 >/dev/null; then\n90- python3 -m py_compile \"$extract/opt-skladchik-reports-monitor/monitor.py\" \"$extract/opt-skladchik-reports-monitor/health_alert.py\" \"$extract/opt-skladchik-reports-monitor/operational_digest.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-runner.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-core\"\n91- py_status=ok\n92- fi\n93: printf '%s_STATUS=OK\\n%s_RUN=%s\\n%s_ARCHIVE_BYTES=%s\\n%s_SHA256_OK=yes\\n%s_TAR_OK=yes\\n%s_REQUIRED_FILES_OK=yes\\n%s_COOKIE_JAR_OK=yes\\n%s_EXTENSION_HEALTH_FILES_OK=yes\\n%s_EXTENSION_SCRIPTS_OK=yes\\n%s_EXTENSION_UNITS_OK=yes\\n%s_SQLITE=%s\\n%s_PY_COMPILE=%s\\n' \"$label\" \"$label\" \"$latest\" \"$label\" \"$(stat -c %s \"$archive\")\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$sqlite_status\" \"$label\" \"$py_status\" >> \"$WORK/result.env\"\n94- log \"${label} OK run=${latest} bytes=$(stat -c %s \"$archive\") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}\"\n95-}\n96-: > \"$WORK/result.env\"\n97-check_one_remote CLOUD_1 \"$CLOUD_1\"\n98-check_one_remote CLOUD_2 \"$CLOUD_2\"\n99-. \"$WORK/result.env\"\n100:printf 'STATUS=OK\\nTS=%s\\nRUN_ID=%s\\nREMOTE_BASE=%s\\nCLOUD_1_STATUS=%s\\nCLOUD_1_RUN=%s\\nCLOUD_2_STATUS=%s\\nCLOUD_2_RUN=%s\\nSHA256_OK=yes\\nTAR_OK=yes\\nREQUIRED_FILES_OK=yes\\nCOOKIE_JAR_OK=yes\\nEXTENSION_HEALTH_FILES_OK=yes\\nEXTENSION_SCRIPTS_OK=yes\\nEXTENSION_UNITS_OK=yes\\nSQLITE_OK=yes\\nPY_COMPILE_OK=yes\\nRESTORE_CHECK_VERSION=4\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$REMOTE_BASE\" \"${CLOUD_1_STATUS:-unknown}\" \"${CLOUD_1_RUN:-}\" \"${CLOUD_2_STATUS:-unknown}\" \"${CLOUD_2_RUN:-}\" > \"$HEALTH\"\n101-find \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\n102-log \"OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}\"\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67 BYTES=753\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status\n1-#!/usr/bin/env bash\n2-set +e\n3-printf 'RESTORE_CHECK_HEALTH_BEGIN\\n'\n4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true\n5-printf 'RESTORE_CHECK_HEALTH_END\\n'\n6-printf 'RESTORE_CHECK_TIMER_BEGIN\\n'\n7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true\n8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true\n9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true\n10-printf 'RESTORE_CHECK_TIMER_END\\n'\n11-printf 'RESTORE_CHECK_LOG_TAIL_BEGIN\\n'\n12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true\n13-printf 'RESTORE_CHECK_LOG_TAIL_END\\n'\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0 BYTES=606\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-status\n1-#!/usr/bin/env bash\n2-set +e\n3-printf 'HEALTH_BEGIN\\n'\n4:cat /var/lib/skladchik-reports-monitor-backup/health.txt 2>/dev/null || true\n5-printf 'HEALTH_END\\n'\n6-printf 'TIMER_BEGIN\\n'\n7:systemctl is-enabled skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/enabled=/' || true\n8:systemctl is-active skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/active=/' || true\n9:systemctl list-timers --all | grep skladchik-reports-monitor-backup || true\n10-printf 'TIMER_END\\n'\n11-printf 'LOG_TAIL_BEGIN\\n'\n12:tail -n 60 /var/log/skladchik-reports-monitor-backup/backup.log 2>/dev/null || true\n13-printf 'LOG_TAIL_END\\n'\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-status\nHEALTH_EXECUTABLE_MATCH_COUNT=5\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0\nHEALTH_SERVICE_MATCH_COUNT=1\nHEALTH_SERVICE_FILE=/etc/systemd/system/homelab-backup-audit.service SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa\nHEALTH_SERVICE_UNIT=homelab-backup-audit.service\nDescription=Homelab unified backup and DR audit\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-audit scheduled\nUser=root\nGroup=root\nResult=exit-code,ExecMainStatus=1,ActiveState=failed,SubState=failed,\nSKLADCHIK_BACKUP_STATE_FILES_BEGIN=1\nMTIME_FLOAT|71|/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\nMTIME_FLOAT|484|/var/lib/skladchik-reports-monitor-backup/health.txt\nSKLADCHIK_BACKUP_STATE_FILES_END=1\nSOURCE_WORKTREE_VALID=true\nSOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614\nSOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9\nSOURCE_PUSHURL_CLASS=DISABLED\nSOURCE_REMOTE_HEAD=UNKNOWN\nSOURCE_HEAD_ALIGNMENT=UNKNOWN\nDECISION=HOLD_BACKUP_1113_PRODUCER_OR_SOURCE_STILL_AMBIGUOUS\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1113_AMBIGUITY\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1112_AMBIGUITY_RCA\nCHANGES_MADE_BY_1113=false\nPRODUCT_MUTATION_BY_1113=false\nVM_MUTATION_BY_1113=false\nCLOUD_MUTATION_BY_1113=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1112_AMBIGUITY_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 6aaa468b..f2759749 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1 STATUS=FAIL RC=3 HOST=pve01 MODE=read-only -COMPONENT=homelab-backup-1111r2-metadata-rca +COMPONENT=homelab-backup-1112-ambiguity-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=7682fd29d69970b8027a5a07ca78e4c4b97a05bc2e5ef1dc75d70b230b1a669a +COMMAND_SHA256=d1e970ab78309057995c785598ad19f335660b3bdad9d91f82f8a58aae349f74 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,14 +24,14 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411 -SANITIZED_OUTPUT_SHA256=745fbfdfd73fd8ea054f372a2703ac589c0afd56c1abfb8883d9e3cf2e3c6411 +RAW_EVIDENCE_SHA256=87b985744c8a04978f232aba11354a2e66388e428b0eeec0a763adafc74e8fa5 +SANITIZED_OUTPUT_SHA256=2beac0e910df9a79ef0fdc3a96ecd450e8b0a17ff922f22c44187d8b2e629d3e OUTPUT_BEGIN -WORKERS2_BACKUP_1111R2_METADATA_RCA_BEGIN=1 -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1 +WORKERS2_BACKUP_1112_AMBIGUITY_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1 MODE=read-only -COMPONENT=homelab-backup-1111r2-metadata-rca -MUTATION_BOUNDARY=NONE;READ_ONLY_SOURCE_AND_STATE_RECONCILE;NO_SYSTEMCTL_START_STOP_RESET;NO_BACKUP;NO_RESTORE;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP +COMPONENT=homelab-backup-1112-ambiguity-rca +MUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_HEALTH_PRODUCER_AND_SOURCE_RECONCILE;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP ERROR_REGISTER_CHECK=OK REFERENCE_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 @@ -45,57 +45,196 @@ BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed SCHEDULER_RC=0 SELFTEST_REPLACEMENT486=PASS SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET -AST_ASSIGN_COUNT_BRANCH=1 -AST_ASSIGN_BRANCH_LINE=22 -AST_ASSIGN_BRANCH_EXPR="cr-2026-1005-backup-v2-health-alerting-remediation" -AUDIT_SOURCE_BRANCH_18=GFSH="/usr/local/libexec/homelab-backup-gfs-policy" -AUDIT_SOURCE_BRANCH_19=GFSP="/etc/homelab-backup/gfs-retention.json" -AUDIT_SOURCE_BRANCH_20=RET="/usr/local/sbin/homelab-backup-retention-gc" -AUDIT_SOURCE_BRANCH_21=WT="/root/homelab-ops-cr-2026-1005" -AUDIT_SOURCE_BRANCH_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" -AUDIT_SOURCE_BRANCH_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") -AUDIT_SOURCE_BRANCH_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") -AUDIT_SOURCE_BRANCH_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] -AUDIT_SOURCE_BRANCH_26= -AST_ASSIGN_COUNT_CLOUDB_HEALTH=1 -AST_ASSIGN_CLOUDB_HEALTH_LINE=24 -AST_ASSIGN_CLOUDB_HEALTH_EXPR=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") -AUDIT_SOURCE_CLOUDB_HEALTH_20=RET="/usr/local/sbin/homelab-backup-retention-gc" -AUDIT_SOURCE_CLOUDB_HEALTH_21=WT="/root/homelab-ops-cr-2026-1005" -AUDIT_SOURCE_CLOUDB_HEALTH_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" -AUDIT_SOURCE_CLOUDB_HEALTH_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") -AUDIT_SOURCE_CLOUDB_HEALTH_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") -AUDIT_SOURCE_CLOUDB_HEALTH_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] -AUDIT_SOURCE_CLOUDB_HEALTH_26= -AUDIT_SOURCE_CLOUDB_HEALTH_27=def run(cmd,timeout=60,env=None): -AUDIT_SOURCE_CLOUDB_HEALTH_28= try: -AST_ASSIGN_COUNT_WT=1 -AST_ASSIGN_WT_LINE=21 -AST_ASSIGN_WT_EXPR="/root/homelab-ops-cr-2026-1005" -AUDIT_SOURCE_WT_17=APPQ="/usr/local/sbin/homelab-app-cloud-quorum-queue" -AUDIT_SOURCE_WT_18=GFSH="/usr/local/libexec/homelab-backup-gfs-policy" -AUDIT_SOURCE_WT_19=GFSP="/etc/homelab-backup/gfs-retention.json" -AUDIT_SOURCE_WT_20=RET="/usr/local/sbin/homelab-backup-retention-gc" -AUDIT_SOURCE_WT_21=WT="/root/homelab-ops-cr-2026-1005" -AUDIT_SOURCE_WT_22=BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" -AUDIT_SOURCE_WT_23=DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") -AUDIT_SOURCE_WT_24=CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") -AUDIT_SOURCE_WT_25=HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] -CLOUDB_HEALTH_CANDIDATE_COUNT=0 -CLOUDB_PRODUCER_COUNT=0 -CLOUDB_SERVICE_MATCH_COUNT=0 -SOURCE_WT=/root/homelab-ops-cr-2026-1005 -SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation -SOURCE_HEAD_ALIGNMENT=UNAVAILABLE -DECISION=HOLD_BACKUP_1112_METADATA_OR_IDENTITY_AMBIGUOUS -NEXT_GATE=RECONCILE_ONLY_BACKUP_1112_AMBIGUITY -TASK_RESULT=HOLD_HOMELAB_BACKUP_1111R2_METADATA_RCA -CHANGES_MADE_BY_1112=false -PRODUCT_MUTATION_BY_1112=false -VM_MUTATION_BY_1112=false -CLOUD_MUTATION_BY_1112=false -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1111R2-METADATA-RCA-1112R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} -WORKERS2_BACKUP_1111R2_METADATA_RCA_END=1 +EXPECTED_CLOUDB_HEALTH_PATH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +CLOUDB_HEALTH_EXISTS=true +CLOUDB_HEALTH_BYTES=71 +CLOUDB_HEALTH_MTIME_EPOCH=1789966497 +CLOUDB_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6 +CLOUDB_HEALTH_MARKERS_BEGIN=1 +STATUS=FAIL +CLOUDB_HEALTH_MARKERS_END=1 +HEALTH_PATH_SOURCE_MATCH_COUNT=8 +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 BYTES=15391 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/homelab-backup-audit +19-GFSP="/etc/homelab-backup/gfs-retention.json" +20-RET="/usr/local/sbin/homelab-backup-retention-gc" +21-WT="/root/homelab-ops-cr-2026-1005" +22-BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" +23-DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") +24:CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +25-HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] +26- +27-def run(cmd,timeout=60,env=None): +28- try: +29- p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env) +-- +231- txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace") +232- kv={} +233- for line in txt.splitlines(): +234- if "=" in line: +235- k,v=line.split("=",1); kv[k]=v +236: cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes") +237-if not cloudb_ok: +238- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"}) +239- +240-source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +241-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/homelab-backup-audit +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc SHA256=f9c9a17c09c542a653df544b74321c797e89cd2dbd726b52287bfd6097b0998a BYTES=29834 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc SHA256=1cff2c1761a8487dd0c7d6e2a0d6b4a6a45710b23e7afdaad1059526b96f396c BYTES=24854 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc SHA256=8acaa25f3d259e2d00d0bccf86a64c616d2fe447bbab492f9e12c4b996334582 BYTES=23194 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 BYTES=5610 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup +1-#!/usr/bin/env bash +2-set -Eeuo pipefail +3-umask 077 +4:CONF=/etc/skladchik-reports-monitor-backup.conf +5-[ -r "$CONF" ] && . "$CONF" +6-: "${EDGE_HOST:=[PRIVATE_IP]}" +7-: "${EDGE_USER:=debian}" +8-: "${SERVICE_NAME:=skladchik-reports-monitor}" +9:: "${LOCAL_ROOT:=/mnt/staging/skladchik-reports-monitor-backup}" +10:: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}" +11-: "${CLOUD_1:=pve01-mail-01-crypt:}" +12-: "${CLOUD_2:=pve01-mail-02-crypt:}" +13-: "${KEEP_DAILY:=30}" +14-: "${CONNECT_TIMEOUT:=20}" +15-SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout="${CONNECT_TIMEOUT}" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] +16-RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +17-DAY="$(date -u +%Y%m%d)" +18-OUTDIR="${LOCAL_ROOT}/daily/${RUN_ID}" +19:LOGDIR=/var/log/skladchik-reports-monitor-backup +20:HEALTH=/var/lib/skladchik-reports-monitor-backup/health.txt +21:LOCK=/run/skladchik-reports-monitor-backup.lock +22-mkdir -p "$OUTDIR" "$LOGDIR" "$(dirname "$HEALTH")" +23-exec 9>"$LOCK" +24-if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi +25-log(){ printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/backup.log"; } +26-fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; } +-- +30-log "START run_id=$RUN_ID edge=${EDGE_USER}@${EDGE_HOST}" +31-ARCHIVE="${OUTDIR}/skladchik-reports-monitor-${RUN_ID}.tar.gz" +32-ssh "${SSH_OPTS[@]}" "${EDGE_USER}@${EDGE_HOST}" sudo bash -s "$RUN_ID" > "$ARCHIVE" <<'REMOTE_SCRIPT' +33-set -Eeuo pipefail +34-RUN_ID="$1" +35:TMP="$(mktemp -d /tmp/skladchik-reports-monitor-backup.XXXXXX)" +36-cleanup(){ rm -rf "$TMP"; } +37-trap cleanup EXIT +38-mkdir -p "$TMP/payload/var-lib-skladchik-reports-monitor" "$TMP/payload/systemd" +39-cp -a /etc/skladchik-reports-monitor "$TMP/payload/etc-skladchik-reports-monitor" 2>/dev/null || true +40-cp -a /opt/skladchik-reports-monitor "$TMP/payload/opt-skladchik-reports-monitor" 2>/dev/null || true +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 BYTES=6851 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check +1-#!/usr/bin/env bash +2-set -Eeuo pipefail +3-umask 077 +4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf +5-[ -r "$CONF" ] && . "$CONF" +6:: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}" +7-: "${CLOUD_1:=pve01-mail-01-crypt:}" +8-: "${CLOUD_2:=pve01-mail-02-crypt:}" +9:: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}" +10-: "${KEEP_LOCAL:=10}" +11:LOGDIR=/var/log/skladchik-reports-monitor-backup +12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock +14-RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +15-WORK="${WORK_ROOT}/${RUN_ID}" +16-mkdir -p "$WORK" "$LOGDIR" "$(dirname "$HEALTH")" +17-exec 9>"$LOCK" +18-if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi +-- +88- py_status=skipped +89- if command -v python3 >/dev/null; then +90- python3 -m py_compile "$extract/opt-skladchik-reports-monitor/monitor.py" "$extract/opt-skladchik-reports-monitor/health_alert.py" "$extract/opt-skladchik-reports-monitor/operational_digest.py" "$extract/opt-skladchik-reports-monitor/event-selftest-runner.py" "$extract/opt-skladchik-reports-monitor/event-selftest-core" +91- py_status=ok +92- fi +93: printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env" +94- log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}" +95-} +96-: > "$WORK/result.env" +97-check_one_remote CLOUD_1 "$CLOUD_1" +98-check_one_remote CLOUD_2 "$CLOUD_2" +99-. "$WORK/result.env" +100:printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH" +101-find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done +102-log "OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}" +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67 BYTES=753 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status +1-#!/usr/bin/env bash +2-set +e +3-printf 'RESTORE_CHECK_HEALTH_BEGIN\n' +4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true +5-printf 'RESTORE_CHECK_HEALTH_END\n' +6-printf 'RESTORE_CHECK_TIMER_BEGIN\n' +7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true +8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true +9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true +10-printf 'RESTORE_CHECK_TIMER_END\n' +11-printf 'RESTORE_CHECK_LOG_TAIL_BEGIN\n' +12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true +13-printf 'RESTORE_CHECK_LOG_TAIL_END\n' +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status +HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0 BYTES=606 +HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-status +1-#!/usr/bin/env bash +2-set +e +3-printf 'HEALTH_BEGIN\n' +4:cat /var/lib/skladchik-reports-monitor-backup/health.txt 2>/dev/null || true +5-printf 'HEALTH_END\n' +6-printf 'TIMER_BEGIN\n' +7:systemctl is-enabled skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/enabled=/' || true +8:systemctl is-active skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/active=/' || true +9:systemctl list-timers --all | grep skladchik-reports-monitor-backup || true +10-printf 'TIMER_END\n' +11-printf 'LOG_TAIL_BEGIN\n' +12:tail -n 60 /var/log/skladchik-reports-monitor-backup/backup.log 2>/dev/null || true +13-printf 'LOG_TAIL_END\n' +HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-status +HEALTH_EXECUTABLE_MATCH_COUNT=5 +HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 +HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 +HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 +HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67 +HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0 +HEALTH_SERVICE_MATCH_COUNT=1 +HEALTH_SERVICE_FILE=/etc/systemd/system/homelab-backup-audit.service SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa +HEALTH_SERVICE_UNIT=homelab-backup-audit.service +Description=Homelab unified backup and DR audit +Type=oneshot +ExecStart=/usr/local/sbin/homelab-backup-audit scheduled +User=root +Group=root +Result=exit-code,ExecMainStatus=1,ActiveState=failed,SubState=failed, +SKLADCHIK_BACKUP_STATE_FILES_BEGIN=1 +MTIME_FLOAT|71|/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +MTIME_FLOAT|484|/var/lib/skladchik-reports-monitor-backup/health.txt +SKLADCHIK_BACKUP_STATE_FILES_END=1 +SOURCE_WORKTREE_VALID=true +SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614 +SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9 +SOURCE_PUSHURL_CLASS=DISABLED +SOURCE_REMOTE_HEAD=UNKNOWN +SOURCE_HEAD_ALIGNMENT=UNKNOWN +DECISION=HOLD_BACKUP_1113_PRODUCER_OR_SOURCE_STILL_AMBIGUOUS +NEXT_GATE=RECONCILE_ONLY_BACKUP_1113_AMBIGUITY +TASK_RESULT=HOLD_HOMELAB_BACKUP_1112_AMBIGUITY_RCA +CHANGES_MADE_BY_1113=false +PRODUCT_MUTATION_BY_1113=false +VM_MUTATION_BY_1113=false +CLOUD_MUTATION_BY_1113=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1112_AMBIGUITY_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END