CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1
STATUS=FAIL
RC=3
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-1112-ambiguity-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=d1e970ab78309057995c785598ad19f335660b3bdad9d91f82f8a58aae349f74
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=3
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=87b985744c8a04978f232aba11354a2e66388e428b0eeec0a763adafc74e8fa5
SANITIZED_OUTPUT_SHA256=2beac0e910df9a79ef0fdc3a96ecd450e8b0a17ff922f22c44187d8b2e629d3e
OUTPUT_BEGIN
WORKERS2_BACKUP_1112_AMBIGUITY_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1
MODE=read-only
COMPONENT=homelab-backup-1112-ambiguity-rca
MUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_HEALTH_PRODUCER_AND_SOURCE_RECONCILE;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP
ERROR_REGISTER_CHECK=OK
REFERENCE_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
VMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612
VM_QUEUE_CURRENT_STATE=success,inactive,dead
BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed
SCHEDULER_RC=0
SELFTEST_REPLACEMENT486=PASS
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
EXPECTED_CLOUDB_HEALTH_PATH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
CLOUDB_HEALTH_EXISTS=true
CLOUDB_HEALTH_BYTES=71
CLOUDB_HEALTH_MTIME_EPOCH=1789966497
CLOUDB_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6
CLOUDB_HEALTH_MARKERS_BEGIN=1
STATUS=FAIL
CLOUDB_HEALTH_MARKERS_END=1
HEALTH_PATH_SOURCE_MATCH_COUNT=8
HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 BYTES=15391
HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/homelab-backup-audit
19-GFSP="/etc/homelab-backup/gfs-retention.json"
20-RET="/usr/local/sbin/homelab-backup-retention-gc"
21-WT="/root/homelab-ops-cr-2026-1005"
22-BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation"
23-DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json")
24:CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
25-HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")]
26-
27-def run(cmd,timeout=60,env=None):
28-    try:
29-        p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)
--
231-    txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace")
232-    kv={}
233-    for line in txt.splitlines():
234-        if "=" in line:
235-            k,v=line.split("=",1); kv[k]=v
236:    cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes")
237-if not cloudb_ok:
238-    issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"})
239-
240-source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
241-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
HEALTH_PATH_CONTEXT_END=/usr/local/sbin/homelab-backup-audit
HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc SHA256=f9c9a17c09c542a653df544b74321c797e89cd2dbd726b52287bfd6097b0998a BYTES=29834
HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc
HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc
HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc SHA256=1cff2c1761a8487dd0c7d6e2a0d6b4a6a45710b23e7afdaad1059526b96f396c BYTES=24854
HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc
HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc
HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc SHA256=8acaa25f3d259e2d00d0bccf86a64c616d2fe447bbab492f9e12c4b996334582 BYTES=23194
HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc
HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc
HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 BYTES=5610
HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup
1-#!/usr/bin/env bash
2-set -Eeuo pipefail
3-umask 077
4:CONF=/etc/skladchik-reports-monitor-backup.conf
5-[ -r "$CONF" ] && . "$CONF"
6-: "${EDGE_HOST:=[PRIVATE_IP]}"
7-: "${EDGE_USER:=debian}"
8-: "${SERVICE_NAME:=skladchik-reports-monitor}"
9:: "${LOCAL_ROOT:=/mnt/staging/skladchik-reports-monitor-backup}"
10:: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}"
11-: "${CLOUD_1:=pve01-mail-01-crypt:}"
12-: "${CLOUD_2:=pve01-mail-02-crypt:}"
13-: "${KEEP_DAILY:=30}"
14-: "${CONNECT_TIMEOUT:=20}"
15-SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout="${CONNECT_TIMEOUT}" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]
16-RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
17-DAY="$(date -u +%Y%m%d)"
18-OUTDIR="${LOCAL_ROOT}/daily/${RUN_ID}"
19:LOGDIR=/var/log/skladchik-reports-monitor-backup
20:HEALTH=/var/lib/skladchik-reports-monitor-backup/health.txt
21:LOCK=/run/skladchik-reports-monitor-backup.lock
22-mkdir -p "$OUTDIR" "$LOGDIR" "$(dirname "$HEALTH")"
23-exec 9>"$LOCK"
24-if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi
25-log(){ printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/backup.log"; }
26-fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; }
--
30-log "START run_id=$RUN_ID edge=${EDGE_USER}@${EDGE_HOST}"
31-ARCHIVE="${OUTDIR}/skladchik-reports-monitor-${RUN_ID}.tar.gz"
32-ssh "${SSH_OPTS[@]}" "${EDGE_USER}@${EDGE_HOST}" sudo bash -s "$RUN_ID" > "$ARCHIVE" <<'REMOTE_SCRIPT'
33-set -Eeuo pipefail
34-RUN_ID="$1"
35:TMP="$(mktemp -d /tmp/skladchik-reports-monitor-backup.XXXXXX)"
36-cleanup(){ rm -rf "$TMP"; }
37-trap cleanup EXIT
38-mkdir -p "$TMP/payload/var-lib-skladchik-reports-monitor" "$TMP/payload/systemd"
39-cp -a /etc/skladchik-reports-monitor "$TMP/payload/etc-skladchik-reports-monitor" 2>/dev/null || true
40-cp -a /opt/skladchik-reports-monitor "$TMP/payload/opt-skladchik-reports-monitor" 2>/dev/null || true
HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup
HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 BYTES=6851
HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check
1-#!/usr/bin/env bash
2-set -Eeuo pipefail
3-umask 077
4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf
5-[ -r "$CONF" ] && . "$CONF"
6:: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}"
7-: "${CLOUD_1:=pve01-mail-01-crypt:}"
8-: "${CLOUD_2:=pve01-mail-02-crypt:}"
9:: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}"
10-: "${KEEP_LOCAL:=10}"
11:LOGDIR=/var/log/skladchik-reports-monitor-backup
12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock
14-RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
15-WORK="${WORK_ROOT}/${RUN_ID}"
16-mkdir -p "$WORK" "$LOGDIR" "$(dirname "$HEALTH")"
17-exec 9>"$LOCK"
18-if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi
--
88-  py_status=skipped
89-  if command -v python3 >/dev/null; then
90-    python3 -m py_compile "$extract/opt-skladchik-reports-monitor/monitor.py" "$extract/opt-skladchik-reports-monitor/health_alert.py" "$extract/opt-skladchik-reports-monitor/operational_digest.py" "$extract/opt-skladchik-reports-monitor/event-selftest-runner.py" "$extract/opt-skladchik-reports-monitor/event-selftest-core"
91-    py_status=ok
92-  fi
93:  printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env"
94-  log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}"
95-}
96-: > "$WORK/result.env"
97-check_one_remote CLOUD_1 "$CLOUD_1"
98-check_one_remote CLOUD_2 "$CLOUD_2"
99-. "$WORK/result.env"
100:printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH"
101-find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done
102-log "OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}"
HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check
HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67 BYTES=753
HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status
1-#!/usr/bin/env bash
2-set +e
3-printf 'RESTORE_CHECK_HEALTH_BEGIN\n'
4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true
5-printf 'RESTORE_CHECK_HEALTH_END\n'
6-printf 'RESTORE_CHECK_TIMER_BEGIN\n'
7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true
8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true
9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true
10-printf 'RESTORE_CHECK_TIMER_END\n'
11-printf 'RESTORE_CHECK_LOG_TAIL_BEGIN\n'
12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true
13-printf 'RESTORE_CHECK_LOG_TAIL_END\n'
HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status
HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0 BYTES=606
HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-status
1-#!/usr/bin/env bash
2-set +e
3-printf 'HEALTH_BEGIN\n'
4:cat /var/lib/skladchik-reports-monitor-backup/health.txt 2>/dev/null || true
5-printf 'HEALTH_END\n'
6-printf 'TIMER_BEGIN\n'
7:systemctl is-enabled skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/enabled=/' || true
8:systemctl is-active skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/active=/' || true
9:systemctl list-timers --all | grep skladchik-reports-monitor-backup || true
10-printf 'TIMER_END\n'
11-printf 'LOG_TAIL_BEGIN\n'
12:tail -n 60 /var/log/skladchik-reports-monitor-backup/backup.log 2>/dev/null || true
13-printf 'LOG_TAIL_END\n'
HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-status
HEALTH_EXECUTABLE_MATCH_COUNT=5
HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3
HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23
HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67
HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0
HEALTH_SERVICE_MATCH_COUNT=1
HEALTH_SERVICE_FILE=/etc/systemd/system/homelab-backup-audit.service SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa
HEALTH_SERVICE_UNIT=homelab-backup-audit.service
Description=Homelab unified backup and DR audit
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-audit scheduled
User=root
Group=root
Result=exit-code,ExecMainStatus=1,ActiveState=failed,SubState=failed,
SKLADCHIK_BACKUP_STATE_FILES_BEGIN=1
MTIME_FLOAT|71|/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
MTIME_FLOAT|484|/var/lib/skladchik-reports-monitor-backup/health.txt
SKLADCHIK_BACKUP_STATE_FILES_END=1
SOURCE_WORKTREE_VALID=true
SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614
SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
SOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
SOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9
SOURCE_PUSHURL_CLASS=DISABLED
SOURCE_REMOTE_HEAD=UNKNOWN
SOURCE_HEAD_ALIGNMENT=UNKNOWN
DECISION=HOLD_BACKUP_1113_PRODUCER_OR_SOURCE_STILL_AMBIGUOUS
NEXT_GATE=RECONCILE_ONLY_BACKUP_1113_AMBIGUITY
TASK_RESULT=HOLD_HOMELAB_BACKUP_1112_AMBIGUITY_RCA
CHANGES_MADE_BY_1113=false
PRODUCT_MUTATION_BY_1113=false
VM_MUTATION_BY_1113=false
CLOUD_MUTATION_BY_1113=false
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
WORKERS2_BACKUP_1112_AMBIGUITY_RCA_END=1

OUTPUT_END
CHAT_OUTPUT_END
