CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1
STATUS=FAIL
RC=3
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-1113-final-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=6a9e35fad42bb449e3bba1af241cc9b4e02b094cc00b77de3741866b6cd5e547
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=3
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a
SANITIZED_OUTPUT_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a
OUTPUT_BEGIN
WORKERS2_BACKUP_1113_FINAL_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1
MODE=read-only
COMPONENT=homelab-backup-1113-final-rca
MUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_RESTORE_CHECK_AND_SOURCE_OFFHOST_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP
ERROR_REGISTER_CHECK=OK
REFERENCE_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
RESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23
BACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3
SCHEDULER_RC=0
SELFTEST_REPLACEMENT486=PASS
SCHEDULER_LIVE_SELECTION={"due_seconds":86400,"enabled":true,"logical_id":"us-netbird"}
VM_QUEUE_CURRENT_STATE=success,inactive,dead
BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed
RESTORE_HEALTH_BEGIN=1
RESTORE_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6
RESTORE_HEALTH_MTIME_EPOCH=1789966497
STATUS=FAIL
TS=2026-09-21T04:54:57Z
RUN_ID=20260921T045457Z
error_rc=1
RESTORE_HEALTH_END=1
BACKUP_HEALTH_BEGIN=1
BACKUP_HEALTH_SHA256=ac71cf99698817816382eb1d888948f76eb5af2782e4755f41b8bc9948853dca
BACKUP_HEALTH_MTIME_EPOCH=1784856521
STATUS=OK
TS=2026-07-24T01:28:41Z
RUN_ID=20260724T012808Z
SOURCE=debian@[PRIVATE_IP]
ARCHIVE=/mnt/staging/skladchik-reports-monitor-backup/daily/20260724T012808Z/skladchik-reports-monitor-20260724T012808Z.tar.gz
SIZE_BYTES=93249
REMOTE_BASE=homelab/skladchik-reports-monitor-backup/daily/20260724T012808Z
CLOUD_1=pve01-mail-01-crypt:
CLOUD_2=pve01-mail-02-crypt:
KEEP_DAILY=30
BACKUP_VERSION=4
COOKIE_JAR_BACKED_UP=yes
EXTENSION_HEALTH_FILES_BACKED_UP=yes
TITLE_IN_EMAIL_SUBJECT=yes
BACKUP_HEALTH_END=1
RESTORE_SOURCE_CONTRACT_BEGIN=1
4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf
11:LOGDIR=/var/log/skladchik-reports-monitor-backup
12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock
14:RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
15:WORK="${WORK_ROOT}/${RUN_ID}"
20:fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; }
22:command -v rclone >/dev/null
28:  if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then
29:    while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort)
31:    rclone copyto "$remote_path" "$out" >/dev/null
36:check_one_remote(){
37:  local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status
39:  latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)"
40:  [ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; }
41:  archive_name="skladchik-reports-monitor-${latest}.tar.gz"
42:  remote_archive="${rbase}/${latest}/${archive_name}"
43:  outdir="${WORK}/${label}-${latest}"
46:  log "${label} DOWNLOAD run=${latest}"
93:  printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env"
94:  log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}"
97:check_one_remote CLOUD_1 "$CLOUD_1"
98:check_one_remote CLOUD_2 "$CLOUD_2"
100:printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH"
RESTORE_SOURCE_CONTRACT_END=1
RESTORE_CONFIG_EXISTS=true
RESTORE_CONFIG_SHA256=c7f63d5707196f9b770cb304333a80fa20d0e9e34554ef11224bd6f2e4208aa5
RESTORE_SERVICE_EXISTS=false
RESTORE_TIMER_EXISTS=false
RESTORE_SERVICE_STATE=success,,,0,inactive,dead
RESTORE_TIMER_STATE=enabled:not-found,active:inactive
RESTORE_SERVICE_UNIT_BEGIN=1
RESTORE_SERVICE_UNIT_END=1
RESTORE_TIMER_UNIT_BEGIN=1
RESTORE_TIMER_UNIT_END=1
RESTORE_JOURNAL_CLASSIFIED_BEGIN=1
RESTORE_JOURNAL_CLASSIFIED_END=1
RESTORE_JOURNAL_SHA256=19c4e28b2f54ea8f217db4e622b0a5c758efa7b77ce2dc0394b6d159568c0b2e
RESTORE_LOG_CLASSIFIED_BEGIN=1
RESTORE_LOG_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
RESTORE_LOG_CLASSIFIED_END=1
SOURCE_WORKTREE_VALID=true
SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614
SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
SOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
SOURCE_TRACKING_HEAD=UNKNOWN
SOURCE_PUSHURL_CLASS=DISABLED
SOURCE_REMOTE_CLASS=HTTP
SOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9
SOURCE_LSREMOTE_RC=0
SOURCE_LSREMOTE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
SOURCE_REMOTE_HEAD=UNKNOWN
RESTORE_SERVICE_BINDING=NOT_PROVEN
DECISION=HOLD_BACKUP_1114_SCOPE_NOT_YET_SAFE
NEXT_GATE=RECONCILE_ONLY_BACKUP_1114_FAILURE
TASK_RESULT=HOLD_HOMELAB_BACKUP_1113_FINAL_RCA
CHANGES_MADE_BY_1114=false
PRODUCT_MUTATION_BY_1114=false
VM_MUTATION_BY_1114=false
CLOUD_MUTATION_BY_1114=false
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
WORKERS2_BACKUP_1113_FINAL_RCA_END=1

OUTPUT_END
CHAT_OUTPUT_END
