runtime: publish SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1

2026-09-22 05:40:45 +00:00
parent 2afa9b559d
commit 4b61839f35
4 changed files with 276 additions and 198 deletions
@@ -0,0 +1,140 @@
+CHAT_OUTPUT_BEGIN
+COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1
+STATUS=FAIL
+RC=3
+HOST=pve01
+MODE=read-only
+COMPONENT=homelab-backup-1113-final-rca
+REFERENCE_REGISTER_CHECK=OK
+REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
+ERROR_REGISTER_CHECK=OK
+ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
+COMMAND_SHA256=6a9e35fad42bb449e3bba1af241cc9b4e02b094cc00b77de3741866b6cd5e547
+DUPLICATE_FAILED_COMMAND_BLOCKED=false
+EXECUTION_STARTED=true
+CHANGE_DECLARED=false
+RESULT_CONTRACT_VALID=true
+RESULT_CONTRACT_STATUS=NOT_APPLICABLE
+RESULT_CONTRACT_ERROR=NONE
+COMMAND_RC=3
+CHANGES_MADE=false
+ROLLBACK_STARTED=false
+ROLLBACK_RESTORED=null
+MUTATION_OUTCOME=NO_MUTATION
+SANITIZED=yes
+SECRETS_INCLUDED=no
+PRIVATE_ADDRESSES_INCLUDED=no
+RAW_EVIDENCE_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a
+SANITIZED_OUTPUT_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a
+OUTPUT_BEGIN
+WORKERS2_BACKUP_1113_FINAL_RCA_BEGIN=1
+COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1
+MODE=read-only
+COMPONENT=homelab-backup-1113-final-rca
+MUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_RESTORE_CHECK_AND_SOURCE_OFFHOST_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP
+ERROR_REGISTER_CHECK=OK
+REFERENCE_CHECK=OK
+REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
+ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
+RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
+WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
+AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
+RESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23
+BACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3
+SCHEDULER_RC=0
+SELFTEST_REPLACEMENT486=PASS
+SCHEDULER_LIVE_SELECTION={"due_seconds":86400,"enabled":true,"logical_id":"us-netbird"}
+VM_QUEUE_CURRENT_STATE=success,inactive,dead
+BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed
+RESTORE_HEALTH_BEGIN=1
+RESTORE_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6
+RESTORE_HEALTH_MTIME_EPOCH=1789966497
+STATUS=FAIL
+TS=2026-09-21T04:54:57Z
+RUN_ID=20260921T045457Z
+error_rc=1
+RESTORE_HEALTH_END=1
+BACKUP_HEALTH_BEGIN=1
+BACKUP_HEALTH_SHA256=ac71cf99698817816382eb1d888948f76eb5af2782e4755f41b8bc9948853dca
+BACKUP_HEALTH_MTIME_EPOCH=1784856521
+STATUS=OK
+TS=2026-07-24T01:28:41Z
+RUN_ID=20260724T012808Z
+SOURCE=debian@[PRIVATE_IP]
+ARCHIVE=/mnt/staging/skladchik-reports-monitor-backup/daily/20260724T012808Z/skladchik-reports-monitor-20260724T012808Z.tar.gz
+SIZE_BYTES=93249
+REMOTE_BASE=homelab/skladchik-reports-monitor-backup/daily/20260724T012808Z
+CLOUD_1=pve01-mail-01-crypt:
+CLOUD_2=pve01-mail-02-crypt:
+KEEP_DAILY=30
+BACKUP_VERSION=4
+COOKIE_JAR_BACKED_UP=yes
+EXTENSION_HEALTH_FILES_BACKED_UP=yes
+TITLE_IN_EMAIL_SUBJECT=yes
+BACKUP_HEALTH_END=1
+RESTORE_SOURCE_CONTRACT_BEGIN=1
+4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf
+11:LOGDIR=/var/log/skladchik-reports-monitor-backup
+12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
+13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock
+14:RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
+15:WORK="${WORK_ROOT}/${RUN_ID}"
+20:fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; }
+22:command -v rclone >/dev/null
+28:  if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then
+29:    while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort)
+31:    rclone copyto "$remote_path" "$out" >/dev/null
+36:check_one_remote(){
+37:  local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status
+39:  latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)"
+40:  [ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; }
+41:  archive_name="skladchik-reports-monitor-${latest}.tar.gz"
+42:  remote_archive="${rbase}/${latest}/${archive_name}"
+43:  outdir="${WORK}/${label}-${latest}"
+46:  log "${label} DOWNLOAD run=${latest}"
+93:  printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env"
+94:  log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}"
+97:check_one_remote CLOUD_1 "$CLOUD_1"
+98:check_one_remote CLOUD_2 "$CLOUD_2"
+100:printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH"
+RESTORE_SOURCE_CONTRACT_END=1
+RESTORE_CONFIG_EXISTS=true
+RESTORE_CONFIG_SHA256=c7f63d5707196f9b770cb304333a80fa20d0e9e34554ef11224bd6f2e4208aa5
+RESTORE_SERVICE_EXISTS=false
+RESTORE_TIMER_EXISTS=false
+RESTORE_SERVICE_STATE=success,,,0,inactive,dead
+RESTORE_TIMER_STATE=enabled:not-found,active:inactive
+RESTORE_SERVICE_UNIT_BEGIN=1
+RESTORE_SERVICE_UNIT_END=1
+RESTORE_TIMER_UNIT_BEGIN=1
+RESTORE_TIMER_UNIT_END=1
+RESTORE_JOURNAL_CLASSIFIED_BEGIN=1
+RESTORE_JOURNAL_CLASSIFIED_END=1
+RESTORE_JOURNAL_SHA256=19c4e28b2f54ea8f217db4e622b0a5c758efa7b77ce2dc0394b6d159568c0b2e
+RESTORE_LOG_CLASSIFIED_BEGIN=1
+RESTORE_LOG_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+RESTORE_LOG_CLASSIFIED_END=1
+SOURCE_WORKTREE_VALID=true
+SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614
+SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
+SOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
+SOURCE_TRACKING_HEAD=UNKNOWN
+SOURCE_PUSHURL_CLASS=DISABLED
+SOURCE_REMOTE_CLASS=HTTP
+SOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9
+SOURCE_LSREMOTE_RC=0
+SOURCE_LSREMOTE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+SOURCE_REMOTE_HEAD=UNKNOWN
+RESTORE_SERVICE_BINDING=NOT_PROVEN
+DECISION=HOLD_BACKUP_1114_SCOPE_NOT_YET_SAFE
+NEXT_GATE=RECONCILE_ONLY_BACKUP_1114_FAILURE
+TASK_RESULT=HOLD_HOMELAB_BACKUP_1113_FINAL_RCA
+CHANGES_MADE_BY_1114=false
+PRODUCT_MUTATION_BY_1114=false
+VM_MUTATION_BY_1114=false
+CLOUD_MUTATION_BY_1114=false
+HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
+WORKERS2_BACKUP_1113_FINAL_RCA_END=1
+
+OUTPUT_END
+CHAT_OUTPUT_END