From 4b61839f358dd77858ee0a3f6438a44b7b187824 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 22 Sep 2026 05:40:45 +0000 Subject: [PATCH] runtime: publish SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1 --- ...-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1.json | 38 +++ ...2-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1.txt | 140 +++++++++ runtime/latest.json | 16 +- runtime/latest.txt | 280 ++++++------------ 4 files changed, 276 insertions(+), 198 deletions(-) create mode 100644 runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1.json create mode 100644 runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1.txt diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1.json b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1.json new file mode 100644 index 00000000..f2e6d92b --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1", + "status": "FAIL", + "rc": 3, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-1113-final-rca", + "started_at_utc": "2026-09-22T05:40:44Z", + "finished_at_utc": "2026-09-22T05:40:44Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "6a9e35fad42bb449e3bba1af241cc9b4e02b094cc00b77de3741866b6cd5e547", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 3, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a", + "sanitized_output_sha256": "8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_1113_FINAL_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1\nMODE=read-only\nCOMPONENT=homelab-backup-1113-final-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_RESTORE_CHECK_AND_SOURCE_OFFHOST_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nRESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23\nBACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION={\"due_seconds\":86400,\"enabled\":true,\"logical_id\":\"us-netbird\"}\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nRESTORE_HEALTH_BEGIN=1\nRESTORE_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6\nRESTORE_HEALTH_MTIME_EPOCH=1789966497\nSTATUS=FAIL\nTS=2026-09-21T04:54:57Z\nRUN_ID=20260921T045457Z\nerror_rc=1\nRESTORE_HEALTH_END=1\nBACKUP_HEALTH_BEGIN=1\nBACKUP_HEALTH_SHA256=ac71cf99698817816382eb1d888948f76eb5af2782e4755f41b8bc9948853dca\nBACKUP_HEALTH_MTIME_EPOCH=1784856521\nSTATUS=OK\nTS=2026-07-24T01:28:41Z\nRUN_ID=20260724T012808Z\nSOURCE=debian@[PRIVATE_IP]\nARCHIVE=/mnt/staging/skladchik-reports-monitor-backup/daily/20260724T012808Z/skladchik-reports-monitor-20260724T012808Z.tar.gz\nSIZE_BYTES=93249\nREMOTE_BASE=homelab/skladchik-reports-monitor-backup/daily/20260724T012808Z\nCLOUD_1=pve01-mail-01-crypt:\nCLOUD_2=pve01-mail-02-crypt:\nKEEP_DAILY=30\nBACKUP_VERSION=4\nCOOKIE_JAR_BACKED_UP=yes\nEXTENSION_HEALTH_FILES_BACKED_UP=yes\nTITLE_IN_EMAIL_SUBJECT=yes\nBACKUP_HEALTH_END=1\nRESTORE_SOURCE_CONTRACT_BEGIN=1\n4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n11:LOGDIR=/var/log/skladchik-reports-monitor-backup\n12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n14:RUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n15:WORK=\"${WORK_ROOT}/${RUN_ID}\"\n20:fail(){ rc=$?; printf 'STATUS=FAIL\\nTS=%s\\nRUN_ID=%s\\nerror_rc=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$rc\" > \"$HEALTH\"; log \"FAIL rc=$rc\"; exit \"$rc\"; }\n22:command -v rclone >/dev/null\n28: if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n29: while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n31: rclone copyto \"$remote_path\" \"$out\" >/dev/null\n36:check_one_remote(){\n37: local label=\"$1\" remote=\"$2\" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status\n39: latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n40: [ -n \"$latest\" ] || { echo \"${label}_STATUS=NO_RUN\" >> \"$WORK/result.env\"; return 20; }\n41: archive_name=\"skladchik-reports-monitor-${latest}.tar.gz\"\n42: remote_archive=\"${rbase}/${latest}/${archive_name}\"\n43: outdir=\"${WORK}/${label}-${latest}\"\n46: log \"${label} DOWNLOAD run=${latest}\"\n93: printf '%s_STATUS=OK\\n%s_RUN=%s\\n%s_ARCHIVE_BYTES=%s\\n%s_SHA256_OK=yes\\n%s_TAR_OK=yes\\n%s_REQUIRED_FILES_OK=yes\\n%s_COOKIE_JAR_OK=yes\\n%s_EXTENSION_HEALTH_FILES_OK=yes\\n%s_EXTENSION_SCRIPTS_OK=yes\\n%s_EXTENSION_UNITS_OK=yes\\n%s_SQLITE=%s\\n%s_PY_COMPILE=%s\\n' \"$label\" \"$label\" \"$latest\" \"$label\" \"$(stat -c %s \"$archive\")\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$sqlite_status\" \"$label\" \"$py_status\" >> \"$WORK/result.env\"\n94: log \"${label} OK run=${latest} bytes=$(stat -c %s \"$archive\") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}\"\n97:check_one_remote CLOUD_1 \"$CLOUD_1\"\n98:check_one_remote CLOUD_2 \"$CLOUD_2\"\n100:printf 'STATUS=OK\\nTS=%s\\nRUN_ID=%s\\nREMOTE_BASE=%s\\nCLOUD_1_STATUS=%s\\nCLOUD_1_RUN=%s\\nCLOUD_2_STATUS=%s\\nCLOUD_2_RUN=%s\\nSHA256_OK=yes\\nTAR_OK=yes\\nREQUIRED_FILES_OK=yes\\nCOOKIE_JAR_OK=yes\\nEXTENSION_HEALTH_FILES_OK=yes\\nEXTENSION_SCRIPTS_OK=yes\\nEXTENSION_UNITS_OK=yes\\nSQLITE_OK=yes\\nPY_COMPILE_OK=yes\\nRESTORE_CHECK_VERSION=4\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$REMOTE_BASE\" \"${CLOUD_1_STATUS:-unknown}\" \"${CLOUD_1_RUN:-}\" \"${CLOUD_2_STATUS:-unknown}\" \"${CLOUD_2_RUN:-}\" > \"$HEALTH\"\nRESTORE_SOURCE_CONTRACT_END=1\nRESTORE_CONFIG_EXISTS=true\nRESTORE_CONFIG_SHA256=c7f63d5707196f9b770cb304333a80fa20d0e9e34554ef11224bd6f2e4208aa5\nRESTORE_SERVICE_EXISTS=false\nRESTORE_TIMER_EXISTS=false\nRESTORE_SERVICE_STATE=success,,,0,inactive,dead\nRESTORE_TIMER_STATE=enabled:not-found,active:inactive\nRESTORE_SERVICE_UNIT_BEGIN=1\nRESTORE_SERVICE_UNIT_END=1\nRESTORE_TIMER_UNIT_BEGIN=1\nRESTORE_TIMER_UNIT_END=1\nRESTORE_JOURNAL_CLASSIFIED_BEGIN=1\nRESTORE_JOURNAL_CLASSIFIED_END=1\nRESTORE_JOURNAL_SHA256=19c4e28b2f54ea8f217db4e622b0a5c758efa7b77ce2dc0394b6d159568c0b2e\nRESTORE_LOG_CLASSIFIED_BEGIN=1\nRESTORE_LOG_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nRESTORE_LOG_CLASSIFIED_END=1\nSOURCE_WORKTREE_VALID=true\nSOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614\nSOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_TRACKING_HEAD=UNKNOWN\nSOURCE_PUSHURL_CLASS=DISABLED\nSOURCE_REMOTE_CLASS=HTTP\nSOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9\nSOURCE_LSREMOTE_RC=0\nSOURCE_LSREMOTE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nSOURCE_REMOTE_HEAD=UNKNOWN\nRESTORE_SERVICE_BINDING=NOT_PROVEN\nDECISION=HOLD_BACKUP_1114_SCOPE_NOT_YET_SAFE\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1114_FAILURE\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1113_FINAL_RCA\nCHANGES_MADE_BY_1114=false\nPRODUCT_MUTATION_BY_1114=false\nVM_MUTATION_BY_1114=false\nCLOUD_MUTATION_BY_1114=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1113_FINAL_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1.txt b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1.txt new file mode 100644 index 00000000..5f4fd32e --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1.txt @@ -0,0 +1,140 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1 +STATUS=FAIL +RC=3 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-1113-final-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=6a9e35fad42bb449e3bba1af241cc9b4e02b094cc00b77de3741866b6cd5e547 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=3 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a +SANITIZED_OUTPUT_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a +OUTPUT_BEGIN +WORKERS2_BACKUP_1113_FINAL_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1 +MODE=read-only +COMPONENT=homelab-backup-1113-final-rca +MUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_RESTORE_CHECK_AND_SOURCE_OFFHOST_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP +ERROR_REGISTER_CHECK=OK +REFERENCE_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 +AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 +RESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 +BACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 +SCHEDULER_RC=0 +SELFTEST_REPLACEMENT486=PASS +SCHEDULER_LIVE_SELECTION={"due_seconds":86400,"enabled":true,"logical_id":"us-netbird"} +VM_QUEUE_CURRENT_STATE=success,inactive,dead +BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed +RESTORE_HEALTH_BEGIN=1 +RESTORE_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6 +RESTORE_HEALTH_MTIME_EPOCH=1789966497 +STATUS=FAIL +TS=2026-09-21T04:54:57Z +RUN_ID=20260921T045457Z +error_rc=1 +RESTORE_HEALTH_END=1 +BACKUP_HEALTH_BEGIN=1 +BACKUP_HEALTH_SHA256=ac71cf99698817816382eb1d888948f76eb5af2782e4755f41b8bc9948853dca +BACKUP_HEALTH_MTIME_EPOCH=1784856521 +STATUS=OK +TS=2026-07-24T01:28:41Z +RUN_ID=20260724T012808Z +SOURCE=debian@[PRIVATE_IP] +ARCHIVE=/mnt/staging/skladchik-reports-monitor-backup/daily/20260724T012808Z/skladchik-reports-monitor-20260724T012808Z.tar.gz +SIZE_BYTES=93249 +REMOTE_BASE=homelab/skladchik-reports-monitor-backup/daily/20260724T012808Z +CLOUD_1=pve01-mail-01-crypt: +CLOUD_2=pve01-mail-02-crypt: +KEEP_DAILY=30 +BACKUP_VERSION=4 +COOKIE_JAR_BACKED_UP=yes +EXTENSION_HEALTH_FILES_BACKED_UP=yes +TITLE_IN_EMAIL_SUBJECT=yes +BACKUP_HEALTH_END=1 +RESTORE_SOURCE_CONTRACT_BEGIN=1 +4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf +11:LOGDIR=/var/log/skladchik-reports-monitor-backup +12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock +14:RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +15:WORK="${WORK_ROOT}/${RUN_ID}" +20:fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; } +22:command -v rclone >/dev/null +28: if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then +29: while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort) +31: rclone copyto "$remote_path" "$out" >/dev/null +36:check_one_remote(){ +37: local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status +39: latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)" +40: [ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; } +41: archive_name="skladchik-reports-monitor-${latest}.tar.gz" +42: remote_archive="${rbase}/${latest}/${archive_name}" +43: outdir="${WORK}/${label}-${latest}" +46: log "${label} DOWNLOAD run=${latest}" +93: printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env" +94: log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}" +97:check_one_remote CLOUD_1 "$CLOUD_1" +98:check_one_remote CLOUD_2 "$CLOUD_2" +100:printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH" +RESTORE_SOURCE_CONTRACT_END=1 +RESTORE_CONFIG_EXISTS=true +RESTORE_CONFIG_SHA256=c7f63d5707196f9b770cb304333a80fa20d0e9e34554ef11224bd6f2e4208aa5 +RESTORE_SERVICE_EXISTS=false +RESTORE_TIMER_EXISTS=false +RESTORE_SERVICE_STATE=success,,,0,inactive,dead +RESTORE_TIMER_STATE=enabled:not-found,active:inactive +RESTORE_SERVICE_UNIT_BEGIN=1 +RESTORE_SERVICE_UNIT_END=1 +RESTORE_TIMER_UNIT_BEGIN=1 +RESTORE_TIMER_UNIT_END=1 +RESTORE_JOURNAL_CLASSIFIED_BEGIN=1 +RESTORE_JOURNAL_CLASSIFIED_END=1 +RESTORE_JOURNAL_SHA256=19c4e28b2f54ea8f217db4e622b0a5c758efa7b77ce2dc0394b6d159568c0b2e +RESTORE_LOG_CLASSIFIED_BEGIN=1 +RESTORE_LOG_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +RESTORE_LOG_CLASSIFIED_END=1 +SOURCE_WORKTREE_VALID=true +SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614 +SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_TRACKING_HEAD=UNKNOWN +SOURCE_PUSHURL_CLASS=DISABLED +SOURCE_REMOTE_CLASS=HTTP +SOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9 +SOURCE_LSREMOTE_RC=0 +SOURCE_LSREMOTE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +SOURCE_REMOTE_HEAD=UNKNOWN +RESTORE_SERVICE_BINDING=NOT_PROVEN +DECISION=HOLD_BACKUP_1114_SCOPE_NOT_YET_SAFE +NEXT_GATE=RECONCILE_ONLY_BACKUP_1114_FAILURE +TASK_RESULT=HOLD_HOMELAB_BACKUP_1113_FINAL_RCA +CHANGES_MADE_BY_1114=false +PRODUCT_MUTATION_BY_1114=false +VM_MUTATION_BY_1114=false +CLOUD_MUTATION_BY_1114=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1113_FINAL_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 4144ca12..f2e6d92b 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1", "status": "FAIL", "rc": 3, "host": "pve01", "mode": "read-only", - "component": "homelab-backup-1112-ambiguity-rca", - "started_at_utc": "2026-09-22T05:32:23Z", - "finished_at_utc": "2026-09-22T05:32:23Z", + "component": "homelab-backup-1113-final-rca", + "started_at_utc": "2026-09-22T05:40:44Z", + "finished_at_utc": "2026-09-22T05:40:44Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "d1e970ab78309057995c785598ad19f335660b3bdad9d91f82f8a58aae349f74", + "command_sha256": "6a9e35fad42bb449e3bba1af241cc9b4e02b094cc00b77de3741866b6cd5e547", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "87b985744c8a04978f232aba11354a2e66388e428b0eeec0a763adafc74e8fa5", - "sanitized_output_sha256": "2beac0e910df9a79ef0fdc3a96ecd450e8b0a17ff922f22c44187d8b2e629d3e", + "raw_evidence_sha256": "8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a", + "sanitized_output_sha256": "8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_BACKUP_1112_AMBIGUITY_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1\nMODE=read-only\nCOMPONENT=homelab-backup-1112-ambiguity-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_HEALTH_PRODUCER_AND_SOURCE_RECONCILE;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nVMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nEXPECTED_CLOUDB_HEALTH_PATH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\nCLOUDB_HEALTH_EXISTS=true\nCLOUDB_HEALTH_BYTES=71\nCLOUDB_HEALTH_MTIME_EPOCH=1789966497\nCLOUDB_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6\nCLOUDB_HEALTH_MARKERS_BEGIN=1\nSTATUS=FAIL\nCLOUDB_HEALTH_MARKERS_END=1\nHEALTH_PATH_SOURCE_MATCH_COUNT=8\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 BYTES=15391\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/homelab-backup-audit\n19-GFSP=\"/etc/homelab-backup/gfs-retention.json\"\n20-RET=\"/usr/local/sbin/homelab-backup-retention-gc\"\n21-WT=\"/root/homelab-ops-cr-2026-1005\"\n22-BRANCH=\"cr-2026-1005-backup-v2-health-alerting-remediation\"\n23-DRSEAL=pathlib.Path(\"/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json\")\n24:CLOUDB_HEALTH=pathlib.Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\n25-HEALTHS=[pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\"),pathlib.Path(\"/var/lib/homelab-health/backup-audit.json\")]\n26-\n27-def run(cmd,timeout=60,env=None):\n28- try:\n29- p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)\n--\n231- txt=CLOUDB_HEALTH.read_text(encoding=\"utf-8\",errors=\"replace\")\n232- kv={}\n233- for line in txt.splitlines():\n234- if \"=\" in line:\n235- k,v=line.split(\"=\",1); kv[k]=v\n236: cloudb_ok=(kv.get(\"STATUS\")==\"OK\" and kv.get(\"CLOUD_2_STATUS\")==\"OK\" and kv.get(\"SHA256_OK\")==\"yes\" and kv.get(\"TAR_OK\")==\"yes\" and kv.get(\"REQUIRED_FILES_OK\")==\"yes\")\n237-if not cloudb_ok:\n238- issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"LATEST_CLOUD_B_DRILL_NOT_OK\"})\n239-\n240-source={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\n241-hrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/homelab-backup-audit\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc SHA256=f9c9a17c09c542a653df544b74321c797e89cd2dbd726b52287bfd6097b0998a BYTES=29834\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc SHA256=1cff2c1761a8487dd0c7d6e2a0d6b4a6a45710b23e7afdaad1059526b96f396c BYTES=24854\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc SHA256=8acaa25f3d259e2d00d0bccf86a64c616d2fe447bbab492f9e12c4b996334582 BYTES=23194\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 BYTES=5610\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup\n1-#!/usr/bin/env bash\n2-set -Eeuo pipefail\n3-umask 077\n4:CONF=/etc/skladchik-reports-monitor-backup.conf\n5-[ -r \"$CONF\" ] && . \"$CONF\"\n6-: \"${EDGE_HOST:=[PRIVATE_IP]}\"\n7-: \"${EDGE_USER:=debian}\"\n8-: \"${SERVICE_NAME:=skladchik-reports-monitor}\"\n9:: \"${LOCAL_ROOT:=/mnt/staging/skladchik-reports-monitor-backup}\"\n10:: \"${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}\"\n11-: \"${CLOUD_1:=pve01-mail-01-crypt:}\"\n12-: \"${CLOUD_2:=pve01-mail-02-crypt:}\"\n13-: \"${KEEP_DAILY:=30}\"\n14-: \"${CONNECT_TIMEOUT:=20}\"\n15-SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=\"${CONNECT_TIMEOUT}\" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\n16-RUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n17-DAY=\"$(date -u +%Y%m%d)\"\n18-OUTDIR=\"${LOCAL_ROOT}/daily/${RUN_ID}\"\n19:LOGDIR=/var/log/skladchik-reports-monitor-backup\n20:HEALTH=/var/lib/skladchik-reports-monitor-backup/health.txt\n21:LOCK=/run/skladchik-reports-monitor-backup.lock\n22-mkdir -p \"$OUTDIR\" \"$LOGDIR\" \"$(dirname \"$HEALTH\")\"\n23-exec 9>\"$LOCK\"\n24-if ! flock -n 9; then echo \"STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$HEALTH\"; exit 0; fi\n25-log(){ printf '%s %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/backup.log\"; }\n26-fail(){ rc=$?; printf 'STATUS=FAIL\\nTS=%s\\nRUN_ID=%s\\nerror_rc=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$rc\" > \"$HEALTH\"; log \"FAIL rc=$rc\"; exit \"$rc\"; }\n--\n30-log \"START run_id=$RUN_ID edge=${EDGE_USER}@${EDGE_HOST}\"\n31-ARCHIVE=\"${OUTDIR}/skladchik-reports-monitor-${RUN_ID}.tar.gz\"\n32-ssh \"${SSH_OPTS[@]}\" \"${EDGE_USER}@${EDGE_HOST}\" sudo bash -s \"$RUN_ID\" > \"$ARCHIVE\" <<'REMOTE_SCRIPT'\n33-set -Eeuo pipefail\n34-RUN_ID=\"$1\"\n35:TMP=\"$(mktemp -d /tmp/skladchik-reports-monitor-backup.XXXXXX)\"\n36-cleanup(){ rm -rf \"$TMP\"; }\n37-trap cleanup EXIT\n38-mkdir -p \"$TMP/payload/var-lib-skladchik-reports-monitor\" \"$TMP/payload/systemd\"\n39-cp -a /etc/skladchik-reports-monitor \"$TMP/payload/etc-skladchik-reports-monitor\" 2>/dev/null || true\n40-cp -a /opt/skladchik-reports-monitor \"$TMP/payload/opt-skladchik-reports-monitor\" 2>/dev/null || true\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 BYTES=6851\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\n1-#!/usr/bin/env bash\n2-set -Eeuo pipefail\n3-umask 077\n4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n5-[ -r \"$CONF\" ] && . \"$CONF\"\n6:: \"${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}\"\n7-: \"${CLOUD_1:=pve01-mail-01-crypt:}\"\n8-: \"${CLOUD_2:=pve01-mail-02-crypt:}\"\n9:: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n10-: \"${KEEP_LOCAL:=10}\"\n11:LOGDIR=/var/log/skladchik-reports-monitor-backup\n12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n14-RUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n15-WORK=\"${WORK_ROOT}/${RUN_ID}\"\n16-mkdir -p \"$WORK\" \"$LOGDIR\" \"$(dirname \"$HEALTH\")\"\n17-exec 9>\"$LOCK\"\n18-if ! flock -n 9; then echo \"STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$HEALTH\"; exit 0; fi\n--\n88- py_status=skipped\n89- if command -v python3 >/dev/null; then\n90- python3 -m py_compile \"$extract/opt-skladchik-reports-monitor/monitor.py\" \"$extract/opt-skladchik-reports-monitor/health_alert.py\" \"$extract/opt-skladchik-reports-monitor/operational_digest.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-runner.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-core\"\n91- py_status=ok\n92- fi\n93: printf '%s_STATUS=OK\\n%s_RUN=%s\\n%s_ARCHIVE_BYTES=%s\\n%s_SHA256_OK=yes\\n%s_TAR_OK=yes\\n%s_REQUIRED_FILES_OK=yes\\n%s_COOKIE_JAR_OK=yes\\n%s_EXTENSION_HEALTH_FILES_OK=yes\\n%s_EXTENSION_SCRIPTS_OK=yes\\n%s_EXTENSION_UNITS_OK=yes\\n%s_SQLITE=%s\\n%s_PY_COMPILE=%s\\n' \"$label\" \"$label\" \"$latest\" \"$label\" \"$(stat -c %s \"$archive\")\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$sqlite_status\" \"$label\" \"$py_status\" >> \"$WORK/result.env\"\n94- log \"${label} OK run=${latest} bytes=$(stat -c %s \"$archive\") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}\"\n95-}\n96-: > \"$WORK/result.env\"\n97-check_one_remote CLOUD_1 \"$CLOUD_1\"\n98-check_one_remote CLOUD_2 \"$CLOUD_2\"\n99-. \"$WORK/result.env\"\n100:printf 'STATUS=OK\\nTS=%s\\nRUN_ID=%s\\nREMOTE_BASE=%s\\nCLOUD_1_STATUS=%s\\nCLOUD_1_RUN=%s\\nCLOUD_2_STATUS=%s\\nCLOUD_2_RUN=%s\\nSHA256_OK=yes\\nTAR_OK=yes\\nREQUIRED_FILES_OK=yes\\nCOOKIE_JAR_OK=yes\\nEXTENSION_HEALTH_FILES_OK=yes\\nEXTENSION_SCRIPTS_OK=yes\\nEXTENSION_UNITS_OK=yes\\nSQLITE_OK=yes\\nPY_COMPILE_OK=yes\\nRESTORE_CHECK_VERSION=4\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$REMOTE_BASE\" \"${CLOUD_1_STATUS:-unknown}\" \"${CLOUD_1_RUN:-}\" \"${CLOUD_2_STATUS:-unknown}\" \"${CLOUD_2_RUN:-}\" > \"$HEALTH\"\n101-find \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\n102-log \"OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}\"\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67 BYTES=753\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status\n1-#!/usr/bin/env bash\n2-set +e\n3-printf 'RESTORE_CHECK_HEALTH_BEGIN\\n'\n4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true\n5-printf 'RESTORE_CHECK_HEALTH_END\\n'\n6-printf 'RESTORE_CHECK_TIMER_BEGIN\\n'\n7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true\n8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true\n9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true\n10-printf 'RESTORE_CHECK_TIMER_END\\n'\n11-printf 'RESTORE_CHECK_LOG_TAIL_BEGIN\\n'\n12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true\n13-printf 'RESTORE_CHECK_LOG_TAIL_END\\n'\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status\nHEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0 BYTES=606\nHEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-status\n1-#!/usr/bin/env bash\n2-set +e\n3-printf 'HEALTH_BEGIN\\n'\n4:cat /var/lib/skladchik-reports-monitor-backup/health.txt 2>/dev/null || true\n5-printf 'HEALTH_END\\n'\n6-printf 'TIMER_BEGIN\\n'\n7:systemctl is-enabled skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/enabled=/' || true\n8:systemctl is-active skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/active=/' || true\n9:systemctl list-timers --all | grep skladchik-reports-monitor-backup || true\n10-printf 'TIMER_END\\n'\n11-printf 'LOG_TAIL_BEGIN\\n'\n12:tail -n 60 /var/log/skladchik-reports-monitor-backup/backup.log 2>/dev/null || true\n13-printf 'LOG_TAIL_END\\n'\nHEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-status\nHEALTH_EXECUTABLE_MATCH_COUNT=5\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67\nHEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0\nHEALTH_SERVICE_MATCH_COUNT=1\nHEALTH_SERVICE_FILE=/etc/systemd/system/homelab-backup-audit.service SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa\nHEALTH_SERVICE_UNIT=homelab-backup-audit.service\nDescription=Homelab unified backup and DR audit\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-audit scheduled\nUser=root\nGroup=root\nResult=exit-code,ExecMainStatus=1,ActiveState=failed,SubState=failed,\nSKLADCHIK_BACKUP_STATE_FILES_BEGIN=1\nMTIME_FLOAT|71|/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\nMTIME_FLOAT|484|/var/lib/skladchik-reports-monitor-backup/health.txt\nSKLADCHIK_BACKUP_STATE_FILES_END=1\nSOURCE_WORKTREE_VALID=true\nSOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614\nSOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9\nSOURCE_PUSHURL_CLASS=DISABLED\nSOURCE_REMOTE_HEAD=UNKNOWN\nSOURCE_HEAD_ALIGNMENT=UNKNOWN\nDECISION=HOLD_BACKUP_1113_PRODUCER_OR_SOURCE_STILL_AMBIGUOUS\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1113_AMBIGUITY\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1112_AMBIGUITY_RCA\nCHANGES_MADE_BY_1113=false\nPRODUCT_MUTATION_BY_1113=false\nVM_MUTATION_BY_1113=false\nCLOUD_MUTATION_BY_1113=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1112_AMBIGUITY_RCA_END=1\n" + "output": "WORKERS2_BACKUP_1113_FINAL_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1\nMODE=read-only\nCOMPONENT=homelab-backup-1113-final-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_RESTORE_CHECK_AND_SOURCE_OFFHOST_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nRESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23\nBACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION={\"due_seconds\":86400,\"enabled\":true,\"logical_id\":\"us-netbird\"}\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nRESTORE_HEALTH_BEGIN=1\nRESTORE_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6\nRESTORE_HEALTH_MTIME_EPOCH=1789966497\nSTATUS=FAIL\nTS=2026-09-21T04:54:57Z\nRUN_ID=20260921T045457Z\nerror_rc=1\nRESTORE_HEALTH_END=1\nBACKUP_HEALTH_BEGIN=1\nBACKUP_HEALTH_SHA256=ac71cf99698817816382eb1d888948f76eb5af2782e4755f41b8bc9948853dca\nBACKUP_HEALTH_MTIME_EPOCH=1784856521\nSTATUS=OK\nTS=2026-07-24T01:28:41Z\nRUN_ID=20260724T012808Z\nSOURCE=debian@[PRIVATE_IP]\nARCHIVE=/mnt/staging/skladchik-reports-monitor-backup/daily/20260724T012808Z/skladchik-reports-monitor-20260724T012808Z.tar.gz\nSIZE_BYTES=93249\nREMOTE_BASE=homelab/skladchik-reports-monitor-backup/daily/20260724T012808Z\nCLOUD_1=pve01-mail-01-crypt:\nCLOUD_2=pve01-mail-02-crypt:\nKEEP_DAILY=30\nBACKUP_VERSION=4\nCOOKIE_JAR_BACKED_UP=yes\nEXTENSION_HEALTH_FILES_BACKED_UP=yes\nTITLE_IN_EMAIL_SUBJECT=yes\nBACKUP_HEALTH_END=1\nRESTORE_SOURCE_CONTRACT_BEGIN=1\n4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n11:LOGDIR=/var/log/skladchik-reports-monitor-backup\n12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n14:RUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n15:WORK=\"${WORK_ROOT}/${RUN_ID}\"\n20:fail(){ rc=$?; printf 'STATUS=FAIL\\nTS=%s\\nRUN_ID=%s\\nerror_rc=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$rc\" > \"$HEALTH\"; log \"FAIL rc=$rc\"; exit \"$rc\"; }\n22:command -v rclone >/dev/null\n28: if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n29: while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n31: rclone copyto \"$remote_path\" \"$out\" >/dev/null\n36:check_one_remote(){\n37: local label=\"$1\" remote=\"$2\" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status\n39: latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n40: [ -n \"$latest\" ] || { echo \"${label}_STATUS=NO_RUN\" >> \"$WORK/result.env\"; return 20; }\n41: archive_name=\"skladchik-reports-monitor-${latest}.tar.gz\"\n42: remote_archive=\"${rbase}/${latest}/${archive_name}\"\n43: outdir=\"${WORK}/${label}-${latest}\"\n46: log \"${label} DOWNLOAD run=${latest}\"\n93: printf '%s_STATUS=OK\\n%s_RUN=%s\\n%s_ARCHIVE_BYTES=%s\\n%s_SHA256_OK=yes\\n%s_TAR_OK=yes\\n%s_REQUIRED_FILES_OK=yes\\n%s_COOKIE_JAR_OK=yes\\n%s_EXTENSION_HEALTH_FILES_OK=yes\\n%s_EXTENSION_SCRIPTS_OK=yes\\n%s_EXTENSION_UNITS_OK=yes\\n%s_SQLITE=%s\\n%s_PY_COMPILE=%s\\n' \"$label\" \"$label\" \"$latest\" \"$label\" \"$(stat -c %s \"$archive\")\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$sqlite_status\" \"$label\" \"$py_status\" >> \"$WORK/result.env\"\n94: log \"${label} OK run=${latest} bytes=$(stat -c %s \"$archive\") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}\"\n97:check_one_remote CLOUD_1 \"$CLOUD_1\"\n98:check_one_remote CLOUD_2 \"$CLOUD_2\"\n100:printf 'STATUS=OK\\nTS=%s\\nRUN_ID=%s\\nREMOTE_BASE=%s\\nCLOUD_1_STATUS=%s\\nCLOUD_1_RUN=%s\\nCLOUD_2_STATUS=%s\\nCLOUD_2_RUN=%s\\nSHA256_OK=yes\\nTAR_OK=yes\\nREQUIRED_FILES_OK=yes\\nCOOKIE_JAR_OK=yes\\nEXTENSION_HEALTH_FILES_OK=yes\\nEXTENSION_SCRIPTS_OK=yes\\nEXTENSION_UNITS_OK=yes\\nSQLITE_OK=yes\\nPY_COMPILE_OK=yes\\nRESTORE_CHECK_VERSION=4\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$REMOTE_BASE\" \"${CLOUD_1_STATUS:-unknown}\" \"${CLOUD_1_RUN:-}\" \"${CLOUD_2_STATUS:-unknown}\" \"${CLOUD_2_RUN:-}\" > \"$HEALTH\"\nRESTORE_SOURCE_CONTRACT_END=1\nRESTORE_CONFIG_EXISTS=true\nRESTORE_CONFIG_SHA256=c7f63d5707196f9b770cb304333a80fa20d0e9e34554ef11224bd6f2e4208aa5\nRESTORE_SERVICE_EXISTS=false\nRESTORE_TIMER_EXISTS=false\nRESTORE_SERVICE_STATE=success,,,0,inactive,dead\nRESTORE_TIMER_STATE=enabled:not-found,active:inactive\nRESTORE_SERVICE_UNIT_BEGIN=1\nRESTORE_SERVICE_UNIT_END=1\nRESTORE_TIMER_UNIT_BEGIN=1\nRESTORE_TIMER_UNIT_END=1\nRESTORE_JOURNAL_CLASSIFIED_BEGIN=1\nRESTORE_JOURNAL_CLASSIFIED_END=1\nRESTORE_JOURNAL_SHA256=19c4e28b2f54ea8f217db4e622b0a5c758efa7b77ce2dc0394b6d159568c0b2e\nRESTORE_LOG_CLASSIFIED_BEGIN=1\nRESTORE_LOG_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nRESTORE_LOG_CLASSIFIED_END=1\nSOURCE_WORKTREE_VALID=true\nSOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614\nSOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_TRACKING_HEAD=UNKNOWN\nSOURCE_PUSHURL_CLASS=DISABLED\nSOURCE_REMOTE_CLASS=HTTP\nSOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9\nSOURCE_LSREMOTE_RC=0\nSOURCE_LSREMOTE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nSOURCE_REMOTE_HEAD=UNKNOWN\nRESTORE_SERVICE_BINDING=NOT_PROVEN\nDECISION=HOLD_BACKUP_1114_SCOPE_NOT_YET_SAFE\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1114_FAILURE\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1113_FINAL_RCA\nCHANGES_MADE_BY_1114=false\nPRODUCT_MUTATION_BY_1114=false\nVM_MUTATION_BY_1114=false\nCLOUD_MUTATION_BY_1114=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1113_FINAL_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index f2759749..5f4fd32e 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1 STATUS=FAIL RC=3 HOST=pve01 MODE=read-only -COMPONENT=homelab-backup-1112-ambiguity-rca +COMPONENT=homelab-backup-1113-final-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=d1e970ab78309057995c785598ad19f335660b3bdad9d91f82f8a58aae349f74 +COMMAND_SHA256=6a9e35fad42bb449e3bba1af241cc9b4e02b094cc00b77de3741866b6cd5e547 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,14 +24,14 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=87b985744c8a04978f232aba11354a2e66388e428b0eeec0a763adafc74e8fa5 -SANITIZED_OUTPUT_SHA256=2beac0e910df9a79ef0fdc3a96ecd450e8b0a17ff922f22c44187d8b2e629d3e +RAW_EVIDENCE_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a +SANITIZED_OUTPUT_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a OUTPUT_BEGIN -WORKERS2_BACKUP_1112_AMBIGUITY_RCA_BEGIN=1 -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1 +WORKERS2_BACKUP_1113_FINAL_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1 MODE=read-only -COMPONENT=homelab-backup-1112-ambiguity-rca -MUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_HEALTH_PRODUCER_AND_SOURCE_RECONCILE;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP +COMPONENT=homelab-backup-1113-final-rca +MUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_RESTORE_CHECK_AND_SOURCE_OFFHOST_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP ERROR_REGISTER_CHECK=OK REFERENCE_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 @@ -39,202 +39,102 @@ ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4 RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 -VMQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 -VM_QUEUE_CURRENT_STATE=success,inactive,dead -BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed +RESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 +BACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 SCHEDULER_RC=0 SELFTEST_REPLACEMENT486=PASS -SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET -EXPECTED_CLOUDB_HEALTH_PATH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt -CLOUDB_HEALTH_EXISTS=true -CLOUDB_HEALTH_BYTES=71 -CLOUDB_HEALTH_MTIME_EPOCH=1789966497 -CLOUDB_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6 -CLOUDB_HEALTH_MARKERS_BEGIN=1 +SCHEDULER_LIVE_SELECTION={"due_seconds":86400,"enabled":true,"logical_id":"us-netbird"} +VM_QUEUE_CURRENT_STATE=success,inactive,dead +BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed +RESTORE_HEALTH_BEGIN=1 +RESTORE_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6 +RESTORE_HEALTH_MTIME_EPOCH=1789966497 STATUS=FAIL -CLOUDB_HEALTH_MARKERS_END=1 -HEALTH_PATH_SOURCE_MATCH_COUNT=8 -HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 BYTES=15391 -HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/homelab-backup-audit -19-GFSP="/etc/homelab-backup/gfs-retention.json" -20-RET="/usr/local/sbin/homelab-backup-retention-gc" -21-WT="/root/homelab-ops-cr-2026-1005" -22-BRANCH="cr-2026-1005-backup-v2-health-alerting-remediation" -23-DRSEAL=pathlib.Path("/var/lib/homelab-admin/seals/site-dr-final-acceptance-v1.json") -24:CLOUDB_HEALTH=pathlib.Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") -25-HEALTHS=[pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json"),pathlib.Path("/var/lib/homelab-health/backup-audit.json")] -26- -27-def run(cmd,timeout=60,env=None): -28- try: -29- p=subprocess.run(cmd,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env) --- -231- txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace") -232- kv={} -233- for line in txt.splitlines(): -234- if "=" in line: -235- k,v=line.split("=",1); kv[k]=v -236: cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes") -237-if not cloudb_ok: -238- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"}) -239- -240-source={"local_head":None,"remote_head":None,"pushurl_disabled":False} -241-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) -HEALTH_PATH_CONTEXT_END=/usr/local/sbin/homelab-backup-audit -HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc SHA256=f9c9a17c09c542a653df544b74321c797e89cd2dbd726b52287bfd6097b0998a BYTES=29834 -HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc -HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisorcpython-313.pyc -HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc SHA256=1cff2c1761a8487dd0c7d6e2a0d6b4a6a45710b23e7afdaad1059526b96f396c BYTES=24854 -HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc -HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-cookie-expiry-watch-20260710T072136Z.cpython-313.pyc -HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc SHA256=8acaa25f3d259e2d00d0bccf86a64c616d2fe447bbab492f9e12c4b996334582 BYTES=23194 -HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc -HEALTH_PATH_CONTEXT_END=/usr/local/sbin/__pycache__/skladchik-reports-monitor-supervisor.pre-session-keepalive-20260710T071605Z.cpython-313.pyc -HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 BYTES=5610 -HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup -1-#!/usr/bin/env bash -2-set -Eeuo pipefail -3-umask 077 -4:CONF=/etc/skladchik-reports-monitor-backup.conf -5-[ -r "$CONF" ] && . "$CONF" -6-: "${EDGE_HOST:=[PRIVATE_IP]}" -7-: "${EDGE_USER:=debian}" -8-: "${SERVICE_NAME:=skladchik-reports-monitor}" -9:: "${LOCAL_ROOT:=/mnt/staging/skladchik-reports-monitor-backup}" -10:: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}" -11-: "${CLOUD_1:=pve01-mail-01-crypt:}" -12-: "${CLOUD_2:=pve01-mail-02-crypt:}" -13-: "${KEEP_DAILY:=30}" -14-: "${CONNECT_TIMEOUT:=20}" -15-SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout="${CONNECT_TIMEOUT}" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] -16-RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" -17-DAY="$(date -u +%Y%m%d)" -18-OUTDIR="${LOCAL_ROOT}/daily/${RUN_ID}" -19:LOGDIR=/var/log/skladchik-reports-monitor-backup -20:HEALTH=/var/lib/skladchik-reports-monitor-backup/health.txt -21:LOCK=/run/skladchik-reports-monitor-backup.lock -22-mkdir -p "$OUTDIR" "$LOGDIR" "$(dirname "$HEALTH")" -23-exec 9>"$LOCK" -24-if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi -25-log(){ printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/backup.log"; } -26-fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; } --- -30-log "START run_id=$RUN_ID edge=${EDGE_USER}@${EDGE_HOST}" -31-ARCHIVE="${OUTDIR}/skladchik-reports-monitor-${RUN_ID}.tar.gz" -32-ssh "${SSH_OPTS[@]}" "${EDGE_USER}@${EDGE_HOST}" sudo bash -s "$RUN_ID" > "$ARCHIVE" <<'REMOTE_SCRIPT' -33-set -Eeuo pipefail -34-RUN_ID="$1" -35:TMP="$(mktemp -d /tmp/skladchik-reports-monitor-backup.XXXXXX)" -36-cleanup(){ rm -rf "$TMP"; } -37-trap cleanup EXIT -38-mkdir -p "$TMP/payload/var-lib-skladchik-reports-monitor" "$TMP/payload/systemd" -39-cp -a /etc/skladchik-reports-monitor "$TMP/payload/etc-skladchik-reports-monitor" 2>/dev/null || true -40-cp -a /opt/skladchik-reports-monitor "$TMP/payload/opt-skladchik-reports-monitor" 2>/dev/null || true -HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup -HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 BYTES=6851 -HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check -1-#!/usr/bin/env bash -2-set -Eeuo pipefail -3-umask 077 +TS=2026-09-21T04:54:57Z +RUN_ID=20260921T045457Z +error_rc=1 +RESTORE_HEALTH_END=1 +BACKUP_HEALTH_BEGIN=1 +BACKUP_HEALTH_SHA256=ac71cf99698817816382eb1d888948f76eb5af2782e4755f41b8bc9948853dca +BACKUP_HEALTH_MTIME_EPOCH=1784856521 +STATUS=OK +TS=2026-07-24T01:28:41Z +RUN_ID=20260724T012808Z +SOURCE=debian@[PRIVATE_IP] +ARCHIVE=/mnt/staging/skladchik-reports-monitor-backup/daily/20260724T012808Z/skladchik-reports-monitor-20260724T012808Z.tar.gz +SIZE_BYTES=93249 +REMOTE_BASE=homelab/skladchik-reports-monitor-backup/daily/20260724T012808Z +CLOUD_1=pve01-mail-01-crypt: +CLOUD_2=pve01-mail-02-crypt: +KEEP_DAILY=30 +BACKUP_VERSION=4 +COOKIE_JAR_BACKED_UP=yes +EXTENSION_HEALTH_FILES_BACKED_UP=yes +TITLE_IN_EMAIL_SUBJECT=yes +BACKUP_HEALTH_END=1 +RESTORE_SOURCE_CONTRACT_BEGIN=1 4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf -5-[ -r "$CONF" ] && . "$CONF" -6:: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}" -7-: "${CLOUD_1:=pve01-mail-01-crypt:}" -8-: "${CLOUD_2:=pve01-mail-02-crypt:}" -9:: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}" -10-: "${KEEP_LOCAL:=10}" 11:LOGDIR=/var/log/skladchik-reports-monitor-backup 12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock -14-RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" -15-WORK="${WORK_ROOT}/${RUN_ID}" -16-mkdir -p "$WORK" "$LOGDIR" "$(dirname "$HEALTH")" -17-exec 9>"$LOCK" -18-if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi --- -88- py_status=skipped -89- if command -v python3 >/dev/null; then -90- python3 -m py_compile "$extract/opt-skladchik-reports-monitor/monitor.py" "$extract/opt-skladchik-reports-monitor/health_alert.py" "$extract/opt-skladchik-reports-monitor/operational_digest.py" "$extract/opt-skladchik-reports-monitor/event-selftest-runner.py" "$extract/opt-skladchik-reports-monitor/event-selftest-core" -91- py_status=ok -92- fi +14:RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +15:WORK="${WORK_ROOT}/${RUN_ID}" +20:fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; } +22:command -v rclone >/dev/null +28: if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then +29: while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort) +31: rclone copyto "$remote_path" "$out" >/dev/null +36:check_one_remote(){ +37: local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status +39: latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)" +40: [ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; } +41: archive_name="skladchik-reports-monitor-${latest}.tar.gz" +42: remote_archive="${rbase}/${latest}/${archive_name}" +43: outdir="${WORK}/${label}-${latest}" +46: log "${label} DOWNLOAD run=${latest}" 93: printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env" -94- log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}" -95-} -96-: > "$WORK/result.env" -97-check_one_remote CLOUD_1 "$CLOUD_1" -98-check_one_remote CLOUD_2 "$CLOUD_2" -99-. "$WORK/result.env" +94: log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}" +97:check_one_remote CLOUD_1 "$CLOUD_1" +98:check_one_remote CLOUD_2 "$CLOUD_2" 100:printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH" -101-find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done -102-log "OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}" -HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check -HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67 BYTES=753 -HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status -1-#!/usr/bin/env bash -2-set +e -3-printf 'RESTORE_CHECK_HEALTH_BEGIN\n' -4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true -5-printf 'RESTORE_CHECK_HEALTH_END\n' -6-printf 'RESTORE_CHECK_TIMER_BEGIN\n' -7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true -8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true -9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true -10-printf 'RESTORE_CHECK_TIMER_END\n' -11-printf 'RESTORE_CHECK_LOG_TAIL_BEGIN\n' -12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true -13-printf 'RESTORE_CHECK_LOG_TAIL_END\n' -HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status -HEALTH_PATH_SOURCE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0 BYTES=606 -HEALTH_PATH_CONTEXT_BEGIN=/usr/local/sbin/skladchik-reports-monitor-backup-status -1-#!/usr/bin/env bash -2-set +e -3-printf 'HEALTH_BEGIN\n' -4:cat /var/lib/skladchik-reports-monitor-backup/health.txt 2>/dev/null || true -5-printf 'HEALTH_END\n' -6-printf 'TIMER_BEGIN\n' -7:systemctl is-enabled skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/enabled=/' || true -8:systemctl is-active skladchik-reports-monitor-backup.timer 2>/dev/null | sed 's/^/active=/' || true -9:systemctl list-timers --all | grep skladchik-reports-monitor-backup || true -10-printf 'TIMER_END\n' -11-printf 'LOG_TAIL_BEGIN\n' -12:tail -n 60 /var/log/skladchik-reports-monitor-backup/backup.log 2>/dev/null || true -13-printf 'LOG_TAIL_END\n' -HEALTH_PATH_CONTEXT_END=/usr/local/sbin/skladchik-reports-monitor-backup-status -HEALTH_EXECUTABLE_MATCH_COUNT=5 -HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/homelab-backup-audit SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 -HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 -HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 -HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status SHA256=92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67 -HEALTH_EXECUTABLE_MATCH=/usr/local/sbin/skladchik-reports-monitor-backup-status SHA256=afbbe3ed8478d2de98456dbfa3f6405ed1a65bd57fec26b98d083f29a08fa5e0 -HEALTH_SERVICE_MATCH_COUNT=1 -HEALTH_SERVICE_FILE=/etc/systemd/system/homelab-backup-audit.service SHA256=7b64362ef0d0d7e59f5305afd642386082a275bba359c3c9b3def97dd6c2c4aa -HEALTH_SERVICE_UNIT=homelab-backup-audit.service -Description=Homelab unified backup and DR audit -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-audit scheduled -User=root -Group=root -Result=exit-code,ExecMainStatus=1,ActiveState=failed,SubState=failed, -SKLADCHIK_BACKUP_STATE_FILES_BEGIN=1 -MTIME_FLOAT|71|/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt -MTIME_FLOAT|484|/var/lib/skladchik-reports-monitor-backup/health.txt -SKLADCHIK_BACKUP_STATE_FILES_END=1 +RESTORE_SOURCE_CONTRACT_END=1 +RESTORE_CONFIG_EXISTS=true +RESTORE_CONFIG_SHA256=c7f63d5707196f9b770cb304333a80fa20d0e9e34554ef11224bd6f2e4208aa5 +RESTORE_SERVICE_EXISTS=false +RESTORE_TIMER_EXISTS=false +RESTORE_SERVICE_STATE=success,,,0,inactive,dead +RESTORE_TIMER_STATE=enabled:not-found,active:inactive +RESTORE_SERVICE_UNIT_BEGIN=1 +RESTORE_SERVICE_UNIT_END=1 +RESTORE_TIMER_UNIT_BEGIN=1 +RESTORE_TIMER_UNIT_END=1 +RESTORE_JOURNAL_CLASSIFIED_BEGIN=1 +RESTORE_JOURNAL_CLASSIFIED_END=1 +RESTORE_JOURNAL_SHA256=19c4e28b2f54ea8f217db4e622b0a5c758efa7b77ce2dc0394b6d159568c0b2e +RESTORE_LOG_CLASSIFIED_BEGIN=1 +RESTORE_LOG_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +RESTORE_LOG_CLASSIFIED_END=1 SOURCE_WORKTREE_VALID=true SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614 SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation SOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation -SOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9 +SOURCE_TRACKING_HEAD=UNKNOWN SOURCE_PUSHURL_CLASS=DISABLED +SOURCE_REMOTE_CLASS=HTTP +SOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9 +SOURCE_LSREMOTE_RC=0 +SOURCE_LSREMOTE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 SOURCE_REMOTE_HEAD=UNKNOWN -SOURCE_HEAD_ALIGNMENT=UNKNOWN -DECISION=HOLD_BACKUP_1113_PRODUCER_OR_SOURCE_STILL_AMBIGUOUS -NEXT_GATE=RECONCILE_ONLY_BACKUP_1113_AMBIGUITY -TASK_RESULT=HOLD_HOMELAB_BACKUP_1112_AMBIGUITY_RCA -CHANGES_MADE_BY_1113=false -PRODUCT_MUTATION_BY_1113=false -VM_MUTATION_BY_1113=false -CLOUD_MUTATION_BY_1113=false -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1112-AMBIGUITY-RCA-1113R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} -WORKERS2_BACKUP_1112_AMBIGUITY_RCA_END=1 +RESTORE_SERVICE_BINDING=NOT_PROVEN +DECISION=HOLD_BACKUP_1114_SCOPE_NOT_YET_SAFE +NEXT_GATE=RECONCILE_ONLY_BACKUP_1114_FAILURE +TASK_RESULT=HOLD_HOMELAB_BACKUP_1113_FINAL_RCA +CHANGES_MADE_BY_1114=false +PRODUCT_MUTATION_BY_1114=false +VM_MUTATION_BY_1114=false +CLOUD_MUTATION_BY_1114=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1113_FINAL_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END