runtime: publish SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1
This commit is contained in:
homelab-runtime-publisher
File diff suppressed because one or more lines are too long
@@ -0,0 +1,327 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1
|
||||
STATUS=FAIL
|
||||
RC=3
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1114-due-restore-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=723d1d786602a2ec8eea15aa8bcad0fba171dfbbd94f08d92cc2f8fa575695b2
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=3
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a
|
||||
SANITIZED_OUTPUT_SHA256=6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1114-due-restore-rca
|
||||
MUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_SCHEDULER_CLOUDB_REMOTE_AND_SOURCE_OFFHOST_CLASSIFICATION;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
REFERENCE_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
|
||||
AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593
|
||||
RESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23
|
||||
BACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3
|
||||
SCHEDULER_RC=0
|
||||
SELFTEST_REPLACEMENT486=PASS
|
||||
SCHEDULER_LIVE_SELECTION={"due_seconds":86400,"enabled":true,"logical_id":"us-netbird"}
|
||||
SCHEDULER_CLASS=US_NETBIRD_DUE
|
||||
US_NETBIRD_STATE_BEGIN=1
|
||||
US_NETBIRD_STATE_SHA256=3fd55f07f513d33d8190aea4eaa952413758b1956cb1ed3677fbb656c20bc38a
|
||||
{"schema":1,"logical_id":"us-netbird","result":"FAIL","generation":"v2-us-netbird-20260921T233658Z-709cf01e1256","updated_at_utc":"2026-09-21T23:37:13Z","last_success_epoch":1789704670,"last_attempt_epoch":1790033833,"retry_after_epoch":1790055433,"consecutive_failures":13,"replica_count":null}
|
||||
US_NETBIRD_STATE_END=1
|
||||
BACKUP_V2_UNIT_STATE_BEGIN=1
|
||||
MainPID=0
|
||||
Result=success
|
||||
ExecMainStartTimestamp=Tue 2026-09-22 08:14:07 MSK
|
||||
ExecMainExitTimestamp=Tue 2026-09-22 08:14:07 MSK
|
||||
ExecMainStatus=0
|
||||
ActiveState=inactive
|
||||
SubState=dead
|
||||
BACKUP_V2_TIMER_ENABLED=enabled
|
||||
BACKUP_V2_TIMER_ACTIVE=active
|
||||
Unit=homelab-backup-v2.service
|
||||
NextElapseUSecRealtime=
|
||||
LastTriggerUSec=Tue 2026-09-22 08:14:07 MSK
|
||||
Result=success
|
||||
BACKUP_V2_UNIT_STATE_END=1
|
||||
BACKUP_V2_JOURNAL_US_NETBIRD_BEGIN=1
|
||||
2026-09-21T14:42:42+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T15:13:12+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T15:13:19+03:00 pve01 homelab-backup-v2[4054724]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T15:13:19+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T15:13:19+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T15:22:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T20:30:35+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T20:30:35+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T20:30:35+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED
|
||||
2026-09-21T20:30:51+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.
|
||||
2026-09-21T20:30:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T21:03:24+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T21:03:57+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T21:03:57+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T21:37:26+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T21:37:49+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T21:37:49+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T22:17:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T22:17:41+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T22:17:41+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T22:49:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:02:25+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:02:25+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:11:13+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:25:05+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:25:05+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:25:14+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:27:32+03:00 pve01 vzdump[1496016]: INFO: Backup job finished successfully
|
||||
2026-09-21T23:28:28+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:28:28+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:28:28+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:28:33+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:30:55+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:30:55+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:30:55+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:31:01+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:31:01+03:00 pve01 homelab-backup-v2[1515875]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-21T23:31:01+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:31:01+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:31:09+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:31:10+03:00 pve01 homelab-backup-v2[1524090]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-21T23:31:10+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:31:10+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:31:18+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:31:18+03:00 pve01 homelab-backup-v2[1526070]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-21T23:31:18+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:31:18+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T00:03:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T00:04:14+03:00 pve01 vzdump[1627438]: INFO: Backup job finished successfully
|
||||
2026-09-22T00:05:17+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T00:05:17+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T00:05:17+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T00:38:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T00:38:32+03:00 pve01 homelab-backup-v2[1741601]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T00:38:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T00:38:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T01:38:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T01:38:16+03:00 pve01 homelab-backup-v2[1935141]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T01:38:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T01:38:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T02:00:52+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T02:02:23+03:00 pve01 vzdump[2000008]: INFO: Backup job finished successfully
|
||||
2026-09-22T02:12:47+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T02:12:47+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T02:12:47+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T02:36:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED
|
||||
2026-09-22T02:37:13+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.
|
||||
2026-09-22T02:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T03:12:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T03:12:32+03:00 pve01 homelab-backup-v2[2223458]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T03:12:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T03:12:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T03:46:25+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T03:46:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T03:46:31+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T04:21:11+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T04:44:11+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T04:44:11+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T04:44:11+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T04:53:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T04:53:59+03:00 pve01 homelab-backup-v2[2612150]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T04:53:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T04:53:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T05:27:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T05:27:59+03:00 pve01 homelab-backup-v2[2723689]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T05:27:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T05:27:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T05:58:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T05:58:16+03:00 pve01 homelab-backup-v2[2823098]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T05:58:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T05:58:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T06:31:19+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T06:32:52+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T06:32:52+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T06:32:52+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T07:09:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T07:09:46+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T07:09:46+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T07:42:15+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T07:42:15+03:00 pve01 homelab-backup-v2[3148085]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T07:42:15+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T07:42:15+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T08:14:07+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T08:14:07+03:00 pve01 homelab-backup-v2[3251854]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T08:14:07+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T08:14:07+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
BACKUP_V2_JOURNAL_US_NETBIRD_END=1
|
||||
RESTORE_SOURCE_SEGMENT_BEGIN=1
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf
|
||||
[ -r "$CONF" ] && . "$CONF"
|
||||
: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}"
|
||||
: "${CLOUD_1:=pve01-mail-01-crypt:}"
|
||||
: "${CLOUD_2:=pve01-mail-02-crypt:}"
|
||||
: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}"
|
||||
: "${KEEP_LOCAL:=10}"
|
||||
LOGDIR=/var/log/skladchik-reports-monitor-backup
|
||||
HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
|
||||
LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock
|
||||
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
WORK="${WORK_ROOT}/${RUN_ID}"
|
||||
mkdir -p "$WORK" "$LOGDIR" "$(dirname "$HEALTH")"
|
||||
exec 9>"$LOCK"
|
||||
if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi
|
||||
log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; }
|
||||
fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; }
|
||||
trap fail ERR
|
||||
command -v rclone >/dev/null
|
||||
log "START run_id=$RUN_ID"
|
||||
cat_chunked(){
|
||||
local remote_path="$1" out="$2" plist part
|
||||
: > "$out"
|
||||
plist="$(mktemp)"
|
||||
if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then
|
||||
while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort)
|
||||
else
|
||||
rclone copyto "$remote_path" "$out" >/dev/null
|
||||
fi
|
||||
rm -f "$plist"
|
||||
[ -s "$out" ]
|
||||
}
|
||||
check_one_remote(){
|
||||
local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status
|
||||
rbase="${remote}${REMOTE_BASE}/daily"
|
||||
latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)"
|
||||
[ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; }
|
||||
archive_name="skladchik-reports-monitor-${latest}.tar.gz"
|
||||
remote_archive="${rbase}/${latest}/${archive_name}"
|
||||
outdir="${WORK}/${label}-${latest}"
|
||||
mkdir -p "$outdir"
|
||||
archive="${outdir}/${archive_name}"
|
||||
log "${label} DOWNLOAD run=${latest}"
|
||||
cat_chunked "$remote_archive" "$archive"
|
||||
cat_chunked "${remote_archive}.sha256" "${archive}.sha256.remote"
|
||||
expected="$(awk '{print $1; exit}' "${archive}.sha256.remote")"
|
||||
actual="$(sha256sum "$archive" | awk '{print $1}')"
|
||||
[ -n "$expected" ] && [ "$expected" = "$actual" ]
|
||||
list="${outdir}/tar-list.txt"
|
||||
tar -tzf "$archive" > "$list"
|
||||
grep -Eq '(^|/)etc-skladchik-reports-monitor/config\.ini$' "$list"
|
||||
grep -Eq '(^|/)etc-skladchik-reports-monitor/secrets\.env$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/monitor\.py$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/health_alert\.py$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/state\.sqlite$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/alert-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/cookies\.jar$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.timer$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/operational_digest\.py$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-runner\.py$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-core$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/weekly-summary-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/age-alert-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/event-selftest-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.timer$' "$list"
|
||||
extract="${outdir}/extract"
|
||||
mkdir -p "$extract"
|
||||
tar -xzf "$archive" -C "$extract"
|
||||
sqlite_status=skipped
|
||||
if command -v sqlite3 >/dev/null && [ -s "$extract/var-lib-skladchik-reports-monitor/state.sqlite" ]; then
|
||||
[ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ]
|
||||
sqlite_status=ok
|
||||
fi
|
||||
py_status=skipped
|
||||
if command -v python3 >/dev/null; then
|
||||
python3 -m py_compile "$extract/opt-skladchik-reports-monitor/monitor.py" "$extract/opt-skladchik-reports-monitor/health_alert.py" "$extract/opt-skladchik-reports-monitor/operational_digest.py" "$extract/opt-skladchik-reports-monitor/event-selftest-runner.py" "$extract/opt-skladchik-reports-monitor/event-selftest-core"
|
||||
py_status=ok
|
||||
fi
|
||||
printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env"
|
||||
log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}"
|
||||
}
|
||||
: > "$WORK/result.env"
|
||||
check_one_remote CLOUD_1 "$CLOUD_1"
|
||||
check_one_remote CLOUD_2 "$CLOUD_2"
|
||||
. "$WORK/result.env"
|
||||
printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH"
|
||||
find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done
|
||||
log "OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}"
|
||||
RESTORE_SOURCE_SEGMENT_END=1
|
||||
RESTORE_SYSTEMD_BINDINGS_BEGIN=1
|
||||
RESTORE_SYSTEMD_BINDING_COUNT=0
|
||||
RESTORE_SYSTEMD_BINDINGS_END=1
|
||||
CLOUD_1_DAILY_LSF_RC=3
|
||||
CLOUD_1_DAILY_STDERR_SHA256=bb3ce24b6401f914e787d66ae4a3fb5460455a41fab035e6ede56f0af463781f
|
||||
2026/09/22 08:47:57 ERROR : : error listing: directory not found
|
||||
2026/09/22 08:47:57 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found
|
||||
CLOUD_2_DAILY_LSF_RC=3
|
||||
CLOUD_2_DAILY_STDERR_SHA256=a3c018181b7d581d2a06e107cb28685794108bc2066f6acc7abdfbc83545fefa
|
||||
2026/09/22 08:47:58 ERROR : : error listing: directory not found
|
||||
2026/09/22 08:47:58 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found
|
||||
CLOUDB_REMOTE_READ_CLASS=INCOMPLETE_OR_ERROR
|
||||
SOURCE_WORKTREE_VALID=true
|
||||
SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614
|
||||
SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_PUSHURL_CLASS=DISABLED
|
||||
SOURCE_REMOTE_HEAD_PROBE_RC=0
|
||||
SOURCE_REMOTE_BRANCH_PROBE_RC=0
|
||||
SOURCE_REMOTE_ALL_PROBE_RC=0
|
||||
SOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b
|
||||
SOURCE_REMOTE_BRANCH_HEAD=ABSENT
|
||||
SOURCE_LOCAL_COMMIT_REMOTE_REF_COUNT=0
|
||||
SOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT
|
||||
DECISION=HOLD_BACKUP_1115_ONE_OR_MORE_PATHS_UNSAFE
|
||||
NEXT_GATE=RECONCILE_ONLY_BACKUP_1115_FAILURE
|
||||
TASK_RESULT=HOLD_HOMELAB_BACKUP_1114_DUE_RESTORE_RCA
|
||||
CHANGES_MADE_BY_1115=false
|
||||
PRODUCT_MUTATION_BY_1115=false
|
||||
VM_MUTATION_BY_1115=false
|
||||
CLOUD_MUTATION_BY_1115=false
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+8
-8
File diff suppressed because one or more lines are too long
+283
-96
@@ -1,15 +1,15 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1
|
||||
STATUS=FAIL
|
||||
RC=3
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1113-final-rca
|
||||
COMPONENT=homelab-backup-1114-due-restore-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=6a9e35fad42bb449e3bba1af241cc9b4e02b094cc00b77de3741866b6cd5e547
|
||||
COMMAND_SHA256=723d1d786602a2ec8eea15aa8bcad0fba171dfbbd94f08d92cc2f8fa575695b2
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
@@ -24,14 +24,14 @@ MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a
|
||||
SANITIZED_OUTPUT_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a
|
||||
RAW_EVIDENCE_SHA256=6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a
|
||||
SANITIZED_OUTPUT_SHA256=6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_BACKUP_1113_FINAL_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1
|
||||
WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-1113-final-rca
|
||||
MUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_RESTORE_CHECK_AND_SOURCE_OFFHOST_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP
|
||||
COMPONENT=homelab-backup-1114-due-restore-rca
|
||||
MUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_SCHEDULER_CLOUDB_REMOTE_AND_SOURCE_OFFHOST_CLASSIFICATION;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
REFERENCE_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
@@ -44,97 +44,284 @@ BACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3
|
||||
SCHEDULER_RC=0
|
||||
SELFTEST_REPLACEMENT486=PASS
|
||||
SCHEDULER_LIVE_SELECTION={"due_seconds":86400,"enabled":true,"logical_id":"us-netbird"}
|
||||
VM_QUEUE_CURRENT_STATE=success,inactive,dead
|
||||
BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed
|
||||
RESTORE_HEALTH_BEGIN=1
|
||||
RESTORE_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6
|
||||
RESTORE_HEALTH_MTIME_EPOCH=1789966497
|
||||
STATUS=FAIL
|
||||
TS=2026-09-21T04:54:57Z
|
||||
RUN_ID=20260921T045457Z
|
||||
error_rc=1
|
||||
RESTORE_HEALTH_END=1
|
||||
BACKUP_HEALTH_BEGIN=1
|
||||
BACKUP_HEALTH_SHA256=ac71cf99698817816382eb1d888948f76eb5af2782e4755f41b8bc9948853dca
|
||||
BACKUP_HEALTH_MTIME_EPOCH=1784856521
|
||||
STATUS=OK
|
||||
TS=2026-07-24T01:28:41Z
|
||||
RUN_ID=20260724T012808Z
|
||||
SOURCE=debian@[PRIVATE_IP]
|
||||
ARCHIVE=/mnt/staging/skladchik-reports-monitor-backup/daily/20260724T012808Z/skladchik-reports-monitor-20260724T012808Z.tar.gz
|
||||
SIZE_BYTES=93249
|
||||
REMOTE_BASE=homelab/skladchik-reports-monitor-backup/daily/20260724T012808Z
|
||||
CLOUD_1=pve01-mail-01-crypt:
|
||||
CLOUD_2=pve01-mail-02-crypt:
|
||||
KEEP_DAILY=30
|
||||
BACKUP_VERSION=4
|
||||
COOKIE_JAR_BACKED_UP=yes
|
||||
EXTENSION_HEALTH_FILES_BACKED_UP=yes
|
||||
TITLE_IN_EMAIL_SUBJECT=yes
|
||||
BACKUP_HEALTH_END=1
|
||||
RESTORE_SOURCE_CONTRACT_BEGIN=1
|
||||
4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf
|
||||
11:LOGDIR=/var/log/skladchik-reports-monitor-backup
|
||||
12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
|
||||
13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock
|
||||
14:RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
15:WORK="${WORK_ROOT}/${RUN_ID}"
|
||||
20:fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; }
|
||||
22:command -v rclone >/dev/null
|
||||
28: if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then
|
||||
29: while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort)
|
||||
31: rclone copyto "$remote_path" "$out" >/dev/null
|
||||
36:check_one_remote(){
|
||||
37: local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status
|
||||
39: latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)"
|
||||
40: [ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; }
|
||||
41: archive_name="skladchik-reports-monitor-${latest}.tar.gz"
|
||||
42: remote_archive="${rbase}/${latest}/${archive_name}"
|
||||
43: outdir="${WORK}/${label}-${latest}"
|
||||
46: log "${label} DOWNLOAD run=${latest}"
|
||||
93: printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env"
|
||||
94: log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}"
|
||||
97:check_one_remote CLOUD_1 "$CLOUD_1"
|
||||
98:check_one_remote CLOUD_2 "$CLOUD_2"
|
||||
100:printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH"
|
||||
RESTORE_SOURCE_CONTRACT_END=1
|
||||
RESTORE_CONFIG_EXISTS=true
|
||||
RESTORE_CONFIG_SHA256=c7f63d5707196f9b770cb304333a80fa20d0e9e34554ef11224bd6f2e4208aa5
|
||||
RESTORE_SERVICE_EXISTS=false
|
||||
RESTORE_TIMER_EXISTS=false
|
||||
RESTORE_SERVICE_STATE=success,,,0,inactive,dead
|
||||
RESTORE_TIMER_STATE=enabled:not-found,active:inactive
|
||||
RESTORE_SERVICE_UNIT_BEGIN=1
|
||||
RESTORE_SERVICE_UNIT_END=1
|
||||
RESTORE_TIMER_UNIT_BEGIN=1
|
||||
RESTORE_TIMER_UNIT_END=1
|
||||
RESTORE_JOURNAL_CLASSIFIED_BEGIN=1
|
||||
RESTORE_JOURNAL_CLASSIFIED_END=1
|
||||
RESTORE_JOURNAL_SHA256=19c4e28b2f54ea8f217db4e622b0a5c758efa7b77ce2dc0394b6d159568c0b2e
|
||||
RESTORE_LOG_CLASSIFIED_BEGIN=1
|
||||
RESTORE_LOG_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
RESTORE_LOG_CLASSIFIED_END=1
|
||||
SCHEDULER_CLASS=US_NETBIRD_DUE
|
||||
US_NETBIRD_STATE_BEGIN=1
|
||||
US_NETBIRD_STATE_SHA256=3fd55f07f513d33d8190aea4eaa952413758b1956cb1ed3677fbb656c20bc38a
|
||||
{"schema":1,"logical_id":"us-netbird","result":"FAIL","generation":"v2-us-netbird-20260921T233658Z-709cf01e1256","updated_at_utc":"2026-09-21T23:37:13Z","last_success_epoch":1789704670,"last_attempt_epoch":1790033833,"retry_after_epoch":1790055433,"consecutive_failures":13,"replica_count":null}
|
||||
US_NETBIRD_STATE_END=1
|
||||
BACKUP_V2_UNIT_STATE_BEGIN=1
|
||||
MainPID=0
|
||||
Result=success
|
||||
ExecMainStartTimestamp=Tue 2026-09-22 08:14:07 MSK
|
||||
ExecMainExitTimestamp=Tue 2026-09-22 08:14:07 MSK
|
||||
ExecMainStatus=0
|
||||
ActiveState=inactive
|
||||
SubState=dead
|
||||
BACKUP_V2_TIMER_ENABLED=enabled
|
||||
BACKUP_V2_TIMER_ACTIVE=active
|
||||
Unit=homelab-backup-v2.service
|
||||
NextElapseUSecRealtime=
|
||||
LastTriggerUSec=Tue 2026-09-22 08:14:07 MSK
|
||||
Result=success
|
||||
BACKUP_V2_UNIT_STATE_END=1
|
||||
BACKUP_V2_JOURNAL_US_NETBIRD_BEGIN=1
|
||||
2026-09-21T14:42:42+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T15:13:12+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T15:13:19+03:00 pve01 homelab-backup-v2[4054724]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T15:13:19+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T15:13:19+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T15:22:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T20:30:35+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T20:30:35+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T20:30:35+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL
|
||||
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED
|
||||
2026-09-21T20:30:51+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.
|
||||
2026-09-21T20:30:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T21:03:24+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T21:03:57+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T21:03:57+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T21:37:26+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T21:37:49+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T21:37:49+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T22:17:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T22:17:41+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T22:17:41+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T22:49:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:02:25+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:02:25+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:11:13+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:25:05+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:25:05+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:25:14+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:27:32+03:00 pve01 vzdump[1496016]: INFO: Backup job finished successfully
|
||||
2026-09-21T23:28:28+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:28:28+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:28:28+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:28:33+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:30:55+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-21T23:30:55+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:30:55+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:31:01+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:31:01+03:00 pve01 homelab-backup-v2[1515875]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-21T23:31:01+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:31:01+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:31:09+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:31:10+03:00 pve01 homelab-backup-v2[1524090]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-21T23:31:10+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:31:10+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-21T23:31:18+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-21T23:31:18+03:00 pve01 homelab-backup-v2[1526070]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-21T23:31:18+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-21T23:31:18+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T00:03:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T00:04:14+03:00 pve01 vzdump[1627438]: INFO: Backup job finished successfully
|
||||
2026-09-22T00:05:17+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T00:05:17+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T00:05:17+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T00:38:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T00:38:32+03:00 pve01 homelab-backup-v2[1741601]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T00:38:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T00:38:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T01:38:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T01:38:16+03:00 pve01 homelab-backup-v2[1935141]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T01:38:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T01:38:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T02:00:52+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T02:02:23+03:00 pve01 vzdump[2000008]: INFO: Backup job finished successfully
|
||||
2026-09-22T02:12:47+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T02:12:47+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T02:12:47+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T02:36:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL
|
||||
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED
|
||||
2026-09-22T02:37:13+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.
|
||||
2026-09-22T02:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T03:12:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T03:12:32+03:00 pve01 homelab-backup-v2[2223458]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T03:12:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T03:12:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T03:46:25+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T03:46:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T03:46:31+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T04:21:11+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T04:44:11+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T04:44:11+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T04:44:11+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T04:53:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T04:53:59+03:00 pve01 homelab-backup-v2[2612150]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T04:53:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T04:53:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T05:27:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T05:27:59+03:00 pve01 homelab-backup-v2[2723689]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T05:27:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T05:27:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T05:58:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T05:58:16+03:00 pve01 homelab-backup-v2[2823098]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T05:58:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T05:58:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T06:31:19+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T06:32:52+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T06:32:52+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T06:32:52+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T07:09:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_RESULT=SUCCESS
|
||||
2026-09-22T07:09:46+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T07:09:46+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T07:42:15+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T07:42:15+03:00 pve01 homelab-backup-v2[3148085]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T07:42:15+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T07:42:15+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
2026-09-22T08:14:07+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
|
||||
2026-09-22T08:14:07+03:00 pve01 homelab-backup-v2[3251854]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET
|
||||
2026-09-22T08:14:07+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.
|
||||
2026-09-22T08:14:07+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.
|
||||
BACKUP_V2_JOURNAL_US_NETBIRD_END=1
|
||||
RESTORE_SOURCE_SEGMENT_BEGIN=1
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf
|
||||
[ -r "$CONF" ] && . "$CONF"
|
||||
: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}"
|
||||
: "${CLOUD_1:=pve01-mail-01-crypt:}"
|
||||
: "${CLOUD_2:=pve01-mail-02-crypt:}"
|
||||
: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}"
|
||||
: "${KEEP_LOCAL:=10}"
|
||||
LOGDIR=/var/log/skladchik-reports-monitor-backup
|
||||
HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
|
||||
LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock
|
||||
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
WORK="${WORK_ROOT}/${RUN_ID}"
|
||||
mkdir -p "$WORK" "$LOGDIR" "$(dirname "$HEALTH")"
|
||||
exec 9>"$LOCK"
|
||||
if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi
|
||||
log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; }
|
||||
fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; }
|
||||
trap fail ERR
|
||||
command -v rclone >/dev/null
|
||||
log "START run_id=$RUN_ID"
|
||||
cat_chunked(){
|
||||
local remote_path="$1" out="$2" plist part
|
||||
: > "$out"
|
||||
plist="$(mktemp)"
|
||||
if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then
|
||||
while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort)
|
||||
else
|
||||
rclone copyto "$remote_path" "$out" >/dev/null
|
||||
fi
|
||||
rm -f "$plist"
|
||||
[ -s "$out" ]
|
||||
}
|
||||
check_one_remote(){
|
||||
local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status
|
||||
rbase="${remote}${REMOTE_BASE}/daily"
|
||||
latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)"
|
||||
[ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; }
|
||||
archive_name="skladchik-reports-monitor-${latest}.tar.gz"
|
||||
remote_archive="${rbase}/${latest}/${archive_name}"
|
||||
outdir="${WORK}/${label}-${latest}"
|
||||
mkdir -p "$outdir"
|
||||
archive="${outdir}/${archive_name}"
|
||||
log "${label} DOWNLOAD run=${latest}"
|
||||
cat_chunked "$remote_archive" "$archive"
|
||||
cat_chunked "${remote_archive}.sha256" "${archive}.sha256.remote"
|
||||
expected="$(awk '{print $1; exit}' "${archive}.sha256.remote")"
|
||||
actual="$(sha256sum "$archive" | awk '{print $1}')"
|
||||
[ -n "$expected" ] && [ "$expected" = "$actual" ]
|
||||
list="${outdir}/tar-list.txt"
|
||||
tar -tzf "$archive" > "$list"
|
||||
grep -Eq '(^|/)etc-skladchik-reports-monitor/config\.ini$' "$list"
|
||||
grep -Eq '(^|/)etc-skladchik-reports-monitor/secrets\.env$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/monitor\.py$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/health_alert\.py$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/state\.sqlite$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/alert-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/cookies\.jar$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.timer$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/operational_digest\.py$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-runner\.py$' "$list"
|
||||
grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-core$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/weekly-summary-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/age-alert-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)var-lib-skladchik-reports-monitor/event-selftest-health\.txt$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.timer$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.service$' "$list"
|
||||
grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.timer$' "$list"
|
||||
extract="${outdir}/extract"
|
||||
mkdir -p "$extract"
|
||||
tar -xzf "$archive" -C "$extract"
|
||||
sqlite_status=skipped
|
||||
if command -v sqlite3 >/dev/null && [ -s "$extract/var-lib-skladchik-reports-monitor/state.sqlite" ]; then
|
||||
[ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ]
|
||||
sqlite_status=ok
|
||||
fi
|
||||
py_status=skipped
|
||||
if command -v python3 >/dev/null; then
|
||||
python3 -m py_compile "$extract/opt-skladchik-reports-monitor/monitor.py" "$extract/opt-skladchik-reports-monitor/health_alert.py" "$extract/opt-skladchik-reports-monitor/operational_digest.py" "$extract/opt-skladchik-reports-monitor/event-selftest-runner.py" "$extract/opt-skladchik-reports-monitor/event-selftest-core"
|
||||
py_status=ok
|
||||
fi
|
||||
printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env"
|
||||
log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}"
|
||||
}
|
||||
: > "$WORK/result.env"
|
||||
check_one_remote CLOUD_1 "$CLOUD_1"
|
||||
check_one_remote CLOUD_2 "$CLOUD_2"
|
||||
. "$WORK/result.env"
|
||||
printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH"
|
||||
find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done
|
||||
log "OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}"
|
||||
RESTORE_SOURCE_SEGMENT_END=1
|
||||
RESTORE_SYSTEMD_BINDINGS_BEGIN=1
|
||||
RESTORE_SYSTEMD_BINDING_COUNT=0
|
||||
RESTORE_SYSTEMD_BINDINGS_END=1
|
||||
CLOUD_1_DAILY_LSF_RC=3
|
||||
CLOUD_1_DAILY_STDERR_SHA256=bb3ce24b6401f914e787d66ae4a3fb5460455a41fab035e6ede56f0af463781f
|
||||
2026/09/22 08:47:57 ERROR : : error listing: directory not found
|
||||
2026/09/22 08:47:57 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found
|
||||
CLOUD_2_DAILY_LSF_RC=3
|
||||
CLOUD_2_DAILY_STDERR_SHA256=a3c018181b7d581d2a06e107cb28685794108bc2066f6acc7abdfbc83545fefa
|
||||
2026/09/22 08:47:58 ERROR : : error listing: directory not found
|
||||
2026/09/22 08:47:58 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found
|
||||
CLOUDB_REMOTE_READ_CLASS=INCOMPLETE_OR_ERROR
|
||||
SOURCE_WORKTREE_VALID=true
|
||||
SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614
|
||||
SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_TRACKING_HEAD=UNKNOWN
|
||||
SOURCE_PUSHURL_CLASS=DISABLED
|
||||
SOURCE_REMOTE_CLASS=HTTP
|
||||
SOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9
|
||||
SOURCE_LSREMOTE_RC=0
|
||||
SOURCE_LSREMOTE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
SOURCE_REMOTE_HEAD=UNKNOWN
|
||||
RESTORE_SERVICE_BINDING=NOT_PROVEN
|
||||
DECISION=HOLD_BACKUP_1114_SCOPE_NOT_YET_SAFE
|
||||
NEXT_GATE=RECONCILE_ONLY_BACKUP_1114_FAILURE
|
||||
TASK_RESULT=HOLD_HOMELAB_BACKUP_1113_FINAL_RCA
|
||||
CHANGES_MADE_BY_1114=false
|
||||
PRODUCT_MUTATION_BY_1114=false
|
||||
VM_MUTATION_BY_1114=false
|
||||
CLOUD_MUTATION_BY_1114=false
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_1113_FINAL_RCA_END=1
|
||||
SOURCE_REMOTE_HEAD_PROBE_RC=0
|
||||
SOURCE_REMOTE_BRANCH_PROBE_RC=0
|
||||
SOURCE_REMOTE_ALL_PROBE_RC=0
|
||||
SOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b
|
||||
SOURCE_REMOTE_BRANCH_HEAD=ABSENT
|
||||
SOURCE_LOCAL_COMMIT_REMOTE_REF_COUNT=0
|
||||
SOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT
|
||||
DECISION=HOLD_BACKUP_1115_ONE_OR_MORE_PATHS_UNSAFE
|
||||
NEXT_GATE=RECONCILE_ONLY_BACKUP_1115_FAILURE
|
||||
TASK_RESULT=HOLD_HOMELAB_BACKUP_1114_DUE_RESTORE_RCA
|
||||
CHANGES_MADE_BY_1115=false
|
||||
PRODUCT_MUTATION_BY_1115=false
|
||||
VM_MUTATION_BY_1115=false
|
||||
CLOUD_MUTATION_BY_1115=false
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user