From 500a1e22ce07ad2a40b8cf9115f5276d32eb71ec Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 22 Sep 2026 05:47:59 +0000 Subject: [PATCH] runtime: publish SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1 --- ...AB-BACKUP-1114-DUE-RESTORE-RCA-1115R1.json | 38 ++ ...LAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1.txt | 327 +++++++++++++++ runtime/latest.json | 16 +- runtime/latest.txt | 379 +++++++++++++----- 4 files changed, 656 insertions(+), 104 deletions(-) create mode 100644 runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1.json create mode 100644 runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1.txt diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1.json b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1.json new file mode 100644 index 00000000..71901d13 --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1", + "status": "FAIL", + "rc": 3, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-1114-due-restore-rca", + "started_at_utc": "2026-09-22T05:47:58Z", + "finished_at_utc": "2026-09-22T05:47:58Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "723d1d786602a2ec8eea15aa8bcad0fba171dfbbd94f08d92cc2f8fa575695b2", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 3, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a", + "sanitized_output_sha256": "6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1\nMODE=read-only\nCOMPONENT=homelab-backup-1114-due-restore-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_SCHEDULER_CLOUDB_REMOTE_AND_SOURCE_OFFHOST_CLASSIFICATION;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nRESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23\nBACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION={\"due_seconds\":86400,\"enabled\":true,\"logical_id\":\"us-netbird\"}\nSCHEDULER_CLASS=US_NETBIRD_DUE\nUS_NETBIRD_STATE_BEGIN=1\nUS_NETBIRD_STATE_SHA256=3fd55f07f513d33d8190aea4eaa952413758b1956cb1ed3677fbb656c20bc38a\n{\"schema\":1,\"logical_id\":\"us-netbird\",\"result\":\"FAIL\",\"generation\":\"v2-us-netbird-20260921T233658Z-709cf01e1256\",\"updated_at_utc\":\"2026-09-21T23:37:13Z\",\"last_success_epoch\":1789704670,\"last_attempt_epoch\":1790033833,\"retry_after_epoch\":1790055433,\"consecutive_failures\":13,\"replica_count\":null}\nUS_NETBIRD_STATE_END=1\nBACKUP_V2_UNIT_STATE_BEGIN=1\nMainPID=0\nResult=success\nExecMainStartTimestamp=Tue 2026-09-22 08:14:07 MSK\nExecMainExitTimestamp=Tue 2026-09-22 08:14:07 MSK\nExecMainStatus=0\nActiveState=inactive\nSubState=dead\nBACKUP_V2_TIMER_ENABLED=enabled\nBACKUP_V2_TIMER_ACTIVE=active\nUnit=homelab-backup-v2.service\nNextElapseUSecRealtime=\nLastTriggerUSec=Tue 2026-09-22 08:14:07 MSK\nResult=success\nBACKUP_V2_UNIT_STATE_END=1\nBACKUP_V2_JOURNAL_US_NETBIRD_BEGIN=1\n2026-09-21T14:42:42+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T15:13:12+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T15:13:19+03:00 pve01 homelab-backup-v2[4054724]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T15:13:19+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T15:13:19+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T15:22:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T20:30:35+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T20:30:35+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T20:30:35+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED\n2026-09-21T20:30:51+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\n2026-09-21T20:30:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T21:03:24+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T21:03:57+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T21:03:57+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T21:37:26+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T21:37:49+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T21:37:49+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T22:17:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T22:17:41+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T22:17:41+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T22:49:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:02:25+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:02:25+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:11:13+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:25:05+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:25:05+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:25:14+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:27:32+03:00 pve01 vzdump[1496016]: INFO: Backup job finished successfully\n2026-09-21T23:28:28+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:28:28+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:28:28+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:28:33+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:30:55+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:30:55+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:30:55+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:31:01+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:31:01+03:00 pve01 homelab-backup-v2[1515875]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-21T23:31:01+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:31:01+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:31:09+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:31:10+03:00 pve01 homelab-backup-v2[1524090]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-21T23:31:10+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:31:10+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:31:18+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:31:18+03:00 pve01 homelab-backup-v2[1526070]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-21T23:31:18+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:31:18+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T00:03:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T00:04:14+03:00 pve01 vzdump[1627438]: INFO: Backup job finished successfully\n2026-09-22T00:05:17+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T00:05:17+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T00:05:17+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T00:38:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T00:38:32+03:00 pve01 homelab-backup-v2[1741601]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T00:38:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T00:38:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T01:38:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T01:38:16+03:00 pve01 homelab-backup-v2[1935141]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T01:38:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T01:38:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T02:00:52+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T02:02:23+03:00 pve01 vzdump[2000008]: INFO: Backup job finished successfully\n2026-09-22T02:12:47+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T02:12:47+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T02:12:47+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T02:36:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED\n2026-09-22T02:37:13+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\n2026-09-22T02:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T03:12:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T03:12:32+03:00 pve01 homelab-backup-v2[2223458]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T03:12:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T03:12:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T03:46:25+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T03:46:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T03:46:31+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T04:21:11+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T04:44:11+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T04:44:11+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T04:44:11+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T04:53:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T04:53:59+03:00 pve01 homelab-backup-v2[2612150]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T04:53:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T04:53:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T05:27:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T05:27:59+03:00 pve01 homelab-backup-v2[2723689]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T05:27:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T05:27:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T05:58:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T05:58:16+03:00 pve01 homelab-backup-v2[2823098]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T05:58:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T05:58:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T06:31:19+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T06:32:52+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T06:32:52+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T06:32:52+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T07:09:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T07:09:46+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T07:09:46+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T07:42:15+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T07:42:15+03:00 pve01 homelab-backup-v2[3148085]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T07:42:15+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T07:42:15+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T08:14:07+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T08:14:07+03:00 pve01 homelab-backup-v2[3251854]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T08:14:07+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T08:14:07+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\nBACKUP_V2_JOURNAL_US_NETBIRD_END=1\nRESTORE_SOURCE_SEGMENT_BEGIN=1\n#!/usr/bin/env bash\nset -Eeuo pipefail\numask 077\nCONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n[ -r \"$CONF\" ] && . \"$CONF\"\n: \"${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}\"\n: \"${CLOUD_1:=pve01-mail-01-crypt:}\"\n: \"${CLOUD_2:=pve01-mail-02-crypt:}\"\n: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n: \"${KEEP_LOCAL:=10}\"\nLOGDIR=/var/log/skladchik-reports-monitor-backup\nHEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\nLOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\nRUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\nWORK=\"${WORK_ROOT}/${RUN_ID}\"\nmkdir -p \"$WORK\" \"$LOGDIR\" \"$(dirname \"$HEALTH\")\"\nexec 9>\"$LOCK\"\nif ! flock -n 9; then echo \"STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$HEALTH\"; exit 0; fi\nlog(){ printf '%s RESTORE_CHECK %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/restore-check.log\"; }\nfail(){ rc=$?; printf 'STATUS=FAIL\\nTS=%s\\nRUN_ID=%s\\nerror_rc=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$rc\" > \"$HEALTH\"; log \"FAIL rc=$rc\"; exit \"$rc\"; }\ntrap fail ERR\ncommand -v rclone >/dev/null\nlog \"START run_id=$RUN_ID\"\ncat_chunked(){\n local remote_path=\"$1\" out=\"$2\" plist part\n : > \"$out\"\n plist=\"$(mktemp)\"\n if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n else\n rclone copyto \"$remote_path\" \"$out\" >/dev/null\n fi\n rm -f \"$plist\"\n [ -s \"$out\" ]\n}\ncheck_one_remote(){\n local label=\"$1\" remote=\"$2\" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status\n rbase=\"${remote}${REMOTE_BASE}/daily\"\n latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n [ -n \"$latest\" ] || { echo \"${label}_STATUS=NO_RUN\" >> \"$WORK/result.env\"; return 20; }\n archive_name=\"skladchik-reports-monitor-${latest}.tar.gz\"\n remote_archive=\"${rbase}/${latest}/${archive_name}\"\n outdir=\"${WORK}/${label}-${latest}\"\n mkdir -p \"$outdir\"\n archive=\"${outdir}/${archive_name}\"\n log \"${label} DOWNLOAD run=${latest}\"\n cat_chunked \"$remote_archive\" \"$archive\"\n cat_chunked \"${remote_archive}.sha256\" \"${archive}.sha256.remote\"\n expected=\"$(awk '{print $1; exit}' \"${archive}.sha256.remote\")\"\n actual=\"$(sha256sum \"$archive\" | awk '{print $1}')\"\n [ -n \"$expected\" ] && [ \"$expected\" = \"$actual\" ]\n list=\"${outdir}/tar-list.txt\"\n tar -tzf \"$archive\" > \"$list\"\n grep -Eq '(^|/)etc-skladchik-reports-monitor/config\\.ini$' \"$list\"\n grep -Eq '(^|/)etc-skladchik-reports-monitor/secrets\\.env$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/monitor\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/health_alert\\.py$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/state\\.sqlite$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/alert-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/cookies\\.jar$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\\.timer$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/operational_digest\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-runner\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-core$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/weekly-summary-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/age-alert-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/event-selftest-health\\.txt$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\\.timer$' \"$list\"\n extract=\"${outdir}/extract\"\n mkdir -p \"$extract\"\n tar -xzf \"$archive\" -C \"$extract\"\n sqlite_status=skipped\n if command -v sqlite3 >/dev/null && [ -s \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" ]; then\n [ \"$(sqlite3 \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" 'PRAGMA integrity_check;' | tail -n 1)\" = \"ok\" ]\n sqlite_status=ok\n fi\n py_status=skipped\n if command -v python3 >/dev/null; then\n python3 -m py_compile \"$extract/opt-skladchik-reports-monitor/monitor.py\" \"$extract/opt-skladchik-reports-monitor/health_alert.py\" \"$extract/opt-skladchik-reports-monitor/operational_digest.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-runner.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-core\"\n py_status=ok\n fi\n printf '%s_STATUS=OK\\n%s_RUN=%s\\n%s_ARCHIVE_BYTES=%s\\n%s_SHA256_OK=yes\\n%s_TAR_OK=yes\\n%s_REQUIRED_FILES_OK=yes\\n%s_COOKIE_JAR_OK=yes\\n%s_EXTENSION_HEALTH_FILES_OK=yes\\n%s_EXTENSION_SCRIPTS_OK=yes\\n%s_EXTENSION_UNITS_OK=yes\\n%s_SQLITE=%s\\n%s_PY_COMPILE=%s\\n' \"$label\" \"$label\" \"$latest\" \"$label\" \"$(stat -c %s \"$archive\")\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$sqlite_status\" \"$label\" \"$py_status\" >> \"$WORK/result.env\"\n log \"${label} OK run=${latest} bytes=$(stat -c %s \"$archive\") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}\"\n}\n: > \"$WORK/result.env\"\ncheck_one_remote CLOUD_1 \"$CLOUD_1\"\ncheck_one_remote CLOUD_2 \"$CLOUD_2\"\n. \"$WORK/result.env\"\nprintf 'STATUS=OK\\nTS=%s\\nRUN_ID=%s\\nREMOTE_BASE=%s\\nCLOUD_1_STATUS=%s\\nCLOUD_1_RUN=%s\\nCLOUD_2_STATUS=%s\\nCLOUD_2_RUN=%s\\nSHA256_OK=yes\\nTAR_OK=yes\\nREQUIRED_FILES_OK=yes\\nCOOKIE_JAR_OK=yes\\nEXTENSION_HEALTH_FILES_OK=yes\\nEXTENSION_SCRIPTS_OK=yes\\nEXTENSION_UNITS_OK=yes\\nSQLITE_OK=yes\\nPY_COMPILE_OK=yes\\nRESTORE_CHECK_VERSION=4\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$REMOTE_BASE\" \"${CLOUD_1_STATUS:-unknown}\" \"${CLOUD_1_RUN:-}\" \"${CLOUD_2_STATUS:-unknown}\" \"${CLOUD_2_RUN:-}\" > \"$HEALTH\"\nfind \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\nlog \"OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}\"\nRESTORE_SOURCE_SEGMENT_END=1\nRESTORE_SYSTEMD_BINDINGS_BEGIN=1\nRESTORE_SYSTEMD_BINDING_COUNT=0\nRESTORE_SYSTEMD_BINDINGS_END=1\nCLOUD_1_DAILY_LSF_RC=3\nCLOUD_1_DAILY_STDERR_SHA256=bb3ce24b6401f914e787d66ae4a3fb5460455a41fab035e6ede56f0af463781f\n2026/09/22 08:47:57 ERROR : : error listing: directory not found\n2026/09/22 08:47:57 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found\nCLOUD_2_DAILY_LSF_RC=3\nCLOUD_2_DAILY_STDERR_SHA256=a3c018181b7d581d2a06e107cb28685794108bc2066f6acc7abdfbc83545fefa\n2026/09/22 08:47:58 ERROR : : error listing: directory not found\n2026/09/22 08:47:58 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found\nCLOUDB_REMOTE_READ_CLASS=INCOMPLETE_OR_ERROR\nSOURCE_WORKTREE_VALID=true\nSOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614\nSOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_PUSHURL_CLASS=DISABLED\nSOURCE_REMOTE_HEAD_PROBE_RC=0\nSOURCE_REMOTE_BRANCH_PROBE_RC=0\nSOURCE_REMOTE_ALL_PROBE_RC=0\nSOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b\nSOURCE_REMOTE_BRANCH_HEAD=ABSENT\nSOURCE_LOCAL_COMMIT_REMOTE_REF_COUNT=0\nSOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT\nDECISION=HOLD_BACKUP_1115_ONE_OR_MORE_PATHS_UNSAFE\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1115_FAILURE\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1114_DUE_RESTORE_RCA\nCHANGES_MADE_BY_1115=false\nPRODUCT_MUTATION_BY_1115=false\nVM_MUTATION_BY_1115=false\nCLOUD_MUTATION_BY_1115=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1114_DUE_RESTORE_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1.txt b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1.txt new file mode 100644 index 00000000..6580ff23 --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1.txt @@ -0,0 +1,327 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1 +STATUS=FAIL +RC=3 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-1114-due-restore-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=723d1d786602a2ec8eea15aa8bcad0fba171dfbbd94f08d92cc2f8fa575695b2 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=3 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a +SANITIZED_OUTPUT_SHA256=6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a +OUTPUT_BEGIN +WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1 +MODE=read-only +COMPONENT=homelab-backup-1114-due-restore-rca +MUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_SCHEDULER_CLOUDB_REMOTE_AND_SOURCE_OFFHOST_CLASSIFICATION;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE +ERROR_REGISTER_CHECK=OK +REFERENCE_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 +AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 +RESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23 +BACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 +SCHEDULER_RC=0 +SELFTEST_REPLACEMENT486=PASS +SCHEDULER_LIVE_SELECTION={"due_seconds":86400,"enabled":true,"logical_id":"us-netbird"} +SCHEDULER_CLASS=US_NETBIRD_DUE +US_NETBIRD_STATE_BEGIN=1 +US_NETBIRD_STATE_SHA256=3fd55f07f513d33d8190aea4eaa952413758b1956cb1ed3677fbb656c20bc38a +{"schema":1,"logical_id":"us-netbird","result":"FAIL","generation":"v2-us-netbird-20260921T233658Z-709cf01e1256","updated_at_utc":"2026-09-21T23:37:13Z","last_success_epoch":1789704670,"last_attempt_epoch":1790033833,"retry_after_epoch":1790055433,"consecutive_failures":13,"replica_count":null} +US_NETBIRD_STATE_END=1 +BACKUP_V2_UNIT_STATE_BEGIN=1 +MainPID=0 +Result=success +ExecMainStartTimestamp=Tue 2026-09-22 08:14:07 MSK +ExecMainExitTimestamp=Tue 2026-09-22 08:14:07 MSK +ExecMainStatus=0 +ActiveState=inactive +SubState=dead +BACKUP_V2_TIMER_ENABLED=enabled +BACKUP_V2_TIMER_ACTIVE=active +Unit=homelab-backup-v2.service +NextElapseUSecRealtime= +LastTriggerUSec=Tue 2026-09-22 08:14:07 MSK +Result=success +BACKUP_V2_UNIT_STATE_END=1 +BACKUP_V2_JOURNAL_US_NETBIRD_BEGIN=1 +2026-09-21T14:42:42+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T15:13:12+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T15:13:19+03:00 pve01 homelab-backup-v2[4054724]: SCHEDULE_RESULT=SUCCESS +2026-09-21T15:13:19+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T15:13:19+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T15:22:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_RESULT=SUCCESS +2026-09-21T20:30:35+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T20:30:35+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T20:30:35+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird +2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL +2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED +2026-09-21T20:30:51+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. +2026-09-21T20:30:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T21:03:24+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_RESULT=SUCCESS +2026-09-21T21:03:57+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T21:03:57+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T21:37:26+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_RESULT=SUCCESS +2026-09-21T21:37:49+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T21:37:49+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T22:17:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_RESULT=SUCCESS +2026-09-21T22:17:41+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T22:17:41+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T22:49:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_RESULT=SUCCESS +2026-09-21T23:02:25+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:02:25+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:11:13+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_RESULT=SUCCESS +2026-09-21T23:25:05+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:25:05+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:25:14+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:27:32+03:00 pve01 vzdump[1496016]: INFO: Backup job finished successfully +2026-09-21T23:28:28+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_RESULT=SUCCESS +2026-09-21T23:28:28+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:28:28+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:28:33+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:30:55+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_RESULT=SUCCESS +2026-09-21T23:30:55+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:30:55+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:31:01+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:31:01+03:00 pve01 homelab-backup-v2[1515875]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-21T23:31:01+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:31:01+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:31:09+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:31:10+03:00 pve01 homelab-backup-v2[1524090]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-21T23:31:10+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:31:10+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:31:18+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:31:18+03:00 pve01 homelab-backup-v2[1526070]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-21T23:31:18+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:31:18+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T00:03:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T00:04:14+03:00 pve01 vzdump[1627438]: INFO: Backup job finished successfully +2026-09-22T00:05:17+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_RESULT=SUCCESS +2026-09-22T00:05:17+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T00:05:17+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T00:38:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T00:38:32+03:00 pve01 homelab-backup-v2[1741601]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T00:38:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T00:38:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T01:38:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T01:38:16+03:00 pve01 homelab-backup-v2[1935141]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T01:38:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T01:38:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T02:00:52+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T02:02:23+03:00 pve01 vzdump[2000008]: INFO: Backup job finished successfully +2026-09-22T02:12:47+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_RESULT=SUCCESS +2026-09-22T02:12:47+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T02:12:47+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T02:36:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird +2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL +2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED +2026-09-22T02:37:13+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. +2026-09-22T02:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T03:12:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T03:12:32+03:00 pve01 homelab-backup-v2[2223458]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T03:12:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T03:12:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T03:46:25+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_RESULT=SUCCESS +2026-09-22T03:46:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T03:46:31+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T04:21:11+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T04:44:11+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_RESULT=SUCCESS +2026-09-22T04:44:11+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T04:44:11+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T04:53:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T04:53:59+03:00 pve01 homelab-backup-v2[2612150]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T04:53:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T04:53:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T05:27:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T05:27:59+03:00 pve01 homelab-backup-v2[2723689]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T05:27:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T05:27:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T05:58:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T05:58:16+03:00 pve01 homelab-backup-v2[2823098]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T05:58:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T05:58:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T06:31:19+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T06:32:52+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_RESULT=SUCCESS +2026-09-22T06:32:52+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T06:32:52+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T07:09:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_RESULT=SUCCESS +2026-09-22T07:09:46+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T07:09:46+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T07:42:15+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T07:42:15+03:00 pve01 homelab-backup-v2[3148085]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T07:42:15+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T07:42:15+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T08:14:07+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T08:14:07+03:00 pve01 homelab-backup-v2[3251854]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T08:14:07+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T08:14:07+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +BACKUP_V2_JOURNAL_US_NETBIRD_END=1 +RESTORE_SOURCE_SEGMENT_BEGIN=1 +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf +[ -r "$CONF" ] && . "$CONF" +: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}" +: "${CLOUD_1:=pve01-mail-01-crypt:}" +: "${CLOUD_2:=pve01-mail-02-crypt:}" +: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}" +: "${KEEP_LOCAL:=10}" +LOGDIR=/var/log/skladchik-reports-monitor-backup +HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +WORK="${WORK_ROOT}/${RUN_ID}" +mkdir -p "$WORK" "$LOGDIR" "$(dirname "$HEALTH")" +exec 9>"$LOCK" +if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi +log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; } +fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; } +trap fail ERR +command -v rclone >/dev/null +log "START run_id=$RUN_ID" +cat_chunked(){ + local remote_path="$1" out="$2" plist part + : > "$out" + plist="$(mktemp)" + if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then + while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort) + else + rclone copyto "$remote_path" "$out" >/dev/null + fi + rm -f "$plist" + [ -s "$out" ] +} +check_one_remote(){ + local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status + rbase="${remote}${REMOTE_BASE}/daily" + latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)" + [ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; } + archive_name="skladchik-reports-monitor-${latest}.tar.gz" + remote_archive="${rbase}/${latest}/${archive_name}" + outdir="${WORK}/${label}-${latest}" + mkdir -p "$outdir" + archive="${outdir}/${archive_name}" + log "${label} DOWNLOAD run=${latest}" + cat_chunked "$remote_archive" "$archive" + cat_chunked "${remote_archive}.sha256" "${archive}.sha256.remote" + expected="$(awk '{print $1; exit}' "${archive}.sha256.remote")" + actual="$(sha256sum "$archive" | awk '{print $1}')" + [ -n "$expected" ] && [ "$expected" = "$actual" ] + list="${outdir}/tar-list.txt" + tar -tzf "$archive" > "$list" + grep -Eq '(^|/)etc-skladchik-reports-monitor/config\.ini$' "$list" + grep -Eq '(^|/)etc-skladchik-reports-monitor/secrets\.env$' "$list" + grep -Eq '(^|/)opt-skladchik-reports-monitor/monitor\.py$' "$list" + grep -Eq '(^|/)opt-skladchik-reports-monitor/health_alert\.py$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/state\.sqlite$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/health\.txt$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/alert-health\.txt$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/cookies\.jar$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor\.timer$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.timer$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.timer$' "$list" + grep -Eq '(^|/)opt-skladchik-reports-monitor/operational_digest\.py$' "$list" + grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-runner\.py$' "$list" + grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-core$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/weekly-summary-health\.txt$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/age-alert-health\.txt$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/event-selftest-health\.txt$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.timer$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.timer$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.timer$' "$list" + extract="${outdir}/extract" + mkdir -p "$extract" + tar -xzf "$archive" -C "$extract" + sqlite_status=skipped + if command -v sqlite3 >/dev/null && [ -s "$extract/var-lib-skladchik-reports-monitor/state.sqlite" ]; then + [ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ] + sqlite_status=ok + fi + py_status=skipped + if command -v python3 >/dev/null; then + python3 -m py_compile "$extract/opt-skladchik-reports-monitor/monitor.py" "$extract/opt-skladchik-reports-monitor/health_alert.py" "$extract/opt-skladchik-reports-monitor/operational_digest.py" "$extract/opt-skladchik-reports-monitor/event-selftest-runner.py" "$extract/opt-skladchik-reports-monitor/event-selftest-core" + py_status=ok + fi + printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env" + log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}" +} +: > "$WORK/result.env" +check_one_remote CLOUD_1 "$CLOUD_1" +check_one_remote CLOUD_2 "$CLOUD_2" +. "$WORK/result.env" +printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH" +find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done +log "OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}" +RESTORE_SOURCE_SEGMENT_END=1 +RESTORE_SYSTEMD_BINDINGS_BEGIN=1 +RESTORE_SYSTEMD_BINDING_COUNT=0 +RESTORE_SYSTEMD_BINDINGS_END=1 +CLOUD_1_DAILY_LSF_RC=3 +CLOUD_1_DAILY_STDERR_SHA256=bb3ce24b6401f914e787d66ae4a3fb5460455a41fab035e6ede56f0af463781f +2026/09/22 08:47:57 ERROR : : error listing: directory not found +2026/09/22 08:47:57 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found +CLOUD_2_DAILY_LSF_RC=3 +CLOUD_2_DAILY_STDERR_SHA256=a3c018181b7d581d2a06e107cb28685794108bc2066f6acc7abdfbc83545fefa +2026/09/22 08:47:58 ERROR : : error listing: directory not found +2026/09/22 08:47:58 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found +CLOUDB_REMOTE_READ_CLASS=INCOMPLETE_OR_ERROR +SOURCE_WORKTREE_VALID=true +SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614 +SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_PUSHURL_CLASS=DISABLED +SOURCE_REMOTE_HEAD_PROBE_RC=0 +SOURCE_REMOTE_BRANCH_PROBE_RC=0 +SOURCE_REMOTE_ALL_PROBE_RC=0 +SOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b +SOURCE_REMOTE_BRANCH_HEAD=ABSENT +SOURCE_LOCAL_COMMIT_REMOTE_REF_COUNT=0 +SOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT +DECISION=HOLD_BACKUP_1115_ONE_OR_MORE_PATHS_UNSAFE +NEXT_GATE=RECONCILE_ONLY_BACKUP_1115_FAILURE +TASK_RESULT=HOLD_HOMELAB_BACKUP_1114_DUE_RESTORE_RCA +CHANGES_MADE_BY_1115=false +PRODUCT_MUTATION_BY_1115=false +VM_MUTATION_BY_1115=false +CLOUD_MUTATION_BY_1115=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index f2e6d92b..71901d13 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1", "status": "FAIL", "rc": 3, "host": "pve01", "mode": "read-only", - "component": "homelab-backup-1113-final-rca", - "started_at_utc": "2026-09-22T05:40:44Z", - "finished_at_utc": "2026-09-22T05:40:44Z", + "component": "homelab-backup-1114-due-restore-rca", + "started_at_utc": "2026-09-22T05:47:58Z", + "finished_at_utc": "2026-09-22T05:47:58Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "6a9e35fad42bb449e3bba1af241cc9b4e02b094cc00b77de3741866b6cd5e547", + "command_sha256": "723d1d786602a2ec8eea15aa8bcad0fba171dfbbd94f08d92cc2f8fa575695b2", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a", - "sanitized_output_sha256": "8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a", + "raw_evidence_sha256": "6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a", + "sanitized_output_sha256": "6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_BACKUP_1113_FINAL_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1\nMODE=read-only\nCOMPONENT=homelab-backup-1113-final-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_RESTORE_CHECK_AND_SOURCE_OFFHOST_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nRESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23\nBACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION={\"due_seconds\":86400,\"enabled\":true,\"logical_id\":\"us-netbird\"}\nVM_QUEUE_CURRENT_STATE=success,inactive,dead\nBACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed\nRESTORE_HEALTH_BEGIN=1\nRESTORE_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6\nRESTORE_HEALTH_MTIME_EPOCH=1789966497\nSTATUS=FAIL\nTS=2026-09-21T04:54:57Z\nRUN_ID=20260921T045457Z\nerror_rc=1\nRESTORE_HEALTH_END=1\nBACKUP_HEALTH_BEGIN=1\nBACKUP_HEALTH_SHA256=ac71cf99698817816382eb1d888948f76eb5af2782e4755f41b8bc9948853dca\nBACKUP_HEALTH_MTIME_EPOCH=1784856521\nSTATUS=OK\nTS=2026-07-24T01:28:41Z\nRUN_ID=20260724T012808Z\nSOURCE=debian@[PRIVATE_IP]\nARCHIVE=/mnt/staging/skladchik-reports-monitor-backup/daily/20260724T012808Z/skladchik-reports-monitor-20260724T012808Z.tar.gz\nSIZE_BYTES=93249\nREMOTE_BASE=homelab/skladchik-reports-monitor-backup/daily/20260724T012808Z\nCLOUD_1=pve01-mail-01-crypt:\nCLOUD_2=pve01-mail-02-crypt:\nKEEP_DAILY=30\nBACKUP_VERSION=4\nCOOKIE_JAR_BACKED_UP=yes\nEXTENSION_HEALTH_FILES_BACKED_UP=yes\nTITLE_IN_EMAIL_SUBJECT=yes\nBACKUP_HEALTH_END=1\nRESTORE_SOURCE_CONTRACT_BEGIN=1\n4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n11:LOGDIR=/var/log/skladchik-reports-monitor-backup\n12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n14:RUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n15:WORK=\"${WORK_ROOT}/${RUN_ID}\"\n20:fail(){ rc=$?; printf 'STATUS=FAIL\\nTS=%s\\nRUN_ID=%s\\nerror_rc=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$rc\" > \"$HEALTH\"; log \"FAIL rc=$rc\"; exit \"$rc\"; }\n22:command -v rclone >/dev/null\n28: if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n29: while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n31: rclone copyto \"$remote_path\" \"$out\" >/dev/null\n36:check_one_remote(){\n37: local label=\"$1\" remote=\"$2\" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status\n39: latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n40: [ -n \"$latest\" ] || { echo \"${label}_STATUS=NO_RUN\" >> \"$WORK/result.env\"; return 20; }\n41: archive_name=\"skladchik-reports-monitor-${latest}.tar.gz\"\n42: remote_archive=\"${rbase}/${latest}/${archive_name}\"\n43: outdir=\"${WORK}/${label}-${latest}\"\n46: log \"${label} DOWNLOAD run=${latest}\"\n93: printf '%s_STATUS=OK\\n%s_RUN=%s\\n%s_ARCHIVE_BYTES=%s\\n%s_SHA256_OK=yes\\n%s_TAR_OK=yes\\n%s_REQUIRED_FILES_OK=yes\\n%s_COOKIE_JAR_OK=yes\\n%s_EXTENSION_HEALTH_FILES_OK=yes\\n%s_EXTENSION_SCRIPTS_OK=yes\\n%s_EXTENSION_UNITS_OK=yes\\n%s_SQLITE=%s\\n%s_PY_COMPILE=%s\\n' \"$label\" \"$label\" \"$latest\" \"$label\" \"$(stat -c %s \"$archive\")\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$sqlite_status\" \"$label\" \"$py_status\" >> \"$WORK/result.env\"\n94: log \"${label} OK run=${latest} bytes=$(stat -c %s \"$archive\") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}\"\n97:check_one_remote CLOUD_1 \"$CLOUD_1\"\n98:check_one_remote CLOUD_2 \"$CLOUD_2\"\n100:printf 'STATUS=OK\\nTS=%s\\nRUN_ID=%s\\nREMOTE_BASE=%s\\nCLOUD_1_STATUS=%s\\nCLOUD_1_RUN=%s\\nCLOUD_2_STATUS=%s\\nCLOUD_2_RUN=%s\\nSHA256_OK=yes\\nTAR_OK=yes\\nREQUIRED_FILES_OK=yes\\nCOOKIE_JAR_OK=yes\\nEXTENSION_HEALTH_FILES_OK=yes\\nEXTENSION_SCRIPTS_OK=yes\\nEXTENSION_UNITS_OK=yes\\nSQLITE_OK=yes\\nPY_COMPILE_OK=yes\\nRESTORE_CHECK_VERSION=4\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$REMOTE_BASE\" \"${CLOUD_1_STATUS:-unknown}\" \"${CLOUD_1_RUN:-}\" \"${CLOUD_2_STATUS:-unknown}\" \"${CLOUD_2_RUN:-}\" > \"$HEALTH\"\nRESTORE_SOURCE_CONTRACT_END=1\nRESTORE_CONFIG_EXISTS=true\nRESTORE_CONFIG_SHA256=c7f63d5707196f9b770cb304333a80fa20d0e9e34554ef11224bd6f2e4208aa5\nRESTORE_SERVICE_EXISTS=false\nRESTORE_TIMER_EXISTS=false\nRESTORE_SERVICE_STATE=success,,,0,inactive,dead\nRESTORE_TIMER_STATE=enabled:not-found,active:inactive\nRESTORE_SERVICE_UNIT_BEGIN=1\nRESTORE_SERVICE_UNIT_END=1\nRESTORE_TIMER_UNIT_BEGIN=1\nRESTORE_TIMER_UNIT_END=1\nRESTORE_JOURNAL_CLASSIFIED_BEGIN=1\nRESTORE_JOURNAL_CLASSIFIED_END=1\nRESTORE_JOURNAL_SHA256=19c4e28b2f54ea8f217db4e622b0a5c758efa7b77ce2dc0394b6d159568c0b2e\nRESTORE_LOG_CLASSIFIED_BEGIN=1\nRESTORE_LOG_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nRESTORE_LOG_CLASSIFIED_END=1\nSOURCE_WORKTREE_VALID=true\nSOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614\nSOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_TRACKING_HEAD=UNKNOWN\nSOURCE_PUSHURL_CLASS=DISABLED\nSOURCE_REMOTE_CLASS=HTTP\nSOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9\nSOURCE_LSREMOTE_RC=0\nSOURCE_LSREMOTE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nSOURCE_REMOTE_HEAD=UNKNOWN\nRESTORE_SERVICE_BINDING=NOT_PROVEN\nDECISION=HOLD_BACKUP_1114_SCOPE_NOT_YET_SAFE\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1114_FAILURE\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1113_FINAL_RCA\nCHANGES_MADE_BY_1114=false\nPRODUCT_MUTATION_BY_1114=false\nVM_MUTATION_BY_1114=false\nCLOUD_MUTATION_BY_1114=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1113_FINAL_RCA_END=1\n" + "output": "WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1\nMODE=read-only\nCOMPONENT=homelab-backup-1114-due-restore-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_SCHEDULER_CLOUDB_REMOTE_AND_SOURCE_OFFHOST_CLASSIFICATION;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nRESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23\nBACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION={\"due_seconds\":86400,\"enabled\":true,\"logical_id\":\"us-netbird\"}\nSCHEDULER_CLASS=US_NETBIRD_DUE\nUS_NETBIRD_STATE_BEGIN=1\nUS_NETBIRD_STATE_SHA256=3fd55f07f513d33d8190aea4eaa952413758b1956cb1ed3677fbb656c20bc38a\n{\"schema\":1,\"logical_id\":\"us-netbird\",\"result\":\"FAIL\",\"generation\":\"v2-us-netbird-20260921T233658Z-709cf01e1256\",\"updated_at_utc\":\"2026-09-21T23:37:13Z\",\"last_success_epoch\":1789704670,\"last_attempt_epoch\":1790033833,\"retry_after_epoch\":1790055433,\"consecutive_failures\":13,\"replica_count\":null}\nUS_NETBIRD_STATE_END=1\nBACKUP_V2_UNIT_STATE_BEGIN=1\nMainPID=0\nResult=success\nExecMainStartTimestamp=Tue 2026-09-22 08:14:07 MSK\nExecMainExitTimestamp=Tue 2026-09-22 08:14:07 MSK\nExecMainStatus=0\nActiveState=inactive\nSubState=dead\nBACKUP_V2_TIMER_ENABLED=enabled\nBACKUP_V2_TIMER_ACTIVE=active\nUnit=homelab-backup-v2.service\nNextElapseUSecRealtime=\nLastTriggerUSec=Tue 2026-09-22 08:14:07 MSK\nResult=success\nBACKUP_V2_UNIT_STATE_END=1\nBACKUP_V2_JOURNAL_US_NETBIRD_BEGIN=1\n2026-09-21T14:42:42+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T15:13:12+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T15:13:19+03:00 pve01 homelab-backup-v2[4054724]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T15:13:19+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T15:13:19+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T15:22:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T20:30:35+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T20:30:35+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T20:30:35+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED\n2026-09-21T20:30:51+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\n2026-09-21T20:30:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T21:03:24+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T21:03:57+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T21:03:57+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T21:37:26+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T21:37:49+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T21:37:49+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T22:17:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T22:17:41+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T22:17:41+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T22:49:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:02:25+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:02:25+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:11:13+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:25:05+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:25:05+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:25:14+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:27:32+03:00 pve01 vzdump[1496016]: INFO: Backup job finished successfully\n2026-09-21T23:28:28+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:28:28+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:28:28+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:28:33+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:30:55+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:30:55+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:30:55+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:31:01+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:31:01+03:00 pve01 homelab-backup-v2[1515875]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-21T23:31:01+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:31:01+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:31:09+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:31:10+03:00 pve01 homelab-backup-v2[1524090]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-21T23:31:10+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:31:10+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:31:18+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:31:18+03:00 pve01 homelab-backup-v2[1526070]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-21T23:31:18+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:31:18+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T00:03:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T00:04:14+03:00 pve01 vzdump[1627438]: INFO: Backup job finished successfully\n2026-09-22T00:05:17+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T00:05:17+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T00:05:17+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T00:38:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T00:38:32+03:00 pve01 homelab-backup-v2[1741601]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T00:38:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T00:38:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T01:38:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T01:38:16+03:00 pve01 homelab-backup-v2[1935141]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T01:38:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T01:38:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T02:00:52+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T02:02:23+03:00 pve01 vzdump[2000008]: INFO: Backup job finished successfully\n2026-09-22T02:12:47+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T02:12:47+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T02:12:47+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T02:36:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED\n2026-09-22T02:37:13+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\n2026-09-22T02:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T03:12:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T03:12:32+03:00 pve01 homelab-backup-v2[2223458]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T03:12:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T03:12:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T03:46:25+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T03:46:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T03:46:31+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T04:21:11+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T04:44:11+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T04:44:11+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T04:44:11+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T04:53:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T04:53:59+03:00 pve01 homelab-backup-v2[2612150]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T04:53:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T04:53:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T05:27:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T05:27:59+03:00 pve01 homelab-backup-v2[2723689]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T05:27:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T05:27:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T05:58:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T05:58:16+03:00 pve01 homelab-backup-v2[2823098]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T05:58:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T05:58:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T06:31:19+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T06:32:52+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T06:32:52+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T06:32:52+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T07:09:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T07:09:46+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T07:09:46+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T07:42:15+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T07:42:15+03:00 pve01 homelab-backup-v2[3148085]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T07:42:15+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T07:42:15+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T08:14:07+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T08:14:07+03:00 pve01 homelab-backup-v2[3251854]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T08:14:07+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T08:14:07+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\nBACKUP_V2_JOURNAL_US_NETBIRD_END=1\nRESTORE_SOURCE_SEGMENT_BEGIN=1\n#!/usr/bin/env bash\nset -Eeuo pipefail\numask 077\nCONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n[ -r \"$CONF\" ] && . \"$CONF\"\n: \"${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}\"\n: \"${CLOUD_1:=pve01-mail-01-crypt:}\"\n: \"${CLOUD_2:=pve01-mail-02-crypt:}\"\n: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n: \"${KEEP_LOCAL:=10}\"\nLOGDIR=/var/log/skladchik-reports-monitor-backup\nHEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\nLOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\nRUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\nWORK=\"${WORK_ROOT}/${RUN_ID}\"\nmkdir -p \"$WORK\" \"$LOGDIR\" \"$(dirname \"$HEALTH\")\"\nexec 9>\"$LOCK\"\nif ! flock -n 9; then echo \"STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$HEALTH\"; exit 0; fi\nlog(){ printf '%s RESTORE_CHECK %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/restore-check.log\"; }\nfail(){ rc=$?; printf 'STATUS=FAIL\\nTS=%s\\nRUN_ID=%s\\nerror_rc=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$rc\" > \"$HEALTH\"; log \"FAIL rc=$rc\"; exit \"$rc\"; }\ntrap fail ERR\ncommand -v rclone >/dev/null\nlog \"START run_id=$RUN_ID\"\ncat_chunked(){\n local remote_path=\"$1\" out=\"$2\" plist part\n : > \"$out\"\n plist=\"$(mktemp)\"\n if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n else\n rclone copyto \"$remote_path\" \"$out\" >/dev/null\n fi\n rm -f \"$plist\"\n [ -s \"$out\" ]\n}\ncheck_one_remote(){\n local label=\"$1\" remote=\"$2\" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status\n rbase=\"${remote}${REMOTE_BASE}/daily\"\n latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n [ -n \"$latest\" ] || { echo \"${label}_STATUS=NO_RUN\" >> \"$WORK/result.env\"; return 20; }\n archive_name=\"skladchik-reports-monitor-${latest}.tar.gz\"\n remote_archive=\"${rbase}/${latest}/${archive_name}\"\n outdir=\"${WORK}/${label}-${latest}\"\n mkdir -p \"$outdir\"\n archive=\"${outdir}/${archive_name}\"\n log \"${label} DOWNLOAD run=${latest}\"\n cat_chunked \"$remote_archive\" \"$archive\"\n cat_chunked \"${remote_archive}.sha256\" \"${archive}.sha256.remote\"\n expected=\"$(awk '{print $1; exit}' \"${archive}.sha256.remote\")\"\n actual=\"$(sha256sum \"$archive\" | awk '{print $1}')\"\n [ -n \"$expected\" ] && [ \"$expected\" = \"$actual\" ]\n list=\"${outdir}/tar-list.txt\"\n tar -tzf \"$archive\" > \"$list\"\n grep -Eq '(^|/)etc-skladchik-reports-monitor/config\\.ini$' \"$list\"\n grep -Eq '(^|/)etc-skladchik-reports-monitor/secrets\\.env$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/monitor\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/health_alert\\.py$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/state\\.sqlite$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/alert-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/cookies\\.jar$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\\.timer$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/operational_digest\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-runner\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-core$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/weekly-summary-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/age-alert-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/event-selftest-health\\.txt$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\\.timer$' \"$list\"\n extract=\"${outdir}/extract\"\n mkdir -p \"$extract\"\n tar -xzf \"$archive\" -C \"$extract\"\n sqlite_status=skipped\n if command -v sqlite3 >/dev/null && [ -s \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" ]; then\n [ \"$(sqlite3 \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" 'PRAGMA integrity_check;' | tail -n 1)\" = \"ok\" ]\n sqlite_status=ok\n fi\n py_status=skipped\n if command -v python3 >/dev/null; then\n python3 -m py_compile \"$extract/opt-skladchik-reports-monitor/monitor.py\" \"$extract/opt-skladchik-reports-monitor/health_alert.py\" \"$extract/opt-skladchik-reports-monitor/operational_digest.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-runner.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-core\"\n py_status=ok\n fi\n printf '%s_STATUS=OK\\n%s_RUN=%s\\n%s_ARCHIVE_BYTES=%s\\n%s_SHA256_OK=yes\\n%s_TAR_OK=yes\\n%s_REQUIRED_FILES_OK=yes\\n%s_COOKIE_JAR_OK=yes\\n%s_EXTENSION_HEALTH_FILES_OK=yes\\n%s_EXTENSION_SCRIPTS_OK=yes\\n%s_EXTENSION_UNITS_OK=yes\\n%s_SQLITE=%s\\n%s_PY_COMPILE=%s\\n' \"$label\" \"$label\" \"$latest\" \"$label\" \"$(stat -c %s \"$archive\")\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$sqlite_status\" \"$label\" \"$py_status\" >> \"$WORK/result.env\"\n log \"${label} OK run=${latest} bytes=$(stat -c %s \"$archive\") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}\"\n}\n: > \"$WORK/result.env\"\ncheck_one_remote CLOUD_1 \"$CLOUD_1\"\ncheck_one_remote CLOUD_2 \"$CLOUD_2\"\n. \"$WORK/result.env\"\nprintf 'STATUS=OK\\nTS=%s\\nRUN_ID=%s\\nREMOTE_BASE=%s\\nCLOUD_1_STATUS=%s\\nCLOUD_1_RUN=%s\\nCLOUD_2_STATUS=%s\\nCLOUD_2_RUN=%s\\nSHA256_OK=yes\\nTAR_OK=yes\\nREQUIRED_FILES_OK=yes\\nCOOKIE_JAR_OK=yes\\nEXTENSION_HEALTH_FILES_OK=yes\\nEXTENSION_SCRIPTS_OK=yes\\nEXTENSION_UNITS_OK=yes\\nSQLITE_OK=yes\\nPY_COMPILE_OK=yes\\nRESTORE_CHECK_VERSION=4\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$REMOTE_BASE\" \"${CLOUD_1_STATUS:-unknown}\" \"${CLOUD_1_RUN:-}\" \"${CLOUD_2_STATUS:-unknown}\" \"${CLOUD_2_RUN:-}\" > \"$HEALTH\"\nfind \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\nlog \"OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}\"\nRESTORE_SOURCE_SEGMENT_END=1\nRESTORE_SYSTEMD_BINDINGS_BEGIN=1\nRESTORE_SYSTEMD_BINDING_COUNT=0\nRESTORE_SYSTEMD_BINDINGS_END=1\nCLOUD_1_DAILY_LSF_RC=3\nCLOUD_1_DAILY_STDERR_SHA256=bb3ce24b6401f914e787d66ae4a3fb5460455a41fab035e6ede56f0af463781f\n2026/09/22 08:47:57 ERROR : : error listing: directory not found\n2026/09/22 08:47:57 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found\nCLOUD_2_DAILY_LSF_RC=3\nCLOUD_2_DAILY_STDERR_SHA256=a3c018181b7d581d2a06e107cb28685794108bc2066f6acc7abdfbc83545fefa\n2026/09/22 08:47:58 ERROR : : error listing: directory not found\n2026/09/22 08:47:58 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found\nCLOUDB_REMOTE_READ_CLASS=INCOMPLETE_OR_ERROR\nSOURCE_WORKTREE_VALID=true\nSOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614\nSOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_PUSHURL_CLASS=DISABLED\nSOURCE_REMOTE_HEAD_PROBE_RC=0\nSOURCE_REMOTE_BRANCH_PROBE_RC=0\nSOURCE_REMOTE_ALL_PROBE_RC=0\nSOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b\nSOURCE_REMOTE_BRANCH_HEAD=ABSENT\nSOURCE_LOCAL_COMMIT_REMOTE_REF_COUNT=0\nSOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT\nDECISION=HOLD_BACKUP_1115_ONE_OR_MORE_PATHS_UNSAFE\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1115_FAILURE\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1114_DUE_RESTORE_RCA\nCHANGES_MADE_BY_1115=false\nPRODUCT_MUTATION_BY_1115=false\nVM_MUTATION_BY_1115=false\nCLOUD_MUTATION_BY_1115=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1114_DUE_RESTORE_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 5f4fd32e..6580ff23 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1 STATUS=FAIL RC=3 HOST=pve01 MODE=read-only -COMPONENT=homelab-backup-1113-final-rca +COMPONENT=homelab-backup-1114-due-restore-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=6a9e35fad42bb449e3bba1af241cc9b4e02b094cc00b77de3741866b6cd5e547 +COMMAND_SHA256=723d1d786602a2ec8eea15aa8bcad0fba171dfbbd94f08d92cc2f8fa575695b2 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,14 +24,14 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a -SANITIZED_OUTPUT_SHA256=8366a96d0a48174ea952d98ae908b85c320fc389b2c75252e67bf54bfe624a2a +RAW_EVIDENCE_SHA256=6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a +SANITIZED_OUTPUT_SHA256=6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a OUTPUT_BEGIN -WORKERS2_BACKUP_1113_FINAL_RCA_BEGIN=1 -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1 +WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1 MODE=read-only -COMPONENT=homelab-backup-1113-final-rca -MUTATION_BOUNDARY=NONE;READ_ONLY_EXACT_RESTORE_CHECK_AND_SOURCE_OFFHOST_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_FILE_WRITE_OUTSIDE_RUNNER_TEMP +COMPONENT=homelab-backup-1114-due-restore-rca +MUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_SCHEDULER_CLOUDB_REMOTE_AND_SOURCE_OFFHOST_CLASSIFICATION;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE ERROR_REGISTER_CHECK=OK REFERENCE_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 @@ -44,97 +44,284 @@ BACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3 SCHEDULER_RC=0 SELFTEST_REPLACEMENT486=PASS SCHEDULER_LIVE_SELECTION={"due_seconds":86400,"enabled":true,"logical_id":"us-netbird"} -VM_QUEUE_CURRENT_STATE=success,inactive,dead -BACKUP_AUDIT_CURRENT_STATE=exit-code,failed,failed -RESTORE_HEALTH_BEGIN=1 -RESTORE_HEALTH_SHA256=c0d8f2008d4f269d13aa9734f4ef856c12c41635e55f1b56917cedaca10380d6 -RESTORE_HEALTH_MTIME_EPOCH=1789966497 -STATUS=FAIL -TS=2026-09-21T04:54:57Z -RUN_ID=20260921T045457Z -error_rc=1 -RESTORE_HEALTH_END=1 -BACKUP_HEALTH_BEGIN=1 -BACKUP_HEALTH_SHA256=ac71cf99698817816382eb1d888948f76eb5af2782e4755f41b8bc9948853dca -BACKUP_HEALTH_MTIME_EPOCH=1784856521 -STATUS=OK -TS=2026-07-24T01:28:41Z -RUN_ID=20260724T012808Z -SOURCE=debian@[PRIVATE_IP] -ARCHIVE=/mnt/staging/skladchik-reports-monitor-backup/daily/20260724T012808Z/skladchik-reports-monitor-20260724T012808Z.tar.gz -SIZE_BYTES=93249 -REMOTE_BASE=homelab/skladchik-reports-monitor-backup/daily/20260724T012808Z -CLOUD_1=pve01-mail-01-crypt: -CLOUD_2=pve01-mail-02-crypt: -KEEP_DAILY=30 -BACKUP_VERSION=4 -COOKIE_JAR_BACKED_UP=yes -EXTENSION_HEALTH_FILES_BACKED_UP=yes -TITLE_IN_EMAIL_SUBJECT=yes -BACKUP_HEALTH_END=1 -RESTORE_SOURCE_CONTRACT_BEGIN=1 -4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf -11:LOGDIR=/var/log/skladchik-reports-monitor-backup -12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt -13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock -14:RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" -15:WORK="${WORK_ROOT}/${RUN_ID}" -20:fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; } -22:command -v rclone >/dev/null -28: if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then -29: while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort) -31: rclone copyto "$remote_path" "$out" >/dev/null -36:check_one_remote(){ -37: local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status -39: latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)" -40: [ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; } -41: archive_name="skladchik-reports-monitor-${latest}.tar.gz" -42: remote_archive="${rbase}/${latest}/${archive_name}" -43: outdir="${WORK}/${label}-${latest}" -46: log "${label} DOWNLOAD run=${latest}" -93: printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env" -94: log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}" -97:check_one_remote CLOUD_1 "$CLOUD_1" -98:check_one_remote CLOUD_2 "$CLOUD_2" -100:printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH" -RESTORE_SOURCE_CONTRACT_END=1 -RESTORE_CONFIG_EXISTS=true -RESTORE_CONFIG_SHA256=c7f63d5707196f9b770cb304333a80fa20d0e9e34554ef11224bd6f2e4208aa5 -RESTORE_SERVICE_EXISTS=false -RESTORE_TIMER_EXISTS=false -RESTORE_SERVICE_STATE=success,,,0,inactive,dead -RESTORE_TIMER_STATE=enabled:not-found,active:inactive -RESTORE_SERVICE_UNIT_BEGIN=1 -RESTORE_SERVICE_UNIT_END=1 -RESTORE_TIMER_UNIT_BEGIN=1 -RESTORE_TIMER_UNIT_END=1 -RESTORE_JOURNAL_CLASSIFIED_BEGIN=1 -RESTORE_JOURNAL_CLASSIFIED_END=1 -RESTORE_JOURNAL_SHA256=19c4e28b2f54ea8f217db4e622b0a5c758efa7b77ce2dc0394b6d159568c0b2e -RESTORE_LOG_CLASSIFIED_BEGIN=1 -RESTORE_LOG_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -RESTORE_LOG_CLASSIFIED_END=1 +SCHEDULER_CLASS=US_NETBIRD_DUE +US_NETBIRD_STATE_BEGIN=1 +US_NETBIRD_STATE_SHA256=3fd55f07f513d33d8190aea4eaa952413758b1956cb1ed3677fbb656c20bc38a +{"schema":1,"logical_id":"us-netbird","result":"FAIL","generation":"v2-us-netbird-20260921T233658Z-709cf01e1256","updated_at_utc":"2026-09-21T23:37:13Z","last_success_epoch":1789704670,"last_attempt_epoch":1790033833,"retry_after_epoch":1790055433,"consecutive_failures":13,"replica_count":null} +US_NETBIRD_STATE_END=1 +BACKUP_V2_UNIT_STATE_BEGIN=1 +MainPID=0 +Result=success +ExecMainStartTimestamp=Tue 2026-09-22 08:14:07 MSK +ExecMainExitTimestamp=Tue 2026-09-22 08:14:07 MSK +ExecMainStatus=0 +ActiveState=inactive +SubState=dead +BACKUP_V2_TIMER_ENABLED=enabled +BACKUP_V2_TIMER_ACTIVE=active +Unit=homelab-backup-v2.service +NextElapseUSecRealtime= +LastTriggerUSec=Tue 2026-09-22 08:14:07 MSK +Result=success +BACKUP_V2_UNIT_STATE_END=1 +BACKUP_V2_JOURNAL_US_NETBIRD_BEGIN=1 +2026-09-21T14:42:42+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T15:13:12+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T15:13:19+03:00 pve01 homelab-backup-v2[4054724]: SCHEDULE_RESULT=SUCCESS +2026-09-21T15:13:19+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T15:13:19+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T15:22:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_RESULT=SUCCESS +2026-09-21T20:30:35+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T20:30:35+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T20:30:35+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird +2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL +2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED +2026-09-21T20:30:51+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. +2026-09-21T20:30:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T21:03:24+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_RESULT=SUCCESS +2026-09-21T21:03:57+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T21:03:57+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T21:37:26+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_RESULT=SUCCESS +2026-09-21T21:37:49+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T21:37:49+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T22:17:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_RESULT=SUCCESS +2026-09-21T22:17:41+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T22:17:41+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T22:49:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_RESULT=SUCCESS +2026-09-21T23:02:25+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:02:25+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:11:13+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_RESULT=SUCCESS +2026-09-21T23:25:05+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:25:05+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:25:14+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:27:32+03:00 pve01 vzdump[1496016]: INFO: Backup job finished successfully +2026-09-21T23:28:28+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_RESULT=SUCCESS +2026-09-21T23:28:28+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:28:28+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:28:33+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:30:55+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_RESULT=SUCCESS +2026-09-21T23:30:55+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:30:55+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:31:01+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:31:01+03:00 pve01 homelab-backup-v2[1515875]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-21T23:31:01+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:31:01+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:31:09+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:31:10+03:00 pve01 homelab-backup-v2[1524090]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-21T23:31:10+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:31:10+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-21T23:31:18+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-21T23:31:18+03:00 pve01 homelab-backup-v2[1526070]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-21T23:31:18+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-21T23:31:18+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T00:03:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T00:04:14+03:00 pve01 vzdump[1627438]: INFO: Backup job finished successfully +2026-09-22T00:05:17+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_RESULT=SUCCESS +2026-09-22T00:05:17+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T00:05:17+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T00:38:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T00:38:32+03:00 pve01 homelab-backup-v2[1741601]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T00:38:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T00:38:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T01:38:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T01:38:16+03:00 pve01 homelab-backup-v2[1935141]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T01:38:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T01:38:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T02:00:52+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T02:02:23+03:00 pve01 vzdump[2000008]: INFO: Backup job finished successfully +2026-09-22T02:12:47+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_RESULT=SUCCESS +2026-09-22T02:12:47+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T02:12:47+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T02:36:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird +2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL +2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED +2026-09-22T02:37:13+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. +2026-09-22T02:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T03:12:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T03:12:32+03:00 pve01 homelab-backup-v2[2223458]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T03:12:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T03:12:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T03:46:25+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_RESULT=SUCCESS +2026-09-22T03:46:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T03:46:31+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T04:21:11+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T04:44:11+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_RESULT=SUCCESS +2026-09-22T04:44:11+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T04:44:11+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T04:53:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T04:53:59+03:00 pve01 homelab-backup-v2[2612150]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T04:53:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T04:53:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T05:27:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T05:27:59+03:00 pve01 homelab-backup-v2[2723689]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T05:27:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T05:27:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T05:58:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T05:58:16+03:00 pve01 homelab-backup-v2[2823098]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T05:58:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T05:58:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T06:31:19+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T06:32:52+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_RESULT=SUCCESS +2026-09-22T06:32:52+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T06:32:52+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T07:09:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_RESULT=SUCCESS +2026-09-22T07:09:46+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T07:09:46+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T07:42:15+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T07:42:15+03:00 pve01 homelab-backup-v2[3148085]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T07:42:15+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T07:42:15+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +2026-09-22T08:14:07+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-22T08:14:07+03:00 pve01 homelab-backup-v2[3251854]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET +2026-09-22T08:14:07+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +2026-09-22T08:14:07+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +BACKUP_V2_JOURNAL_US_NETBIRD_END=1 +RESTORE_SOURCE_SEGMENT_BEGIN=1 +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf +[ -r "$CONF" ] && . "$CONF" +: "${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}" +: "${CLOUD_1:=pve01-mail-01-crypt:}" +: "${CLOUD_2:=pve01-mail-02-crypt:}" +: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}" +: "${KEEP_LOCAL:=10}" +LOGDIR=/var/log/skladchik-reports-monitor-backup +HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +WORK="${WORK_ROOT}/${RUN_ID}" +mkdir -p "$WORK" "$LOGDIR" "$(dirname "$HEALTH")" +exec 9>"$LOCK" +if ! flock -n 9; then echo "STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEALTH"; exit 0; fi +log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; } +fail(){ rc=$?; printf 'STATUS=FAIL\nTS=%s\nRUN_ID=%s\nerror_rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$rc" > "$HEALTH"; log "FAIL rc=$rc"; exit "$rc"; } +trap fail ERR +command -v rclone >/dev/null +log "START run_id=$RUN_ID" +cat_chunked(){ + local remote_path="$1" out="$2" plist part + : > "$out" + plist="$(mktemp)" + if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then + while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort) + else + rclone copyto "$remote_path" "$out" >/dev/null + fi + rm -f "$plist" + [ -s "$out" ] +} +check_one_remote(){ + local label="$1" remote="$2" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status + rbase="${remote}${REMOTE_BASE}/daily" + latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)" + [ -n "$latest" ] || { echo "${label}_STATUS=NO_RUN" >> "$WORK/result.env"; return 20; } + archive_name="skladchik-reports-monitor-${latest}.tar.gz" + remote_archive="${rbase}/${latest}/${archive_name}" + outdir="${WORK}/${label}-${latest}" + mkdir -p "$outdir" + archive="${outdir}/${archive_name}" + log "${label} DOWNLOAD run=${latest}" + cat_chunked "$remote_archive" "$archive" + cat_chunked "${remote_archive}.sha256" "${archive}.sha256.remote" + expected="$(awk '{print $1; exit}' "${archive}.sha256.remote")" + actual="$(sha256sum "$archive" | awk '{print $1}')" + [ -n "$expected" ] && [ "$expected" = "$actual" ] + list="${outdir}/tar-list.txt" + tar -tzf "$archive" > "$list" + grep -Eq '(^|/)etc-skladchik-reports-monitor/config\.ini$' "$list" + grep -Eq '(^|/)etc-skladchik-reports-monitor/secrets\.env$' "$list" + grep -Eq '(^|/)opt-skladchik-reports-monitor/monitor\.py$' "$list" + grep -Eq '(^|/)opt-skladchik-reports-monitor/health_alert\.py$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/state\.sqlite$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/health\.txt$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/alert-health\.txt$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/cookies\.jar$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor\.timer$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\.timer$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\.timer$' "$list" + grep -Eq '(^|/)opt-skladchik-reports-monitor/operational_digest\.py$' "$list" + grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-runner\.py$' "$list" + grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-core$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/weekly-summary-health\.txt$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/age-alert-health\.txt$' "$list" + grep -Eq '(^|/)var-lib-skladchik-reports-monitor/event-selftest-health\.txt$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\.timer$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\.timer$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.service$' "$list" + grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\.timer$' "$list" + extract="${outdir}/extract" + mkdir -p "$extract" + tar -xzf "$archive" -C "$extract" + sqlite_status=skipped + if command -v sqlite3 >/dev/null && [ -s "$extract/var-lib-skladchik-reports-monitor/state.sqlite" ]; then + [ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ] + sqlite_status=ok + fi + py_status=skipped + if command -v python3 >/dev/null; then + python3 -m py_compile "$extract/opt-skladchik-reports-monitor/monitor.py" "$extract/opt-skladchik-reports-monitor/health_alert.py" "$extract/opt-skladchik-reports-monitor/operational_digest.py" "$extract/opt-skladchik-reports-monitor/event-selftest-runner.py" "$extract/opt-skladchik-reports-monitor/event-selftest-core" + py_status=ok + fi + printf '%s_STATUS=OK\n%s_RUN=%s\n%s_ARCHIVE_BYTES=%s\n%s_SHA256_OK=yes\n%s_TAR_OK=yes\n%s_REQUIRED_FILES_OK=yes\n%s_COOKIE_JAR_OK=yes\n%s_EXTENSION_HEALTH_FILES_OK=yes\n%s_EXTENSION_SCRIPTS_OK=yes\n%s_EXTENSION_UNITS_OK=yes\n%s_SQLITE=%s\n%s_PY_COMPILE=%s\n' "$label" "$label" "$latest" "$label" "$(stat -c %s "$archive")" "$label" "$label" "$label" "$label" "$label" "$label" "$label" "$sqlite_status" "$label" "$py_status" >> "$WORK/result.env" + log "${label} OK run=${latest} bytes=$(stat -c %s "$archive") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}" +} +: > "$WORK/result.env" +check_one_remote CLOUD_1 "$CLOUD_1" +check_one_remote CLOUD_2 "$CLOUD_2" +. "$WORK/result.env" +printf 'STATUS=OK\nTS=%s\nRUN_ID=%s\nREMOTE_BASE=%s\nCLOUD_1_STATUS=%s\nCLOUD_1_RUN=%s\nCLOUD_2_STATUS=%s\nCLOUD_2_RUN=%s\nSHA256_OK=yes\nTAR_OK=yes\nREQUIRED_FILES_OK=yes\nCOOKIE_JAR_OK=yes\nEXTENSION_HEALTH_FILES_OK=yes\nEXTENSION_SCRIPTS_OK=yes\nEXTENSION_UNITS_OK=yes\nSQLITE_OK=yes\nPY_COMPILE_OK=yes\nRESTORE_CHECK_VERSION=4\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$REMOTE_BASE" "${CLOUD_1_STATUS:-unknown}" "${CLOUD_1_RUN:-}" "${CLOUD_2_STATUS:-unknown}" "${CLOUD_2_RUN:-}" > "$HEALTH" +find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done +log "OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}" +RESTORE_SOURCE_SEGMENT_END=1 +RESTORE_SYSTEMD_BINDINGS_BEGIN=1 +RESTORE_SYSTEMD_BINDING_COUNT=0 +RESTORE_SYSTEMD_BINDINGS_END=1 +CLOUD_1_DAILY_LSF_RC=3 +CLOUD_1_DAILY_STDERR_SHA256=bb3ce24b6401f914e787d66ae4a3fb5460455a41fab035e6ede56f0af463781f +2026/09/22 08:47:57 ERROR : : error listing: directory not found +2026/09/22 08:47:57 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found +CLOUD_2_DAILY_LSF_RC=3 +CLOUD_2_DAILY_STDERR_SHA256=a3c018181b7d581d2a06e107cb28685794108bc2066f6acc7abdfbc83545fefa +2026/09/22 08:47:58 ERROR : : error listing: directory not found +2026/09/22 08:47:58 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found +CLOUDB_REMOTE_READ_CLASS=INCOMPLETE_OR_ERROR SOURCE_WORKTREE_VALID=true SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614 SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation -SOURCE_EXPECTED_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation -SOURCE_TRACKING_HEAD=UNKNOWN SOURCE_PUSHURL_CLASS=DISABLED -SOURCE_REMOTE_CLASS=HTTP -SOURCE_REMOTE_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9 -SOURCE_LSREMOTE_RC=0 -SOURCE_LSREMOTE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -SOURCE_REMOTE_HEAD=UNKNOWN -RESTORE_SERVICE_BINDING=NOT_PROVEN -DECISION=HOLD_BACKUP_1114_SCOPE_NOT_YET_SAFE -NEXT_GATE=RECONCILE_ONLY_BACKUP_1114_FAILURE -TASK_RESULT=HOLD_HOMELAB_BACKUP_1113_FINAL_RCA -CHANGES_MADE_BY_1114=false -PRODUCT_MUTATION_BY_1114=false -VM_MUTATION_BY_1114=false -CLOUD_MUTATION_BY_1114=false -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1113-FINAL-RCA-1114R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} -WORKERS2_BACKUP_1113_FINAL_RCA_END=1 +SOURCE_REMOTE_HEAD_PROBE_RC=0 +SOURCE_REMOTE_BRANCH_PROBE_RC=0 +SOURCE_REMOTE_ALL_PROBE_RC=0 +SOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b +SOURCE_REMOTE_BRANCH_HEAD=ABSENT +SOURCE_LOCAL_COMMIT_REMOTE_REF_COUNT=0 +SOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT +DECISION=HOLD_BACKUP_1115_ONE_OR_MORE_PATHS_UNSAFE +NEXT_GATE=RECONCILE_ONLY_BACKUP_1115_FAILURE +TASK_RESULT=HOLD_HOMELAB_BACKUP_1114_DUE_RESTORE_RCA +CHANGES_MADE_BY_1115=false +PRODUCT_MUTATION_BY_1115=false +VM_MUTATION_BY_1115=false +CLOUD_MUTATION_BY_1115=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END