homelab-runtime-publisher
39 lines
25 KiB
JSON
39 lines
25 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"channel": "homelab-runtime",
|
|
"command_id": "SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1",
|
|
"status": "FAIL",
|
|
"rc": 3,
|
|
"host": "pve01",
|
|
"mode": "read-only",
|
|
"component": "homelab-backup-1114-due-restore-rca",
|
|
"started_at_utc": "2026-09-22T05:47:58Z",
|
|
"finished_at_utc": "2026-09-22T05:47:58Z",
|
|
"reference_register_checked": true,
|
|
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
|
|
"error_register_checked": true,
|
|
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
|
|
"command_sha256": "723d1d786602a2ec8eea15aa8bcad0fba171dfbbd94f08d92cc2f8fa575695b2",
|
|
"duplicate_failed_command_blocked": false,
|
|
"block_reason": null,
|
|
"execution_started": true,
|
|
"change_declared": false,
|
|
"result_contract_valid": true,
|
|
"result_contract_status": null,
|
|
"result_contract_error": null,
|
|
"command_rc": 3,
|
|
"changes_made": false,
|
|
"rollback_started": false,
|
|
"rollback_restored": null,
|
|
"mutation_outcome": "NO_MUTATION",
|
|
"sanitized": true,
|
|
"secrets_included": false,
|
|
"private_addresses_included": false,
|
|
"raw_evidence_retained_locally": true,
|
|
"raw_evidence_sha256": "6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a",
|
|
"sanitized_output_sha256": "6d664682ffbfc73b8e0770be8526d10db48c853f575a6ac881344d91ac4cdf1a",
|
|
"output_truncated_in_json": false,
|
|
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
|
"output": "WORKERS2_BACKUP_1114_DUE_RESTORE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1\nMODE=read-only\nCOMPONENT=homelab-backup-1114-due-restore-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_SCHEDULER_CLOUDB_REMOTE_AND_SOURCE_OFFHOST_CLASSIFICATION;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nRESTORE_SHA256=abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23\nBACKUP_SHA256=1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION={\"due_seconds\":86400,\"enabled\":true,\"logical_id\":\"us-netbird\"}\nSCHEDULER_CLASS=US_NETBIRD_DUE\nUS_NETBIRD_STATE_BEGIN=1\nUS_NETBIRD_STATE_SHA256=3fd55f07f513d33d8190aea4eaa952413758b1956cb1ed3677fbb656c20bc38a\n{\"schema\":1,\"logical_id\":\"us-netbird\",\"result\":\"FAIL\",\"generation\":\"v2-us-netbird-20260921T233658Z-709cf01e1256\",\"updated_at_utc\":\"2026-09-21T23:37:13Z\",\"last_success_epoch\":1789704670,\"last_attempt_epoch\":1790033833,\"retry_after_epoch\":1790055433,\"consecutive_failures\":13,\"replica_count\":null}\nUS_NETBIRD_STATE_END=1\nBACKUP_V2_UNIT_STATE_BEGIN=1\nMainPID=0\nResult=success\nExecMainStartTimestamp=Tue 2026-09-22 08:14:07 MSK\nExecMainExitTimestamp=Tue 2026-09-22 08:14:07 MSK\nExecMainStatus=0\nActiveState=inactive\nSubState=dead\nBACKUP_V2_TIMER_ENABLED=enabled\nBACKUP_V2_TIMER_ACTIVE=active\nUnit=homelab-backup-v2.service\nNextElapseUSecRealtime=\nLastTriggerUSec=Tue 2026-09-22 08:14:07 MSK\nResult=success\nBACKUP_V2_UNIT_STATE_END=1\nBACKUP_V2_JOURNAL_US_NETBIRD_BEGIN=1\n2026-09-21T14:42:42+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T15:13:12+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T15:13:19+03:00 pve01 homelab-backup-v2[4054724]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T15:13:19+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T15:13:19+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T15:22:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T20:30:35+03:00 pve01 homelab-backup-v2[4090639]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T20:30:35+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T20:30:35+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T20:30:35+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED\n2026-09-21T20:30:51+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\n2026-09-21T20:30:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T21:03:24+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T21:03:57+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T21:03:57+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T21:37:26+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T21:37:49+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T21:37:49+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T22:17:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T22:17:41+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T22:17:41+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T22:49:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:02:25+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:02:25+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:11:13+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:25:05+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:25:05+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:25:14+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:27:32+03:00 pve01 vzdump[1496016]: INFO: Backup job finished successfully\n2026-09-21T23:28:28+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:28:28+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:28:28+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:28:33+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:30:55+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_RESULT=SUCCESS\n2026-09-21T23:30:55+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:30:55+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:31:01+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:31:01+03:00 pve01 homelab-backup-v2[1515875]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-21T23:31:01+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:31:01+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:31:09+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:31:10+03:00 pve01 homelab-backup-v2[1524090]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-21T23:31:10+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:31:10+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-21T23:31:18+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-21T23:31:18+03:00 pve01 homelab-backup-v2[1526070]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-21T23:31:18+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-21T23:31:18+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T00:03:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T00:04:14+03:00 pve01 vzdump[1627438]: INFO: Backup job finished successfully\n2026-09-22T00:05:17+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T00:05:17+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T00:05:17+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T00:38:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T00:38:32+03:00 pve01 homelab-backup-v2[1741601]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T00:38:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T00:38:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T01:38:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T01:38:16+03:00 pve01 homelab-backup-v2[1935141]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T01:38:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T01:38:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T02:00:52+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T02:02:23+03:00 pve01 vzdump[2000008]: INFO: Backup job finished successfully\n2026-09-22T02:12:47+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T02:12:47+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T02:12:47+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T02:36:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED\n2026-09-22T02:37:13+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'.\n2026-09-22T02:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T03:12:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T03:12:32+03:00 pve01 homelab-backup-v2[2223458]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T03:12:32+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T03:12:32+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T03:46:25+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T03:46:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T03:46:31+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T04:21:11+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T04:44:11+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T04:44:11+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T04:44:11+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T04:53:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T04:53:59+03:00 pve01 homelab-backup-v2[2612150]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T04:53:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T04:53:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T05:27:59+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T05:27:59+03:00 pve01 homelab-backup-v2[2723689]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T05:27:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T05:27:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T05:58:16+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T05:58:16+03:00 pve01 homelab-backup-v2[2823098]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T05:58:16+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T05:58:16+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T06:31:19+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T06:32:52+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T06:32:52+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T06:32:52+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T07:09:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_RESULT=SUCCESS\n2026-09-22T07:09:46+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T07:09:46+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T07:42:15+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T07:42:15+03:00 pve01 homelab-backup-v2[3148085]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T07:42:15+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T07:42:15+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-22T08:14:07+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-22T08:14:07+03:00 pve01 homelab-backup-v2[3251854]: SCHEDULE_DECISION=DEFER_NO_DUE_TARGET\n2026-09-22T08:14:07+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\n2026-09-22T08:14:07+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\nBACKUP_V2_JOURNAL_US_NETBIRD_END=1\nRESTORE_SOURCE_SEGMENT_BEGIN=1\n#!/usr/bin/env bash\nset -Eeuo pipefail\numask 077\nCONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n[ -r \"$CONF\" ] && . \"$CONF\"\n: \"${REMOTE_BASE:=homelab/skladchik-reports-monitor-backup}\"\n: \"${CLOUD_1:=pve01-mail-01-crypt:}\"\n: \"${CLOUD_2:=pve01-mail-02-crypt:}\"\n: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n: \"${KEEP_LOCAL:=10}\"\nLOGDIR=/var/log/skladchik-reports-monitor-backup\nHEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\nLOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\nRUN_ID=\"$(date -u +%Y%m%dT%H%M%SZ)\"\nWORK=\"${WORK_ROOT}/${RUN_ID}\"\nmkdir -p \"$WORK\" \"$LOGDIR\" \"$(dirname \"$HEALTH\")\"\nexec 9>\"$LOCK\"\nif ! flock -n 9; then echo \"STATUS=SKIP_LOCKED TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" > \"$HEALTH\"; exit 0; fi\nlog(){ printf '%s RESTORE_CHECK %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/restore-check.log\"; }\nfail(){ rc=$?; printf 'STATUS=FAIL\\nTS=%s\\nRUN_ID=%s\\nerror_rc=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$rc\" > \"$HEALTH\"; log \"FAIL rc=$rc\"; exit \"$rc\"; }\ntrap fail ERR\ncommand -v rclone >/dev/null\nlog \"START run_id=$RUN_ID\"\ncat_chunked(){\n local remote_path=\"$1\" out=\"$2\" plist part\n : > \"$out\"\n plist=\"$(mktemp)\"\n if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n else\n rclone copyto \"$remote_path\" \"$out\" >/dev/null\n fi\n rm -f \"$plist\"\n [ -s \"$out\" ]\n}\ncheck_one_remote(){\n local label=\"$1\" remote=\"$2\" rbase latest archive_name remote_archive outdir archive sha_remote expected actual list extract sqlite_status py_status\n rbase=\"${remote}${REMOTE_BASE}/daily\"\n latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n [ -n \"$latest\" ] || { echo \"${label}_STATUS=NO_RUN\" >> \"$WORK/result.env\"; return 20; }\n archive_name=\"skladchik-reports-monitor-${latest}.tar.gz\"\n remote_archive=\"${rbase}/${latest}/${archive_name}\"\n outdir=\"${WORK}/${label}-${latest}\"\n mkdir -p \"$outdir\"\n archive=\"${outdir}/${archive_name}\"\n log \"${label} DOWNLOAD run=${latest}\"\n cat_chunked \"$remote_archive\" \"$archive\"\n cat_chunked \"${remote_archive}.sha256\" \"${archive}.sha256.remote\"\n expected=\"$(awk '{print $1; exit}' \"${archive}.sha256.remote\")\"\n actual=\"$(sha256sum \"$archive\" | awk '{print $1}')\"\n [ -n \"$expected\" ] && [ \"$expected\" = \"$actual\" ]\n list=\"${outdir}/tar-list.txt\"\n tar -tzf \"$archive\" > \"$list\"\n grep -Eq '(^|/)etc-skladchik-reports-monitor/config\\.ini$' \"$list\"\n grep -Eq '(^|/)etc-skladchik-reports-monitor/secrets\\.env$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/monitor\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/health_alert\\.py$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/state\\.sqlite$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/alert-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/cookies\\.jar$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-watchdog\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-heartbeat\\.timer$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/operational_digest\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-runner\\.py$' \"$list\"\n grep -Eq '(^|/)opt-skladchik-reports-monitor/event-selftest-core$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/weekly-summary-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/age-alert-health\\.txt$' \"$list\"\n grep -Eq '(^|/)var-lib-skladchik-reports-monitor/event-selftest-health\\.txt$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-weekly-summary\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-age-alert\\.timer$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\\.service$' \"$list\"\n grep -Eq '(^|/)systemd/skladchik-reports-monitor-event-selftest\\.timer$' \"$list\"\n extract=\"${outdir}/extract\"\n mkdir -p \"$extract\"\n tar -xzf \"$archive\" -C \"$extract\"\n sqlite_status=skipped\n if command -v sqlite3 >/dev/null && [ -s \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" ]; then\n [ \"$(sqlite3 \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" 'PRAGMA integrity_check;' | tail -n 1)\" = \"ok\" ]\n sqlite_status=ok\n fi\n py_status=skipped\n if command -v python3 >/dev/null; then\n python3 -m py_compile \"$extract/opt-skladchik-reports-monitor/monitor.py\" \"$extract/opt-skladchik-reports-monitor/health_alert.py\" \"$extract/opt-skladchik-reports-monitor/operational_digest.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-runner.py\" \"$extract/opt-skladchik-reports-monitor/event-selftest-core\"\n py_status=ok\n fi\n printf '%s_STATUS=OK\\n%s_RUN=%s\\n%s_ARCHIVE_BYTES=%s\\n%s_SHA256_OK=yes\\n%s_TAR_OK=yes\\n%s_REQUIRED_FILES_OK=yes\\n%s_COOKIE_JAR_OK=yes\\n%s_EXTENSION_HEALTH_FILES_OK=yes\\n%s_EXTENSION_SCRIPTS_OK=yes\\n%s_EXTENSION_UNITS_OK=yes\\n%s_SQLITE=%s\\n%s_PY_COMPILE=%s\\n' \"$label\" \"$label\" \"$latest\" \"$label\" \"$(stat -c %s \"$archive\")\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$label\" \"$sqlite_status\" \"$label\" \"$py_status\" >> \"$WORK/result.env\"\n log \"${label} OK run=${latest} bytes=$(stat -c %s \"$archive\") cookie_jar=ok sqlite=${sqlite_status} py=${py_status}\"\n}\n: > \"$WORK/result.env\"\ncheck_one_remote CLOUD_1 \"$CLOUD_1\"\ncheck_one_remote CLOUD_2 \"$CLOUD_2\"\n. \"$WORK/result.env\"\nprintf 'STATUS=OK\\nTS=%s\\nRUN_ID=%s\\nREMOTE_BASE=%s\\nCLOUD_1_STATUS=%s\\nCLOUD_1_RUN=%s\\nCLOUD_2_STATUS=%s\\nCLOUD_2_RUN=%s\\nSHA256_OK=yes\\nTAR_OK=yes\\nREQUIRED_FILES_OK=yes\\nCOOKIE_JAR_OK=yes\\nEXTENSION_HEALTH_FILES_OK=yes\\nEXTENSION_SCRIPTS_OK=yes\\nEXTENSION_UNITS_OK=yes\\nSQLITE_OK=yes\\nPY_COMPILE_OK=yes\\nRESTORE_CHECK_VERSION=4\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$RUN_ID\" \"$REMOTE_BASE\" \"${CLOUD_1_STATUS:-unknown}\" \"${CLOUD_1_RUN:-}\" \"${CLOUD_2_STATUS:-unknown}\" \"${CLOUD_2_RUN:-}\" > \"$HEALTH\"\nfind \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\nlog \"OK run_id=$RUN_ID cloud1=${CLOUD_1_RUN:-} cloud2=${CLOUD_2_RUN:-}\"\nRESTORE_SOURCE_SEGMENT_END=1\nRESTORE_SYSTEMD_BINDINGS_BEGIN=1\nRESTORE_SYSTEMD_BINDING_COUNT=0\nRESTORE_SYSTEMD_BINDINGS_END=1\nCLOUD_1_DAILY_LSF_RC=3\nCLOUD_1_DAILY_STDERR_SHA256=bb3ce24b6401f914e787d66ae4a3fb5460455a41fab035e6ede56f0af463781f\n2026/09/22 08:47:57 ERROR : : error listing: directory not found\n2026/09/22 08:47:57 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found\nCLOUD_2_DAILY_LSF_RC=3\nCLOUD_2_DAILY_STDERR_SHA256=a3c018181b7d581d2a06e107cb28685794108bc2066f6acc7abdfbc83545fefa\n2026/09/22 08:47:58 ERROR : : error listing: directory not found\n2026/09/22 08:47:58 Failed to lsf with 2 errors: last error was: error in ListJSON: directory not found\nCLOUDB_REMOTE_READ_CLASS=INCOMPLETE_OR_ERROR\nSOURCE_WORKTREE_VALID=true\nSOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614\nSOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_PUSHURL_CLASS=DISABLED\nSOURCE_REMOTE_HEAD_PROBE_RC=0\nSOURCE_REMOTE_BRANCH_PROBE_RC=0\nSOURCE_REMOTE_ALL_PROBE_RC=0\nSOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b\nSOURCE_REMOTE_BRANCH_HEAD=ABSENT\nSOURCE_LOCAL_COMMIT_REMOTE_REF_COUNT=0\nSOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT\nDECISION=HOLD_BACKUP_1115_ONE_OR_MORE_PATHS_UNSAFE\nNEXT_GATE=RECONCILE_ONLY_BACKUP_1115_FAILURE\nTASK_RESULT=HOLD_HOMELAB_BACKUP_1114_DUE_RESTORE_RCA\nCHANGES_MADE_BY_1115=false\nPRODUCT_MUTATION_BY_1115=false\nVM_MUTATION_BY_1115=false\nCLOUD_MUTATION_BY_1115=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1114-DUE-RESTORE-RCA-1115R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1114_DUE_RESTORE_RCA_END=1\n"
|
|
}
|