diff --git a/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1.json b/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1.json new file mode 100644 index 00000000..05430ff2 --- /dev/null +++ b/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-alert-noise-rca", + "started_at_utc": "2026-09-25T08:43:34Z", + "finished_at_utc": "2026-09-25T08:43:34Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "6bf3ecfb4a7968445da820af52f396bb757d5a620f7ca04ca2a18ed4a1b9f524", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd", + "sanitized_output_sha256": "04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_ALERT_NOISE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1\nMODE=read-only\nCOMPONENT=homelab-backup-alert-noise-rca\nMUTATION_BOUNDARY=READ_ONLY;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_ALERT_POST;NO_MAIL;NO_BACKUP;NO_GIT_WRITE\nALERT_SCRIPT_SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006\nALERT_SCRIPT_BYTES=5857\nALERT_SERVICE_SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d\nALERT_TIMER_SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a\nALERT_SERVICE_BEGIN=1\n[Unit]\nDescription=Workers2 backup health alert via existing alertmanager backup integration\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-alertmanager-backup-run health-alert\nALERT_SERVICE_END=1\nALERT_TIMER_BEGIN=1\n[Unit]\nDescription=Workers2 backup health alert schedule\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=15min\nRandomizedDelaySec=30s\nPersistent=true\nUnit=homelab-alertmanager-backup-health.service\n\n[Install]\nWantedBy=timers.target\nALERT_TIMER_END=1\nALERT_SCRIPT_RELEVANT_BEGIN=1\n2:if [[ \"${1:-}\" == \"health-alert\" || \"${1:-}\" == \"email-test\" ]]; then\n4: exec /usr/bin/python3 -I -B -c 'import base64,os;src=base64.b64decode(\"aW1wb3J0IHN5cyxvcyxqc29uLHJlLHN1YnByb2Nlc3MscGF0aGxpYix0aW1lCm1vZGU9c3lzLmFyZ3ZbMV0KQVVESVQ9Ii91c3IvbG9jYWwvc2Jpbi9ob21lbGFiLWJhY2t1cC1hdWRpdCIKSEVBTFRIPXBhdGhsaWIuUGF0aCgiL3Zhci9saWIvaG9tZWxhYi1oZWFsdGgvYmFja3VwLWF1ZGl0LXYyLmpzb24iKQpTVEFURT1wYXRobGliLlBhdGgoIi92YXIvbGliL2hvbWVsYWItYWxlcnRpbmcvYmFja3VwLWFsZXJ0LXN0YXRlLmpzb24iKQpVU0VSQ0ZHPXBhdGhsaWIuUGF0aCgiL2V0Yy9wdmUvdXNlci5jZmciKQpkZWYgcmVjaXBpZW50KCk6CiAgICB0ZXh0PVVTRVJDRkcucmVhZF90ZXh0KGVuY29kaW5nPSJ1dGYtOCIsZXJyb3JzPSJyZXBsYWNlIikKICAgIG09cmUuZmluZGFsbChyIig/aSlcYltBLVowLTkuXyUrLV0rQFtBLVowLTkuLV0rXC5bQS1aXXsyLH1cYiIsdGV4dCkKICAgIGlmIGxlbihtKSE9MTogcmFpc2UgU3lzdGVtRXhpdCg2NSkKICAgIHJldHVybiBtWzBdCmRlZiBzZW5kKHN1YmplY3QsYm9keSk6CiAgICByY3B0PXJlY2lwaWVudCgpCiAgICBtc2c9ZiJGcm9tOiBob21lbGFiLWFsZXJ0c0BncmFtMS5ydVxuVG86IHtyY3B0fVxuU3ViamVjdDoge3N1YmplY3R9XG5Db250ZW50LVR5cGU6IHRleHQvcGxhaW47IGNoYXJzZXQ9VVRGLThcblxue2JvZHl9XG4iCiAgICBwPXN1YnByb2Nlc3MucnVuKFsiL3Vzci9zYmluL3NlbmRtYWlsIiwiLXQiXSxpbnB1dD1tc2csdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKICAgIGlmIHAucmV0dXJuY29kZSE9MDogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCmRlZiBrZXkoaSk6CiAgICByZXR1cm4gInwiLmpvaW4oc3RyKGkuZ2V0KHgsIiIpKSBmb3IgeCBpbiAoImNvZGUiLCJsb2dpY2FsX2lkIiwicm9vdF9jYXVzZSIpKQppZiBtb2RlPT0iZW1haWwtdGVzdCI6CiAgICBzZW5kKCJXb3JrZXJzMiBiYWNrdXAgYWxlcnRpbmcgdGVzdCIsIldvcmtlcnMyIGJhY2t1cC9EUiBhbGVydGluZyByb3V0ZSB0ZXN0LiBJZiB5b3UgcmVjZWl2ZWQgdGhpcywgdGhlIGxvY2FsIE1UQSBhY2NlcHRlZCB0aGUgY29udHJvbGxlZCB0ZXN0IGdlbmVyYXRlZCBkdXJpbmcgYmFja3VwIGNsb3N1cmUuIikKICAgIHByaW50KCJFTUFJTF9URVNUX0FDQ0VQVEVEPVlFUyIpCiAgICByYWlzZSBTeXN0ZW1FeGl0KDApCmlmIG1vZGUhPSJoZWFsdGgtYWxlcnQiOiByYWlzZSBTeXN0ZW1FeGl0KDY0KQpwPXN1YnByb2Nlc3MucnVuKFtBVURJVCwid3JpdGUtaGVhbHRoIl0sdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKaWYgcC5yZXR1cm5jb2RlIG5vdCBpbiAoMCwxKTogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCnRyeTogaD1qc29uLmxvYWRzKEhFQUxUSC5yZWFkX3RleHQoZW5jb2Rpbmc9InV0Zi04IikpCmV4Y2VwdCBFeGNlcHRpb246IHJhaXNlIFN5c3RlbUV4aXQoNjkpCmlzc3Vlcz1oLmdldCgiaXNzdWVzIikgaWYgaXNpbnN0YW5jZShoLmdldCgiaXNzdWVzIiksbGlzdCkgZWxzZSBbXQpjdXI9e2tleShpKTppIGZvciBpIGluIGlzc3VlcyBpZiBpc2luc3RhbmNlKGksZGljdCl9Cm9sZD17fQppZiBTVEFURS5pc19maWxlKCk6CiAgICB0cnk6CiAgICAgICAgb2xkaj1qc29uLmxvYWRzKFNUQVRFLnJlYWRfdGV4dChlbmNvZGluZz0idXRmLTgiKSkKICAgICAgICBvbGQ9e3g6e30gZm9yIHggaW4gb2xkai5nZXQoImlzc3VlX2tleXMiLFtdKSBpZiBpc2luc3RhbmNlKHgsc3RyKX0KICAgIGV4Y2VwdCBFeGNlcHRpb246IG9sZD17fQpjaGFuZ2VkPXNldChjdXIpIT1zZXQob2xkKQppZiBjaGFuZ2VkOgogICAgaWYgY3VyOgogICAgICAgIGxpbmVzPVsiV29ya2VyczIgYmFja3VwL0RSIGFsZXJ0IHN0YXRlIGNoYW5nZWQuIiwiIl0KICAgICAgICBmb3IgayBpbiBzb3J0ZWQoY3VyKToKICAgICAgICAgICAgaT1jdXJba10KICAgICAgICAgICAgbGluZXMuYXBwZW5kKGYie2kuZ2V0KCdzZXZlcml0eScsJ1VOS05PV04nKX0ge2kuZ2V0KCdjb2RlJywnVU5LTk9XTicpfSBsb2dpY2FsX2lkPXtpLmdldCgnbG9naWNhbF9pZCcsJy0nKX0gcm9vdF9jYXVzZT17aS5nZXQoJ3Jvb3RfY2F1c2UnLCdVTktOT1dOJyl9IikKICAgICAgICBzZW5kKGYiW1dvcmtlcnMyIEJBQ0tVUCBBTEVSVF0ge2xlbihjdXIpfSBhY3RpdmUgaXNzdWUocykiLCJcbiIuam9pbihsaW5lcykpCiAgICAgICAgcHJpbnQoIkFMRVJUX0VNQUlMX0FDQ0VQVEVEPVlFUyIpCiAgICBlbGlmIG9sZDoKICAgICAgICBzZW5kKCJbV29ya2VyczIgQkFDS1VQIFJFQ09WRVJZXSBhbGwgaXNzdWVzIGNsZWFyZWQiLCJXb3JrZXJzMiBiYWNrdXAvRFIgYXVkaXQgaXMgR1JFRU4uIEFsbCBwcmV2aW91c2x5IGFjdGl2ZSBiYWNrdXAgaXNzdWVzIGFyZSBjbGVhcmVkLiIpCiAgICAgICAgcHJpbnQoIlJFQ09WRVJZX0VNQUlMX0FDQ0VQVEVEPVlFUyIpClNUQVRFLnBhcmVudC5ta2RpcihwYXJlbnRzPVRydWUsZXhpc3Rfb2s9VHJ1ZSkKdG1wPVNUQVRFLndpdGhfbmFtZShTVEFURS5uYW1lKyIudG1wIikKdG1wLndyaXRlX3RleHQoanNvbi5kdW1wcyh7InNjaGVtYSI6ImhvbWVsYWItYmFja3VwLWFsZXJ0LXN0YXRlLXYxIiwidXBkYXRlZF9lcG9jaCI6aW50KHRpbWUudGltZSgpKSwiaXNzdWVfa2V5cyI6c29ydGVkKGN1cil9LHNlcGFyYXRvcnM9KCIsIiwiOiIpLHNvcnRfa2V5cz1UcnVlKSsiXG4iLGVuY29kaW5nPSJ1dGYtOCIpCm9zLmNobW9kKHRtcCwwbzYwMCk7IG9zLnJlcGxhY2UodG1wLFNUQVRFKQpwcmludCgiSEVBTFRIX0FMRVJUX1NUQVRFPU9LIikK\").decode(\"utf-8\");exec(compile(src,\"workers2-backup-alert\",\"exec\"))' \"$MODE\"\n9:APP=\"alertmanager\"\n19:fail(){ echo \"[ERROR] alertmanager backup failed; kept: $STAGE\"; }\n21:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$MON\" \"sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data\" > \"$STAGE/remote.tgz\"\n24:test -s \"$STAGE/payload/opt/stacks/observability-lite/alertmanager/alertmanager.yml\"\n26:jq -n --arg app \"$APP\" --arg host \"monitoring\" --arg engine \"files+alertmanager-config-check\" --arg config \"ok\" --argjson files \"$FILES\" \"{app:\\$app,engine:\\$engine,source_host:\\$host,source_paths:[\\\"monitoring:/opt/stacks/observability-lite/alertmanager\\\",\\\"monitoring:/opt/stacks/observability-lite/compose.yaml\\\",\\\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\\\"],config_check:\\$config,file_count:\\$files,note:\\\"Alertmanager configuration and runtime state in encrypted two-cloud backup\\\"}\" > \"$STAGE/metadata.json\"\n30:echo ALERTMANAGER_BACKUP_RUN=OK\nALERT_SCRIPT_RELEVANT_END=1\nALERT_UNIT_STATE_BEGIN=1\nUNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success\nUNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success\nNEXT LEFT LAST PASSED UNIT ACTIVATES\nFri 2026-09-25 11:52:11 MSK 8min Fri 2026-09-25 11:37:00 MSK 6min ago homelab-alertmanager-backup-health.timer homelab-alertmanager-backup-health.service\n\n1 timers listed.\nALERT_UNIT_STATE_END=1\nALERT_RECENT_JOURNAL_BEGIN=1\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nALERT_RECENT_JOURNAL_END=1\nCURRENT_HEALTH_BEGIN=1\nHEALTH_FILE=/var/lib/homelab-health/backup-audit-v2.json\nHEALTH_STATUS=RED\nHEALTH_ISSUE_COUNT=3\nHEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM\nHEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK\nHEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT\nHEALTH_FILE=/var/lib/homelab-health/backup-audit.json\nHEALTH_STATUS=RED\nHEALTH_ISSUE_COUNT=3\nHEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM\nHEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK\nHEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT\nCURRENT_HEALTH_END=1\nSOURCE_CANDIDATES_BEGIN=1\nSOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-run SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006\nSOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.timer SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a\nSOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.service SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d\nSOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a\nSOURCE_CANDIDATES_END=1\nDECISION=RCA_COMPLETE_ALERT_NOISE_FILTER_CHANGE_READY\nNEXT_GATE=CHANGE_ALERT_NOISE_FILTER_KEEP_CRITICAL_ALERTS\nTASK_RESULT=PASS_HOMELAB_BACKUP_ALERT_NOISE_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_ALERT_NOISE_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1.txt b/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1.txt new file mode 100644 index 00000000..de4dc396 --- /dev/null +++ b/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1.txt @@ -0,0 +1,188 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-alert-noise-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=6bf3ecfb4a7968445da820af52f396bb757d5a620f7ca04ca2a18ed4a1b9f524 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd +SANITIZED_OUTPUT_SHA256=04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd +OUTPUT_BEGIN +WORKERS2_BACKUP_ALERT_NOISE_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1 +MODE=read-only +COMPONENT=homelab-backup-alert-noise-rca +MUTATION_BOUNDARY=READ_ONLY;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_ALERT_POST;NO_MAIL;NO_BACKUP;NO_GIT_WRITE +ALERT_SCRIPT_SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006 +ALERT_SCRIPT_BYTES=5857 +ALERT_SERVICE_SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d +ALERT_TIMER_SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a +ALERT_SERVICE_BEGIN=1 +[Unit] +Description=Workers2 backup health alert via existing alertmanager backup integration +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/homelab-alertmanager-backup-run health-alert +ALERT_SERVICE_END=1 +ALERT_TIMER_BEGIN=1 +[Unit] +Description=Workers2 backup health alert schedule + +[Timer] +OnBootSec=5min +OnUnitActiveSec=15min +RandomizedDelaySec=30s +Persistent=true +Unit=homelab-alertmanager-backup-health.service + +[Install] +WantedBy=timers.target +ALERT_TIMER_END=1 +ALERT_SCRIPT_RELEVANT_BEGIN=1 +2:if [[ "${1:-}" == "health-alert" || "${1:-}" == "email-test" ]]; then +4: exec /usr/bin/python3 -I -B -c 'import base64,os;src=base64.b64decode("aW1wb3J0IHN5cyxvcyxqc29uLHJlLHN1YnByb2Nlc3MscGF0aGxpYix0aW1lCm1vZGU9c3lzLmFyZ3ZbMV0KQVVESVQ9Ii91c3IvbG9jYWwvc2Jpbi9ob21lbGFiLWJhY2t1cC1hdWRpdCIKSEVBTFRIPXBhdGhsaWIuUGF0aCgiL3Zhci9saWIvaG9tZWxhYi1oZWFsdGgvYmFja3VwLWF1ZGl0LXYyLmpzb24iKQpTVEFURT1wYXRobGliLlBhdGgoIi92YXIvbGliL2hvbWVsYWItYWxlcnRpbmcvYmFja3VwLWFsZXJ0LXN0YXRlLmpzb24iKQpVU0VSQ0ZHPXBhdGhsaWIuUGF0aCgiL2V0Yy9wdmUvdXNlci5jZmciKQpkZWYgcmVjaXBpZW50KCk6CiAgICB0ZXh0PVVTRVJDRkcucmVhZF90ZXh0KGVuY29kaW5nPSJ1dGYtOCIsZXJyb3JzPSJyZXBsYWNlIikKICAgIG09cmUuZmluZGFsbChyIig/aSlcYltBLVowLTkuXyUrLV0rQFtBLVowLTkuLV0rXC5bQS1aXXsyLH1cYiIsdGV4dCkKICAgIGlmIGxlbihtKSE9MTogcmFpc2UgU3lzdGVtRXhpdCg2NSkKICAgIHJldHVybiBtWzBdCmRlZiBzZW5kKHN1YmplY3QsYm9keSk6CiAgICByY3B0PXJlY2lwaWVudCgpCiAgICBtc2c9ZiJGcm9tOiBob21lbGFiLWFsZXJ0c0BncmFtMS5ydVxuVG86IHtyY3B0fVxuU3ViamVjdDoge3N1YmplY3R9XG5Db250ZW50LVR5cGU6IHRleHQvcGxhaW47IGNoYXJzZXQ9VVRGLThcblxue2JvZHl9XG4iCiAgICBwPXN1YnByb2Nlc3MucnVuKFsiL3Vzci9zYmluL3NlbmRtYWlsIiwiLXQiXSxpbnB1dD1tc2csdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKICAgIGlmIHAucmV0dXJuY29kZSE9MDogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCmRlZiBrZXkoaSk6CiAgICByZXR1cm4gInwiLmpvaW4oc3RyKGkuZ2V0KHgsIiIpKSBmb3IgeCBpbiAoImNvZGUiLCJsb2dpY2FsX2lkIiwicm9vdF9jYXVzZSIpKQppZiBtb2RlPT0iZW1haWwtdGVzdCI6CiAgICBzZW5kKCJXb3JrZXJzMiBiYWNrdXAgYWxlcnRpbmcgdGVzdCIsIldvcmtlcnMyIGJhY2t1cC9EUiBhbGVydGluZyByb3V0ZSB0ZXN0LiBJZiB5b3UgcmVjZWl2ZWQgdGhpcywgdGhlIGxvY2FsIE1UQSBhY2NlcHRlZCB0aGUgY29udHJvbGxlZCB0ZXN0IGdlbmVyYXRlZCBkdXJpbmcgYmFja3VwIGNsb3N1cmUuIikKICAgIHByaW50KCJFTUFJTF9URVNUX0FDQ0VQVEVEPVlFUyIpCiAgICByYWlzZSBTeXN0ZW1FeGl0KDApCmlmIG1vZGUhPSJoZWFsdGgtYWxlcnQiOiByYWlzZSBTeXN0ZW1FeGl0KDY0KQpwPXN1YnByb2Nlc3MucnVuKFtBVURJVCwid3JpdGUtaGVhbHRoIl0sdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKaWYgcC5yZXR1cm5jb2RlIG5vdCBpbiAoMCwxKTogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCnRyeTogaD1qc29uLmxvYWRzKEhFQUxUSC5yZWFkX3RleHQoZW5jb2Rpbmc9InV0Zi04IikpCmV4Y2VwdCBFeGNlcHRpb246IHJhaXNlIFN5c3RlbUV4aXQoNjkpCmlzc3Vlcz1oLmdldCgiaXNzdWVzIikgaWYgaXNpbnN0YW5jZShoLmdldCgiaXNzdWVzIiksbGlzdCkgZWxzZSBbXQpjdXI9e2tleShpKTppIGZvciBpIGluIGlzc3VlcyBpZiBpc2luc3RhbmNlKGksZGljdCl9Cm9sZD17fQppZiBTVEFURS5pc19maWxlKCk6CiAgICB0cnk6CiAgICAgICAgb2xkaj1qc29uLmxvYWRzKFNUQVRFLnJlYWRfdGV4dChlbmNvZGluZz0idXRmLTgiKSkKICAgICAgICBvbGQ9e3g6e30gZm9yIHggaW4gb2xkai5nZXQoImlzc3VlX2tleXMiLFtdKSBpZiBpc2luc3RhbmNlKHgsc3RyKX0KICAgIGV4Y2VwdCBFeGNlcHRpb246IG9sZD17fQpjaGFuZ2VkPXNldChjdXIpIT1zZXQob2xkKQppZiBjaGFuZ2VkOgogICAgaWYgY3VyOgogICAgICAgIGxpbmVzPVsiV29ya2VyczIgYmFja3VwL0RSIGFsZXJ0IHN0YXRlIGNoYW5nZWQuIiwiIl0KICAgICAgICBmb3IgayBpbiBzb3J0ZWQoY3VyKToKICAgICAgICAgICAgaT1jdXJba10KICAgICAgICAgICAgbGluZXMuYXBwZW5kKGYie2kuZ2V0KCdzZXZlcml0eScsJ1VOS05PV04nKX0ge2kuZ2V0KCdjb2RlJywnVU5LTk9XTicpfSBsb2dpY2FsX2lkPXtpLmdldCgnbG9naWNhbF9pZCcsJy0nKX0gcm9vdF9jYXVzZT17aS5nZXQoJ3Jvb3RfY2F1c2UnLCdVTktOT1dOJyl9IikKICAgICAgICBzZW5kKGYiW1dvcmtlcnMyIEJBQ0tVUCBBTEVSVF0ge2xlbihjdXIpfSBhY3RpdmUgaXNzdWUocykiLCJcbiIuam9pbihsaW5lcykpCiAgICAgICAgcHJpbnQoIkFMRVJUX0VNQUlMX0FDQ0VQVEVEPVlFUyIpCiAgICBlbGlmIG9sZDoKICAgICAgICBzZW5kKCJbV29ya2VyczIgQkFDS1VQIFJFQ09WRVJZXSBhbGwgaXNzdWVzIGNsZWFyZWQiLCJXb3JrZXJzMiBiYWNrdXAvRFIgYXVkaXQgaXMgR1JFRU4uIEFsbCBwcmV2aW91c2x5IGFjdGl2ZSBiYWNrdXAgaXNzdWVzIGFyZSBjbGVhcmVkLiIpCiAgICAgICAgcHJpbnQoIlJFQ09WRVJZX0VNQUlMX0FDQ0VQVEVEPVlFUyIpClNUQVRFLnBhcmVudC5ta2RpcihwYXJlbnRzPVRydWUsZXhpc3Rfb2s9VHJ1ZSkKdG1wPVNUQVRFLndpdGhfbmFtZShTVEFURS5uYW1lKyIudG1wIikKdG1wLndyaXRlX3RleHQoanNvbi5kdW1wcyh7InNjaGVtYSI6ImhvbWVsYWItYmFja3VwLWFsZXJ0LXN0YXRlLXYxIiwidXBkYXRlZF9lcG9jaCI6aW50KHRpbWUudGltZSgpKSwiaXNzdWVfa2V5cyI6c29ydGVkKGN1cil9LHNlcGFyYXRvcnM9KCIsIiwiOiIpLHNvcnRfa2V5cz1UcnVlKSsiXG4iLGVuY29kaW5nPSJ1dGYtOCIpCm9zLmNobW9kKHRtcCwwbzYwMCk7IG9zLnJlcGxhY2UodG1wLFNUQVRFKQpwcmludCgiSEVBTFRIX0FMRVJUX1NUQVRFPU9LIikK").decode("utf-8");exec(compile(src,"workers2-backup-alert","exec"))' "$MODE" +9:APP="alertmanager" +19:fail(){ echo "[ERROR] alertmanager backup failed; kept: $STAGE"; } +21:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$MON" "sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data" > "$STAGE/remote.tgz" +24:test -s "$STAGE/payload/opt/stacks/observability-lite/alertmanager/alertmanager.yml" +26:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+alertmanager-config-check" --arg config "ok" --argjson files "$FILES" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"monitoring:/opt/stacks/observability-lite/alertmanager\",\"monitoring:/opt/stacks/observability-lite/compose.yaml\",\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\"],config_check:\$config,file_count:\$files,note:\"Alertmanager configuration and runtime state in encrypted two-cloud backup\"}" > "$STAGE/metadata.json" +30:echo ALERTMANAGER_BACKUP_RUN=OK +ALERT_SCRIPT_RELEVANT_END=1 +ALERT_UNIT_STATE_BEGIN=1 +UNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success +UNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success +NEXT LEFT LAST PASSED UNIT ACTIVATES +Fri 2026-09-25 11:52:11 MSK 8min Fri 2026-09-25 11:37:00 MSK 6min ago homelab-alertmanager-backup-health.timer homelab-alertmanager-backup-health.service + +1 timers listed. +ALERT_UNIT_STATE_END=1 +ALERT_RECENT_JOURNAL_BEGIN=1 +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +ALERT_RECENT_JOURNAL_END=1 +CURRENT_HEALTH_BEGIN=1 +HEALTH_FILE=/var/lib/homelab-health/backup-audit-v2.json +HEALTH_STATUS=RED +HEALTH_ISSUE_COUNT=3 +HEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM +HEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK +HEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT +HEALTH_FILE=/var/lib/homelab-health/backup-audit.json +HEALTH_STATUS=RED +HEALTH_ISSUE_COUNT=3 +HEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM +HEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK +HEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT +CURRENT_HEALTH_END=1 +SOURCE_CANDIDATES_BEGIN=1 +SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-run SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006 +SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.timer SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a +SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.service SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d +SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a +SOURCE_CANDIDATES_END=1 +DECISION=RCA_COMPLETE_ALERT_NOISE_FILTER_CHANGE_READY +NEXT_GATE=CHANGE_ALERT_NOISE_FILTER_KEEP_CRITICAL_ALERTS +TASK_RESULT=PASS_HOMELAB_BACKUP_ALERT_NOISE_RCA +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_ALERT_NOISE_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index bdd084cd..05430ff2 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260925-SKLADCHIK-CHROMEUSA-STATE-RECON-73233R1", - "status": "FAIL", - "rc": 77, + "command_id": "SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1", + "status": "OK", + "rc": 0, "host": "pve01", "mode": "read-only", - "component": "skladchik-chromeusa-state-recon", - "started_at_utc": "2026-09-25T08:40:32Z", - "finished_at_utc": "2026-09-25T08:40:32Z", + "component": "homelab-backup-alert-noise-rca", + "started_at_utc": "2026-09-25T08:43:34Z", + "finished_at_utc": "2026-09-25T08:43:34Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "aa348612a48d2382aa654488b684f7622047b1805949cee46b5d7f1ca0fb556b", + "command_sha256": "6bf3ecfb4a7968445da820af52f396bb757d5a620f7ca04ca2a18ed4a1b9f524", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -21,7 +21,7 @@ "result_contract_valid": true, "result_contract_status": null, "result_contract_error": null, - "command_rc": 77, + "command_rc": 0, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "f77bdfefbd3afb070861e4a0707a9dc535787fe2bf6b2943b771c3373ad7f95a", - "sanitized_output_sha256": "f77bdfefbd3afb070861e4a0707a9dc535787fe2bf6b2943b771c3373ad7f95a", + "raw_evidence_sha256": "04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd", + "sanitized_output_sha256": "04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_SKLADCHIK_CHROMEUSA_STATE_RECON_BEGIN=1\nCOMMAND_ID=SUPPORT-260925-SKLADCHIK-CHROMEUSA-STATE-RECON-73233R1\nMODE=read-only\nCOMPONENT=skladchik-chromeusa-state-recon\nMUTATION_BOUNDARY=NONE;READ_ONLY_WINDOWS_CHROMEUSA_LAUNCH_STATE_RECONSTRUCTION;LOCAL_TEMP_EVIDENCE_ONLY;NO_VPS_PROBE;NO_AUTH_TO_VPS;NO_TUNNEL;NO_REGISTRY_WRITE;NO_TASK_WRITE;NO_SHORTCUT_WRITE;NO_RUNKEY_WRITE;NO_HOSTKEY_CACHE_WRITE;NO_PROXY_WRITE;NO_APP_CONFIG_WRITE;NO_SERVICE_ACTION;NO_NETBIRD_CHANGE;NO_ROUTE_CHANGE;NO_GIT_WRITE_EXCEPT_RUNNER_RESULT\nPREDECESSOR_EXACT=SUPPORT-260925-SKLADCHIK-PUTTYVPS73231-STAGE-RECONCILE-73232R1\nPREDECESSOR_DECISION=WINDOWS_PUTTY_NO_CURRENT_VPS_HOSTKEY_CACHE_ANCHOR\nPREDECESSOR_NEXT_GATE=WINDOWS_CHROMEUSA_STATE_RECONSTRUCTION_READ_ONLY\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nFAILURE=local_audit_not_ok\nDECISION=CHROMEUSA_STATE_RECONSTRUCTION_INFRASTRUCTURE_FAIL\nNEXT_GATE=READ_EXACT_73233_AND_FIX_ONLY_PROVEN_BLOCKER\nTASK_RESULT=FAIL_73233_INFRASTRUCTURE\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260925-SKLADCHIK-CHROMEUSA-STATE-RECON-73233R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "WORKERS2_BACKUP_ALERT_NOISE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1\nMODE=read-only\nCOMPONENT=homelab-backup-alert-noise-rca\nMUTATION_BOUNDARY=READ_ONLY;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_ALERT_POST;NO_MAIL;NO_BACKUP;NO_GIT_WRITE\nALERT_SCRIPT_SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006\nALERT_SCRIPT_BYTES=5857\nALERT_SERVICE_SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d\nALERT_TIMER_SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a\nALERT_SERVICE_BEGIN=1\n[Unit]\nDescription=Workers2 backup health alert via existing alertmanager backup integration\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-alertmanager-backup-run health-alert\nALERT_SERVICE_END=1\nALERT_TIMER_BEGIN=1\n[Unit]\nDescription=Workers2 backup health alert schedule\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=15min\nRandomizedDelaySec=30s\nPersistent=true\nUnit=homelab-alertmanager-backup-health.service\n\n[Install]\nWantedBy=timers.target\nALERT_TIMER_END=1\nALERT_SCRIPT_RELEVANT_BEGIN=1\n2:if [[ \"${1:-}\" == \"health-alert\" || \"${1:-}\" == \"email-test\" ]]; then\n4: exec /usr/bin/python3 -I -B -c 'import base64,os;src=base64.b64decode(\"aW1wb3J0IHN5cyxvcyxqc29uLHJlLHN1YnByb2Nlc3MscGF0aGxpYix0aW1lCm1vZGU9c3lzLmFyZ3ZbMV0KQVVESVQ9Ii91c3IvbG9jYWwvc2Jpbi9ob21lbGFiLWJhY2t1cC1hdWRpdCIKSEVBTFRIPXBhdGhsaWIuUGF0aCgiL3Zhci9saWIvaG9tZWxhYi1oZWFsdGgvYmFja3VwLWF1ZGl0LXYyLmpzb24iKQpTVEFURT1wYXRobGliLlBhdGgoIi92YXIvbGliL2hvbWVsYWItYWxlcnRpbmcvYmFja3VwLWFsZXJ0LXN0YXRlLmpzb24iKQpVU0VSQ0ZHPXBhdGhsaWIuUGF0aCgiL2V0Yy9wdmUvdXNlci5jZmciKQpkZWYgcmVjaXBpZW50KCk6CiAgICB0ZXh0PVVTRVJDRkcucmVhZF90ZXh0KGVuY29kaW5nPSJ1dGYtOCIsZXJyb3JzPSJyZXBsYWNlIikKICAgIG09cmUuZmluZGFsbChyIig/aSlcYltBLVowLTkuXyUrLV0rQFtBLVowLTkuLV0rXC5bQS1aXXsyLH1cYiIsdGV4dCkKICAgIGlmIGxlbihtKSE9MTogcmFpc2UgU3lzdGVtRXhpdCg2NSkKICAgIHJldHVybiBtWzBdCmRlZiBzZW5kKHN1YmplY3QsYm9keSk6CiAgICByY3B0PXJlY2lwaWVudCgpCiAgICBtc2c9ZiJGcm9tOiBob21lbGFiLWFsZXJ0c0BncmFtMS5ydVxuVG86IHtyY3B0fVxuU3ViamVjdDoge3N1YmplY3R9XG5Db250ZW50LVR5cGU6IHRleHQvcGxhaW47IGNoYXJzZXQ9VVRGLThcblxue2JvZHl9XG4iCiAgICBwPXN1YnByb2Nlc3MucnVuKFsiL3Vzci9zYmluL3NlbmRtYWlsIiwiLXQiXSxpbnB1dD1tc2csdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKICAgIGlmIHAucmV0dXJuY29kZSE9MDogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCmRlZiBrZXkoaSk6CiAgICByZXR1cm4gInwiLmpvaW4oc3RyKGkuZ2V0KHgsIiIpKSBmb3IgeCBpbiAoImNvZGUiLCJsb2dpY2FsX2lkIiwicm9vdF9jYXVzZSIpKQppZiBtb2RlPT0iZW1haWwtdGVzdCI6CiAgICBzZW5kKCJXb3JrZXJzMiBiYWNrdXAgYWxlcnRpbmcgdGVzdCIsIldvcmtlcnMyIGJhY2t1cC9EUiBhbGVydGluZyByb3V0ZSB0ZXN0LiBJZiB5b3UgcmVjZWl2ZWQgdGhpcywgdGhlIGxvY2FsIE1UQSBhY2NlcHRlZCB0aGUgY29udHJvbGxlZCB0ZXN0IGdlbmVyYXRlZCBkdXJpbmcgYmFja3VwIGNsb3N1cmUuIikKICAgIHByaW50KCJFTUFJTF9URVNUX0FDQ0VQVEVEPVlFUyIpCiAgICByYWlzZSBTeXN0ZW1FeGl0KDApCmlmIG1vZGUhPSJoZWFsdGgtYWxlcnQiOiByYWlzZSBTeXN0ZW1FeGl0KDY0KQpwPXN1YnByb2Nlc3MucnVuKFtBVURJVCwid3JpdGUtaGVhbHRoIl0sdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKaWYgcC5yZXR1cm5jb2RlIG5vdCBpbiAoMCwxKTogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCnRyeTogaD1qc29uLmxvYWRzKEhFQUxUSC5yZWFkX3RleHQoZW5jb2Rpbmc9InV0Zi04IikpCmV4Y2VwdCBFeGNlcHRpb246IHJhaXNlIFN5c3RlbUV4aXQoNjkpCmlzc3Vlcz1oLmdldCgiaXNzdWVzIikgaWYgaXNpbnN0YW5jZShoLmdldCgiaXNzdWVzIiksbGlzdCkgZWxzZSBbXQpjdXI9e2tleShpKTppIGZvciBpIGluIGlzc3VlcyBpZiBpc2luc3RhbmNlKGksZGljdCl9Cm9sZD17fQppZiBTVEFURS5pc19maWxlKCk6CiAgICB0cnk6CiAgICAgICAgb2xkaj1qc29uLmxvYWRzKFNUQVRFLnJlYWRfdGV4dChlbmNvZGluZz0idXRmLTgiKSkKICAgICAgICBvbGQ9e3g6e30gZm9yIHggaW4gb2xkai5nZXQoImlzc3VlX2tleXMiLFtdKSBpZiBpc2luc3RhbmNlKHgsc3RyKX0KICAgIGV4Y2VwdCBFeGNlcHRpb246IG9sZD17fQpjaGFuZ2VkPXNldChjdXIpIT1zZXQob2xkKQppZiBjaGFuZ2VkOgogICAgaWYgY3VyOgogICAgICAgIGxpbmVzPVsiV29ya2VyczIgYmFja3VwL0RSIGFsZXJ0IHN0YXRlIGNoYW5nZWQuIiwiIl0KICAgICAgICBmb3IgayBpbiBzb3J0ZWQoY3VyKToKICAgICAgICAgICAgaT1jdXJba10KICAgICAgICAgICAgbGluZXMuYXBwZW5kKGYie2kuZ2V0KCdzZXZlcml0eScsJ1VOS05PV04nKX0ge2kuZ2V0KCdjb2RlJywnVU5LTk9XTicpfSBsb2dpY2FsX2lkPXtpLmdldCgnbG9naWNhbF9pZCcsJy0nKX0gcm9vdF9jYXVzZT17aS5nZXQoJ3Jvb3RfY2F1c2UnLCdVTktOT1dOJyl9IikKICAgICAgICBzZW5kKGYiW1dvcmtlcnMyIEJBQ0tVUCBBTEVSVF0ge2xlbihjdXIpfSBhY3RpdmUgaXNzdWUocykiLCJcbiIuam9pbihsaW5lcykpCiAgICAgICAgcHJpbnQoIkFMRVJUX0VNQUlMX0FDQ0VQVEVEPVlFUyIpCiAgICBlbGlmIG9sZDoKICAgICAgICBzZW5kKCJbV29ya2VyczIgQkFDS1VQIFJFQ09WRVJZXSBhbGwgaXNzdWVzIGNsZWFyZWQiLCJXb3JrZXJzMiBiYWNrdXAvRFIgYXVkaXQgaXMgR1JFRU4uIEFsbCBwcmV2aW91c2x5IGFjdGl2ZSBiYWNrdXAgaXNzdWVzIGFyZSBjbGVhcmVkLiIpCiAgICAgICAgcHJpbnQoIlJFQ09WRVJZX0VNQUlMX0FDQ0VQVEVEPVlFUyIpClNUQVRFLnBhcmVudC5ta2RpcihwYXJlbnRzPVRydWUsZXhpc3Rfb2s9VHJ1ZSkKdG1wPVNUQVRFLndpdGhfbmFtZShTVEFURS5uYW1lKyIudG1wIikKdG1wLndyaXRlX3RleHQoanNvbi5kdW1wcyh7InNjaGVtYSI6ImhvbWVsYWItYmFja3VwLWFsZXJ0LXN0YXRlLXYxIiwidXBkYXRlZF9lcG9jaCI6aW50KHRpbWUudGltZSgpKSwiaXNzdWVfa2V5cyI6c29ydGVkKGN1cil9LHNlcGFyYXRvcnM9KCIsIiwiOiIpLHNvcnRfa2V5cz1UcnVlKSsiXG4iLGVuY29kaW5nPSJ1dGYtOCIpCm9zLmNobW9kKHRtcCwwbzYwMCk7IG9zLnJlcGxhY2UodG1wLFNUQVRFKQpwcmludCgiSEVBTFRIX0FMRVJUX1NUQVRFPU9LIikK\").decode(\"utf-8\");exec(compile(src,\"workers2-backup-alert\",\"exec\"))' \"$MODE\"\n9:APP=\"alertmanager\"\n19:fail(){ echo \"[ERROR] alertmanager backup failed; kept: $STAGE\"; }\n21:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$MON\" \"sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data\" > \"$STAGE/remote.tgz\"\n24:test -s \"$STAGE/payload/opt/stacks/observability-lite/alertmanager/alertmanager.yml\"\n26:jq -n --arg app \"$APP\" --arg host \"monitoring\" --arg engine \"files+alertmanager-config-check\" --arg config \"ok\" --argjson files \"$FILES\" \"{app:\\$app,engine:\\$engine,source_host:\\$host,source_paths:[\\\"monitoring:/opt/stacks/observability-lite/alertmanager\\\",\\\"monitoring:/opt/stacks/observability-lite/compose.yaml\\\",\\\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\\\"],config_check:\\$config,file_count:\\$files,note:\\\"Alertmanager configuration and runtime state in encrypted two-cloud backup\\\"}\" > \"$STAGE/metadata.json\"\n30:echo ALERTMANAGER_BACKUP_RUN=OK\nALERT_SCRIPT_RELEVANT_END=1\nALERT_UNIT_STATE_BEGIN=1\nUNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success\nUNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success\nNEXT LEFT LAST PASSED UNIT ACTIVATES\nFri 2026-09-25 11:52:11 MSK 8min Fri 2026-09-25 11:37:00 MSK 6min ago homelab-alertmanager-backup-health.timer homelab-alertmanager-backup-health.service\n\n1 timers listed.\nALERT_UNIT_STATE_END=1\nALERT_RECENT_JOURNAL_BEGIN=1\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nALERT_RECENT_JOURNAL_END=1\nCURRENT_HEALTH_BEGIN=1\nHEALTH_FILE=/var/lib/homelab-health/backup-audit-v2.json\nHEALTH_STATUS=RED\nHEALTH_ISSUE_COUNT=3\nHEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM\nHEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK\nHEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT\nHEALTH_FILE=/var/lib/homelab-health/backup-audit.json\nHEALTH_STATUS=RED\nHEALTH_ISSUE_COUNT=3\nHEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM\nHEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK\nHEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT\nCURRENT_HEALTH_END=1\nSOURCE_CANDIDATES_BEGIN=1\nSOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-run SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006\nSOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.timer SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a\nSOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.service SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d\nSOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a\nSOURCE_CANDIDATES_END=1\nDECISION=RCA_COMPLETE_ALERT_NOISE_FILTER_CHANGE_READY\nNEXT_GATE=CHANGE_ALERT_NOISE_FILTER_KEEP_CRITICAL_ALERTS\nTASK_RESULT=PASS_HOMELAB_BACKUP_ALERT_NOISE_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_ALERT_NOISE_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index a75b522b..de4dc396 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260925-SKLADCHIK-CHROMEUSA-STATE-RECON-73233R1 -STATUS=FAIL -RC=77 +COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1 +STATUS=OK +RC=0 HOST=pve01 MODE=read-only -COMPONENT=skladchik-chromeusa-state-recon +COMPONENT=homelab-backup-alert-noise-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=aa348612a48d2382aa654488b684f7622047b1805949cee46b5d7f1ca0fb556b +COMMAND_SHA256=6bf3ecfb4a7968445da820af52f396bb757d5a620f7ca04ca2a18ed4a1b9f524 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=77 +COMMAND_RC=0 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,25 +24,165 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=f77bdfefbd3afb070861e4a0707a9dc535787fe2bf6b2943b771c3373ad7f95a -SANITIZED_OUTPUT_SHA256=f77bdfefbd3afb070861e4a0707a9dc535787fe2bf6b2943b771c3373ad7f95a +RAW_EVIDENCE_SHA256=04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd +SANITIZED_OUTPUT_SHA256=04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd OUTPUT_BEGIN -WORKERS2_SKLADCHIK_CHROMEUSA_STATE_RECON_BEGIN=1 -COMMAND_ID=SUPPORT-260925-SKLADCHIK-CHROMEUSA-STATE-RECON-73233R1 +WORKERS2_BACKUP_ALERT_NOISE_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1 MODE=read-only -COMPONENT=skladchik-chromeusa-state-recon -MUTATION_BOUNDARY=NONE;READ_ONLY_WINDOWS_CHROMEUSA_LAUNCH_STATE_RECONSTRUCTION;LOCAL_TEMP_EVIDENCE_ONLY;NO_VPS_PROBE;NO_AUTH_TO_VPS;NO_TUNNEL;NO_REGISTRY_WRITE;NO_TASK_WRITE;NO_SHORTCUT_WRITE;NO_RUNKEY_WRITE;NO_HOSTKEY_CACHE_WRITE;NO_PROXY_WRITE;NO_APP_CONFIG_WRITE;NO_SERVICE_ACTION;NO_NETBIRD_CHANGE;NO_ROUTE_CHANGE;NO_GIT_WRITE_EXCEPT_RUNNER_RESULT -PREDECESSOR_EXACT=SUPPORT-260925-SKLADCHIK-PUTTYVPS73231-STAGE-RECONCILE-73232R1 -PREDECESSOR_DECISION=WINDOWS_PUTTY_NO_CURRENT_VPS_HOSTKEY_CACHE_ANCHOR -PREDECESSOR_NEXT_GATE=WINDOWS_CHROMEUSA_STATE_RECONSTRUCTION_READ_ONLY -REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 -ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -FAILURE=local_audit_not_ok -DECISION=CHROMEUSA_STATE_RECONSTRUCTION_INFRASTRUCTURE_FAIL -NEXT_GATE=READ_EXACT_73233_AND_FIX_ONLY_PROVEN_BLOCKER -TASK_RESULT=FAIL_73233_INFRASTRUCTURE -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-SKLADCHIK-CHROMEUSA-STATE-RECON-73233R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +COMPONENT=homelab-backup-alert-noise-rca +MUTATION_BOUNDARY=READ_ONLY;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_ALERT_POST;NO_MAIL;NO_BACKUP;NO_GIT_WRITE +ALERT_SCRIPT_SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006 +ALERT_SCRIPT_BYTES=5857 +ALERT_SERVICE_SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d +ALERT_TIMER_SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a +ALERT_SERVICE_BEGIN=1 +[Unit] +Description=Workers2 backup health alert via existing alertmanager backup integration +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/homelab-alertmanager-backup-run health-alert +ALERT_SERVICE_END=1 +ALERT_TIMER_BEGIN=1 +[Unit] +Description=Workers2 backup health alert schedule + +[Timer] +OnBootSec=5min +OnUnitActiveSec=15min +RandomizedDelaySec=30s +Persistent=true +Unit=homelab-alertmanager-backup-health.service + +[Install] +WantedBy=timers.target +ALERT_TIMER_END=1 +ALERT_SCRIPT_RELEVANT_BEGIN=1 +2:if [[ "${1:-}" == "health-alert" || "${1:-}" == "email-test" ]]; then +4: exec /usr/bin/python3 -I -B -c 'import base64,os;src=base64.b64decode("aW1wb3J0IHN5cyxvcyxqc29uLHJlLHN1YnByb2Nlc3MscGF0aGxpYix0aW1lCm1vZGU9c3lzLmFyZ3ZbMV0KQVVESVQ9Ii91c3IvbG9jYWwvc2Jpbi9ob21lbGFiLWJhY2t1cC1hdWRpdCIKSEVBTFRIPXBhdGhsaWIuUGF0aCgiL3Zhci9saWIvaG9tZWxhYi1oZWFsdGgvYmFja3VwLWF1ZGl0LXYyLmpzb24iKQpTVEFURT1wYXRobGliLlBhdGgoIi92YXIvbGliL2hvbWVsYWItYWxlcnRpbmcvYmFja3VwLWFsZXJ0LXN0YXRlLmpzb24iKQpVU0VSQ0ZHPXBhdGhsaWIuUGF0aCgiL2V0Yy9wdmUvdXNlci5jZmciKQpkZWYgcmVjaXBpZW50KCk6CiAgICB0ZXh0PVVTRVJDRkcucmVhZF90ZXh0KGVuY29kaW5nPSJ1dGYtOCIsZXJyb3JzPSJyZXBsYWNlIikKICAgIG09cmUuZmluZGFsbChyIig/aSlcYltBLVowLTkuXyUrLV0rQFtBLVowLTkuLV0rXC5bQS1aXXsyLH1cYiIsdGV4dCkKICAgIGlmIGxlbihtKSE9MTogcmFpc2UgU3lzdGVtRXhpdCg2NSkKICAgIHJldHVybiBtWzBdCmRlZiBzZW5kKHN1YmplY3QsYm9keSk6CiAgICByY3B0PXJlY2lwaWVudCgpCiAgICBtc2c9ZiJGcm9tOiBob21lbGFiLWFsZXJ0c0BncmFtMS5ydVxuVG86IHtyY3B0fVxuU3ViamVjdDoge3N1YmplY3R9XG5Db250ZW50LVR5cGU6IHRleHQvcGxhaW47IGNoYXJzZXQ9VVRGLThcblxue2JvZHl9XG4iCiAgICBwPXN1YnByb2Nlc3MucnVuKFsiL3Vzci9zYmluL3NlbmRtYWlsIiwiLXQiXSxpbnB1dD1tc2csdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKICAgIGlmIHAucmV0dXJuY29kZSE9MDogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCmRlZiBrZXkoaSk6CiAgICByZXR1cm4gInwiLmpvaW4oc3RyKGkuZ2V0KHgsIiIpKSBmb3IgeCBpbiAoImNvZGUiLCJsb2dpY2FsX2lkIiwicm9vdF9jYXVzZSIpKQppZiBtb2RlPT0iZW1haWwtdGVzdCI6CiAgICBzZW5kKCJXb3JrZXJzMiBiYWNrdXAgYWxlcnRpbmcgdGVzdCIsIldvcmtlcnMyIGJhY2t1cC9EUiBhbGVydGluZyByb3V0ZSB0ZXN0LiBJZiB5b3UgcmVjZWl2ZWQgdGhpcywgdGhlIGxvY2FsIE1UQSBhY2NlcHRlZCB0aGUgY29udHJvbGxlZCB0ZXN0IGdlbmVyYXRlZCBkdXJpbmcgYmFja3VwIGNsb3N1cmUuIikKICAgIHByaW50KCJFTUFJTF9URVNUX0FDQ0VQVEVEPVlFUyIpCiAgICByYWlzZSBTeXN0ZW1FeGl0KDApCmlmIG1vZGUhPSJoZWFsdGgtYWxlcnQiOiByYWlzZSBTeXN0ZW1FeGl0KDY0KQpwPXN1YnByb2Nlc3MucnVuKFtBVURJVCwid3JpdGUtaGVhbHRoIl0sdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKaWYgcC5yZXR1cm5jb2RlIG5vdCBpbiAoMCwxKTogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCnRyeTogaD1qc29uLmxvYWRzKEhFQUxUSC5yZWFkX3RleHQoZW5jb2Rpbmc9InV0Zi04IikpCmV4Y2VwdCBFeGNlcHRpb246IHJhaXNlIFN5c3RlbUV4aXQoNjkpCmlzc3Vlcz1oLmdldCgiaXNzdWVzIikgaWYgaXNpbnN0YW5jZShoLmdldCgiaXNzdWVzIiksbGlzdCkgZWxzZSBbXQpjdXI9e2tleShpKTppIGZvciBpIGluIGlzc3VlcyBpZiBpc2luc3RhbmNlKGksZGljdCl9Cm9sZD17fQppZiBTVEFURS5pc19maWxlKCk6CiAgICB0cnk6CiAgICAgICAgb2xkaj1qc29uLmxvYWRzKFNUQVRFLnJlYWRfdGV4dChlbmNvZGluZz0idXRmLTgiKSkKICAgICAgICBvbGQ9e3g6e30gZm9yIHggaW4gb2xkai5nZXQoImlzc3VlX2tleXMiLFtdKSBpZiBpc2luc3RhbmNlKHgsc3RyKX0KICAgIGV4Y2VwdCBFeGNlcHRpb246IG9sZD17fQpjaGFuZ2VkPXNldChjdXIpIT1zZXQob2xkKQppZiBjaGFuZ2VkOgogICAgaWYgY3VyOgogICAgICAgIGxpbmVzPVsiV29ya2VyczIgYmFja3VwL0RSIGFsZXJ0IHN0YXRlIGNoYW5nZWQuIiwiIl0KICAgICAgICBmb3IgayBpbiBzb3J0ZWQoY3VyKToKICAgICAgICAgICAgaT1jdXJba10KICAgICAgICAgICAgbGluZXMuYXBwZW5kKGYie2kuZ2V0KCdzZXZlcml0eScsJ1VOS05PV04nKX0ge2kuZ2V0KCdjb2RlJywnVU5LTk9XTicpfSBsb2dpY2FsX2lkPXtpLmdldCgnbG9naWNhbF9pZCcsJy0nKX0gcm9vdF9jYXVzZT17aS5nZXQoJ3Jvb3RfY2F1c2UnLCdVTktOT1dOJyl9IikKICAgICAgICBzZW5kKGYiW1dvcmtlcnMyIEJBQ0tVUCBBTEVSVF0ge2xlbihjdXIpfSBhY3RpdmUgaXNzdWUocykiLCJcbiIuam9pbihsaW5lcykpCiAgICAgICAgcHJpbnQoIkFMRVJUX0VNQUlMX0FDQ0VQVEVEPVlFUyIpCiAgICBlbGlmIG9sZDoKICAgICAgICBzZW5kKCJbV29ya2VyczIgQkFDS1VQIFJFQ09WRVJZXSBhbGwgaXNzdWVzIGNsZWFyZWQiLCJXb3JrZXJzMiBiYWNrdXAvRFIgYXVkaXQgaXMgR1JFRU4uIEFsbCBwcmV2aW91c2x5IGFjdGl2ZSBiYWNrdXAgaXNzdWVzIGFyZSBjbGVhcmVkLiIpCiAgICAgICAgcHJpbnQoIlJFQ09WRVJZX0VNQUlMX0FDQ0VQVEVEPVlFUyIpClNUQVRFLnBhcmVudC5ta2RpcihwYXJlbnRzPVRydWUsZXhpc3Rfb2s9VHJ1ZSkKdG1wPVNUQVRFLndpdGhfbmFtZShTVEFURS5uYW1lKyIudG1wIikKdG1wLndyaXRlX3RleHQoanNvbi5kdW1wcyh7InNjaGVtYSI6ImhvbWVsYWItYmFja3VwLWFsZXJ0LXN0YXRlLXYxIiwidXBkYXRlZF9lcG9jaCI6aW50KHRpbWUudGltZSgpKSwiaXNzdWVfa2V5cyI6c29ydGVkKGN1cil9LHNlcGFyYXRvcnM9KCIsIiwiOiIpLHNvcnRfa2V5cz1UcnVlKSsiXG4iLGVuY29kaW5nPSJ1dGYtOCIpCm9zLmNobW9kKHRtcCwwbzYwMCk7IG9zLnJlcGxhY2UodG1wLFNUQVRFKQpwcmludCgiSEVBTFRIX0FMRVJUX1NUQVRFPU9LIikK").decode("utf-8");exec(compile(src,"workers2-backup-alert","exec"))' "$MODE" +9:APP="alertmanager" +19:fail(){ echo "[ERROR] alertmanager backup failed; kept: $STAGE"; } +21:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$MON" "sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data" > "$STAGE/remote.tgz" +24:test -s "$STAGE/payload/opt/stacks/observability-lite/alertmanager/alertmanager.yml" +26:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+alertmanager-config-check" --arg config "ok" --argjson files "$FILES" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"monitoring:/opt/stacks/observability-lite/alertmanager\",\"monitoring:/opt/stacks/observability-lite/compose.yaml\",\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\"],config_check:\$config,file_count:\$files,note:\"Alertmanager configuration and runtime state in encrypted two-cloud backup\"}" > "$STAGE/metadata.json" +30:echo ALERTMANAGER_BACKUP_RUN=OK +ALERT_SCRIPT_RELEVANT_END=1 +ALERT_UNIT_STATE_BEGIN=1 +UNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success +UNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success +NEXT LEFT LAST PASSED UNIT ACTIVATES +Fri 2026-09-25 11:52:11 MSK 8min Fri 2026-09-25 11:37:00 MSK 6min ago homelab-alertmanager-backup-health.timer homelab-alertmanager-backup-health.service + +1 timers listed. +ALERT_UNIT_STATE_END=1 +ALERT_RECENT_JOURNAL_BEGIN=1 +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +ALERT_RECENT_JOURNAL_END=1 +CURRENT_HEALTH_BEGIN=1 +HEALTH_FILE=/var/lib/homelab-health/backup-audit-v2.json +HEALTH_STATUS=RED +HEALTH_ISSUE_COUNT=3 +HEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM +HEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK +HEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT +HEALTH_FILE=/var/lib/homelab-health/backup-audit.json +HEALTH_STATUS=RED +HEALTH_ISSUE_COUNT=3 +HEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM +HEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK +HEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT +CURRENT_HEALTH_END=1 +SOURCE_CANDIDATES_BEGIN=1 +SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-run SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006 +SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.timer SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a +SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.service SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d +SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a +SOURCE_CANDIDATES_END=1 +DECISION=RCA_COMPLETE_ALERT_NOISE_FILTER_CHANGE_READY +NEXT_GATE=CHANGE_ALERT_NOISE_FILTER_KEEP_CRITICAL_ALERTS +TASK_RESULT=PASS_HOMELAB_BACKUP_ALERT_NOISE_RCA +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_ALERT_NOISE_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END