runtime: publish SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1
This commit is contained in:
homelab-runtime-publisher
File diff suppressed because one or more lines are too long
@@ -0,0 +1,188 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-alert-noise-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=6bf3ecfb4a7968445da820af52f396bb757d5a620f7ca04ca2a18ed4a1b9f524
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=0
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd
|
||||
SANITIZED_OUTPUT_SHA256=04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_BACKUP_ALERT_NOISE_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-alert-noise-rca
|
||||
MUTATION_BOUNDARY=READ_ONLY;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_ALERT_POST;NO_MAIL;NO_BACKUP;NO_GIT_WRITE
|
||||
ALERT_SCRIPT_SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006
|
||||
ALERT_SCRIPT_BYTES=5857
|
||||
ALERT_SERVICE_SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d
|
||||
ALERT_TIMER_SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a
|
||||
ALERT_SERVICE_BEGIN=1
|
||||
[Unit]
|
||||
Description=Workers2 backup health alert via existing alertmanager backup integration
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/homelab-alertmanager-backup-run health-alert
|
||||
ALERT_SERVICE_END=1
|
||||
ALERT_TIMER_BEGIN=1
|
||||
[Unit]
|
||||
Description=Workers2 backup health alert schedule
|
||||
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec=15min
|
||||
RandomizedDelaySec=30s
|
||||
Persistent=true
|
||||
Unit=homelab-alertmanager-backup-health.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
ALERT_TIMER_END=1
|
||||
ALERT_SCRIPT_RELEVANT_BEGIN=1
|
||||
2:if [[ "${1:-}" == "health-alert" || "${1:-}" == "email-test" ]]; then
|
||||
4: exec /usr/bin/python3 -I -B -c 'import base64,os;src=base64.b64decode("aW1wb3J0IHN5cyxvcyxqc29uLHJlLHN1YnByb2Nlc3MscGF0aGxpYix0aW1lCm1vZGU9c3lzLmFyZ3ZbMV0KQVVESVQ9Ii91c3IvbG9jYWwvc2Jpbi9ob21lbGFiLWJhY2t1cC1hdWRpdCIKSEVBTFRIPXBhdGhsaWIuUGF0aCgiL3Zhci9saWIvaG9tZWxhYi1oZWFsdGgvYmFja3VwLWF1ZGl0LXYyLmpzb24iKQpTVEFURT1wYXRobGliLlBhdGgoIi92YXIvbGliL2hvbWVsYWItYWxlcnRpbmcvYmFja3VwLWFsZXJ0LXN0YXRlLmpzb24iKQpVU0VSQ0ZHPXBhdGhsaWIuUGF0aCgiL2V0Yy9wdmUvdXNlci5jZmciKQpkZWYgcmVjaXBpZW50KCk6CiAgICB0ZXh0PVVTRVJDRkcucmVhZF90ZXh0KGVuY29kaW5nPSJ1dGYtOCIsZXJyb3JzPSJyZXBsYWNlIikKICAgIG09cmUuZmluZGFsbChyIig/aSlcYltBLVowLTkuXyUrLV0rQFtBLVowLTkuLV0rXC5bQS1aXXsyLH1cYiIsdGV4dCkKICAgIGlmIGxlbihtKSE9MTogcmFpc2UgU3lzdGVtRXhpdCg2NSkKICAgIHJldHVybiBtWzBdCmRlZiBzZW5kKHN1YmplY3QsYm9keSk6CiAgICByY3B0PXJlY2lwaWVudCgpCiAgICBtc2c9ZiJGcm9tOiBob21lbGFiLWFsZXJ0c0BncmFtMS5ydVxuVG86IHtyY3B0fVxuU3ViamVjdDoge3N1YmplY3R9XG5Db250ZW50LVR5cGU6IHRleHQvcGxhaW47IGNoYXJzZXQ9VVRGLThcblxue2JvZHl9XG4iCiAgICBwPXN1YnByb2Nlc3MucnVuKFsiL3Vzci9zYmluL3NlbmRtYWlsIiwiLXQiXSxpbnB1dD1tc2csdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKICAgIGlmIHAucmV0dXJuY29kZSE9MDogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCmRlZiBrZXkoaSk6CiAgICByZXR1cm4gInwiLmpvaW4oc3RyKGkuZ2V0KHgsIiIpKSBmb3IgeCBpbiAoImNvZGUiLCJsb2dpY2FsX2lkIiwicm9vdF9jYXVzZSIpKQppZiBtb2RlPT0iZW1haWwtdGVzdCI6CiAgICBzZW5kKCJXb3JrZXJzMiBiYWNrdXAgYWxlcnRpbmcgdGVzdCIsIldvcmtlcnMyIGJhY2t1cC9EUiBhbGVydGluZyByb3V0ZSB0ZXN0LiBJZiB5b3UgcmVjZWl2ZWQgdGhpcywgdGhlIGxvY2FsIE1UQSBhY2NlcHRlZCB0aGUgY29udHJvbGxlZCB0ZXN0IGdlbmVyYXRlZCBkdXJpbmcgYmFja3VwIGNsb3N1cmUuIikKICAgIHByaW50KCJFTUFJTF9URVNUX0FDQ0VQVEVEPVlFUyIpCiAgICByYWlzZSBTeXN0ZW1FeGl0KDApCmlmIG1vZGUhPSJoZWFsdGgtYWxlcnQiOiByYWlzZSBTeXN0ZW1FeGl0KDY0KQpwPXN1YnByb2Nlc3MucnVuKFtBVURJVCwid3JpdGUtaGVhbHRoIl0sdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKaWYgcC5yZXR1cm5jb2RlIG5vdCBpbiAoMCwxKTogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCnRyeTogaD1qc29uLmxvYWRzKEhFQUxUSC5yZWFkX3RleHQoZW5jb2Rpbmc9InV0Zi04IikpCmV4Y2VwdCBFeGNlcHRpb246IHJhaXNlIFN5c3RlbUV4aXQoNjkpCmlzc3Vlcz1oLmdldCgiaXNzdWVzIikgaWYgaXNpbnN0YW5jZShoLmdldCgiaXNzdWVzIiksbGlzdCkgZWxzZSBbXQpjdXI9e2tleShpKTppIGZvciBpIGluIGlzc3VlcyBpZiBpc2luc3RhbmNlKGksZGljdCl9Cm9sZD17fQppZiBTVEFURS5pc19maWxlKCk6CiAgICB0cnk6CiAgICAgICAgb2xkaj1qc29uLmxvYWRzKFNUQVRFLnJlYWRfdGV4dChlbmNvZGluZz0idXRmLTgiKSkKICAgICAgICBvbGQ9e3g6e30gZm9yIHggaW4gb2xkai5nZXQoImlzc3VlX2tleXMiLFtdKSBpZiBpc2luc3RhbmNlKHgsc3RyKX0KICAgIGV4Y2VwdCBFeGNlcHRpb246IG9sZD17fQpjaGFuZ2VkPXNldChjdXIpIT1zZXQob2xkKQppZiBjaGFuZ2VkOgogICAgaWYgY3VyOgogICAgICAgIGxpbmVzPVsiV29ya2VyczIgYmFja3VwL0RSIGFsZXJ0IHN0YXRlIGNoYW5nZWQuIiwiIl0KICAgICAgICBmb3IgayBpbiBzb3J0ZWQoY3VyKToKICAgICAgICAgICAgaT1jdXJba10KICAgICAgICAgICAgbGluZXMuYXBwZW5kKGYie2kuZ2V0KCdzZXZlcml0eScsJ1VOS05PV04nKX0ge2kuZ2V0KCdjb2RlJywnVU5LTk9XTicpfSBsb2dpY2FsX2lkPXtpLmdldCgnbG9naWNhbF9pZCcsJy0nKX0gcm9vdF9jYXVzZT17aS5nZXQoJ3Jvb3RfY2F1c2UnLCdVTktOT1dOJyl9IikKICAgICAgICBzZW5kKGYiW1dvcmtlcnMyIEJBQ0tVUCBBTEVSVF0ge2xlbihjdXIpfSBhY3RpdmUgaXNzdWUocykiLCJcbiIuam9pbihsaW5lcykpCiAgICAgICAgcHJpbnQoIkFMRVJUX0VNQUlMX0FDQ0VQVEVEPVlFUyIpCiAgICBlbGlmIG9sZDoKICAgICAgICBzZW5kKCJbV29ya2VyczIgQkFDS1VQIFJFQ09WRVJZXSBhbGwgaXNzdWVzIGNsZWFyZWQiLCJXb3JrZXJzMiBiYWNrdXAvRFIgYXVkaXQgaXMgR1JFRU4uIEFsbCBwcmV2aW91c2x5IGFjdGl2ZSBiYWNrdXAgaXNzdWVzIGFyZSBjbGVhcmVkLiIpCiAgICAgICAgcHJpbnQoIlJFQ09WRVJZX0VNQUlMX0FDQ0VQVEVEPVlFUyIpClNUQVRFLnBhcmVudC5ta2RpcihwYXJlbnRzPVRydWUsZXhpc3Rfb2s9VHJ1ZSkKdG1wPVNUQVRFLndpdGhfbmFtZShTVEFURS5uYW1lKyIudG1wIikKdG1wLndyaXRlX3RleHQoanNvbi5kdW1wcyh7InNjaGVtYSI6ImhvbWVsYWItYmFja3VwLWFsZXJ0LXN0YXRlLXYxIiwidXBkYXRlZF9lcG9jaCI6aW50KHRpbWUudGltZSgpKSwiaXNzdWVfa2V5cyI6c29ydGVkKGN1cil9LHNlcGFyYXRvcnM9KCIsIiwiOiIpLHNvcnRfa2V5cz1UcnVlKSsiXG4iLGVuY29kaW5nPSJ1dGYtOCIpCm9zLmNobW9kKHRtcCwwbzYwMCk7IG9zLnJlcGxhY2UodG1wLFNUQVRFKQpwcmludCgiSEVBTFRIX0FMRVJUX1NUQVRFPU9LIikK").decode("utf-8");exec(compile(src,"workers2-backup-alert","exec"))' "$MODE"
|
||||
9:APP="alertmanager"
|
||||
19:fail(){ echo "[ERROR] alertmanager backup failed; kept: $STAGE"; }
|
||||
21:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$MON" "sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data" > "$STAGE/remote.tgz"
|
||||
24:test -s "$STAGE/payload/opt/stacks/observability-lite/alertmanager/alertmanager.yml"
|
||||
26:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+alertmanager-config-check" --arg config "ok" --argjson files "$FILES" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"monitoring:/opt/stacks/observability-lite/alertmanager\",\"monitoring:/opt/stacks/observability-lite/compose.yaml\",\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\"],config_check:\$config,file_count:\$files,note:\"Alertmanager configuration and runtime state in encrypted two-cloud backup\"}" > "$STAGE/metadata.json"
|
||||
30:echo ALERTMANAGER_BACKUP_RUN=OK
|
||||
ALERT_SCRIPT_RELEVANT_END=1
|
||||
ALERT_UNIT_STATE_BEGIN=1
|
||||
UNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success
|
||||
UNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success
|
||||
NEXT LEFT LAST PASSED UNIT ACTIVATES
|
||||
Fri 2026-09-25 11:52:11 MSK 8min Fri 2026-09-25 11:37:00 MSK 6min ago homelab-alertmanager-backup-health.timer homelab-alertmanager-backup-health.service
|
||||
|
||||
1 timers listed.
|
||||
ALERT_UNIT_STATE_END=1
|
||||
ALERT_RECENT_JOURNAL_BEGIN=1
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
ALERT_RECENT_JOURNAL_END=1
|
||||
CURRENT_HEALTH_BEGIN=1
|
||||
HEALTH_FILE=/var/lib/homelab-health/backup-audit-v2.json
|
||||
HEALTH_STATUS=RED
|
||||
HEALTH_ISSUE_COUNT=3
|
||||
HEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
HEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK
|
||||
HEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT
|
||||
HEALTH_FILE=/var/lib/homelab-health/backup-audit.json
|
||||
HEALTH_STATUS=RED
|
||||
HEALTH_ISSUE_COUNT=3
|
||||
HEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
HEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK
|
||||
HEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT
|
||||
CURRENT_HEALTH_END=1
|
||||
SOURCE_CANDIDATES_BEGIN=1
|
||||
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-run SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006
|
||||
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.timer SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a
|
||||
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.service SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d
|
||||
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a
|
||||
SOURCE_CANDIDATES_END=1
|
||||
DECISION=RCA_COMPLETE_ALERT_NOISE_FILTER_CHANGE_READY
|
||||
NEXT_GATE=CHANGE_ALERT_NOISE_FILTER_KEEP_CRITICAL_ALERTS
|
||||
TASK_RESULT=PASS_HOMELAB_BACKUP_ALERT_NOISE_RCA
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_ALERT_NOISE_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+11
-11
File diff suppressed because one or more lines are too long
+163
-23
@@ -1,22 +1,22 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260925-SKLADCHIK-CHROMEUSA-STATE-RECON-73233R1
|
||||
STATUS=FAIL
|
||||
RC=77
|
||||
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=skladchik-chromeusa-state-recon
|
||||
COMPONENT=homelab-backup-alert-noise-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=aa348612a48d2382aa654488b684f7622047b1805949cee46b5d7f1ca0fb556b
|
||||
COMMAND_SHA256=6bf3ecfb4a7968445da820af52f396bb757d5a620f7ca04ca2a18ed4a1b9f524
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=77
|
||||
COMMAND_RC=0
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
@@ -24,25 +24,165 @@ MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=f77bdfefbd3afb070861e4a0707a9dc535787fe2bf6b2943b771c3373ad7f95a
|
||||
SANITIZED_OUTPUT_SHA256=f77bdfefbd3afb070861e4a0707a9dc535787fe2bf6b2943b771c3373ad7f95a
|
||||
RAW_EVIDENCE_SHA256=04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd
|
||||
SANITIZED_OUTPUT_SHA256=04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_SKLADCHIK_CHROMEUSA_STATE_RECON_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260925-SKLADCHIK-CHROMEUSA-STATE-RECON-73233R1
|
||||
WORKERS2_BACKUP_ALERT_NOISE_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1
|
||||
MODE=read-only
|
||||
COMPONENT=skladchik-chromeusa-state-recon
|
||||
MUTATION_BOUNDARY=NONE;READ_ONLY_WINDOWS_CHROMEUSA_LAUNCH_STATE_RECONSTRUCTION;LOCAL_TEMP_EVIDENCE_ONLY;NO_VPS_PROBE;NO_AUTH_TO_VPS;NO_TUNNEL;NO_REGISTRY_WRITE;NO_TASK_WRITE;NO_SHORTCUT_WRITE;NO_RUNKEY_WRITE;NO_HOSTKEY_CACHE_WRITE;NO_PROXY_WRITE;NO_APP_CONFIG_WRITE;NO_SERVICE_ACTION;NO_NETBIRD_CHANGE;NO_ROUTE_CHANGE;NO_GIT_WRITE_EXCEPT_RUNNER_RESULT
|
||||
PREDECESSOR_EXACT=SUPPORT-260925-SKLADCHIK-PUTTYVPS73231-STAGE-RECONCILE-73232R1
|
||||
PREDECESSOR_DECISION=WINDOWS_PUTTY_NO_CURRENT_VPS_HOSTKEY_CACHE_ANCHOR
|
||||
PREDECESSOR_NEXT_GATE=WINDOWS_CHROMEUSA_STATE_RECONSTRUCTION_READ_ONLY
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||
FAILURE=local_audit_not_ok
|
||||
DECISION=CHROMEUSA_STATE_RECONSTRUCTION_INFRASTRUCTURE_FAIL
|
||||
NEXT_GATE=READ_EXACT_73233_AND_FIX_ONLY_PROVEN_BLOCKER
|
||||
TASK_RESULT=FAIL_73233_INFRASTRUCTURE
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-SKLADCHIK-CHROMEUSA-STATE-RECON-73233R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
COMPONENT=homelab-backup-alert-noise-rca
|
||||
MUTATION_BOUNDARY=READ_ONLY;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_ALERT_POST;NO_MAIL;NO_BACKUP;NO_GIT_WRITE
|
||||
ALERT_SCRIPT_SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006
|
||||
ALERT_SCRIPT_BYTES=5857
|
||||
ALERT_SERVICE_SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d
|
||||
ALERT_TIMER_SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a
|
||||
ALERT_SERVICE_BEGIN=1
|
||||
[Unit]
|
||||
Description=Workers2 backup health alert via existing alertmanager backup integration
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/homelab-alertmanager-backup-run health-alert
|
||||
ALERT_SERVICE_END=1
|
||||
ALERT_TIMER_BEGIN=1
|
||||
[Unit]
|
||||
Description=Workers2 backup health alert schedule
|
||||
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec=15min
|
||||
RandomizedDelaySec=30s
|
||||
Persistent=true
|
||||
Unit=homelab-alertmanager-backup-health.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
ALERT_TIMER_END=1
|
||||
ALERT_SCRIPT_RELEVANT_BEGIN=1
|
||||
2:if [[ "${1:-}" == "health-alert" || "${1:-}" == "email-test" ]]; then
|
||||
4: exec /usr/bin/python3 -I -B -c 'import base64,os;src=base64.b64decode("aW1wb3J0IHN5cyxvcyxqc29uLHJlLHN1YnByb2Nlc3MscGF0aGxpYix0aW1lCm1vZGU9c3lzLmFyZ3ZbMV0KQVVESVQ9Ii91c3IvbG9jYWwvc2Jpbi9ob21lbGFiLWJhY2t1cC1hdWRpdCIKSEVBTFRIPXBhdGhsaWIuUGF0aCgiL3Zhci9saWIvaG9tZWxhYi1oZWFsdGgvYmFja3VwLWF1ZGl0LXYyLmpzb24iKQpTVEFURT1wYXRobGliLlBhdGgoIi92YXIvbGliL2hvbWVsYWItYWxlcnRpbmcvYmFja3VwLWFsZXJ0LXN0YXRlLmpzb24iKQpVU0VSQ0ZHPXBhdGhsaWIuUGF0aCgiL2V0Yy9wdmUvdXNlci5jZmciKQpkZWYgcmVjaXBpZW50KCk6CiAgICB0ZXh0PVVTRVJDRkcucmVhZF90ZXh0KGVuY29kaW5nPSJ1dGYtOCIsZXJyb3JzPSJyZXBsYWNlIikKICAgIG09cmUuZmluZGFsbChyIig/aSlcYltBLVowLTkuXyUrLV0rQFtBLVowLTkuLV0rXC5bQS1aXXsyLH1cYiIsdGV4dCkKICAgIGlmIGxlbihtKSE9MTogcmFpc2UgU3lzdGVtRXhpdCg2NSkKICAgIHJldHVybiBtWzBdCmRlZiBzZW5kKHN1YmplY3QsYm9keSk6CiAgICByY3B0PXJlY2lwaWVudCgpCiAgICBtc2c9ZiJGcm9tOiBob21lbGFiLWFsZXJ0c0BncmFtMS5ydVxuVG86IHtyY3B0fVxuU3ViamVjdDoge3N1YmplY3R9XG5Db250ZW50LVR5cGU6IHRleHQvcGxhaW47IGNoYXJzZXQ9VVRGLThcblxue2JvZHl9XG4iCiAgICBwPXN1YnByb2Nlc3MucnVuKFsiL3Vzci9zYmluL3NlbmRtYWlsIiwiLXQiXSxpbnB1dD1tc2csdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKICAgIGlmIHAucmV0dXJuY29kZSE9MDogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCmRlZiBrZXkoaSk6CiAgICByZXR1cm4gInwiLmpvaW4oc3RyKGkuZ2V0KHgsIiIpKSBmb3IgeCBpbiAoImNvZGUiLCJsb2dpY2FsX2lkIiwicm9vdF9jYXVzZSIpKQppZiBtb2RlPT0iZW1haWwtdGVzdCI6CiAgICBzZW5kKCJXb3JrZXJzMiBiYWNrdXAgYWxlcnRpbmcgdGVzdCIsIldvcmtlcnMyIGJhY2t1cC9EUiBhbGVydGluZyByb3V0ZSB0ZXN0LiBJZiB5b3UgcmVjZWl2ZWQgdGhpcywgdGhlIGxvY2FsIE1UQSBhY2NlcHRlZCB0aGUgY29udHJvbGxlZCB0ZXN0IGdlbmVyYXRlZCBkdXJpbmcgYmFja3VwIGNsb3N1cmUuIikKICAgIHByaW50KCJFTUFJTF9URVNUX0FDQ0VQVEVEPVlFUyIpCiAgICByYWlzZSBTeXN0ZW1FeGl0KDApCmlmIG1vZGUhPSJoZWFsdGgtYWxlcnQiOiByYWlzZSBTeXN0ZW1FeGl0KDY0KQpwPXN1YnByb2Nlc3MucnVuKFtBVURJVCwid3JpdGUtaGVhbHRoIl0sdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKaWYgcC5yZXR1cm5jb2RlIG5vdCBpbiAoMCwxKTogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCnRyeTogaD1qc29uLmxvYWRzKEhFQUxUSC5yZWFkX3RleHQoZW5jb2Rpbmc9InV0Zi04IikpCmV4Y2VwdCBFeGNlcHRpb246IHJhaXNlIFN5c3RlbUV4aXQoNjkpCmlzc3Vlcz1oLmdldCgiaXNzdWVzIikgaWYgaXNpbnN0YW5jZShoLmdldCgiaXNzdWVzIiksbGlzdCkgZWxzZSBbXQpjdXI9e2tleShpKTppIGZvciBpIGluIGlzc3VlcyBpZiBpc2luc3RhbmNlKGksZGljdCl9Cm9sZD17fQppZiBTVEFURS5pc19maWxlKCk6CiAgICB0cnk6CiAgICAgICAgb2xkaj1qc29uLmxvYWRzKFNUQVRFLnJlYWRfdGV4dChlbmNvZGluZz0idXRmLTgiKSkKICAgICAgICBvbGQ9e3g6e30gZm9yIHggaW4gb2xkai5nZXQoImlzc3VlX2tleXMiLFtdKSBpZiBpc2luc3RhbmNlKHgsc3RyKX0KICAgIGV4Y2VwdCBFeGNlcHRpb246IG9sZD17fQpjaGFuZ2VkPXNldChjdXIpIT1zZXQob2xkKQppZiBjaGFuZ2VkOgogICAgaWYgY3VyOgogICAgICAgIGxpbmVzPVsiV29ya2VyczIgYmFja3VwL0RSIGFsZXJ0IHN0YXRlIGNoYW5nZWQuIiwiIl0KICAgICAgICBmb3IgayBpbiBzb3J0ZWQoY3VyKToKICAgICAgICAgICAgaT1jdXJba10KICAgICAgICAgICAgbGluZXMuYXBwZW5kKGYie2kuZ2V0KCdzZXZlcml0eScsJ1VOS05PV04nKX0ge2kuZ2V0KCdjb2RlJywnVU5LTk9XTicpfSBsb2dpY2FsX2lkPXtpLmdldCgnbG9naWNhbF9pZCcsJy0nKX0gcm9vdF9jYXVzZT17aS5nZXQoJ3Jvb3RfY2F1c2UnLCdVTktOT1dOJyl9IikKICAgICAgICBzZW5kKGYiW1dvcmtlcnMyIEJBQ0tVUCBBTEVSVF0ge2xlbihjdXIpfSBhY3RpdmUgaXNzdWUocykiLCJcbiIuam9pbihsaW5lcykpCiAgICAgICAgcHJpbnQoIkFMRVJUX0VNQUlMX0FDQ0VQVEVEPVlFUyIpCiAgICBlbGlmIG9sZDoKICAgICAgICBzZW5kKCJbV29ya2VyczIgQkFDS1VQIFJFQ09WRVJZXSBhbGwgaXNzdWVzIGNsZWFyZWQiLCJXb3JrZXJzMiBiYWNrdXAvRFIgYXVkaXQgaXMgR1JFRU4uIEFsbCBwcmV2aW91c2x5IGFjdGl2ZSBiYWNrdXAgaXNzdWVzIGFyZSBjbGVhcmVkLiIpCiAgICAgICAgcHJpbnQoIlJFQ09WRVJZX0VNQUlMX0FDQ0VQVEVEPVlFUyIpClNUQVRFLnBhcmVudC5ta2RpcihwYXJlbnRzPVRydWUsZXhpc3Rfb2s9VHJ1ZSkKdG1wPVNUQVRFLndpdGhfbmFtZShTVEFURS5uYW1lKyIudG1wIikKdG1wLndyaXRlX3RleHQoanNvbi5kdW1wcyh7InNjaGVtYSI6ImhvbWVsYWItYmFja3VwLWFsZXJ0LXN0YXRlLXYxIiwidXBkYXRlZF9lcG9jaCI6aW50KHRpbWUudGltZSgpKSwiaXNzdWVfa2V5cyI6c29ydGVkKGN1cil9LHNlcGFyYXRvcnM9KCIsIiwiOiIpLHNvcnRfa2V5cz1UcnVlKSsiXG4iLGVuY29kaW5nPSJ1dGYtOCIpCm9zLmNobW9kKHRtcCwwbzYwMCk7IG9zLnJlcGxhY2UodG1wLFNUQVRFKQpwcmludCgiSEVBTFRIX0FMRVJUX1NUQVRFPU9LIikK").decode("utf-8");exec(compile(src,"workers2-backup-alert","exec"))' "$MODE"
|
||||
9:APP="alertmanager"
|
||||
19:fail(){ echo "[ERROR] alertmanager backup failed; kept: $STAGE"; }
|
||||
21:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$MON" "sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data" > "$STAGE/remote.tgz"
|
||||
24:test -s "$STAGE/payload/opt/stacks/observability-lite/alertmanager/alertmanager.yml"
|
||||
26:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+alertmanager-config-check" --arg config "ok" --argjson files "$FILES" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"monitoring:/opt/stacks/observability-lite/alertmanager\",\"monitoring:/opt/stacks/observability-lite/compose.yaml\",\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\"],config_check:\$config,file_count:\$files,note:\"Alertmanager configuration and runtime state in encrypted two-cloud backup\"}" > "$STAGE/metadata.json"
|
||||
30:echo ALERTMANAGER_BACKUP_RUN=OK
|
||||
ALERT_SCRIPT_RELEVANT_END=1
|
||||
ALERT_UNIT_STATE_BEGIN=1
|
||||
UNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success
|
||||
UNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success
|
||||
NEXT LEFT LAST PASSED UNIT ACTIVATES
|
||||
Fri 2026-09-25 11:52:11 MSK 8min Fri 2026-09-25 11:37:00 MSK 6min ago homelab-alertmanager-backup-health.timer homelab-alertmanager-backup-health.service
|
||||
|
||||
1 timers listed.
|
||||
ALERT_UNIT_STATE_END=1
|
||||
ALERT_RECENT_JOURNAL_BEGIN=1
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
ALERT_RECENT_JOURNAL_END=1
|
||||
CURRENT_HEALTH_BEGIN=1
|
||||
HEALTH_FILE=/var/lib/homelab-health/backup-audit-v2.json
|
||||
HEALTH_STATUS=RED
|
||||
HEALTH_ISSUE_COUNT=3
|
||||
HEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
HEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK
|
||||
HEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT
|
||||
HEALTH_FILE=/var/lib/homelab-health/backup-audit.json
|
||||
HEALTH_STATUS=RED
|
||||
HEALTH_ISSUE_COUNT=3
|
||||
HEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM
|
||||
HEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK
|
||||
HEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT
|
||||
CURRENT_HEALTH_END=1
|
||||
SOURCE_CANDIDATES_BEGIN=1
|
||||
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-run SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006
|
||||
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.timer SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a
|
||||
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.service SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d
|
||||
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a
|
||||
SOURCE_CANDIDATES_END=1
|
||||
DECISION=RCA_COMPLETE_ALERT_NOISE_FILTER_CHANGE_READY
|
||||
NEXT_GATE=CHANGE_ALERT_NOISE_FILTER_KEEP_CRITICAL_ALERTS
|
||||
TASK_RESULT=PASS_HOMELAB_BACKUP_ALERT_NOISE_RCA
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_ALERT_NOISE_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user