CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-alert-noise-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=6bf3ecfb4a7968445da820af52f396bb757d5a620f7ca04ca2a18ed4a1b9f524
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd
SANITIZED_OUTPUT_SHA256=04c7046c9668bedc808178ae0954639abc14300274e53c34923a2c3a193f5cbd
OUTPUT_BEGIN
WORKERS2_BACKUP_ALERT_NOISE_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1
MODE=read-only
COMPONENT=homelab-backup-alert-noise-rca
MUTATION_BOUNDARY=READ_ONLY;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_ALERT_POST;NO_MAIL;NO_BACKUP;NO_GIT_WRITE
ALERT_SCRIPT_SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006
ALERT_SCRIPT_BYTES=5857
ALERT_SERVICE_SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d
ALERT_TIMER_SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a
ALERT_SERVICE_BEGIN=1
[Unit]
Description=Workers2 backup health alert via existing alertmanager backup integration
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-alertmanager-backup-run health-alert
ALERT_SERVICE_END=1
ALERT_TIMER_BEGIN=1
[Unit]
Description=Workers2 backup health alert schedule

[Timer]
OnBootSec=5min
OnUnitActiveSec=15min
RandomizedDelaySec=30s
Persistent=true
Unit=homelab-alertmanager-backup-health.service

[Install]
WantedBy=timers.target
ALERT_TIMER_END=1
ALERT_SCRIPT_RELEVANT_BEGIN=1
2:if [[ "${1:-}" == "health-alert" || "${1:-}" == "email-test" ]]; then
4:  exec /usr/bin/python3 -I -B -c 'import base64,os;src=base64.b64decode("aW1wb3J0IHN5cyxvcyxqc29uLHJlLHN1YnByb2Nlc3MscGF0aGxpYix0aW1lCm1vZGU9c3lzLmFyZ3ZbMV0KQVVESVQ9Ii91c3IvbG9jYWwvc2Jpbi9ob21lbGFiLWJhY2t1cC1hdWRpdCIKSEVBTFRIPXBhdGhsaWIuUGF0aCgiL3Zhci9saWIvaG9tZWxhYi1oZWFsdGgvYmFja3VwLWF1ZGl0LXYyLmpzb24iKQpTVEFURT1wYXRobGliLlBhdGgoIi92YXIvbGliL2hvbWVsYWItYWxlcnRpbmcvYmFja3VwLWFsZXJ0LXN0YXRlLmpzb24iKQpVU0VSQ0ZHPXBhdGhsaWIuUGF0aCgiL2V0Yy9wdmUvdXNlci5jZmciKQpkZWYgcmVjaXBpZW50KCk6CiAgICB0ZXh0PVVTRVJDRkcucmVhZF90ZXh0KGVuY29kaW5nPSJ1dGYtOCIsZXJyb3JzPSJyZXBsYWNlIikKICAgIG09cmUuZmluZGFsbChyIig/aSlcYltBLVowLTkuXyUrLV0rQFtBLVowLTkuLV0rXC5bQS1aXXsyLH1cYiIsdGV4dCkKICAgIGlmIGxlbihtKSE9MTogcmFpc2UgU3lzdGVtRXhpdCg2NSkKICAgIHJldHVybiBtWzBdCmRlZiBzZW5kKHN1YmplY3QsYm9keSk6CiAgICByY3B0PXJlY2lwaWVudCgpCiAgICBtc2c9ZiJGcm9tOiBob21lbGFiLWFsZXJ0c0BncmFtMS5ydVxuVG86IHtyY3B0fVxuU3ViamVjdDoge3N1YmplY3R9XG5Db250ZW50LVR5cGU6IHRleHQvcGxhaW47IGNoYXJzZXQ9VVRGLThcblxue2JvZHl9XG4iCiAgICBwPXN1YnByb2Nlc3MucnVuKFsiL3Vzci9zYmluL3NlbmRtYWlsIiwiLXQiXSxpbnB1dD1tc2csdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKICAgIGlmIHAucmV0dXJuY29kZSE9MDogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCmRlZiBrZXkoaSk6CiAgICByZXR1cm4gInwiLmpvaW4oc3RyKGkuZ2V0KHgsIiIpKSBmb3IgeCBpbiAoImNvZGUiLCJsb2dpY2FsX2lkIiwicm9vdF9jYXVzZSIpKQppZiBtb2RlPT0iZW1haWwtdGVzdCI6CiAgICBzZW5kKCJXb3JrZXJzMiBiYWNrdXAgYWxlcnRpbmcgdGVzdCIsIldvcmtlcnMyIGJhY2t1cC9EUiBhbGVydGluZyByb3V0ZSB0ZXN0LiBJZiB5b3UgcmVjZWl2ZWQgdGhpcywgdGhlIGxvY2FsIE1UQSBhY2NlcHRlZCB0aGUgY29udHJvbGxlZCB0ZXN0IGdlbmVyYXRlZCBkdXJpbmcgYmFja3VwIGNsb3N1cmUuIikKICAgIHByaW50KCJFTUFJTF9URVNUX0FDQ0VQVEVEPVlFUyIpCiAgICByYWlzZSBTeXN0ZW1FeGl0KDApCmlmIG1vZGUhPSJoZWFsdGgtYWxlcnQiOiByYWlzZSBTeXN0ZW1FeGl0KDY0KQpwPXN1YnByb2Nlc3MucnVuKFtBVURJVCwid3JpdGUtaGVhbHRoIl0sdGV4dD1UcnVlLHN0ZG91dD1zdWJwcm9jZXNzLlBJUEUsc3RkZXJyPXN1YnByb2Nlc3MuUElQRSkKaWYgcC5yZXR1cm5jb2RlIG5vdCBpbiAoMCwxKTogcmFpc2UgU3lzdGVtRXhpdChwLnJldHVybmNvZGUgb3IgNzUpCnRyeTogaD1qc29uLmxvYWRzKEhFQUxUSC5yZWFkX3RleHQoZW5jb2Rpbmc9InV0Zi04IikpCmV4Y2VwdCBFeGNlcHRpb246IHJhaXNlIFN5c3RlbUV4aXQoNjkpCmlzc3Vlcz1oLmdldCgiaXNzdWVzIikgaWYgaXNpbnN0YW5jZShoLmdldCgiaXNzdWVzIiksbGlzdCkgZWxzZSBbXQpjdXI9e2tleShpKTppIGZvciBpIGluIGlzc3VlcyBpZiBpc2luc3RhbmNlKGksZGljdCl9Cm9sZD17fQppZiBTVEFURS5pc19maWxlKCk6CiAgICB0cnk6CiAgICAgICAgb2xkaj1qc29uLmxvYWRzKFNUQVRFLnJlYWRfdGV4dChlbmNvZGluZz0idXRmLTgiKSkKICAgICAgICBvbGQ9e3g6e30gZm9yIHggaW4gb2xkai5nZXQoImlzc3VlX2tleXMiLFtdKSBpZiBpc2luc3RhbmNlKHgsc3RyKX0KICAgIGV4Y2VwdCBFeGNlcHRpb246IG9sZD17fQpjaGFuZ2VkPXNldChjdXIpIT1zZXQob2xkKQppZiBjaGFuZ2VkOgogICAgaWYgY3VyOgogICAgICAgIGxpbmVzPVsiV29ya2VyczIgYmFja3VwL0RSIGFsZXJ0IHN0YXRlIGNoYW5nZWQuIiwiIl0KICAgICAgICBmb3IgayBpbiBzb3J0ZWQoY3VyKToKICAgICAgICAgICAgaT1jdXJba10KICAgICAgICAgICAgbGluZXMuYXBwZW5kKGYie2kuZ2V0KCdzZXZlcml0eScsJ1VOS05PV04nKX0ge2kuZ2V0KCdjb2RlJywnVU5LTk9XTicpfSBsb2dpY2FsX2lkPXtpLmdldCgnbG9naWNhbF9pZCcsJy0nKX0gcm9vdF9jYXVzZT17aS5nZXQoJ3Jvb3RfY2F1c2UnLCdVTktOT1dOJyl9IikKICAgICAgICBzZW5kKGYiW1dvcmtlcnMyIEJBQ0tVUCBBTEVSVF0ge2xlbihjdXIpfSBhY3RpdmUgaXNzdWUocykiLCJcbiIuam9pbihsaW5lcykpCiAgICAgICAgcHJpbnQoIkFMRVJUX0VNQUlMX0FDQ0VQVEVEPVlFUyIpCiAgICBlbGlmIG9sZDoKICAgICAgICBzZW5kKCJbV29ya2VyczIgQkFDS1VQIFJFQ09WRVJZXSBhbGwgaXNzdWVzIGNsZWFyZWQiLCJXb3JrZXJzMiBiYWNrdXAvRFIgYXVkaXQgaXMgR1JFRU4uIEFsbCBwcmV2aW91c2x5IGFjdGl2ZSBiYWNrdXAgaXNzdWVzIGFyZSBjbGVhcmVkLiIpCiAgICAgICAgcHJpbnQoIlJFQ09WRVJZX0VNQUlMX0FDQ0VQVEVEPVlFUyIpClNUQVRFLnBhcmVudC5ta2RpcihwYXJlbnRzPVRydWUsZXhpc3Rfb2s9VHJ1ZSkKdG1wPVNUQVRFLndpdGhfbmFtZShTVEFURS5uYW1lKyIudG1wIikKdG1wLndyaXRlX3RleHQoanNvbi5kdW1wcyh7InNjaGVtYSI6ImhvbWVsYWItYmFja3VwLWFsZXJ0LXN0YXRlLXYxIiwidXBkYXRlZF9lcG9jaCI6aW50KHRpbWUudGltZSgpKSwiaXNzdWVfa2V5cyI6c29ydGVkKGN1cil9LHNlcGFyYXRvcnM9KCIsIiwiOiIpLHNvcnRfa2V5cz1UcnVlKSsiXG4iLGVuY29kaW5nPSJ1dGYtOCIpCm9zLmNobW9kKHRtcCwwbzYwMCk7IG9zLnJlcGxhY2UodG1wLFNUQVRFKQpwcmludCgiSEVBTFRIX0FMRVJUX1NUQVRFPU9LIikK").decode("utf-8");exec(compile(src,"workers2-backup-alert","exec"))' "$MODE"
9:APP="alertmanager"
19:fail(){ echo "[ERROR] alertmanager backup failed; kept: $STAGE"; }
21:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$MON" "sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data" > "$STAGE/remote.tgz"
24:test -s "$STAGE/payload/opt/stacks/observability-lite/alertmanager/alertmanager.yml"
26:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+alertmanager-config-check" --arg config "ok" --argjson files "$FILES" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"monitoring:/opt/stacks/observability-lite/alertmanager\",\"monitoring:/opt/stacks/observability-lite/compose.yaml\",\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\"],config_check:\$config,file_count:\$files,note:\"Alertmanager configuration and runtime state in encrypted two-cloud backup\"}" > "$STAGE/metadata.json"
30:echo ALERTMANAGER_BACKUP_RUN=OK
ALERT_SCRIPT_RELEVANT_END=1
ALERT_UNIT_STATE_BEGIN=1
UNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success
UNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success
NEXT                        LEFT LAST                          PASSED UNIT                                     ACTIVATES
Fri 2026-09-25 11:52:11 MSK 8min Fri 2026-09-25 11:37:00 MSK 6min ago homelab-alertmanager-backup-health.timer homelab-alertmanager-backup-health.service

1 timers listed.
ALERT_UNIT_STATE_END=1
ALERT_RECENT_JOURNAL_BEGIN=1
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
ALERT_EMAIL_ACCEPTED=YES
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
ALERT_EMAIL_ACCEPTED=YES
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
ALERT_EMAIL_ACCEPTED=YES
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
ALERT_EMAIL_ACCEPTED=YES
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
ALERT_EMAIL_ACCEPTED=YES
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
ALERT_EMAIL_ACCEPTED=YES
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
ALERT_EMAIL_ACCEPTED=YES
HEALTH_ALERT_STATE=OK
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
ALERT_RECENT_JOURNAL_END=1
CURRENT_HEALTH_BEGIN=1
HEALTH_FILE=/var/lib/homelab-health/backup-audit-v2.json
HEALTH_STATUS=RED
HEALTH_ISSUE_COUNT=3
HEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM
HEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK
HEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT
HEALTH_FILE=/var/lib/homelab-health/backup-audit.json
HEALTH_STATUS=RED
HEALTH_ISSUE_COUNT=3
HEALTH_ISSUE=HIGH|RPO_BREACH|vm110|NO_FRESH_V2_OR_SEALED_QUORUM
HEALTH_ISSUE=HIGH|LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK
HEALTH_ISSUE=MEDIUM|SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT
CURRENT_HEALTH_END=1
SOURCE_CANDIDATES_BEGIN=1
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-run SHA256=de7c46079a98da2f05bf9cd254d5b385bbd154487311fc79bf54fe1556086006
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.timer SHA256=c3bf285723b53949a1ad57a195255a097337db49ffe9c7c714aedd58772b903a
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-health.service SHA256=cd3660d31757d9225b939b7730d0fc96ff1f71317e3df4f0143acb0ffd5a229d
SOURCE_CANDIDATE=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a
SOURCE_CANDIDATES_END=1
DECISION=RCA_COMPLETE_ALERT_NOISE_FILTER_CHANGE_READY
NEXT_GATE=CHANGE_ALERT_NOISE_FILTER_KEEP_CRITICAL_ALERTS
TASK_RESULT=PASS_HOMELAB_BACKUP_ALERT_NOISE_RCA
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-ALERT-NOISE-RCA-1219R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
WORKERS2_BACKUP_ALERT_NOISE_RCA_END=1

OUTPUT_END
CHAT_OUTPUT_END
