homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-09-28 04:40:17 +00:00
parent b7b43bed23
commit 053dfd3eb8
4 changed files with 480 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1.json
+38
View File
File diff suppressed because one or more lines are too long
runtime/history/SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1.txt
+238
View File
@@ -0,0 +1,238 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-alert-noise-root-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=9ae99a8877a8c63075030326f98f8146f9c933c9cbcb7bd076b9bbcfeeb0e1a3
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=9c2e7b15aaf9e4270699137f0f21aa91d7e88a1ae44c347b734ba55e3bbe2b81
SANITIZED_OUTPUT_SHA256=9c2e7b15aaf9e4270699137f0f21aa91d7e88a1ae44c347b734ba55e3bbe2b81
OUTPUT_BEGIN
BACKUP_ALERT_NOISE_ROOT_RCA_BEGIN=true
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1
MODE=read-only
COMPONENT=homelab-backup-alert-noise-root-rca
MUTATION_BOUNDARY=READ_ONLY_SYSTEMD_CONTEXT_GIT_REMOTE_PROBES_AUDIT_ALL_SOURCE_DISCOVERY_AND_HISTORICAL_CORRELATION;NO_ALERT_EXECUTION;NO_MAIL;NO_BACKUP;NO_LOCK_ACQUIRE;NO_CONFIG_WRITE;NO_SCRIPT_WRITE;NO_SYSTEMD_ACTION;NO_TIMER_CHANGE;NO_HEALTH_WRITE;NO_GIT_WRITE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_NETBIRD_SKLADCHIK_CHANGE
=== AUTHORITY ===
HOST=pve01
CLUSTER_QUORATE=Yes
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
SOURCE_1237_STATUS=OK
SOURCE_1237_RC=0
SOURCE_1237_DECISION=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED
SOURCE_1237_NEXT_GATE=READ_EXACT_1237_AND_FIX_ONLY_PROVEN_ALERT_NOISE_CAUSE
=== LIVE_IDENTITIES ===
WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
ALERT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9
=== SOURCE_BRANCH_AUTHORITY ===
SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
SOURCE_WORKTREE_CLEAN=true
SOURCE_ORIGIN_SCHEME=https
SOURCE_ORIGIN_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9
SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
SOURCE_LOCAL_REMOTE_EXACT=true
=== SYSTEMD_UNIT_CONTEXT ===
SYSTEMD_SHOW_BEGIN=homelab-backup-audit.service
ExecStart={ path=/usr/local/sbin/homelab-backup-audit ; argv[]=/usr/local/sbin/homelab-backup-audit scheduled ; ignore_errors=no ; start_time=[Mon 2026-09-28 07:36:16 MSK] ; stop_time=[Mon 2026-09-28 07:36:20 MSK] ; pid=2104808 ; code=exited ; status=1 }
Environment=
WorkingDirectory=
RootDirectory=
User=root
Group=root
DynamicUser=no
PAMName=
SYSTEMD_SHOW_END=homelab-backup-audit.service
SYSTEMD_CAT_BEGIN=homelab-backup-audit.service
# /etc/systemd/system/homelab-backup-audit.service
[Unit]
Description=Homelab unified backup and DR audit
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-audit scheduled
User=root
Group=root
UMask=0077
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
SYSTEMD_CAT_END=homelab-backup-audit.service
SYSTEMD_SHOW_BEGIN=homelab-alertmanager-backup-health.service
ExecStart={ path=/usr/local/sbin/homelab-alertmanager-backup-run ; argv[]=/usr/local/sbin/homelab-alertmanager-backup-run health-alert ; ignore_errors=no ; start_time=[Mon 2026-09-28 07:31:56 MSK] ; stop_time=[Mon 2026-09-28 07:32:00 MSK] ; pid=2089737 ; code=exited ; status=0 }
Environment=
WorkingDirectory=
RootDirectory=
User=
Group=
DynamicUser=no
PAMName=
SYSTEMD_SHOW_END=homelab-alertmanager-backup-health.service
SYSTEMD_CAT_BEGIN=homelab-alertmanager-backup-health.service
# /etc/systemd/system/homelab-alertmanager-backup-health.service
[Unit]
Description=Workers2 backup health alert via existing alertmanager backup integration
After=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-alertmanager-backup-run health-alert
SYSTEMD_CAT_END=homelab-alertmanager-backup-health.service
SYSTEM_MANAGER_ENV_RELEVANT_BEGIN
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin
SYSTEM_MANAGER_ENV_RELEVANT_END
=== GIT_CONFIG_CONTEXT ===
GIT_CONFIG_MODE=minimal OUT_BYTES=0 ERR_BYTES=21 OUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ERR_SHA256=6a8a72c20480deb7191693468ae1348ea123ecd0c5ff9baf8693dd293e9112cb
GIT_CONFIG_MODE=nohome OUT_BYTES=0 ERR_BYTES=21 OUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ERR_SHA256=6a8a72c20480deb7191693468ae1348ea123ecd0c5ff9baf8693dd293e9112cb
GIT_CONFIG_MODE=homeroot OUT_BYTES=346 ERR_BYTES=0 OUT_SHA256=9a188d2d29e9701bac3d2fe0cc4f11283f36fca0a941b1b3aeaf10d3b47fdc7e ERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
=== REMOTE_CONTEXT_PROBES ===
REMOTE_PROBE=normal RC=0 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 EXACT=true STDERR_BYTES=0 STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
REMOTE_PROBE=minimal RC=128 HEAD=NONE EXACT=false STDERR_BYTES=85 STDERR_SHA256=adbd757d7820290ef0868baee0cb62cc4953c3831e45981d82cb0ed5faed79cc
REMOTE_PROBE_STDERR_BEGIN=minimal
fatal: could not read Username for 'https://git.gram1.ru': No such device or address
REMOTE_PROBE_STDERR_END=minimal
REMOTE_PROBE=nohome RC=128 HEAD=NONE EXACT=false STDERR_BYTES=85 STDERR_SHA256=adbd757d7820290ef0868baee0cb62cc4953c3831e45981d82cb0ed5faed79cc
REMOTE_PROBE_STDERR_BEGIN=nohome
fatal: could not read Username for 'https://git.gram1.ru': No such device or address
REMOTE_PROBE_STDERR_END=nohome
REMOTE_PROBE=homeroot RC=0 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 EXACT=true STDERR_BYTES=0 STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
=== AUDIT_ALL_CONTEXT_PROBES ===
AUDIT_CONTEXT=minimal RC=1 SOURCE_OFFHOST_ISSUES=1 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
AUDIT_CONTEXT_SUMMARY=minimal|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
AUDIT_CONTEXT=nohome RC=1 SOURCE_OFFHOST_ISSUES=1 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
AUDIT_CONTEXT_SUMMARY=nohome|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
AUDIT_CONTEXT=homeroot RC=1 SOURCE_OFFHOST_ISSUES=0 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
AUDIT_CONTEXT_SUMMARY=homeroot|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
AUDIT_CONTEXT=normal RC=1 SOURCE_OFFHOST_ISSUES=0 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
AUDIT_CONTEXT_SUMMARY=normal|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
AUDIT_ALL_CONTEXT_HEALTH_NO_WRITE=true
=== LOCK_OWNER_CORRELATION ===
CURRENT_BACKUP_ACTIVE=activating
CURRENT_BACKUP_MAINPID=1856481
CURRENT_BACKUP_RESULT=success
CURRENT_BACKUP_EXEC_STATUS=0
homelab-backup- 1856481 FLOCK WRITE /run/homelab-backup-heavy.lock
CURRENT_HEAVY_FUSER= 1856481
RAW1237_PATH=/var/lib/homelab-chat-run/raw/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1-20260928T043051Z.log
25:UNIT=homelab-backup-v2.service ACTIVE=activating ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=1856481
238:LOCK_FUSER=/run/homelab-backup-heavy.lock| 1856481
239:LOCK_FREE=/run/homelab-backup-heavy.lock|false
249:homelab-backup- 1856481 FLOCK WRITE /run/homelab-backup-heavy.lock
=== AUDIT_LOCK_LOGIC_STATIC ===
LOCK_LOOP_SOURCE_BEGIN
for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]:
p=pathlib.Path(lock)
if p.exists():
lrc,_,_=run(["flock","-n",str(p),"-c","true"],5)
if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
LOCK_LOOP_SOURCE_END
LOCK_LOOP_CALLS=issues.append,p.exists,pathlib.Path,run,str
=== ALERT_COUNTER_LOGIC_STATIC ===
ALERT_LOGIC|0028|if mode!="health-alert": raise SystemExit(64)
ALERT_LOGIC|0029|p=subprocess.run([AUDIT,"write-health"],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
ALERT_LOGIC|0030|if p.returncode not in (0,1): raise SystemExit(p.returncode or 75)
ALERT_LOGIC|0031|try:
ALERT_LOGIC|0032| h=json.loads(HEALTH.read_text(encoding="utf-8"))
ALERT_LOGIC|0033|except Exception:
ALERT_LOGIC|0034| raise SystemExit(69)
ALERT_LOGIC|0035|issues=h.get("issues") if isinstance(h.get("issues"),list) else []
ALERT_LOGIC|0036|cur={key(i):i for i in issues if isinstance(i,dict)}
ALERT_LOGIC|0037|codes=set(str(i.get("code","")) for i in cur.values())
ALERT_LOGIC|0038|oldj={}
ALERT_LOGIC|0039|if STATE.is_file():
ALERT_LOGIC|0040| try:
ALERT_LOGIC|0041| oldj=json.loads(STATE.read_text(encoding="utf-8"))
ALERT_LOGIC|0042| if not isinstance(oldj,dict): oldj={}
ALERT_LOGIC|0043| except Exception:
ALERT_LOGIC|0044| oldj={}
ALERT_LOGIC|0045|migrating=oldj.get("schema")!=SCHEMA
ALERT_LOGIC|0046|prev_counts=oldj.get("code_counts") if isinstance(oldj.get("code_counts"),dict) and not migrating else {}
ALERT_LOGIC|0047|counts={}
ALERT_LOGIC|0048|for code in codes:
ALERT_LOGIC|0049| try: prev=int(prev_counts.get(code,0))
ALERT_LOGIC|0050| except Exception: prev=0
ALERT_LOGIC|0051| counts[code]=prev+1
ALERT_LOGIC|0052|actionable={}
ALERT_LOGIC|0053|for k,i in cur.items():
ALERT_LOGIC|0054| code=str(i.get("code",""))
ALERT_LOGIC|0055| need=int(THRESHOLDS.get(code,1))
ALERT_LOGIC|0056| if counts.get(code,0)>=need:
ALERT_LOGIC|0057| actionable[k]=i
ALERT_LOGIC|0058|tokens=[REDACTED]
ALERT_LOGIC|0082|STATE.parent.mkdir(parents=True,exist_ok=True)
ALERT_LOGIC|0083|tmp=STATE.with_name(STATE.name+".tmp")
ALERT_LOGIC|0084|payload={
ALERT_LOGIC|0085| "schema":SCHEMA,
ALERT_LOGIC|0086| "updated_epoch":int(time.time()),
ALERT_LOGIC|0087| "issue_keys":sorted(actionable),
ALERT_LOGIC|0088| "notify_tokens":[REDACTED]
ALERT_LOGIC|0089| "raw_issue_keys":sorted(cur),
ALERT_LOGIC|0090| "code_counts":{k:int(v) for k,v in sorted(counts.items())},
ALERT_LOGIC|0091| "migration_silent":bool(migrating)
ALERT_LOGIC|0092|}
ALERT_LOGIC|0093|tmp.write_text(json.dumps(payload,separators=(",",":"),sort_keys=True)+"\n",encoding="utf-8")
ALERT_LOGIC|0094|os.chmod(tmp,0o600)
ALERT_LOGIC|0095|os.replace(tmp,STATE)
=== CANONICAL_SOURCE_CANDIDATES ===
SOURCE_CANDIDATE|name=homelab-backup-audit|exact=false|sha256=1474c9ce58bbc1b3ae7437261d1bdb923232926fb2943c7c830253f8b284724c|path=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-audit
SOURCE_CANDIDATE_TOTAL|name=homelab-backup-audit|count=1|exact_count=0
SOURCE_CANDIDATE|name=homelab-alertmanager-backup-run|exact=true|sha256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9|path=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-run
SOURCE_CANDIDATE|name=homelab-alertmanager-backup-run|exact=false|sha256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a|path=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run
SOURCE_CANDIDATE_TOTAL|name=homelab-alertmanager-backup-run|count=2|exact_count=1
=== SOURCE_RELEVANT_GREP ===
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:188:for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]:
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:192: if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:292:source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:299: lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:300: if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:301:if source["remote_head"]!=source["local_head"]:
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}")
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:14:HEAVY_LOCK=/run/homelab-backup-heavy.lock
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:424: exec 5>"$LOCK"; flock -n 5 || die GC_LOCK_BUSY 75
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:434: exec 5>"$LOCK"; flock -n 5 || { printf 'GC_SCHEDULED=SKIP|reason=GC_LOCK_BUSY\n'; return 0; }
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:444: if ! acquire_gc_locks; then write_health "$p1" true 0; printf 'GC_SCHEDULED=SKIP|reason=BACKUP_LOCK_BUSY|candidates=%s|open_transactions=%s\n' "$count" "$open_tx"; return 0; fi
changes/CR-2026-1005/runtime/homelab-backup-v2-app-scheduled:6:HEAVY_LOCK=/run/homelab-backup-heavy.lock
changes/CR-2026-1005/runtime/homelab-backup-v2-app-scheduled:19:exec 9>"$QUEUE_LOCK"; if ! flock -n 9; then printf 'APP_V2_SCHEDULED=SKIP|reason=APP_QUEUE_LOCK_BUSY\n'; exit 0; fi
changes/CR-2026-1005/runtime/homelab-backup-v2-app-scheduled:20:exec 8>"$HEAVY_LOCK"; if ! flock -n 8; then printf 'APP_V2_SCHEDULED=SKIP|reason=GLOBAL_HEAVY_LOCK_BUSY\n'; exit 0; fi
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-audit:159:flock -n 9 || die AUDIT_LOCK_BUSY 75
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-audit:422:for lock in /run/homelab-backup-heavy.lock /run/lock/homelab-vm-cloud-quorum-queue.lock /run/lock/homelab-app-cloud-quorum-queue.lock /run/lock/homelab-backup-dispatcher.lock; do
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:14:HEAVY_LOCK=/run/homelab-backup-heavy.lock
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:424: exec 5>"$LOCK"; flock -n 5 || die GC_LOCK_BUSY 75
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:434: exec 5>"$LOCK"; flock -n 5 || { printf 'GC_SCHEDULED=SKIP|reason=GC_LOCK_BUSY\n'; return 0; }
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:444: if ! acquire_gc_locks; then write_health "$p1" true 0; printf 'GC_SCHEDULED=SKIP|reason=BACKUP_LOCK_BUSY|candidates=%s|open_transactions=%s\n' "$count" "$open_tx"; return 0; fi
tasks/snikket-vm150-rebuild-v2/assets/vm150-runtime.py:60:BACKUP_LOCKS=['/run/lock/homelab-backup-dispatcher.lock','/run/lock/homelab-vm-cloud-quorum-queue.lock','/run/homelab-backup-heavy.lock']
tasks/snikket-vm150-rebuild-v2/assets/vm150-runtime.py:506: f.close(); die('LOCK_BUSY_'+pathlib.Path(q).name,75)
TASK_RESULT=PASS_BACKUP_ALERT_NOISE_ROOT_RCA_COLLECTED
DECISION=PASS_BACKUP_ALERT_NOISE_ROOT_RCA_COLLECTED
NEXT_GATE=READ_EXACT_1238_AND_APPLY_ONLY_PROVEN_ALERT_NOISE_FIX
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+8 -8
View File
File diff suppressed because one or more lines are too long
runtime/latest.txt
+196 -413
View File
@@ -1,15 +1,15 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-alert-spam-current-rca
COMPONENT=homelab-backup-alert-noise-root-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=e205fb5749bd8d2a466e79a56e8d9f3a66537e7dcbd1c635202ab6df607b736b
COMMAND_SHA256=9ae99a8877a8c63075030326f98f8146f9c933c9cbcb7bd076b9bbcfeeb0e1a3
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
@@ -24,432 +24,215 @@ MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=c64c77856d1b796c0414be59dd0089c9687ec505d26bb8b4111ab68ee967e5a2
SANITIZED_OUTPUT_SHA256=c64c77856d1b796c0414be59dd0089c9687ec505d26bb8b4111ab68ee967e5a2
RAW_EVIDENCE_SHA256=9c2e7b15aaf9e4270699137f0f21aa91d7e88a1ae44c347b734ba55e3bbe2b81
SANITIZED_OUTPUT_SHA256=9c2e7b15aaf9e4270699137f0f21aa91d7e88a1ae44c347b734ba55e3bbe2b81
OUTPUT_BEGIN
BACKUP_ALERT_SPAM_CURRENT_RCA_BEGIN=true
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1
BACKUP_ALERT_NOISE_ROOT_RCA_BEGIN=true
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1
MODE=read-only
COMPONENT=homelab-backup-alert-spam-current-rca
MUTATION_BOUNDARY=READ_ONLY_ALERT_HEALTH_STATE_SOURCE_JOURNAL_LOCK_AND_QUIET_AUDIT;NO_ALERT_EXECUTION;NO_MAIL;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_SYSTEMD_ACTION;NO_TIMER_CHANGE;NO_HEALTH_WRITE;NO_GIT_WRITE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_NETBIRD_SKLADCHIK_CHANGE
COMPONENT=homelab-backup-alert-noise-root-rca
MUTATION_BOUNDARY=READ_ONLY_SYSTEMD_CONTEXT_GIT_REMOTE_PROBES_AUDIT_ALL_SOURCE_DISCOVERY_AND_HISTORICAL_CORRELATION;NO_ALERT_EXECUTION;NO_MAIL;NO_BACKUP;NO_LOCK_ACQUIRE;NO_CONFIG_WRITE;NO_SCRIPT_WRITE;NO_SYSTEMD_ACTION;NO_TIMER_CHANGE;NO_HEALTH_WRITE;NO_GIT_WRITE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_NETBIRD_SKLADCHIK_CHANGE
=== AUTHORITY ===
HOST=pve01
CLUSTER_QUORATE=Yes
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
SOURCE_1236_STATUS=OK
SOURCE_1236_RC=0
SOURCE_1236_CHANGES_MADE=false
SOURCE_1236_DECISION=HOLD_1236_V2_SERVICE_LAST_RUN_NOT_SUCCESS
SOURCE_1236_NEXT_GATE=RECONCILE_POST_RECOVERY_SCHEDULER
SOURCE_1236_POST_RECOVERY_V2_FAILED_COUNT=0
SOURCE_1236_POST_RECOVERY_FRESH_HELPER_FAILED_COUNT=0
SOURCE_1236_POST_RECOVERY_SCHEDULE_SUCCESS_COUNT=7
SOURCE_1237_STATUS=OK
SOURCE_1237_RC=0
SOURCE_1237_DECISION=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED
SOURCE_1237_NEXT_GATE=READ_EXACT_1237_AND_FIX_ONLY_PROVEN_ALERT_NOISE_CAUSE
=== LIVE_IDENTITIES ===
WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
ALERT_SCRIPT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9
=== CURRENT_UNIT_STATE ===
UNIT=homelab-backup-v2.service ACTIVE=activating ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=1856481
UNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=
UNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code EXEC_MAIN_STATUS=1 MAINPID=0
UNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=
UNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=0
UNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=
=== CURRENT_HEALTH_FILES ===
HEALTH1_EXISTS=true
HEALTH1_SHA256=d89fb55b92f2b9908e2d2d8da3b9a0d0db6d2fe84c6e6d37bcec1a86e4384222
HEALTH1_JSON_OK=true
HEALTH1_SCHEMA=homelab-backup-audit-v3
HEALTH1_STATUS=RED
HEALTH1_GENERATED_AT_EPOCH=1790568997
HEALTH1_AGE_SECONDS=855
HEALTH1_ISSUE_COUNT=2
HEALTH1_ISSUE_1={"code":"LOCK_BUSY","logical_id":"/run/homelab-backup-heavy.lock","root_cause":"ACTIVE_OR_ORPHAN_LOCK","severity":"HIGH"}
HEALTH1_ISSUE_2={"code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT","severity":"MEDIUM"}
HEALTH1_SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
HEALTH1_SOURCE_REMOTE_HEAD=None
HEALTH1_SOURCE_PUSHURL_DISABLED=true
HEALTH1_SOURCE_EXACT=false
HEALTH2_EXISTS=true
HEALTH2_SHA256=d89fb55b92f2b9908e2d2d8da3b9a0d0db6d2fe84c6e6d37bcec1a86e4384222
HEALTH2_JSON_OK=true
HEALTH2_SCHEMA=homelab-backup-audit-v3
HEALTH2_STATUS=RED
HEALTH2_GENERATED_AT_EPOCH=1790568997
HEALTH2_AGE_SECONDS=855
HEALTH2_ISSUE_COUNT=2
HEALTH2_ISSUE_1={"code":"LOCK_BUSY","logical_id":"/run/homelab-backup-heavy.lock","root_cause":"ACTIVE_OR_ORPHAN_LOCK","severity":"HIGH"}
HEALTH2_ISSUE_2={"code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT","severity":"MEDIUM"}
HEALTH2_SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
HEALTH2_SOURCE_REMOTE_HEAD=None
HEALTH2_SOURCE_PUSHURL_DISABLED=true
HEALTH2_SOURCE_EXACT=false
HEALTH_FILES_IDENTICAL=true
HEALTH_EPOCH_VALID=true
=== CURRENT_ALERT_STATE ===
ALERT_STATE_EXISTS=true
ALERT_STATE_SHA256=493ce50563cb31030f2b16ff4ad722a7ac5ec211a77b2e89625491c057e8827d
ALERT_STATE_JSON_OK=true
ALERT_STATE_KEYS=code_counts,issue_keys,migration_silent,notify_tokens,raw_issue_keys,schema,updated_epoch
ALERT_STATE_SAFE_JSON={"code_counts":{"LOCK_BUSY":4,"SOURCE_OFFHOST_MISSING":263},"issue_keys":["LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK","SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT"],"migration_silent":false,"notify_tokens":"[REDACTED]","raw_issue_keys":["LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK","SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT"],"schema":"homelab-backup-alert-state-v2","updated_epoch":1790569001}
ALERT_STATE_SAFE_TRUNCATED=false
ALERT_STATE_FIELD_SCHEMA="homelab-backup-alert-state-v2"
ALERT_STATE_FIELD_UPDATED_EPOCH=1790569001
ALERT_STATE_FIELD_ISSUE_KEYS=["LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK","SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT"]
=== ALERT_FILTER_SOURCE ===
ALERT_EMBEDDED_SOURCE_FOUND=true
ALERT_EMBEDDED_SOURCE_SHA256=344306105083cd8bc64e19d451f7f66c17accc43a2f413538d2f9e2687b2540e
ALERT_EMBEDDED_SOURCE_BEGIN
0001:import sys,os,json,re,subprocess,pathlib,time
0002:mode=sys.argv[1]
0003:AUDIT="/usr/local/sbin/homelab-backup-audit"
0004:HEALTH=pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json")
0005:STATE=pathlib.Path("/var/lib/homelab-alerting/backup-alert-state.json")
0006:USERCFG=pathlib.Path("/etc/pve/user.cfg")
0007:SCHEMA="homelab-backup-alert-state-v2"
0008:THRESHOLDS={"RPO_BREACH":2,"SOURCE_OFFHOST_MISSING":2,"LOCK_BUSY":3}
0009:def recipient():
0010: text=USERCFG.read_text(encoding="utf-8",errors="replace")
0011: m=re.findall(r"(?i)\b[A-Z0-9._+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b",text)
0012: if len(m)!=1: raise SystemExit(65)
0013: return m[0]
0014:def send(subject,body):
0015: rcpt=recipient()
0016: msg=f"From: h***@gram1.ru\nTo: {rcpt}\nSubject: {subject}\nContent-Type: text/plain; charset=UTF-8\n\n{body}\n"
0017: p=subprocess.run(["/usr/sbin/sendmail","-t"],input=msg,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
0018: if p.returncode!=0: raise SystemExit(p.returncode or 75)
0019:def key(i):
0020: return "|".join(str(i.get(x,"")) for x in ("code","logical_id","root_cause"))
0021:def token(i):[REDACTED]
0022: code=str(i.get("code",""))
0023: return "group:"+code if code in THRESHOLDS else "issue:"+key(i)
0024:if mode=="email-test":
0025: send("Workers2 backup alerting test","Workers2 backup/DR alerting route test. If you received this, the local MTA accepted the controlled test generated during backup closure.")
0026: print("EMAIL_TEST_ACCEPTED=YES")
0027: raise SystemExit(0)
0028:if mode!="health-alert": raise SystemExit(64)
0029:p=subprocess.run([AUDIT,"write-health"],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
0030:if p.returncode not in (0,1): raise SystemExit(p.returncode or 75)
0031:try:
0032: h=json.loads(HEALTH.read_text(encoding="utf-8"))
0033:except Exception:
0034: raise SystemExit(69)
0035:issues=h.get("issues") if isinstance(h.get("issues"),list) else []
0036:cur={key(i):i for i in issues if isinstance(i,dict)}
0037:codes=set(str(i.get("code","")) for i in cur.values())
0038:oldj={}
0039:if STATE.is_file():
0040: try:
0041: oldj=json.loads(STATE.read_text(encoding="utf-8"))
0042: if not isinstance(oldj,dict): oldj={}
0043: except Exception:
0044: oldj={}
0045:migrating=oldj.get("schema")!=SCHEMA
0046:prev_counts=oldj.get("code_counts") if isinstance(oldj.get("code_counts"),dict) and not migrating else {}
0047:counts={}
0048:for code in codes:
0049: try: prev=int(prev_counts.get(code,0))
0050: except Exception: prev=0
0051: counts[code]=prev+1
0052:actionable={}
0053:for k,i in cur.items():
0054: code=str(i.get("code",""))
0055: need=int(THRESHOLDS.get(code,1))
0056: if counts.get(code,0)>=need:
0057: actionable[k]=i
0058:tokens=[REDACTED]
0059:old_tokens=[REDACTED]
0060:if not migrating:
0061: old_tokens=[REDACTED]
0062:added=[REDACTED]
0063:removed=[REDACTED]
0064:if added:
0065: lines=[
0066: "Workers2 backup/DR actionable alert state changed.",
0067: "",
0068: "Transient debounce policy: RPO_BREACH=2 checks; SOURCE_OFFHOST_MISSING=2 checks; LOCK_BUSY=3 checks; all other issue codes=immediate.",
0069: ""
0070: ]
0071: for k in sorted(actionable):
0072: i=actionable[k]
0073: lines.append(f"{i.get('severity','UNKNOWN')} {i.get('code','UNKNOWN')} logical_id={i.get('logical_id','-')} root_cause={i.get('root_cause','UNKNOWN')}")
0074: suppressed=len(cur)-len(actionable)
0075: if suppressed:
0076: lines.extend(["",f"Suppressed transient issue(s) still below threshold: {suppressed}"])
0077: send(f"[Workers2 BACKUP ALERT] {len(actionable)} actionable issue(s)","\n".join(lines))
0078: print("ALERT_EMAIL_ACCEPTED=YES")
0079:elif old_tokens and not tokens:[REDACTED]
0080: send("[Workers2 BACKUP RECOVERY] all actionable issues cleared","Workers2 backup/DR actionable audit state is clear. All previously notified actionable backup issues are cleared.")
0081: print("RECOVERY_EMAIL_ACCEPTED=YES")
0082:STATE.parent.mkdir(parents=True,exist_ok=True)
0083:tmp=STATE.with_name(STATE.name+".tmp")
0084:payload={
0085: "schema":SCHEMA,
0086: "updated_epoch":int(time.time()),
0087: "issue_keys":sorted(actionable),
0088: "notify_tokens":[REDACTED]
0089: "raw_issue_keys":sorted(cur),
0090: "code_counts":{k:int(v) for k,v in sorted(counts.items())},
0091: "migration_silent":bool(migrating)
0092:}
0093:tmp.write_text(json.dumps(payload,separators=(",",":"),sort_keys=True)+"\n",encoding="utf-8")
0094:os.chmod(tmp,0o600)
0095:os.replace(tmp,STATE)
0096:print("ALERT_FILTER_SCHEMA="+SCHEMA)
0097:print("ALERT_RAW_ISSUE_COUNT="+str(len(cur)))
0098:print("ALERT_ACTIONABLE_ISSUE_COUNT="+str(len(actionable)))
0099:print("ALERT_NOTIFY_TOKEN_COUNT=[REDACTED]
0100:print("ALERT_ADDED_TOKEN_COUNT=[REDACTED]
0101:print("ALERT_REMOVED_TOKEN_COUNT=[REDACTED]
0102:print("ALERT_SUPPRESSED_TRANSIENT_COUNT="+str(len(cur)-len(actionable)))
0103:print("ALERT_STATE_MIGRATION_SILENT="+("true" if migrating else "false"))
0104:print("HEALTH_ALERT_STATE=OK")
ALERT_EMBEDDED_SOURCE_END
=== AUDIT_SOURCE_RELEVANT ===
183- ("homelab-alertmanager-backup-health.timer","ALERT_TIMER_BAD"),
184-]:
185- ok,a,e=unit_ok(unit)
186- if not ok: issues.append({"severity":"HIGH","code":code,"logical_id":unit,"root_cause":"UNIT_"+a+"_"+e})
187-
188:for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]:
189- p=pathlib.Path(lock)
190- if p.exists():
191: lrc,_,_=run(["flock","-n",str(p),"-c","true"],5)
192: if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
193-
194-env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG)
195-grc,gout,_=run([GFSH,"validate"],20,env)
196-if grc!=0 or "GFS_POLICY=OK" not in gout:
197- issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"})
--
287-except Exception:
288- cloudb_ok=False
289-if not cloudb_ok:
290- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"})
291-
292:source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
293-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
294:if hrc==0: source["local_head"]=ho.strip()
295-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10)
296-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10)
297-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED")
298-if frc==0 and fo.strip():
299- lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
300: if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
301:if source["remote_head"]!=source["local_head"]:
302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
303-
304-status="GREEN" if not issues else "RED"
305-for t in targets:
306- print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None))
307-for i in issues:
308- print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i))
309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}")
310-health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")}
311-if MODE in ("write-health","scheduled"):
312- for hp in HEALTHS:
313- hp.parent.mkdir(parents=True,exist_ok=True)
314- tmp=hp.with_name(hp.name+".tmp")
=== LIVE_SOURCE_HEAD_RECHECK ===
ALERT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9
=== SOURCE_BRANCH_AUTHORITY ===
SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
SOURCE_WORKTREE_CLEAN=true
SOURCE_ORIGIN_SCHEME=https
SOURCE_ORIGIN_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9
SOURCE_REMOTE_ATTEMPT=1 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true
SOURCE_REMOTE_ATTEMPT=2 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true
SOURCE_REMOTE_ATTEMPT=3 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true
SOURCE_REMOTE_ATTEMPT=4 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true
SOURCE_REMOTE_ATTEMPT=5 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true
=== CURRENT_LOCK_OWNERSHIP ===
LOCK_META=/run/homelab-backup-heavy.lock|inode=3474|600|root:root|2026-09-28 03:27:07.425060217 +0300
LOCK_FUSER=/run/homelab-backup-heavy.lock| 1856481
LOCK_FREE=/run/homelab-backup-heavy.lock|false
LOCK_META=/run/lock/homelab-backup-dispatcher.lock|inode=9|600|root:root|2026-09-28 03:27:07.423060188 +0300
LOCK_FUSER=/run/lock/homelab-backup-dispatcher.lock|
LOCK_FREE=/run/lock/homelab-backup-dispatcher.lock|true
LOCK_META=/run/lock/homelab-vm-cloud-quorum-queue.lock|inode=2944|600|root:root|2026-09-28 03:27:07.424060203 +0300
LOCK_FUSER=/run/lock/homelab-vm-cloud-quorum-queue.lock|
LOCK_FREE=/run/lock/homelab-vm-cloud-quorum-queue.lock|true
LOCK_META=/run/lock/homelab-app-cloud-quorum-queue.lock|inode=15|600|root:root|2026-09-28 03:27:07.425060217 +0300
LOCK_FUSER=/run/lock/homelab-app-cloud-quorum-queue.lock|
LOCK_FREE=/run/lock/homelab-app-cloud-quorum-queue.lock|true
SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
SOURCE_LOCAL_REMOTE_EXACT=true
=== SYSTEMD_UNIT_CONTEXT ===
SYSTEMD_SHOW_BEGIN=homelab-backup-audit.service
ExecStart={ path=/usr/local/sbin/homelab-backup-audit ; argv[]=/usr/local/sbin/homelab-backup-audit scheduled ; ignore_errors=no ; start_time=[Mon 2026-09-28 07:36:16 MSK] ; stop_time=[Mon 2026-09-28 07:36:20 MSK] ; pid=2104808 ; code=exited ; status=1 }
Environment=
WorkingDirectory=
RootDirectory=
User=root
Group=root
DynamicUser=no
PAMName=
SYSTEMD_SHOW_END=homelab-backup-audit.service
SYSTEMD_CAT_BEGIN=homelab-backup-audit.service
# /etc/systemd/system/homelab-backup-audit.service
[Unit]
Description=Homelab unified backup and DR audit
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-audit scheduled
User=root
Group=root
UMask=0077
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
SYSTEMD_CAT_END=homelab-backup-audit.service
SYSTEMD_SHOW_BEGIN=homelab-alertmanager-backup-health.service
ExecStart={ path=/usr/local/sbin/homelab-alertmanager-backup-run ; argv[]=/usr/local/sbin/homelab-alertmanager-backup-run health-alert ; ignore_errors=no ; start_time=[Mon 2026-09-28 07:31:56 MSK] ; stop_time=[Mon 2026-09-28 07:32:00 MSK] ; pid=2089737 ; code=exited ; status=0 }
Environment=
WorkingDirectory=
RootDirectory=
User=
Group=
DynamicUser=no
PAMName=
SYSTEMD_SHOW_END=homelab-alertmanager-backup-health.service
SYSTEMD_CAT_BEGIN=homelab-alertmanager-backup-health.service
# /etc/systemd/system/homelab-alertmanager-backup-health.service
[Unit]
Description=Workers2 backup health alert via existing alertmanager backup integration
After=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-alertmanager-backup-run health-alert
SYSTEMD_CAT_END=homelab-alertmanager-backup-health.service
SYSTEM_MANAGER_ENV_RELEVANT_BEGIN
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin
SYSTEM_MANAGER_ENV_RELEVANT_END
=== GIT_CONFIG_CONTEXT ===
GIT_CONFIG_MODE=minimal OUT_BYTES=0 ERR_BYTES=21 OUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ERR_SHA256=6a8a72c20480deb7191693468ae1348ea123ecd0c5ff9baf8693dd293e9112cb
GIT_CONFIG_MODE=nohome OUT_BYTES=0 ERR_BYTES=21 OUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ERR_SHA256=6a8a72c20480deb7191693468ae1348ea123ecd0c5ff9baf8693dd293e9112cb
GIT_CONFIG_MODE=homeroot OUT_BYTES=346 ERR_BYTES=0 OUT_SHA256=9a188d2d29e9701bac3d2fe0cc4f11283f36fca0a941b1b3aeaf10d3b47fdc7e ERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
=== REMOTE_CONTEXT_PROBES ===
REMOTE_PROBE=normal RC=0 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 EXACT=true STDERR_BYTES=0 STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
REMOTE_PROBE=minimal RC=128 HEAD=NONE EXACT=false STDERR_BYTES=85 STDERR_SHA256=adbd757d7820290ef0868baee0cb62cc4953c3831e45981d82cb0ed5faed79cc
REMOTE_PROBE_STDERR_BEGIN=minimal
fatal: could not read Username for 'https://git.gram1.ru': No such device or address
REMOTE_PROBE_STDERR_END=minimal
REMOTE_PROBE=nohome RC=128 HEAD=NONE EXACT=false STDERR_BYTES=85 STDERR_SHA256=adbd757d7820290ef0868baee0cb62cc4953c3831e45981d82cb0ed5faed79cc
REMOTE_PROBE_STDERR_BEGIN=nohome
fatal: could not read Username for 'https://git.gram1.ru': No such device or address
REMOTE_PROBE_STDERR_END=nohome
REMOTE_PROBE=homeroot RC=0 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 EXACT=true STDERR_BYTES=0 STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
=== AUDIT_ALL_CONTEXT_PROBES ===
AUDIT_CONTEXT=minimal RC=1 SOURCE_OFFHOST_ISSUES=1 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
AUDIT_CONTEXT_SUMMARY=minimal|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
AUDIT_CONTEXT=nohome RC=1 SOURCE_OFFHOST_ISSUES=1 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
AUDIT_CONTEXT_SUMMARY=nohome|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
AUDIT_CONTEXT=homeroot RC=1 SOURCE_OFFHOST_ISSUES=0 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
AUDIT_CONTEXT_SUMMARY=homeroot|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
AUDIT_CONTEXT=normal RC=1 SOURCE_OFFHOST_ISSUES=0 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
AUDIT_CONTEXT_SUMMARY=normal|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
AUDIT_ALL_CONTEXT_HEALTH_NO_WRITE=true
=== LOCK_OWNER_CORRELATION ===
CURRENT_BACKUP_ACTIVE=activating
CURRENT_BACKUP_MAINPID=1856481
CURRENT_BACKUP_RESULT=success
CURRENT_BACKUP_EXEC_STATUS=0
homelab-backup- 1856481 FLOCK WRITE /run/homelab-backup-heavy.lock
=== ALERT_EMAIL_TIME_CORRELATION_0701_MSK ===
2026-09-28T06:19:01.361470+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...
2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T06:36:20.423980+03:00 pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
2026-09-28T06:36:20.424205+03:00 pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T07:06:48.304891+03:00 pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
2026-09-28T07:06:48.305137+03:00 pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
2026-09-28T06:15:16.319082+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
2026-09-28T06:15:20.661916+03:00 pve01 homelab-alertmanager-backup-run[1835343]: HEALTH_ALERT_STATE=OK
2026-09-28T06:15:20.666144+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
2026-09-28T06:30:42.414371+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
2026-09-28T06:30:46.367196+03:00 pve01 homelab-alertmanager-backup-run[1886013]: HEALTH_ALERT_STATE=OK
2026-09-28T06:30:46.370827+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
2026-09-28T06:46:01.347189+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
2026-09-28T06:46:05.406037+03:00 pve01 homelab-alertmanager-backup-run[1945314]: HEALTH_ALERT_STATE=OK
2026-09-28T06:46:05.410491+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
2026-09-28T07:01:32.217324+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: ALERT_EMAIL_ACCEPTED=YES
2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: HEALTH_ALERT_STATE=OK
2026-09-28T07:01:36.278152+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
2026-09-28T07:16:37.585569+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
2026-09-28T07:16:41.913014+03:00 pve01 homelab-alertmanager-backup-run[2048228]: HEALTH_ALERT_STATE=OK
2026-09-28T07:16:41.916768+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
=== ALERT_FREQUENCY_LAST12H ===
ALERT_RUN_START_COUNT=0
ALERT_EMAIL_ACCEPTED_COUNT=3
RECOVERY_EMAIL_ACCEPTED_COUNT=0
HEALTH_ALERT_STATE_OK_COUNT=47
2026-09-27T19:32:04.991437+03:00 pve01 homelab-alertmanager-backup-run[3909103]: HEALTH_ALERT_STATE=OK
2026-09-27T19:47:36.328545+03:00 pve01 homelab-alertmanager-backup-run[3959645]: HEALTH_ALERT_STATE=OK
2026-09-27T20:02:46.359055+03:00 pve01 homelab-alertmanager-backup-run[4008840]: HEALTH_ALERT_STATE=OK
2026-09-27T20:18:05.225521+03:00 pve01 homelab-alertmanager-backup-run[4059785]: HEALTH_ALERT_STATE=OK
2026-09-27T20:33:20.585967+03:00 pve01 homelab-alertmanager-backup-run[4108890]: HEALTH_ALERT_STATE=OK
2026-09-27T20:48:36.460451+03:00 pve01 homelab-alertmanager-backup-run[4148152]: HEALTH_ALERT_STATE=OK
2026-09-27T21:04:08.100470+03:00 pve01 homelab-alertmanager-backup-run[4070]: HEALTH_ALERT_STATE=OK
2026-09-27T21:19:36.336976+03:00 pve01 homelab-alertmanager-backup-run[54137]: HEALTH_ALERT_STATE=OK
2026-09-27T21:35:05.706232+03:00 pve01 homelab-alertmanager-backup-run[104752]: HEALTH_ALERT_STATE=OK
2026-09-27T21:50:23.015203+03:00 pve01 homelab-alertmanager-backup-run[154365]: HEALTH_ALERT_STATE=OK
2026-09-27T22:05:36.461278+03:00 pve01 homelab-alertmanager-backup-run[202960]: HEALTH_ALERT_STATE=OK
2026-09-27T22:21:05.253758+03:00 pve01 homelab-alertmanager-backup-run[242449]: HEALTH_ALERT_STATE=OK
2026-09-27T22:36:15.399283+03:00 pve01 homelab-alertmanager-backup-run[292331]: HEALTH_ALERT_STATE=OK
2026-09-27T22:51:31.819800+03:00 pve01 homelab-alertmanager-backup-run[343850]: HEALTH_ALERT_STATE=OK
2026-09-27T23:06:53.759023+03:00 pve01 homelab-alertmanager-backup-run[396329]: HEALTH_ALERT_STATE=OK
2026-09-27T23:22:20.677831+03:00 pve01 homelab-alertmanager-backup-run[461385]: HEALTH_ALERT_STATE=OK
2026-09-27T23:37:36.973241+03:00 pve01 homelab-alertmanager-backup-run[513251]: HEALTH_ALERT_STATE=OK
2026-09-27T23:52:43.434843+03:00 pve01 homelab-alertmanager-backup-run[566888]: ALERT_EMAIL_ACCEPTED=YES
2026-09-27T23:52:43.434843+03:00 pve01 homelab-alertmanager-backup-run[566888]: HEALTH_ALERT_STATE=OK
2026-09-28T00:08:05.576769+03:00 pve01 homelab-alertmanager-backup-run[607129]: HEALTH_ALERT_STATE=OK
2026-09-28T00:23:21.212626+03:00 pve01 homelab-alertmanager-backup-run[660884]: HEALTH_ALERT_STATE=OK
2026-09-28T00:38:36.269886+03:00 pve01 homelab-alertmanager-backup-run[711652]: HEALTH_ALERT_STATE=OK
2026-09-28T00:53:55.974540+03:00 pve01 homelab-alertmanager-backup-run[762988]: HEALTH_ALERT_STATE=OK
2026-09-28T01:09:05.464301+03:00 pve01 homelab-alertmanager-backup-run[812499]: HEALTH_ALERT_STATE=OK
2026-09-28T01:24:16.127247+03:00 pve01 homelab-alertmanager-backup-run[862903]: HEALTH_ALERT_STATE=OK
2026-09-28T01:39:41.256513+03:00 pve01 homelab-alertmanager-backup-run[919688]: HEALTH_ALERT_STATE=OK
2026-09-28T01:54:54.298806+03:00 pve01 homelab-alertmanager-backup-run[958250]: HEALTH_ALERT_STATE=OK
2026-09-28T02:10:05.486220+03:00 pve01 homelab-alertmanager-backup-run[1009352]: HEALTH_ALERT_STATE=OK
2026-09-28T02:25:22.594856+03:00 pve01 homelab-alertmanager-backup-run[1058040]: HEALTH_ALERT_STATE=OK
2026-09-28T02:40:53.144072+03:00 pve01 homelab-alertmanager-backup-run[1108677]: HEALTH_ALERT_STATE=OK
2026-09-28T02:56:20.122916+03:00 pve01 homelab-alertmanager-backup-run[1157287]: HEALTH_ALERT_STATE=OK
2026-09-28T03:11:36.224287+03:00 pve01 homelab-alertmanager-backup-run[1207541]: HEALTH_ALERT_STATE=OK
2026-09-28T03:26:56.104164+03:00 pve01 homelab-alertmanager-backup-run[1267608]: HEALTH_ALERT_STATE=OK
2026-09-28T03:42:16.163161+03:00 pve01 homelab-alertmanager-backup-run[1337091]: HEALTH_ALERT_STATE=OK
2026-09-28T03:57:35.911704+03:00 pve01 homelab-alertmanager-backup-run[1388112]: HEALTH_ALERT_STATE=OK
2026-09-28T04:12:45.302638+03:00 pve01 homelab-alertmanager-backup-run[1439953]: ALERT_EMAIL_ACCEPTED=YES
2026-09-28T04:12:45.302638+03:00 pve01 homelab-alertmanager-backup-run[1439953]: HEALTH_ALERT_STATE=OK
2026-09-28T04:28:16.066840+03:00 pve01 homelab-alertmanager-backup-run[1491056]: HEALTH_ALERT_STATE=OK
2026-09-28T04:43:26.035041+03:00 pve01 homelab-alertmanager-backup-run[1543022]: HEALTH_ALERT_STATE=OK
2026-09-28T04:58:29.541024+03:00 pve01 homelab-alertmanager-backup-run[1593789]: HEALTH_ALERT_STATE=OK
2026-09-28T05:13:39.334685+03:00 pve01 homelab-alertmanager-backup-run[1645768]: HEALTH_ALERT_STATE=OK
2026-09-28T05:28:53.353807+03:00 pve01 homelab-alertmanager-backup-run[1685761]: HEALTH_ALERT_STATE=OK
2026-09-28T05:44:20.155523+03:00 pve01 homelab-alertmanager-backup-run[1736193]: HEALTH_ALERT_STATE=OK
2026-09-28T06:00:05.422013+03:00 pve01 homelab-alertmanager-backup-run[1785685]: HEALTH_ALERT_STATE=OK
2026-09-28T06:15:20.661916+03:00 pve01 homelab-alertmanager-backup-run[1835343]: HEALTH_ALERT_STATE=OK
2026-09-28T06:30:46.367196+03:00 pve01 homelab-alertmanager-backup-run[1886013]: HEALTH_ALERT_STATE=OK
2026-09-28T06:46:05.406037+03:00 pve01 homelab-alertmanager-backup-run[1945314]: HEALTH_ALERT_STATE=OK
2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: ALERT_EMAIL_ACCEPTED=YES
2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: HEALTH_ALERT_STATE=OK
2026-09-28T07:16:41.913014+03:00 pve01 homelab-alertmanager-backup-run[2048228]: HEALTH_ALERT_STATE=OK
=== AUDIT_FREQUENCY_LAST12H ===
AUDIT_GREEN_SUMMARY_COUNT=7
AUDIT_RED_SUMMARY_COUNT=17
AUDIT_LOCK_BUSY_ISSUE_COUNT=9
AUDIT_SOURCE_OFFHOST_ISSUE_COUNT=0
2026-09-27T19:37:02.093711+03:00 pve01 homelab-backup-audit[3923461]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-27T20:06:15.522779+03:00 pve01 homelab-backup-audit[4022565]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-27T20:36:05.450336+03:00 pve01 homelab-backup-audit[4111037]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-27T21:06:02.094425+03:00 pve01 homelab-backup-audit[16838]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-27T21:37:05.002224+03:00 pve01 homelab-backup-audit[106421]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-27T22:06:07.004513+03:00 pve01 homelab-backup-audit[203921]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-27T22:35:26.212759+03:00 pve01 homelab-backup-audit[291167]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-27T23:06:54.987894+03:00 pve01 homelab-backup-audit[395212]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-27T23:36:37.162587+03:00 pve01 homelab-backup-audit[500802]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-27T23:36:37.162587+03:00 pve01 homelab-backup-audit[500802]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=6|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T00:06:21.135619+03:00 pve01 homelab-backup-audit[605334]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-28T00:06:21.135619+03:00 pve01 homelab-backup-audit[605334]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T00:36:10.445081+03:00 pve01 homelab-backup-audit[707255]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-28T00:36:10.445081+03:00 pve01 homelab-backup-audit[707255]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T01:06:01.997372+03:00 pve01 homelab-backup-audit[799187]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T01:36:02.006941+03:00 pve01 homelab-backup-audit[906059]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T02:05:36.356288+03:00 pve01 homelab-backup-audit[995508]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T02:36:15.265015+03:00 pve01 homelab-backup-audit[1094502]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T03:05:20.453132+03:00 pve01 homelab-backup-audit[1192518]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T03:35:20.147380+03:00 pve01 homelab-backup-audit[1320691]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-28T03:35:20.147380+03:00 pve01 homelab-backup-audit[1320691]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T04:06:45.928910+03:00 pve01 homelab-backup-audit[1424254]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-28T04:06:45.928910+03:00 pve01 homelab-backup-audit[1424254]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T04:35:20.240415+03:00 pve01 homelab-backup-audit[1514892]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-28T04:35:20.240415+03:00 pve01 homelab-backup-audit[1514892]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T05:05:52.974415+03:00 pve01 homelab-backup-audit[1617767]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-28T05:05:52.974415+03:00 pve01 homelab-backup-audit[1617767]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T05:37:02.145613+03:00 pve01 homelab-backup-audit[1720751]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T06:06:02.032037+03:00 pve01 homelab-backup-audit[1807928]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
=== HEALTH_GENERATION_CORRELATION ===
HEALTH_GENERATED_AT_EPOCH=1790568997
HEALTH_CORRELATION_FROM=1790568817
HEALTH_CORRELATION_TO=1790569177
=== FRESH_QUIET_AUDIT ===
QUIET_AUDIT_WAIT|attempt=1|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=2|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=3|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=4|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=5|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=6|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=7|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=8|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=9|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=10|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=11|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=12|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=13|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=14|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=15|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=16|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=17|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=18|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=19|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=20|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=21|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=22|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=23|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=24|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=25|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=26|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=27|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=28|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=29|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=30|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=31|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=32|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=33|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=34|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=35|reason=SERVICE_BUSY|active=activating|pid=1856481
QUIET_AUDIT_WAIT|attempt=36|reason=SERVICE_BUSY|active=activating|pid=1856481
FRESH_QUIET_AUDIT_RC=NO_QUIET_WINDOW
FRESH_AUDIT_HEALTH_NO_WRITE=false
=== CANONICAL_ALERT_SOURCE_AUTHORITY ===
CANONICAL_ALERT_PATH=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run
CANONICAL_ALERT_EXISTS=true
CANONICAL_ALERT_SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a
CANONICAL_ALERT_MATCH_LIVE=false
CANONICAL_SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
CANONICAL_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
CANONICAL_SOURCE_EXACT=true
TASK_RESULT=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED
DECISION=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED
NEXT_GATE=READ_EXACT_1237_AND_FIX_ONLY_PROVEN_ALERT_NOISE_CAUSE
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
CURRENT_HEAVY_FUSER= 1856481
RAW1237_PATH=/var/lib/homelab-chat-run/raw/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1-20260928T043051Z.log
25:UNIT=homelab-backup-v2.service ACTIVE=activating ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=1856481
238:LOCK_FUSER=/run/homelab-backup-heavy.lock| 1856481
239:LOCK_FREE=/run/homelab-backup-heavy.lock|false
249:homelab-backup- 1856481 FLOCK WRITE /run/homelab-backup-heavy.lock
=== AUDIT_LOCK_LOGIC_STATIC ===
LOCK_LOOP_SOURCE_BEGIN
for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]:
p=pathlib.Path(lock)
if p.exists():
lrc,_,_=run(["flock","-n",str(p),"-c","true"],5)
if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
LOCK_LOOP_SOURCE_END
LOCK_LOOP_CALLS=issues.append,p.exists,pathlib.Path,run,str
=== ALERT_COUNTER_LOGIC_STATIC ===
ALERT_LOGIC|0028|if mode!="health-alert": raise SystemExit(64)
ALERT_LOGIC|0029|p=subprocess.run([AUDIT,"write-health"],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
ALERT_LOGIC|0030|if p.returncode not in (0,1): raise SystemExit(p.returncode or 75)
ALERT_LOGIC|0031|try:
ALERT_LOGIC|0032| h=json.loads(HEALTH.read_text(encoding="utf-8"))
ALERT_LOGIC|0033|except Exception:
ALERT_LOGIC|0034| raise SystemExit(69)
ALERT_LOGIC|0035|issues=h.get("issues") if isinstance(h.get("issues"),list) else []
ALERT_LOGIC|0036|cur={key(i):i for i in issues if isinstance(i,dict)}
ALERT_LOGIC|0037|codes=set(str(i.get("code","")) for i in cur.values())
ALERT_LOGIC|0038|oldj={}
ALERT_LOGIC|0039|if STATE.is_file():
ALERT_LOGIC|0040| try:
ALERT_LOGIC|0041| oldj=json.loads(STATE.read_text(encoding="utf-8"))
ALERT_LOGIC|0042| if not isinstance(oldj,dict): oldj={}
ALERT_LOGIC|0043| except Exception:
ALERT_LOGIC|0044| oldj={}
ALERT_LOGIC|0045|migrating=oldj.get("schema")!=SCHEMA
ALERT_LOGIC|0046|prev_counts=oldj.get("code_counts") if isinstance(oldj.get("code_counts"),dict) and not migrating else {}
ALERT_LOGIC|0047|counts={}
ALERT_LOGIC|0048|for code in codes:
ALERT_LOGIC|0049| try: prev=int(prev_counts.get(code,0))
ALERT_LOGIC|0050| except Exception: prev=0
ALERT_LOGIC|0051| counts[code]=prev+1
ALERT_LOGIC|0052|actionable={}
ALERT_LOGIC|0053|for k,i in cur.items():
ALERT_LOGIC|0054| code=str(i.get("code",""))
ALERT_LOGIC|0055| need=int(THRESHOLDS.get(code,1))
ALERT_LOGIC|0056| if counts.get(code,0)>=need:
ALERT_LOGIC|0057| actionable[k]=i
ALERT_LOGIC|0058|tokens=[REDACTED]
ALERT_LOGIC|0082|STATE.parent.mkdir(parents=True,exist_ok=True)
ALERT_LOGIC|0083|tmp=STATE.with_name(STATE.name+".tmp")
ALERT_LOGIC|0084|payload={
ALERT_LOGIC|0085| "schema":SCHEMA,
ALERT_LOGIC|0086| "updated_epoch":int(time.time()),
ALERT_LOGIC|0087| "issue_keys":sorted(actionable),
ALERT_LOGIC|0088| "notify_tokens":[REDACTED]
ALERT_LOGIC|0089| "raw_issue_keys":sorted(cur),
ALERT_LOGIC|0090| "code_counts":{k:int(v) for k,v in sorted(counts.items())},
ALERT_LOGIC|0091| "migration_silent":bool(migrating)
ALERT_LOGIC|0092|}
ALERT_LOGIC|0093|tmp.write_text(json.dumps(payload,separators=(",",":"),sort_keys=True)+"\n",encoding="utf-8")
ALERT_LOGIC|0094|os.chmod(tmp,0o600)
ALERT_LOGIC|0095|os.replace(tmp,STATE)
=== CANONICAL_SOURCE_CANDIDATES ===
SOURCE_CANDIDATE|name=homelab-backup-audit|exact=false|sha256=1474c9ce58bbc1b3ae7437261d1bdb923232926fb2943c7c830253f8b284724c|path=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-audit
SOURCE_CANDIDATE_TOTAL|name=homelab-backup-audit|count=1|exact_count=0
SOURCE_CANDIDATE|name=homelab-alertmanager-backup-run|exact=true|sha256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9|path=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-run
SOURCE_CANDIDATE|name=homelab-alertmanager-backup-run|exact=false|sha256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a|path=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run
SOURCE_CANDIDATE_TOTAL|name=homelab-alertmanager-backup-run|count=2|exact_count=1
=== SOURCE_RELEVANT_GREP ===
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:188:for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]:
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:192: if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:292:source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:299: lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:300: if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:301:if source["remote_head"]!=source["local_head"]:
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}")
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:14:HEAVY_LOCK=/run/homelab-backup-heavy.lock
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:424: exec 5>"$LOCK"; flock -n 5 || die GC_LOCK_BUSY 75
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:434: exec 5>"$LOCK"; flock -n 5 || { printf 'GC_SCHEDULED=SKIP|reason=GC_LOCK_BUSY\n'; return 0; }
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:444: if ! acquire_gc_locks; then write_health "$p1" true 0; printf 'GC_SCHEDULED=SKIP|reason=BACKUP_LOCK_BUSY|candidates=%s|open_transactions=%s\n' "$count" "$open_tx"; return 0; fi
changes/CR-2026-1005/runtime/homelab-backup-v2-app-scheduled:6:HEAVY_LOCK=/run/homelab-backup-heavy.lock
changes/CR-2026-1005/runtime/homelab-backup-v2-app-scheduled:19:exec 9>"$QUEUE_LOCK"; if ! flock -n 9; then printf 'APP_V2_SCHEDULED=SKIP|reason=APP_QUEUE_LOCK_BUSY\n'; exit 0; fi
changes/CR-2026-1005/runtime/homelab-backup-v2-app-scheduled:20:exec 8>"$HEAVY_LOCK"; if ! flock -n 8; then printf 'APP_V2_SCHEDULED=SKIP|reason=GLOBAL_HEAVY_LOCK_BUSY\n'; exit 0; fi
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-audit:159:flock -n 9 || die AUDIT_LOCK_BUSY 75
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-audit:422:for lock in /run/homelab-backup-heavy.lock /run/lock/homelab-vm-cloud-quorum-queue.lock /run/lock/homelab-app-cloud-quorum-queue.lock /run/lock/homelab-backup-dispatcher.lock; do
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:14:HEAVY_LOCK=/run/homelab-backup-heavy.lock
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:424: exec 5>"$LOCK"; flock -n 5 || die GC_LOCK_BUSY 75
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:434: exec 5>"$LOCK"; flock -n 5 || { printf 'GC_SCHEDULED=SKIP|reason=GC_LOCK_BUSY\n'; return 0; }
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:444: if ! acquire_gc_locks; then write_health "$p1" true 0; printf 'GC_SCHEDULED=SKIP|reason=BACKUP_LOCK_BUSY|candidates=%s|open_transactions=%s\n' "$count" "$open_tx"; return 0; fi
tasks/snikket-vm150-rebuild-v2/assets/vm150-runtime.py:60:BACKUP_LOCKS=['/run/lock/homelab-backup-dispatcher.lock','/run/lock/homelab-vm-cloud-quorum-queue.lock','/run/homelab-backup-heavy.lock']
tasks/snikket-vm150-rebuild-v2/assets/vm150-runtime.py:506: f.close(); die('LOCK_BUSY_'+pathlib.Path(q).name,75)
TASK_RESULT=PASS_BACKUP_ALERT_NOISE_ROOT_RCA_COLLECTED
DECISION=PASS_BACKUP_ALERT_NOISE_ROOT_RCA_COLLECTED
NEXT_GATE=READ_EXACT_1238_AND_APPLY_ONLY_PROVEN_ALERT_NOISE_FIX
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END