From b7b43bed2377f18702a0c90a75810ca4b92f1f64 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Mon, 28 Sep 2026 04:34:03 +0000 Subject: [PATCH] runtime: publish SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1 --- ...-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1.json | 38 ++ ...B-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1.txt | 455 ++++++++++++++++ runtime/latest.json | 16 +- runtime/latest.txt | 499 +++++++++++++++--- 4 files changed, 916 insertions(+), 92 deletions(-) create mode 100644 runtime/history/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1.json create mode 100644 runtime/history/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1.txt diff --git a/runtime/history/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1.json b/runtime/history/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1.json new file mode 100644 index 00000000..df9be427 --- /dev/null +++ b/runtime/history/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-alert-spam-current-rca", + "started_at_utc": "2026-09-28T04:30:51Z", + "finished_at_utc": "2026-09-28T04:34:02Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "e205fb5749bd8d2a466e79a56e8d9f3a66537e7dcbd1c635202ab6df607b736b", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "c64c77856d1b796c0414be59dd0089c9687ec505d26bb8b4111ab68ee967e5a2", + "sanitized_output_sha256": "c64c77856d1b796c0414be59dd0089c9687ec505d26bb8b4111ab68ee967e5a2", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "BACKUP_ALERT_SPAM_CURRENT_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1\nMODE=read-only\nCOMPONENT=homelab-backup-alert-spam-current-rca\nMUTATION_BOUNDARY=READ_ONLY_ALERT_HEALTH_STATE_SOURCE_JOURNAL_LOCK_AND_QUIET_AUDIT;NO_ALERT_EXECUTION;NO_MAIL;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_SYSTEMD_ACTION;NO_TIMER_CHANGE;NO_HEALTH_WRITE;NO_GIT_WRITE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1236_STATUS=OK\nSOURCE_1236_RC=0\nSOURCE_1236_CHANGES_MADE=false\nSOURCE_1236_DECISION=HOLD_1236_V2_SERVICE_LAST_RUN_NOT_SUCCESS\nSOURCE_1236_NEXT_GATE=RECONCILE_POST_RECOVERY_SCHEDULER\nSOURCE_1236_POST_RECOVERY_V2_FAILED_COUNT=0\nSOURCE_1236_POST_RECOVERY_FRESH_HELPER_FAILED_COUNT=0\nSOURCE_1236_POST_RECOVERY_SCHEDULE_SUCCESS_COUNT=7\n=== LIVE_IDENTITIES ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nALERT_SCRIPT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9\n=== CURRENT_UNIT_STATE ===\nUNIT=homelab-backup-v2.service ACTIVE=activating ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=1856481\nUNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=\nUNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code EXEC_MAIN_STATUS=1 MAINPID=0\nUNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=\nUNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=0\nUNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=\n=== CURRENT_HEALTH_FILES ===\nHEALTH1_EXISTS=true\nHEALTH1_SHA256=d89fb55b92f2b9908e2d2d8da3b9a0d0db6d2fe84c6e6d37bcec1a86e4384222\nHEALTH1_JSON_OK=true\nHEALTH1_SCHEMA=homelab-backup-audit-v3\nHEALTH1_STATUS=RED\nHEALTH1_GENERATED_AT_EPOCH=1790568997\nHEALTH1_AGE_SECONDS=855\nHEALTH1_ISSUE_COUNT=2\nHEALTH1_ISSUE_1={\"code\":\"LOCK_BUSY\",\"logical_id\":\"/run/homelab-backup-heavy.lock\",\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\",\"severity\":\"HIGH\"}\nHEALTH1_ISSUE_2={\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\",\"severity\":\"MEDIUM\"}\nHEALTH1_SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nHEALTH1_SOURCE_REMOTE_HEAD=None\nHEALTH1_SOURCE_PUSHURL_DISABLED=true\nHEALTH1_SOURCE_EXACT=false\nHEALTH2_EXISTS=true\nHEALTH2_SHA256=d89fb55b92f2b9908e2d2d8da3b9a0d0db6d2fe84c6e6d37bcec1a86e4384222\nHEALTH2_JSON_OK=true\nHEALTH2_SCHEMA=homelab-backup-audit-v3\nHEALTH2_STATUS=RED\nHEALTH2_GENERATED_AT_EPOCH=1790568997\nHEALTH2_AGE_SECONDS=855\nHEALTH2_ISSUE_COUNT=2\nHEALTH2_ISSUE_1={\"code\":\"LOCK_BUSY\",\"logical_id\":\"/run/homelab-backup-heavy.lock\",\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\",\"severity\":\"HIGH\"}\nHEALTH2_ISSUE_2={\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\",\"severity\":\"MEDIUM\"}\nHEALTH2_SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nHEALTH2_SOURCE_REMOTE_HEAD=None\nHEALTH2_SOURCE_PUSHURL_DISABLED=true\nHEALTH2_SOURCE_EXACT=false\nHEALTH_FILES_IDENTICAL=true\nHEALTH_EPOCH_VALID=true\n=== CURRENT_ALERT_STATE ===\nALERT_STATE_EXISTS=true\nALERT_STATE_SHA256=493ce50563cb31030f2b16ff4ad722a7ac5ec211a77b2e89625491c057e8827d\nALERT_STATE_JSON_OK=true\nALERT_STATE_KEYS=code_counts,issue_keys,migration_silent,notify_tokens,raw_issue_keys,schema,updated_epoch\nALERT_STATE_SAFE_JSON={\"code_counts\":{\"LOCK_BUSY\":4,\"SOURCE_OFFHOST_MISSING\":263},\"issue_keys\":[\"LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK\",\"SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT\"],\"migration_silent\":false,\"notify_tokens\":\"[REDACTED]\",\"raw_issue_keys\":[\"LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK\",\"SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT\"],\"schema\":\"homelab-backup-alert-state-v2\",\"updated_epoch\":1790569001}\nALERT_STATE_SAFE_TRUNCATED=false\nALERT_STATE_FIELD_SCHEMA=\"homelab-backup-alert-state-v2\"\nALERT_STATE_FIELD_UPDATED_EPOCH=1790569001\nALERT_STATE_FIELD_ISSUE_KEYS=[\"LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK\",\"SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT\"]\n=== ALERT_FILTER_SOURCE ===\nALERT_EMBEDDED_SOURCE_FOUND=true\nALERT_EMBEDDED_SOURCE_SHA256=344306105083cd8bc64e19d451f7f66c17accc43a2f413538d2f9e2687b2540e\nALERT_EMBEDDED_SOURCE_BEGIN\n0001:import sys,os,json,re,subprocess,pathlib,time\n0002:mode=sys.argv[1]\n0003:AUDIT=\"/usr/local/sbin/homelab-backup-audit\"\n0004:HEALTH=pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\")\n0005:STATE=pathlib.Path(\"/var/lib/homelab-alerting/backup-alert-state.json\")\n0006:USERCFG=pathlib.Path(\"/etc/pve/user.cfg\")\n0007:SCHEMA=\"homelab-backup-alert-state-v2\"\n0008:THRESHOLDS={\"RPO_BREACH\":2,\"SOURCE_OFFHOST_MISSING\":2,\"LOCK_BUSY\":3}\n0009:def recipient():\n0010: text=USERCFG.read_text(encoding=\"utf-8\",errors=\"replace\")\n0011: m=re.findall(r\"(?i)\\b[A-Z0-9._+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}\\b\",text)\n0012: if len(m)!=1: raise SystemExit(65)\n0013: return m[0]\n0014:def send(subject,body):\n0015: rcpt=recipient()\n0016: msg=f\"From: h***@gram1.ru\\nTo: {rcpt}\\nSubject: {subject}\\nContent-Type: text/plain; charset=UTF-8\\n\\n{body}\\n\"\n0017: p=subprocess.run([\"/usr/sbin/sendmail\",\"-t\"],input=msg,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)\n0018: if p.returncode!=0: raise SystemExit(p.returncode or 75)\n0019:def key(i):\n0020: return \"|\".join(str(i.get(x,\"\")) for x in (\"code\",\"logical_id\",\"root_cause\"))\n0021:def token(i):[REDACTED]\n0022: code=str(i.get(\"code\",\"\"))\n0023: return \"group:\"+code if code in THRESHOLDS else \"issue:\"+key(i)\n0024:if mode==\"email-test\":\n0025: send(\"Workers2 backup alerting test\",\"Workers2 backup/DR alerting route test. If you received this, the local MTA accepted the controlled test generated during backup closure.\")\n0026: print(\"EMAIL_TEST_ACCEPTED=YES\")\n0027: raise SystemExit(0)\n0028:if mode!=\"health-alert\": raise SystemExit(64)\n0029:p=subprocess.run([AUDIT,\"write-health\"],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)\n0030:if p.returncode not in (0,1): raise SystemExit(p.returncode or 75)\n0031:try:\n0032: h=json.loads(HEALTH.read_text(encoding=\"utf-8\"))\n0033:except Exception:\n0034: raise SystemExit(69)\n0035:issues=h.get(\"issues\") if isinstance(h.get(\"issues\"),list) else []\n0036:cur={key(i):i for i in issues if isinstance(i,dict)}\n0037:codes=set(str(i.get(\"code\",\"\")) for i in cur.values())\n0038:oldj={}\n0039:if STATE.is_file():\n0040: try:\n0041: oldj=json.loads(STATE.read_text(encoding=\"utf-8\"))\n0042: if not isinstance(oldj,dict): oldj={}\n0043: except Exception:\n0044: oldj={}\n0045:migrating=oldj.get(\"schema\")!=SCHEMA\n0046:prev_counts=oldj.get(\"code_counts\") if isinstance(oldj.get(\"code_counts\"),dict) and not migrating else {}\n0047:counts={}\n0048:for code in codes:\n0049: try: prev=int(prev_counts.get(code,0))\n0050: except Exception: prev=0\n0051: counts[code]=prev+1\n0052:actionable={}\n0053:for k,i in cur.items():\n0054: code=str(i.get(\"code\",\"\"))\n0055: need=int(THRESHOLDS.get(code,1))\n0056: if counts.get(code,0)>=need:\n0057: actionable[k]=i\n0058:tokens=[REDACTED]\n0059:old_tokens=[REDACTED]\n0060:if not migrating:\n0061: old_tokens=[REDACTED]\n0062:added=[REDACTED]\n0063:removed=[REDACTED]\n0064:if added:\n0065: lines=[\n0066: \"Workers2 backup/DR actionable alert state changed.\",\n0067: \"\",\n0068: \"Transient debounce policy: RPO_BREACH=2 checks; SOURCE_OFFHOST_MISSING=2 checks; LOCK_BUSY=3 checks; all other issue codes=immediate.\",\n0069: \"\"\n0070: ]\n0071: for k in sorted(actionable):\n0072: i=actionable[k]\n0073: lines.append(f\"{i.get('severity','UNKNOWN')} {i.get('code','UNKNOWN')} logical_id={i.get('logical_id','-')} root_cause={i.get('root_cause','UNKNOWN')}\")\n0074: suppressed=len(cur)-len(actionable)\n0075: if suppressed:\n0076: lines.extend([\"\",f\"Suppressed transient issue(s) still below threshold: {suppressed}\"])\n0077: send(f\"[Workers2 BACKUP ALERT] {len(actionable)} actionable issue(s)\",\"\\n\".join(lines))\n0078: print(\"ALERT_EMAIL_ACCEPTED=YES\")\n0079:elif old_tokens and not tokens:[REDACTED]\n0080: send(\"[Workers2 BACKUP RECOVERY] all actionable issues cleared\",\"Workers2 backup/DR actionable audit state is clear. All previously notified actionable backup issues are cleared.\")\n0081: print(\"RECOVERY_EMAIL_ACCEPTED=YES\")\n0082:STATE.parent.mkdir(parents=True,exist_ok=True)\n0083:tmp=STATE.with_name(STATE.name+\".tmp\")\n0084:payload={\n0085: \"schema\":SCHEMA,\n0086: \"updated_epoch\":int(time.time()),\n0087: \"issue_keys\":sorted(actionable),\n0088: \"notify_tokens\":[REDACTED]\n0089: \"raw_issue_keys\":sorted(cur),\n0090: \"code_counts\":{k:int(v) for k,v in sorted(counts.items())},\n0091: \"migration_silent\":bool(migrating)\n0092:}\n0093:tmp.write_text(json.dumps(payload,separators=(\",\",\":\"),sort_keys=True)+\"\\n\",encoding=\"utf-8\")\n0094:os.chmod(tmp,0o600)\n0095:os.replace(tmp,STATE)\n0096:print(\"ALERT_FILTER_SCHEMA=\"+SCHEMA)\n0097:print(\"ALERT_RAW_ISSUE_COUNT=\"+str(len(cur)))\n0098:print(\"ALERT_ACTIONABLE_ISSUE_COUNT=\"+str(len(actionable)))\n0099:print(\"ALERT_NOTIFY_TOKEN_COUNT=[REDACTED]\n0100:print(\"ALERT_ADDED_TOKEN_COUNT=[REDACTED]\n0101:print(\"ALERT_REMOVED_TOKEN_COUNT=[REDACTED]\n0102:print(\"ALERT_SUPPRESSED_TRANSIENT_COUNT=\"+str(len(cur)-len(actionable)))\n0103:print(\"ALERT_STATE_MIGRATION_SILENT=\"+(\"true\" if migrating else \"false\"))\n0104:print(\"HEALTH_ALERT_STATE=OK\")\nALERT_EMBEDDED_SOURCE_END\n=== AUDIT_SOURCE_RELEVANT ===\n183- (\"homelab-alertmanager-backup-health.timer\",\"ALERT_TIMER_BAD\"),\n184-]:\n185- ok,a,e=unit_ok(unit)\n186- if not ok: issues.append({\"severity\":\"HIGH\",\"code\":code,\"logical_id\":unit,\"root_cause\":\"UNIT_\"+a+\"_\"+e})\n187-\n188:for lock in [\"/run/homelab-backup-heavy.lock\",\"/run/lock/homelab-app-cloud-quorum-queue.lock\",\"/run/lock/homelab-vm-cloud-quorum-queue.lock\",\"/run/lock/homelab-backup-dispatcher.lock\"]:\n189- p=pathlib.Path(lock)\n190- if p.exists():\n191: lrc,_,_=run([\"flock\",\"-n\",str(p),\"-c\",\"true\"],5)\n192: if lrc!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"LOCK_BUSY\",\"logical_id\":lock,\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\"})\n193-\n194-env=os.environ.copy(); env[\"HOMELAB_BACKUP_GFS_POLICY\"]=GFSP; env[\"HOMELAB_BACKUP_TARGET_REGISTRY\"]=str(REG)\n195-grc,gout,_=run([GFSH,\"validate\"],20,env)\n196-if grc!=0 or \"GFS_POLICY=OK\" not in gout:\n197- issues.append({\"severity\":\"CRITICAL\",\"code\":\"GFS_INVALID\",\"logical_id\":\"-\",\"root_cause\":\"GFS_VALIDATE_FAILED\"})\n--\n287-except Exception:\n288- cloudb_ok=False\n289-if not cloudb_ok:\n290- issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK\"})\n291-\n292:source={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\n293-hrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\n294:if hrc==0: source[\"local_head\"]=ho.strip()\n295-frc,fo,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.url\"],10)\n296-prc,po,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.pushurl\"],10)\n297-source[\"pushurl_disabled\"]=(prc==0 and po.strip()==\"DISABLED\")\n298-if frc==0 and fo.strip():\n299- lrc,lo,_=run([\"git\",\"ls-remote\",fo.strip(),\"refs/heads/\"+BRANCH],30)\n300: if lrc==0 and lo.strip(): source[\"remote_head\"]=lo.split()[0]\n301:if source[\"remote_head\"]!=source[\"local_head\"]:\n302: issues.append({\"severity\":\"MEDIUM\",\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\"})\n303-\n304-status=\"GREEN\" if not issues else \"RED\"\n305-for t in targets:\n306- print(\"TARGET|\"+\"|\".join(f\"{k}={v}\" for k,v in t.items() if v is not None))\n307-for i in issues:\n308- print(\"ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}\".format(**i))\n309:print(f\"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}\")\n310-health={\"schema\":\"homelab-backup-audit-v3\",\"status\":status,\"generated_at_epoch\":now,\"targets\":targets,\"issues\":issues,\"cloud\":cloud,\"source\":source,\"cloud_b_restore_drill\":cloudb_ok,\"dr_proven\":bool(isinstance(dr,dict) and dr.get(\"status\")==\"SEALED\")}\n311-if MODE in (\"write-health\",\"scheduled\"):\n312- for hp in HEALTHS:\n313- hp.parent.mkdir(parents=True,exist_ok=True)\n314- tmp=hp.with_name(hp.name+\".tmp\")\n=== LIVE_SOURCE_HEAD_RECHECK ===\nSOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_WORKTREE_CLEAN=true\nSOURCE_ORIGIN_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9\nSOURCE_REMOTE_ATTEMPT=1 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true\nSOURCE_REMOTE_ATTEMPT=2 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true\nSOURCE_REMOTE_ATTEMPT=3 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true\nSOURCE_REMOTE_ATTEMPT=4 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true\nSOURCE_REMOTE_ATTEMPT=5 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true\n=== CURRENT_LOCK_OWNERSHIP ===\nLOCK_META=/run/homelab-backup-heavy.lock|inode=3474|600|root:root|2026-09-28 03:27:07.425060217 +0300\nLOCK_FUSER=/run/homelab-backup-heavy.lock| 1856481\nLOCK_FREE=/run/homelab-backup-heavy.lock|false\nLOCK_META=/run/lock/homelab-backup-dispatcher.lock|inode=9|600|root:root|2026-09-28 03:27:07.423060188 +0300\nLOCK_FUSER=/run/lock/homelab-backup-dispatcher.lock|\nLOCK_FREE=/run/lock/homelab-backup-dispatcher.lock|true\nLOCK_META=/run/lock/homelab-vm-cloud-quorum-queue.lock|inode=2944|600|root:root|2026-09-28 03:27:07.424060203 +0300\nLOCK_FUSER=/run/lock/homelab-vm-cloud-quorum-queue.lock|\nLOCK_FREE=/run/lock/homelab-vm-cloud-quorum-queue.lock|true\nLOCK_META=/run/lock/homelab-app-cloud-quorum-queue.lock|inode=15|600|root:root|2026-09-28 03:27:07.425060217 +0300\nLOCK_FUSER=/run/lock/homelab-app-cloud-quorum-queue.lock|\nLOCK_FREE=/run/lock/homelab-app-cloud-quorum-queue.lock|true\nhomelab-backup- 1856481 FLOCK WRITE /run/homelab-backup-heavy.lock\n=== ALERT_EMAIL_TIME_CORRELATION_0701_MSK ===\n2026-09-28T06:19:01.361470+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T06:36:20.423980+03:00 pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-28T06:36:20.424205+03:00 pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\n2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T07:06:48.304891+03:00 pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-28T07:06:48.305137+03:00 pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\n2026-09-28T06:15:16.319082+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\n2026-09-28T06:15:20.661916+03:00 pve01 homelab-alertmanager-backup-run[1835343]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:15:20.666144+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\n2026-09-28T06:30:42.414371+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\n2026-09-28T06:30:46.367196+03:00 pve01 homelab-alertmanager-backup-run[1886013]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:30:46.370827+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\n2026-09-28T06:46:01.347189+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\n2026-09-28T06:46:05.406037+03:00 pve01 homelab-alertmanager-backup-run[1945314]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:46:05.410491+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\n2026-09-28T07:01:32.217324+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\n2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: ALERT_EMAIL_ACCEPTED=YES\n2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: HEALTH_ALERT_STATE=OK\n2026-09-28T07:01:36.278152+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\n2026-09-28T07:16:37.585569+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\n2026-09-28T07:16:41.913014+03:00 pve01 homelab-alertmanager-backup-run[2048228]: HEALTH_ALERT_STATE=OK\n2026-09-28T07:16:41.916768+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\n=== ALERT_FREQUENCY_LAST12H ===\nALERT_RUN_START_COUNT=0\nALERT_EMAIL_ACCEPTED_COUNT=3\nRECOVERY_EMAIL_ACCEPTED_COUNT=0\nHEALTH_ALERT_STATE_OK_COUNT=47\n2026-09-27T19:32:04.991437+03:00 pve01 homelab-alertmanager-backup-run[3909103]: HEALTH_ALERT_STATE=OK\n2026-09-27T19:47:36.328545+03:00 pve01 homelab-alertmanager-backup-run[3959645]: HEALTH_ALERT_STATE=OK\n2026-09-27T20:02:46.359055+03:00 pve01 homelab-alertmanager-backup-run[4008840]: HEALTH_ALERT_STATE=OK\n2026-09-27T20:18:05.225521+03:00 pve01 homelab-alertmanager-backup-run[4059785]: HEALTH_ALERT_STATE=OK\n2026-09-27T20:33:20.585967+03:00 pve01 homelab-alertmanager-backup-run[4108890]: HEALTH_ALERT_STATE=OK\n2026-09-27T20:48:36.460451+03:00 pve01 homelab-alertmanager-backup-run[4148152]: HEALTH_ALERT_STATE=OK\n2026-09-27T21:04:08.100470+03:00 pve01 homelab-alertmanager-backup-run[4070]: HEALTH_ALERT_STATE=OK\n2026-09-27T21:19:36.336976+03:00 pve01 homelab-alertmanager-backup-run[54137]: HEALTH_ALERT_STATE=OK\n2026-09-27T21:35:05.706232+03:00 pve01 homelab-alertmanager-backup-run[104752]: HEALTH_ALERT_STATE=OK\n2026-09-27T21:50:23.015203+03:00 pve01 homelab-alertmanager-backup-run[154365]: HEALTH_ALERT_STATE=OK\n2026-09-27T22:05:36.461278+03:00 pve01 homelab-alertmanager-backup-run[202960]: HEALTH_ALERT_STATE=OK\n2026-09-27T22:21:05.253758+03:00 pve01 homelab-alertmanager-backup-run[242449]: HEALTH_ALERT_STATE=OK\n2026-09-27T22:36:15.399283+03:00 pve01 homelab-alertmanager-backup-run[292331]: HEALTH_ALERT_STATE=OK\n2026-09-27T22:51:31.819800+03:00 pve01 homelab-alertmanager-backup-run[343850]: HEALTH_ALERT_STATE=OK\n2026-09-27T23:06:53.759023+03:00 pve01 homelab-alertmanager-backup-run[396329]: HEALTH_ALERT_STATE=OK\n2026-09-27T23:22:20.677831+03:00 pve01 homelab-alertmanager-backup-run[461385]: HEALTH_ALERT_STATE=OK\n2026-09-27T23:37:36.973241+03:00 pve01 homelab-alertmanager-backup-run[513251]: HEALTH_ALERT_STATE=OK\n2026-09-27T23:52:43.434843+03:00 pve01 homelab-alertmanager-backup-run[566888]: ALERT_EMAIL_ACCEPTED=YES\n2026-09-27T23:52:43.434843+03:00 pve01 homelab-alertmanager-backup-run[566888]: HEALTH_ALERT_STATE=OK\n2026-09-28T00:08:05.576769+03:00 pve01 homelab-alertmanager-backup-run[607129]: HEALTH_ALERT_STATE=OK\n2026-09-28T00:23:21.212626+03:00 pve01 homelab-alertmanager-backup-run[660884]: HEALTH_ALERT_STATE=OK\n2026-09-28T00:38:36.269886+03:00 pve01 homelab-alertmanager-backup-run[711652]: HEALTH_ALERT_STATE=OK\n2026-09-28T00:53:55.974540+03:00 pve01 homelab-alertmanager-backup-run[762988]: HEALTH_ALERT_STATE=OK\n2026-09-28T01:09:05.464301+03:00 pve01 homelab-alertmanager-backup-run[812499]: HEALTH_ALERT_STATE=OK\n2026-09-28T01:24:16.127247+03:00 pve01 homelab-alertmanager-backup-run[862903]: HEALTH_ALERT_STATE=OK\n2026-09-28T01:39:41.256513+03:00 pve01 homelab-alertmanager-backup-run[919688]: HEALTH_ALERT_STATE=OK\n2026-09-28T01:54:54.298806+03:00 pve01 homelab-alertmanager-backup-run[958250]: HEALTH_ALERT_STATE=OK\n2026-09-28T02:10:05.486220+03:00 pve01 homelab-alertmanager-backup-run[1009352]: HEALTH_ALERT_STATE=OK\n2026-09-28T02:25:22.594856+03:00 pve01 homelab-alertmanager-backup-run[1058040]: HEALTH_ALERT_STATE=OK\n2026-09-28T02:40:53.144072+03:00 pve01 homelab-alertmanager-backup-run[1108677]: HEALTH_ALERT_STATE=OK\n2026-09-28T02:56:20.122916+03:00 pve01 homelab-alertmanager-backup-run[1157287]: HEALTH_ALERT_STATE=OK\n2026-09-28T03:11:36.224287+03:00 pve01 homelab-alertmanager-backup-run[1207541]: HEALTH_ALERT_STATE=OK\n2026-09-28T03:26:56.104164+03:00 pve01 homelab-alertmanager-backup-run[1267608]: HEALTH_ALERT_STATE=OK\n2026-09-28T03:42:16.163161+03:00 pve01 homelab-alertmanager-backup-run[1337091]: HEALTH_ALERT_STATE=OK\n2026-09-28T03:57:35.911704+03:00 pve01 homelab-alertmanager-backup-run[1388112]: HEALTH_ALERT_STATE=OK\n2026-09-28T04:12:45.302638+03:00 pve01 homelab-alertmanager-backup-run[1439953]: ALERT_EMAIL_ACCEPTED=YES\n2026-09-28T04:12:45.302638+03:00 pve01 homelab-alertmanager-backup-run[1439953]: HEALTH_ALERT_STATE=OK\n2026-09-28T04:28:16.066840+03:00 pve01 homelab-alertmanager-backup-run[1491056]: HEALTH_ALERT_STATE=OK\n2026-09-28T04:43:26.035041+03:00 pve01 homelab-alertmanager-backup-run[1543022]: HEALTH_ALERT_STATE=OK\n2026-09-28T04:58:29.541024+03:00 pve01 homelab-alertmanager-backup-run[1593789]: HEALTH_ALERT_STATE=OK\n2026-09-28T05:13:39.334685+03:00 pve01 homelab-alertmanager-backup-run[1645768]: HEALTH_ALERT_STATE=OK\n2026-09-28T05:28:53.353807+03:00 pve01 homelab-alertmanager-backup-run[1685761]: HEALTH_ALERT_STATE=OK\n2026-09-28T05:44:20.155523+03:00 pve01 homelab-alertmanager-backup-run[1736193]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:00:05.422013+03:00 pve01 homelab-alertmanager-backup-run[1785685]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:15:20.661916+03:00 pve01 homelab-alertmanager-backup-run[1835343]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:30:46.367196+03:00 pve01 homelab-alertmanager-backup-run[1886013]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:46:05.406037+03:00 pve01 homelab-alertmanager-backup-run[1945314]: HEALTH_ALERT_STATE=OK\n2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: ALERT_EMAIL_ACCEPTED=YES\n2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: HEALTH_ALERT_STATE=OK\n2026-09-28T07:16:41.913014+03:00 pve01 homelab-alertmanager-backup-run[2048228]: HEALTH_ALERT_STATE=OK\n=== AUDIT_FREQUENCY_LAST12H ===\nAUDIT_GREEN_SUMMARY_COUNT=7\nAUDIT_RED_SUMMARY_COUNT=17\nAUDIT_LOCK_BUSY_ISSUE_COUNT=9\nAUDIT_SOURCE_OFFHOST_ISSUE_COUNT=0\n2026-09-27T19:37:02.093711+03:00 pve01 homelab-backup-audit[3923461]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T20:06:15.522779+03:00 pve01 homelab-backup-audit[4022565]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T20:36:05.450336+03:00 pve01 homelab-backup-audit[4111037]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T21:06:02.094425+03:00 pve01 homelab-backup-audit[16838]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T21:37:05.002224+03:00 pve01 homelab-backup-audit[106421]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T22:06:07.004513+03:00 pve01 homelab-backup-audit[203921]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T22:35:26.212759+03:00 pve01 homelab-backup-audit[291167]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T23:06:54.987894+03:00 pve01 homelab-backup-audit[395212]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T23:36:37.162587+03:00 pve01 homelab-backup-audit[500802]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-27T23:36:37.162587+03:00 pve01 homelab-backup-audit[500802]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=6|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T00:06:21.135619+03:00 pve01 homelab-backup-audit[605334]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T00:06:21.135619+03:00 pve01 homelab-backup-audit[605334]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T00:36:10.445081+03:00 pve01 homelab-backup-audit[707255]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T00:36:10.445081+03:00 pve01 homelab-backup-audit[707255]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T01:06:01.997372+03:00 pve01 homelab-backup-audit[799187]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T01:36:02.006941+03:00 pve01 homelab-backup-audit[906059]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T02:05:36.356288+03:00 pve01 homelab-backup-audit[995508]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T02:36:15.265015+03:00 pve01 homelab-backup-audit[1094502]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T03:05:20.453132+03:00 pve01 homelab-backup-audit[1192518]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T03:35:20.147380+03:00 pve01 homelab-backup-audit[1320691]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T03:35:20.147380+03:00 pve01 homelab-backup-audit[1320691]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T04:06:45.928910+03:00 pve01 homelab-backup-audit[1424254]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T04:06:45.928910+03:00 pve01 homelab-backup-audit[1424254]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T04:35:20.240415+03:00 pve01 homelab-backup-audit[1514892]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T04:35:20.240415+03:00 pve01 homelab-backup-audit[1514892]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T05:05:52.974415+03:00 pve01 homelab-backup-audit[1617767]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T05:05:52.974415+03:00 pve01 homelab-backup-audit[1617767]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T05:37:02.145613+03:00 pve01 homelab-backup-audit[1720751]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T06:06:02.032037+03:00 pve01 homelab-backup-audit[1807928]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n=== HEALTH_GENERATION_CORRELATION ===\nHEALTH_GENERATED_AT_EPOCH=1790568997\nHEALTH_CORRELATION_FROM=1790568817\nHEALTH_CORRELATION_TO=1790569177\n=== FRESH_QUIET_AUDIT ===\nQUIET_AUDIT_WAIT|attempt=1|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=2|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=3|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=4|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=5|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=6|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=7|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=8|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=9|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=10|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=11|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=12|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=13|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=14|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=15|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=16|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=17|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=18|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=19|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=20|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=21|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=22|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=23|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=24|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=25|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=26|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=27|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=28|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=29|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=30|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=31|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=32|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=33|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=34|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=35|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=36|reason=SERVICE_BUSY|active=activating|pid=1856481\nFRESH_QUIET_AUDIT_RC=NO_QUIET_WINDOW\nFRESH_AUDIT_HEALTH_NO_WRITE=false\n=== CANONICAL_ALERT_SOURCE_AUTHORITY ===\nCANONICAL_ALERT_PATH=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run\nCANONICAL_ALERT_EXISTS=true\nCANONICAL_ALERT_SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a\nCANONICAL_ALERT_MATCH_LIVE=false\nCANONICAL_SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nCANONICAL_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nCANONICAL_SOURCE_EXACT=true\nTASK_RESULT=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED\nDECISION=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED\nNEXT_GATE=READ_EXACT_1237_AND_FIX_ONLY_PROVEN_ALERT_NOISE_CAUSE\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1.txt b/runtime/history/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1.txt new file mode 100644 index 00000000..ca8bc20f --- /dev/null +++ b/runtime/history/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1.txt @@ -0,0 +1,455 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-alert-spam-current-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=e205fb5749bd8d2a466e79a56e8d9f3a66537e7dcbd1c635202ab6df607b736b +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=c64c77856d1b796c0414be59dd0089c9687ec505d26bb8b4111ab68ee967e5a2 +SANITIZED_OUTPUT_SHA256=c64c77856d1b796c0414be59dd0089c9687ec505d26bb8b4111ab68ee967e5a2 +OUTPUT_BEGIN +BACKUP_ALERT_SPAM_CURRENT_RCA_BEGIN=true +COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1 +MODE=read-only +COMPONENT=homelab-backup-alert-spam-current-rca +MUTATION_BOUNDARY=READ_ONLY_ALERT_HEALTH_STATE_SOURCE_JOURNAL_LOCK_AND_QUIET_AUDIT;NO_ALERT_EXECUTION;NO_MAIL;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_SYSTEMD_ACTION;NO_TIMER_CHANGE;NO_HEALTH_WRITE;NO_GIT_WRITE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_NETBIRD_SKLADCHIK_CHANGE +=== AUTHORITY === +HOST=pve01 +CLUSTER_QUORATE=Yes +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +SOURCE_1236_STATUS=OK +SOURCE_1236_RC=0 +SOURCE_1236_CHANGES_MADE=false +SOURCE_1236_DECISION=HOLD_1236_V2_SERVICE_LAST_RUN_NOT_SUCCESS +SOURCE_1236_NEXT_GATE=RECONCILE_POST_RECOVERY_SCHEDULER +SOURCE_1236_POST_RECOVERY_V2_FAILED_COUNT=0 +SOURCE_1236_POST_RECOVERY_FRESH_HELPER_FAILED_COUNT=0 +SOURCE_1236_POST_RECOVERY_SCHEDULE_SUCCESS_COUNT=7 +=== LIVE_IDENTITIES === +WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c +HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 +VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad +AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +ALERT_SCRIPT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9 +=== CURRENT_UNIT_STATE === +UNIT=homelab-backup-v2.service ACTIVE=activating ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=1856481 +UNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID= +UNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code EXEC_MAIN_STATUS=1 MAINPID=0 +UNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID= +UNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=0 +UNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID= +=== CURRENT_HEALTH_FILES === +HEALTH1_EXISTS=true +HEALTH1_SHA256=d89fb55b92f2b9908e2d2d8da3b9a0d0db6d2fe84c6e6d37bcec1a86e4384222 +HEALTH1_JSON_OK=true +HEALTH1_SCHEMA=homelab-backup-audit-v3 +HEALTH1_STATUS=RED +HEALTH1_GENERATED_AT_EPOCH=1790568997 +HEALTH1_AGE_SECONDS=855 +HEALTH1_ISSUE_COUNT=2 +HEALTH1_ISSUE_1={"code":"LOCK_BUSY","logical_id":"/run/homelab-backup-heavy.lock","root_cause":"ACTIVE_OR_ORPHAN_LOCK","severity":"HIGH"} +HEALTH1_ISSUE_2={"code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT","severity":"MEDIUM"} +HEALTH1_SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +HEALTH1_SOURCE_REMOTE_HEAD=None +HEALTH1_SOURCE_PUSHURL_DISABLED=true +HEALTH1_SOURCE_EXACT=false +HEALTH2_EXISTS=true +HEALTH2_SHA256=d89fb55b92f2b9908e2d2d8da3b9a0d0db6d2fe84c6e6d37bcec1a86e4384222 +HEALTH2_JSON_OK=true +HEALTH2_SCHEMA=homelab-backup-audit-v3 +HEALTH2_STATUS=RED +HEALTH2_GENERATED_AT_EPOCH=1790568997 +HEALTH2_AGE_SECONDS=855 +HEALTH2_ISSUE_COUNT=2 +HEALTH2_ISSUE_1={"code":"LOCK_BUSY","logical_id":"/run/homelab-backup-heavy.lock","root_cause":"ACTIVE_OR_ORPHAN_LOCK","severity":"HIGH"} +HEALTH2_ISSUE_2={"code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT","severity":"MEDIUM"} +HEALTH2_SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +HEALTH2_SOURCE_REMOTE_HEAD=None +HEALTH2_SOURCE_PUSHURL_DISABLED=true +HEALTH2_SOURCE_EXACT=false +HEALTH_FILES_IDENTICAL=true +HEALTH_EPOCH_VALID=true +=== CURRENT_ALERT_STATE === +ALERT_STATE_EXISTS=true +ALERT_STATE_SHA256=493ce50563cb31030f2b16ff4ad722a7ac5ec211a77b2e89625491c057e8827d +ALERT_STATE_JSON_OK=true +ALERT_STATE_KEYS=code_counts,issue_keys,migration_silent,notify_tokens,raw_issue_keys,schema,updated_epoch +ALERT_STATE_SAFE_JSON={"code_counts":{"LOCK_BUSY":4,"SOURCE_OFFHOST_MISSING":263},"issue_keys":["LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK","SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT"],"migration_silent":false,"notify_tokens":"[REDACTED]","raw_issue_keys":["LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK","SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT"],"schema":"homelab-backup-alert-state-v2","updated_epoch":1790569001} +ALERT_STATE_SAFE_TRUNCATED=false +ALERT_STATE_FIELD_SCHEMA="homelab-backup-alert-state-v2" +ALERT_STATE_FIELD_UPDATED_EPOCH=1790569001 +ALERT_STATE_FIELD_ISSUE_KEYS=["LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK","SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT"] +=== ALERT_FILTER_SOURCE === +ALERT_EMBEDDED_SOURCE_FOUND=true +ALERT_EMBEDDED_SOURCE_SHA256=344306105083cd8bc64e19d451f7f66c17accc43a2f413538d2f9e2687b2540e +ALERT_EMBEDDED_SOURCE_BEGIN +0001:import sys,os,json,re,subprocess,pathlib,time +0002:mode=sys.argv[1] +0003:AUDIT="/usr/local/sbin/homelab-backup-audit" +0004:HEALTH=pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json") +0005:STATE=pathlib.Path("/var/lib/homelab-alerting/backup-alert-state.json") +0006:USERCFG=pathlib.Path("/etc/pve/user.cfg") +0007:SCHEMA="homelab-backup-alert-state-v2" +0008:THRESHOLDS={"RPO_BREACH":2,"SOURCE_OFFHOST_MISSING":2,"LOCK_BUSY":3} +0009:def recipient(): +0010: text=USERCFG.read_text(encoding="utf-8",errors="replace") +0011: m=re.findall(r"(?i)\b[A-Z0-9._+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b",text) +0012: if len(m)!=1: raise SystemExit(65) +0013: return m[0] +0014:def send(subject,body): +0015: rcpt=recipient() +0016: msg=f"From: h***@gram1.ru\nTo: {rcpt}\nSubject: {subject}\nContent-Type: text/plain; charset=UTF-8\n\n{body}\n" +0017: p=subprocess.run(["/usr/sbin/sendmail","-t"],input=msg,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) +0018: if p.returncode!=0: raise SystemExit(p.returncode or 75) +0019:def key(i): +0020: return "|".join(str(i.get(x,"")) for x in ("code","logical_id","root_cause")) +0021:def token(i):[REDACTED] +0022: code=str(i.get("code","")) +0023: return "group:"+code if code in THRESHOLDS else "issue:"+key(i) +0024:if mode=="email-test": +0025: send("Workers2 backup alerting test","Workers2 backup/DR alerting route test. If you received this, the local MTA accepted the controlled test generated during backup closure.") +0026: print("EMAIL_TEST_ACCEPTED=YES") +0027: raise SystemExit(0) +0028:if mode!="health-alert": raise SystemExit(64) +0029:p=subprocess.run([AUDIT,"write-health"],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) +0030:if p.returncode not in (0,1): raise SystemExit(p.returncode or 75) +0031:try: +0032: h=json.loads(HEALTH.read_text(encoding="utf-8")) +0033:except Exception: +0034: raise SystemExit(69) +0035:issues=h.get("issues") if isinstance(h.get("issues"),list) else [] +0036:cur={key(i):i for i in issues if isinstance(i,dict)} +0037:codes=set(str(i.get("code","")) for i in cur.values()) +0038:oldj={} +0039:if STATE.is_file(): +0040: try: +0041: oldj=json.loads(STATE.read_text(encoding="utf-8")) +0042: if not isinstance(oldj,dict): oldj={} +0043: except Exception: +0044: oldj={} +0045:migrating=oldj.get("schema")!=SCHEMA +0046:prev_counts=oldj.get("code_counts") if isinstance(oldj.get("code_counts"),dict) and not migrating else {} +0047:counts={} +0048:for code in codes: +0049: try: prev=int(prev_counts.get(code,0)) +0050: except Exception: prev=0 +0051: counts[code]=prev+1 +0052:actionable={} +0053:for k,i in cur.items(): +0054: code=str(i.get("code","")) +0055: need=int(THRESHOLDS.get(code,1)) +0056: if counts.get(code,0)>=need: +0057: actionable[k]=i +0058:tokens=[REDACTED] +0059:old_tokens=[REDACTED] +0060:if not migrating: +0061: old_tokens=[REDACTED] +0062:added=[REDACTED] +0063:removed=[REDACTED] +0064:if added: +0065: lines=[ +0066: "Workers2 backup/DR actionable alert state changed.", +0067: "", +0068: "Transient debounce policy: RPO_BREACH=2 checks; SOURCE_OFFHOST_MISSING=2 checks; LOCK_BUSY=3 checks; all other issue codes=immediate.", +0069: "" +0070: ] +0071: for k in sorted(actionable): +0072: i=actionable[k] +0073: lines.append(f"{i.get('severity','UNKNOWN')} {i.get('code','UNKNOWN')} logical_id={i.get('logical_id','-')} root_cause={i.get('root_cause','UNKNOWN')}") +0074: suppressed=len(cur)-len(actionable) +0075: if suppressed: +0076: lines.extend(["",f"Suppressed transient issue(s) still below threshold: {suppressed}"]) +0077: send(f"[Workers2 BACKUP ALERT] {len(actionable)} actionable issue(s)","\n".join(lines)) +0078: print("ALERT_EMAIL_ACCEPTED=YES") +0079:elif old_tokens and not tokens:[REDACTED] +0080: send("[Workers2 BACKUP RECOVERY] all actionable issues cleared","Workers2 backup/DR actionable audit state is clear. All previously notified actionable backup issues are cleared.") +0081: print("RECOVERY_EMAIL_ACCEPTED=YES") +0082:STATE.parent.mkdir(parents=True,exist_ok=True) +0083:tmp=STATE.with_name(STATE.name+".tmp") +0084:payload={ +0085: "schema":SCHEMA, +0086: "updated_epoch":int(time.time()), +0087: "issue_keys":sorted(actionable), +0088: "notify_tokens":[REDACTED] +0089: "raw_issue_keys":sorted(cur), +0090: "code_counts":{k:int(v) for k,v in sorted(counts.items())}, +0091: "migration_silent":bool(migrating) +0092:} +0093:tmp.write_text(json.dumps(payload,separators=(",",":"),sort_keys=True)+"\n",encoding="utf-8") +0094:os.chmod(tmp,0o600) +0095:os.replace(tmp,STATE) +0096:print("ALERT_FILTER_SCHEMA="+SCHEMA) +0097:print("ALERT_RAW_ISSUE_COUNT="+str(len(cur))) +0098:print("ALERT_ACTIONABLE_ISSUE_COUNT="+str(len(actionable))) +0099:print("ALERT_NOTIFY_TOKEN_COUNT=[REDACTED] +0100:print("ALERT_ADDED_TOKEN_COUNT=[REDACTED] +0101:print("ALERT_REMOVED_TOKEN_COUNT=[REDACTED] +0102:print("ALERT_SUPPRESSED_TRANSIENT_COUNT="+str(len(cur)-len(actionable))) +0103:print("ALERT_STATE_MIGRATION_SILENT="+("true" if migrating else "false")) +0104:print("HEALTH_ALERT_STATE=OK") +ALERT_EMBEDDED_SOURCE_END +=== AUDIT_SOURCE_RELEVANT === +183- ("homelab-alertmanager-backup-health.timer","ALERT_TIMER_BAD"), +184-]: +185- ok,a,e=unit_ok(unit) +186- if not ok: issues.append({"severity":"HIGH","code":code,"logical_id":unit,"root_cause":"UNIT_"+a+"_"+e}) +187- +188:for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]: +189- p=pathlib.Path(lock) +190- if p.exists(): +191: lrc,_,_=run(["flock","-n",str(p),"-c","true"],5) +192: if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"}) +193- +194-env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG) +195-grc,gout,_=run([GFSH,"validate"],20,env) +196-if grc!=0 or "GFS_POLICY=OK" not in gout: +197- issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"}) +-- +287-except Exception: +288- cloudb_ok=False +289-if not cloudb_ok: +290- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"}) +291- +292:source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +293-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +294:if hrc==0: source["local_head"]=ho.strip() +295-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) +296-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) +297-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") +298-if frc==0 and fo.strip(): +299- lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) +300: if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] +301:if source["remote_head"]!=source["local_head"]: +302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) +303- +304-status="GREEN" if not issues else "RED" +305-for t in targets: +306- print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None)) +307-for i in issues: +308- print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i)) +309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}") +310-health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")} +311-if MODE in ("write-health","scheduled"): +312- for hp in HEALTHS: +313- hp.parent.mkdir(parents=True,exist_ok=True) +314- tmp=hp.with_name(hp.name+".tmp") +=== LIVE_SOURCE_HEAD_RECHECK === +SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_WORKTREE_CLEAN=true +SOURCE_ORIGIN_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9 +SOURCE_REMOTE_ATTEMPT=1 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true +SOURCE_REMOTE_ATTEMPT=2 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true +SOURCE_REMOTE_ATTEMPT=3 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true +SOURCE_REMOTE_ATTEMPT=4 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true +SOURCE_REMOTE_ATTEMPT=5 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true +=== CURRENT_LOCK_OWNERSHIP === +LOCK_META=/run/homelab-backup-heavy.lock|inode=3474|600|root:root|2026-09-28 03:27:07.425060217 +0300 +LOCK_FUSER=/run/homelab-backup-heavy.lock| 1856481 +LOCK_FREE=/run/homelab-backup-heavy.lock|false +LOCK_META=/run/lock/homelab-backup-dispatcher.lock|inode=9|600|root:root|2026-09-28 03:27:07.423060188 +0300 +LOCK_FUSER=/run/lock/homelab-backup-dispatcher.lock| +LOCK_FREE=/run/lock/homelab-backup-dispatcher.lock|true +LOCK_META=/run/lock/homelab-vm-cloud-quorum-queue.lock|inode=2944|600|root:root|2026-09-28 03:27:07.424060203 +0300 +LOCK_FUSER=/run/lock/homelab-vm-cloud-quorum-queue.lock| +LOCK_FREE=/run/lock/homelab-vm-cloud-quorum-queue.lock|true +LOCK_META=/run/lock/homelab-app-cloud-quorum-queue.lock|inode=15|600|root:root|2026-09-28 03:27:07.425060217 +0300 +LOCK_FUSER=/run/lock/homelab-app-cloud-quorum-queue.lock| +LOCK_FREE=/run/lock/homelab-app-cloud-quorum-queue.lock|true +homelab-backup- 1856481 FLOCK WRITE /run/homelab-backup-heavy.lock +=== ALERT_EMAIL_TIME_CORRELATION_0701_MSK === +2026-09-28T06:19:01.361470+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T06:36:20.423980+03:00 pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE +2026-09-28T06:36:20.424205+03:00 pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. +2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T07:06:48.304891+03:00 pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE +2026-09-28T07:06:48.305137+03:00 pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. +2026-09-28T06:15:16.319082+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +2026-09-28T06:15:20.661916+03:00 pve01 homelab-alertmanager-backup-run[1835343]: HEALTH_ALERT_STATE=OK +2026-09-28T06:15:20.666144+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +2026-09-28T06:30:42.414371+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +2026-09-28T06:30:46.367196+03:00 pve01 homelab-alertmanager-backup-run[1886013]: HEALTH_ALERT_STATE=OK +2026-09-28T06:30:46.370827+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +2026-09-28T06:46:01.347189+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +2026-09-28T06:46:05.406037+03:00 pve01 homelab-alertmanager-backup-run[1945314]: HEALTH_ALERT_STATE=OK +2026-09-28T06:46:05.410491+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +2026-09-28T07:01:32.217324+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: ALERT_EMAIL_ACCEPTED=YES +2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: HEALTH_ALERT_STATE=OK +2026-09-28T07:01:36.278152+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +2026-09-28T07:16:37.585569+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +2026-09-28T07:16:41.913014+03:00 pve01 homelab-alertmanager-backup-run[2048228]: HEALTH_ALERT_STATE=OK +2026-09-28T07:16:41.916768+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +=== ALERT_FREQUENCY_LAST12H === +ALERT_RUN_START_COUNT=0 +ALERT_EMAIL_ACCEPTED_COUNT=3 +RECOVERY_EMAIL_ACCEPTED_COUNT=0 +HEALTH_ALERT_STATE_OK_COUNT=47 +2026-09-27T19:32:04.991437+03:00 pve01 homelab-alertmanager-backup-run[3909103]: HEALTH_ALERT_STATE=OK +2026-09-27T19:47:36.328545+03:00 pve01 homelab-alertmanager-backup-run[3959645]: HEALTH_ALERT_STATE=OK +2026-09-27T20:02:46.359055+03:00 pve01 homelab-alertmanager-backup-run[4008840]: HEALTH_ALERT_STATE=OK +2026-09-27T20:18:05.225521+03:00 pve01 homelab-alertmanager-backup-run[4059785]: HEALTH_ALERT_STATE=OK +2026-09-27T20:33:20.585967+03:00 pve01 homelab-alertmanager-backup-run[4108890]: HEALTH_ALERT_STATE=OK +2026-09-27T20:48:36.460451+03:00 pve01 homelab-alertmanager-backup-run[4148152]: HEALTH_ALERT_STATE=OK +2026-09-27T21:04:08.100470+03:00 pve01 homelab-alertmanager-backup-run[4070]: HEALTH_ALERT_STATE=OK +2026-09-27T21:19:36.336976+03:00 pve01 homelab-alertmanager-backup-run[54137]: HEALTH_ALERT_STATE=OK +2026-09-27T21:35:05.706232+03:00 pve01 homelab-alertmanager-backup-run[104752]: HEALTH_ALERT_STATE=OK +2026-09-27T21:50:23.015203+03:00 pve01 homelab-alertmanager-backup-run[154365]: HEALTH_ALERT_STATE=OK +2026-09-27T22:05:36.461278+03:00 pve01 homelab-alertmanager-backup-run[202960]: HEALTH_ALERT_STATE=OK +2026-09-27T22:21:05.253758+03:00 pve01 homelab-alertmanager-backup-run[242449]: HEALTH_ALERT_STATE=OK +2026-09-27T22:36:15.399283+03:00 pve01 homelab-alertmanager-backup-run[292331]: HEALTH_ALERT_STATE=OK +2026-09-27T22:51:31.819800+03:00 pve01 homelab-alertmanager-backup-run[343850]: HEALTH_ALERT_STATE=OK +2026-09-27T23:06:53.759023+03:00 pve01 homelab-alertmanager-backup-run[396329]: HEALTH_ALERT_STATE=OK +2026-09-27T23:22:20.677831+03:00 pve01 homelab-alertmanager-backup-run[461385]: HEALTH_ALERT_STATE=OK +2026-09-27T23:37:36.973241+03:00 pve01 homelab-alertmanager-backup-run[513251]: HEALTH_ALERT_STATE=OK +2026-09-27T23:52:43.434843+03:00 pve01 homelab-alertmanager-backup-run[566888]: ALERT_EMAIL_ACCEPTED=YES +2026-09-27T23:52:43.434843+03:00 pve01 homelab-alertmanager-backup-run[566888]: HEALTH_ALERT_STATE=OK +2026-09-28T00:08:05.576769+03:00 pve01 homelab-alertmanager-backup-run[607129]: HEALTH_ALERT_STATE=OK +2026-09-28T00:23:21.212626+03:00 pve01 homelab-alertmanager-backup-run[660884]: HEALTH_ALERT_STATE=OK +2026-09-28T00:38:36.269886+03:00 pve01 homelab-alertmanager-backup-run[711652]: HEALTH_ALERT_STATE=OK +2026-09-28T00:53:55.974540+03:00 pve01 homelab-alertmanager-backup-run[762988]: HEALTH_ALERT_STATE=OK +2026-09-28T01:09:05.464301+03:00 pve01 homelab-alertmanager-backup-run[812499]: HEALTH_ALERT_STATE=OK +2026-09-28T01:24:16.127247+03:00 pve01 homelab-alertmanager-backup-run[862903]: HEALTH_ALERT_STATE=OK +2026-09-28T01:39:41.256513+03:00 pve01 homelab-alertmanager-backup-run[919688]: HEALTH_ALERT_STATE=OK +2026-09-28T01:54:54.298806+03:00 pve01 homelab-alertmanager-backup-run[958250]: HEALTH_ALERT_STATE=OK +2026-09-28T02:10:05.486220+03:00 pve01 homelab-alertmanager-backup-run[1009352]: HEALTH_ALERT_STATE=OK +2026-09-28T02:25:22.594856+03:00 pve01 homelab-alertmanager-backup-run[1058040]: HEALTH_ALERT_STATE=OK +2026-09-28T02:40:53.144072+03:00 pve01 homelab-alertmanager-backup-run[1108677]: HEALTH_ALERT_STATE=OK +2026-09-28T02:56:20.122916+03:00 pve01 homelab-alertmanager-backup-run[1157287]: HEALTH_ALERT_STATE=OK +2026-09-28T03:11:36.224287+03:00 pve01 homelab-alertmanager-backup-run[1207541]: HEALTH_ALERT_STATE=OK +2026-09-28T03:26:56.104164+03:00 pve01 homelab-alertmanager-backup-run[1267608]: HEALTH_ALERT_STATE=OK +2026-09-28T03:42:16.163161+03:00 pve01 homelab-alertmanager-backup-run[1337091]: HEALTH_ALERT_STATE=OK +2026-09-28T03:57:35.911704+03:00 pve01 homelab-alertmanager-backup-run[1388112]: HEALTH_ALERT_STATE=OK +2026-09-28T04:12:45.302638+03:00 pve01 homelab-alertmanager-backup-run[1439953]: ALERT_EMAIL_ACCEPTED=YES +2026-09-28T04:12:45.302638+03:00 pve01 homelab-alertmanager-backup-run[1439953]: HEALTH_ALERT_STATE=OK +2026-09-28T04:28:16.066840+03:00 pve01 homelab-alertmanager-backup-run[1491056]: HEALTH_ALERT_STATE=OK +2026-09-28T04:43:26.035041+03:00 pve01 homelab-alertmanager-backup-run[1543022]: HEALTH_ALERT_STATE=OK +2026-09-28T04:58:29.541024+03:00 pve01 homelab-alertmanager-backup-run[1593789]: HEALTH_ALERT_STATE=OK +2026-09-28T05:13:39.334685+03:00 pve01 homelab-alertmanager-backup-run[1645768]: HEALTH_ALERT_STATE=OK +2026-09-28T05:28:53.353807+03:00 pve01 homelab-alertmanager-backup-run[1685761]: HEALTH_ALERT_STATE=OK +2026-09-28T05:44:20.155523+03:00 pve01 homelab-alertmanager-backup-run[1736193]: HEALTH_ALERT_STATE=OK +2026-09-28T06:00:05.422013+03:00 pve01 homelab-alertmanager-backup-run[1785685]: HEALTH_ALERT_STATE=OK +2026-09-28T06:15:20.661916+03:00 pve01 homelab-alertmanager-backup-run[1835343]: HEALTH_ALERT_STATE=OK +2026-09-28T06:30:46.367196+03:00 pve01 homelab-alertmanager-backup-run[1886013]: HEALTH_ALERT_STATE=OK +2026-09-28T06:46:05.406037+03:00 pve01 homelab-alertmanager-backup-run[1945314]: HEALTH_ALERT_STATE=OK +2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: ALERT_EMAIL_ACCEPTED=YES +2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: HEALTH_ALERT_STATE=OK +2026-09-28T07:16:41.913014+03:00 pve01 homelab-alertmanager-backup-run[2048228]: HEALTH_ALERT_STATE=OK +=== AUDIT_FREQUENCY_LAST12H === +AUDIT_GREEN_SUMMARY_COUNT=7 +AUDIT_RED_SUMMARY_COUNT=17 +AUDIT_LOCK_BUSY_ISSUE_COUNT=9 +AUDIT_SOURCE_OFFHOST_ISSUE_COUNT=0 +2026-09-27T19:37:02.093711+03:00 pve01 homelab-backup-audit[3923461]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T20:06:15.522779+03:00 pve01 homelab-backup-audit[4022565]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T20:36:05.450336+03:00 pve01 homelab-backup-audit[4111037]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T21:06:02.094425+03:00 pve01 homelab-backup-audit[16838]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T21:37:05.002224+03:00 pve01 homelab-backup-audit[106421]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T22:06:07.004513+03:00 pve01 homelab-backup-audit[203921]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T22:35:26.212759+03:00 pve01 homelab-backup-audit[291167]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T23:06:54.987894+03:00 pve01 homelab-backup-audit[395212]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T23:36:37.162587+03:00 pve01 homelab-backup-audit[500802]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-27T23:36:37.162587+03:00 pve01 homelab-backup-audit[500802]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=6|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T00:06:21.135619+03:00 pve01 homelab-backup-audit[605334]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T00:06:21.135619+03:00 pve01 homelab-backup-audit[605334]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T00:36:10.445081+03:00 pve01 homelab-backup-audit[707255]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T00:36:10.445081+03:00 pve01 homelab-backup-audit[707255]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T01:06:01.997372+03:00 pve01 homelab-backup-audit[799187]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T01:36:02.006941+03:00 pve01 homelab-backup-audit[906059]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T02:05:36.356288+03:00 pve01 homelab-backup-audit[995508]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T02:36:15.265015+03:00 pve01 homelab-backup-audit[1094502]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T03:05:20.453132+03:00 pve01 homelab-backup-audit[1192518]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T03:35:20.147380+03:00 pve01 homelab-backup-audit[1320691]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T03:35:20.147380+03:00 pve01 homelab-backup-audit[1320691]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T04:06:45.928910+03:00 pve01 homelab-backup-audit[1424254]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T04:06:45.928910+03:00 pve01 homelab-backup-audit[1424254]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T04:35:20.240415+03:00 pve01 homelab-backup-audit[1514892]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T04:35:20.240415+03:00 pve01 homelab-backup-audit[1514892]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T05:05:52.974415+03:00 pve01 homelab-backup-audit[1617767]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T05:05:52.974415+03:00 pve01 homelab-backup-audit[1617767]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T05:37:02.145613+03:00 pve01 homelab-backup-audit[1720751]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T06:06:02.032037+03:00 pve01 homelab-backup-audit[1807928]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +=== HEALTH_GENERATION_CORRELATION === +HEALTH_GENERATED_AT_EPOCH=1790568997 +HEALTH_CORRELATION_FROM=1790568817 +HEALTH_CORRELATION_TO=1790569177 +=== FRESH_QUIET_AUDIT === +QUIET_AUDIT_WAIT|attempt=1|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=2|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=3|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=4|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=5|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=6|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=7|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=8|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=9|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=10|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=11|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=12|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=13|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=14|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=15|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=16|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=17|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=18|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=19|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=20|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=21|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=22|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=23|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=24|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=25|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=26|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=27|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=28|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=29|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=30|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=31|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=32|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=33|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=34|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=35|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=36|reason=SERVICE_BUSY|active=activating|pid=1856481 +FRESH_QUIET_AUDIT_RC=NO_QUIET_WINDOW +FRESH_AUDIT_HEALTH_NO_WRITE=false +=== CANONICAL_ALERT_SOURCE_AUTHORITY === +CANONICAL_ALERT_PATH=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run +CANONICAL_ALERT_EXISTS=true +CANONICAL_ALERT_SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a +CANONICAL_ALERT_MATCH_LIVE=false +CANONICAL_SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +CANONICAL_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +CANONICAL_SOURCE_EXACT=true +TASK_RESULT=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED +DECISION=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED +NEXT_GATE=READ_EXACT_1237_AND_FIX_ONLY_PROVEN_ALERT_NOISE_CAUSE +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index f1d5810e..df9be427 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260928-HOMELAB-BACKUP-V2-FINAL-ACCEPTANCE-CLOSEOUT-1236R1", + "command_id": "SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "homelab-backup-v2-final-acceptance-closeout", - "started_at_utc": "2026-09-28T04:24:30Z", - "finished_at_utc": "2026-09-28T04:24:32Z", + "component": "homelab-backup-alert-spam-current-rca", + "started_at_utc": "2026-09-28T04:30:51Z", + "finished_at_utc": "2026-09-28T04:34:02Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "ae0922db9c4c4ecb922eff1bc2dbc83ca34800f18e36edd72d9ff12921aff878", + "command_sha256": "e205fb5749bd8d2a466e79a56e8d9f3a66537e7dcbd1c635202ab6df607b736b", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "92d863c807a78fd412105892222e8b6204cabb0751fc53ff09613fbe11d0394c", - "sanitized_output_sha256": "92d863c807a78fd412105892222e8b6204cabb0751fc53ff09613fbe11d0394c", + "raw_evidence_sha256": "c64c77856d1b796c0414be59dd0089c9687ec505d26bb8b4111ab68ee967e5a2", + "sanitized_output_sha256": "c64c77856d1b796c0414be59dd0089c9687ec505d26bb8b4111ab68ee967e5a2", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "BACKUP_V2_FINAL_ACCEPTANCE_CLOSEOUT_BEGIN=true\nCOMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-V2-FINAL-ACCEPTANCE-CLOSEOUT-1236R1\nMODE=read-only\nCOMPONENT=homelab-backup-v2-final-acceptance-closeout\nMUTATION_BOUNDARY=FINAL_OBSERVATIONAL_ACCEPTANCE_ONLY;NO_BACKUP;NO_PERSISTENT_LOCK_OWNERSHIP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_SYSTEMD_ACTION;NO_TIMER_CHANGE;NO_AUDIT_HEALTH_WRITE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== SOURCE_AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1235_STATUS=OK\nSOURCE_1235_RC=0\nSOURCE_1235_DECISION=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN\nSOURCE_1235_NEXT_GATE=FINAL_BACKUP_ACCEPTANCE_CLOSEOUT\nSOURCE_1235_FINAL_AUDIT_RC=0\nSOURCE_1235_FINAL_ISSUE_COUNT=0\nSOURCE_1235_FINAL_RPO_COUNT=0\nSOURCE_1235_FINAL_NON_RPO_COUNT=0\nSOURCE_1235_TARGETED_REFRESH_COUNT=11\nSOURCE_1235_SUCCESS_MAIL_POLICY_ACCEPTED=true\nSOURCE_1235_RECOVERY_END_EPOCH=1790545685\n=== LIVE_IMMUTABLE_IDENTITIES ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nRELEASE_META=500:root:root\nRELEASE_FILESET=entry504.py fresh_lxc_helper.py lxc_adapter.py producer_exec_helper.py qemu_adapter.py scheduler_adapter.py\nRELEASE_FILESET_EXPECTED=entry504.py fresh_lxc_helper.py lxc_adapter.py producer_exec_helper.py qemu_adapter.py scheduler_adapter.py\nRELEASE_FILE=fresh_lxc_helper.py SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 META=400:root:root\nRELEASE_FILE=producer_exec_helper.py SHA256=b3b97099bcd7126670f866e81faae6a42a96d869d9989e6a67f6ea6c871c90b3 META=400:root:root\nRELEASE_FILE=lxc_adapter.py SHA256=3fa7d194010d76f7c8bc95035ae8876262e9337903e310f45f45df88c266dc8d META=400:root:root\nRELEASE_FILE=qemu_adapter.py SHA256=1906db42bd34bbe85eb46fd9797b3572d46e0a41c29d1c29be5f9df5e448f15b META=400:root:root\nRELEASE_FILE=scheduler_adapter.py SHA256=a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96 META=400:root:root\nRELEASE_FILE=entry504.py SHA256=ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793 META=400:root:root\n=== ROLLBACK_AUTHORITY ===\nROLLBACK_PATH=/var/lib/homelab-change-rollback/SUPPORT-260927-HOMELAB-BACKUP-NOTIFICATION-COMPAT-HOTFIX-1230R1/homelab-backup-v2.before\nROLLBACK_EXISTS=true\nROLLBACK_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\n=== CANONICAL_SOURCE_AUTHORITY ===\nSOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_WORKTREE_CLEAN=true\n=== UNIT_AND_TIMER_CURRENT ===\nUNIT=homelab-backup-v2.service ACTIVE=activating ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=1856481\nUNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=\nUNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code EXEC_MAIN_STATUS=1 MAINPID=0\nUNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=\n=== POST_RECOVERY_SCHEDULER_JOURNAL ===\n2026-09-28T01:12:52.217428+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-28T01:25:14.480529+03:00 pve01 homelab-backup-v2[815211]: SCHEDULE_TARGET=xf-codevipe\n2026-09-28T01:25:14.480529+03:00 pve01 homelab-backup-v2[815211]: SCHEDULE_RESULT=SUCCESS\n2026-09-28T01:25:14.488885+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-28T01:49:49.378146+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-28T02:03:38.623364+03:00 pve01 homelab-backup-v2[943938]: SCHEDULE_TARGET=xf-zakrutim\n2026-09-28T02:03:38.623364+03:00 pve01 homelab-backup-v2[943938]: SCHEDULE_RESULT=SUCCESS\n2026-09-28T02:03:38.630266+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-28T02:29:32.216618+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-28T02:29:49.327002+03:00 pve01 homelab-backup-v2[1072063]: SCHEDULE_TARGET=vm210\n2026-09-28T02:30:55.417267+03:00 pve01 homelab-backup-v2[1072063]: SCHEDULE_RESULT=SUCCESS\n2026-09-28T02:30:55.424354+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-28T03:01:22.218704+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-28T03:01:54.164165+03:00 pve01 homelab-backup-v2[1179144]: SCHEDULE_TARGET=vm211\n2026-09-28T03:03:04.842495+03:00 pve01 homelab-backup-v2[1179144]: SCHEDULE_RESULT=SUCCESS\n2026-09-28T03:03:04.849383+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-28T03:34:32.216619+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-28T05:14:16.727192+03:00 pve01 homelab-backup-v2[1320207]: SCHEDULE_TARGET=xf-dsmods\n2026-09-28T05:14:16.727192+03:00 pve01 homelab-backup-v2[1320207]: SCHEDULE_RESULT=SUCCESS\n2026-09-28T05:14:16.733465+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-28T05:14:16.735708+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-28T05:24:43.293853+03:00 pve01 homelab-backup-v2[1647429]: SCHEDULE_TARGET=xf-kingofwolk\n2026-09-28T05:24:43.293853+03:00 pve01 homelab-backup-v2[1647429]: SCHEDULE_RESULT=SUCCESS\n2026-09-28T05:24:43.300102+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-28T05:45:12.220837+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-28T05:52:51.548422+03:00 pve01 homelab-backup-v2[1737475]: SCHEDULE_TARGET=xf-newfi\n2026-09-28T05:52:51.548422+03:00 pve01 homelab-backup-v2[1737475]: SCHEDULE_RESULT=SUCCESS\n2026-09-28T05:52:51.557165+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\n2026-09-28T06:19:01.361470+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\nPOST_RECOVERY_V2_START_COUNT=8\nPOST_RECOVERY_V2_FINISHED_COUNT=7\nPOST_RECOVERY_V2_FAILED_COUNT=0\nPOST_RECOVERY_FRESH_HELPER_FAILED_COUNT=0\nPOST_RECOVERY_SCHEDULE_SUCCESS_COUNT=7\nPOST_RECOVERY_SCHEDULE_DEFER_COUNT=0\nV2_SERVICE_RESULT=success\nV2_SERVICE_EXEC_MAIN_STATUS=0\nV2_SERVICE_ACTIVE=activating\nBACKUP_PROJECT_COMPLETE=false\nTASK_RESULT=HOLD_BACKUP_V2_FINAL_ACCEPTANCE_CLOSEOUT\nDECISION=HOLD_1236_V2_SERVICE_LAST_RUN_NOT_SUCCESS\nNEXT_GATE=RECONCILE_POST_RECOVERY_SCHEDULER\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260928-HOMELAB-BACKUP-V2-FINAL-ACCEPTANCE-CLOSEOUT-1236R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "BACKUP_ALERT_SPAM_CURRENT_RCA_BEGIN=true\nCOMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1\nMODE=read-only\nCOMPONENT=homelab-backup-alert-spam-current-rca\nMUTATION_BOUNDARY=READ_ONLY_ALERT_HEALTH_STATE_SOURCE_JOURNAL_LOCK_AND_QUIET_AUDIT;NO_ALERT_EXECUTION;NO_MAIL;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_SYSTEMD_ACTION;NO_TIMER_CHANGE;NO_HEALTH_WRITE;NO_GIT_WRITE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1236_STATUS=OK\nSOURCE_1236_RC=0\nSOURCE_1236_CHANGES_MADE=false\nSOURCE_1236_DECISION=HOLD_1236_V2_SERVICE_LAST_RUN_NOT_SUCCESS\nSOURCE_1236_NEXT_GATE=RECONCILE_POST_RECOVERY_SCHEDULER\nSOURCE_1236_POST_RECOVERY_V2_FAILED_COUNT=0\nSOURCE_1236_POST_RECOVERY_FRESH_HELPER_FAILED_COUNT=0\nSOURCE_1236_POST_RECOVERY_SCHEDULE_SUCCESS_COUNT=7\n=== LIVE_IDENTITIES ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nALERT_SCRIPT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9\n=== CURRENT_UNIT_STATE ===\nUNIT=homelab-backup-v2.service ACTIVE=activating ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=1856481\nUNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=\nUNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code EXEC_MAIN_STATUS=1 MAINPID=0\nUNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=\nUNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=0\nUNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID=\n=== CURRENT_HEALTH_FILES ===\nHEALTH1_EXISTS=true\nHEALTH1_SHA256=d89fb55b92f2b9908e2d2d8da3b9a0d0db6d2fe84c6e6d37bcec1a86e4384222\nHEALTH1_JSON_OK=true\nHEALTH1_SCHEMA=homelab-backup-audit-v3\nHEALTH1_STATUS=RED\nHEALTH1_GENERATED_AT_EPOCH=1790568997\nHEALTH1_AGE_SECONDS=855\nHEALTH1_ISSUE_COUNT=2\nHEALTH1_ISSUE_1={\"code\":\"LOCK_BUSY\",\"logical_id\":\"/run/homelab-backup-heavy.lock\",\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\",\"severity\":\"HIGH\"}\nHEALTH1_ISSUE_2={\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\",\"severity\":\"MEDIUM\"}\nHEALTH1_SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nHEALTH1_SOURCE_REMOTE_HEAD=None\nHEALTH1_SOURCE_PUSHURL_DISABLED=true\nHEALTH1_SOURCE_EXACT=false\nHEALTH2_EXISTS=true\nHEALTH2_SHA256=d89fb55b92f2b9908e2d2d8da3b9a0d0db6d2fe84c6e6d37bcec1a86e4384222\nHEALTH2_JSON_OK=true\nHEALTH2_SCHEMA=homelab-backup-audit-v3\nHEALTH2_STATUS=RED\nHEALTH2_GENERATED_AT_EPOCH=1790568997\nHEALTH2_AGE_SECONDS=855\nHEALTH2_ISSUE_COUNT=2\nHEALTH2_ISSUE_1={\"code\":\"LOCK_BUSY\",\"logical_id\":\"/run/homelab-backup-heavy.lock\",\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\",\"severity\":\"HIGH\"}\nHEALTH2_ISSUE_2={\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\",\"severity\":\"MEDIUM\"}\nHEALTH2_SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nHEALTH2_SOURCE_REMOTE_HEAD=None\nHEALTH2_SOURCE_PUSHURL_DISABLED=true\nHEALTH2_SOURCE_EXACT=false\nHEALTH_FILES_IDENTICAL=true\nHEALTH_EPOCH_VALID=true\n=== CURRENT_ALERT_STATE ===\nALERT_STATE_EXISTS=true\nALERT_STATE_SHA256=493ce50563cb31030f2b16ff4ad722a7ac5ec211a77b2e89625491c057e8827d\nALERT_STATE_JSON_OK=true\nALERT_STATE_KEYS=code_counts,issue_keys,migration_silent,notify_tokens,raw_issue_keys,schema,updated_epoch\nALERT_STATE_SAFE_JSON={\"code_counts\":{\"LOCK_BUSY\":4,\"SOURCE_OFFHOST_MISSING\":263},\"issue_keys\":[\"LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK\",\"SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT\"],\"migration_silent\":false,\"notify_tokens\":\"[REDACTED]\",\"raw_issue_keys\":[\"LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK\",\"SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT\"],\"schema\":\"homelab-backup-alert-state-v2\",\"updated_epoch\":1790569001}\nALERT_STATE_SAFE_TRUNCATED=false\nALERT_STATE_FIELD_SCHEMA=\"homelab-backup-alert-state-v2\"\nALERT_STATE_FIELD_UPDATED_EPOCH=1790569001\nALERT_STATE_FIELD_ISSUE_KEYS=[\"LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK\",\"SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT\"]\n=== ALERT_FILTER_SOURCE ===\nALERT_EMBEDDED_SOURCE_FOUND=true\nALERT_EMBEDDED_SOURCE_SHA256=344306105083cd8bc64e19d451f7f66c17accc43a2f413538d2f9e2687b2540e\nALERT_EMBEDDED_SOURCE_BEGIN\n0001:import sys,os,json,re,subprocess,pathlib,time\n0002:mode=sys.argv[1]\n0003:AUDIT=\"/usr/local/sbin/homelab-backup-audit\"\n0004:HEALTH=pathlib.Path(\"/var/lib/homelab-health/backup-audit-v2.json\")\n0005:STATE=pathlib.Path(\"/var/lib/homelab-alerting/backup-alert-state.json\")\n0006:USERCFG=pathlib.Path(\"/etc/pve/user.cfg\")\n0007:SCHEMA=\"homelab-backup-alert-state-v2\"\n0008:THRESHOLDS={\"RPO_BREACH\":2,\"SOURCE_OFFHOST_MISSING\":2,\"LOCK_BUSY\":3}\n0009:def recipient():\n0010: text=USERCFG.read_text(encoding=\"utf-8\",errors=\"replace\")\n0011: m=re.findall(r\"(?i)\\b[A-Z0-9._+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}\\b\",text)\n0012: if len(m)!=1: raise SystemExit(65)\n0013: return m[0]\n0014:def send(subject,body):\n0015: rcpt=recipient()\n0016: msg=f\"From: h***@gram1.ru\\nTo: {rcpt}\\nSubject: {subject}\\nContent-Type: text/plain; charset=UTF-8\\n\\n{body}\\n\"\n0017: p=subprocess.run([\"/usr/sbin/sendmail\",\"-t\"],input=msg,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)\n0018: if p.returncode!=0: raise SystemExit(p.returncode or 75)\n0019:def key(i):\n0020: return \"|\".join(str(i.get(x,\"\")) for x in (\"code\",\"logical_id\",\"root_cause\"))\n0021:def token(i):[REDACTED]\n0022: code=str(i.get(\"code\",\"\"))\n0023: return \"group:\"+code if code in THRESHOLDS else \"issue:\"+key(i)\n0024:if mode==\"email-test\":\n0025: send(\"Workers2 backup alerting test\",\"Workers2 backup/DR alerting route test. If you received this, the local MTA accepted the controlled test generated during backup closure.\")\n0026: print(\"EMAIL_TEST_ACCEPTED=YES\")\n0027: raise SystemExit(0)\n0028:if mode!=\"health-alert\": raise SystemExit(64)\n0029:p=subprocess.run([AUDIT,\"write-health\"],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)\n0030:if p.returncode not in (0,1): raise SystemExit(p.returncode or 75)\n0031:try:\n0032: h=json.loads(HEALTH.read_text(encoding=\"utf-8\"))\n0033:except Exception:\n0034: raise SystemExit(69)\n0035:issues=h.get(\"issues\") if isinstance(h.get(\"issues\"),list) else []\n0036:cur={key(i):i for i in issues if isinstance(i,dict)}\n0037:codes=set(str(i.get(\"code\",\"\")) for i in cur.values())\n0038:oldj={}\n0039:if STATE.is_file():\n0040: try:\n0041: oldj=json.loads(STATE.read_text(encoding=\"utf-8\"))\n0042: if not isinstance(oldj,dict): oldj={}\n0043: except Exception:\n0044: oldj={}\n0045:migrating=oldj.get(\"schema\")!=SCHEMA\n0046:prev_counts=oldj.get(\"code_counts\") if isinstance(oldj.get(\"code_counts\"),dict) and not migrating else {}\n0047:counts={}\n0048:for code in codes:\n0049: try: prev=int(prev_counts.get(code,0))\n0050: except Exception: prev=0\n0051: counts[code]=prev+1\n0052:actionable={}\n0053:for k,i in cur.items():\n0054: code=str(i.get(\"code\",\"\"))\n0055: need=int(THRESHOLDS.get(code,1))\n0056: if counts.get(code,0)>=need:\n0057: actionable[k]=i\n0058:tokens=[REDACTED]\n0059:old_tokens=[REDACTED]\n0060:if not migrating:\n0061: old_tokens=[REDACTED]\n0062:added=[REDACTED]\n0063:removed=[REDACTED]\n0064:if added:\n0065: lines=[\n0066: \"Workers2 backup/DR actionable alert state changed.\",\n0067: \"\",\n0068: \"Transient debounce policy: RPO_BREACH=2 checks; SOURCE_OFFHOST_MISSING=2 checks; LOCK_BUSY=3 checks; all other issue codes=immediate.\",\n0069: \"\"\n0070: ]\n0071: for k in sorted(actionable):\n0072: i=actionable[k]\n0073: lines.append(f\"{i.get('severity','UNKNOWN')} {i.get('code','UNKNOWN')} logical_id={i.get('logical_id','-')} root_cause={i.get('root_cause','UNKNOWN')}\")\n0074: suppressed=len(cur)-len(actionable)\n0075: if suppressed:\n0076: lines.extend([\"\",f\"Suppressed transient issue(s) still below threshold: {suppressed}\"])\n0077: send(f\"[Workers2 BACKUP ALERT] {len(actionable)} actionable issue(s)\",\"\\n\".join(lines))\n0078: print(\"ALERT_EMAIL_ACCEPTED=YES\")\n0079:elif old_tokens and not tokens:[REDACTED]\n0080: send(\"[Workers2 BACKUP RECOVERY] all actionable issues cleared\",\"Workers2 backup/DR actionable audit state is clear. All previously notified actionable backup issues are cleared.\")\n0081: print(\"RECOVERY_EMAIL_ACCEPTED=YES\")\n0082:STATE.parent.mkdir(parents=True,exist_ok=True)\n0083:tmp=STATE.with_name(STATE.name+\".tmp\")\n0084:payload={\n0085: \"schema\":SCHEMA,\n0086: \"updated_epoch\":int(time.time()),\n0087: \"issue_keys\":sorted(actionable),\n0088: \"notify_tokens\":[REDACTED]\n0089: \"raw_issue_keys\":sorted(cur),\n0090: \"code_counts\":{k:int(v) for k,v in sorted(counts.items())},\n0091: \"migration_silent\":bool(migrating)\n0092:}\n0093:tmp.write_text(json.dumps(payload,separators=(\",\",\":\"),sort_keys=True)+\"\\n\",encoding=\"utf-8\")\n0094:os.chmod(tmp,0o600)\n0095:os.replace(tmp,STATE)\n0096:print(\"ALERT_FILTER_SCHEMA=\"+SCHEMA)\n0097:print(\"ALERT_RAW_ISSUE_COUNT=\"+str(len(cur)))\n0098:print(\"ALERT_ACTIONABLE_ISSUE_COUNT=\"+str(len(actionable)))\n0099:print(\"ALERT_NOTIFY_TOKEN_COUNT=[REDACTED]\n0100:print(\"ALERT_ADDED_TOKEN_COUNT=[REDACTED]\n0101:print(\"ALERT_REMOVED_TOKEN_COUNT=[REDACTED]\n0102:print(\"ALERT_SUPPRESSED_TRANSIENT_COUNT=\"+str(len(cur)-len(actionable)))\n0103:print(\"ALERT_STATE_MIGRATION_SILENT=\"+(\"true\" if migrating else \"false\"))\n0104:print(\"HEALTH_ALERT_STATE=OK\")\nALERT_EMBEDDED_SOURCE_END\n=== AUDIT_SOURCE_RELEVANT ===\n183- (\"homelab-alertmanager-backup-health.timer\",\"ALERT_TIMER_BAD\"),\n184-]:\n185- ok,a,e=unit_ok(unit)\n186- if not ok: issues.append({\"severity\":\"HIGH\",\"code\":code,\"logical_id\":unit,\"root_cause\":\"UNIT_\"+a+\"_\"+e})\n187-\n188:for lock in [\"/run/homelab-backup-heavy.lock\",\"/run/lock/homelab-app-cloud-quorum-queue.lock\",\"/run/lock/homelab-vm-cloud-quorum-queue.lock\",\"/run/lock/homelab-backup-dispatcher.lock\"]:\n189- p=pathlib.Path(lock)\n190- if p.exists():\n191: lrc,_,_=run([\"flock\",\"-n\",str(p),\"-c\",\"true\"],5)\n192: if lrc!=0: issues.append({\"severity\":\"HIGH\",\"code\":\"LOCK_BUSY\",\"logical_id\":lock,\"root_cause\":\"ACTIVE_OR_ORPHAN_LOCK\"})\n193-\n194-env=os.environ.copy(); env[\"HOMELAB_BACKUP_GFS_POLICY\"]=GFSP; env[\"HOMELAB_BACKUP_TARGET_REGISTRY\"]=str(REG)\n195-grc,gout,_=run([GFSH,\"validate\"],20,env)\n196-if grc!=0 or \"GFS_POLICY=OK\" not in gout:\n197- issues.append({\"severity\":\"CRITICAL\",\"code\":\"GFS_INVALID\",\"logical_id\":\"-\",\"root_cause\":\"GFS_VALIDATE_FAILED\"})\n--\n287-except Exception:\n288- cloudb_ok=False\n289-if not cloudb_ok:\n290- issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK\"})\n291-\n292:source={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\n293-hrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\n294:if hrc==0: source[\"local_head\"]=ho.strip()\n295-frc,fo,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.url\"],10)\n296-prc,po,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.pushurl\"],10)\n297-source[\"pushurl_disabled\"]=(prc==0 and po.strip()==\"DISABLED\")\n298-if frc==0 and fo.strip():\n299- lrc,lo,_=run([\"git\",\"ls-remote\",fo.strip(),\"refs/heads/\"+BRANCH],30)\n300: if lrc==0 and lo.strip(): source[\"remote_head\"]=lo.split()[0]\n301:if source[\"remote_head\"]!=source[\"local_head\"]:\n302: issues.append({\"severity\":\"MEDIUM\",\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\"})\n303-\n304-status=\"GREEN\" if not issues else \"RED\"\n305-for t in targets:\n306- print(\"TARGET|\"+\"|\".join(f\"{k}={v}\" for k,v in t.items() if v is not None))\n307-for i in issues:\n308- print(\"ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}\".format(**i))\n309:print(f\"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}\")\n310-health={\"schema\":\"homelab-backup-audit-v3\",\"status\":status,\"generated_at_epoch\":now,\"targets\":targets,\"issues\":issues,\"cloud\":cloud,\"source\":source,\"cloud_b_restore_drill\":cloudb_ok,\"dr_proven\":bool(isinstance(dr,dict) and dr.get(\"status\")==\"SEALED\")}\n311-if MODE in (\"write-health\",\"scheduled\"):\n312- for hp in HEALTHS:\n313- hp.parent.mkdir(parents=True,exist_ok=True)\n314- tmp=hp.with_name(hp.name+\".tmp\")\n=== LIVE_SOURCE_HEAD_RECHECK ===\nSOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_WORKTREE_CLEAN=true\nSOURCE_ORIGIN_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9\nSOURCE_REMOTE_ATTEMPT=1 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true\nSOURCE_REMOTE_ATTEMPT=2 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true\nSOURCE_REMOTE_ATTEMPT=3 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true\nSOURCE_REMOTE_ATTEMPT=4 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true\nSOURCE_REMOTE_ATTEMPT=5 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true\n=== CURRENT_LOCK_OWNERSHIP ===\nLOCK_META=/run/homelab-backup-heavy.lock|inode=3474|600|root:root|2026-09-28 03:27:07.425060217 +0300\nLOCK_FUSER=/run/homelab-backup-heavy.lock| 1856481\nLOCK_FREE=/run/homelab-backup-heavy.lock|false\nLOCK_META=/run/lock/homelab-backup-dispatcher.lock|inode=9|600|root:root|2026-09-28 03:27:07.423060188 +0300\nLOCK_FUSER=/run/lock/homelab-backup-dispatcher.lock|\nLOCK_FREE=/run/lock/homelab-backup-dispatcher.lock|true\nLOCK_META=/run/lock/homelab-vm-cloud-quorum-queue.lock|inode=2944|600|root:root|2026-09-28 03:27:07.424060203 +0300\nLOCK_FUSER=/run/lock/homelab-vm-cloud-quorum-queue.lock|\nLOCK_FREE=/run/lock/homelab-vm-cloud-quorum-queue.lock|true\nLOCK_META=/run/lock/homelab-app-cloud-quorum-queue.lock|inode=15|600|root:root|2026-09-28 03:27:07.425060217 +0300\nLOCK_FUSER=/run/lock/homelab-app-cloud-quorum-queue.lock|\nLOCK_FREE=/run/lock/homelab-app-cloud-quorum-queue.lock|true\nhomelab-backup- 1856481 FLOCK WRITE /run/homelab-backup-heavy.lock\n=== ALERT_EMAIL_TIME_CORRELATION_0701_MSK ===\n2026-09-28T06:19:01.361470+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\n2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T06:36:20.423980+03:00 pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-28T06:36:20.424205+03:00 pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\n2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T07:06:48.304891+03:00 pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\n2026-09-28T07:06:48.305137+03:00 pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\n2026-09-28T06:15:16.319082+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\n2026-09-28T06:15:20.661916+03:00 pve01 homelab-alertmanager-backup-run[1835343]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:15:20.666144+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\n2026-09-28T06:30:42.414371+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\n2026-09-28T06:30:46.367196+03:00 pve01 homelab-alertmanager-backup-run[1886013]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:30:46.370827+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\n2026-09-28T06:46:01.347189+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\n2026-09-28T06:46:05.406037+03:00 pve01 homelab-alertmanager-backup-run[1945314]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:46:05.410491+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\n2026-09-28T07:01:32.217324+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\n2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: ALERT_EMAIL_ACCEPTED=YES\n2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: HEALTH_ALERT_STATE=OK\n2026-09-28T07:01:36.278152+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\n2026-09-28T07:16:37.585569+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\n2026-09-28T07:16:41.913014+03:00 pve01 homelab-alertmanager-backup-run[2048228]: HEALTH_ALERT_STATE=OK\n2026-09-28T07:16:41.916768+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\n=== ALERT_FREQUENCY_LAST12H ===\nALERT_RUN_START_COUNT=0\nALERT_EMAIL_ACCEPTED_COUNT=3\nRECOVERY_EMAIL_ACCEPTED_COUNT=0\nHEALTH_ALERT_STATE_OK_COUNT=47\n2026-09-27T19:32:04.991437+03:00 pve01 homelab-alertmanager-backup-run[3909103]: HEALTH_ALERT_STATE=OK\n2026-09-27T19:47:36.328545+03:00 pve01 homelab-alertmanager-backup-run[3959645]: HEALTH_ALERT_STATE=OK\n2026-09-27T20:02:46.359055+03:00 pve01 homelab-alertmanager-backup-run[4008840]: HEALTH_ALERT_STATE=OK\n2026-09-27T20:18:05.225521+03:00 pve01 homelab-alertmanager-backup-run[4059785]: HEALTH_ALERT_STATE=OK\n2026-09-27T20:33:20.585967+03:00 pve01 homelab-alertmanager-backup-run[4108890]: HEALTH_ALERT_STATE=OK\n2026-09-27T20:48:36.460451+03:00 pve01 homelab-alertmanager-backup-run[4148152]: HEALTH_ALERT_STATE=OK\n2026-09-27T21:04:08.100470+03:00 pve01 homelab-alertmanager-backup-run[4070]: HEALTH_ALERT_STATE=OK\n2026-09-27T21:19:36.336976+03:00 pve01 homelab-alertmanager-backup-run[54137]: HEALTH_ALERT_STATE=OK\n2026-09-27T21:35:05.706232+03:00 pve01 homelab-alertmanager-backup-run[104752]: HEALTH_ALERT_STATE=OK\n2026-09-27T21:50:23.015203+03:00 pve01 homelab-alertmanager-backup-run[154365]: HEALTH_ALERT_STATE=OK\n2026-09-27T22:05:36.461278+03:00 pve01 homelab-alertmanager-backup-run[202960]: HEALTH_ALERT_STATE=OK\n2026-09-27T22:21:05.253758+03:00 pve01 homelab-alertmanager-backup-run[242449]: HEALTH_ALERT_STATE=OK\n2026-09-27T22:36:15.399283+03:00 pve01 homelab-alertmanager-backup-run[292331]: HEALTH_ALERT_STATE=OK\n2026-09-27T22:51:31.819800+03:00 pve01 homelab-alertmanager-backup-run[343850]: HEALTH_ALERT_STATE=OK\n2026-09-27T23:06:53.759023+03:00 pve01 homelab-alertmanager-backup-run[396329]: HEALTH_ALERT_STATE=OK\n2026-09-27T23:22:20.677831+03:00 pve01 homelab-alertmanager-backup-run[461385]: HEALTH_ALERT_STATE=OK\n2026-09-27T23:37:36.973241+03:00 pve01 homelab-alertmanager-backup-run[513251]: HEALTH_ALERT_STATE=OK\n2026-09-27T23:52:43.434843+03:00 pve01 homelab-alertmanager-backup-run[566888]: ALERT_EMAIL_ACCEPTED=YES\n2026-09-27T23:52:43.434843+03:00 pve01 homelab-alertmanager-backup-run[566888]: HEALTH_ALERT_STATE=OK\n2026-09-28T00:08:05.576769+03:00 pve01 homelab-alertmanager-backup-run[607129]: HEALTH_ALERT_STATE=OK\n2026-09-28T00:23:21.212626+03:00 pve01 homelab-alertmanager-backup-run[660884]: HEALTH_ALERT_STATE=OK\n2026-09-28T00:38:36.269886+03:00 pve01 homelab-alertmanager-backup-run[711652]: HEALTH_ALERT_STATE=OK\n2026-09-28T00:53:55.974540+03:00 pve01 homelab-alertmanager-backup-run[762988]: HEALTH_ALERT_STATE=OK\n2026-09-28T01:09:05.464301+03:00 pve01 homelab-alertmanager-backup-run[812499]: HEALTH_ALERT_STATE=OK\n2026-09-28T01:24:16.127247+03:00 pve01 homelab-alertmanager-backup-run[862903]: HEALTH_ALERT_STATE=OK\n2026-09-28T01:39:41.256513+03:00 pve01 homelab-alertmanager-backup-run[919688]: HEALTH_ALERT_STATE=OK\n2026-09-28T01:54:54.298806+03:00 pve01 homelab-alertmanager-backup-run[958250]: HEALTH_ALERT_STATE=OK\n2026-09-28T02:10:05.486220+03:00 pve01 homelab-alertmanager-backup-run[1009352]: HEALTH_ALERT_STATE=OK\n2026-09-28T02:25:22.594856+03:00 pve01 homelab-alertmanager-backup-run[1058040]: HEALTH_ALERT_STATE=OK\n2026-09-28T02:40:53.144072+03:00 pve01 homelab-alertmanager-backup-run[1108677]: HEALTH_ALERT_STATE=OK\n2026-09-28T02:56:20.122916+03:00 pve01 homelab-alertmanager-backup-run[1157287]: HEALTH_ALERT_STATE=OK\n2026-09-28T03:11:36.224287+03:00 pve01 homelab-alertmanager-backup-run[1207541]: HEALTH_ALERT_STATE=OK\n2026-09-28T03:26:56.104164+03:00 pve01 homelab-alertmanager-backup-run[1267608]: HEALTH_ALERT_STATE=OK\n2026-09-28T03:42:16.163161+03:00 pve01 homelab-alertmanager-backup-run[1337091]: HEALTH_ALERT_STATE=OK\n2026-09-28T03:57:35.911704+03:00 pve01 homelab-alertmanager-backup-run[1388112]: HEALTH_ALERT_STATE=OK\n2026-09-28T04:12:45.302638+03:00 pve01 homelab-alertmanager-backup-run[1439953]: ALERT_EMAIL_ACCEPTED=YES\n2026-09-28T04:12:45.302638+03:00 pve01 homelab-alertmanager-backup-run[1439953]: HEALTH_ALERT_STATE=OK\n2026-09-28T04:28:16.066840+03:00 pve01 homelab-alertmanager-backup-run[1491056]: HEALTH_ALERT_STATE=OK\n2026-09-28T04:43:26.035041+03:00 pve01 homelab-alertmanager-backup-run[1543022]: HEALTH_ALERT_STATE=OK\n2026-09-28T04:58:29.541024+03:00 pve01 homelab-alertmanager-backup-run[1593789]: HEALTH_ALERT_STATE=OK\n2026-09-28T05:13:39.334685+03:00 pve01 homelab-alertmanager-backup-run[1645768]: HEALTH_ALERT_STATE=OK\n2026-09-28T05:28:53.353807+03:00 pve01 homelab-alertmanager-backup-run[1685761]: HEALTH_ALERT_STATE=OK\n2026-09-28T05:44:20.155523+03:00 pve01 homelab-alertmanager-backup-run[1736193]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:00:05.422013+03:00 pve01 homelab-alertmanager-backup-run[1785685]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:15:20.661916+03:00 pve01 homelab-alertmanager-backup-run[1835343]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:30:46.367196+03:00 pve01 homelab-alertmanager-backup-run[1886013]: HEALTH_ALERT_STATE=OK\n2026-09-28T06:46:05.406037+03:00 pve01 homelab-alertmanager-backup-run[1945314]: HEALTH_ALERT_STATE=OK\n2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: ALERT_EMAIL_ACCEPTED=YES\n2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: HEALTH_ALERT_STATE=OK\n2026-09-28T07:16:41.913014+03:00 pve01 homelab-alertmanager-backup-run[2048228]: HEALTH_ALERT_STATE=OK\n=== AUDIT_FREQUENCY_LAST12H ===\nAUDIT_GREEN_SUMMARY_COUNT=7\nAUDIT_RED_SUMMARY_COUNT=17\nAUDIT_LOCK_BUSY_ISSUE_COUNT=9\nAUDIT_SOURCE_OFFHOST_ISSUE_COUNT=0\n2026-09-27T19:37:02.093711+03:00 pve01 homelab-backup-audit[3923461]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T20:06:15.522779+03:00 pve01 homelab-backup-audit[4022565]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T20:36:05.450336+03:00 pve01 homelab-backup-audit[4111037]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T21:06:02.094425+03:00 pve01 homelab-backup-audit[16838]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T21:37:05.002224+03:00 pve01 homelab-backup-audit[106421]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T22:06:07.004513+03:00 pve01 homelab-backup-audit[203921]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T22:35:26.212759+03:00 pve01 homelab-backup-audit[291167]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T23:06:54.987894+03:00 pve01 homelab-backup-audit[395212]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-27T23:36:37.162587+03:00 pve01 homelab-backup-audit[500802]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-27T23:36:37.162587+03:00 pve01 homelab-backup-audit[500802]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=6|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T00:06:21.135619+03:00 pve01 homelab-backup-audit[605334]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T00:06:21.135619+03:00 pve01 homelab-backup-audit[605334]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T00:36:10.445081+03:00 pve01 homelab-backup-audit[707255]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T00:36:10.445081+03:00 pve01 homelab-backup-audit[707255]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T01:06:01.997372+03:00 pve01 homelab-backup-audit[799187]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T01:36:02.006941+03:00 pve01 homelab-backup-audit[906059]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T02:05:36.356288+03:00 pve01 homelab-backup-audit[995508]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T02:36:15.265015+03:00 pve01 homelab-backup-audit[1094502]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T03:05:20.453132+03:00 pve01 homelab-backup-audit[1192518]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T03:35:20.147380+03:00 pve01 homelab-backup-audit[1320691]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T03:35:20.147380+03:00 pve01 homelab-backup-audit[1320691]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T04:06:45.928910+03:00 pve01 homelab-backup-audit[1424254]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T04:06:45.928910+03:00 pve01 homelab-backup-audit[1424254]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T04:35:20.240415+03:00 pve01 homelab-backup-audit[1514892]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T04:35:20.240415+03:00 pve01 homelab-backup-audit[1514892]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T05:05:52.974415+03:00 pve01 homelab-backup-audit[1617767]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T05:05:52.974415+03:00 pve01 homelab-backup-audit[1617767]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T05:37:02.145613+03:00 pve01 homelab-backup-audit[1720751]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T06:06:02.032037+03:00 pve01 homelab-backup-audit[1807928]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\n2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\n=== HEALTH_GENERATION_CORRELATION ===\nHEALTH_GENERATED_AT_EPOCH=1790568997\nHEALTH_CORRELATION_FROM=1790568817\nHEALTH_CORRELATION_TO=1790569177\n=== FRESH_QUIET_AUDIT ===\nQUIET_AUDIT_WAIT|attempt=1|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=2|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=3|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=4|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=5|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=6|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=7|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=8|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=9|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=10|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=11|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=12|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=13|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=14|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=15|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=16|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=17|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=18|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=19|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=20|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=21|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=22|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=23|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=24|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=25|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=26|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=27|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=28|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=29|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=30|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=31|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=32|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=33|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=34|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=35|reason=SERVICE_BUSY|active=activating|pid=1856481\nQUIET_AUDIT_WAIT|attempt=36|reason=SERVICE_BUSY|active=activating|pid=1856481\nFRESH_QUIET_AUDIT_RC=NO_QUIET_WINDOW\nFRESH_AUDIT_HEALTH_NO_WRITE=false\n=== CANONICAL_ALERT_SOURCE_AUTHORITY ===\nCANONICAL_ALERT_PATH=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run\nCANONICAL_ALERT_EXISTS=true\nCANONICAL_ALERT_SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a\nCANONICAL_ALERT_MATCH_LIVE=false\nCANONICAL_SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nCANONICAL_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nCANONICAL_SOURCE_EXACT=true\nTASK_RESULT=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED\nDECISION=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED\nNEXT_GATE=READ_EXACT_1237_AND_FIX_ONLY_PROVEN_ALERT_NOISE_CAUSE\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 7542c8de..ca8bc20f 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-V2-FINAL-ACCEPTANCE-CLOSEOUT-1236R1 +COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=homelab-backup-v2-final-acceptance-closeout +COMPONENT=homelab-backup-alert-spam-current-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=ae0922db9c4c4ecb922eff1bc2dbc83ca34800f18e36edd72d9ff12921aff878 +COMMAND_SHA256=e205fb5749bd8d2a466e79a56e8d9f3a66537e7dcbd1c635202ab6df607b736b DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,101 +24,432 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=92d863c807a78fd412105892222e8b6204cabb0751fc53ff09613fbe11d0394c -SANITIZED_OUTPUT_SHA256=92d863c807a78fd412105892222e8b6204cabb0751fc53ff09613fbe11d0394c +RAW_EVIDENCE_SHA256=c64c77856d1b796c0414be59dd0089c9687ec505d26bb8b4111ab68ee967e5a2 +SANITIZED_OUTPUT_SHA256=c64c77856d1b796c0414be59dd0089c9687ec505d26bb8b4111ab68ee967e5a2 OUTPUT_BEGIN -BACKUP_V2_FINAL_ACCEPTANCE_CLOSEOUT_BEGIN=true -COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-V2-FINAL-ACCEPTANCE-CLOSEOUT-1236R1 +BACKUP_ALERT_SPAM_CURRENT_RCA_BEGIN=true +COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1 MODE=read-only -COMPONENT=homelab-backup-v2-final-acceptance-closeout -MUTATION_BOUNDARY=FINAL_OBSERVATIONAL_ACCEPTANCE_ONLY;NO_BACKUP;NO_PERSISTENT_LOCK_OWNERSHIP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_SYSTEMD_ACTION;NO_TIMER_CHANGE;NO_AUDIT_HEALTH_WRITE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MAIL;NO_GIT_WRITE;NO_NETBIRD_SKLADCHIK_CHANGE -=== SOURCE_AUTHORITY === +COMPONENT=homelab-backup-alert-spam-current-rca +MUTATION_BOUNDARY=READ_ONLY_ALERT_HEALTH_STATE_SOURCE_JOURNAL_LOCK_AND_QUIET_AUDIT;NO_ALERT_EXECUTION;NO_MAIL;NO_BACKUP;NO_CONFIG_WRITE;NO_WRAPPER_WRITE;NO_RELEASE_WRITE;NO_SYSTEMD_ACTION;NO_TIMER_CHANGE;NO_HEALTH_WRITE;NO_GIT_WRITE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_NETBIRD_SKLADCHIK_CHANGE +=== AUTHORITY === HOST=pve01 CLUSTER_QUORATE=Yes RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -SOURCE_1235_STATUS=OK -SOURCE_1235_RC=0 -SOURCE_1235_DECISION=PASS_LOCK_AWARE_RPO_RECOVERY_GREEN -SOURCE_1235_NEXT_GATE=FINAL_BACKUP_ACCEPTANCE_CLOSEOUT -SOURCE_1235_FINAL_AUDIT_RC=0 -SOURCE_1235_FINAL_ISSUE_COUNT=0 -SOURCE_1235_FINAL_RPO_COUNT=0 -SOURCE_1235_FINAL_NON_RPO_COUNT=0 -SOURCE_1235_TARGETED_REFRESH_COUNT=11 -SOURCE_1235_SUCCESS_MAIL_POLICY_ACCEPTED=true -SOURCE_1235_RECOVERY_END_EPOCH=1790545685 -=== LIVE_IMMUTABLE_IDENTITIES === +SOURCE_1236_STATUS=OK +SOURCE_1236_RC=0 +SOURCE_1236_CHANGES_MADE=false +SOURCE_1236_DECISION=HOLD_1236_V2_SERVICE_LAST_RUN_NOT_SUCCESS +SOURCE_1236_NEXT_GATE=RECONCILE_POST_RECOVERY_SCHEDULER +SOURCE_1236_POST_RECOVERY_V2_FAILED_COUNT=0 +SOURCE_1236_POST_RECOVERY_FRESH_HELPER_FAILED_COUNT=0 +SOURCE_1236_POST_RECOVERY_SCHEDULE_SUCCESS_COUNT=7 +=== LIVE_IDENTITIES === WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a -RELEASE_META=500:root:root -RELEASE_FILESET=entry504.py fresh_lxc_helper.py lxc_adapter.py producer_exec_helper.py qemu_adapter.py scheduler_adapter.py -RELEASE_FILESET_EXPECTED=entry504.py fresh_lxc_helper.py lxc_adapter.py producer_exec_helper.py qemu_adapter.py scheduler_adapter.py -RELEASE_FILE=fresh_lxc_helper.py SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860 META=400:root:root -RELEASE_FILE=producer_exec_helper.py SHA256=b3b97099bcd7126670f866e81faae6a42a96d869d9989e6a67f6ea6c871c90b3 META=400:root:root -RELEASE_FILE=lxc_adapter.py SHA256=3fa7d194010d76f7c8bc95035ae8876262e9337903e310f45f45df88c266dc8d META=400:root:root -RELEASE_FILE=qemu_adapter.py SHA256=1906db42bd34bbe85eb46fd9797b3572d46e0a41c29d1c29be5f9df5e448f15b META=400:root:root -RELEASE_FILE=scheduler_adapter.py SHA256=a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96 META=400:root:root -RELEASE_FILE=entry504.py SHA256=ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793 META=400:root:root -=== ROLLBACK_AUTHORITY === -ROLLBACK_PATH=/var/lib/homelab-change-rollback/SUPPORT-260927-HOMELAB-BACKUP-NOTIFICATION-COMPAT-HOTFIX-1230R1/homelab-backup-v2.before -ROLLBACK_EXISTS=true -ROLLBACK_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 -=== CANONICAL_SOURCE_AUTHORITY === -SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 -SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation -SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 -SOURCE_WORKTREE_CLEAN=true -=== UNIT_AND_TIMER_CURRENT === +ALERT_SCRIPT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9 +=== CURRENT_UNIT_STATE === UNIT=homelab-backup-v2.service ACTIVE=activating ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=1856481 UNIT=homelab-backup-v2.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID= UNIT=homelab-backup-audit.service ACTIVE=failed ENABLED=static RESULT=exit-code EXEC_MAIN_STATUS=1 MAINPID=0 UNIT=homelab-backup-audit.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID= -=== POST_RECOVERY_SCHEDULER_JOURNAL === -2026-09-28T01:12:52.217428+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... -2026-09-28T01:25:14.480529+03:00 pve01 homelab-backup-v2[815211]: SCHEDULE_TARGET=xf-codevipe -2026-09-28T01:25:14.480529+03:00 pve01 homelab-backup-v2[815211]: SCHEDULE_RESULT=SUCCESS -2026-09-28T01:25:14.488885+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. -2026-09-28T01:49:49.378146+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... -2026-09-28T02:03:38.623364+03:00 pve01 homelab-backup-v2[943938]: SCHEDULE_TARGET=xf-zakrutim -2026-09-28T02:03:38.623364+03:00 pve01 homelab-backup-v2[943938]: SCHEDULE_RESULT=SUCCESS -2026-09-28T02:03:38.630266+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. -2026-09-28T02:29:32.216618+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... -2026-09-28T02:29:49.327002+03:00 pve01 homelab-backup-v2[1072063]: SCHEDULE_TARGET=vm210 -2026-09-28T02:30:55.417267+03:00 pve01 homelab-backup-v2[1072063]: SCHEDULE_RESULT=SUCCESS -2026-09-28T02:30:55.424354+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. -2026-09-28T03:01:22.218704+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... -2026-09-28T03:01:54.164165+03:00 pve01 homelab-backup-v2[1179144]: SCHEDULE_TARGET=vm211 -2026-09-28T03:03:04.842495+03:00 pve01 homelab-backup-v2[1179144]: SCHEDULE_RESULT=SUCCESS -2026-09-28T03:03:04.849383+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. -2026-09-28T03:34:32.216619+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... -2026-09-28T05:14:16.727192+03:00 pve01 homelab-backup-v2[1320207]: SCHEDULE_TARGET=xf-dsmods -2026-09-28T05:14:16.727192+03:00 pve01 homelab-backup-v2[1320207]: SCHEDULE_RESULT=SUCCESS -2026-09-28T05:14:16.733465+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. -2026-09-28T05:14:16.735708+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... -2026-09-28T05:24:43.293853+03:00 pve01 homelab-backup-v2[1647429]: SCHEDULE_TARGET=xf-kingofwolk -2026-09-28T05:24:43.293853+03:00 pve01 homelab-backup-v2[1647429]: SCHEDULE_RESULT=SUCCESS -2026-09-28T05:24:43.300102+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. -2026-09-28T05:45:12.220837+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... -2026-09-28T05:52:51.548422+03:00 pve01 homelab-backup-v2[1737475]: SCHEDULE_TARGET=xf-newfi -2026-09-28T05:52:51.548422+03:00 pve01 homelab-backup-v2[1737475]: SCHEDULE_RESULT=SUCCESS -2026-09-28T05:52:51.557165+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +UNIT=homelab-alertmanager-backup-health.service ACTIVE=inactive ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=0 +UNIT=homelab-alertmanager-backup-health.timer ACTIVE=active ENABLED=enabled RESULT=success EXEC_MAIN_STATUS= MAINPID= +=== CURRENT_HEALTH_FILES === +HEALTH1_EXISTS=true +HEALTH1_SHA256=d89fb55b92f2b9908e2d2d8da3b9a0d0db6d2fe84c6e6d37bcec1a86e4384222 +HEALTH1_JSON_OK=true +HEALTH1_SCHEMA=homelab-backup-audit-v3 +HEALTH1_STATUS=RED +HEALTH1_GENERATED_AT_EPOCH=1790568997 +HEALTH1_AGE_SECONDS=855 +HEALTH1_ISSUE_COUNT=2 +HEALTH1_ISSUE_1={"code":"LOCK_BUSY","logical_id":"/run/homelab-backup-heavy.lock","root_cause":"ACTIVE_OR_ORPHAN_LOCK","severity":"HIGH"} +HEALTH1_ISSUE_2={"code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT","severity":"MEDIUM"} +HEALTH1_SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +HEALTH1_SOURCE_REMOTE_HEAD=None +HEALTH1_SOURCE_PUSHURL_DISABLED=true +HEALTH1_SOURCE_EXACT=false +HEALTH2_EXISTS=true +HEALTH2_SHA256=d89fb55b92f2b9908e2d2d8da3b9a0d0db6d2fe84c6e6d37bcec1a86e4384222 +HEALTH2_JSON_OK=true +HEALTH2_SCHEMA=homelab-backup-audit-v3 +HEALTH2_STATUS=RED +HEALTH2_GENERATED_AT_EPOCH=1790568997 +HEALTH2_AGE_SECONDS=855 +HEALTH2_ISSUE_COUNT=2 +HEALTH2_ISSUE_1={"code":"LOCK_BUSY","logical_id":"/run/homelab-backup-heavy.lock","root_cause":"ACTIVE_OR_ORPHAN_LOCK","severity":"HIGH"} +HEALTH2_ISSUE_2={"code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT","severity":"MEDIUM"} +HEALTH2_SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +HEALTH2_SOURCE_REMOTE_HEAD=None +HEALTH2_SOURCE_PUSHURL_DISABLED=true +HEALTH2_SOURCE_EXACT=false +HEALTH_FILES_IDENTICAL=true +HEALTH_EPOCH_VALID=true +=== CURRENT_ALERT_STATE === +ALERT_STATE_EXISTS=true +ALERT_STATE_SHA256=493ce50563cb31030f2b16ff4ad722a7ac5ec211a77b2e89625491c057e8827d +ALERT_STATE_JSON_OK=true +ALERT_STATE_KEYS=code_counts,issue_keys,migration_silent,notify_tokens,raw_issue_keys,schema,updated_epoch +ALERT_STATE_SAFE_JSON={"code_counts":{"LOCK_BUSY":4,"SOURCE_OFFHOST_MISSING":263},"issue_keys":["LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK","SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT"],"migration_silent":false,"notify_tokens":"[REDACTED]","raw_issue_keys":["LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK","SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT"],"schema":"homelab-backup-alert-state-v2","updated_epoch":1790569001} +ALERT_STATE_SAFE_TRUNCATED=false +ALERT_STATE_FIELD_SCHEMA="homelab-backup-alert-state-v2" +ALERT_STATE_FIELD_UPDATED_EPOCH=1790569001 +ALERT_STATE_FIELD_ISSUE_KEYS=["LOCK_BUSY|/run/homelab-backup-heavy.lock|ACTIVE_OR_ORPHAN_LOCK","SOURCE_OFFHOST_MISSING|-|CR_BRANCH_SHA_NOT_EXACT"] +=== ALERT_FILTER_SOURCE === +ALERT_EMBEDDED_SOURCE_FOUND=true +ALERT_EMBEDDED_SOURCE_SHA256=344306105083cd8bc64e19d451f7f66c17accc43a2f413538d2f9e2687b2540e +ALERT_EMBEDDED_SOURCE_BEGIN +0001:import sys,os,json,re,subprocess,pathlib,time +0002:mode=sys.argv[1] +0003:AUDIT="/usr/local/sbin/homelab-backup-audit" +0004:HEALTH=pathlib.Path("/var/lib/homelab-health/backup-audit-v2.json") +0005:STATE=pathlib.Path("/var/lib/homelab-alerting/backup-alert-state.json") +0006:USERCFG=pathlib.Path("/etc/pve/user.cfg") +0007:SCHEMA="homelab-backup-alert-state-v2" +0008:THRESHOLDS={"RPO_BREACH":2,"SOURCE_OFFHOST_MISSING":2,"LOCK_BUSY":3} +0009:def recipient(): +0010: text=USERCFG.read_text(encoding="utf-8",errors="replace") +0011: m=re.findall(r"(?i)\b[A-Z0-9._+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b",text) +0012: if len(m)!=1: raise SystemExit(65) +0013: return m[0] +0014:def send(subject,body): +0015: rcpt=recipient() +0016: msg=f"From: h***@gram1.ru\nTo: {rcpt}\nSubject: {subject}\nContent-Type: text/plain; charset=UTF-8\n\n{body}\n" +0017: p=subprocess.run(["/usr/sbin/sendmail","-t"],input=msg,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) +0018: if p.returncode!=0: raise SystemExit(p.returncode or 75) +0019:def key(i): +0020: return "|".join(str(i.get(x,"")) for x in ("code","logical_id","root_cause")) +0021:def token(i):[REDACTED] +0022: code=str(i.get("code","")) +0023: return "group:"+code if code in THRESHOLDS else "issue:"+key(i) +0024:if mode=="email-test": +0025: send("Workers2 backup alerting test","Workers2 backup/DR alerting route test. If you received this, the local MTA accepted the controlled test generated during backup closure.") +0026: print("EMAIL_TEST_ACCEPTED=YES") +0027: raise SystemExit(0) +0028:if mode!="health-alert": raise SystemExit(64) +0029:p=subprocess.run([AUDIT,"write-health"],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) +0030:if p.returncode not in (0,1): raise SystemExit(p.returncode or 75) +0031:try: +0032: h=json.loads(HEALTH.read_text(encoding="utf-8")) +0033:except Exception: +0034: raise SystemExit(69) +0035:issues=h.get("issues") if isinstance(h.get("issues"),list) else [] +0036:cur={key(i):i for i in issues if isinstance(i,dict)} +0037:codes=set(str(i.get("code","")) for i in cur.values()) +0038:oldj={} +0039:if STATE.is_file(): +0040: try: +0041: oldj=json.loads(STATE.read_text(encoding="utf-8")) +0042: if not isinstance(oldj,dict): oldj={} +0043: except Exception: +0044: oldj={} +0045:migrating=oldj.get("schema")!=SCHEMA +0046:prev_counts=oldj.get("code_counts") if isinstance(oldj.get("code_counts"),dict) and not migrating else {} +0047:counts={} +0048:for code in codes: +0049: try: prev=int(prev_counts.get(code,0)) +0050: except Exception: prev=0 +0051: counts[code]=prev+1 +0052:actionable={} +0053:for k,i in cur.items(): +0054: code=str(i.get("code","")) +0055: need=int(THRESHOLDS.get(code,1)) +0056: if counts.get(code,0)>=need: +0057: actionable[k]=i +0058:tokens=[REDACTED] +0059:old_tokens=[REDACTED] +0060:if not migrating: +0061: old_tokens=[REDACTED] +0062:added=[REDACTED] +0063:removed=[REDACTED] +0064:if added: +0065: lines=[ +0066: "Workers2 backup/DR actionable alert state changed.", +0067: "", +0068: "Transient debounce policy: RPO_BREACH=2 checks; SOURCE_OFFHOST_MISSING=2 checks; LOCK_BUSY=3 checks; all other issue codes=immediate.", +0069: "" +0070: ] +0071: for k in sorted(actionable): +0072: i=actionable[k] +0073: lines.append(f"{i.get('severity','UNKNOWN')} {i.get('code','UNKNOWN')} logical_id={i.get('logical_id','-')} root_cause={i.get('root_cause','UNKNOWN')}") +0074: suppressed=len(cur)-len(actionable) +0075: if suppressed: +0076: lines.extend(["",f"Suppressed transient issue(s) still below threshold: {suppressed}"]) +0077: send(f"[Workers2 BACKUP ALERT] {len(actionable)} actionable issue(s)","\n".join(lines)) +0078: print("ALERT_EMAIL_ACCEPTED=YES") +0079:elif old_tokens and not tokens:[REDACTED] +0080: send("[Workers2 BACKUP RECOVERY] all actionable issues cleared","Workers2 backup/DR actionable audit state is clear. All previously notified actionable backup issues are cleared.") +0081: print("RECOVERY_EMAIL_ACCEPTED=YES") +0082:STATE.parent.mkdir(parents=True,exist_ok=True) +0083:tmp=STATE.with_name(STATE.name+".tmp") +0084:payload={ +0085: "schema":SCHEMA, +0086: "updated_epoch":int(time.time()), +0087: "issue_keys":sorted(actionable), +0088: "notify_tokens":[REDACTED] +0089: "raw_issue_keys":sorted(cur), +0090: "code_counts":{k:int(v) for k,v in sorted(counts.items())}, +0091: "migration_silent":bool(migrating) +0092:} +0093:tmp.write_text(json.dumps(payload,separators=(",",":"),sort_keys=True)+"\n",encoding="utf-8") +0094:os.chmod(tmp,0o600) +0095:os.replace(tmp,STATE) +0096:print("ALERT_FILTER_SCHEMA="+SCHEMA) +0097:print("ALERT_RAW_ISSUE_COUNT="+str(len(cur))) +0098:print("ALERT_ACTIONABLE_ISSUE_COUNT="+str(len(actionable))) +0099:print("ALERT_NOTIFY_TOKEN_COUNT=[REDACTED] +0100:print("ALERT_ADDED_TOKEN_COUNT=[REDACTED] +0101:print("ALERT_REMOVED_TOKEN_COUNT=[REDACTED] +0102:print("ALERT_SUPPRESSED_TRANSIENT_COUNT="+str(len(cur)-len(actionable))) +0103:print("ALERT_STATE_MIGRATION_SILENT="+("true" if migrating else "false")) +0104:print("HEALTH_ALERT_STATE=OK") +ALERT_EMBEDDED_SOURCE_END +=== AUDIT_SOURCE_RELEVANT === +183- ("homelab-alertmanager-backup-health.timer","ALERT_TIMER_BAD"), +184-]: +185- ok,a,e=unit_ok(unit) +186- if not ok: issues.append({"severity":"HIGH","code":code,"logical_id":unit,"root_cause":"UNIT_"+a+"_"+e}) +187- +188:for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]: +189- p=pathlib.Path(lock) +190- if p.exists(): +191: lrc,_,_=run(["flock","-n",str(p),"-c","true"],5) +192: if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"}) +193- +194-env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG) +195-grc,gout,_=run([GFSH,"validate"],20,env) +196-if grc!=0 or "GFS_POLICY=OK" not in gout: +197- issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"}) +-- +287-except Exception: +288- cloudb_ok=False +289-if not cloudb_ok: +290- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"}) +291- +292:source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +293-hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +294:if hrc==0: source["local_head"]=ho.strip() +295-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) +296-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) +297-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") +298-if frc==0 and fo.strip(): +299- lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) +300: if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] +301:if source["remote_head"]!=source["local_head"]: +302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) +303- +304-status="GREEN" if not issues else "RED" +305-for t in targets: +306- print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None)) +307-for i in issues: +308- print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i)) +309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}") +310-health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")} +311-if MODE in ("write-health","scheduled"): +312- for hp in HEALTHS: +313- hp.parent.mkdir(parents=True,exist_ok=True) +314- tmp=hp.with_name(hp.name+".tmp") +=== LIVE_SOURCE_HEAD_RECHECK === +SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_WORKTREE_CLEAN=true +SOURCE_ORIGIN_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9 +SOURCE_REMOTE_ATTEMPT=1 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true +SOURCE_REMOTE_ATTEMPT=2 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true +SOURCE_REMOTE_ATTEMPT=3 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true +SOURCE_REMOTE_ATTEMPT=4 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true +SOURCE_REMOTE_ATTEMPT=5 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 STDERR_BYTES=0 EXACT=true +=== CURRENT_LOCK_OWNERSHIP === +LOCK_META=/run/homelab-backup-heavy.lock|inode=3474|600|root:root|2026-09-28 03:27:07.425060217 +0300 +LOCK_FUSER=/run/homelab-backup-heavy.lock| 1856481 +LOCK_FREE=/run/homelab-backup-heavy.lock|false +LOCK_META=/run/lock/homelab-backup-dispatcher.lock|inode=9|600|root:root|2026-09-28 03:27:07.423060188 +0300 +LOCK_FUSER=/run/lock/homelab-backup-dispatcher.lock| +LOCK_FREE=/run/lock/homelab-backup-dispatcher.lock|true +LOCK_META=/run/lock/homelab-vm-cloud-quorum-queue.lock|inode=2944|600|root:root|2026-09-28 03:27:07.424060203 +0300 +LOCK_FUSER=/run/lock/homelab-vm-cloud-quorum-queue.lock| +LOCK_FREE=/run/lock/homelab-vm-cloud-quorum-queue.lock|true +LOCK_META=/run/lock/homelab-app-cloud-quorum-queue.lock|inode=15|600|root:root|2026-09-28 03:27:07.425060217 +0300 +LOCK_FUSER=/run/lock/homelab-app-cloud-quorum-queue.lock| +LOCK_FREE=/run/lock/homelab-app-cloud-quorum-queue.lock|true +homelab-backup- 1856481 FLOCK WRITE /run/homelab-backup-heavy.lock +=== ALERT_EMAIL_TIME_CORRELATION_0701_MSK === 2026-09-28T06:19:01.361470+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... -POST_RECOVERY_V2_START_COUNT=8 -POST_RECOVERY_V2_FINISHED_COUNT=7 -POST_RECOVERY_V2_FAILED_COUNT=0 -POST_RECOVERY_FRESH_HELPER_FAILED_COUNT=0 -POST_RECOVERY_SCHEDULE_SUCCESS_COUNT=7 -POST_RECOVERY_SCHEDULE_DEFER_COUNT=0 -V2_SERVICE_RESULT=success -V2_SERVICE_EXEC_MAIN_STATUS=0 -V2_SERVICE_ACTIVE=activating -BACKUP_PROJECT_COMPLETE=false -TASK_RESULT=HOLD_BACKUP_V2_FINAL_ACCEPTANCE_CLOSEOUT -DECISION=HOLD_1236_V2_SERVICE_LAST_RUN_NOT_SUCCESS -NEXT_GATE=RECONCILE_POST_RECOVERY_SCHEDULER -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260928-HOMELAB-BACKUP-V2-FINAL-ACCEPTANCE-CLOSEOUT-1236R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T06:36:20.423980+03:00 pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE +2026-09-28T06:36:20.424205+03:00 pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. +2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T07:06:48.304891+03:00 pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE +2026-09-28T07:06:48.305137+03:00 pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. +2026-09-28T06:15:16.319082+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +2026-09-28T06:15:20.661916+03:00 pve01 homelab-alertmanager-backup-run[1835343]: HEALTH_ALERT_STATE=OK +2026-09-28T06:15:20.666144+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +2026-09-28T06:30:42.414371+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +2026-09-28T06:30:46.367196+03:00 pve01 homelab-alertmanager-backup-run[1886013]: HEALTH_ALERT_STATE=OK +2026-09-28T06:30:46.370827+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +2026-09-28T06:46:01.347189+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +2026-09-28T06:46:05.406037+03:00 pve01 homelab-alertmanager-backup-run[1945314]: HEALTH_ALERT_STATE=OK +2026-09-28T06:46:05.410491+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +2026-09-28T07:01:32.217324+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: ALERT_EMAIL_ACCEPTED=YES +2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: HEALTH_ALERT_STATE=OK +2026-09-28T07:01:36.278152+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +2026-09-28T07:16:37.585569+03:00 pve01 systemd[1]: Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +2026-09-28T07:16:41.913014+03:00 pve01 homelab-alertmanager-backup-run[2048228]: HEALTH_ALERT_STATE=OK +2026-09-28T07:16:41.916768+03:00 pve01 systemd[1]: Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +=== ALERT_FREQUENCY_LAST12H === +ALERT_RUN_START_COUNT=0 +ALERT_EMAIL_ACCEPTED_COUNT=3 +RECOVERY_EMAIL_ACCEPTED_COUNT=0 +HEALTH_ALERT_STATE_OK_COUNT=47 +2026-09-27T19:32:04.991437+03:00 pve01 homelab-alertmanager-backup-run[3909103]: HEALTH_ALERT_STATE=OK +2026-09-27T19:47:36.328545+03:00 pve01 homelab-alertmanager-backup-run[3959645]: HEALTH_ALERT_STATE=OK +2026-09-27T20:02:46.359055+03:00 pve01 homelab-alertmanager-backup-run[4008840]: HEALTH_ALERT_STATE=OK +2026-09-27T20:18:05.225521+03:00 pve01 homelab-alertmanager-backup-run[4059785]: HEALTH_ALERT_STATE=OK +2026-09-27T20:33:20.585967+03:00 pve01 homelab-alertmanager-backup-run[4108890]: HEALTH_ALERT_STATE=OK +2026-09-27T20:48:36.460451+03:00 pve01 homelab-alertmanager-backup-run[4148152]: HEALTH_ALERT_STATE=OK +2026-09-27T21:04:08.100470+03:00 pve01 homelab-alertmanager-backup-run[4070]: HEALTH_ALERT_STATE=OK +2026-09-27T21:19:36.336976+03:00 pve01 homelab-alertmanager-backup-run[54137]: HEALTH_ALERT_STATE=OK +2026-09-27T21:35:05.706232+03:00 pve01 homelab-alertmanager-backup-run[104752]: HEALTH_ALERT_STATE=OK +2026-09-27T21:50:23.015203+03:00 pve01 homelab-alertmanager-backup-run[154365]: HEALTH_ALERT_STATE=OK +2026-09-27T22:05:36.461278+03:00 pve01 homelab-alertmanager-backup-run[202960]: HEALTH_ALERT_STATE=OK +2026-09-27T22:21:05.253758+03:00 pve01 homelab-alertmanager-backup-run[242449]: HEALTH_ALERT_STATE=OK +2026-09-27T22:36:15.399283+03:00 pve01 homelab-alertmanager-backup-run[292331]: HEALTH_ALERT_STATE=OK +2026-09-27T22:51:31.819800+03:00 pve01 homelab-alertmanager-backup-run[343850]: HEALTH_ALERT_STATE=OK +2026-09-27T23:06:53.759023+03:00 pve01 homelab-alertmanager-backup-run[396329]: HEALTH_ALERT_STATE=OK +2026-09-27T23:22:20.677831+03:00 pve01 homelab-alertmanager-backup-run[461385]: HEALTH_ALERT_STATE=OK +2026-09-27T23:37:36.973241+03:00 pve01 homelab-alertmanager-backup-run[513251]: HEALTH_ALERT_STATE=OK +2026-09-27T23:52:43.434843+03:00 pve01 homelab-alertmanager-backup-run[566888]: ALERT_EMAIL_ACCEPTED=YES +2026-09-27T23:52:43.434843+03:00 pve01 homelab-alertmanager-backup-run[566888]: HEALTH_ALERT_STATE=OK +2026-09-28T00:08:05.576769+03:00 pve01 homelab-alertmanager-backup-run[607129]: HEALTH_ALERT_STATE=OK +2026-09-28T00:23:21.212626+03:00 pve01 homelab-alertmanager-backup-run[660884]: HEALTH_ALERT_STATE=OK +2026-09-28T00:38:36.269886+03:00 pve01 homelab-alertmanager-backup-run[711652]: HEALTH_ALERT_STATE=OK +2026-09-28T00:53:55.974540+03:00 pve01 homelab-alertmanager-backup-run[762988]: HEALTH_ALERT_STATE=OK +2026-09-28T01:09:05.464301+03:00 pve01 homelab-alertmanager-backup-run[812499]: HEALTH_ALERT_STATE=OK +2026-09-28T01:24:16.127247+03:00 pve01 homelab-alertmanager-backup-run[862903]: HEALTH_ALERT_STATE=OK +2026-09-28T01:39:41.256513+03:00 pve01 homelab-alertmanager-backup-run[919688]: HEALTH_ALERT_STATE=OK +2026-09-28T01:54:54.298806+03:00 pve01 homelab-alertmanager-backup-run[958250]: HEALTH_ALERT_STATE=OK +2026-09-28T02:10:05.486220+03:00 pve01 homelab-alertmanager-backup-run[1009352]: HEALTH_ALERT_STATE=OK +2026-09-28T02:25:22.594856+03:00 pve01 homelab-alertmanager-backup-run[1058040]: HEALTH_ALERT_STATE=OK +2026-09-28T02:40:53.144072+03:00 pve01 homelab-alertmanager-backup-run[1108677]: HEALTH_ALERT_STATE=OK +2026-09-28T02:56:20.122916+03:00 pve01 homelab-alertmanager-backup-run[1157287]: HEALTH_ALERT_STATE=OK +2026-09-28T03:11:36.224287+03:00 pve01 homelab-alertmanager-backup-run[1207541]: HEALTH_ALERT_STATE=OK +2026-09-28T03:26:56.104164+03:00 pve01 homelab-alertmanager-backup-run[1267608]: HEALTH_ALERT_STATE=OK +2026-09-28T03:42:16.163161+03:00 pve01 homelab-alertmanager-backup-run[1337091]: HEALTH_ALERT_STATE=OK +2026-09-28T03:57:35.911704+03:00 pve01 homelab-alertmanager-backup-run[1388112]: HEALTH_ALERT_STATE=OK +2026-09-28T04:12:45.302638+03:00 pve01 homelab-alertmanager-backup-run[1439953]: ALERT_EMAIL_ACCEPTED=YES +2026-09-28T04:12:45.302638+03:00 pve01 homelab-alertmanager-backup-run[1439953]: HEALTH_ALERT_STATE=OK +2026-09-28T04:28:16.066840+03:00 pve01 homelab-alertmanager-backup-run[1491056]: HEALTH_ALERT_STATE=OK +2026-09-28T04:43:26.035041+03:00 pve01 homelab-alertmanager-backup-run[1543022]: HEALTH_ALERT_STATE=OK +2026-09-28T04:58:29.541024+03:00 pve01 homelab-alertmanager-backup-run[1593789]: HEALTH_ALERT_STATE=OK +2026-09-28T05:13:39.334685+03:00 pve01 homelab-alertmanager-backup-run[1645768]: HEALTH_ALERT_STATE=OK +2026-09-28T05:28:53.353807+03:00 pve01 homelab-alertmanager-backup-run[1685761]: HEALTH_ALERT_STATE=OK +2026-09-28T05:44:20.155523+03:00 pve01 homelab-alertmanager-backup-run[1736193]: HEALTH_ALERT_STATE=OK +2026-09-28T06:00:05.422013+03:00 pve01 homelab-alertmanager-backup-run[1785685]: HEALTH_ALERT_STATE=OK +2026-09-28T06:15:20.661916+03:00 pve01 homelab-alertmanager-backup-run[1835343]: HEALTH_ALERT_STATE=OK +2026-09-28T06:30:46.367196+03:00 pve01 homelab-alertmanager-backup-run[1886013]: HEALTH_ALERT_STATE=OK +2026-09-28T06:46:05.406037+03:00 pve01 homelab-alertmanager-backup-run[1945314]: HEALTH_ALERT_STATE=OK +2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: ALERT_EMAIL_ACCEPTED=YES +2026-09-28T07:01:36.274466+03:00 pve01 homelab-alertmanager-backup-run[1988992]: HEALTH_ALERT_STATE=OK +2026-09-28T07:16:41.913014+03:00 pve01 homelab-alertmanager-backup-run[2048228]: HEALTH_ALERT_STATE=OK +=== AUDIT_FREQUENCY_LAST12H === +AUDIT_GREEN_SUMMARY_COUNT=7 +AUDIT_RED_SUMMARY_COUNT=17 +AUDIT_LOCK_BUSY_ISSUE_COUNT=9 +AUDIT_SOURCE_OFFHOST_ISSUE_COUNT=0 +2026-09-27T19:37:02.093711+03:00 pve01 homelab-backup-audit[3923461]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T20:06:15.522779+03:00 pve01 homelab-backup-audit[4022565]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T20:36:05.450336+03:00 pve01 homelab-backup-audit[4111037]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T21:06:02.094425+03:00 pve01 homelab-backup-audit[16838]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T21:37:05.002224+03:00 pve01 homelab-backup-audit[106421]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T22:06:07.004513+03:00 pve01 homelab-backup-audit[203921]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T22:35:26.212759+03:00 pve01 homelab-backup-audit[291167]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=12|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T23:06:54.987894+03:00 pve01 homelab-backup-audit[395212]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=11|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-27T23:36:37.162587+03:00 pve01 homelab-backup-audit[500802]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-27T23:36:37.162587+03:00 pve01 homelab-backup-audit[500802]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=6|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T00:06:21.135619+03:00 pve01 homelab-backup-audit[605334]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T00:06:21.135619+03:00 pve01 homelab-backup-audit[605334]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=4|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T00:36:10.445081+03:00 pve01 homelab-backup-audit[707255]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T00:36:10.445081+03:00 pve01 homelab-backup-audit[707255]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T01:06:01.997372+03:00 pve01 homelab-backup-audit[799187]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T01:36:02.006941+03:00 pve01 homelab-backup-audit[906059]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T02:05:36.356288+03:00 pve01 homelab-backup-audit[995508]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T02:36:15.265015+03:00 pve01 homelab-backup-audit[1094502]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T03:05:20.453132+03:00 pve01 homelab-backup-audit[1192518]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T03:35:20.147380+03:00 pve01 homelab-backup-audit[1320691]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T03:35:20.147380+03:00 pve01 homelab-backup-audit[1320691]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T04:06:45.928910+03:00 pve01 homelab-backup-audit[1424254]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T04:06:45.928910+03:00 pve01 homelab-backup-audit[1424254]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T04:35:20.240415+03:00 pve01 homelab-backup-audit[1514892]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T04:35:20.240415+03:00 pve01 homelab-backup-audit[1514892]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T05:05:52.974415+03:00 pve01 homelab-backup-audit[1617767]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T05:05:52.974415+03:00 pve01 homelab-backup-audit[1617767]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T05:37:02.145613+03:00 pve01 homelab-backup-audit[1720751]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T06:06:02.032037+03:00 pve01 homelab-backup-audit[1807928]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T06:36:20.418325+03:00 pve01 homelab-backup-audit[1908813]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +2026-09-28T07:06:48.299695+03:00 pve01 homelab-backup-audit[2011656]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +=== HEALTH_GENERATION_CORRELATION === +HEALTH_GENERATED_AT_EPOCH=1790568997 +HEALTH_CORRELATION_FROM=1790568817 +HEALTH_CORRELATION_TO=1790569177 +=== FRESH_QUIET_AUDIT === +QUIET_AUDIT_WAIT|attempt=1|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=2|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=3|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=4|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=5|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=6|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=7|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=8|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=9|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=10|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=11|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=12|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=13|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=14|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=15|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=16|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=17|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=18|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=19|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=20|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=21|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=22|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=23|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=24|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=25|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=26|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=27|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=28|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=29|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=30|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=31|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=32|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=33|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=34|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=35|reason=SERVICE_BUSY|active=activating|pid=1856481 +QUIET_AUDIT_WAIT|attempt=36|reason=SERVICE_BUSY|active=activating|pid=1856481 +FRESH_QUIET_AUDIT_RC=NO_QUIET_WINDOW +FRESH_AUDIT_HEALTH_NO_WRITE=false +=== CANONICAL_ALERT_SOURCE_AUTHORITY === +CANONICAL_ALERT_PATH=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run +CANONICAL_ALERT_EXISTS=true +CANONICAL_ALERT_SHA256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a +CANONICAL_ALERT_MATCH_LIVE=false +CANONICAL_SOURCE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +CANONICAL_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663 +CANONICAL_SOURCE_EXACT=true +TASK_RESULT=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED +DECISION=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED +NEXT_GATE=READ_EXACT_1237_AND_FIX_ONLY_PROVEN_ALERT_NOISE_CAUSE +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END