homelab-runtime-publisher
112 lines
6.6 KiB
Plaintext
112 lines
6.6 KiB
Plaintext
CHAT_OUTPUT_BEGIN
|
|
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1
|
|
STATUS=OK
|
|
RC=0
|
|
HOST=pve01
|
|
MODE=read-only
|
|
COMPONENT=homelab-backup-source-offhost-rca
|
|
REFERENCE_REGISTER_CHECK=OK
|
|
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
|
ERROR_REGISTER_CHECK=OK
|
|
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
|
COMMAND_SHA256=34d36e4fc4a57591b0bda606d45a984ce67b48165c8d5ded708db29a3645cfc2
|
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
|
EXECUTION_STARTED=true
|
|
CHANGE_DECLARED=false
|
|
RESULT_CONTRACT_VALID=true
|
|
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
|
RESULT_CONTRACT_ERROR=NONE
|
|
COMMAND_RC=0
|
|
CHANGES_MADE=false
|
|
ROLLBACK_STARTED=false
|
|
ROLLBACK_RESTORED=null
|
|
MUTATION_OUTCOME=NO_MUTATION
|
|
SANITIZED=yes
|
|
SECRETS_INCLUDED=no
|
|
PRIVATE_ADDRESSES_INCLUDED=no
|
|
RAW_EVIDENCE_SHA256=a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826
|
|
SANITIZED_OUTPUT_SHA256=a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826
|
|
OUTPUT_BEGIN
|
|
WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_BEGIN=1
|
|
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1
|
|
MODE=read-only
|
|
COMPONENT=homelab-backup-source-offhost-rca
|
|
MUTATION_BOUNDARY=READ_ONLY;NO_GIT_FETCH;NO_GIT_PULL;NO_GIT_RESET;NO_GIT_COMMIT;NO_GIT_PUSH;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_BACKUP;NO_RESTORE;NO_RETENTION;NO_DELETE;NO_MAIL
|
|
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
|
SOURCE_LOCAL_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd
|
|
SOURCE_REMOTE_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd
|
|
SOURCE_ACCEPTED_1126_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd
|
|
SOURCE_LOCAL_EQUALS_REMOTE=true
|
|
SOURCE_LOCAL_EQUALS_ACCEPTED_1126=true
|
|
SOURCE_REMOTE_EQUALS_ACCEPTED_1126=true
|
|
SOURCE_RELATION=EXACT
|
|
SOURCE_AHEAD_COUNT=0
|
|
SOURCE_BEHIND_COUNT=0
|
|
SOURCE_REMOTE_OBJECT_PRESENT_LOCALLY=true
|
|
SOURCE_TRACKING_HEAD=refs/remotes/origin/cr-2026-1005-backup-v2-health-alerting-remediation
|
|
SOURCE_TRACKING_EQUALS_LOCAL=false
|
|
SOURCE_TRACKING_EQUALS_REMOTE=false
|
|
SOURCE_WORKTREE_STATUS_COUNT=0
|
|
SOURCE_WORKTREE_STATUS_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
|
SOURCE_FETCH_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9
|
|
SOURCE_PUSH_URL_LITERAL_DOT=false
|
|
SOURCE_PUSH_URL_SHA256=0c7ef33e451eadb5c230e83d5155fe481df0dccf42529851a31110a88a776d12
|
|
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
|
|
AUDIT_SOURCE_OFFHOST_LOGIC_BEGIN=1
|
|
289-if not cloudb_ok:
|
|
290- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"})
|
|
291-
|
|
292-source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
|
|
293:hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
|
|
294-if hrc==0: source["local_head"]=ho.strip()
|
|
295-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10)
|
|
296-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10)
|
|
297-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED")
|
|
298-if frc==0 and fo.strip():
|
|
299: lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
|
|
300- if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
|
|
301-if source["remote_head"]!=source["local_head"]:
|
|
302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
|
|
303-
|
|
304-status="GREEN" if not issues else "RED"
|
|
305-for t in targets:
|
|
306- print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None))
|
|
307-for i in issues:
|
|
308- print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i))
|
|
309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}")
|
|
310-health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")}
|
|
311-if MODE in ("write-health","scheduled"):
|
|
312- for hp in HEALTHS:
|
|
313- hp.parent.mkdir(parents=True,exist_ok=True)
|
|
AUDIT_SOURCE_OFFHOST_LOGIC_END=1
|
|
AUDIT_JOURNAL_RECENT_BEGIN=1
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
|
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
|
AUDIT_JOURNAL_RECENT_END=1
|
|
ALERT_JOURNAL_RECENT_BEGIN=1
|
|
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
|
ALERT_EMAIL_ACCEPTED=YES
|
|
HEALTH_ALERT_STATE=OK
|
|
homelab-alertmanager-backup-health.service: Deactivated successfully.
|
|
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
|
homelab-alertmanager-backup-health.service: Consumed 2.441s CPU time, 154M memory peak.
|
|
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
|
HEALTH_ALERT_STATE=OK
|
|
homelab-alertmanager-backup-health.service: Deactivated successfully.
|
|
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
|
homelab-alertmanager-backup-health.service: Consumed 2.706s CPU time, 153.6M memory peak.
|
|
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
|
HEALTH_ALERT_STATE=OK
|
|
homelab-alertmanager-backup-health.service: Deactivated successfully.
|
|
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
|
homelab-alertmanager-backup-health.service: Consumed 2.523s CPU time, 154.5M memory peak.
|
|
ALERT_JOURNAL_RECENT_END=1
|
|
DECISION=SOURCE_OFFHOST_CURRENTLY_EXACT
|
|
NEXT_GATE=ANALYZE_SOURCE_OFFHOST_RCA
|
|
TASK_RESULT=PASS_HOMELAB_BACKUP_SOURCE_OFFHOST_RCA
|
|
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
|
WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_END=1
|
|
|
|
OUTPUT_END
|
|
CHAT_OUTPUT_END
|