runtime: publish SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1
This commit is contained in:
homelab-runtime-publisher
File diff suppressed because one or more lines are too long
@@ -0,0 +1,111 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-source-offhost-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=34d36e4fc4a57591b0bda606d45a984ce67b48165c8d5ded708db29a3645cfc2
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=0
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826
|
||||
SANITIZED_OUTPUT_SHA256=a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826
|
||||
OUTPUT_BEGIN
|
||||
WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-source-offhost-rca
|
||||
MUTATION_BOUNDARY=READ_ONLY;NO_GIT_FETCH;NO_GIT_PULL;NO_GIT_RESET;NO_GIT_COMMIT;NO_GIT_PUSH;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_BACKUP;NO_RESTORE;NO_RETENTION;NO_DELETE;NO_MAIL
|
||||
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_LOCAL_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd
|
||||
SOURCE_REMOTE_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd
|
||||
SOURCE_ACCEPTED_1126_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd
|
||||
SOURCE_LOCAL_EQUALS_REMOTE=true
|
||||
SOURCE_LOCAL_EQUALS_ACCEPTED_1126=true
|
||||
SOURCE_REMOTE_EQUALS_ACCEPTED_1126=true
|
||||
SOURCE_RELATION=EXACT
|
||||
SOURCE_AHEAD_COUNT=0
|
||||
SOURCE_BEHIND_COUNT=0
|
||||
SOURCE_REMOTE_OBJECT_PRESENT_LOCALLY=true
|
||||
SOURCE_TRACKING_HEAD=refs/remotes/origin/cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_TRACKING_EQUALS_LOCAL=false
|
||||
SOURCE_TRACKING_EQUALS_REMOTE=false
|
||||
SOURCE_WORKTREE_STATUS_COUNT=0
|
||||
SOURCE_WORKTREE_STATUS_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
SOURCE_FETCH_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9
|
||||
SOURCE_PUSH_URL_LITERAL_DOT=false
|
||||
SOURCE_PUSH_URL_SHA256=0c7ef33e451eadb5c230e83d5155fe481df0dccf42529851a31110a88a776d12
|
||||
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
|
||||
AUDIT_SOURCE_OFFHOST_LOGIC_BEGIN=1
|
||||
289-if not cloudb_ok:
|
||||
290- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"})
|
||||
291-
|
||||
292-source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
|
||||
293:hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
|
||||
294-if hrc==0: source["local_head"]=ho.strip()
|
||||
295-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10)
|
||||
296-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10)
|
||||
297-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED")
|
||||
298-if frc==0 and fo.strip():
|
||||
299: lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
|
||||
300- if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
|
||||
301-if source["remote_head"]!=source["local_head"]:
|
||||
302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
|
||||
303-
|
||||
304-status="GREEN" if not issues else "RED"
|
||||
305-for t in targets:
|
||||
306- print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None))
|
||||
307-for i in issues:
|
||||
308- print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i))
|
||||
309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}")
|
||||
310-health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")}
|
||||
311-if MODE in ("write-health","scheduled"):
|
||||
312- for hp in HEALTHS:
|
||||
313- hp.parent.mkdir(parents=True,exist_ok=True)
|
||||
AUDIT_SOURCE_OFFHOST_LOGIC_END=1
|
||||
AUDIT_JOURNAL_RECENT_BEGIN=1
|
||||
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
AUDIT_JOURNAL_RECENT_END=1
|
||||
ALERT_JOURNAL_RECENT_BEGIN=1
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
homelab-alertmanager-backup-health.service: Deactivated successfully.
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
homelab-alertmanager-backup-health.service: Consumed 2.441s CPU time, 154M memory peak.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
homelab-alertmanager-backup-health.service: Deactivated successfully.
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
homelab-alertmanager-backup-health.service: Consumed 2.706s CPU time, 153.6M memory peak.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
homelab-alertmanager-backup-health.service: Deactivated successfully.
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
homelab-alertmanager-backup-health.service: Consumed 2.523s CPU time, 154.5M memory peak.
|
||||
ALERT_JOURNAL_RECENT_END=1
|
||||
DECISION=SOURCE_OFFHOST_CURRENTLY_EXACT
|
||||
NEXT_GATE=ANALYZE_SOURCE_OFFHOST_RCA
|
||||
TASK_RESULT=PASS_HOMELAB_BACKUP_SOURCE_OFFHOST_RCA
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+8
-8
File diff suppressed because one or more lines are too long
+84
-6
@@ -1,15 +1,15 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-CARRIER-DIRECT-1215R1
|
||||
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-carrier-direct
|
||||
COMPONENT=homelab-backup-source-offhost-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=6935a01d8bbc652fce22ac71ca61ac799b5ed334895f0e8fda50b95426c92d53
|
||||
COMMAND_SHA256=34d36e4fc4a57591b0bda606d45a984ce67b48165c8d5ded708db29a3645cfc2
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
@@ -24,10 +24,88 @@ MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=20ac1ba45517f1e7dcab38c59e201e291d8fc13e6a6c92b4f1f456fc363665f2
|
||||
SANITIZED_OUTPUT_SHA256=20ac1ba45517f1e7dcab38c59e201e291d8fc13e6a6c92b4f1f456fc363665f2
|
||||
RAW_EVIDENCE_SHA256=a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826
|
||||
SANITIZED_OUTPUT_SHA256=a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826
|
||||
OUTPUT_BEGIN
|
||||
1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-CARRIER-DIRECT-1215R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}printf: warning: ignoring excess arguments, starting with ‘WORKERS2_BACKUP_CARRIER_DIRECT_END=1’
|
||||
WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_BEGIN=1
|
||||
COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-source-offhost-rca
|
||||
MUTATION_BOUNDARY=READ_ONLY;NO_GIT_FETCH;NO_GIT_PULL;NO_GIT_RESET;NO_GIT_COMMIT;NO_GIT_PUSH;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_BACKUP;NO_RESTORE;NO_RETENTION;NO_DELETE;NO_MAIL
|
||||
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_LOCAL_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd
|
||||
SOURCE_REMOTE_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd
|
||||
SOURCE_ACCEPTED_1126_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd
|
||||
SOURCE_LOCAL_EQUALS_REMOTE=true
|
||||
SOURCE_LOCAL_EQUALS_ACCEPTED_1126=true
|
||||
SOURCE_REMOTE_EQUALS_ACCEPTED_1126=true
|
||||
SOURCE_RELATION=EXACT
|
||||
SOURCE_AHEAD_COUNT=0
|
||||
SOURCE_BEHIND_COUNT=0
|
||||
SOURCE_REMOTE_OBJECT_PRESENT_LOCALLY=true
|
||||
SOURCE_TRACKING_HEAD=refs/remotes/origin/cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_TRACKING_EQUALS_LOCAL=false
|
||||
SOURCE_TRACKING_EQUALS_REMOTE=false
|
||||
SOURCE_WORKTREE_STATUS_COUNT=0
|
||||
SOURCE_WORKTREE_STATUS_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
SOURCE_FETCH_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9
|
||||
SOURCE_PUSH_URL_LITERAL_DOT=false
|
||||
SOURCE_PUSH_URL_SHA256=0c7ef33e451eadb5c230e83d5155fe481df0dccf42529851a31110a88a776d12
|
||||
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
|
||||
AUDIT_SOURCE_OFFHOST_LOGIC_BEGIN=1
|
||||
289-if not cloudb_ok:
|
||||
290- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"})
|
||||
291-
|
||||
292-source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
|
||||
293:hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10)
|
||||
294-if hrc==0: source["local_head"]=ho.strip()
|
||||
295-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10)
|
||||
296-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10)
|
||||
297-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED")
|
||||
298-if frc==0 and fo.strip():
|
||||
299: lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
|
||||
300- if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
|
||||
301-if source["remote_head"]!=source["local_head"]:
|
||||
302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
|
||||
303-
|
||||
304-status="GREEN" if not issues else "RED"
|
||||
305-for t in targets:
|
||||
306- print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None))
|
||||
307-for i in issues:
|
||||
308- print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i))
|
||||
309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}")
|
||||
310-health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")}
|
||||
311-if MODE in ("write-health","scheduled"):
|
||||
312- for hp in HEALTHS:
|
||||
313- hp.parent.mkdir(parents=True,exist_ok=True)
|
||||
AUDIT_SOURCE_OFFHOST_LOGIC_END=1
|
||||
AUDIT_JOURNAL_RECENT_BEGIN=1
|
||||
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
AUDIT_JOURNAL_RECENT_END=1
|
||||
ALERT_JOURNAL_RECENT_BEGIN=1
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
ALERT_EMAIL_ACCEPTED=YES
|
||||
HEALTH_ALERT_STATE=OK
|
||||
homelab-alertmanager-backup-health.service: Deactivated successfully.
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
homelab-alertmanager-backup-health.service: Consumed 2.441s CPU time, 154M memory peak.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
homelab-alertmanager-backup-health.service: Deactivated successfully.
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
homelab-alertmanager-backup-health.service: Consumed 2.706s CPU time, 153.6M memory peak.
|
||||
Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...
|
||||
HEALTH_ALERT_STATE=OK
|
||||
homelab-alertmanager-backup-health.service: Deactivated successfully.
|
||||
Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.
|
||||
homelab-alertmanager-backup-health.service: Consumed 2.523s CPU time, 154.5M memory peak.
|
||||
ALERT_JOURNAL_RECENT_END=1
|
||||
DECISION=SOURCE_OFFHOST_CURRENTLY_EXACT
|
||||
NEXT_GATE=ANALYZE_SOURCE_OFFHOST_RCA
|
||||
TASK_RESULT=PASS_HOMELAB_BACKUP_SOURCE_OFFHOST_RCA
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_END=1
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user