diff --git a/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1.json b/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1.json new file mode 100644 index 00000000..9c36ec5e --- /dev/null +++ b/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-source-offhost-rca", + "started_at_utc": "2026-09-25T03:16:08Z", + "finished_at_utc": "2026-09-25T03:16:09Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "34d36e4fc4a57591b0bda606d45a984ce67b48165c8d5ded708db29a3645cfc2", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826", + "sanitized_output_sha256": "a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1\nMODE=read-only\nCOMPONENT=homelab-backup-source-offhost-rca\nMUTATION_BOUNDARY=READ_ONLY;NO_GIT_FETCH;NO_GIT_PULL;NO_GIT_RESET;NO_GIT_COMMIT;NO_GIT_PUSH;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_BACKUP;NO_RESTORE;NO_RETENTION;NO_DELETE;NO_MAIL\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_LOCAL_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd\nSOURCE_REMOTE_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd\nSOURCE_ACCEPTED_1126_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd\nSOURCE_LOCAL_EQUALS_REMOTE=true\nSOURCE_LOCAL_EQUALS_ACCEPTED_1126=true\nSOURCE_REMOTE_EQUALS_ACCEPTED_1126=true\nSOURCE_RELATION=EXACT\nSOURCE_AHEAD_COUNT=0\nSOURCE_BEHIND_COUNT=0\nSOURCE_REMOTE_OBJECT_PRESENT_LOCALLY=true\nSOURCE_TRACKING_HEAD=refs/remotes/origin/cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_TRACKING_EQUALS_LOCAL=false\nSOURCE_TRACKING_EQUALS_REMOTE=false\nSOURCE_WORKTREE_STATUS_COUNT=0\nSOURCE_WORKTREE_STATUS_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nSOURCE_FETCH_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9\nSOURCE_PUSH_URL_LITERAL_DOT=false\nSOURCE_PUSH_URL_SHA256=0c7ef33e451eadb5c230e83d5155fe481df0dccf42529851a31110a88a776d12\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nAUDIT_SOURCE_OFFHOST_LOGIC_BEGIN=1\n289-if not cloudb_ok:\n290- issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK\"})\n291-\n292-source={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\n293:hrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\n294-if hrc==0: source[\"local_head\"]=ho.strip()\n295-frc,fo,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.url\"],10)\n296-prc,po,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.pushurl\"],10)\n297-source[\"pushurl_disabled\"]=(prc==0 and po.strip()==\"DISABLED\")\n298-if frc==0 and fo.strip():\n299: lrc,lo,_=run([\"git\",\"ls-remote\",fo.strip(),\"refs/heads/\"+BRANCH],30)\n300- if lrc==0 and lo.strip(): source[\"remote_head\"]=lo.split()[0]\n301-if source[\"remote_head\"]!=source[\"local_head\"]:\n302: issues.append({\"severity\":\"MEDIUM\",\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\"})\n303-\n304-status=\"GREEN\" if not issues else \"RED\"\n305-for t in targets:\n306- print(\"TARGET|\"+\"|\".join(f\"{k}={v}\" for k,v in t.items() if v is not None))\n307-for i in issues:\n308- print(\"ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}\".format(**i))\n309:print(f\"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}\")\n310-health={\"schema\":\"homelab-backup-audit-v3\",\"status\":status,\"generated_at_epoch\":now,\"targets\":targets,\"issues\":issues,\"cloud\":cloud,\"source\":source,\"cloud_b_restore_drill\":cloudb_ok,\"dr_proven\":bool(isinstance(dr,dict) and dr.get(\"status\")==\"SEALED\")}\n311-if MODE in (\"write-health\",\"scheduled\"):\n312- for hp in HEALTHS:\n313- hp.parent.mkdir(parents=True,exist_ok=True)\nAUDIT_SOURCE_OFFHOST_LOGIC_END=1\nAUDIT_JOURNAL_RECENT_BEGIN=1\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nAUDIT_JOURNAL_RECENT_END=1\nALERT_JOURNAL_RECENT_BEGIN=1\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nhomelab-alertmanager-backup-health.service: Deactivated successfully.\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nhomelab-alertmanager-backup-health.service: Consumed 2.441s CPU time, 154M memory peak.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nhomelab-alertmanager-backup-health.service: Deactivated successfully.\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nhomelab-alertmanager-backup-health.service: Consumed 2.706s CPU time, 153.6M memory peak.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nhomelab-alertmanager-backup-health.service: Deactivated successfully.\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nhomelab-alertmanager-backup-health.service: Consumed 2.523s CPU time, 154.5M memory peak.\nALERT_JOURNAL_RECENT_END=1\nDECISION=SOURCE_OFFHOST_CURRENTLY_EXACT\nNEXT_GATE=ANALYZE_SOURCE_OFFHOST_RCA\nTASK_RESULT=PASS_HOMELAB_BACKUP_SOURCE_OFFHOST_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_SOURCE_OFFHOST_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1.txt b/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1.txt new file mode 100644 index 00000000..1f6315b4 --- /dev/null +++ b/runtime/history/SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1.txt @@ -0,0 +1,111 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-source-offhost-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=34d36e4fc4a57591b0bda606d45a984ce67b48165c8d5ded708db29a3645cfc2 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826 +SANITIZED_OUTPUT_SHA256=a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826 +OUTPUT_BEGIN +WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1 +MODE=read-only +COMPONENT=homelab-backup-source-offhost-rca +MUTATION_BOUNDARY=READ_ONLY;NO_GIT_FETCH;NO_GIT_PULL;NO_GIT_RESET;NO_GIT_COMMIT;NO_GIT_PUSH;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_BACKUP;NO_RESTORE;NO_RETENTION;NO_DELETE;NO_MAIL +SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_LOCAL_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd +SOURCE_REMOTE_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd +SOURCE_ACCEPTED_1126_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd +SOURCE_LOCAL_EQUALS_REMOTE=true +SOURCE_LOCAL_EQUALS_ACCEPTED_1126=true +SOURCE_REMOTE_EQUALS_ACCEPTED_1126=true +SOURCE_RELATION=EXACT +SOURCE_AHEAD_COUNT=0 +SOURCE_BEHIND_COUNT=0 +SOURCE_REMOTE_OBJECT_PRESENT_LOCALLY=true +SOURCE_TRACKING_HEAD=refs/remotes/origin/cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_TRACKING_EQUALS_LOCAL=false +SOURCE_TRACKING_EQUALS_REMOTE=false +SOURCE_WORKTREE_STATUS_COUNT=0 +SOURCE_WORKTREE_STATUS_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +SOURCE_FETCH_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9 +SOURCE_PUSH_URL_LITERAL_DOT=false +SOURCE_PUSH_URL_SHA256=0c7ef33e451eadb5c230e83d5155fe481df0dccf42529851a31110a88a776d12 +AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +AUDIT_SOURCE_OFFHOST_LOGIC_BEGIN=1 +289-if not cloudb_ok: +290- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"}) +291- +292-source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +293:hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +294-if hrc==0: source["local_head"]=ho.strip() +295-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) +296-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) +297-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") +298-if frc==0 and fo.strip(): +299: lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) +300- if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] +301-if source["remote_head"]!=source["local_head"]: +302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) +303- +304-status="GREEN" if not issues else "RED" +305-for t in targets: +306- print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None)) +307-for i in issues: +308- print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i)) +309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}") +310-health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")} +311-if MODE in ("write-health","scheduled"): +312- for hp in HEALTHS: +313- hp.parent.mkdir(parents=True,exist_ok=True) +AUDIT_SOURCE_OFFHOST_LOGIC_END=1 +AUDIT_JOURNAL_RECENT_BEGIN=1 +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +AUDIT_JOURNAL_RECENT_END=1 +ALERT_JOURNAL_RECENT_BEGIN=1 +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +homelab-alertmanager-backup-health.service: Deactivated successfully. +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +homelab-alertmanager-backup-health.service: Consumed 2.441s CPU time, 154M memory peak. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +homelab-alertmanager-backup-health.service: Deactivated successfully. +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +homelab-alertmanager-backup-health.service: Consumed 2.706s CPU time, 153.6M memory peak. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +homelab-alertmanager-backup-health.service: Deactivated successfully. +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +homelab-alertmanager-backup-health.service: Consumed 2.523s CPU time, 154.5M memory peak. +ALERT_JOURNAL_RECENT_END=1 +DECISION=SOURCE_OFFHOST_CURRENTLY_EXACT +NEXT_GATE=ANALYZE_SOURCE_OFFHOST_RCA +TASK_RESULT=PASS_HOMELAB_BACKUP_SOURCE_OFFHOST_RCA +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 85377b04..9c36ec5e 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260925-HOMELAB-BACKUP-CARRIER-DIRECT-1215R1", + "command_id": "SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "homelab-backup-carrier-direct", - "started_at_utc": "2026-09-25T03:12:44Z", - "finished_at_utc": "2026-09-25T03:12:44Z", + "component": "homelab-backup-source-offhost-rca", + "started_at_utc": "2026-09-25T03:16:08Z", + "finished_at_utc": "2026-09-25T03:16:09Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "6935a01d8bbc652fce22ac71ca61ac799b5ed334895f0e8fda50b95426c92d53", + "command_sha256": "34d36e4fc4a57591b0bda606d45a984ce67b48165c8d5ded708db29a3645cfc2", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "20ac1ba45517f1e7dcab38c59e201e291d8fc13e6a6c92b4f1f456fc363665f2", - "sanitized_output_sha256": "20ac1ba45517f1e7dcab38c59e201e291d8fc13e6a6c92b4f1f456fc363665f2", + "raw_evidence_sha256": "a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826", + "sanitized_output_sha256": "a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "1,\"command_id\":\"SUPPORT-260925-HOMELAB-BACKUP-CARRIER-DIRECT-1215R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}printf: warning: ignoring excess arguments, starting with ‘WORKERS2_BACKUP_CARRIER_DIRECT_END=1’\n" + "output": "WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1\nMODE=read-only\nCOMPONENT=homelab-backup-source-offhost-rca\nMUTATION_BOUNDARY=READ_ONLY;NO_GIT_FETCH;NO_GIT_PULL;NO_GIT_RESET;NO_GIT_COMMIT;NO_GIT_PUSH;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_BACKUP;NO_RESTORE;NO_RETENTION;NO_DELETE;NO_MAIL\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_LOCAL_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd\nSOURCE_REMOTE_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd\nSOURCE_ACCEPTED_1126_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd\nSOURCE_LOCAL_EQUALS_REMOTE=true\nSOURCE_LOCAL_EQUALS_ACCEPTED_1126=true\nSOURCE_REMOTE_EQUALS_ACCEPTED_1126=true\nSOURCE_RELATION=EXACT\nSOURCE_AHEAD_COUNT=0\nSOURCE_BEHIND_COUNT=0\nSOURCE_REMOTE_OBJECT_PRESENT_LOCALLY=true\nSOURCE_TRACKING_HEAD=refs/remotes/origin/cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_TRACKING_EQUALS_LOCAL=false\nSOURCE_TRACKING_EQUALS_REMOTE=false\nSOURCE_WORKTREE_STATUS_COUNT=0\nSOURCE_WORKTREE_STATUS_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nSOURCE_FETCH_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9\nSOURCE_PUSH_URL_LITERAL_DOT=false\nSOURCE_PUSH_URL_SHA256=0c7ef33e451eadb5c230e83d5155fe481df0dccf42529851a31110a88a776d12\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nAUDIT_SOURCE_OFFHOST_LOGIC_BEGIN=1\n289-if not cloudb_ok:\n290- issues.append({\"severity\":\"CRITICAL\",\"code\":\"CLOUD_B_RESTORE_DRILL_BAD\",\"logical_id\":\"app-skladchik-reports-monitor\",\"root_cause\":\"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK\"})\n291-\n292-source={\"local_head\":None,\"remote_head\":None,\"pushurl_disabled\":False}\n293:hrc,ho,_=run([\"git\",\"-C\",WT,\"rev-parse\",\"HEAD\"],10)\n294-if hrc==0: source[\"local_head\"]=ho.strip()\n295-frc,fo,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.url\"],10)\n296-prc,po,_=run([\"git\",\"-C\",WT,\"config\",\"--get\",\"remote.origin.pushurl\"],10)\n297-source[\"pushurl_disabled\"]=(prc==0 and po.strip()==\"DISABLED\")\n298-if frc==0 and fo.strip():\n299: lrc,lo,_=run([\"git\",\"ls-remote\",fo.strip(),\"refs/heads/\"+BRANCH],30)\n300- if lrc==0 and lo.strip(): source[\"remote_head\"]=lo.split()[0]\n301-if source[\"remote_head\"]!=source[\"local_head\"]:\n302: issues.append({\"severity\":\"MEDIUM\",\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\"})\n303-\n304-status=\"GREEN\" if not issues else \"RED\"\n305-for t in targets:\n306- print(\"TARGET|\"+\"|\".join(f\"{k}={v}\" for k,v in t.items() if v is not None))\n307-for i in issues:\n308- print(\"ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}\".format(**i))\n309:print(f\"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}\")\n310-health={\"schema\":\"homelab-backup-audit-v3\",\"status\":status,\"generated_at_epoch\":now,\"targets\":targets,\"issues\":issues,\"cloud\":cloud,\"source\":source,\"cloud_b_restore_drill\":cloudb_ok,\"dr_proven\":bool(isinstance(dr,dict) and dr.get(\"status\")==\"SEALED\")}\n311-if MODE in (\"write-health\",\"scheduled\"):\n312- for hp in HEALTHS:\n313- hp.parent.mkdir(parents=True,exist_ok=True)\nAUDIT_SOURCE_OFFHOST_LOGIC_END=1\nAUDIT_JOURNAL_RECENT_BEGIN=1\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nAUDIT_JOURNAL_RECENT_END=1\nALERT_JOURNAL_RECENT_BEGIN=1\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nALERT_EMAIL_ACCEPTED=YES\nHEALTH_ALERT_STATE=OK\nhomelab-alertmanager-backup-health.service: Deactivated successfully.\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nhomelab-alertmanager-backup-health.service: Consumed 2.441s CPU time, 154M memory peak.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nhomelab-alertmanager-backup-health.service: Deactivated successfully.\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nhomelab-alertmanager-backup-health.service: Consumed 2.706s CPU time, 153.6M memory peak.\nStarting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration...\nHEALTH_ALERT_STATE=OK\nhomelab-alertmanager-backup-health.service: Deactivated successfully.\nFinished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration.\nhomelab-alertmanager-backup-health.service: Consumed 2.523s CPU time, 154.5M memory peak.\nALERT_JOURNAL_RECENT_END=1\nDECISION=SOURCE_OFFHOST_CURRENTLY_EXACT\nNEXT_GATE=ANALYZE_SOURCE_OFFHOST_RCA\nTASK_RESULT=PASS_HOMELAB_BACKUP_SOURCE_OFFHOST_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_SOURCE_OFFHOST_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 5911b0f6..1f6315b4 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-CARRIER-DIRECT-1215R1 +COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1217R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=homelab-backup-carrier-direct +COMPONENT=homelab-backup-source-offhost-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=6935a01d8bbc652fce22ac71ca61ac799b5ed334895f0e8fda50b95426c92d53 +COMMAND_SHA256=34d36e4fc4a57591b0bda606d45a984ce67b48165c8d5ded708db29a3645cfc2 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,10 +24,88 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=20ac1ba45517f1e7dcab38c59e201e291d8fc13e6a6c92b4f1f456fc363665f2 -SANITIZED_OUTPUT_SHA256=20ac1ba45517f1e7dcab38c59e201e291d8fc13e6a6c92b4f1f456fc363665f2 +RAW_EVIDENCE_SHA256=a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826 +SANITIZED_OUTPUT_SHA256=a88b18992cc4f26ac9728ac5d6f551da5b1de0b34b3f23c6187bc93637fbe826 OUTPUT_BEGIN -1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-CARRIER-DIRECT-1215R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}printf: warning: ignoring excess arguments, starting with ‘WORKERS2_BACKUP_CARRIER_DIRECT_END=1’ +WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1 +MODE=read-only +COMPONENT=homelab-backup-source-offhost-rca +MUTATION_BOUNDARY=READ_ONLY;NO_GIT_FETCH;NO_GIT_PULL;NO_GIT_RESET;NO_GIT_COMMIT;NO_GIT_PUSH;NO_CONFIG_WRITE;NO_SYSTEMD_ACTION;NO_BACKUP;NO_RESTORE;NO_RETENTION;NO_DELETE;NO_MAIL +SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_LOCAL_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd +SOURCE_REMOTE_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd +SOURCE_ACCEPTED_1126_HEAD=6401ea1e99988b9d2550de544c9e45bd7e829bdd +SOURCE_LOCAL_EQUALS_REMOTE=true +SOURCE_LOCAL_EQUALS_ACCEPTED_1126=true +SOURCE_REMOTE_EQUALS_ACCEPTED_1126=true +SOURCE_RELATION=EXACT +SOURCE_AHEAD_COUNT=0 +SOURCE_BEHIND_COUNT=0 +SOURCE_REMOTE_OBJECT_PRESENT_LOCALLY=true +SOURCE_TRACKING_HEAD=refs/remotes/origin/cr-2026-1005-backup-v2-health-alerting-remediation +SOURCE_TRACKING_EQUALS_LOCAL=false +SOURCE_TRACKING_EQUALS_REMOTE=false +SOURCE_WORKTREE_STATUS_COUNT=0 +SOURCE_WORKTREE_STATUS_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +SOURCE_FETCH_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9 +SOURCE_PUSH_URL_LITERAL_DOT=false +SOURCE_PUSH_URL_SHA256=0c7ef33e451eadb5c230e83d5155fe481df0dccf42529851a31110a88a776d12 +AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +AUDIT_SOURCE_OFFHOST_LOGIC_BEGIN=1 +289-if not cloudb_ok: +290- issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"CANONICAL_CLOUD_B_PAYLOAD_OR_RESTORE_EVIDENCE_NOT_OK"}) +291- +292-source={"local_head":None,"remote_head":None,"pushurl_disabled":False} +293:hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) +294-if hrc==0: source["local_head"]=ho.strip() +295-frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) +296-prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) +297-source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") +298-if frc==0 and fo.strip(): +299: lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) +300- if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] +301-if source["remote_head"]!=source["local_head"]: +302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) +303- +304-status="GREEN" if not issues else "RED" +305-for t in targets: +306- print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None)) +307-for i in issues: +308- print("ISSUE|severity={severity}|code={code}|logical_id={logical_id}|root_cause={root_cause}".format(**i)) +309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}") +310-health={"schema":"homelab-backup-audit-v3","status":status,"generated_at_epoch":now,"targets":targets,"issues":issues,"cloud":cloud,"source":source,"cloud_b_restore_drill":cloudb_ok,"dr_proven":bool(isinstance(dr,dict) and dr.get("status")=="SEALED")} +311-if MODE in ("write-health","scheduled"): +312- for hp in HEALTHS: +313- hp.parent.mkdir(parents=True,exist_ok=True) +AUDIT_SOURCE_OFFHOST_LOGIC_END=1 +AUDIT_JOURNAL_RECENT_BEGIN=1 +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=GREEN|ACTIVE_COUNT=16|ISSUE_COUNT=0|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +AUDIT_JOURNAL_RECENT_END=1 +ALERT_JOURNAL_RECENT_BEGIN=1 +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +ALERT_EMAIL_ACCEPTED=YES +HEALTH_ALERT_STATE=OK +homelab-alertmanager-backup-health.service: Deactivated successfully. +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +homelab-alertmanager-backup-health.service: Consumed 2.441s CPU time, 154M memory peak. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +homelab-alertmanager-backup-health.service: Deactivated successfully. +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +homelab-alertmanager-backup-health.service: Consumed 2.706s CPU time, 153.6M memory peak. +Starting homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration... +HEALTH_ALERT_STATE=OK +homelab-alertmanager-backup-health.service: Deactivated successfully. +Finished homelab-alertmanager-backup-health.service - Workers2 backup health alert via existing alertmanager backup integration. +homelab-alertmanager-backup-health.service: Consumed 2.523s CPU time, 154.5M memory peak. +ALERT_JOURNAL_RECENT_END=1 +DECISION=SOURCE_OFFHOST_CURRENTLY_EXACT +NEXT_GATE=ANALYZE_SOURCE_OFFHOST_RCA +TASK_RESULT=PASS_HOMELAB_BACKUP_SOURCE_OFFHOST_RCA +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260925-HOMELAB-BACKUP-SOURCE-OFFHOST-RCA-1216R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_SOURCE_OFFHOST_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END