runtime: publish SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1
This commit is contained in:
+38
@@ -0,0 +1,38 @@
|
|||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"channel": "homelab-runtime",
|
||||||
|
"command_id": "SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1",
|
||||||
|
"status": "OK",
|
||||||
|
"rc": 0,
|
||||||
|
"host": "pve01",
|
||||||
|
"mode": "read-only",
|
||||||
|
"component": "infra-dr-backupv2-control-backup-logic-focus-diagnostic",
|
||||||
|
"started_at_utc": "2026-09-11T23:44:54Z",
|
||||||
|
"finished_at_utc": "2026-09-11T23:44:54Z",
|
||||||
|
"reference_register_checked": true,
|
||||||
|
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
|
||||||
|
"error_register_checked": true,
|
||||||
|
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
|
||||||
|
"command_sha256": "f3ab0a154eda3d947c6b078120e2088b7d0721f992b74d9da8e78af6568708ca",
|
||||||
|
"duplicate_failed_command_blocked": false,
|
||||||
|
"block_reason": null,
|
||||||
|
"execution_started": true,
|
||||||
|
"change_declared": false,
|
||||||
|
"result_contract_valid": true,
|
||||||
|
"result_contract_status": null,
|
||||||
|
"result_contract_error": null,
|
||||||
|
"command_rc": 0,
|
||||||
|
"changes_made": false,
|
||||||
|
"rollback_started": false,
|
||||||
|
"rollback_restored": null,
|
||||||
|
"mutation_outcome": "NO_MUTATION",
|
||||||
|
"sanitized": true,
|
||||||
|
"secrets_included": false,
|
||||||
|
"private_addresses_included": false,
|
||||||
|
"raw_evidence_retained_locally": true,
|
||||||
|
"raw_evidence_sha256": "72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974",
|
||||||
|
"sanitized_output_sha256": "72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974",
|
||||||
|
"output_truncated_in_json": false,
|
||||||
|
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
||||||
|
"output": "DIAG258_BEGIN\nTARGET_COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1\nTARGET_ATTEMPT_DIR=/root/homelab-run-SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1\nTARGET_ATTEMPT_DIR_EXISTS=NO\nMATCHING_257_ATTEMPT_DIR_COUNT=0\nEVIDENCE_LAYER=NO_257_SERVER_ATTEMPT_DIRECTORY\nINTERPRETATION=257_DID_NOT_REACH_RUNNER_RESERVATION\nFILES_WRITTEN_BY_DIAG258=0\nRCLONE_COMMANDS_BY_DIAG258=0\nCLOUD_MUTATIONS_BY_DIAG258=0\nDIAG258_END\n"
|
||||||
|
}
|
||||||
+43
@@ -0,0 +1,43 @@
|
|||||||
|
CHAT_OUTPUT_BEGIN
|
||||||
|
COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1
|
||||||
|
STATUS=OK
|
||||||
|
RC=0
|
||||||
|
HOST=pve01
|
||||||
|
MODE=read-only
|
||||||
|
COMPONENT=infra-dr-backupv2-control-backup-logic-focus-diagnostic
|
||||||
|
REFERENCE_REGISTER_CHECK=OK
|
||||||
|
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||||
|
ERROR_REGISTER_CHECK=OK
|
||||||
|
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||||
|
COMMAND_SHA256=f3ab0a154eda3d947c6b078120e2088b7d0721f992b74d9da8e78af6568708ca
|
||||||
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||||
|
EXECUTION_STARTED=true
|
||||||
|
CHANGE_DECLARED=false
|
||||||
|
RESULT_CONTRACT_VALID=true
|
||||||
|
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||||
|
RESULT_CONTRACT_ERROR=NONE
|
||||||
|
COMMAND_RC=0
|
||||||
|
CHANGES_MADE=false
|
||||||
|
ROLLBACK_STARTED=false
|
||||||
|
ROLLBACK_RESTORED=null
|
||||||
|
MUTATION_OUTCOME=NO_MUTATION
|
||||||
|
SANITIZED=yes
|
||||||
|
SECRETS_INCLUDED=no
|
||||||
|
PRIVATE_ADDRESSES_INCLUDED=no
|
||||||
|
RAW_EVIDENCE_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974
|
||||||
|
SANITIZED_OUTPUT_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974
|
||||||
|
OUTPUT_BEGIN
|
||||||
|
DIAG258_BEGIN
|
||||||
|
TARGET_COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1
|
||||||
|
TARGET_ATTEMPT_DIR=/root/homelab-run-SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1
|
||||||
|
TARGET_ATTEMPT_DIR_EXISTS=NO
|
||||||
|
MATCHING_257_ATTEMPT_DIR_COUNT=0
|
||||||
|
EVIDENCE_LAYER=NO_257_SERVER_ATTEMPT_DIRECTORY
|
||||||
|
INTERPRETATION=257_DID_NOT_REACH_RUNNER_RESERVATION
|
||||||
|
FILES_WRITTEN_BY_DIAG258=0
|
||||||
|
RCLONE_COMMANDS_BY_DIAG258=0
|
||||||
|
CLOUD_MUTATIONS_BY_DIAG258=0
|
||||||
|
DIAG258_END
|
||||||
|
|
||||||
|
OUTPUT_END
|
||||||
|
CHAT_OUTPUT_END
|
||||||
+11
-11
File diff suppressed because one or more lines are too long
+19
-106
@@ -1,22 +1,22 @@
|
|||||||
CHAT_OUTPUT_BEGIN
|
CHAT_OUTPUT_BEGIN
|
||||||
COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1
|
COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1
|
||||||
STATUS=FAIL
|
STATUS=OK
|
||||||
RC=2
|
RC=0
|
||||||
HOST=pve01
|
HOST=pve01
|
||||||
MODE=read-only
|
MODE=read-only
|
||||||
COMPONENT=infra-dr-backupv2-control-backup-logic-focus
|
COMPONENT=infra-dr-backupv2-control-backup-logic-focus-diagnostic
|
||||||
REFERENCE_REGISTER_CHECK=OK
|
REFERENCE_REGISTER_CHECK=OK
|
||||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||||
ERROR_REGISTER_CHECK=OK
|
ERROR_REGISTER_CHECK=OK
|
||||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||||
COMMAND_SHA256=9980bab4ad121c27772e38557dd65e4015aa4f3f718e5fcdc6e8dff43be4cd03
|
COMMAND_SHA256=f3ab0a154eda3d947c6b078120e2088b7d0721f992b74d9da8e78af6568708ca
|
||||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||||
EXECUTION_STARTED=true
|
EXECUTION_STARTED=true
|
||||||
CHANGE_DECLARED=false
|
CHANGE_DECLARED=false
|
||||||
RESULT_CONTRACT_VALID=true
|
RESULT_CONTRACT_VALID=true
|
||||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||||
RESULT_CONTRACT_ERROR=NONE
|
RESULT_CONTRACT_ERROR=NONE
|
||||||
COMMAND_RC=2
|
COMMAND_RC=0
|
||||||
CHANGES_MADE=false
|
CHANGES_MADE=false
|
||||||
ROLLBACK_STARTED=false
|
ROLLBACK_STARTED=false
|
||||||
ROLLBACK_RESTORED=null
|
ROLLBACK_RESTORED=null
|
||||||
@@ -24,107 +24,20 @@ MUTATION_OUTCOME=NO_MUTATION
|
|||||||
SANITIZED=yes
|
SANITIZED=yes
|
||||||
SECRETS_INCLUDED=no
|
SECRETS_INCLUDED=no
|
||||||
PRIVATE_ADDRESSES_INCLUDED=no
|
PRIVATE_ADDRESSES_INCLUDED=no
|
||||||
RAW_EVIDENCE_SHA256=cb86b6a96cfd10465cf8955bc75edde6147f7b65633d31dd3f958e33501c2a82
|
RAW_EVIDENCE_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974
|
||||||
SANITIZED_OUTPUT_SHA256=e987624faf1aaf226e4a886a8bc963d4096bd01e5f5a5f3d283c195d15db6c98
|
SANITIZED_OUTPUT_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974
|
||||||
OUTPUT_BEGIN
|
OUTPUT_BEGIN
|
||||||
FOCUS256_BEGIN
|
DIAG258_BEGIN
|
||||||
COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1
|
TARGET_COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1
|
||||||
MODE=READ_ONLY_LOCAL_SOURCE_LOGIC_FOCUS
|
TARGET_ATTEMPT_DIR=/root/homelab-run-SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1
|
||||||
CLOUD_CALLS_BY_256=0
|
TARGET_ATTEMPT_DIR_EXISTS=NO
|
||||||
RCLONE_COMMANDS_EXECUTED_BY_256=0
|
MATCHING_257_ATTEMPT_DIR_COUNT=0
|
||||||
ERROR_REGISTER_CHECK=READ
|
EVIDENCE_LAYER=NO_257_SERVER_ATTEMPT_DIRECTORY
|
||||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
INTERPRETATION=257_DID_NOT_REACH_RUNNER_RESERVATION
|
||||||
REFERENCE_CHECK=READ
|
FILES_WRITTEN_BY_DIAG258=0
|
||||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
RCLONE_COMMANDS_BY_DIAG258=0
|
||||||
AUTHORITY_FRESH_READ=OK
|
CLOUD_MUTATIONS_BY_DIAG258=0
|
||||||
TARGET_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57
|
DIAG258_END
|
||||||
TARGET_SHA_MATCH=YES
|
|
||||||
UNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250
|
|
||||||
UNIT_SHA_MATCH=YES
|
|
||||||
TARGET_LINE_COUNT=150
|
|
||||||
FOCUS_FUNCTION=prepare_rclone
|
|
||||||
FOCUS_SPAN=66-69
|
|
||||||
FOCUS_BODY_BEGIN=prepare_rclone
|
|
||||||
L066=prepare_rclone(){
|
|
||||||
L067= local dst="$1"; [[ -f "$RCLONE_CONFIG_SOURCE" && ! -L "$RCLONE_CONFIG_SOURCE" ]] || die RCLONE_CONFIG_UNSAFE 66
|
|
||||||
L068= install -m 0600 "$RCLONE_CONFIG_SOURCE" "$dst"
|
|
||||||
L069=}
|
|
||||||
FOCUS_BODY_END=prepare_rclone
|
|
||||||
FOCUS_FUNCTION=run_backup
|
|
||||||
FOCUS_SPAN=109-150
|
|
||||||
FOCUS_BODY_BEGIN=run_backup
|
|
||||||
L109=run_backup(){
|
|
||||||
L110= local force="${1:-0}" decision line state gen payload outdir now
|
|
||||||
L111= mkdir -p "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" /run/lock; chmod 0700 "$STATE_ROOT" "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT"
|
|
||||||
L112= exec 9>"$LOCK"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\n'; return 0; }
|
|
||||||
L113= local incompletes
|
|
||||||
L114= incompletes="$(find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(.status!="SEALED")|.generation' 2>/dev/null || true)"
|
|
||||||
L115= [[ "$(grep -c . <<<"$incompletes" || true)" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68
|
|
||||||
L116= line="$(scan)"; printf '%s\n' "$line"; decision="$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,"",$i);print $i}}' <<<"$line")"
|
|
||||||
L117= gen="$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$line")"
|
|
||||||
L118= if [[ "$decision" == SKIP_FRESH && "$force" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\n' "$gen"; return 0; fi
|
|
||||||
L119= if [[ "$decision" == BACKUP_REVERIFY ]]; then
|
|
||||||
L120= if [[ "$force" != 1 ]]; then verify_generation "$gen"; return 0; fi
|
|
||||||
L121= # Forced refresh deliberately creates a new generation instead of requiring
|
|
||||||
L122= # a legacy generation to satisfy the current bundle schema.
|
|
||||||
L123= decision=BACKUP_DUE
|
|
||||||
L124= fi
|
|
||||||
L125= if [[ "$decision" == BACKUP_RESUME ]]; then
|
|
||||||
L126= state="$STATE_DIR/$gen.json"; payload="$(jq -er '.payload_path' "$state")"; [[ -s "$payload" && ! -L "$payload" ]] || die INCOMPLETE_PAYLOAD_MISSING 68
|
|
||||||
L127= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die <LONG_TOKEN_REDACTED> 68
|
|
||||||
L128= mark_cloud_write
|
|
||||||
L129= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"; verify_generation "$gen"; printf 'CONTROL_BACKUP_RESULT=SEALED_RESUMED GENERATION=%s\n' "$gen"; return 0
|
|
||||||
L130= fi
|
|
||||||
L131= case "$decision" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die "DECISION_INVALID_$decision" 65;; esac
|
|
||||||
L132= exec 8>"$HEAVY_LOCK"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\n'; return 0; }
|
|
||||||
L133= now="$(date -u +%Y%m%dT%H%M%SZ)"; gen="control-bootstrap-v2-pve01-$now"; outdir="$STAGE_ROOT/$gen"
|
|
||||||
L134= "$BUILDER" build --generation "$gen" --output-dir "$outdir" >/dev/null
|
|
||||||
L135= payload="$outdir/$gen.tar.gz"; [[ -s "$payload" && ! -L "$payload" ]] || die BUILT_PAYLOAD_MISSING 68
|
|
||||||
L136= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die <LONG_TOKEN_REDACTED> 68
|
|
||||||
L137= exec 8>&-
|
|
||||||
L138= mark_cloud_write
|
|
||||||
L139= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
||||||
L140= rm -rf --one-file-system "$outdir" 2>/dev/null || true
|
|
||||||
L141= verify_generation "$gen"
|
|
||||||
L142= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\n' "$gen"
|
|
||||||
L143=}
|
|
||||||
L144=case "${1:-}" in
|
|
||||||
L145= scan) [[ "$#" == 1 ]] || die USAGE 64; scan;;
|
|
||||||
L146= run) [[ "$#" == 1 ]] || die USAGE 64; run_backup 0;;
|
|
||||||
L147= force) [[ "$#" == 1 ]] || die USAGE 64; run_backup 1;;
|
|
||||||
L148= verify) [[ "$#" == 2 ]] || die USAGE 64; verify_generation "$2";;
|
|
||||||
L149= *) die USAGE_scan_run_force_verify 64;;
|
|
||||||
L150=esac
|
|
||||||
FOCUS_BODY_END=run_backup
|
|
||||||
FOCUS_FUNCTION=verify_generation
|
|
||||||
FOCUS_SPAN=87-108
|
|
||||||
FOCUS_BODY_BEGIN=verify_generation
|
|
||||||
L087=verify_generation(){
|
|
||||||
L088= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now
|
|
||||||
L089= gen="$1"
|
|
||||||
L090= state="$STATE_DIR/$gen.json"
|
|
||||||
L091= [[ -s "$state" && ! -L "$state" ]] || die VERIFY_STATE_UNSAFE 68
|
|
||||||
L092= "$TRANSPORT" verify-sealed "$LOGICAL_ID" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null
|
|
||||||
L093= mkdir -p "$VERIFY_ROOT"; chmod 0700 "$VERIFY_ROOT"
|
|
||||||
L094= work="$(mktemp -d "$VERIFY_ROOT/.verify.XXXXXX")"; cfg="$work/[SENSITIVE_FILE]"; prepare_rclone "$cfg"
|
|
||||||
L095= cleanup_verify(){ rm -rf --one-file-system "$work"; }; trap cleanup_verify EXIT
|
|
||||||
L096= base_a="$(jq -er '.remote_a_base' "$state")"; base_b="$(jq -er '.remote_b_base' "$state")"; a="$work/A.tar.gz"; b="$work/B.tar.gz"
|
|
||||||
L097= reconstruct "$base_a" "$a" "$cfg" "$state" A; reconstruct "$base_b" "$b" "$cfg" "$state" B
|
|
||||||
L098= cmp -s "$a" "$b" || die CLOUD_A_B_PAYLOAD_MISMATCH 68
|
|
||||||
L099= verify_json="$("$BUILDER" verify --archive "$a" --expected-generation "$gen")" || die <LONG_TOKEN_REDACTED> 68
|
|
||||||
L100=<REDACTED_SENSITIVE_LINE>
|
|
||||||
L101= archive_sha="$(jq -er '.archive_sha256' "$state")"; mkdir -p "$EVIDENCE_ROOT" /var/lib/homelab-health; chmod 0700 "$EVIDENCE_ROOT"
|
|
||||||
L102= tmp="$EVIDENCE_ROOT/.$gen.new.$$"; now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
||||||
L103=<REDACTED_SENSITIVE_LINE>
|
|
||||||
L104= chmod 0600 "$tmp"; mv -f "$tmp" "$EVIDENCE_ROOT/$gen.json"
|
|
||||||
L105= tmp="${HEALTH}.new.$$"; jq -n --arg gen "$gen" --arg state "$state" --arg evidence "$EVIDENCE_ROOT/$gen.json" --arg sha "$archive_sha" --arg now "$now" '{schema:"homelab-control-backup-health-v1",status:"OK",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshness:"FRESH",full_site_dr_proven:false,updated_at_utc:$now}' >"$tmp"; chmod 0644 "$tmp"; mv -f "$tmp" "$HEALTH"
|
|
||||||
L106= trap - EXIT; cleanup_verify
|
|
||||||
L107= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\n' "$gen" "$EVIDENCE_ROOT/$gen.json"
|
|
||||||
L108=}
|
|
||||||
FOCUS_BODY_END=verify_generation
|
|
||||||
FOCUS256_STOP=FOCUS_FUNCTION_MISSING
|
|
||||||
FOCUS256_DETAIL=cleanup_verify
|
|
||||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
|
||||||
|
|
||||||
OUTPUT_END
|
OUTPUT_END
|
||||||
CHAT_OUTPUT_END
|
CHAT_OUTPUT_END
|
||||||
|
|||||||
Reference in New Issue
Block a user