runtime: publish SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1
This commit is contained in:
homelab-runtime-publisher
File diff suppressed because one or more lines are too long
@@ -0,0 +1,237 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=infra-dr-backupv2-control-backup-source
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=09220124de7fa21ffc8fbc498052fa8ab92cee5d72b61fe34976f30307a58400
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=0
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=3762076dc1704d241330f8e0aa300dc7cbca2c80ec5ccc24137e59d67f7bd34b
|
||||
SANITIZED_OUTPUT_SHA256=497e4996869d853f0466d28fe81745576dffca2ccbe7ed64998990ed695d898b
|
||||
OUTPUT_BEGIN
|
||||
SOURCE259_BEGIN
|
||||
COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1
|
||||
MODE=READ_ONLY_HASH_BOUND_SOURCE_EXPORT
|
||||
SOURCE_EXECUTION=FORBIDDEN
|
||||
ERROR_REGISTER_CHECK=READ
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
REFERENCE_CHECK=READ
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
AUTHORITY_FRESH_READ=OK
|
||||
ENGINE_PATH=/usr/local/sbin/homelab-control-backup
|
||||
ENGINE_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57
|
||||
ENGINE_LINE_COUNT=150
|
||||
ENGINE_REDACTED_LINE_COUNT=13
|
||||
ENGINE_SOURCE_BEGIN
|
||||
ENGINE_L0001_P001_OF001=#!/usr/bin/env bash
|
||||
ENGINE_L0002_P001_OF001=set -Eeuo pipefail
|
||||
ENGINE_L0003_P001_OF001=umask 077
|
||||
ENGINE_L0004_P001_OF001=LOGICAL_ID=control-bootstrap-v2
|
||||
ENGINE_L0005_P001_OF001=DUE_SECONDS=43200
|
||||
ENGINE_L0006_P001_OF001=SLA_SECONDS=86400
|
||||
ENGINE_L0007_P001_OF001=STATE_ROOT=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0008_P001_OF001=STATE_DIR=$STATE_ROOT/$LOGICAL_ID
|
||||
ENGINE_L0009_P001_OF001=EVIDENCE_ROOT=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0010_P001_OF001=HEALTH=<LONG_TOKEN_REDACTED>.json
|
||||
ENGINE_L0011_P001_OF001=STAGE_ROOT=/mnt/staging/homelab-control-auto
|
||||
ENGINE_L0012_P001_OF001=VERIFY_ROOT=/mnt/staging/homelab-control-verify
|
||||
ENGINE_L0013_P001_OF001=LOCK=/run/lock/homelab-control-backup.lock
|
||||
ENGINE_L0014_P001_OF001=HEAVY_LOCK=/run/homelab-backup-heavy.lock
|
||||
ENGINE_L0015_P001_OF001=RCLONE_CONFIG_SOURCE=${RCLONE_CONFIG:-/root/.config/rclone/[SENSITIVE_FILE]}
|
||||
ENGINE_L0016_P001_OF001=TRANSPORT=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0017_P001_OF001=TRANSPORT_SHA=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0018_P001_OF001=BUILDER=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0019_P001_OF001=BUILDER_SHA=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0020_P001_OF001=REMOTE_A=pve01-mail-01-crypt:homelab-backups
|
||||
ENGINE_L0021_P001_OF001=REMOTE_B=pve01-mail-02-crypt:homelab-backups
|
||||
ENGINE_L0022_P001_OF001=CLOUD_WRITE_MARKER=${HOMELAB_CONTROL_CLOUD_WRITE_MARKER:-}
|
||||
ENGINE_L0023_P001_OF001=
|
||||
ENGINE_L0024_P001_OF001=die(){ printf 'CONTROL_BACKUP_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; }
|
||||
ENGINE_L0025_P001_OF002=for c in jq date find sort tail cut sha256sum stat flock rclone mktemp cmp awk mkdir rm install xargs grep cat mv chmod; do command -v "$c" >/dev/null || die "T
|
||||
ENGINE_L0025_P002_OF002=OOL_MISSING_$c" 69; done
|
||||
ENGINE_L0026_P001_OF001=[[ -x "$TRANSPORT" && ! -L "$TRANSPORT" ]] || die TRANSPORT_UNSAFE 69
|
||||
ENGINE_L0027_P001_OF001=[[ "$(sha256sum "$TRANSPORT"|awk '{print $1}')" == "$TRANSPORT_SHA" ]] || die TRANSPORT_SHA_DRIFT 65
|
||||
ENGINE_L0028_P001_OF001=[[ -x "$BUILDER" && ! -L "$BUILDER" ]] || die BUILDER_UNSAFE 69
|
||||
ENGINE_L0029_P001_OF001=[[ "$(sha256sum "$BUILDER"|awk '{print $1}')" == "$BUILDER_SHA" ]] || die BUILDER_SHA_DRIFT 65
|
||||
ENGINE_L0030_P001_OF001=
|
||||
ENGINE_L0031_P001_OF001=latest_state(){
|
||||
ENGINE_L0032_P001_OF001= [[ -d "$STATE_DIR" && ! -L "$STATE_DIR" ]] || return 0
|
||||
ENGINE_L0033_P001_OF001= find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print 2>/dev/null | LC_ALL=C sort | tail -1
|
||||
ENGINE_L0034_P001_OF001=}
|
||||
ENGINE_L0035_P001_OF001=state_decision(){
|
||||
ENGINE_L0036_P001_OF001= local state="$1" now="$2" status gen upd epoch age ev sha
|
||||
ENGINE_L0037_P001_OF001= if [[ -z "$state" ]]; then printf 'DECISION=BACKUP_UNVERIFIED REASON=NO_STATE AGE_SECONDS=-1\n'; return 0; fi
|
||||
ENGINE_L0038_P001_OF001= [[ -s "$state" && ! -L "$state" ]] || { printf 'DECISION=BACKUP_UNVERIFIED REASON=STATE_UNSAFE AGE_SECONDS=-1\n'; return 0; }
|
||||
ENGINE_L0039_P001_OF001= status="$(jq -r '.status // "INVALID"' "$state")"; gen="$(jq -r '.generation // "UNKNOWN"' "$state")"
|
||||
ENGINE_L0040_P001_OF001= case "$status" in
|
||||
ENGINE_L0041_P001_OF001= PREPARING|A_VERIFIED|PREPARED|DELETE_AUTHORIZED)
|
||||
ENGINE_L0042_P001_OF001= printf 'DECISION=BACKUP_RESUME GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\n' "$gen" "$state"; return 0;;
|
||||
ENGINE_L0043_P001_OF001= SEALED) ;;
|
||||
ENGINE_L0044_P001_OF001= *) printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=STATE_STATUS_INVALID AGE_SECONDS=-1\n' "$gen"; return 0;;
|
||||
ENGINE_L0045_P001_OF001= esac
|
||||
ENGINE_L0046_P001_OF001= sha="$(jq -r '.archive_sha256 // ""' "$state")"; upd="$(jq -r '.sealed_at_utc // .updated_at_utc // ""' "$state")"
|
||||
ENGINE_L0047_P001_OF001= epoch="$(date -u -d "$upd" +%s 2>/dev/null || true)"
|
||||
ENGINE_L0048_P001_OF002= [[ "$epoch" =~ ^[0-9]+$ && "$epoch" -le "$now" ]] || { printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=TIMESTAMP_INVALID AGE_SECONDS=-1\n' "$gen"; ret
|
||||
ENGINE_L0048_P002_OF002=urn 0; }
|
||||
ENGINE_L0049_P001_OF001= ev="$EVIDENCE_ROOT/$gen.json"
|
||||
ENGINE_L0050_P001_OF001=<REDACTED_SENSITIVE_LINE>
|
||||
ENGINE_L0051_P001_OF001= printf 'DECISION=BACKUP_REVERIFY GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\n' "$gen" "$state"; return 0
|
||||
ENGINE_L0052_P001_OF001= fi
|
||||
ENGINE_L0053_P001_OF001= age=$((now-epoch))
|
||||
ENGINE_L0054_P001_OF001= if (( age < DUE_SECONDS )); then d=SKIP_FRESH; elif (( age < SLA_SECONDS )); then d=BACKUP_DUE; else d=BACKUP_STALE; fi
|
||||
ENGINE_L0055_P001_OF001= printf 'DECISION=%s GENERATION=%s STATE_FILE=%s AGE_SECONDS=%s DUE_SECONDS=%s SLA_SECONDS=%s\n' "$d" "$gen" "$state" "$age" "$DUE_SECONDS" "$SLA_SECONDS"
|
||||
ENGINE_L0056_P001_OF001=}
|
||||
ENGINE_L0057_P001_OF001=scan(){ local now state; now="$(date -u +%s)"; state="$(latest_state)"; state_decision "$state" "$now"; }
|
||||
ENGINE_L0058_P001_OF001=mark_cloud_write(){
|
||||
ENGINE_L0059_P001_OF001= [[ -z "$CLOUD_WRITE_MARKER" ]] && return 0
|
||||
ENGINE_L0060_P001_OF001= [[ "$CLOUD_WRITE_MARKER" == /* ]] || die CLOUD_WRITE_MARKER_NOT_ABSOLUTE 65
|
||||
ENGINE_L0061_P001_OF001= [[ ! -L "$CLOUD_WRITE_MARKER" ]] || die CLOUD_WRITE_MARKER_SYMLINK 65
|
||||
ENGINE_L0062_P001_OF001= printf '%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$CLOUD_WRITE_MARKER"
|
||||
ENGINE_L0063_P001_OF001= chmod 0600 "$CLOUD_WRITE_MARKER"
|
||||
ENGINE_L0064_P001_OF001=}
|
||||
ENGINE_L0065_P001_OF001=
|
||||
ENGINE_L0066_P001_OF001=prepare_rclone(){
|
||||
ENGINE_L0067_P001_OF001= local dst="$1"; [[ -f "$RCLONE_CONFIG_SOURCE" && ! -L "$RCLONE_CONFIG_SOURCE" ]] || die RCLONE_CONFIG_UNSAFE 66
|
||||
ENGINE_L0068_P001_OF001= install -m 0600 "$RCLONE_CONFIG_SOURCE" "$dst"
|
||||
ENGINE_L0069_P001_OF001=}
|
||||
ENGINE_L0070_P001_OF001=reconstruct(){
|
||||
ENGINE_L0071_P001_OF001= local base="$1" out="$2" cfg="$3" state="$4" label="$5" manifest part idx name size hash got_size got_hash expected_manifest expected_archive
|
||||
ENGINE_L0072_P001_OF001= expected_manifest="$(jq -er '.parts_manifest_sha256' "$state")"; expected_archive="$(jq -er '.archive_sha256' "$state")"
|
||||
ENGINE_L0073_P001_OF001= manifest="${out}.PARTS.tsv"; : >"$out"
|
||||
ENGINE_L0074_P001_OF001= rclone --config "$cfg" cat "${base%/}/PARTS.tsv" >"$manifest" || die "REMOTE_${label}_MANIFEST_READ_FAILED" 74
|
||||
ENGINE_L0075_P001_OF001= [[ "$(sha256sum "$manifest"|awk '{print $1}')" == "$expected_manifest" ]] || die "REMOTE_${label}_MANIFEST_SHA_MISMATCH" 68
|
||||
ENGINE_L0076_P001_OF001= idx=0
|
||||
ENGINE_L0077_P001_OF001= while IFS=$'\t' read -r row name size hash; do
|
||||
ENGINE_L0078_P001_OF002= [[ "$row" == "$(printf '%08d' "$idx")" && "$name" == part-????????.bin && "$size" =~ ^[1-9][0-9]*$ && "$hash" =~ ^[0-9a-f]{64}$ ]] || die "REMOTE_${label}_M
|
||||
ENGINE_L0078_P002_OF002=ANIFEST_ROW_INVALID" 68
|
||||
ENGINE_L0079_P001_OF001= part="${out}.part"; rm -f "$part"; rclone --config "$cfg" cat "${base%/}/payload.parts/$name" >"$part" || die "REMOTE_${label}_PART_READ_FAILED" 74
|
||||
ENGINE_L0080_P001_OF001= got_size="$(stat -c %s "$part")"; got_hash="$(sha256sum "$part"|awk '{print $1}')"
|
||||
ENGINE_L0081_P001_OF001= [[ "$got_size" == "$size" && "$got_hash" == "$hash" ]] || die "REMOTE_${label}_PART_VERIFY_FAILED" 68
|
||||
ENGINE_L0082_P001_OF001= cat "$part" >>"$out"; rm -f "$part"; idx=$((idx+1))
|
||||
ENGINE_L0083_P001_OF001= done <"$manifest"
|
||||
ENGINE_L0084_P001_OF001= [[ "$(sha256sum "$out"|awk '{print $1}')" == "$expected_archive" ]] || die "REMOTE_${label}_ARCHIVE_SHA_MISMATCH" 68
|
||||
ENGINE_L0085_P001_OF001= rm -f "$manifest"
|
||||
ENGINE_L0086_P001_OF001=}
|
||||
ENGINE_L0087_P001_OF001=verify_generation(){
|
||||
ENGINE_L0088_P001_OF001= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now
|
||||
ENGINE_L0089_P001_OF001= gen="$1"
|
||||
ENGINE_L0090_P001_OF001= state="$STATE_DIR/$gen.json"
|
||||
ENGINE_L0091_P001_OF001= [[ -s "$state" && ! -L "$state" ]] || die VERIFY_STATE_UNSAFE 68
|
||||
ENGINE_L0092_P001_OF001= "$TRANSPORT" verify-sealed "$LOGICAL_ID" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null
|
||||
ENGINE_L0093_P001_OF001= mkdir -p "$VERIFY_ROOT"; chmod 0700 "$VERIFY_ROOT"
|
||||
ENGINE_L0094_P001_OF001= work="$(mktemp -d "$VERIFY_ROOT/.verify.XXXXXX")"; cfg="$work/[SENSITIVE_FILE]"; prepare_rclone "$cfg"
|
||||
ENGINE_L0095_P001_OF001= cleanup_verify(){ rm -rf --one-file-system "$work"; }; trap cleanup_verify EXIT
|
||||
ENGINE_L0096_P001_OF001= base_a="$(jq -er '.remote_a_base' "$state")"; base_b="$(jq -er '.remote_b_base' "$state")"; a="$work/A.tar.gz"; b="$work/B.tar.gz"
|
||||
ENGINE_L0097_P001_OF001= reconstruct "$base_a" "$a" "$cfg" "$state" A; reconstruct "$base_b" "$b" "$cfg" "$state" B
|
||||
ENGINE_L0098_P001_OF001= cmp -s "$a" "$b" || die CLOUD_A_B_PAYLOAD_MISMATCH 68
|
||||
ENGINE_L0099_P001_OF001= verify_json="$("$BUILDER" verify --archive "$a" --expected-generation "$gen")" || die <LONG_TOKEN_REDACTED> 68
|
||||
ENGINE_L0100_P001_OF001=<REDACTED_SENSITIVE_LINE>
|
||||
ENGINE_L0101_P001_OF001= archive_sha="$(jq -er '.archive_sha256' "$state")"; mkdir -p "$EVIDENCE_ROOT" /var/lib/homelab-health; chmod 0700 "$EVIDENCE_ROOT"
|
||||
ENGINE_L0102_P001_OF001= tmp="$EVIDENCE_ROOT/.$gen.new.$$"; now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
ENGINE_L0103_P001_OF001=<REDACTED_SENSITIVE_LINE>
|
||||
ENGINE_L0104_P001_OF001= chmod 0600 "$tmp"; mv -f "$tmp" "$EVIDENCE_ROOT/$gen.json"
|
||||
ENGINE_L0105_P001_OF003= tmp="${HEALTH}.new.$$"; jq -n --arg gen "$gen" --arg state "$state" --arg evidence "$EVIDENCE_ROOT/$gen.json" --arg sha "$archive_sha" --arg now "$now" '{sche
|
||||
ENGINE_L0105_P002_OF003=ma:"homelab-control-backup-health-v1",status:"OK",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshn
|
||||
ENGINE_L0105_P003_OF003=ess:"FRESH",full_site_dr_proven:false,updated_at_utc:$now}' >"$tmp"; chmod 0644 "$tmp"; mv -f "$tmp" "$HEALTH"
|
||||
ENGINE_L0106_P001_OF001= trap - EXIT; cleanup_verify
|
||||
ENGINE_L0107_P001_OF001= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\n' "$gen" "$EVIDENCE_ROOT/$gen.json"
|
||||
ENGINE_L0108_P001_OF001=}
|
||||
ENGINE_L0109_P001_OF001=run_backup(){
|
||||
ENGINE_L0110_P001_OF001= local force="${1:-0}" decision line state gen payload outdir now
|
||||
ENGINE_L0111_P001_OF002= mkdir -p "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" /run/lock; chmod 0700 "$STATE_ROOT" "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_R
|
||||
ENGINE_L0111_P002_OF002=OOT"
|
||||
ENGINE_L0112_P001_OF001= exec 9>"$LOCK"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\n'; return 0; }
|
||||
ENGINE_L0113_P001_OF001= local incompletes
|
||||
ENGINE_L0114_P001_OF002= incompletes="$(find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(
|
||||
ENGINE_L0114_P002_OF002=.status!="SEALED")|.generation' 2>/dev/null || true)"
|
||||
ENGINE_L0115_P001_OF001= [[ "$(grep -c . <<<"$incompletes" || true)" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68
|
||||
ENGINE_L0116_P001_OF001= line="$(scan)"; printf '%s\n' "$line"; decision="$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,"",$i);print $i}}' <<<"$line")"
|
||||
ENGINE_L0117_P001_OF001= gen="$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$line")"
|
||||
ENGINE_L0118_P001_OF001= if [[ "$decision" == SKIP_FRESH && "$force" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\n' "$gen"; return 0; fi
|
||||
ENGINE_L0119_P001_OF001= if [[ "$decision" == BACKUP_REVERIFY ]]; then
|
||||
ENGINE_L0120_P001_OF001= if [[ "$force" != 1 ]]; then verify_generation "$gen"; return 0; fi
|
||||
ENGINE_L0121_P001_OF001= # Forced refresh deliberately creates a new generation instead of requiring
|
||||
ENGINE_L0122_P001_OF001= # a legacy generation to satisfy the current bundle schema.
|
||||
ENGINE_L0123_P001_OF001= decision=BACKUP_DUE
|
||||
ENGINE_L0124_P001_OF001= fi
|
||||
ENGINE_L0125_P001_OF001= if [[ "$decision" == BACKUP_RESUME ]]; then
|
||||
ENGINE_L0126_P001_OF001= state="$STATE_DIR/$gen.json"; payload="$(jq -er '.payload_path' "$state")"; [[ -s "$payload" && ! -L "$payload" ]] || die INCOMPLETE_PAYLOAD_MISSING 68
|
||||
ENGINE_L0127_P001_OF001= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die <LONG_TOKEN_REDACTED> 68
|
||||
ENGINE_L0128_P001_OF001= mark_cloud_write
|
||||
ENGINE_L0129_P001_OF002= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"; verify_generation "$gen"; printf 'CONTROL_BACKUP_RESULT=SE
|
||||
ENGINE_L0129_P002_OF002=ALED_RESUMED GENERATION=%s\n' "$gen"; return 0
|
||||
ENGINE_L0130_P001_OF001= fi
|
||||
ENGINE_L0131_P001_OF001= case "$decision" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die "DECISION_INVALID_$decision" 65;; esac
|
||||
ENGINE_L0132_P001_OF001= exec 8>"$HEAVY_LOCK"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\n'; return 0; }
|
||||
ENGINE_L0133_P001_OF001= now="$(date -u +%Y%m%dT%H%M%SZ)"; gen="control-bootstrap-v2-pve01-$now"; outdir="$STAGE_ROOT/$gen"
|
||||
ENGINE_L0134_P001_OF001= "$BUILDER" build --generation "$gen" --output-dir "$outdir" >/dev/null
|
||||
ENGINE_L0135_P001_OF001= payload="$outdir/$gen.tar.gz"; [[ -s "$payload" && ! -L "$payload" ]] || die BUILT_PAYLOAD_MISSING 68
|
||||
ENGINE_L0136_P001_OF001= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die <LONG_TOKEN_REDACTED> 68
|
||||
ENGINE_L0137_P001_OF001= exec 8>&-
|
||||
ENGINE_L0138_P001_OF001= mark_cloud_write
|
||||
ENGINE_L0139_P001_OF001= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
||||
ENGINE_L0140_P001_OF001= rm -rf --one-file-system "$outdir" 2>/dev/null || true
|
||||
ENGINE_L0141_P001_OF001= verify_generation "$gen"
|
||||
ENGINE_L0142_P001_OF001= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\n' "$gen"
|
||||
ENGINE_L0143_P001_OF001=}
|
||||
ENGINE_L0144_P001_OF001=case "${1:-}" in
|
||||
ENGINE_L0145_P001_OF001= scan) [[ "$#" == 1 ]] || die USAGE 64; scan;;
|
||||
ENGINE_L0146_P001_OF001= run) [[ "$#" == 1 ]] || die USAGE 64; run_backup 0;;
|
||||
ENGINE_L0147_P001_OF001= force) [[ "$#" == 1 ]] || die USAGE 64; run_backup 1;;
|
||||
ENGINE_L0148_P001_OF001= verify) [[ "$#" == 2 ]] || die USAGE 64; verify_generation "$2";;
|
||||
ENGINE_L0149_P001_OF001= *) die USAGE_scan_run_force_verify 64;;
|
||||
ENGINE_L0150_P001_OF001=esac
|
||||
ENGINE_SOURCE_END
|
||||
UNIT_PATH=/etc/systemd/system/homelab-control-backup.service
|
||||
UNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250
|
||||
UNIT_LINE_COUNT=16
|
||||
UNIT_REDACTED_LINE_COUNT=3
|
||||
UNIT_SOURCE_BEGIN
|
||||
UNIT_L0001_P001_OF001=[Unit]
|
||||
UNIT_L0002_P001_OF001=Description=Homelab recurring CONTROL cloud quorum backup
|
||||
UNIT_L0003_P001_OF001=After=network-online.target
|
||||
UNIT_L0004_P001_OF001=Wants=network-online.target
|
||||
UNIT_L0005_P001_OF001=ConditionPathExists=<LONG_TOKEN_REDACTED>
|
||||
UNIT_L0006_P001_OF001=ConditionPathExists=<LONG_TOKEN_REDACTED>
|
||||
UNIT_L0007_P001_OF001=
|
||||
UNIT_L0008_P001_OF001=[Service]
|
||||
UNIT_L0009_P001_OF001=Type=oneshot
|
||||
UNIT_L0010_P001_OF001=ExecStart=<LONG_TOKEN_REDACTED> run
|
||||
UNIT_L0011_P001_OF001=User=root
|
||||
UNIT_L0012_P001_OF001=Group=root
|
||||
UNIT_L0013_P001_OF001=UMask=0077
|
||||
UNIT_L0014_P001_OF001=Nice=10
|
||||
UNIT_L0015_P001_OF001=IOSchedulingClass=best-effort
|
||||
UNIT_L0016_P001_OF001=IOSchedulingPriority=7
|
||||
UNIT_SOURCE_END
|
||||
SOURCE_TRUNCATED_LINES=0
|
||||
SOURCE_FUNCTION_PARSER_USED=NO
|
||||
BACKUP_ENGINE_EXECUTIONS=0
|
||||
RCLONE_COMMANDS_EXECUTED=0
|
||||
CLOUD_CALLS=0
|
||||
FILES_WRITTEN_BY_PAYLOAD=0
|
||||
SYSTEMD_MUTATIONS=0
|
||||
SOURCE259_READ_COMPLETE=YES
|
||||
SOURCE259_END
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+8
-8
File diff suppressed because one or more lines are too long
+210
-16
@@ -1,15 +1,15 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1
|
||||
COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=infra-dr-backupv2-control-backup-logic-focus-diagnostic
|
||||
COMPONENT=infra-dr-backupv2-control-backup-source
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=f3ab0a154eda3d947c6b078120e2088b7d0721f992b74d9da8e78af6568708ca
|
||||
COMMAND_SHA256=09220124de7fa21ffc8fbc498052fa8ab92cee5d72b61fe34976f30307a58400
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
@@ -24,20 +24,214 @@ MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974
|
||||
SANITIZED_OUTPUT_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974
|
||||
RAW_EVIDENCE_SHA256=3762076dc1704d241330f8e0aa300dc7cbca2c80ec5ccc24137e59d67f7bd34b
|
||||
SANITIZED_OUTPUT_SHA256=497e4996869d853f0466d28fe81745576dffca2ccbe7ed64998990ed695d898b
|
||||
OUTPUT_BEGIN
|
||||
DIAG258_BEGIN
|
||||
TARGET_COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1
|
||||
TARGET_ATTEMPT_DIR=/root/homelab-run-SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1
|
||||
TARGET_ATTEMPT_DIR_EXISTS=NO
|
||||
MATCHING_257_ATTEMPT_DIR_COUNT=0
|
||||
EVIDENCE_LAYER=NO_257_SERVER_ATTEMPT_DIRECTORY
|
||||
INTERPRETATION=257_DID_NOT_REACH_RUNNER_RESERVATION
|
||||
FILES_WRITTEN_BY_DIAG258=0
|
||||
RCLONE_COMMANDS_BY_DIAG258=0
|
||||
CLOUD_MUTATIONS_BY_DIAG258=0
|
||||
DIAG258_END
|
||||
SOURCE259_BEGIN
|
||||
COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1
|
||||
MODE=READ_ONLY_HASH_BOUND_SOURCE_EXPORT
|
||||
SOURCE_EXECUTION=FORBIDDEN
|
||||
ERROR_REGISTER_CHECK=READ
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
REFERENCE_CHECK=READ
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
AUTHORITY_FRESH_READ=OK
|
||||
ENGINE_PATH=/usr/local/sbin/homelab-control-backup
|
||||
ENGINE_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57
|
||||
ENGINE_LINE_COUNT=150
|
||||
ENGINE_REDACTED_LINE_COUNT=13
|
||||
ENGINE_SOURCE_BEGIN
|
||||
ENGINE_L0001_P001_OF001=#!/usr/bin/env bash
|
||||
ENGINE_L0002_P001_OF001=set -Eeuo pipefail
|
||||
ENGINE_L0003_P001_OF001=umask 077
|
||||
ENGINE_L0004_P001_OF001=LOGICAL_ID=control-bootstrap-v2
|
||||
ENGINE_L0005_P001_OF001=DUE_SECONDS=43200
|
||||
ENGINE_L0006_P001_OF001=SLA_SECONDS=86400
|
||||
ENGINE_L0007_P001_OF001=STATE_ROOT=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0008_P001_OF001=STATE_DIR=$STATE_ROOT/$LOGICAL_ID
|
||||
ENGINE_L0009_P001_OF001=EVIDENCE_ROOT=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0010_P001_OF001=HEALTH=<LONG_TOKEN_REDACTED>.json
|
||||
ENGINE_L0011_P001_OF001=STAGE_ROOT=/mnt/staging/homelab-control-auto
|
||||
ENGINE_L0012_P001_OF001=VERIFY_ROOT=/mnt/staging/homelab-control-verify
|
||||
ENGINE_L0013_P001_OF001=LOCK=/run/lock/homelab-control-backup.lock
|
||||
ENGINE_L0014_P001_OF001=HEAVY_LOCK=/run/homelab-backup-heavy.lock
|
||||
ENGINE_L0015_P001_OF001=RCLONE_CONFIG_SOURCE=${RCLONE_CONFIG:-/root/.config/rclone/[SENSITIVE_FILE]}
|
||||
ENGINE_L0016_P001_OF001=TRANSPORT=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0017_P001_OF001=TRANSPORT_SHA=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0018_P001_OF001=BUILDER=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0019_P001_OF001=BUILDER_SHA=<LONG_TOKEN_REDACTED>
|
||||
ENGINE_L0020_P001_OF001=REMOTE_A=pve01-mail-01-crypt:homelab-backups
|
||||
ENGINE_L0021_P001_OF001=REMOTE_B=pve01-mail-02-crypt:homelab-backups
|
||||
ENGINE_L0022_P001_OF001=CLOUD_WRITE_MARKER=${HOMELAB_CONTROL_CLOUD_WRITE_MARKER:-}
|
||||
ENGINE_L0023_P001_OF001=
|
||||
ENGINE_L0024_P001_OF001=die(){ printf 'CONTROL_BACKUP_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; }
|
||||
ENGINE_L0025_P001_OF002=for c in jq date find sort tail cut sha256sum stat flock rclone mktemp cmp awk mkdir rm install xargs grep cat mv chmod; do command -v "$c" >/dev/null || die "T
|
||||
ENGINE_L0025_P002_OF002=OOL_MISSING_$c" 69; done
|
||||
ENGINE_L0026_P001_OF001=[[ -x "$TRANSPORT" && ! -L "$TRANSPORT" ]] || die TRANSPORT_UNSAFE 69
|
||||
ENGINE_L0027_P001_OF001=[[ "$(sha256sum "$TRANSPORT"|awk '{print $1}')" == "$TRANSPORT_SHA" ]] || die TRANSPORT_SHA_DRIFT 65
|
||||
ENGINE_L0028_P001_OF001=[[ -x "$BUILDER" && ! -L "$BUILDER" ]] || die BUILDER_UNSAFE 69
|
||||
ENGINE_L0029_P001_OF001=[[ "$(sha256sum "$BUILDER"|awk '{print $1}')" == "$BUILDER_SHA" ]] || die BUILDER_SHA_DRIFT 65
|
||||
ENGINE_L0030_P001_OF001=
|
||||
ENGINE_L0031_P001_OF001=latest_state(){
|
||||
ENGINE_L0032_P001_OF001= [[ -d "$STATE_DIR" && ! -L "$STATE_DIR" ]] || return 0
|
||||
ENGINE_L0033_P001_OF001= find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print 2>/dev/null | LC_ALL=C sort | tail -1
|
||||
ENGINE_L0034_P001_OF001=}
|
||||
ENGINE_L0035_P001_OF001=state_decision(){
|
||||
ENGINE_L0036_P001_OF001= local state="$1" now="$2" status gen upd epoch age ev sha
|
||||
ENGINE_L0037_P001_OF001= if [[ -z "$state" ]]; then printf 'DECISION=BACKUP_UNVERIFIED REASON=NO_STATE AGE_SECONDS=-1\n'; return 0; fi
|
||||
ENGINE_L0038_P001_OF001= [[ -s "$state" && ! -L "$state" ]] || { printf 'DECISION=BACKUP_UNVERIFIED REASON=STATE_UNSAFE AGE_SECONDS=-1\n'; return 0; }
|
||||
ENGINE_L0039_P001_OF001= status="$(jq -r '.status // "INVALID"' "$state")"; gen="$(jq -r '.generation // "UNKNOWN"' "$state")"
|
||||
ENGINE_L0040_P001_OF001= case "$status" in
|
||||
ENGINE_L0041_P001_OF001= PREPARING|A_VERIFIED|PREPARED|DELETE_AUTHORIZED)
|
||||
ENGINE_L0042_P001_OF001= printf 'DECISION=BACKUP_RESUME GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\n' "$gen" "$state"; return 0;;
|
||||
ENGINE_L0043_P001_OF001= SEALED) ;;
|
||||
ENGINE_L0044_P001_OF001= *) printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=STATE_STATUS_INVALID AGE_SECONDS=-1\n' "$gen"; return 0;;
|
||||
ENGINE_L0045_P001_OF001= esac
|
||||
ENGINE_L0046_P001_OF001= sha="$(jq -r '.archive_sha256 // ""' "$state")"; upd="$(jq -r '.sealed_at_utc // .updated_at_utc // ""' "$state")"
|
||||
ENGINE_L0047_P001_OF001= epoch="$(date -u -d "$upd" +%s 2>/dev/null || true)"
|
||||
ENGINE_L0048_P001_OF002= [[ "$epoch" =~ ^[0-9]+$ && "$epoch" -le "$now" ]] || { printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=TIMESTAMP_INVALID AGE_SECONDS=-1\n' "$gen"; ret
|
||||
ENGINE_L0048_P002_OF002=urn 0; }
|
||||
ENGINE_L0049_P001_OF001= ev="$EVIDENCE_ROOT/$gen.json"
|
||||
ENGINE_L0050_P001_OF001=<REDACTED_SENSITIVE_LINE>
|
||||
ENGINE_L0051_P001_OF001= printf 'DECISION=BACKUP_REVERIFY GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\n' "$gen" "$state"; return 0
|
||||
ENGINE_L0052_P001_OF001= fi
|
||||
ENGINE_L0053_P001_OF001= age=$((now-epoch))
|
||||
ENGINE_L0054_P001_OF001= if (( age < DUE_SECONDS )); then d=SKIP_FRESH; elif (( age < SLA_SECONDS )); then d=BACKUP_DUE; else d=BACKUP_STALE; fi
|
||||
ENGINE_L0055_P001_OF001= printf 'DECISION=%s GENERATION=%s STATE_FILE=%s AGE_SECONDS=%s DUE_SECONDS=%s SLA_SECONDS=%s\n' "$d" "$gen" "$state" "$age" "$DUE_SECONDS" "$SLA_SECONDS"
|
||||
ENGINE_L0056_P001_OF001=}
|
||||
ENGINE_L0057_P001_OF001=scan(){ local now state; now="$(date -u +%s)"; state="$(latest_state)"; state_decision "$state" "$now"; }
|
||||
ENGINE_L0058_P001_OF001=mark_cloud_write(){
|
||||
ENGINE_L0059_P001_OF001= [[ -z "$CLOUD_WRITE_MARKER" ]] && return 0
|
||||
ENGINE_L0060_P001_OF001= [[ "$CLOUD_WRITE_MARKER" == /* ]] || die CLOUD_WRITE_MARKER_NOT_ABSOLUTE 65
|
||||
ENGINE_L0061_P001_OF001= [[ ! -L "$CLOUD_WRITE_MARKER" ]] || die CLOUD_WRITE_MARKER_SYMLINK 65
|
||||
ENGINE_L0062_P001_OF001= printf '%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$CLOUD_WRITE_MARKER"
|
||||
ENGINE_L0063_P001_OF001= chmod 0600 "$CLOUD_WRITE_MARKER"
|
||||
ENGINE_L0064_P001_OF001=}
|
||||
ENGINE_L0065_P001_OF001=
|
||||
ENGINE_L0066_P001_OF001=prepare_rclone(){
|
||||
ENGINE_L0067_P001_OF001= local dst="$1"; [[ -f "$RCLONE_CONFIG_SOURCE" && ! -L "$RCLONE_CONFIG_SOURCE" ]] || die RCLONE_CONFIG_UNSAFE 66
|
||||
ENGINE_L0068_P001_OF001= install -m 0600 "$RCLONE_CONFIG_SOURCE" "$dst"
|
||||
ENGINE_L0069_P001_OF001=}
|
||||
ENGINE_L0070_P001_OF001=reconstruct(){
|
||||
ENGINE_L0071_P001_OF001= local base="$1" out="$2" cfg="$3" state="$4" label="$5" manifest part idx name size hash got_size got_hash expected_manifest expected_archive
|
||||
ENGINE_L0072_P001_OF001= expected_manifest="$(jq -er '.parts_manifest_sha256' "$state")"; expected_archive="$(jq -er '.archive_sha256' "$state")"
|
||||
ENGINE_L0073_P001_OF001= manifest="${out}.PARTS.tsv"; : >"$out"
|
||||
ENGINE_L0074_P001_OF001= rclone --config "$cfg" cat "${base%/}/PARTS.tsv" >"$manifest" || die "REMOTE_${label}_MANIFEST_READ_FAILED" 74
|
||||
ENGINE_L0075_P001_OF001= [[ "$(sha256sum "$manifest"|awk '{print $1}')" == "$expected_manifest" ]] || die "REMOTE_${label}_MANIFEST_SHA_MISMATCH" 68
|
||||
ENGINE_L0076_P001_OF001= idx=0
|
||||
ENGINE_L0077_P001_OF001= while IFS=$'\t' read -r row name size hash; do
|
||||
ENGINE_L0078_P001_OF002= [[ "$row" == "$(printf '%08d' "$idx")" && "$name" == part-????????.bin && "$size" =~ ^[1-9][0-9]*$ && "$hash" =~ ^[0-9a-f]{64}$ ]] || die "REMOTE_${label}_M
|
||||
ENGINE_L0078_P002_OF002=ANIFEST_ROW_INVALID" 68
|
||||
ENGINE_L0079_P001_OF001= part="${out}.part"; rm -f "$part"; rclone --config "$cfg" cat "${base%/}/payload.parts/$name" >"$part" || die "REMOTE_${label}_PART_READ_FAILED" 74
|
||||
ENGINE_L0080_P001_OF001= got_size="$(stat -c %s "$part")"; got_hash="$(sha256sum "$part"|awk '{print $1}')"
|
||||
ENGINE_L0081_P001_OF001= [[ "$got_size" == "$size" && "$got_hash" == "$hash" ]] || die "REMOTE_${label}_PART_VERIFY_FAILED" 68
|
||||
ENGINE_L0082_P001_OF001= cat "$part" >>"$out"; rm -f "$part"; idx=$((idx+1))
|
||||
ENGINE_L0083_P001_OF001= done <"$manifest"
|
||||
ENGINE_L0084_P001_OF001= [[ "$(sha256sum "$out"|awk '{print $1}')" == "$expected_archive" ]] || die "REMOTE_${label}_ARCHIVE_SHA_MISMATCH" 68
|
||||
ENGINE_L0085_P001_OF001= rm -f "$manifest"
|
||||
ENGINE_L0086_P001_OF001=}
|
||||
ENGINE_L0087_P001_OF001=verify_generation(){
|
||||
ENGINE_L0088_P001_OF001= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now
|
||||
ENGINE_L0089_P001_OF001= gen="$1"
|
||||
ENGINE_L0090_P001_OF001= state="$STATE_DIR/$gen.json"
|
||||
ENGINE_L0091_P001_OF001= [[ -s "$state" && ! -L "$state" ]] || die VERIFY_STATE_UNSAFE 68
|
||||
ENGINE_L0092_P001_OF001= "$TRANSPORT" verify-sealed "$LOGICAL_ID" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null
|
||||
ENGINE_L0093_P001_OF001= mkdir -p "$VERIFY_ROOT"; chmod 0700 "$VERIFY_ROOT"
|
||||
ENGINE_L0094_P001_OF001= work="$(mktemp -d "$VERIFY_ROOT/.verify.XXXXXX")"; cfg="$work/[SENSITIVE_FILE]"; prepare_rclone "$cfg"
|
||||
ENGINE_L0095_P001_OF001= cleanup_verify(){ rm -rf --one-file-system "$work"; }; trap cleanup_verify EXIT
|
||||
ENGINE_L0096_P001_OF001= base_a="$(jq -er '.remote_a_base' "$state")"; base_b="$(jq -er '.remote_b_base' "$state")"; a="$work/A.tar.gz"; b="$work/B.tar.gz"
|
||||
ENGINE_L0097_P001_OF001= reconstruct "$base_a" "$a" "$cfg" "$state" A; reconstruct "$base_b" "$b" "$cfg" "$state" B
|
||||
ENGINE_L0098_P001_OF001= cmp -s "$a" "$b" || die CLOUD_A_B_PAYLOAD_MISMATCH 68
|
||||
ENGINE_L0099_P001_OF001= verify_json="$("$BUILDER" verify --archive "$a" --expected-generation "$gen")" || die <LONG_TOKEN_REDACTED> 68
|
||||
ENGINE_L0100_P001_OF001=<REDACTED_SENSITIVE_LINE>
|
||||
ENGINE_L0101_P001_OF001= archive_sha="$(jq -er '.archive_sha256' "$state")"; mkdir -p "$EVIDENCE_ROOT" /var/lib/homelab-health; chmod 0700 "$EVIDENCE_ROOT"
|
||||
ENGINE_L0102_P001_OF001= tmp="$EVIDENCE_ROOT/.$gen.new.$$"; now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
ENGINE_L0103_P001_OF001=<REDACTED_SENSITIVE_LINE>
|
||||
ENGINE_L0104_P001_OF001= chmod 0600 "$tmp"; mv -f "$tmp" "$EVIDENCE_ROOT/$gen.json"
|
||||
ENGINE_L0105_P001_OF003= tmp="${HEALTH}.new.$$"; jq -n --arg gen "$gen" --arg state "$state" --arg evidence "$EVIDENCE_ROOT/$gen.json" --arg sha "$archive_sha" --arg now "$now" '{sche
|
||||
ENGINE_L0105_P002_OF003=ma:"homelab-control-backup-health-v1",status:"OK",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshn
|
||||
ENGINE_L0105_P003_OF003=ess:"FRESH",full_site_dr_proven:false,updated_at_utc:$now}' >"$tmp"; chmod 0644 "$tmp"; mv -f "$tmp" "$HEALTH"
|
||||
ENGINE_L0106_P001_OF001= trap - EXIT; cleanup_verify
|
||||
ENGINE_L0107_P001_OF001= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\n' "$gen" "$EVIDENCE_ROOT/$gen.json"
|
||||
ENGINE_L0108_P001_OF001=}
|
||||
ENGINE_L0109_P001_OF001=run_backup(){
|
||||
ENGINE_L0110_P001_OF001= local force="${1:-0}" decision line state gen payload outdir now
|
||||
ENGINE_L0111_P001_OF002= mkdir -p "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" /run/lock; chmod 0700 "$STATE_ROOT" "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_R
|
||||
ENGINE_L0111_P002_OF002=OOT"
|
||||
ENGINE_L0112_P001_OF001= exec 9>"$LOCK"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\n'; return 0; }
|
||||
ENGINE_L0113_P001_OF001= local incompletes
|
||||
ENGINE_L0114_P001_OF002= incompletes="$(find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(
|
||||
ENGINE_L0114_P002_OF002=.status!="SEALED")|.generation' 2>/dev/null || true)"
|
||||
ENGINE_L0115_P001_OF001= [[ "$(grep -c . <<<"$incompletes" || true)" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68
|
||||
ENGINE_L0116_P001_OF001= line="$(scan)"; printf '%s\n' "$line"; decision="$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,"",$i);print $i}}' <<<"$line")"
|
||||
ENGINE_L0117_P001_OF001= gen="$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$line")"
|
||||
ENGINE_L0118_P001_OF001= if [[ "$decision" == SKIP_FRESH && "$force" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\n' "$gen"; return 0; fi
|
||||
ENGINE_L0119_P001_OF001= if [[ "$decision" == BACKUP_REVERIFY ]]; then
|
||||
ENGINE_L0120_P001_OF001= if [[ "$force" != 1 ]]; then verify_generation "$gen"; return 0; fi
|
||||
ENGINE_L0121_P001_OF001= # Forced refresh deliberately creates a new generation instead of requiring
|
||||
ENGINE_L0122_P001_OF001= # a legacy generation to satisfy the current bundle schema.
|
||||
ENGINE_L0123_P001_OF001= decision=BACKUP_DUE
|
||||
ENGINE_L0124_P001_OF001= fi
|
||||
ENGINE_L0125_P001_OF001= if [[ "$decision" == BACKUP_RESUME ]]; then
|
||||
ENGINE_L0126_P001_OF001= state="$STATE_DIR/$gen.json"; payload="$(jq -er '.payload_path' "$state")"; [[ -s "$payload" && ! -L "$payload" ]] || die INCOMPLETE_PAYLOAD_MISSING 68
|
||||
ENGINE_L0127_P001_OF001= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die <LONG_TOKEN_REDACTED> 68
|
||||
ENGINE_L0128_P001_OF001= mark_cloud_write
|
||||
ENGINE_L0129_P001_OF002= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"; verify_generation "$gen"; printf 'CONTROL_BACKUP_RESULT=SE
|
||||
ENGINE_L0129_P002_OF002=ALED_RESUMED GENERATION=%s\n' "$gen"; return 0
|
||||
ENGINE_L0130_P001_OF001= fi
|
||||
ENGINE_L0131_P001_OF001= case "$decision" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die "DECISION_INVALID_$decision" 65;; esac
|
||||
ENGINE_L0132_P001_OF001= exec 8>"$HEAVY_LOCK"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\n'; return 0; }
|
||||
ENGINE_L0133_P001_OF001= now="$(date -u +%Y%m%dT%H%M%SZ)"; gen="control-bootstrap-v2-pve01-$now"; outdir="$STAGE_ROOT/$gen"
|
||||
ENGINE_L0134_P001_OF001= "$BUILDER" build --generation "$gen" --output-dir "$outdir" >/dev/null
|
||||
ENGINE_L0135_P001_OF001= payload="$outdir/$gen.tar.gz"; [[ -s "$payload" && ! -L "$payload" ]] || die BUILT_PAYLOAD_MISSING 68
|
||||
ENGINE_L0136_P001_OF001= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die <LONG_TOKEN_REDACTED> 68
|
||||
ENGINE_L0137_P001_OF001= exec 8>&-
|
||||
ENGINE_L0138_P001_OF001= mark_cloud_write
|
||||
ENGINE_L0139_P001_OF001= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
||||
ENGINE_L0140_P001_OF001= rm -rf --one-file-system "$outdir" 2>/dev/null || true
|
||||
ENGINE_L0141_P001_OF001= verify_generation "$gen"
|
||||
ENGINE_L0142_P001_OF001= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\n' "$gen"
|
||||
ENGINE_L0143_P001_OF001=}
|
||||
ENGINE_L0144_P001_OF001=case "${1:-}" in
|
||||
ENGINE_L0145_P001_OF001= scan) [[ "$#" == 1 ]] || die USAGE 64; scan;;
|
||||
ENGINE_L0146_P001_OF001= run) [[ "$#" == 1 ]] || die USAGE 64; run_backup 0;;
|
||||
ENGINE_L0147_P001_OF001= force) [[ "$#" == 1 ]] || die USAGE 64; run_backup 1;;
|
||||
ENGINE_L0148_P001_OF001= verify) [[ "$#" == 2 ]] || die USAGE 64; verify_generation "$2";;
|
||||
ENGINE_L0149_P001_OF001= *) die USAGE_scan_run_force_verify 64;;
|
||||
ENGINE_L0150_P001_OF001=esac
|
||||
ENGINE_SOURCE_END
|
||||
UNIT_PATH=/etc/systemd/system/homelab-control-backup.service
|
||||
UNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250
|
||||
UNIT_LINE_COUNT=16
|
||||
UNIT_REDACTED_LINE_COUNT=3
|
||||
UNIT_SOURCE_BEGIN
|
||||
UNIT_L0001_P001_OF001=[Unit]
|
||||
UNIT_L0002_P001_OF001=Description=Homelab recurring CONTROL cloud quorum backup
|
||||
UNIT_L0003_P001_OF001=After=network-online.target
|
||||
UNIT_L0004_P001_OF001=Wants=network-online.target
|
||||
UNIT_L0005_P001_OF001=ConditionPathExists=<LONG_TOKEN_REDACTED>
|
||||
UNIT_L0006_P001_OF001=ConditionPathExists=<LONG_TOKEN_REDACTED>
|
||||
UNIT_L0007_P001_OF001=
|
||||
UNIT_L0008_P001_OF001=[Service]
|
||||
UNIT_L0009_P001_OF001=Type=oneshot
|
||||
UNIT_L0010_P001_OF001=ExecStart=<LONG_TOKEN_REDACTED> run
|
||||
UNIT_L0011_P001_OF001=User=root
|
||||
UNIT_L0012_P001_OF001=Group=root
|
||||
UNIT_L0013_P001_OF001=UMask=0077
|
||||
UNIT_L0014_P001_OF001=Nice=10
|
||||
UNIT_L0015_P001_OF001=IOSchedulingClass=best-effort
|
||||
UNIT_L0016_P001_OF001=IOSchedulingPriority=7
|
||||
UNIT_SOURCE_END
|
||||
SOURCE_TRUNCATED_LINES=0
|
||||
SOURCE_FUNCTION_PARSER_USED=NO
|
||||
BACKUP_ENGINE_EXECUTIONS=0
|
||||
RCLONE_COMMANDS_EXECUTED=0
|
||||
CLOUD_CALLS=0
|
||||
FILES_WRITTEN_BY_PAYLOAD=0
|
||||
SYSTEMD_MUTATIONS=0
|
||||
SOURCE259_READ_COMPLETE=YES
|
||||
SOURCE259_END
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user