From d04e1ec5355df6e21e84d3275f9c41b2aa5c2b4f Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Sat, 12 Sep 2026 00:02:19 +0000 Subject: [PATCH] runtime: publish SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1 --- ...-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1.json | 38 +++ ...R-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1.txt | 237 ++++++++++++++++++ runtime/latest.json | 16 +- runtime/latest.txt | 226 +++++++++++++++-- 4 files changed, 493 insertions(+), 24 deletions(-) create mode 100644 runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1.json create mode 100644 runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1.txt diff --git a/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1.json b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1.json new file mode 100644 index 00000000..dad1d4c0 --- /dev/null +++ b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "infra-dr-backupv2-control-backup-source", + "started_at_utc": "2026-09-12T00:02:19Z", + "finished_at_utc": "2026-09-12T00:02:19Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "09220124de7fa21ffc8fbc498052fa8ab92cee5d72b61fe34976f30307a58400", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "3762076dc1704d241330f8e0aa300dc7cbca2c80ec5ccc24137e59d67f7bd34b", + "sanitized_output_sha256": "497e4996869d853f0466d28fe81745576dffca2ccbe7ed64998990ed695d898b", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "SOURCE259_BEGIN\nCOMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1\nMODE=READ_ONLY_HASH_BOUND_SOURCE_EXPORT\nSOURCE_EXECUTION=FORBIDDEN\nERROR_REGISTER_CHECK=READ\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nREFERENCE_CHECK=READ\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nAUTHORITY_FRESH_READ=OK\nENGINE_PATH=/usr/local/sbin/homelab-control-backup\nENGINE_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57\nENGINE_LINE_COUNT=150\nENGINE_REDACTED_LINE_COUNT=13\nENGINE_SOURCE_BEGIN\nENGINE_L0001_P001_OF001=#!/usr/bin/env bash\nENGINE_L0002_P001_OF001=set -Eeuo pipefail\nENGINE_L0003_P001_OF001=umask 077\nENGINE_L0004_P001_OF001=LOGICAL_ID=control-bootstrap-v2\nENGINE_L0005_P001_OF001=DUE_SECONDS=43200\nENGINE_L0006_P001_OF001=SLA_SECONDS=86400\nENGINE_L0007_P001_OF001=STATE_ROOT=\nENGINE_L0008_P001_OF001=STATE_DIR=$STATE_ROOT/$LOGICAL_ID\nENGINE_L0009_P001_OF001=EVIDENCE_ROOT=\nENGINE_L0010_P001_OF001=HEALTH=.json\nENGINE_L0011_P001_OF001=STAGE_ROOT=/mnt/staging/homelab-control-auto\nENGINE_L0012_P001_OF001=VERIFY_ROOT=/mnt/staging/homelab-control-verify\nENGINE_L0013_P001_OF001=LOCK=/run/lock/homelab-control-backup.lock\nENGINE_L0014_P001_OF001=HEAVY_LOCK=/run/homelab-backup-heavy.lock\nENGINE_L0015_P001_OF001=RCLONE_CONFIG_SOURCE=${RCLONE_CONFIG:-/root/.config/rclone/[SENSITIVE_FILE]}\nENGINE_L0016_P001_OF001=TRANSPORT=\nENGINE_L0017_P001_OF001=TRANSPORT_SHA=\nENGINE_L0018_P001_OF001=BUILDER=\nENGINE_L0019_P001_OF001=BUILDER_SHA=\nENGINE_L0020_P001_OF001=REMOTE_A=pve01-mail-01-crypt:homelab-backups\nENGINE_L0021_P001_OF001=REMOTE_B=pve01-mail-02-crypt:homelab-backups\nENGINE_L0022_P001_OF001=CLOUD_WRITE_MARKER=${HOMELAB_CONTROL_CLOUD_WRITE_MARKER:-}\nENGINE_L0023_P001_OF001=\nENGINE_L0024_P001_OF001=die(){ printf 'CONTROL_BACKUP_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }\nENGINE_L0025_P001_OF002=for c in jq date find sort tail cut sha256sum stat flock rclone mktemp cmp awk mkdir rm install xargs grep cat mv chmod; do command -v \"$c\" >/dev/null || die \"T\nENGINE_L0025_P002_OF002=OOL_MISSING_$c\" 69; done\nENGINE_L0026_P001_OF001=[[ -x \"$TRANSPORT\" && ! -L \"$TRANSPORT\" ]] || die TRANSPORT_UNSAFE 69\nENGINE_L0027_P001_OF001=[[ \"$(sha256sum \"$TRANSPORT\"|awk '{print $1}')\" == \"$TRANSPORT_SHA\" ]] || die TRANSPORT_SHA_DRIFT 65\nENGINE_L0028_P001_OF001=[[ -x \"$BUILDER\" && ! -L \"$BUILDER\" ]] || die BUILDER_UNSAFE 69\nENGINE_L0029_P001_OF001=[[ \"$(sha256sum \"$BUILDER\"|awk '{print $1}')\" == \"$BUILDER_SHA\" ]] || die BUILDER_SHA_DRIFT 65\nENGINE_L0030_P001_OF001=\nENGINE_L0031_P001_OF001=latest_state(){\nENGINE_L0032_P001_OF001= [[ -d \"$STATE_DIR\" && ! -L \"$STATE_DIR\" ]] || return 0\nENGINE_L0033_P001_OF001= find \"$STATE_DIR\" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print 2>/dev/null | LC_ALL=C sort | tail -1\nENGINE_L0034_P001_OF001=}\nENGINE_L0035_P001_OF001=state_decision(){\nENGINE_L0036_P001_OF001= local state=\"$1\" now=\"$2\" status gen upd epoch age ev sha\nENGINE_L0037_P001_OF001= if [[ -z \"$state\" ]]; then printf 'DECISION=BACKUP_UNVERIFIED REASON=NO_STATE AGE_SECONDS=-1\\n'; return 0; fi\nENGINE_L0038_P001_OF001= [[ -s \"$state\" && ! -L \"$state\" ]] || { printf 'DECISION=BACKUP_UNVERIFIED REASON=STATE_UNSAFE AGE_SECONDS=-1\\n'; return 0; }\nENGINE_L0039_P001_OF001= status=\"$(jq -r '.status // \"INVALID\"' \"$state\")\"; gen=\"$(jq -r '.generation // \"UNKNOWN\"' \"$state\")\"\nENGINE_L0040_P001_OF001= case \"$status\" in\nENGINE_L0041_P001_OF001= PREPARING|A_VERIFIED|PREPARED|DELETE_AUTHORIZED)\nENGINE_L0042_P001_OF001= printf 'DECISION=BACKUP_RESUME GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\\n' \"$gen\" \"$state\"; return 0;;\nENGINE_L0043_P001_OF001= SEALED) ;;\nENGINE_L0044_P001_OF001= *) printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=STATE_STATUS_INVALID AGE_SECONDS=-1\\n' \"$gen\"; return 0;;\nENGINE_L0045_P001_OF001= esac\nENGINE_L0046_P001_OF001= sha=\"$(jq -r '.archive_sha256 // \"\"' \"$state\")\"; upd=\"$(jq -r '.sealed_at_utc // .updated_at_utc // \"\"' \"$state\")\"\nENGINE_L0047_P001_OF001= epoch=\"$(date -u -d \"$upd\" +%s 2>/dev/null || true)\"\nENGINE_L0048_P001_OF002= [[ \"$epoch\" =~ ^[0-9]+$ && \"$epoch\" -le \"$now\" ]] || { printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=TIMESTAMP_INVALID AGE_SECONDS=-1\\n' \"$gen\"; ret\nENGINE_L0048_P002_OF002=urn 0; }\nENGINE_L0049_P001_OF001= ev=\"$EVIDENCE_ROOT/$gen.json\"\nENGINE_L0050_P001_OF001=\nENGINE_L0051_P001_OF001= printf 'DECISION=BACKUP_REVERIFY GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\\n' \"$gen\" \"$state\"; return 0\nENGINE_L0052_P001_OF001= fi\nENGINE_L0053_P001_OF001= age=$((now-epoch))\nENGINE_L0054_P001_OF001= if (( age < DUE_SECONDS )); then d=SKIP_FRESH; elif (( age < SLA_SECONDS )); then d=BACKUP_DUE; else d=BACKUP_STALE; fi\nENGINE_L0055_P001_OF001= printf 'DECISION=%s GENERATION=%s STATE_FILE=%s AGE_SECONDS=%s DUE_SECONDS=%s SLA_SECONDS=%s\\n' \"$d\" \"$gen\" \"$state\" \"$age\" \"$DUE_SECONDS\" \"$SLA_SECONDS\"\nENGINE_L0056_P001_OF001=}\nENGINE_L0057_P001_OF001=scan(){ local now state; now=\"$(date -u +%s)\"; state=\"$(latest_state)\"; state_decision \"$state\" \"$now\"; }\nENGINE_L0058_P001_OF001=mark_cloud_write(){\nENGINE_L0059_P001_OF001= [[ -z \"$CLOUD_WRITE_MARKER\" ]] && return 0\nENGINE_L0060_P001_OF001= [[ \"$CLOUD_WRITE_MARKER\" == /* ]] || die CLOUD_WRITE_MARKER_NOT_ABSOLUTE 65\nENGINE_L0061_P001_OF001= [[ ! -L \"$CLOUD_WRITE_MARKER\" ]] || die CLOUD_WRITE_MARKER_SYMLINK 65\nENGINE_L0062_P001_OF001= printf '%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" >\"$CLOUD_WRITE_MARKER\"\nENGINE_L0063_P001_OF001= chmod 0600 \"$CLOUD_WRITE_MARKER\"\nENGINE_L0064_P001_OF001=}\nENGINE_L0065_P001_OF001=\nENGINE_L0066_P001_OF001=prepare_rclone(){\nENGINE_L0067_P001_OF001= local dst=\"$1\"; [[ -f \"$RCLONE_CONFIG_SOURCE\" && ! -L \"$RCLONE_CONFIG_SOURCE\" ]] || die RCLONE_CONFIG_UNSAFE 66\nENGINE_L0068_P001_OF001= install -m 0600 \"$RCLONE_CONFIG_SOURCE\" \"$dst\"\nENGINE_L0069_P001_OF001=}\nENGINE_L0070_P001_OF001=reconstruct(){\nENGINE_L0071_P001_OF001= local base=\"$1\" out=\"$2\" cfg=\"$3\" state=\"$4\" label=\"$5\" manifest part idx name size hash got_size got_hash expected_manifest expected_archive\nENGINE_L0072_P001_OF001= expected_manifest=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"; expected_archive=\"$(jq -er '.archive_sha256' \"$state\")\"\nENGINE_L0073_P001_OF001= manifest=\"${out}.PARTS.tsv\"; : >\"$out\"\nENGINE_L0074_P001_OF001= rclone --config \"$cfg\" cat \"${base%/}/PARTS.tsv\" >\"$manifest\" || die \"REMOTE_${label}_MANIFEST_READ_FAILED\" 74\nENGINE_L0075_P001_OF001= [[ \"$(sha256sum \"$manifest\"|awk '{print $1}')\" == \"$expected_manifest\" ]] || die \"REMOTE_${label}_MANIFEST_SHA_MISMATCH\" 68\nENGINE_L0076_P001_OF001= idx=0\nENGINE_L0077_P001_OF001= while IFS=$'\\t' read -r row name size hash; do\nENGINE_L0078_P001_OF002= [[ \"$row\" == \"$(printf '%08d' \"$idx\")\" && \"$name\" == part-????????.bin && \"$size\" =~ ^[1-9][0-9]*$ && \"$hash\" =~ ^[0-9a-f]{64}$ ]] || die \"REMOTE_${label}_M\nENGINE_L0078_P002_OF002=ANIFEST_ROW_INVALID\" 68\nENGINE_L0079_P001_OF001= part=\"${out}.part\"; rm -f \"$part\"; rclone --config \"$cfg\" cat \"${base%/}/payload.parts/$name\" >\"$part\" || die \"REMOTE_${label}_PART_READ_FAILED\" 74\nENGINE_L0080_P001_OF001= got_size=\"$(stat -c %s \"$part\")\"; got_hash=\"$(sha256sum \"$part\"|awk '{print $1}')\"\nENGINE_L0081_P001_OF001= [[ \"$got_size\" == \"$size\" && \"$got_hash\" == \"$hash\" ]] || die \"REMOTE_${label}_PART_VERIFY_FAILED\" 68\nENGINE_L0082_P001_OF001= cat \"$part\" >>\"$out\"; rm -f \"$part\"; idx=$((idx+1))\nENGINE_L0083_P001_OF001= done <\"$manifest\"\nENGINE_L0084_P001_OF001= [[ \"$(sha256sum \"$out\"|awk '{print $1}')\" == \"$expected_archive\" ]] || die \"REMOTE_${label}_ARCHIVE_SHA_MISMATCH\" 68\nENGINE_L0085_P001_OF001= rm -f \"$manifest\"\nENGINE_L0086_P001_OF001=}\nENGINE_L0087_P001_OF001=verify_generation(){\nENGINE_L0088_P001_OF001= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now\nENGINE_L0089_P001_OF001= gen=\"$1\"\nENGINE_L0090_P001_OF001= state=\"$STATE_DIR/$gen.json\"\nENGINE_L0091_P001_OF001= [[ -s \"$state\" && ! -L \"$state\" ]] || die VERIFY_STATE_UNSAFE 68\nENGINE_L0092_P001_OF001= \"$TRANSPORT\" verify-sealed \"$LOGICAL_ID\" \"$gen\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\" >/dev/null\nENGINE_L0093_P001_OF001= mkdir -p \"$VERIFY_ROOT\"; chmod 0700 \"$VERIFY_ROOT\"\nENGINE_L0094_P001_OF001= work=\"$(mktemp -d \"$VERIFY_ROOT/.verify.XXXXXX\")\"; cfg=\"$work/[SENSITIVE_FILE]\"; prepare_rclone \"$cfg\"\nENGINE_L0095_P001_OF001= cleanup_verify(){ rm -rf --one-file-system \"$work\"; }; trap cleanup_verify EXIT\nENGINE_L0096_P001_OF001= base_a=\"$(jq -er '.remote_a_base' \"$state\")\"; base_b=\"$(jq -er '.remote_b_base' \"$state\")\"; a=\"$work/A.tar.gz\"; b=\"$work/B.tar.gz\"\nENGINE_L0097_P001_OF001= reconstruct \"$base_a\" \"$a\" \"$cfg\" \"$state\" A; reconstruct \"$base_b\" \"$b\" \"$cfg\" \"$state\" B\nENGINE_L0098_P001_OF001= cmp -s \"$a\" \"$b\" || die CLOUD_A_B_PAYLOAD_MISMATCH 68\nENGINE_L0099_P001_OF001= verify_json=\"$(\"$BUILDER\" verify --archive \"$a\" --expected-generation \"$gen\")\" || die 68\nENGINE_L0100_P001_OF001=\nENGINE_L0101_P001_OF001= archive_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; mkdir -p \"$EVIDENCE_ROOT\" /var/lib/homelab-health; chmod 0700 \"$EVIDENCE_ROOT\"\nENGINE_L0102_P001_OF001= tmp=\"$EVIDENCE_ROOT/.$gen.new.$$\"; now=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nENGINE_L0103_P001_OF001=\nENGINE_L0104_P001_OF001= chmod 0600 \"$tmp\"; mv -f \"$tmp\" \"$EVIDENCE_ROOT/$gen.json\"\nENGINE_L0105_P001_OF003= tmp=\"${HEALTH}.new.$$\"; jq -n --arg gen \"$gen\" --arg state \"$state\" --arg evidence \"$EVIDENCE_ROOT/$gen.json\" --arg sha \"$archive_sha\" --arg now \"$now\" '{sche\nENGINE_L0105_P002_OF003=ma:\"homelab-control-backup-health-v1\",status:\"OK\",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshn\nENGINE_L0105_P003_OF003=ess:\"FRESH\",full_site_dr_proven:false,updated_at_utc:$now}' >\"$tmp\"; chmod 0644 \"$tmp\"; mv -f \"$tmp\" \"$HEALTH\"\nENGINE_L0106_P001_OF001= trap - EXIT; cleanup_verify\nENGINE_L0107_P001_OF001= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\\n' \"$gen\" \"$EVIDENCE_ROOT/$gen.json\"\nENGINE_L0108_P001_OF001=}\nENGINE_L0109_P001_OF001=run_backup(){\nENGINE_L0110_P001_OF001= local force=\"${1:-0}\" decision line state gen payload outdir now\nENGINE_L0111_P001_OF002= mkdir -p \"$STATE_DIR\" \"$EVIDENCE_ROOT\" \"$STAGE_ROOT\" \"$VERIFY_ROOT\" /run/lock; chmod 0700 \"$STATE_ROOT\" \"$STATE_DIR\" \"$EVIDENCE_ROOT\" \"$STAGE_ROOT\" \"$VERIFY_R\nENGINE_L0111_P002_OF002=OOT\"\nENGINE_L0112_P001_OF001= exec 9>\"$LOCK\"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\\n'; return 0; }\nENGINE_L0113_P001_OF001= local incompletes\nENGINE_L0114_P001_OF002= incompletes=\"$(find \"$STATE_DIR\" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(\nENGINE_L0114_P002_OF002=.status!=\"SEALED\")|.generation' 2>/dev/null || true)\"\nENGINE_L0115_P001_OF001= [[ \"$(grep -c . <<<\"$incompletes\" || true)\" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68\nENGINE_L0116_P001_OF001= line=\"$(scan)\"; printf '%s\\n' \"$line\"; decision=\"$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$line\")\"\nENGINE_L0117_P001_OF001= gen=\"$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$line\")\"\nENGINE_L0118_P001_OF001= if [[ \"$decision\" == SKIP_FRESH && \"$force\" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\\n' \"$gen\"; return 0; fi\nENGINE_L0119_P001_OF001= if [[ \"$decision\" == BACKUP_REVERIFY ]]; then\nENGINE_L0120_P001_OF001= if [[ \"$force\" != 1 ]]; then verify_generation \"$gen\"; return 0; fi\nENGINE_L0121_P001_OF001= # Forced refresh deliberately creates a new generation instead of requiring\nENGINE_L0122_P001_OF001= # a legacy generation to satisfy the current bundle schema.\nENGINE_L0123_P001_OF001= decision=BACKUP_DUE\nENGINE_L0124_P001_OF001= fi\nENGINE_L0125_P001_OF001= if [[ \"$decision\" == BACKUP_RESUME ]]; then\nENGINE_L0126_P001_OF001= state=\"$STATE_DIR/$gen.json\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; [[ -s \"$payload\" && ! -L \"$payload\" ]] || die INCOMPLETE_PAYLOAD_MISSING 68\nENGINE_L0127_P001_OF001= \"$BUILDER\" verify --archive \"$payload\" --expected-generation \"$gen\" >/dev/null || die 68\nENGINE_L0128_P001_OF001= mark_cloud_write\nENGINE_L0129_P001_OF002= \"$TRANSPORT\" backup-delete \"$LOGICAL_ID\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"; verify_generation \"$gen\"; printf 'CONTROL_BACKUP_RESULT=SE\nENGINE_L0129_P002_OF002=ALED_RESUMED GENERATION=%s\\n' \"$gen\"; return 0\nENGINE_L0130_P001_OF001= fi\nENGINE_L0131_P001_OF001= case \"$decision\" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die \"DECISION_INVALID_$decision\" 65;; esac\nENGINE_L0132_P001_OF001= exec 8>\"$HEAVY_LOCK\"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\\n'; return 0; }\nENGINE_L0133_P001_OF001= now=\"$(date -u +%Y%m%dT%H%M%SZ)\"; gen=\"control-bootstrap-v2-pve01-$now\"; outdir=\"$STAGE_ROOT/$gen\"\nENGINE_L0134_P001_OF001= \"$BUILDER\" build --generation \"$gen\" --output-dir \"$outdir\" >/dev/null\nENGINE_L0135_P001_OF001= payload=\"$outdir/$gen.tar.gz\"; [[ -s \"$payload\" && ! -L \"$payload\" ]] || die BUILT_PAYLOAD_MISSING 68\nENGINE_L0136_P001_OF001= \"$BUILDER\" verify --archive \"$payload\" --expected-generation \"$gen\" >/dev/null || die 68\nENGINE_L0137_P001_OF001= exec 8>&-\nENGINE_L0138_P001_OF001= mark_cloud_write\nENGINE_L0139_P001_OF001= \"$TRANSPORT\" backup-delete \"$LOGICAL_ID\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nENGINE_L0140_P001_OF001= rm -rf --one-file-system \"$outdir\" 2>/dev/null || true\nENGINE_L0141_P001_OF001= verify_generation \"$gen\"\nENGINE_L0142_P001_OF001= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\\n' \"$gen\"\nENGINE_L0143_P001_OF001=}\nENGINE_L0144_P001_OF001=case \"${1:-}\" in\nENGINE_L0145_P001_OF001= scan) [[ \"$#\" == 1 ]] || die USAGE 64; scan;;\nENGINE_L0146_P001_OF001= run) [[ \"$#\" == 1 ]] || die USAGE 64; run_backup 0;;\nENGINE_L0147_P001_OF001= force) [[ \"$#\" == 1 ]] || die USAGE 64; run_backup 1;;\nENGINE_L0148_P001_OF001= verify) [[ \"$#\" == 2 ]] || die USAGE 64; verify_generation \"$2\";;\nENGINE_L0149_P001_OF001= *) die USAGE_scan_run_force_verify 64;;\nENGINE_L0150_P001_OF001=esac\nENGINE_SOURCE_END\nUNIT_PATH=/etc/systemd/system/homelab-control-backup.service\nUNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250\nUNIT_LINE_COUNT=16\nUNIT_REDACTED_LINE_COUNT=3\nUNIT_SOURCE_BEGIN\nUNIT_L0001_P001_OF001=[Unit]\nUNIT_L0002_P001_OF001=Description=Homelab recurring CONTROL cloud quorum backup\nUNIT_L0003_P001_OF001=After=network-online.target\nUNIT_L0004_P001_OF001=Wants=network-online.target\nUNIT_L0005_P001_OF001=ConditionPathExists=\nUNIT_L0006_P001_OF001=ConditionPathExists=\nUNIT_L0007_P001_OF001=\nUNIT_L0008_P001_OF001=[Service]\nUNIT_L0009_P001_OF001=Type=oneshot\nUNIT_L0010_P001_OF001=ExecStart= run\nUNIT_L0011_P001_OF001=User=root\nUNIT_L0012_P001_OF001=Group=root\nUNIT_L0013_P001_OF001=UMask=0077\nUNIT_L0014_P001_OF001=Nice=10\nUNIT_L0015_P001_OF001=IOSchedulingClass=best-effort\nUNIT_L0016_P001_OF001=IOSchedulingPriority=7\nUNIT_SOURCE_END\nSOURCE_TRUNCATED_LINES=0\nSOURCE_FUNCTION_PARSER_USED=NO\nBACKUP_ENGINE_EXECUTIONS=0\nRCLONE_COMMANDS_EXECUTED=0\nCLOUD_CALLS=0\nFILES_WRITTEN_BY_PAYLOAD=0\nSYSTEMD_MUTATIONS=0\nSOURCE259_READ_COMPLETE=YES\nSOURCE259_END\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1.txt b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1.txt new file mode 100644 index 00000000..ff8bbddc --- /dev/null +++ b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1.txt @@ -0,0 +1,237 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=infra-dr-backupv2-control-backup-source +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=09220124de7fa21ffc8fbc498052fa8ab92cee5d72b61fe34976f30307a58400 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=3762076dc1704d241330f8e0aa300dc7cbca2c80ec5ccc24137e59d67f7bd34b +SANITIZED_OUTPUT_SHA256=497e4996869d853f0466d28fe81745576dffca2ccbe7ed64998990ed695d898b +OUTPUT_BEGIN +SOURCE259_BEGIN +COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1 +MODE=READ_ONLY_HASH_BOUND_SOURCE_EXPORT +SOURCE_EXECUTION=FORBIDDEN +ERROR_REGISTER_CHECK=READ +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +REFERENCE_CHECK=READ +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +AUTHORITY_FRESH_READ=OK +ENGINE_PATH=/usr/local/sbin/homelab-control-backup +ENGINE_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57 +ENGINE_LINE_COUNT=150 +ENGINE_REDACTED_LINE_COUNT=13 +ENGINE_SOURCE_BEGIN +ENGINE_L0001_P001_OF001=#!/usr/bin/env bash +ENGINE_L0002_P001_OF001=set -Eeuo pipefail +ENGINE_L0003_P001_OF001=umask 077 +ENGINE_L0004_P001_OF001=LOGICAL_ID=control-bootstrap-v2 +ENGINE_L0005_P001_OF001=DUE_SECONDS=43200 +ENGINE_L0006_P001_OF001=SLA_SECONDS=86400 +ENGINE_L0007_P001_OF001=STATE_ROOT= +ENGINE_L0008_P001_OF001=STATE_DIR=$STATE_ROOT/$LOGICAL_ID +ENGINE_L0009_P001_OF001=EVIDENCE_ROOT= +ENGINE_L0010_P001_OF001=HEALTH=.json +ENGINE_L0011_P001_OF001=STAGE_ROOT=/mnt/staging/homelab-control-auto +ENGINE_L0012_P001_OF001=VERIFY_ROOT=/mnt/staging/homelab-control-verify +ENGINE_L0013_P001_OF001=LOCK=/run/lock/homelab-control-backup.lock +ENGINE_L0014_P001_OF001=HEAVY_LOCK=/run/homelab-backup-heavy.lock +ENGINE_L0015_P001_OF001=RCLONE_CONFIG_SOURCE=${RCLONE_CONFIG:-/root/.config/rclone/[SENSITIVE_FILE]} +ENGINE_L0016_P001_OF001=TRANSPORT= +ENGINE_L0017_P001_OF001=TRANSPORT_SHA= +ENGINE_L0018_P001_OF001=BUILDER= +ENGINE_L0019_P001_OF001=BUILDER_SHA= +ENGINE_L0020_P001_OF001=REMOTE_A=pve01-mail-01-crypt:homelab-backups +ENGINE_L0021_P001_OF001=REMOTE_B=pve01-mail-02-crypt:homelab-backups +ENGINE_L0022_P001_OF001=CLOUD_WRITE_MARKER=${HOMELAB_CONTROL_CLOUD_WRITE_MARKER:-} +ENGINE_L0023_P001_OF001= +ENGINE_L0024_P001_OF001=die(){ printf 'CONTROL_BACKUP_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; } +ENGINE_L0025_P001_OF002=for c in jq date find sort tail cut sha256sum stat flock rclone mktemp cmp awk mkdir rm install xargs grep cat mv chmod; do command -v "$c" >/dev/null || die "T +ENGINE_L0025_P002_OF002=OOL_MISSING_$c" 69; done +ENGINE_L0026_P001_OF001=[[ -x "$TRANSPORT" && ! -L "$TRANSPORT" ]] || die TRANSPORT_UNSAFE 69 +ENGINE_L0027_P001_OF001=[[ "$(sha256sum "$TRANSPORT"|awk '{print $1}')" == "$TRANSPORT_SHA" ]] || die TRANSPORT_SHA_DRIFT 65 +ENGINE_L0028_P001_OF001=[[ -x "$BUILDER" && ! -L "$BUILDER" ]] || die BUILDER_UNSAFE 69 +ENGINE_L0029_P001_OF001=[[ "$(sha256sum "$BUILDER"|awk '{print $1}')" == "$BUILDER_SHA" ]] || die BUILDER_SHA_DRIFT 65 +ENGINE_L0030_P001_OF001= +ENGINE_L0031_P001_OF001=latest_state(){ +ENGINE_L0032_P001_OF001= [[ -d "$STATE_DIR" && ! -L "$STATE_DIR" ]] || return 0 +ENGINE_L0033_P001_OF001= find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print 2>/dev/null | LC_ALL=C sort | tail -1 +ENGINE_L0034_P001_OF001=} +ENGINE_L0035_P001_OF001=state_decision(){ +ENGINE_L0036_P001_OF001= local state="$1" now="$2" status gen upd epoch age ev sha +ENGINE_L0037_P001_OF001= if [[ -z "$state" ]]; then printf 'DECISION=BACKUP_UNVERIFIED REASON=NO_STATE AGE_SECONDS=-1\n'; return 0; fi +ENGINE_L0038_P001_OF001= [[ -s "$state" && ! -L "$state" ]] || { printf 'DECISION=BACKUP_UNVERIFIED REASON=STATE_UNSAFE AGE_SECONDS=-1\n'; return 0; } +ENGINE_L0039_P001_OF001= status="$(jq -r '.status // "INVALID"' "$state")"; gen="$(jq -r '.generation // "UNKNOWN"' "$state")" +ENGINE_L0040_P001_OF001= case "$status" in +ENGINE_L0041_P001_OF001= PREPARING|A_VERIFIED|PREPARED|DELETE_AUTHORIZED) +ENGINE_L0042_P001_OF001= printf 'DECISION=BACKUP_RESUME GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\n' "$gen" "$state"; return 0;; +ENGINE_L0043_P001_OF001= SEALED) ;; +ENGINE_L0044_P001_OF001= *) printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=STATE_STATUS_INVALID AGE_SECONDS=-1\n' "$gen"; return 0;; +ENGINE_L0045_P001_OF001= esac +ENGINE_L0046_P001_OF001= sha="$(jq -r '.archive_sha256 // ""' "$state")"; upd="$(jq -r '.sealed_at_utc // .updated_at_utc // ""' "$state")" +ENGINE_L0047_P001_OF001= epoch="$(date -u -d "$upd" +%s 2>/dev/null || true)" +ENGINE_L0048_P001_OF002= [[ "$epoch" =~ ^[0-9]+$ && "$epoch" -le "$now" ]] || { printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=TIMESTAMP_INVALID AGE_SECONDS=-1\n' "$gen"; ret +ENGINE_L0048_P002_OF002=urn 0; } +ENGINE_L0049_P001_OF001= ev="$EVIDENCE_ROOT/$gen.json" +ENGINE_L0050_P001_OF001= +ENGINE_L0051_P001_OF001= printf 'DECISION=BACKUP_REVERIFY GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\n' "$gen" "$state"; return 0 +ENGINE_L0052_P001_OF001= fi +ENGINE_L0053_P001_OF001= age=$((now-epoch)) +ENGINE_L0054_P001_OF001= if (( age < DUE_SECONDS )); then d=SKIP_FRESH; elif (( age < SLA_SECONDS )); then d=BACKUP_DUE; else d=BACKUP_STALE; fi +ENGINE_L0055_P001_OF001= printf 'DECISION=%s GENERATION=%s STATE_FILE=%s AGE_SECONDS=%s DUE_SECONDS=%s SLA_SECONDS=%s\n' "$d" "$gen" "$state" "$age" "$DUE_SECONDS" "$SLA_SECONDS" +ENGINE_L0056_P001_OF001=} +ENGINE_L0057_P001_OF001=scan(){ local now state; now="$(date -u +%s)"; state="$(latest_state)"; state_decision "$state" "$now"; } +ENGINE_L0058_P001_OF001=mark_cloud_write(){ +ENGINE_L0059_P001_OF001= [[ -z "$CLOUD_WRITE_MARKER" ]] && return 0 +ENGINE_L0060_P001_OF001= [[ "$CLOUD_WRITE_MARKER" == /* ]] || die CLOUD_WRITE_MARKER_NOT_ABSOLUTE 65 +ENGINE_L0061_P001_OF001= [[ ! -L "$CLOUD_WRITE_MARKER" ]] || die CLOUD_WRITE_MARKER_SYMLINK 65 +ENGINE_L0062_P001_OF001= printf '%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$CLOUD_WRITE_MARKER" +ENGINE_L0063_P001_OF001= chmod 0600 "$CLOUD_WRITE_MARKER" +ENGINE_L0064_P001_OF001=} +ENGINE_L0065_P001_OF001= +ENGINE_L0066_P001_OF001=prepare_rclone(){ +ENGINE_L0067_P001_OF001= local dst="$1"; [[ -f "$RCLONE_CONFIG_SOURCE" && ! -L "$RCLONE_CONFIG_SOURCE" ]] || die RCLONE_CONFIG_UNSAFE 66 +ENGINE_L0068_P001_OF001= install -m 0600 "$RCLONE_CONFIG_SOURCE" "$dst" +ENGINE_L0069_P001_OF001=} +ENGINE_L0070_P001_OF001=reconstruct(){ +ENGINE_L0071_P001_OF001= local base="$1" out="$2" cfg="$3" state="$4" label="$5" manifest part idx name size hash got_size got_hash expected_manifest expected_archive +ENGINE_L0072_P001_OF001= expected_manifest="$(jq -er '.parts_manifest_sha256' "$state")"; expected_archive="$(jq -er '.archive_sha256' "$state")" +ENGINE_L0073_P001_OF001= manifest="${out}.PARTS.tsv"; : >"$out" +ENGINE_L0074_P001_OF001= rclone --config "$cfg" cat "${base%/}/PARTS.tsv" >"$manifest" || die "REMOTE_${label}_MANIFEST_READ_FAILED" 74 +ENGINE_L0075_P001_OF001= [[ "$(sha256sum "$manifest"|awk '{print $1}')" == "$expected_manifest" ]] || die "REMOTE_${label}_MANIFEST_SHA_MISMATCH" 68 +ENGINE_L0076_P001_OF001= idx=0 +ENGINE_L0077_P001_OF001= while IFS=$'\t' read -r row name size hash; do +ENGINE_L0078_P001_OF002= [[ "$row" == "$(printf '%08d' "$idx")" && "$name" == part-????????.bin && "$size" =~ ^[1-9][0-9]*$ && "$hash" =~ ^[0-9a-f]{64}$ ]] || die "REMOTE_${label}_M +ENGINE_L0078_P002_OF002=ANIFEST_ROW_INVALID" 68 +ENGINE_L0079_P001_OF001= part="${out}.part"; rm -f "$part"; rclone --config "$cfg" cat "${base%/}/payload.parts/$name" >"$part" || die "REMOTE_${label}_PART_READ_FAILED" 74 +ENGINE_L0080_P001_OF001= got_size="$(stat -c %s "$part")"; got_hash="$(sha256sum "$part"|awk '{print $1}')" +ENGINE_L0081_P001_OF001= [[ "$got_size" == "$size" && "$got_hash" == "$hash" ]] || die "REMOTE_${label}_PART_VERIFY_FAILED" 68 +ENGINE_L0082_P001_OF001= cat "$part" >>"$out"; rm -f "$part"; idx=$((idx+1)) +ENGINE_L0083_P001_OF001= done <"$manifest" +ENGINE_L0084_P001_OF001= [[ "$(sha256sum "$out"|awk '{print $1}')" == "$expected_archive" ]] || die "REMOTE_${label}_ARCHIVE_SHA_MISMATCH" 68 +ENGINE_L0085_P001_OF001= rm -f "$manifest" +ENGINE_L0086_P001_OF001=} +ENGINE_L0087_P001_OF001=verify_generation(){ +ENGINE_L0088_P001_OF001= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now +ENGINE_L0089_P001_OF001= gen="$1" +ENGINE_L0090_P001_OF001= state="$STATE_DIR/$gen.json" +ENGINE_L0091_P001_OF001= [[ -s "$state" && ! -L "$state" ]] || die VERIFY_STATE_UNSAFE 68 +ENGINE_L0092_P001_OF001= "$TRANSPORT" verify-sealed "$LOGICAL_ID" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null +ENGINE_L0093_P001_OF001= mkdir -p "$VERIFY_ROOT"; chmod 0700 "$VERIFY_ROOT" +ENGINE_L0094_P001_OF001= work="$(mktemp -d "$VERIFY_ROOT/.verify.XXXXXX")"; cfg="$work/[SENSITIVE_FILE]"; prepare_rclone "$cfg" +ENGINE_L0095_P001_OF001= cleanup_verify(){ rm -rf --one-file-system "$work"; }; trap cleanup_verify EXIT +ENGINE_L0096_P001_OF001= base_a="$(jq -er '.remote_a_base' "$state")"; base_b="$(jq -er '.remote_b_base' "$state")"; a="$work/A.tar.gz"; b="$work/B.tar.gz" +ENGINE_L0097_P001_OF001= reconstruct "$base_a" "$a" "$cfg" "$state" A; reconstruct "$base_b" "$b" "$cfg" "$state" B +ENGINE_L0098_P001_OF001= cmp -s "$a" "$b" || die CLOUD_A_B_PAYLOAD_MISMATCH 68 +ENGINE_L0099_P001_OF001= verify_json="$("$BUILDER" verify --archive "$a" --expected-generation "$gen")" || die 68 +ENGINE_L0100_P001_OF001= +ENGINE_L0101_P001_OF001= archive_sha="$(jq -er '.archive_sha256' "$state")"; mkdir -p "$EVIDENCE_ROOT" /var/lib/homelab-health; chmod 0700 "$EVIDENCE_ROOT" +ENGINE_L0102_P001_OF001= tmp="$EVIDENCE_ROOT/.$gen.new.$$"; now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +ENGINE_L0103_P001_OF001= +ENGINE_L0104_P001_OF001= chmod 0600 "$tmp"; mv -f "$tmp" "$EVIDENCE_ROOT/$gen.json" +ENGINE_L0105_P001_OF003= tmp="${HEALTH}.new.$$"; jq -n --arg gen "$gen" --arg state "$state" --arg evidence "$EVIDENCE_ROOT/$gen.json" --arg sha "$archive_sha" --arg now "$now" '{sche +ENGINE_L0105_P002_OF003=ma:"homelab-control-backup-health-v1",status:"OK",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshn +ENGINE_L0105_P003_OF003=ess:"FRESH",full_site_dr_proven:false,updated_at_utc:$now}' >"$tmp"; chmod 0644 "$tmp"; mv -f "$tmp" "$HEALTH" +ENGINE_L0106_P001_OF001= trap - EXIT; cleanup_verify +ENGINE_L0107_P001_OF001= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\n' "$gen" "$EVIDENCE_ROOT/$gen.json" +ENGINE_L0108_P001_OF001=} +ENGINE_L0109_P001_OF001=run_backup(){ +ENGINE_L0110_P001_OF001= local force="${1:-0}" decision line state gen payload outdir now +ENGINE_L0111_P001_OF002= mkdir -p "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" /run/lock; chmod 0700 "$STATE_ROOT" "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_R +ENGINE_L0111_P002_OF002=OOT" +ENGINE_L0112_P001_OF001= exec 9>"$LOCK"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\n'; return 0; } +ENGINE_L0113_P001_OF001= local incompletes +ENGINE_L0114_P001_OF002= incompletes="$(find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select( +ENGINE_L0114_P002_OF002=.status!="SEALED")|.generation' 2>/dev/null || true)" +ENGINE_L0115_P001_OF001= [[ "$(grep -c . <<<"$incompletes" || true)" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68 +ENGINE_L0116_P001_OF001= line="$(scan)"; printf '%s\n' "$line"; decision="$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,"",$i);print $i}}' <<<"$line")" +ENGINE_L0117_P001_OF001= gen="$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$line")" +ENGINE_L0118_P001_OF001= if [[ "$decision" == SKIP_FRESH && "$force" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\n' "$gen"; return 0; fi +ENGINE_L0119_P001_OF001= if [[ "$decision" == BACKUP_REVERIFY ]]; then +ENGINE_L0120_P001_OF001= if [[ "$force" != 1 ]]; then verify_generation "$gen"; return 0; fi +ENGINE_L0121_P001_OF001= # Forced refresh deliberately creates a new generation instead of requiring +ENGINE_L0122_P001_OF001= # a legacy generation to satisfy the current bundle schema. +ENGINE_L0123_P001_OF001= decision=BACKUP_DUE +ENGINE_L0124_P001_OF001= fi +ENGINE_L0125_P001_OF001= if [[ "$decision" == BACKUP_RESUME ]]; then +ENGINE_L0126_P001_OF001= state="$STATE_DIR/$gen.json"; payload="$(jq -er '.payload_path' "$state")"; [[ -s "$payload" && ! -L "$payload" ]] || die INCOMPLETE_PAYLOAD_MISSING 68 +ENGINE_L0127_P001_OF001= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die 68 +ENGINE_L0128_P001_OF001= mark_cloud_write +ENGINE_L0129_P001_OF002= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"; verify_generation "$gen"; printf 'CONTROL_BACKUP_RESULT=SE +ENGINE_L0129_P002_OF002=ALED_RESUMED GENERATION=%s\n' "$gen"; return 0 +ENGINE_L0130_P001_OF001= fi +ENGINE_L0131_P001_OF001= case "$decision" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die "DECISION_INVALID_$decision" 65;; esac +ENGINE_L0132_P001_OF001= exec 8>"$HEAVY_LOCK"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\n'; return 0; } +ENGINE_L0133_P001_OF001= now="$(date -u +%Y%m%dT%H%M%SZ)"; gen="control-bootstrap-v2-pve01-$now"; outdir="$STAGE_ROOT/$gen" +ENGINE_L0134_P001_OF001= "$BUILDER" build --generation "$gen" --output-dir "$outdir" >/dev/null +ENGINE_L0135_P001_OF001= payload="$outdir/$gen.tar.gz"; [[ -s "$payload" && ! -L "$payload" ]] || die BUILT_PAYLOAD_MISSING 68 +ENGINE_L0136_P001_OF001= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die 68 +ENGINE_L0137_P001_OF001= exec 8>&- +ENGINE_L0138_P001_OF001= mark_cloud_write +ENGINE_L0139_P001_OF001= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +ENGINE_L0140_P001_OF001= rm -rf --one-file-system "$outdir" 2>/dev/null || true +ENGINE_L0141_P001_OF001= verify_generation "$gen" +ENGINE_L0142_P001_OF001= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\n' "$gen" +ENGINE_L0143_P001_OF001=} +ENGINE_L0144_P001_OF001=case "${1:-}" in +ENGINE_L0145_P001_OF001= scan) [[ "$#" == 1 ]] || die USAGE 64; scan;; +ENGINE_L0146_P001_OF001= run) [[ "$#" == 1 ]] || die USAGE 64; run_backup 0;; +ENGINE_L0147_P001_OF001= force) [[ "$#" == 1 ]] || die USAGE 64; run_backup 1;; +ENGINE_L0148_P001_OF001= verify) [[ "$#" == 2 ]] || die USAGE 64; verify_generation "$2";; +ENGINE_L0149_P001_OF001= *) die USAGE_scan_run_force_verify 64;; +ENGINE_L0150_P001_OF001=esac +ENGINE_SOURCE_END +UNIT_PATH=/etc/systemd/system/homelab-control-backup.service +UNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250 +UNIT_LINE_COUNT=16 +UNIT_REDACTED_LINE_COUNT=3 +UNIT_SOURCE_BEGIN +UNIT_L0001_P001_OF001=[Unit] +UNIT_L0002_P001_OF001=Description=Homelab recurring CONTROL cloud quorum backup +UNIT_L0003_P001_OF001=After=network-online.target +UNIT_L0004_P001_OF001=Wants=network-online.target +UNIT_L0005_P001_OF001=ConditionPathExists= +UNIT_L0006_P001_OF001=ConditionPathExists= +UNIT_L0007_P001_OF001= +UNIT_L0008_P001_OF001=[Service] +UNIT_L0009_P001_OF001=Type=oneshot +UNIT_L0010_P001_OF001=ExecStart= run +UNIT_L0011_P001_OF001=User=root +UNIT_L0012_P001_OF001=Group=root +UNIT_L0013_P001_OF001=UMask=0077 +UNIT_L0014_P001_OF001=Nice=10 +UNIT_L0015_P001_OF001=IOSchedulingClass=best-effort +UNIT_L0016_P001_OF001=IOSchedulingPriority=7 +UNIT_SOURCE_END +SOURCE_TRUNCATED_LINES=0 +SOURCE_FUNCTION_PARSER_USED=NO +BACKUP_ENGINE_EXECUTIONS=0 +RCLONE_COMMANDS_EXECUTED=0 +CLOUD_CALLS=0 +FILES_WRITTEN_BY_PAYLOAD=0 +SYSTEMD_MUTATIONS=0 +SOURCE259_READ_COMPLETE=YES +SOURCE259_END +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 47140ef6..dad1d4c0 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1", + "command_id": "SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "infra-dr-backupv2-control-backup-logic-focus-diagnostic", - "started_at_utc": "2026-09-11T23:44:54Z", - "finished_at_utc": "2026-09-11T23:44:54Z", + "component": "infra-dr-backupv2-control-backup-source", + "started_at_utc": "2026-09-12T00:02:19Z", + "finished_at_utc": "2026-09-12T00:02:19Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "f3ab0a154eda3d947c6b078120e2088b7d0721f992b74d9da8e78af6568708ca", + "command_sha256": "09220124de7fa21ffc8fbc498052fa8ab92cee5d72b61fe34976f30307a58400", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974", - "sanitized_output_sha256": "72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974", + "raw_evidence_sha256": "3762076dc1704d241330f8e0aa300dc7cbca2c80ec5ccc24137e59d67f7bd34b", + "sanitized_output_sha256": "497e4996869d853f0466d28fe81745576dffca2ccbe7ed64998990ed695d898b", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "DIAG258_BEGIN\nTARGET_COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1\nTARGET_ATTEMPT_DIR=/root/homelab-run-SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1\nTARGET_ATTEMPT_DIR_EXISTS=NO\nMATCHING_257_ATTEMPT_DIR_COUNT=0\nEVIDENCE_LAYER=NO_257_SERVER_ATTEMPT_DIRECTORY\nINTERPRETATION=257_DID_NOT_REACH_RUNNER_RESERVATION\nFILES_WRITTEN_BY_DIAG258=0\nRCLONE_COMMANDS_BY_DIAG258=0\nCLOUD_MUTATIONS_BY_DIAG258=0\nDIAG258_END\n" + "output": "SOURCE259_BEGIN\nCOMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1\nMODE=READ_ONLY_HASH_BOUND_SOURCE_EXPORT\nSOURCE_EXECUTION=FORBIDDEN\nERROR_REGISTER_CHECK=READ\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nREFERENCE_CHECK=READ\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nAUTHORITY_FRESH_READ=OK\nENGINE_PATH=/usr/local/sbin/homelab-control-backup\nENGINE_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57\nENGINE_LINE_COUNT=150\nENGINE_REDACTED_LINE_COUNT=13\nENGINE_SOURCE_BEGIN\nENGINE_L0001_P001_OF001=#!/usr/bin/env bash\nENGINE_L0002_P001_OF001=set -Eeuo pipefail\nENGINE_L0003_P001_OF001=umask 077\nENGINE_L0004_P001_OF001=LOGICAL_ID=control-bootstrap-v2\nENGINE_L0005_P001_OF001=DUE_SECONDS=43200\nENGINE_L0006_P001_OF001=SLA_SECONDS=86400\nENGINE_L0007_P001_OF001=STATE_ROOT=\nENGINE_L0008_P001_OF001=STATE_DIR=$STATE_ROOT/$LOGICAL_ID\nENGINE_L0009_P001_OF001=EVIDENCE_ROOT=\nENGINE_L0010_P001_OF001=HEALTH=.json\nENGINE_L0011_P001_OF001=STAGE_ROOT=/mnt/staging/homelab-control-auto\nENGINE_L0012_P001_OF001=VERIFY_ROOT=/mnt/staging/homelab-control-verify\nENGINE_L0013_P001_OF001=LOCK=/run/lock/homelab-control-backup.lock\nENGINE_L0014_P001_OF001=HEAVY_LOCK=/run/homelab-backup-heavy.lock\nENGINE_L0015_P001_OF001=RCLONE_CONFIG_SOURCE=${RCLONE_CONFIG:-/root/.config/rclone/[SENSITIVE_FILE]}\nENGINE_L0016_P001_OF001=TRANSPORT=\nENGINE_L0017_P001_OF001=TRANSPORT_SHA=\nENGINE_L0018_P001_OF001=BUILDER=\nENGINE_L0019_P001_OF001=BUILDER_SHA=\nENGINE_L0020_P001_OF001=REMOTE_A=pve01-mail-01-crypt:homelab-backups\nENGINE_L0021_P001_OF001=REMOTE_B=pve01-mail-02-crypt:homelab-backups\nENGINE_L0022_P001_OF001=CLOUD_WRITE_MARKER=${HOMELAB_CONTROL_CLOUD_WRITE_MARKER:-}\nENGINE_L0023_P001_OF001=\nENGINE_L0024_P001_OF001=die(){ printf 'CONTROL_BACKUP_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }\nENGINE_L0025_P001_OF002=for c in jq date find sort tail cut sha256sum stat flock rclone mktemp cmp awk mkdir rm install xargs grep cat mv chmod; do command -v \"$c\" >/dev/null || die \"T\nENGINE_L0025_P002_OF002=OOL_MISSING_$c\" 69; done\nENGINE_L0026_P001_OF001=[[ -x \"$TRANSPORT\" && ! -L \"$TRANSPORT\" ]] || die TRANSPORT_UNSAFE 69\nENGINE_L0027_P001_OF001=[[ \"$(sha256sum \"$TRANSPORT\"|awk '{print $1}')\" == \"$TRANSPORT_SHA\" ]] || die TRANSPORT_SHA_DRIFT 65\nENGINE_L0028_P001_OF001=[[ -x \"$BUILDER\" && ! -L \"$BUILDER\" ]] || die BUILDER_UNSAFE 69\nENGINE_L0029_P001_OF001=[[ \"$(sha256sum \"$BUILDER\"|awk '{print $1}')\" == \"$BUILDER_SHA\" ]] || die BUILDER_SHA_DRIFT 65\nENGINE_L0030_P001_OF001=\nENGINE_L0031_P001_OF001=latest_state(){\nENGINE_L0032_P001_OF001= [[ -d \"$STATE_DIR\" && ! -L \"$STATE_DIR\" ]] || return 0\nENGINE_L0033_P001_OF001= find \"$STATE_DIR\" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print 2>/dev/null | LC_ALL=C sort | tail -1\nENGINE_L0034_P001_OF001=}\nENGINE_L0035_P001_OF001=state_decision(){\nENGINE_L0036_P001_OF001= local state=\"$1\" now=\"$2\" status gen upd epoch age ev sha\nENGINE_L0037_P001_OF001= if [[ -z \"$state\" ]]; then printf 'DECISION=BACKUP_UNVERIFIED REASON=NO_STATE AGE_SECONDS=-1\\n'; return 0; fi\nENGINE_L0038_P001_OF001= [[ -s \"$state\" && ! -L \"$state\" ]] || { printf 'DECISION=BACKUP_UNVERIFIED REASON=STATE_UNSAFE AGE_SECONDS=-1\\n'; return 0; }\nENGINE_L0039_P001_OF001= status=\"$(jq -r '.status // \"INVALID\"' \"$state\")\"; gen=\"$(jq -r '.generation // \"UNKNOWN\"' \"$state\")\"\nENGINE_L0040_P001_OF001= case \"$status\" in\nENGINE_L0041_P001_OF001= PREPARING|A_VERIFIED|PREPARED|DELETE_AUTHORIZED)\nENGINE_L0042_P001_OF001= printf 'DECISION=BACKUP_RESUME GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\\n' \"$gen\" \"$state\"; return 0;;\nENGINE_L0043_P001_OF001= SEALED) ;;\nENGINE_L0044_P001_OF001= *) printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=STATE_STATUS_INVALID AGE_SECONDS=-1\\n' \"$gen\"; return 0;;\nENGINE_L0045_P001_OF001= esac\nENGINE_L0046_P001_OF001= sha=\"$(jq -r '.archive_sha256 // \"\"' \"$state\")\"; upd=\"$(jq -r '.sealed_at_utc // .updated_at_utc // \"\"' \"$state\")\"\nENGINE_L0047_P001_OF001= epoch=\"$(date -u -d \"$upd\" +%s 2>/dev/null || true)\"\nENGINE_L0048_P001_OF002= [[ \"$epoch\" =~ ^[0-9]+$ && \"$epoch\" -le \"$now\" ]] || { printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=TIMESTAMP_INVALID AGE_SECONDS=-1\\n' \"$gen\"; ret\nENGINE_L0048_P002_OF002=urn 0; }\nENGINE_L0049_P001_OF001= ev=\"$EVIDENCE_ROOT/$gen.json\"\nENGINE_L0050_P001_OF001=\nENGINE_L0051_P001_OF001= printf 'DECISION=BACKUP_REVERIFY GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\\n' \"$gen\" \"$state\"; return 0\nENGINE_L0052_P001_OF001= fi\nENGINE_L0053_P001_OF001= age=$((now-epoch))\nENGINE_L0054_P001_OF001= if (( age < DUE_SECONDS )); then d=SKIP_FRESH; elif (( age < SLA_SECONDS )); then d=BACKUP_DUE; else d=BACKUP_STALE; fi\nENGINE_L0055_P001_OF001= printf 'DECISION=%s GENERATION=%s STATE_FILE=%s AGE_SECONDS=%s DUE_SECONDS=%s SLA_SECONDS=%s\\n' \"$d\" \"$gen\" \"$state\" \"$age\" \"$DUE_SECONDS\" \"$SLA_SECONDS\"\nENGINE_L0056_P001_OF001=}\nENGINE_L0057_P001_OF001=scan(){ local now state; now=\"$(date -u +%s)\"; state=\"$(latest_state)\"; state_decision \"$state\" \"$now\"; }\nENGINE_L0058_P001_OF001=mark_cloud_write(){\nENGINE_L0059_P001_OF001= [[ -z \"$CLOUD_WRITE_MARKER\" ]] && return 0\nENGINE_L0060_P001_OF001= [[ \"$CLOUD_WRITE_MARKER\" == /* ]] || die CLOUD_WRITE_MARKER_NOT_ABSOLUTE 65\nENGINE_L0061_P001_OF001= [[ ! -L \"$CLOUD_WRITE_MARKER\" ]] || die CLOUD_WRITE_MARKER_SYMLINK 65\nENGINE_L0062_P001_OF001= printf '%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" >\"$CLOUD_WRITE_MARKER\"\nENGINE_L0063_P001_OF001= chmod 0600 \"$CLOUD_WRITE_MARKER\"\nENGINE_L0064_P001_OF001=}\nENGINE_L0065_P001_OF001=\nENGINE_L0066_P001_OF001=prepare_rclone(){\nENGINE_L0067_P001_OF001= local dst=\"$1\"; [[ -f \"$RCLONE_CONFIG_SOURCE\" && ! -L \"$RCLONE_CONFIG_SOURCE\" ]] || die RCLONE_CONFIG_UNSAFE 66\nENGINE_L0068_P001_OF001= install -m 0600 \"$RCLONE_CONFIG_SOURCE\" \"$dst\"\nENGINE_L0069_P001_OF001=}\nENGINE_L0070_P001_OF001=reconstruct(){\nENGINE_L0071_P001_OF001= local base=\"$1\" out=\"$2\" cfg=\"$3\" state=\"$4\" label=\"$5\" manifest part idx name size hash got_size got_hash expected_manifest expected_archive\nENGINE_L0072_P001_OF001= expected_manifest=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"; expected_archive=\"$(jq -er '.archive_sha256' \"$state\")\"\nENGINE_L0073_P001_OF001= manifest=\"${out}.PARTS.tsv\"; : >\"$out\"\nENGINE_L0074_P001_OF001= rclone --config \"$cfg\" cat \"${base%/}/PARTS.tsv\" >\"$manifest\" || die \"REMOTE_${label}_MANIFEST_READ_FAILED\" 74\nENGINE_L0075_P001_OF001= [[ \"$(sha256sum \"$manifest\"|awk '{print $1}')\" == \"$expected_manifest\" ]] || die \"REMOTE_${label}_MANIFEST_SHA_MISMATCH\" 68\nENGINE_L0076_P001_OF001= idx=0\nENGINE_L0077_P001_OF001= while IFS=$'\\t' read -r row name size hash; do\nENGINE_L0078_P001_OF002= [[ \"$row\" == \"$(printf '%08d' \"$idx\")\" && \"$name\" == part-????????.bin && \"$size\" =~ ^[1-9][0-9]*$ && \"$hash\" =~ ^[0-9a-f]{64}$ ]] || die \"REMOTE_${label}_M\nENGINE_L0078_P002_OF002=ANIFEST_ROW_INVALID\" 68\nENGINE_L0079_P001_OF001= part=\"${out}.part\"; rm -f \"$part\"; rclone --config \"$cfg\" cat \"${base%/}/payload.parts/$name\" >\"$part\" || die \"REMOTE_${label}_PART_READ_FAILED\" 74\nENGINE_L0080_P001_OF001= got_size=\"$(stat -c %s \"$part\")\"; got_hash=\"$(sha256sum \"$part\"|awk '{print $1}')\"\nENGINE_L0081_P001_OF001= [[ \"$got_size\" == \"$size\" && \"$got_hash\" == \"$hash\" ]] || die \"REMOTE_${label}_PART_VERIFY_FAILED\" 68\nENGINE_L0082_P001_OF001= cat \"$part\" >>\"$out\"; rm -f \"$part\"; idx=$((idx+1))\nENGINE_L0083_P001_OF001= done <\"$manifest\"\nENGINE_L0084_P001_OF001= [[ \"$(sha256sum \"$out\"|awk '{print $1}')\" == \"$expected_archive\" ]] || die \"REMOTE_${label}_ARCHIVE_SHA_MISMATCH\" 68\nENGINE_L0085_P001_OF001= rm -f \"$manifest\"\nENGINE_L0086_P001_OF001=}\nENGINE_L0087_P001_OF001=verify_generation(){\nENGINE_L0088_P001_OF001= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now\nENGINE_L0089_P001_OF001= gen=\"$1\"\nENGINE_L0090_P001_OF001= state=\"$STATE_DIR/$gen.json\"\nENGINE_L0091_P001_OF001= [[ -s \"$state\" && ! -L \"$state\" ]] || die VERIFY_STATE_UNSAFE 68\nENGINE_L0092_P001_OF001= \"$TRANSPORT\" verify-sealed \"$LOGICAL_ID\" \"$gen\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\" >/dev/null\nENGINE_L0093_P001_OF001= mkdir -p \"$VERIFY_ROOT\"; chmod 0700 \"$VERIFY_ROOT\"\nENGINE_L0094_P001_OF001= work=\"$(mktemp -d \"$VERIFY_ROOT/.verify.XXXXXX\")\"; cfg=\"$work/[SENSITIVE_FILE]\"; prepare_rclone \"$cfg\"\nENGINE_L0095_P001_OF001= cleanup_verify(){ rm -rf --one-file-system \"$work\"; }; trap cleanup_verify EXIT\nENGINE_L0096_P001_OF001= base_a=\"$(jq -er '.remote_a_base' \"$state\")\"; base_b=\"$(jq -er '.remote_b_base' \"$state\")\"; a=\"$work/A.tar.gz\"; b=\"$work/B.tar.gz\"\nENGINE_L0097_P001_OF001= reconstruct \"$base_a\" \"$a\" \"$cfg\" \"$state\" A; reconstruct \"$base_b\" \"$b\" \"$cfg\" \"$state\" B\nENGINE_L0098_P001_OF001= cmp -s \"$a\" \"$b\" || die CLOUD_A_B_PAYLOAD_MISMATCH 68\nENGINE_L0099_P001_OF001= verify_json=\"$(\"$BUILDER\" verify --archive \"$a\" --expected-generation \"$gen\")\" || die 68\nENGINE_L0100_P001_OF001=\nENGINE_L0101_P001_OF001= archive_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; mkdir -p \"$EVIDENCE_ROOT\" /var/lib/homelab-health; chmod 0700 \"$EVIDENCE_ROOT\"\nENGINE_L0102_P001_OF001= tmp=\"$EVIDENCE_ROOT/.$gen.new.$$\"; now=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nENGINE_L0103_P001_OF001=\nENGINE_L0104_P001_OF001= chmod 0600 \"$tmp\"; mv -f \"$tmp\" \"$EVIDENCE_ROOT/$gen.json\"\nENGINE_L0105_P001_OF003= tmp=\"${HEALTH}.new.$$\"; jq -n --arg gen \"$gen\" --arg state \"$state\" --arg evidence \"$EVIDENCE_ROOT/$gen.json\" --arg sha \"$archive_sha\" --arg now \"$now\" '{sche\nENGINE_L0105_P002_OF003=ma:\"homelab-control-backup-health-v1\",status:\"OK\",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshn\nENGINE_L0105_P003_OF003=ess:\"FRESH\",full_site_dr_proven:false,updated_at_utc:$now}' >\"$tmp\"; chmod 0644 \"$tmp\"; mv -f \"$tmp\" \"$HEALTH\"\nENGINE_L0106_P001_OF001= trap - EXIT; cleanup_verify\nENGINE_L0107_P001_OF001= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\\n' \"$gen\" \"$EVIDENCE_ROOT/$gen.json\"\nENGINE_L0108_P001_OF001=}\nENGINE_L0109_P001_OF001=run_backup(){\nENGINE_L0110_P001_OF001= local force=\"${1:-0}\" decision line state gen payload outdir now\nENGINE_L0111_P001_OF002= mkdir -p \"$STATE_DIR\" \"$EVIDENCE_ROOT\" \"$STAGE_ROOT\" \"$VERIFY_ROOT\" /run/lock; chmod 0700 \"$STATE_ROOT\" \"$STATE_DIR\" \"$EVIDENCE_ROOT\" \"$STAGE_ROOT\" \"$VERIFY_R\nENGINE_L0111_P002_OF002=OOT\"\nENGINE_L0112_P001_OF001= exec 9>\"$LOCK\"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\\n'; return 0; }\nENGINE_L0113_P001_OF001= local incompletes\nENGINE_L0114_P001_OF002= incompletes=\"$(find \"$STATE_DIR\" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(\nENGINE_L0114_P002_OF002=.status!=\"SEALED\")|.generation' 2>/dev/null || true)\"\nENGINE_L0115_P001_OF001= [[ \"$(grep -c . <<<\"$incompletes\" || true)\" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68\nENGINE_L0116_P001_OF001= line=\"$(scan)\"; printf '%s\\n' \"$line\"; decision=\"$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$line\")\"\nENGINE_L0117_P001_OF001= gen=\"$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$line\")\"\nENGINE_L0118_P001_OF001= if [[ \"$decision\" == SKIP_FRESH && \"$force\" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\\n' \"$gen\"; return 0; fi\nENGINE_L0119_P001_OF001= if [[ \"$decision\" == BACKUP_REVERIFY ]]; then\nENGINE_L0120_P001_OF001= if [[ \"$force\" != 1 ]]; then verify_generation \"$gen\"; return 0; fi\nENGINE_L0121_P001_OF001= # Forced refresh deliberately creates a new generation instead of requiring\nENGINE_L0122_P001_OF001= # a legacy generation to satisfy the current bundle schema.\nENGINE_L0123_P001_OF001= decision=BACKUP_DUE\nENGINE_L0124_P001_OF001= fi\nENGINE_L0125_P001_OF001= if [[ \"$decision\" == BACKUP_RESUME ]]; then\nENGINE_L0126_P001_OF001= state=\"$STATE_DIR/$gen.json\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; [[ -s \"$payload\" && ! -L \"$payload\" ]] || die INCOMPLETE_PAYLOAD_MISSING 68\nENGINE_L0127_P001_OF001= \"$BUILDER\" verify --archive \"$payload\" --expected-generation \"$gen\" >/dev/null || die 68\nENGINE_L0128_P001_OF001= mark_cloud_write\nENGINE_L0129_P001_OF002= \"$TRANSPORT\" backup-delete \"$LOGICAL_ID\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"; verify_generation \"$gen\"; printf 'CONTROL_BACKUP_RESULT=SE\nENGINE_L0129_P002_OF002=ALED_RESUMED GENERATION=%s\\n' \"$gen\"; return 0\nENGINE_L0130_P001_OF001= fi\nENGINE_L0131_P001_OF001= case \"$decision\" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die \"DECISION_INVALID_$decision\" 65;; esac\nENGINE_L0132_P001_OF001= exec 8>\"$HEAVY_LOCK\"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\\n'; return 0; }\nENGINE_L0133_P001_OF001= now=\"$(date -u +%Y%m%dT%H%M%SZ)\"; gen=\"control-bootstrap-v2-pve01-$now\"; outdir=\"$STAGE_ROOT/$gen\"\nENGINE_L0134_P001_OF001= \"$BUILDER\" build --generation \"$gen\" --output-dir \"$outdir\" >/dev/null\nENGINE_L0135_P001_OF001= payload=\"$outdir/$gen.tar.gz\"; [[ -s \"$payload\" && ! -L \"$payload\" ]] || die BUILT_PAYLOAD_MISSING 68\nENGINE_L0136_P001_OF001= \"$BUILDER\" verify --archive \"$payload\" --expected-generation \"$gen\" >/dev/null || die 68\nENGINE_L0137_P001_OF001= exec 8>&-\nENGINE_L0138_P001_OF001= mark_cloud_write\nENGINE_L0139_P001_OF001= \"$TRANSPORT\" backup-delete \"$LOGICAL_ID\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nENGINE_L0140_P001_OF001= rm -rf --one-file-system \"$outdir\" 2>/dev/null || true\nENGINE_L0141_P001_OF001= verify_generation \"$gen\"\nENGINE_L0142_P001_OF001= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\\n' \"$gen\"\nENGINE_L0143_P001_OF001=}\nENGINE_L0144_P001_OF001=case \"${1:-}\" in\nENGINE_L0145_P001_OF001= scan) [[ \"$#\" == 1 ]] || die USAGE 64; scan;;\nENGINE_L0146_P001_OF001= run) [[ \"$#\" == 1 ]] || die USAGE 64; run_backup 0;;\nENGINE_L0147_P001_OF001= force) [[ \"$#\" == 1 ]] || die USAGE 64; run_backup 1;;\nENGINE_L0148_P001_OF001= verify) [[ \"$#\" == 2 ]] || die USAGE 64; verify_generation \"$2\";;\nENGINE_L0149_P001_OF001= *) die USAGE_scan_run_force_verify 64;;\nENGINE_L0150_P001_OF001=esac\nENGINE_SOURCE_END\nUNIT_PATH=/etc/systemd/system/homelab-control-backup.service\nUNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250\nUNIT_LINE_COUNT=16\nUNIT_REDACTED_LINE_COUNT=3\nUNIT_SOURCE_BEGIN\nUNIT_L0001_P001_OF001=[Unit]\nUNIT_L0002_P001_OF001=Description=Homelab recurring CONTROL cloud quorum backup\nUNIT_L0003_P001_OF001=After=network-online.target\nUNIT_L0004_P001_OF001=Wants=network-online.target\nUNIT_L0005_P001_OF001=ConditionPathExists=\nUNIT_L0006_P001_OF001=ConditionPathExists=\nUNIT_L0007_P001_OF001=\nUNIT_L0008_P001_OF001=[Service]\nUNIT_L0009_P001_OF001=Type=oneshot\nUNIT_L0010_P001_OF001=ExecStart= run\nUNIT_L0011_P001_OF001=User=root\nUNIT_L0012_P001_OF001=Group=root\nUNIT_L0013_P001_OF001=UMask=0077\nUNIT_L0014_P001_OF001=Nice=10\nUNIT_L0015_P001_OF001=IOSchedulingClass=best-effort\nUNIT_L0016_P001_OF001=IOSchedulingPriority=7\nUNIT_SOURCE_END\nSOURCE_TRUNCATED_LINES=0\nSOURCE_FUNCTION_PARSER_USED=NO\nBACKUP_ENGINE_EXECUTIONS=0\nRCLONE_COMMANDS_EXECUTED=0\nCLOUD_CALLS=0\nFILES_WRITTEN_BY_PAYLOAD=0\nSYSTEMD_MUTATIONS=0\nSOURCE259_READ_COMPLETE=YES\nSOURCE259_END\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 271e21d5..ff8bbddc 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1 +COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=infra-dr-backupv2-control-backup-logic-focus-diagnostic +COMPONENT=infra-dr-backupv2-control-backup-source REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=f3ab0a154eda3d947c6b078120e2088b7d0721f992b74d9da8e78af6568708ca +COMMAND_SHA256=09220124de7fa21ffc8fbc498052fa8ab92cee5d72b61fe34976f30307a58400 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,20 +24,214 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974 -SANITIZED_OUTPUT_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974 +RAW_EVIDENCE_SHA256=3762076dc1704d241330f8e0aa300dc7cbca2c80ec5ccc24137e59d67f7bd34b +SANITIZED_OUTPUT_SHA256=497e4996869d853f0466d28fe81745576dffca2ccbe7ed64998990ed695d898b OUTPUT_BEGIN -DIAG258_BEGIN -TARGET_COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1 -TARGET_ATTEMPT_DIR=/root/homelab-run-SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1 -TARGET_ATTEMPT_DIR_EXISTS=NO -MATCHING_257_ATTEMPT_DIR_COUNT=0 -EVIDENCE_LAYER=NO_257_SERVER_ATTEMPT_DIRECTORY -INTERPRETATION=257_DID_NOT_REACH_RUNNER_RESERVATION -FILES_WRITTEN_BY_DIAG258=0 -RCLONE_COMMANDS_BY_DIAG258=0 -CLOUD_MUTATIONS_BY_DIAG258=0 -DIAG258_END +SOURCE259_BEGIN +COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1 +MODE=READ_ONLY_HASH_BOUND_SOURCE_EXPORT +SOURCE_EXECUTION=FORBIDDEN +ERROR_REGISTER_CHECK=READ +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +REFERENCE_CHECK=READ +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +AUTHORITY_FRESH_READ=OK +ENGINE_PATH=/usr/local/sbin/homelab-control-backup +ENGINE_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57 +ENGINE_LINE_COUNT=150 +ENGINE_REDACTED_LINE_COUNT=13 +ENGINE_SOURCE_BEGIN +ENGINE_L0001_P001_OF001=#!/usr/bin/env bash +ENGINE_L0002_P001_OF001=set -Eeuo pipefail +ENGINE_L0003_P001_OF001=umask 077 +ENGINE_L0004_P001_OF001=LOGICAL_ID=control-bootstrap-v2 +ENGINE_L0005_P001_OF001=DUE_SECONDS=43200 +ENGINE_L0006_P001_OF001=SLA_SECONDS=86400 +ENGINE_L0007_P001_OF001=STATE_ROOT= +ENGINE_L0008_P001_OF001=STATE_DIR=$STATE_ROOT/$LOGICAL_ID +ENGINE_L0009_P001_OF001=EVIDENCE_ROOT= +ENGINE_L0010_P001_OF001=HEALTH=.json +ENGINE_L0011_P001_OF001=STAGE_ROOT=/mnt/staging/homelab-control-auto +ENGINE_L0012_P001_OF001=VERIFY_ROOT=/mnt/staging/homelab-control-verify +ENGINE_L0013_P001_OF001=LOCK=/run/lock/homelab-control-backup.lock +ENGINE_L0014_P001_OF001=HEAVY_LOCK=/run/homelab-backup-heavy.lock +ENGINE_L0015_P001_OF001=RCLONE_CONFIG_SOURCE=${RCLONE_CONFIG:-/root/.config/rclone/[SENSITIVE_FILE]} +ENGINE_L0016_P001_OF001=TRANSPORT= +ENGINE_L0017_P001_OF001=TRANSPORT_SHA= +ENGINE_L0018_P001_OF001=BUILDER= +ENGINE_L0019_P001_OF001=BUILDER_SHA= +ENGINE_L0020_P001_OF001=REMOTE_A=pve01-mail-01-crypt:homelab-backups +ENGINE_L0021_P001_OF001=REMOTE_B=pve01-mail-02-crypt:homelab-backups +ENGINE_L0022_P001_OF001=CLOUD_WRITE_MARKER=${HOMELAB_CONTROL_CLOUD_WRITE_MARKER:-} +ENGINE_L0023_P001_OF001= +ENGINE_L0024_P001_OF001=die(){ printf 'CONTROL_BACKUP_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; } +ENGINE_L0025_P001_OF002=for c in jq date find sort tail cut sha256sum stat flock rclone mktemp cmp awk mkdir rm install xargs grep cat mv chmod; do command -v "$c" >/dev/null || die "T +ENGINE_L0025_P002_OF002=OOL_MISSING_$c" 69; done +ENGINE_L0026_P001_OF001=[[ -x "$TRANSPORT" && ! -L "$TRANSPORT" ]] || die TRANSPORT_UNSAFE 69 +ENGINE_L0027_P001_OF001=[[ "$(sha256sum "$TRANSPORT"|awk '{print $1}')" == "$TRANSPORT_SHA" ]] || die TRANSPORT_SHA_DRIFT 65 +ENGINE_L0028_P001_OF001=[[ -x "$BUILDER" && ! -L "$BUILDER" ]] || die BUILDER_UNSAFE 69 +ENGINE_L0029_P001_OF001=[[ "$(sha256sum "$BUILDER"|awk '{print $1}')" == "$BUILDER_SHA" ]] || die BUILDER_SHA_DRIFT 65 +ENGINE_L0030_P001_OF001= +ENGINE_L0031_P001_OF001=latest_state(){ +ENGINE_L0032_P001_OF001= [[ -d "$STATE_DIR" && ! -L "$STATE_DIR" ]] || return 0 +ENGINE_L0033_P001_OF001= find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print 2>/dev/null | LC_ALL=C sort | tail -1 +ENGINE_L0034_P001_OF001=} +ENGINE_L0035_P001_OF001=state_decision(){ +ENGINE_L0036_P001_OF001= local state="$1" now="$2" status gen upd epoch age ev sha +ENGINE_L0037_P001_OF001= if [[ -z "$state" ]]; then printf 'DECISION=BACKUP_UNVERIFIED REASON=NO_STATE AGE_SECONDS=-1\n'; return 0; fi +ENGINE_L0038_P001_OF001= [[ -s "$state" && ! -L "$state" ]] || { printf 'DECISION=BACKUP_UNVERIFIED REASON=STATE_UNSAFE AGE_SECONDS=-1\n'; return 0; } +ENGINE_L0039_P001_OF001= status="$(jq -r '.status // "INVALID"' "$state")"; gen="$(jq -r '.generation // "UNKNOWN"' "$state")" +ENGINE_L0040_P001_OF001= case "$status" in +ENGINE_L0041_P001_OF001= PREPARING|A_VERIFIED|PREPARED|DELETE_AUTHORIZED) +ENGINE_L0042_P001_OF001= printf 'DECISION=BACKUP_RESUME GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\n' "$gen" "$state"; return 0;; +ENGINE_L0043_P001_OF001= SEALED) ;; +ENGINE_L0044_P001_OF001= *) printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=STATE_STATUS_INVALID AGE_SECONDS=-1\n' "$gen"; return 0;; +ENGINE_L0045_P001_OF001= esac +ENGINE_L0046_P001_OF001= sha="$(jq -r '.archive_sha256 // ""' "$state")"; upd="$(jq -r '.sealed_at_utc // .updated_at_utc // ""' "$state")" +ENGINE_L0047_P001_OF001= epoch="$(date -u -d "$upd" +%s 2>/dev/null || true)" +ENGINE_L0048_P001_OF002= [[ "$epoch" =~ ^[0-9]+$ && "$epoch" -le "$now" ]] || { printf 'DECISION=BACKUP_UNVERIFIED GENERATION=%s REASON=TIMESTAMP_INVALID AGE_SECONDS=-1\n' "$gen"; ret +ENGINE_L0048_P002_OF002=urn 0; } +ENGINE_L0049_P001_OF001= ev="$EVIDENCE_ROOT/$gen.json" +ENGINE_L0050_P001_OF001= +ENGINE_L0051_P001_OF001= printf 'DECISION=BACKUP_REVERIFY GENERATION=%s STATE_FILE=%s AGE_SECONDS=-1\n' "$gen" "$state"; return 0 +ENGINE_L0052_P001_OF001= fi +ENGINE_L0053_P001_OF001= age=$((now-epoch)) +ENGINE_L0054_P001_OF001= if (( age < DUE_SECONDS )); then d=SKIP_FRESH; elif (( age < SLA_SECONDS )); then d=BACKUP_DUE; else d=BACKUP_STALE; fi +ENGINE_L0055_P001_OF001= printf 'DECISION=%s GENERATION=%s STATE_FILE=%s AGE_SECONDS=%s DUE_SECONDS=%s SLA_SECONDS=%s\n' "$d" "$gen" "$state" "$age" "$DUE_SECONDS" "$SLA_SECONDS" +ENGINE_L0056_P001_OF001=} +ENGINE_L0057_P001_OF001=scan(){ local now state; now="$(date -u +%s)"; state="$(latest_state)"; state_decision "$state" "$now"; } +ENGINE_L0058_P001_OF001=mark_cloud_write(){ +ENGINE_L0059_P001_OF001= [[ -z "$CLOUD_WRITE_MARKER" ]] && return 0 +ENGINE_L0060_P001_OF001= [[ "$CLOUD_WRITE_MARKER" == /* ]] || die CLOUD_WRITE_MARKER_NOT_ABSOLUTE 65 +ENGINE_L0061_P001_OF001= [[ ! -L "$CLOUD_WRITE_MARKER" ]] || die CLOUD_WRITE_MARKER_SYMLINK 65 +ENGINE_L0062_P001_OF001= printf '%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$CLOUD_WRITE_MARKER" +ENGINE_L0063_P001_OF001= chmod 0600 "$CLOUD_WRITE_MARKER" +ENGINE_L0064_P001_OF001=} +ENGINE_L0065_P001_OF001= +ENGINE_L0066_P001_OF001=prepare_rclone(){ +ENGINE_L0067_P001_OF001= local dst="$1"; [[ -f "$RCLONE_CONFIG_SOURCE" && ! -L "$RCLONE_CONFIG_SOURCE" ]] || die RCLONE_CONFIG_UNSAFE 66 +ENGINE_L0068_P001_OF001= install -m 0600 "$RCLONE_CONFIG_SOURCE" "$dst" +ENGINE_L0069_P001_OF001=} +ENGINE_L0070_P001_OF001=reconstruct(){ +ENGINE_L0071_P001_OF001= local base="$1" out="$2" cfg="$3" state="$4" label="$5" manifest part idx name size hash got_size got_hash expected_manifest expected_archive +ENGINE_L0072_P001_OF001= expected_manifest="$(jq -er '.parts_manifest_sha256' "$state")"; expected_archive="$(jq -er '.archive_sha256' "$state")" +ENGINE_L0073_P001_OF001= manifest="${out}.PARTS.tsv"; : >"$out" +ENGINE_L0074_P001_OF001= rclone --config "$cfg" cat "${base%/}/PARTS.tsv" >"$manifest" || die "REMOTE_${label}_MANIFEST_READ_FAILED" 74 +ENGINE_L0075_P001_OF001= [[ "$(sha256sum "$manifest"|awk '{print $1}')" == "$expected_manifest" ]] || die "REMOTE_${label}_MANIFEST_SHA_MISMATCH" 68 +ENGINE_L0076_P001_OF001= idx=0 +ENGINE_L0077_P001_OF001= while IFS=$'\t' read -r row name size hash; do +ENGINE_L0078_P001_OF002= [[ "$row" == "$(printf '%08d' "$idx")" && "$name" == part-????????.bin && "$size" =~ ^[1-9][0-9]*$ && "$hash" =~ ^[0-9a-f]{64}$ ]] || die "REMOTE_${label}_M +ENGINE_L0078_P002_OF002=ANIFEST_ROW_INVALID" 68 +ENGINE_L0079_P001_OF001= part="${out}.part"; rm -f "$part"; rclone --config "$cfg" cat "${base%/}/payload.parts/$name" >"$part" || die "REMOTE_${label}_PART_READ_FAILED" 74 +ENGINE_L0080_P001_OF001= got_size="$(stat -c %s "$part")"; got_hash="$(sha256sum "$part"|awk '{print $1}')" +ENGINE_L0081_P001_OF001= [[ "$got_size" == "$size" && "$got_hash" == "$hash" ]] || die "REMOTE_${label}_PART_VERIFY_FAILED" 68 +ENGINE_L0082_P001_OF001= cat "$part" >>"$out"; rm -f "$part"; idx=$((idx+1)) +ENGINE_L0083_P001_OF001= done <"$manifest" +ENGINE_L0084_P001_OF001= [[ "$(sha256sum "$out"|awk '{print $1}')" == "$expected_archive" ]] || die "REMOTE_${label}_ARCHIVE_SHA_MISMATCH" 68 +ENGINE_L0085_P001_OF001= rm -f "$manifest" +ENGINE_L0086_P001_OF001=} +ENGINE_L0087_P001_OF001=verify_generation(){ +ENGINE_L0088_P001_OF001= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now +ENGINE_L0089_P001_OF001= gen="$1" +ENGINE_L0090_P001_OF001= state="$STATE_DIR/$gen.json" +ENGINE_L0091_P001_OF001= [[ -s "$state" && ! -L "$state" ]] || die VERIFY_STATE_UNSAFE 68 +ENGINE_L0092_P001_OF001= "$TRANSPORT" verify-sealed "$LOGICAL_ID" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null +ENGINE_L0093_P001_OF001= mkdir -p "$VERIFY_ROOT"; chmod 0700 "$VERIFY_ROOT" +ENGINE_L0094_P001_OF001= work="$(mktemp -d "$VERIFY_ROOT/.verify.XXXXXX")"; cfg="$work/[SENSITIVE_FILE]"; prepare_rclone "$cfg" +ENGINE_L0095_P001_OF001= cleanup_verify(){ rm -rf --one-file-system "$work"; }; trap cleanup_verify EXIT +ENGINE_L0096_P001_OF001= base_a="$(jq -er '.remote_a_base' "$state")"; base_b="$(jq -er '.remote_b_base' "$state")"; a="$work/A.tar.gz"; b="$work/B.tar.gz" +ENGINE_L0097_P001_OF001= reconstruct "$base_a" "$a" "$cfg" "$state" A; reconstruct "$base_b" "$b" "$cfg" "$state" B +ENGINE_L0098_P001_OF001= cmp -s "$a" "$b" || die CLOUD_A_B_PAYLOAD_MISMATCH 68 +ENGINE_L0099_P001_OF001= verify_json="$("$BUILDER" verify --archive "$a" --expected-generation "$gen")" || die 68 +ENGINE_L0100_P001_OF001= +ENGINE_L0101_P001_OF001= archive_sha="$(jq -er '.archive_sha256' "$state")"; mkdir -p "$EVIDENCE_ROOT" /var/lib/homelab-health; chmod 0700 "$EVIDENCE_ROOT" +ENGINE_L0102_P001_OF001= tmp="$EVIDENCE_ROOT/.$gen.new.$$"; now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +ENGINE_L0103_P001_OF001= +ENGINE_L0104_P001_OF001= chmod 0600 "$tmp"; mv -f "$tmp" "$EVIDENCE_ROOT/$gen.json" +ENGINE_L0105_P001_OF003= tmp="${HEALTH}.new.$$"; jq -n --arg gen "$gen" --arg state "$state" --arg evidence "$EVIDENCE_ROOT/$gen.json" --arg sha "$archive_sha" --arg now "$now" '{sche +ENGINE_L0105_P002_OF003=ma:"homelab-control-backup-health-v1",status:"OK",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshn +ENGINE_L0105_P003_OF003=ess:"FRESH",full_site_dr_proven:false,updated_at_utc:$now}' >"$tmp"; chmod 0644 "$tmp"; mv -f "$tmp" "$HEALTH" +ENGINE_L0106_P001_OF001= trap - EXIT; cleanup_verify +ENGINE_L0107_P001_OF001= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\n' "$gen" "$EVIDENCE_ROOT/$gen.json" +ENGINE_L0108_P001_OF001=} +ENGINE_L0109_P001_OF001=run_backup(){ +ENGINE_L0110_P001_OF001= local force="${1:-0}" decision line state gen payload outdir now +ENGINE_L0111_P001_OF002= mkdir -p "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" /run/lock; chmod 0700 "$STATE_ROOT" "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_R +ENGINE_L0111_P002_OF002=OOT" +ENGINE_L0112_P001_OF001= exec 9>"$LOCK"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\n'; return 0; } +ENGINE_L0113_P001_OF001= local incompletes +ENGINE_L0114_P001_OF002= incompletes="$(find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select( +ENGINE_L0114_P002_OF002=.status!="SEALED")|.generation' 2>/dev/null || true)" +ENGINE_L0115_P001_OF001= [[ "$(grep -c . <<<"$incompletes" || true)" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68 +ENGINE_L0116_P001_OF001= line="$(scan)"; printf '%s\n' "$line"; decision="$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,"",$i);print $i}}' <<<"$line")" +ENGINE_L0117_P001_OF001= gen="$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$line")" +ENGINE_L0118_P001_OF001= if [[ "$decision" == SKIP_FRESH && "$force" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\n' "$gen"; return 0; fi +ENGINE_L0119_P001_OF001= if [[ "$decision" == BACKUP_REVERIFY ]]; then +ENGINE_L0120_P001_OF001= if [[ "$force" != 1 ]]; then verify_generation "$gen"; return 0; fi +ENGINE_L0121_P001_OF001= # Forced refresh deliberately creates a new generation instead of requiring +ENGINE_L0122_P001_OF001= # a legacy generation to satisfy the current bundle schema. +ENGINE_L0123_P001_OF001= decision=BACKUP_DUE +ENGINE_L0124_P001_OF001= fi +ENGINE_L0125_P001_OF001= if [[ "$decision" == BACKUP_RESUME ]]; then +ENGINE_L0126_P001_OF001= state="$STATE_DIR/$gen.json"; payload="$(jq -er '.payload_path' "$state")"; [[ -s "$payload" && ! -L "$payload" ]] || die INCOMPLETE_PAYLOAD_MISSING 68 +ENGINE_L0127_P001_OF001= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die 68 +ENGINE_L0128_P001_OF001= mark_cloud_write +ENGINE_L0129_P001_OF002= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"; verify_generation "$gen"; printf 'CONTROL_BACKUP_RESULT=SE +ENGINE_L0129_P002_OF002=ALED_RESUMED GENERATION=%s\n' "$gen"; return 0 +ENGINE_L0130_P001_OF001= fi +ENGINE_L0131_P001_OF001= case "$decision" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die "DECISION_INVALID_$decision" 65;; esac +ENGINE_L0132_P001_OF001= exec 8>"$HEAVY_LOCK"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\n'; return 0; } +ENGINE_L0133_P001_OF001= now="$(date -u +%Y%m%dT%H%M%SZ)"; gen="control-bootstrap-v2-pve01-$now"; outdir="$STAGE_ROOT/$gen" +ENGINE_L0134_P001_OF001= "$BUILDER" build --generation "$gen" --output-dir "$outdir" >/dev/null +ENGINE_L0135_P001_OF001= payload="$outdir/$gen.tar.gz"; [[ -s "$payload" && ! -L "$payload" ]] || die BUILT_PAYLOAD_MISSING 68 +ENGINE_L0136_P001_OF001= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die 68 +ENGINE_L0137_P001_OF001= exec 8>&- +ENGINE_L0138_P001_OF001= mark_cloud_write +ENGINE_L0139_P001_OF001= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +ENGINE_L0140_P001_OF001= rm -rf --one-file-system "$outdir" 2>/dev/null || true +ENGINE_L0141_P001_OF001= verify_generation "$gen" +ENGINE_L0142_P001_OF001= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\n' "$gen" +ENGINE_L0143_P001_OF001=} +ENGINE_L0144_P001_OF001=case "${1:-}" in +ENGINE_L0145_P001_OF001= scan) [[ "$#" == 1 ]] || die USAGE 64; scan;; +ENGINE_L0146_P001_OF001= run) [[ "$#" == 1 ]] || die USAGE 64; run_backup 0;; +ENGINE_L0147_P001_OF001= force) [[ "$#" == 1 ]] || die USAGE 64; run_backup 1;; +ENGINE_L0148_P001_OF001= verify) [[ "$#" == 2 ]] || die USAGE 64; verify_generation "$2";; +ENGINE_L0149_P001_OF001= *) die USAGE_scan_run_force_verify 64;; +ENGINE_L0150_P001_OF001=esac +ENGINE_SOURCE_END +UNIT_PATH=/etc/systemd/system/homelab-control-backup.service +UNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250 +UNIT_LINE_COUNT=16 +UNIT_REDACTED_LINE_COUNT=3 +UNIT_SOURCE_BEGIN +UNIT_L0001_P001_OF001=[Unit] +UNIT_L0002_P001_OF001=Description=Homelab recurring CONTROL cloud quorum backup +UNIT_L0003_P001_OF001=After=network-online.target +UNIT_L0004_P001_OF001=Wants=network-online.target +UNIT_L0005_P001_OF001=ConditionPathExists= +UNIT_L0006_P001_OF001=ConditionPathExists= +UNIT_L0007_P001_OF001= +UNIT_L0008_P001_OF001=[Service] +UNIT_L0009_P001_OF001=Type=oneshot +UNIT_L0010_P001_OF001=ExecStart= run +UNIT_L0011_P001_OF001=User=root +UNIT_L0012_P001_OF001=Group=root +UNIT_L0013_P001_OF001=UMask=0077 +UNIT_L0014_P001_OF001=Nice=10 +UNIT_L0015_P001_OF001=IOSchedulingClass=best-effort +UNIT_L0016_P001_OF001=IOSchedulingPriority=7 +UNIT_SOURCE_END +SOURCE_TRUNCATED_LINES=0 +SOURCE_FUNCTION_PARSER_USED=NO +BACKUP_ENGINE_EXECUTIONS=0 +RCLONE_COMMANDS_EXECUTED=0 +CLOUD_CALLS=0 +FILES_WRITTEN_BY_PAYLOAD=0 +SYSTEMD_MUTATIONS=0 +SOURCE259_READ_COMPLETE=YES +SOURCE259_END +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-SOURCE-259R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END