diff --git a/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1.json b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1.json new file mode 100644 index 00000000..11d2833a --- /dev/null +++ b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1", + "status": "FAIL", + "rc": 2, + "host": "pve01", + "mode": "read-only", + "component": "infra-dr-backupv2-control-backup-logic-focus", + "started_at_utc": "2026-09-11T23:37:18Z", + "finished_at_utc": "2026-09-11T23:37:18Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "9980bab4ad121c27772e38557dd65e4015aa4f3f718e5fcdc6e8dff43be4cd03", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 2, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "cb86b6a96cfd10465cf8955bc75edde6147f7b65633d31dd3f958e33501c2a82", + "sanitized_output_sha256": "e987624faf1aaf226e4a886a8bc963d4096bd01e5f5a5f3d283c195d15db6c98", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "FOCUS256_BEGIN\nCOMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1\nMODE=READ_ONLY_LOCAL_SOURCE_LOGIC_FOCUS\nCLOUD_CALLS_BY_256=0\nRCLONE_COMMANDS_EXECUTED_BY_256=0\nERROR_REGISTER_CHECK=READ\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nREFERENCE_CHECK=READ\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nAUTHORITY_FRESH_READ=OK\nTARGET_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57\nTARGET_SHA_MATCH=YES\nUNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250\nUNIT_SHA_MATCH=YES\nTARGET_LINE_COUNT=150\nFOCUS_FUNCTION=prepare_rclone\nFOCUS_SPAN=66-69\nFOCUS_BODY_BEGIN=prepare_rclone\nL066=prepare_rclone(){\nL067= local dst=\"$1\"; [[ -f \"$RCLONE_CONFIG_SOURCE\" && ! -L \"$RCLONE_CONFIG_SOURCE\" ]] || die RCLONE_CONFIG_UNSAFE 66\nL068= install -m 0600 \"$RCLONE_CONFIG_SOURCE\" \"$dst\"\nL069=}\nFOCUS_BODY_END=prepare_rclone\nFOCUS_FUNCTION=run_backup\nFOCUS_SPAN=109-150\nFOCUS_BODY_BEGIN=run_backup\nL109=run_backup(){\nL110= local force=\"${1:-0}\" decision line state gen payload outdir now\nL111= mkdir -p \"$STATE_DIR\" \"$EVIDENCE_ROOT\" \"$STAGE_ROOT\" \"$VERIFY_ROOT\" /run/lock; chmod 0700 \"$STATE_ROOT\" \"$STATE_DIR\" \"$EVIDENCE_ROOT\" \"$STAGE_ROOT\" \"$VERIFY_ROOT\"\nL112= exec 9>\"$LOCK\"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\\n'; return 0; }\nL113= local incompletes\nL114= incompletes=\"$(find \"$STATE_DIR\" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(.status!=\"SEALED\")|.generation' 2>/dev/null || true)\"\nL115= [[ \"$(grep -c . <<<\"$incompletes\" || true)\" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68\nL116= line=\"$(scan)\"; printf '%s\\n' \"$line\"; decision=\"$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$line\")\"\nL117= gen=\"$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$line\")\"\nL118= if [[ \"$decision\" == SKIP_FRESH && \"$force\" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\\n' \"$gen\"; return 0; fi\nL119= if [[ \"$decision\" == BACKUP_REVERIFY ]]; then\nL120= if [[ \"$force\" != 1 ]]; then verify_generation \"$gen\"; return 0; fi\nL121= # Forced refresh deliberately creates a new generation instead of requiring\nL122= # a legacy generation to satisfy the current bundle schema.\nL123= decision=BACKUP_DUE\nL124= fi\nL125= if [[ \"$decision\" == BACKUP_RESUME ]]; then\nL126= state=\"$STATE_DIR/$gen.json\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; [[ -s \"$payload\" && ! -L \"$payload\" ]] || die INCOMPLETE_PAYLOAD_MISSING 68\nL127= \"$BUILDER\" verify --archive \"$payload\" --expected-generation \"$gen\" >/dev/null || die 68\nL128= mark_cloud_write\nL129= \"$TRANSPORT\" backup-delete \"$LOGICAL_ID\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"; verify_generation \"$gen\"; printf 'CONTROL_BACKUP_RESULT=SEALED_RESUMED GENERATION=%s\\n' \"$gen\"; return 0\nL130= fi\nL131= case \"$decision\" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die \"DECISION_INVALID_$decision\" 65;; esac\nL132= exec 8>\"$HEAVY_LOCK\"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\\n'; return 0; }\nL133= now=\"$(date -u +%Y%m%dT%H%M%SZ)\"; gen=\"control-bootstrap-v2-pve01-$now\"; outdir=\"$STAGE_ROOT/$gen\"\nL134= \"$BUILDER\" build --generation \"$gen\" --output-dir \"$outdir\" >/dev/null\nL135= payload=\"$outdir/$gen.tar.gz\"; [[ -s \"$payload\" && ! -L \"$payload\" ]] || die BUILT_PAYLOAD_MISSING 68\nL136= \"$BUILDER\" verify --archive \"$payload\" --expected-generation \"$gen\" >/dev/null || die 68\nL137= exec 8>&-\nL138= mark_cloud_write\nL139= \"$TRANSPORT\" backup-delete \"$LOGICAL_ID\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nL140= rm -rf --one-file-system \"$outdir\" 2>/dev/null || true\nL141= verify_generation \"$gen\"\nL142= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\\n' \"$gen\"\nL143=}\nL144=case \"${1:-}\" in\nL145= scan) [[ \"$#\" == 1 ]] || die USAGE 64; scan;;\nL146= run) [[ \"$#\" == 1 ]] || die USAGE 64; run_backup 0;;\nL147= force) [[ \"$#\" == 1 ]] || die USAGE 64; run_backup 1;;\nL148= verify) [[ \"$#\" == 2 ]] || die USAGE 64; verify_generation \"$2\";;\nL149= *) die USAGE_scan_run_force_verify 64;;\nL150=esac\nFOCUS_BODY_END=run_backup\nFOCUS_FUNCTION=verify_generation\nFOCUS_SPAN=87-108\nFOCUS_BODY_BEGIN=verify_generation\nL087=verify_generation(){\nL088= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now\nL089= gen=\"$1\"\nL090= state=\"$STATE_DIR/$gen.json\"\nL091= [[ -s \"$state\" && ! -L \"$state\" ]] || die VERIFY_STATE_UNSAFE 68\nL092= \"$TRANSPORT\" verify-sealed \"$LOGICAL_ID\" \"$gen\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\" >/dev/null\nL093= mkdir -p \"$VERIFY_ROOT\"; chmod 0700 \"$VERIFY_ROOT\"\nL094= work=\"$(mktemp -d \"$VERIFY_ROOT/.verify.XXXXXX\")\"; cfg=\"$work/[SENSITIVE_FILE]\"; prepare_rclone \"$cfg\"\nL095= cleanup_verify(){ rm -rf --one-file-system \"$work\"; }; trap cleanup_verify EXIT\nL096= base_a=\"$(jq -er '.remote_a_base' \"$state\")\"; base_b=\"$(jq -er '.remote_b_base' \"$state\")\"; a=\"$work/A.tar.gz\"; b=\"$work/B.tar.gz\"\nL097= reconstruct \"$base_a\" \"$a\" \"$cfg\" \"$state\" A; reconstruct \"$base_b\" \"$b\" \"$cfg\" \"$state\" B\nL098= cmp -s \"$a\" \"$b\" || die CLOUD_A_B_PAYLOAD_MISMATCH 68\nL099= verify_json=\"$(\"$BUILDER\" verify --archive \"$a\" --expected-generation \"$gen\")\" || die 68\nL100=\nL101= archive_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; mkdir -p \"$EVIDENCE_ROOT\" /var/lib/homelab-health; chmod 0700 \"$EVIDENCE_ROOT\"\nL102= tmp=\"$EVIDENCE_ROOT/.$gen.new.$$\"; now=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nL103=\nL104= chmod 0600 \"$tmp\"; mv -f \"$tmp\" \"$EVIDENCE_ROOT/$gen.json\"\nL105= tmp=\"${HEALTH}.new.$$\"; jq -n --arg gen \"$gen\" --arg state \"$state\" --arg evidence \"$EVIDENCE_ROOT/$gen.json\" --arg sha \"$archive_sha\" --arg now \"$now\" '{schema:\"homelab-control-backup-health-v1\",status:\"OK\",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshness:\"FRESH\",full_site_dr_proven:false,updated_at_utc:$now}' >\"$tmp\"; chmod 0644 \"$tmp\"; mv -f \"$tmp\" \"$HEALTH\"\nL106= trap - EXIT; cleanup_verify\nL107= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\\n' \"$gen\" \"$EVIDENCE_ROOT/$gen.json\"\nL108=}\nFOCUS_BODY_END=verify_generation\nFOCUS256_STOP=FOCUS_FUNCTION_MISSING\nFOCUS256_DETAIL=cleanup_verify\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1.txt b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1.txt new file mode 100644 index 00000000..97cf9111 --- /dev/null +++ b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1.txt @@ -0,0 +1,130 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1 +STATUS=FAIL +RC=2 +HOST=pve01 +MODE=read-only +COMPONENT=infra-dr-backupv2-control-backup-logic-focus +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=9980bab4ad121c27772e38557dd65e4015aa4f3f718e5fcdc6e8dff43be4cd03 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=2 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=cb86b6a96cfd10465cf8955bc75edde6147f7b65633d31dd3f958e33501c2a82 +SANITIZED_OUTPUT_SHA256=e987624faf1aaf226e4a886a8bc963d4096bd01e5f5a5f3d283c195d15db6c98 +OUTPUT_BEGIN +FOCUS256_BEGIN +COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1 +MODE=READ_ONLY_LOCAL_SOURCE_LOGIC_FOCUS +CLOUD_CALLS_BY_256=0 +RCLONE_COMMANDS_EXECUTED_BY_256=0 +ERROR_REGISTER_CHECK=READ +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +REFERENCE_CHECK=READ +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +AUTHORITY_FRESH_READ=OK +TARGET_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57 +TARGET_SHA_MATCH=YES +UNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250 +UNIT_SHA_MATCH=YES +TARGET_LINE_COUNT=150 +FOCUS_FUNCTION=prepare_rclone +FOCUS_SPAN=66-69 +FOCUS_BODY_BEGIN=prepare_rclone +L066=prepare_rclone(){ +L067= local dst="$1"; [[ -f "$RCLONE_CONFIG_SOURCE" && ! -L "$RCLONE_CONFIG_SOURCE" ]] || die RCLONE_CONFIG_UNSAFE 66 +L068= install -m 0600 "$RCLONE_CONFIG_SOURCE" "$dst" +L069=} +FOCUS_BODY_END=prepare_rclone +FOCUS_FUNCTION=run_backup +FOCUS_SPAN=109-150 +FOCUS_BODY_BEGIN=run_backup +L109=run_backup(){ +L110= local force="${1:-0}" decision line state gen payload outdir now +L111= mkdir -p "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" /run/lock; chmod 0700 "$STATE_ROOT" "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" +L112= exec 9>"$LOCK"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\n'; return 0; } +L113= local incompletes +L114= incompletes="$(find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(.status!="SEALED")|.generation' 2>/dev/null || true)" +L115= [[ "$(grep -c . <<<"$incompletes" || true)" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68 +L116= line="$(scan)"; printf '%s\n' "$line"; decision="$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,"",$i);print $i}}' <<<"$line")" +L117= gen="$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$line")" +L118= if [[ "$decision" == SKIP_FRESH && "$force" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\n' "$gen"; return 0; fi +L119= if [[ "$decision" == BACKUP_REVERIFY ]]; then +L120= if [[ "$force" != 1 ]]; then verify_generation "$gen"; return 0; fi +L121= # Forced refresh deliberately creates a new generation instead of requiring +L122= # a legacy generation to satisfy the current bundle schema. +L123= decision=BACKUP_DUE +L124= fi +L125= if [[ "$decision" == BACKUP_RESUME ]]; then +L126= state="$STATE_DIR/$gen.json"; payload="$(jq -er '.payload_path' "$state")"; [[ -s "$payload" && ! -L "$payload" ]] || die INCOMPLETE_PAYLOAD_MISSING 68 +L127= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die 68 +L128= mark_cloud_write +L129= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"; verify_generation "$gen"; printf 'CONTROL_BACKUP_RESULT=SEALED_RESUMED GENERATION=%s\n' "$gen"; return 0 +L130= fi +L131= case "$decision" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die "DECISION_INVALID_$decision" 65;; esac +L132= exec 8>"$HEAVY_LOCK"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\n'; return 0; } +L133= now="$(date -u +%Y%m%dT%H%M%SZ)"; gen="control-bootstrap-v2-pve01-$now"; outdir="$STAGE_ROOT/$gen" +L134= "$BUILDER" build --generation "$gen" --output-dir "$outdir" >/dev/null +L135= payload="$outdir/$gen.tar.gz"; [[ -s "$payload" && ! -L "$payload" ]] || die BUILT_PAYLOAD_MISSING 68 +L136= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die 68 +L137= exec 8>&- +L138= mark_cloud_write +L139= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +L140= rm -rf --one-file-system "$outdir" 2>/dev/null || true +L141= verify_generation "$gen" +L142= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\n' "$gen" +L143=} +L144=case "${1:-}" in +L145= scan) [[ "$#" == 1 ]] || die USAGE 64; scan;; +L146= run) [[ "$#" == 1 ]] || die USAGE 64; run_backup 0;; +L147= force) [[ "$#" == 1 ]] || die USAGE 64; run_backup 1;; +L148= verify) [[ "$#" == 2 ]] || die USAGE 64; verify_generation "$2";; +L149= *) die USAGE_scan_run_force_verify 64;; +L150=esac +FOCUS_BODY_END=run_backup +FOCUS_FUNCTION=verify_generation +FOCUS_SPAN=87-108 +FOCUS_BODY_BEGIN=verify_generation +L087=verify_generation(){ +L088= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now +L089= gen="$1" +L090= state="$STATE_DIR/$gen.json" +L091= [[ -s "$state" && ! -L "$state" ]] || die VERIFY_STATE_UNSAFE 68 +L092= "$TRANSPORT" verify-sealed "$LOGICAL_ID" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null +L093= mkdir -p "$VERIFY_ROOT"; chmod 0700 "$VERIFY_ROOT" +L094= work="$(mktemp -d "$VERIFY_ROOT/.verify.XXXXXX")"; cfg="$work/[SENSITIVE_FILE]"; prepare_rclone "$cfg" +L095= cleanup_verify(){ rm -rf --one-file-system "$work"; }; trap cleanup_verify EXIT +L096= base_a="$(jq -er '.remote_a_base' "$state")"; base_b="$(jq -er '.remote_b_base' "$state")"; a="$work/A.tar.gz"; b="$work/B.tar.gz" +L097= reconstruct "$base_a" "$a" "$cfg" "$state" A; reconstruct "$base_b" "$b" "$cfg" "$state" B +L098= cmp -s "$a" "$b" || die CLOUD_A_B_PAYLOAD_MISMATCH 68 +L099= verify_json="$("$BUILDER" verify --archive "$a" --expected-generation "$gen")" || die 68 +L100= +L101= archive_sha="$(jq -er '.archive_sha256' "$state")"; mkdir -p "$EVIDENCE_ROOT" /var/lib/homelab-health; chmod 0700 "$EVIDENCE_ROOT" +L102= tmp="$EVIDENCE_ROOT/.$gen.new.$$"; now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +L103= +L104= chmod 0600 "$tmp"; mv -f "$tmp" "$EVIDENCE_ROOT/$gen.json" +L105= tmp="${HEALTH}.new.$$"; jq -n --arg gen "$gen" --arg state "$state" --arg evidence "$EVIDENCE_ROOT/$gen.json" --arg sha "$archive_sha" --arg now "$now" '{schema:"homelab-control-backup-health-v1",status:"OK",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshness:"FRESH",full_site_dr_proven:false,updated_at_utc:$now}' >"$tmp"; chmod 0644 "$tmp"; mv -f "$tmp" "$HEALTH" +L106= trap - EXIT; cleanup_verify +L107= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\n' "$gen" "$EVIDENCE_ROOT/$gen.json" +L108=} +FOCUS_BODY_END=verify_generation +FOCUS256_STOP=FOCUS_FUNCTION_MISSING +FOCUS256_DETAIL=cleanup_verify +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index ebacfdd9..11d2833a 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-255R1", - "status": "OK", - "rc": 0, + "command_id": "SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1", + "status": "FAIL", + "rc": 2, "host": "pve01", "mode": "read-only", - "component": "infra-dr-backupv2-control-backup-logic", - "started_at_utc": "2026-09-11T23:33:29Z", - "finished_at_utc": "2026-09-11T23:33:29Z", + "component": "infra-dr-backupv2-control-backup-logic-focus", + "started_at_utc": "2026-09-11T23:37:18Z", + "finished_at_utc": "2026-09-11T23:37:18Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "2c1d227e011446eb686ecf97218c70c89f651616f45b4bb3e68f32bba3533ace", + "command_sha256": "9980bab4ad121c27772e38557dd65e4015aa4f3f718e5fcdc6e8dff43be4cd03", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -21,7 +21,7 @@ "result_contract_valid": true, "result_contract_status": null, "result_contract_error": null, - "command_rc": 0, + "command_rc": 2, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "4be463d9015eac40cac32a68aa1b48e571e45d59f953244a55f8eba2d6e159cb", - "sanitized_output_sha256": "f367fb4787ad044b2e839dc8ca1d49f1687c27df5a650940aeacd85661a27130", + "raw_evidence_sha256": "cb86b6a96cfd10465cf8955bc75edde6147f7b65633d31dd3f958e33501c2a82", + "sanitized_output_sha256": "e987624faf1aaf226e4a886a8bc963d4096bd01e5f5a5f3d283c195d15db6c98", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "LOGIC255_BEGIN\nCOMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-255R1\nMODE=READ_ONLY_LOCAL_SOURCE_LOGIC\nCLOUD_CALLS_BY_255=0\nRCLONE_COMMANDS_EXECUTED_BY_255=0\nERROR_REGISTER_CHECK=READ\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nREFERENCE_CHECK=READ\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nAUTHORITY_FRESH_READ=OK\nTARGET_PATH=/usr/local/sbin/homelab-control-backup\nTARGET_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57\nTARGET_SHA_MATCH=YES\nUNIT_PATH=/etc/systemd/system/homelab-control-backup.service\nUNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250\nUNIT_SHA_MATCH=YES\nTARGET_LINE_COUNT=150\nFUNCTION_COUNT=0\nSOURCE_SKELETON_BEGIN\nSRC_LINE_001=#!/usr/bin/env bash\nSRC_LINE_002=set -Eeuo pipefail\nSRC_LINE_003=umask 077\nSRC_LINE_004=LOGICAL_ID=control-bootstrap-v2\nSRC_LINE_005=DUE_SECONDS=43200\nSRC_LINE_006=SLA_SECONDS=86400\nSRC_LINE_007=STATE_ROOT=\nSRC_LINE_008=STATE_DIR=$STATE_ROOT/$LOGICAL_ID\nSRC_LINE_009=EVIDENCE_ROOT=\nSRC_LINE_010=HEALTH=.json\nSRC_LINE_011=STAGE_ROOT=\nSRC_LINE_012=VERIFY_ROOT=\nSRC_LINE_013=LOCK=.lock\nSRC_LINE_014=HEAVY_LOCK=/run/homelab-backup-heavy.lock\nSRC_LINE_015=RCLONE_CONFIG_SOURCE=${RCLONE_CONFIG:-/root/.config/rclone/[SENSITIVE_FILE]}\nSRC_LINE_016=TRANSPORT=\nSRC_LINE_017=TRANSPORT_SHA=\nSRC_LINE_018=BUILDER=\nSRC_LINE_019=BUILDER_SHA=\nSRC_LINE_020=REMOTE_A=pve01-mail-01-crypt:homelab-backups\nSRC_LINE_021=REMOTE_B=pve01-mail-02-crypt:homelab-backups\nSRC_LINE_022=CLOUD_WRITE_MARKER=${:-}\nSRC_LINE_023=\nSRC_LINE_024=die(){ printf 'CONTROL_BACKUP_ERROR=%s' \"$1\" >&2; exit \"${2:-65}\"; }\nSRC_LINE_025=for c in jq date find sort tail cut sha256sum stat flock rclone mktemp cmp awk mkdir rm install xargs grep cat mv chmod; do command -v \"$c\" >/dev/null || die \"TOOL_MISSING_$c\" 69; done\nSRC_LINE_026=[[ -x \"$TRANSPORT\" && ! -L \"$TRANSPORT\" ]] || die TRANSPORT_UNSAFE 69\nSRC_LINE_027=[[ \"$(sha256sum \"$TRANSPORT\"|awk '{print $1}')\" == \"$TRANSPORT_SHA\" ]] || die TRANSPORT_SHA_DRIFT 65\nSRC_LINE_028=[[ -x \"$BUILDER\" && ! -L \"$BUILDER\" ]] || die BUILDER_UNSAFE 69\nSRC_LINE_029=[[ \"$(sha256sum \"$BUILDER\"|awk '{print $1}')\" == \"$BUILDER_SHA\" ]] || die BUILDER_SHA_DRIFT 65\nSRC_LINE_030=\nSOURCE_SKELETON_END\nSOURCE_REDACTED_LINE_COUNT=0\nFIXED_REMOTE_PVE01_MAIL_01_CRYPT_LINES=20\nFIXED_REMOTE_PVE01_MAIL_02_CRYPT_LINES=21\nMARKER_RCLONE_LINES=15,25,66,67,68,74,79,94\nMARKER_SEALED_LINES=43,46,92,114,129,142\nMARKER_VERIFY_SEALED_LINES=92\nMARKER_BACKUP_DELETE_LINES=129,139\nMARKER_LOCAL_CLEANUP_LINES=NONE\nMARKER_RUN_BACKUP_LINES=109,146,147\nMARKER_VERIFY_GENERATION_LINES=87,120,129,141,148\nMARKER_CLEANUP_VERIFY_LINES=95,106\nMARKER_RECONSTRUCT_LINES=70,97,103\nMARKER_SPLIT_LINES=NONE\nMARKER_536870912_LINES=NONE\nMARKER_512M_LINES=NONE\nMARKER_PART_LINES=71,72,73,74,78,79,80,81,82\nUNIT_LINE_COUNT=16\nUNIT_SKELETON_BEGIN\nUNIT_LINE_001=[Unit]\nUNIT_LINE_002=Description=Homelab recurring CONTROL cloud quorum backup\nUNIT_LINE_003=After=network-online.target\nUNIT_LINE_004=Wants=network-online.target\nUNIT_LINE_005=ConditionPathExists=\nUNIT_LINE_006=ConditionPathExists=\nUNIT_LINE_007=\nUNIT_LINE_008=[Service]\nUNIT_LINE_009=Type=oneshot\nUNIT_LINE_010=ExecStart= run\nUNIT_LINE_011=User=root\nUNIT_LINE_012=Group=root\nUNIT_LINE_013=UMask=0077\nUNIT_LINE_014=Nice=10\nUNIT_LINE_015=IOSchedulingClass=best-effort\nUNIT_LINE_016=IOSchedulingPriority=7\nUNIT_SKELETON_END\nUNIT_REDACTED_LINE_COUNT=0\nFILES_WRITTEN_BY_PAYLOAD=0\nSYSTEMD_MUTATIONS_BY_255=0\nLOGIC255_READY_FOR_MINIMAL_PATCH=YES\nLOGIC255_END\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-255R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "FOCUS256_BEGIN\nCOMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1\nMODE=READ_ONLY_LOCAL_SOURCE_LOGIC_FOCUS\nCLOUD_CALLS_BY_256=0\nRCLONE_COMMANDS_EXECUTED_BY_256=0\nERROR_REGISTER_CHECK=READ\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nREFERENCE_CHECK=READ\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nAUTHORITY_FRESH_READ=OK\nTARGET_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57\nTARGET_SHA_MATCH=YES\nUNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250\nUNIT_SHA_MATCH=YES\nTARGET_LINE_COUNT=150\nFOCUS_FUNCTION=prepare_rclone\nFOCUS_SPAN=66-69\nFOCUS_BODY_BEGIN=prepare_rclone\nL066=prepare_rclone(){\nL067= local dst=\"$1\"; [[ -f \"$RCLONE_CONFIG_SOURCE\" && ! -L \"$RCLONE_CONFIG_SOURCE\" ]] || die RCLONE_CONFIG_UNSAFE 66\nL068= install -m 0600 \"$RCLONE_CONFIG_SOURCE\" \"$dst\"\nL069=}\nFOCUS_BODY_END=prepare_rclone\nFOCUS_FUNCTION=run_backup\nFOCUS_SPAN=109-150\nFOCUS_BODY_BEGIN=run_backup\nL109=run_backup(){\nL110= local force=\"${1:-0}\" decision line state gen payload outdir now\nL111= mkdir -p \"$STATE_DIR\" \"$EVIDENCE_ROOT\" \"$STAGE_ROOT\" \"$VERIFY_ROOT\" /run/lock; chmod 0700 \"$STATE_ROOT\" \"$STATE_DIR\" \"$EVIDENCE_ROOT\" \"$STAGE_ROOT\" \"$VERIFY_ROOT\"\nL112= exec 9>\"$LOCK\"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\\n'; return 0; }\nL113= local incompletes\nL114= incompletes=\"$(find \"$STATE_DIR\" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(.status!=\"SEALED\")|.generation' 2>/dev/null || true)\"\nL115= [[ \"$(grep -c . <<<\"$incompletes\" || true)\" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68\nL116= line=\"$(scan)\"; printf '%s\\n' \"$line\"; decision=\"$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$line\")\"\nL117= gen=\"$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$line\")\"\nL118= if [[ \"$decision\" == SKIP_FRESH && \"$force\" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\\n' \"$gen\"; return 0; fi\nL119= if [[ \"$decision\" == BACKUP_REVERIFY ]]; then\nL120= if [[ \"$force\" != 1 ]]; then verify_generation \"$gen\"; return 0; fi\nL121= # Forced refresh deliberately creates a new generation instead of requiring\nL122= # a legacy generation to satisfy the current bundle schema.\nL123= decision=BACKUP_DUE\nL124= fi\nL125= if [[ \"$decision\" == BACKUP_RESUME ]]; then\nL126= state=\"$STATE_DIR/$gen.json\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; [[ -s \"$payload\" && ! -L \"$payload\" ]] || die INCOMPLETE_PAYLOAD_MISSING 68\nL127= \"$BUILDER\" verify --archive \"$payload\" --expected-generation \"$gen\" >/dev/null || die 68\nL128= mark_cloud_write\nL129= \"$TRANSPORT\" backup-delete \"$LOGICAL_ID\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"; verify_generation \"$gen\"; printf 'CONTROL_BACKUP_RESULT=SEALED_RESUMED GENERATION=%s\\n' \"$gen\"; return 0\nL130= fi\nL131= case \"$decision\" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die \"DECISION_INVALID_$decision\" 65;; esac\nL132= exec 8>\"$HEAVY_LOCK\"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\\n'; return 0; }\nL133= now=\"$(date -u +%Y%m%dT%H%M%SZ)\"; gen=\"control-bootstrap-v2-pve01-$now\"; outdir=\"$STAGE_ROOT/$gen\"\nL134= \"$BUILDER\" build --generation \"$gen\" --output-dir \"$outdir\" >/dev/null\nL135= payload=\"$outdir/$gen.tar.gz\"; [[ -s \"$payload\" && ! -L \"$payload\" ]] || die BUILT_PAYLOAD_MISSING 68\nL136= \"$BUILDER\" verify --archive \"$payload\" --expected-generation \"$gen\" >/dev/null || die 68\nL137= exec 8>&-\nL138= mark_cloud_write\nL139= \"$TRANSPORT\" backup-delete \"$LOGICAL_ID\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nL140= rm -rf --one-file-system \"$outdir\" 2>/dev/null || true\nL141= verify_generation \"$gen\"\nL142= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\\n' \"$gen\"\nL143=}\nL144=case \"${1:-}\" in\nL145= scan) [[ \"$#\" == 1 ]] || die USAGE 64; scan;;\nL146= run) [[ \"$#\" == 1 ]] || die USAGE 64; run_backup 0;;\nL147= force) [[ \"$#\" == 1 ]] || die USAGE 64; run_backup 1;;\nL148= verify) [[ \"$#\" == 2 ]] || die USAGE 64; verify_generation \"$2\";;\nL149= *) die USAGE_scan_run_force_verify 64;;\nL150=esac\nFOCUS_BODY_END=run_backup\nFOCUS_FUNCTION=verify_generation\nFOCUS_SPAN=87-108\nFOCUS_BODY_BEGIN=verify_generation\nL087=verify_generation(){\nL088= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now\nL089= gen=\"$1\"\nL090= state=\"$STATE_DIR/$gen.json\"\nL091= [[ -s \"$state\" && ! -L \"$state\" ]] || die VERIFY_STATE_UNSAFE 68\nL092= \"$TRANSPORT\" verify-sealed \"$LOGICAL_ID\" \"$gen\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\" >/dev/null\nL093= mkdir -p \"$VERIFY_ROOT\"; chmod 0700 \"$VERIFY_ROOT\"\nL094= work=\"$(mktemp -d \"$VERIFY_ROOT/.verify.XXXXXX\")\"; cfg=\"$work/[SENSITIVE_FILE]\"; prepare_rclone \"$cfg\"\nL095= cleanup_verify(){ rm -rf --one-file-system \"$work\"; }; trap cleanup_verify EXIT\nL096= base_a=\"$(jq -er '.remote_a_base' \"$state\")\"; base_b=\"$(jq -er '.remote_b_base' \"$state\")\"; a=\"$work/A.tar.gz\"; b=\"$work/B.tar.gz\"\nL097= reconstruct \"$base_a\" \"$a\" \"$cfg\" \"$state\" A; reconstruct \"$base_b\" \"$b\" \"$cfg\" \"$state\" B\nL098= cmp -s \"$a\" \"$b\" || die CLOUD_A_B_PAYLOAD_MISMATCH 68\nL099= verify_json=\"$(\"$BUILDER\" verify --archive \"$a\" --expected-generation \"$gen\")\" || die 68\nL100=\nL101= archive_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; mkdir -p \"$EVIDENCE_ROOT\" /var/lib/homelab-health; chmod 0700 \"$EVIDENCE_ROOT\"\nL102= tmp=\"$EVIDENCE_ROOT/.$gen.new.$$\"; now=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nL103=\nL104= chmod 0600 \"$tmp\"; mv -f \"$tmp\" \"$EVIDENCE_ROOT/$gen.json\"\nL105= tmp=\"${HEALTH}.new.$$\"; jq -n --arg gen \"$gen\" --arg state \"$state\" --arg evidence \"$EVIDENCE_ROOT/$gen.json\" --arg sha \"$archive_sha\" --arg now \"$now\" '{schema:\"homelab-control-backup-health-v1\",status:\"OK\",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshness:\"FRESH\",full_site_dr_proven:false,updated_at_utc:$now}' >\"$tmp\"; chmod 0644 \"$tmp\"; mv -f \"$tmp\" \"$HEALTH\"\nL106= trap - EXIT; cleanup_verify\nL107= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\\n' \"$gen\" \"$EVIDENCE_ROOT/$gen.json\"\nL108=}\nFOCUS_BODY_END=verify_generation\nFOCUS256_STOP=FOCUS_FUNCTION_MISSING\nFOCUS256_DETAIL=cleanup_verify\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 3d53ef7a..97cf9111 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-255R1 -STATUS=OK -RC=0 +COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1 +STATUS=FAIL +RC=2 HOST=pve01 MODE=read-only -COMPONENT=infra-dr-backupv2-control-backup-logic +COMPONENT=infra-dr-backupv2-control-backup-logic-focus REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=2c1d227e011446eb686ecf97218c70c89f651616f45b4bb3e68f32bba3533ace +COMMAND_SHA256=9980bab4ad121c27772e38557dd65e4015aa4f3f718e5fcdc6e8dff43be4cd03 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=0 +COMMAND_RC=2 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,100 +24,107 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=4be463d9015eac40cac32a68aa1b48e571e45d59f953244a55f8eba2d6e159cb -SANITIZED_OUTPUT_SHA256=f367fb4787ad044b2e839dc8ca1d49f1687c27df5a650940aeacd85661a27130 +RAW_EVIDENCE_SHA256=cb86b6a96cfd10465cf8955bc75edde6147f7b65633d31dd3f958e33501c2a82 +SANITIZED_OUTPUT_SHA256=e987624faf1aaf226e4a886a8bc963d4096bd01e5f5a5f3d283c195d15db6c98 OUTPUT_BEGIN -LOGIC255_BEGIN -COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-255R1 -MODE=READ_ONLY_LOCAL_SOURCE_LOGIC -CLOUD_CALLS_BY_255=0 -RCLONE_COMMANDS_EXECUTED_BY_255=0 +FOCUS256_BEGIN +COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1 +MODE=READ_ONLY_LOCAL_SOURCE_LOGIC_FOCUS +CLOUD_CALLS_BY_256=0 +RCLONE_COMMANDS_EXECUTED_BY_256=0 ERROR_REGISTER_CHECK=READ ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 REFERENCE_CHECK=READ REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 AUTHORITY_FRESH_READ=OK -TARGET_PATH=/usr/local/sbin/homelab-control-backup TARGET_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57 TARGET_SHA_MATCH=YES -UNIT_PATH=/etc/systemd/system/homelab-control-backup.service UNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250 UNIT_SHA_MATCH=YES TARGET_LINE_COUNT=150 -FUNCTION_COUNT=0 -SOURCE_SKELETON_BEGIN -SRC_LINE_001=#!/usr/bin/env bash -SRC_LINE_002=set -Eeuo pipefail -SRC_LINE_003=umask 077 -SRC_LINE_004=LOGICAL_ID=control-bootstrap-v2 -SRC_LINE_005=DUE_SECONDS=43200 -SRC_LINE_006=SLA_SECONDS=86400 -SRC_LINE_007=STATE_ROOT= -SRC_LINE_008=STATE_DIR=$STATE_ROOT/$LOGICAL_ID -SRC_LINE_009=EVIDENCE_ROOT= -SRC_LINE_010=HEALTH=.json -SRC_LINE_011=STAGE_ROOT= -SRC_LINE_012=VERIFY_ROOT= -SRC_LINE_013=LOCK=.lock -SRC_LINE_014=HEAVY_LOCK=/run/homelab-backup-heavy.lock -SRC_LINE_015=RCLONE_CONFIG_SOURCE=${RCLONE_CONFIG:-/root/.config/rclone/[SENSITIVE_FILE]} -SRC_LINE_016=TRANSPORT= -SRC_LINE_017=TRANSPORT_SHA= -SRC_LINE_018=BUILDER= -SRC_LINE_019=BUILDER_SHA= -SRC_LINE_020=REMOTE_A=pve01-mail-01-crypt:homelab-backups -SRC_LINE_021=REMOTE_B=pve01-mail-02-crypt:homelab-backups -SRC_LINE_022=CLOUD_WRITE_MARKER=${:-} -SRC_LINE_023= -SRC_LINE_024=die(){ printf 'CONTROL_BACKUP_ERROR=%s' "$1" >&2; exit "${2:-65}"; } -SRC_LINE_025=for c in jq date find sort tail cut sha256sum stat flock rclone mktemp cmp awk mkdir rm install xargs grep cat mv chmod; do command -v "$c" >/dev/null || die "TOOL_MISSING_$c" 69; done -SRC_LINE_026=[[ -x "$TRANSPORT" && ! -L "$TRANSPORT" ]] || die TRANSPORT_UNSAFE 69 -SRC_LINE_027=[[ "$(sha256sum "$TRANSPORT"|awk '{print $1}')" == "$TRANSPORT_SHA" ]] || die TRANSPORT_SHA_DRIFT 65 -SRC_LINE_028=[[ -x "$BUILDER" && ! -L "$BUILDER" ]] || die BUILDER_UNSAFE 69 -SRC_LINE_029=[[ "$(sha256sum "$BUILDER"|awk '{print $1}')" == "$BUILDER_SHA" ]] || die BUILDER_SHA_DRIFT 65 -SRC_LINE_030= -SOURCE_SKELETON_END -SOURCE_REDACTED_LINE_COUNT=0 -FIXED_REMOTE_PVE01_MAIL_01_CRYPT_LINES=20 -FIXED_REMOTE_PVE01_MAIL_02_CRYPT_LINES=21 -MARKER_RCLONE_LINES=15,25,66,67,68,74,79,94 -MARKER_SEALED_LINES=43,46,92,114,129,142 -MARKER_VERIFY_SEALED_LINES=92 -MARKER_BACKUP_DELETE_LINES=129,139 -MARKER_LOCAL_CLEANUP_LINES=NONE -MARKER_RUN_BACKUP_LINES=109,146,147 -MARKER_VERIFY_GENERATION_LINES=87,120,129,141,148 -MARKER_CLEANUP_VERIFY_LINES=95,106 -MARKER_RECONSTRUCT_LINES=70,97,103 -MARKER_SPLIT_LINES=NONE -MARKER_536870912_LINES=NONE -MARKER_512M_LINES=NONE -MARKER_PART_LINES=71,72,73,74,78,79,80,81,82 -UNIT_LINE_COUNT=16 -UNIT_SKELETON_BEGIN -UNIT_LINE_001=[Unit] -UNIT_LINE_002=Description=Homelab recurring CONTROL cloud quorum backup -UNIT_LINE_003=After=network-online.target -UNIT_LINE_004=Wants=network-online.target -UNIT_LINE_005=ConditionPathExists= -UNIT_LINE_006=ConditionPathExists= -UNIT_LINE_007= -UNIT_LINE_008=[Service] -UNIT_LINE_009=Type=oneshot -UNIT_LINE_010=ExecStart= run -UNIT_LINE_011=User=root -UNIT_LINE_012=Group=root -UNIT_LINE_013=UMask=0077 -UNIT_LINE_014=Nice=10 -UNIT_LINE_015=IOSchedulingClass=best-effort -UNIT_LINE_016=IOSchedulingPriority=7 -UNIT_SKELETON_END -UNIT_REDACTED_LINE_COUNT=0 -FILES_WRITTEN_BY_PAYLOAD=0 -SYSTEMD_MUTATIONS_BY_255=0 -LOGIC255_READY_FOR_MINIMAL_PATCH=YES -LOGIC255_END -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-255R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +FOCUS_FUNCTION=prepare_rclone +FOCUS_SPAN=66-69 +FOCUS_BODY_BEGIN=prepare_rclone +L066=prepare_rclone(){ +L067= local dst="$1"; [[ -f "$RCLONE_CONFIG_SOURCE" && ! -L "$RCLONE_CONFIG_SOURCE" ]] || die RCLONE_CONFIG_UNSAFE 66 +L068= install -m 0600 "$RCLONE_CONFIG_SOURCE" "$dst" +L069=} +FOCUS_BODY_END=prepare_rclone +FOCUS_FUNCTION=run_backup +FOCUS_SPAN=109-150 +FOCUS_BODY_BEGIN=run_backup +L109=run_backup(){ +L110= local force="${1:-0}" decision line state gen payload outdir now +L111= mkdir -p "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" /run/lock; chmod 0700 "$STATE_ROOT" "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" +L112= exec 9>"$LOCK"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\n'; return 0; } +L113= local incompletes +L114= incompletes="$(find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(.status!="SEALED")|.generation' 2>/dev/null || true)" +L115= [[ "$(grep -c . <<<"$incompletes" || true)" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68 +L116= line="$(scan)"; printf '%s\n' "$line"; decision="$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,"",$i);print $i}}' <<<"$line")" +L117= gen="$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$line")" +L118= if [[ "$decision" == SKIP_FRESH && "$force" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\n' "$gen"; return 0; fi +L119= if [[ "$decision" == BACKUP_REVERIFY ]]; then +L120= if [[ "$force" != 1 ]]; then verify_generation "$gen"; return 0; fi +L121= # Forced refresh deliberately creates a new generation instead of requiring +L122= # a legacy generation to satisfy the current bundle schema. +L123= decision=BACKUP_DUE +L124= fi +L125= if [[ "$decision" == BACKUP_RESUME ]]; then +L126= state="$STATE_DIR/$gen.json"; payload="$(jq -er '.payload_path' "$state")"; [[ -s "$payload" && ! -L "$payload" ]] || die INCOMPLETE_PAYLOAD_MISSING 68 +L127= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die 68 +L128= mark_cloud_write +L129= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"; verify_generation "$gen"; printf 'CONTROL_BACKUP_RESULT=SEALED_RESUMED GENERATION=%s\n' "$gen"; return 0 +L130= fi +L131= case "$decision" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die "DECISION_INVALID_$decision" 65;; esac +L132= exec 8>"$HEAVY_LOCK"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\n'; return 0; } +L133= now="$(date -u +%Y%m%dT%H%M%SZ)"; gen="control-bootstrap-v2-pve01-$now"; outdir="$STAGE_ROOT/$gen" +L134= "$BUILDER" build --generation "$gen" --output-dir "$outdir" >/dev/null +L135= payload="$outdir/$gen.tar.gz"; [[ -s "$payload" && ! -L "$payload" ]] || die BUILT_PAYLOAD_MISSING 68 +L136= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die 68 +L137= exec 8>&- +L138= mark_cloud_write +L139= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +L140= rm -rf --one-file-system "$outdir" 2>/dev/null || true +L141= verify_generation "$gen" +L142= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\n' "$gen" +L143=} +L144=case "${1:-}" in +L145= scan) [[ "$#" == 1 ]] || die USAGE 64; scan;; +L146= run) [[ "$#" == 1 ]] || die USAGE 64; run_backup 0;; +L147= force) [[ "$#" == 1 ]] || die USAGE 64; run_backup 1;; +L148= verify) [[ "$#" == 2 ]] || die USAGE 64; verify_generation "$2";; +L149= *) die USAGE_scan_run_force_verify 64;; +L150=esac +FOCUS_BODY_END=run_backup +FOCUS_FUNCTION=verify_generation +FOCUS_SPAN=87-108 +FOCUS_BODY_BEGIN=verify_generation +L087=verify_generation(){ +L088= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now +L089= gen="$1" +L090= state="$STATE_DIR/$gen.json" +L091= [[ -s "$state" && ! -L "$state" ]] || die VERIFY_STATE_UNSAFE 68 +L092= "$TRANSPORT" verify-sealed "$LOGICAL_ID" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null +L093= mkdir -p "$VERIFY_ROOT"; chmod 0700 "$VERIFY_ROOT" +L094= work="$(mktemp -d "$VERIFY_ROOT/.verify.XXXXXX")"; cfg="$work/[SENSITIVE_FILE]"; prepare_rclone "$cfg" +L095= cleanup_verify(){ rm -rf --one-file-system "$work"; }; trap cleanup_verify EXIT +L096= base_a="$(jq -er '.remote_a_base' "$state")"; base_b="$(jq -er '.remote_b_base' "$state")"; a="$work/A.tar.gz"; b="$work/B.tar.gz" +L097= reconstruct "$base_a" "$a" "$cfg" "$state" A; reconstruct "$base_b" "$b" "$cfg" "$state" B +L098= cmp -s "$a" "$b" || die CLOUD_A_B_PAYLOAD_MISMATCH 68 +L099= verify_json="$("$BUILDER" verify --archive "$a" --expected-generation "$gen")" || die 68 +L100= +L101= archive_sha="$(jq -er '.archive_sha256' "$state")"; mkdir -p "$EVIDENCE_ROOT" /var/lib/homelab-health; chmod 0700 "$EVIDENCE_ROOT" +L102= tmp="$EVIDENCE_ROOT/.$gen.new.$$"; now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +L103= +L104= chmod 0600 "$tmp"; mv -f "$tmp" "$EVIDENCE_ROOT/$gen.json" +L105= tmp="${HEALTH}.new.$$"; jq -n --arg gen "$gen" --arg state "$state" --arg evidence "$EVIDENCE_ROOT/$gen.json" --arg sha "$archive_sha" --arg now "$now" '{schema:"homelab-control-backup-health-v1",status:"OK",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshness:"FRESH",full_site_dr_proven:false,updated_at_utc:$now}' >"$tmp"; chmod 0644 "$tmp"; mv -f "$tmp" "$HEALTH" +L106= trap - EXIT; cleanup_verify +L107= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\n' "$gen" "$EVIDENCE_ROOT/$gen.json" +L108=} +FOCUS_BODY_END=verify_generation +FOCUS256_STOP=FOCUS_FUNCTION_MISSING +FOCUS256_DETAIL=cleanup_verify +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END