diff --git a/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1.json b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1.json new file mode 100644 index 00000000..47140ef6 --- /dev/null +++ b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "infra-dr-backupv2-control-backup-logic-focus-diagnostic", + "started_at_utc": "2026-09-11T23:44:54Z", + "finished_at_utc": "2026-09-11T23:44:54Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "f3ab0a154eda3d947c6b078120e2088b7d0721f992b74d9da8e78af6568708ca", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974", + "sanitized_output_sha256": "72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "DIAG258_BEGIN\nTARGET_COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1\nTARGET_ATTEMPT_DIR=/root/homelab-run-SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1\nTARGET_ATTEMPT_DIR_EXISTS=NO\nMATCHING_257_ATTEMPT_DIR_COUNT=0\nEVIDENCE_LAYER=NO_257_SERVER_ATTEMPT_DIRECTORY\nINTERPRETATION=257_DID_NOT_REACH_RUNNER_RESERVATION\nFILES_WRITTEN_BY_DIAG258=0\nRCLONE_COMMANDS_BY_DIAG258=0\nCLOUD_MUTATIONS_BY_DIAG258=0\nDIAG258_END\n" +} diff --git a/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1.txt b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1.txt new file mode 100644 index 00000000..271e21d5 --- /dev/null +++ b/runtime/history/SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1.txt @@ -0,0 +1,43 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=infra-dr-backupv2-control-backup-logic-focus-diagnostic +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=f3ab0a154eda3d947c6b078120e2088b7d0721f992b74d9da8e78af6568708ca +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974 +SANITIZED_OUTPUT_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974 +OUTPUT_BEGIN +DIAG258_BEGIN +TARGET_COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1 +TARGET_ATTEMPT_DIR=/root/homelab-run-SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1 +TARGET_ATTEMPT_DIR_EXISTS=NO +MATCHING_257_ATTEMPT_DIR_COUNT=0 +EVIDENCE_LAYER=NO_257_SERVER_ATTEMPT_DIRECTORY +INTERPRETATION=257_DID_NOT_REACH_RUNNER_RESERVATION +FILES_WRITTEN_BY_DIAG258=0 +RCLONE_COMMANDS_BY_DIAG258=0 +CLOUD_MUTATIONS_BY_DIAG258=0 +DIAG258_END + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 11d2833a..47140ef6 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1", - "status": "FAIL", - "rc": 2, + "command_id": "SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1", + "status": "OK", + "rc": 0, "host": "pve01", "mode": "read-only", - "component": "infra-dr-backupv2-control-backup-logic-focus", - "started_at_utc": "2026-09-11T23:37:18Z", - "finished_at_utc": "2026-09-11T23:37:18Z", + "component": "infra-dr-backupv2-control-backup-logic-focus-diagnostic", + "started_at_utc": "2026-09-11T23:44:54Z", + "finished_at_utc": "2026-09-11T23:44:54Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "9980bab4ad121c27772e38557dd65e4015aa4f3f718e5fcdc6e8dff43be4cd03", + "command_sha256": "f3ab0a154eda3d947c6b078120e2088b7d0721f992b74d9da8e78af6568708ca", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -21,7 +21,7 @@ "result_contract_valid": true, "result_contract_status": null, "result_contract_error": null, - "command_rc": 2, + "command_rc": 0, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "cb86b6a96cfd10465cf8955bc75edde6147f7b65633d31dd3f958e33501c2a82", - "sanitized_output_sha256": "e987624faf1aaf226e4a886a8bc963d4096bd01e5f5a5f3d283c195d15db6c98", + "raw_evidence_sha256": "72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974", + "sanitized_output_sha256": "72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "FOCUS256_BEGIN\nCOMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1\nMODE=READ_ONLY_LOCAL_SOURCE_LOGIC_FOCUS\nCLOUD_CALLS_BY_256=0\nRCLONE_COMMANDS_EXECUTED_BY_256=0\nERROR_REGISTER_CHECK=READ\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nREFERENCE_CHECK=READ\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nAUTHORITY_FRESH_READ=OK\nTARGET_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57\nTARGET_SHA_MATCH=YES\nUNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250\nUNIT_SHA_MATCH=YES\nTARGET_LINE_COUNT=150\nFOCUS_FUNCTION=prepare_rclone\nFOCUS_SPAN=66-69\nFOCUS_BODY_BEGIN=prepare_rclone\nL066=prepare_rclone(){\nL067= local dst=\"$1\"; [[ -f \"$RCLONE_CONFIG_SOURCE\" && ! -L \"$RCLONE_CONFIG_SOURCE\" ]] || die RCLONE_CONFIG_UNSAFE 66\nL068= install -m 0600 \"$RCLONE_CONFIG_SOURCE\" \"$dst\"\nL069=}\nFOCUS_BODY_END=prepare_rclone\nFOCUS_FUNCTION=run_backup\nFOCUS_SPAN=109-150\nFOCUS_BODY_BEGIN=run_backup\nL109=run_backup(){\nL110= local force=\"${1:-0}\" decision line state gen payload outdir now\nL111= mkdir -p \"$STATE_DIR\" \"$EVIDENCE_ROOT\" \"$STAGE_ROOT\" \"$VERIFY_ROOT\" /run/lock; chmod 0700 \"$STATE_ROOT\" \"$STATE_DIR\" \"$EVIDENCE_ROOT\" \"$STAGE_ROOT\" \"$VERIFY_ROOT\"\nL112= exec 9>\"$LOCK\"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\\n'; return 0; }\nL113= local incompletes\nL114= incompletes=\"$(find \"$STATE_DIR\" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(.status!=\"SEALED\")|.generation' 2>/dev/null || true)\"\nL115= [[ \"$(grep -c . <<<\"$incompletes\" || true)\" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68\nL116= line=\"$(scan)\"; printf '%s\\n' \"$line\"; decision=\"$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,\"\",$i);print $i}}' <<<\"$line\")\"\nL117= gen=\"$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$line\")\"\nL118= if [[ \"$decision\" == SKIP_FRESH && \"$force\" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\\n' \"$gen\"; return 0; fi\nL119= if [[ \"$decision\" == BACKUP_REVERIFY ]]; then\nL120= if [[ \"$force\" != 1 ]]; then verify_generation \"$gen\"; return 0; fi\nL121= # Forced refresh deliberately creates a new generation instead of requiring\nL122= # a legacy generation to satisfy the current bundle schema.\nL123= decision=BACKUP_DUE\nL124= fi\nL125= if [[ \"$decision\" == BACKUP_RESUME ]]; then\nL126= state=\"$STATE_DIR/$gen.json\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; [[ -s \"$payload\" && ! -L \"$payload\" ]] || die INCOMPLETE_PAYLOAD_MISSING 68\nL127= \"$BUILDER\" verify --archive \"$payload\" --expected-generation \"$gen\" >/dev/null || die 68\nL128= mark_cloud_write\nL129= \"$TRANSPORT\" backup-delete \"$LOGICAL_ID\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"; verify_generation \"$gen\"; printf 'CONTROL_BACKUP_RESULT=SEALED_RESUMED GENERATION=%s\\n' \"$gen\"; return 0\nL130= fi\nL131= case \"$decision\" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die \"DECISION_INVALID_$decision\" 65;; esac\nL132= exec 8>\"$HEAVY_LOCK\"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\\n'; return 0; }\nL133= now=\"$(date -u +%Y%m%dT%H%M%SZ)\"; gen=\"control-bootstrap-v2-pve01-$now\"; outdir=\"$STAGE_ROOT/$gen\"\nL134= \"$BUILDER\" build --generation \"$gen\" --output-dir \"$outdir\" >/dev/null\nL135= payload=\"$outdir/$gen.tar.gz\"; [[ -s \"$payload\" && ! -L \"$payload\" ]] || die BUILT_PAYLOAD_MISSING 68\nL136= \"$BUILDER\" verify --archive \"$payload\" --expected-generation \"$gen\" >/dev/null || die 68\nL137= exec 8>&-\nL138= mark_cloud_write\nL139= \"$TRANSPORT\" backup-delete \"$LOGICAL_ID\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nL140= rm -rf --one-file-system \"$outdir\" 2>/dev/null || true\nL141= verify_generation \"$gen\"\nL142= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\\n' \"$gen\"\nL143=}\nL144=case \"${1:-}\" in\nL145= scan) [[ \"$#\" == 1 ]] || die USAGE 64; scan;;\nL146= run) [[ \"$#\" == 1 ]] || die USAGE 64; run_backup 0;;\nL147= force) [[ \"$#\" == 1 ]] || die USAGE 64; run_backup 1;;\nL148= verify) [[ \"$#\" == 2 ]] || die USAGE 64; verify_generation \"$2\";;\nL149= *) die USAGE_scan_run_force_verify 64;;\nL150=esac\nFOCUS_BODY_END=run_backup\nFOCUS_FUNCTION=verify_generation\nFOCUS_SPAN=87-108\nFOCUS_BODY_BEGIN=verify_generation\nL087=verify_generation(){\nL088= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now\nL089= gen=\"$1\"\nL090= state=\"$STATE_DIR/$gen.json\"\nL091= [[ -s \"$state\" && ! -L \"$state\" ]] || die VERIFY_STATE_UNSAFE 68\nL092= \"$TRANSPORT\" verify-sealed \"$LOGICAL_ID\" \"$gen\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\" >/dev/null\nL093= mkdir -p \"$VERIFY_ROOT\"; chmod 0700 \"$VERIFY_ROOT\"\nL094= work=\"$(mktemp -d \"$VERIFY_ROOT/.verify.XXXXXX\")\"; cfg=\"$work/[SENSITIVE_FILE]\"; prepare_rclone \"$cfg\"\nL095= cleanup_verify(){ rm -rf --one-file-system \"$work\"; }; trap cleanup_verify EXIT\nL096= base_a=\"$(jq -er '.remote_a_base' \"$state\")\"; base_b=\"$(jq -er '.remote_b_base' \"$state\")\"; a=\"$work/A.tar.gz\"; b=\"$work/B.tar.gz\"\nL097= reconstruct \"$base_a\" \"$a\" \"$cfg\" \"$state\" A; reconstruct \"$base_b\" \"$b\" \"$cfg\" \"$state\" B\nL098= cmp -s \"$a\" \"$b\" || die CLOUD_A_B_PAYLOAD_MISMATCH 68\nL099= verify_json=\"$(\"$BUILDER\" verify --archive \"$a\" --expected-generation \"$gen\")\" || die 68\nL100=\nL101= archive_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; mkdir -p \"$EVIDENCE_ROOT\" /var/lib/homelab-health; chmod 0700 \"$EVIDENCE_ROOT\"\nL102= tmp=\"$EVIDENCE_ROOT/.$gen.new.$$\"; now=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nL103=\nL104= chmod 0600 \"$tmp\"; mv -f \"$tmp\" \"$EVIDENCE_ROOT/$gen.json\"\nL105= tmp=\"${HEALTH}.new.$$\"; jq -n --arg gen \"$gen\" --arg state \"$state\" --arg evidence \"$EVIDENCE_ROOT/$gen.json\" --arg sha \"$archive_sha\" --arg now \"$now\" '{schema:\"homelab-control-backup-health-v1\",status:\"OK\",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshness:\"FRESH\",full_site_dr_proven:false,updated_at_utc:$now}' >\"$tmp\"; chmod 0644 \"$tmp\"; mv -f \"$tmp\" \"$HEALTH\"\nL106= trap - EXIT; cleanup_verify\nL107= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\\n' \"$gen\" \"$EVIDENCE_ROOT/$gen.json\"\nL108=}\nFOCUS_BODY_END=verify_generation\nFOCUS256_STOP=FOCUS_FUNCTION_MISSING\nFOCUS256_DETAIL=cleanup_verify\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "DIAG258_BEGIN\nTARGET_COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1\nTARGET_ATTEMPT_DIR=/root/homelab-run-SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1\nTARGET_ATTEMPT_DIR_EXISTS=NO\nMATCHING_257_ATTEMPT_DIR_COUNT=0\nEVIDENCE_LAYER=NO_257_SERVER_ATTEMPT_DIRECTORY\nINTERPRETATION=257_DID_NOT_REACH_RUNNER_RESERVATION\nFILES_WRITTEN_BY_DIAG258=0\nRCLONE_COMMANDS_BY_DIAG258=0\nCLOUD_MUTATIONS_BY_DIAG258=0\nDIAG258_END\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 97cf9111..271e21d5 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1 -STATUS=FAIL -RC=2 +COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-DIAG-258R1 +STATUS=OK +RC=0 HOST=pve01 MODE=read-only -COMPONENT=infra-dr-backupv2-control-backup-logic-focus +COMPONENT=infra-dr-backupv2-control-backup-logic-focus-diagnostic REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=9980bab4ad121c27772e38557dd65e4015aa4f3f718e5fcdc6e8dff43be4cd03 +COMMAND_SHA256=f3ab0a154eda3d947c6b078120e2088b7d0721f992b74d9da8e78af6568708ca DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=2 +COMMAND_RC=0 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,107 +24,20 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=cb86b6a96cfd10465cf8955bc75edde6147f7b65633d31dd3f958e33501c2a82 -SANITIZED_OUTPUT_SHA256=e987624faf1aaf226e4a886a8bc963d4096bd01e5f5a5f3d283c195d15db6c98 +RAW_EVIDENCE_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974 +SANITIZED_OUTPUT_SHA256=72357b45b8350f455d8c94c8defca9c8c33a1fd92d1aa4b6db2c1e6bc00fc974 OUTPUT_BEGIN -FOCUS256_BEGIN -COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1 -MODE=READ_ONLY_LOCAL_SOURCE_LOGIC_FOCUS -CLOUD_CALLS_BY_256=0 -RCLONE_COMMANDS_EXECUTED_BY_256=0 -ERROR_REGISTER_CHECK=READ -ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -REFERENCE_CHECK=READ -REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 -AUTHORITY_FRESH_READ=OK -TARGET_SHA256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57 -TARGET_SHA_MATCH=YES -UNIT_SHA256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250 -UNIT_SHA_MATCH=YES -TARGET_LINE_COUNT=150 -FOCUS_FUNCTION=prepare_rclone -FOCUS_SPAN=66-69 -FOCUS_BODY_BEGIN=prepare_rclone -L066=prepare_rclone(){ -L067= local dst="$1"; [[ -f "$RCLONE_CONFIG_SOURCE" && ! -L "$RCLONE_CONFIG_SOURCE" ]] || die RCLONE_CONFIG_UNSAFE 66 -L068= install -m 0600 "$RCLONE_CONFIG_SOURCE" "$dst" -L069=} -FOCUS_BODY_END=prepare_rclone -FOCUS_FUNCTION=run_backup -FOCUS_SPAN=109-150 -FOCUS_BODY_BEGIN=run_backup -L109=run_backup(){ -L110= local force="${1:-0}" decision line state gen payload outdir now -L111= mkdir -p "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" /run/lock; chmod 0700 "$STATE_ROOT" "$STATE_DIR" "$EVIDENCE_ROOT" "$STAGE_ROOT" "$VERIFY_ROOT" -L112= exec 9>"$LOCK"; flock -n 9 || { printf 'CONTROL_BACKUP_DEFERRED=LOCK_BUSY\n'; return 0; } -L113= local incompletes -L114= incompletes="$(find "$STATE_DIR" -maxdepth 1 -type f -name 'control-bootstrap-v2-pve01-????????T??????Z.json' -print0 2>/dev/null | xargs -0 -r jq -r 'select(.status!="SEALED")|.generation' 2>/dev/null || true)" -L115= [[ "$(grep -c . <<<"$incompletes" || true)" -le 1 ]] || die MULTIPLE_INCOMPLETE_GENERATIONS 68 -L116= line="$(scan)"; printf '%s\n' "$line"; decision="$(awk '{for(i=1;i<=NF;i++)if($i~/^DECISION=/){sub(/^DECISION=/,"",$i);print $i}}' <<<"$line")" -L117= gen="$(awk '{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$line")" -L118= if [[ "$decision" == SKIP_FRESH && "$force" != 1 ]]; then printf 'CONTROL_BACKUP_RESULT=SKIP_FRESH GENERATION=%s\n' "$gen"; return 0; fi -L119= if [[ "$decision" == BACKUP_REVERIFY ]]; then -L120= if [[ "$force" != 1 ]]; then verify_generation "$gen"; return 0; fi -L121= # Forced refresh deliberately creates a new generation instead of requiring -L122= # a legacy generation to satisfy the current bundle schema. -L123= decision=BACKUP_DUE -L124= fi -L125= if [[ "$decision" == BACKUP_RESUME ]]; then -L126= state="$STATE_DIR/$gen.json"; payload="$(jq -er '.payload_path' "$state")"; [[ -s "$payload" && ! -L "$payload" ]] || die INCOMPLETE_PAYLOAD_MISSING 68 -L127= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die 68 -L128= mark_cloud_write -L129= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"; verify_generation "$gen"; printf 'CONTROL_BACKUP_RESULT=SEALED_RESUMED GENERATION=%s\n' "$gen"; return 0 -L130= fi -L131= case "$decision" in BACKUP_DUE|BACKUP_STALE|BACKUP_UNVERIFIED|SKIP_FRESH) ;; *) die "DECISION_INVALID_$decision" 65;; esac -L132= exec 8>"$HEAVY_LOCK"; flock -n 8 || { printf 'CONTROL_BACKUP_DEFERRED=HEAVY_LOCK_BUSY\n'; return 0; } -L133= now="$(date -u +%Y%m%dT%H%M%SZ)"; gen="control-bootstrap-v2-pve01-$now"; outdir="$STAGE_ROOT/$gen" -L134= "$BUILDER" build --generation "$gen" --output-dir "$outdir" >/dev/null -L135= payload="$outdir/$gen.tar.gz"; [[ -s "$payload" && ! -L "$payload" ]] || die BUILT_PAYLOAD_MISSING 68 -L136= "$BUILDER" verify --archive "$payload" --expected-generation "$gen" >/dev/null || die 68 -L137= exec 8>&- -L138= mark_cloud_write -L139= "$TRANSPORT" backup-delete "$LOGICAL_ID" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -L140= rm -rf --one-file-system "$outdir" 2>/dev/null || true -L141= verify_generation "$gen" -L142= printf 'CONTROL_BACKUP_RESULT=SEALED_NEW GENERATION=%s\n' "$gen" -L143=} -L144=case "${1:-}" in -L145= scan) [[ "$#" == 1 ]] || die USAGE 64; scan;; -L146= run) [[ "$#" == 1 ]] || die USAGE 64; run_backup 0;; -L147= force) [[ "$#" == 1 ]] || die USAGE 64; run_backup 1;; -L148= verify) [[ "$#" == 2 ]] || die USAGE 64; verify_generation "$2";; -L149= *) die USAGE_scan_run_force_verify 64;; -L150=esac -FOCUS_BODY_END=run_backup -FOCUS_FUNCTION=verify_generation -FOCUS_SPAN=87-108 -FOCUS_BODY_BEGIN=verify_generation -L087=verify_generation(){ -L088= local gen state work cfg base_a base_b a b verify_json archive_sha tmp now -L089= gen="$1" -L090= state="$STATE_DIR/$gen.json" -L091= [[ -s "$state" && ! -L "$state" ]] || die VERIFY_STATE_UNSAFE 68 -L092= "$TRANSPORT" verify-sealed "$LOGICAL_ID" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null -L093= mkdir -p "$VERIFY_ROOT"; chmod 0700 "$VERIFY_ROOT" -L094= work="$(mktemp -d "$VERIFY_ROOT/.verify.XXXXXX")"; cfg="$work/[SENSITIVE_FILE]"; prepare_rclone "$cfg" -L095= cleanup_verify(){ rm -rf --one-file-system "$work"; }; trap cleanup_verify EXIT -L096= base_a="$(jq -er '.remote_a_base' "$state")"; base_b="$(jq -er '.remote_b_base' "$state")"; a="$work/A.tar.gz"; b="$work/B.tar.gz" -L097= reconstruct "$base_a" "$a" "$cfg" "$state" A; reconstruct "$base_b" "$b" "$cfg" "$state" B -L098= cmp -s "$a" "$b" || die CLOUD_A_B_PAYLOAD_MISMATCH 68 -L099= verify_json="$("$BUILDER" verify --archive "$a" --expected-generation "$gen")" || die 68 -L100= -L101= archive_sha="$(jq -er '.archive_sha256' "$state")"; mkdir -p "$EVIDENCE_ROOT" /var/lib/homelab-health; chmod 0700 "$EVIDENCE_ROOT" -L102= tmp="$EVIDENCE_ROOT/.$gen.new.$$"; now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -L103= -L104= chmod 0600 "$tmp"; mv -f "$tmp" "$EVIDENCE_ROOT/$gen.json" -L105= tmp="${HEALTH}.new.$$"; jq -n --arg gen "$gen" --arg state "$state" --arg evidence "$EVIDENCE_ROOT/$gen.json" --arg sha "$archive_sha" --arg now "$now" '{schema:"homelab-control-backup-health-v1",status:"OK",generation:$gen,state_file:$state,restore_evidence:$evidence,archive_sha256:$sha,cloud_quorum:2,control_freshness:"FRESH",full_site_dr_proven:false,updated_at_utc:$now}' >"$tmp"; chmod 0644 "$tmp"; mv -f "$tmp" "$HEALTH" -L106= trap - EXIT; cleanup_verify -L107= printf 'CONTROL_VERIFY=OK GENERATION=%s EVIDENCE=%s\n' "$gen" "$EVIDENCE_ROOT/$gen.json" -L108=} -FOCUS_BODY_END=verify_generation -FOCUS256_STOP=FOCUS_FUNCTION_MISSING -FOCUS256_DETAIL=cleanup_verify -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-256R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} +DIAG258_BEGIN +TARGET_COMMAND_ID=SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1 +TARGET_ATTEMPT_DIR=/root/homelab-run-SUPPORT-260912-INFRA-DR-BACKUPV2-CONTROL-BACKUP-LOGIC-FOCUS-257R1 +TARGET_ATTEMPT_DIR_EXISTS=NO +MATCHING_257_ATTEMPT_DIR_COUNT=0 +EVIDENCE_LAYER=NO_257_SERVER_ATTEMPT_DIRECTORY +INTERPRETATION=257_DID_NOT_REACH_RUNNER_RESERVATION +FILES_WRITTEN_BY_DIAG258=0 +RCLONE_COMMANDS_BY_DIAG258=0 +CLOUD_MUTATIONS_BY_DIAG258=0 +DIAG258_END OUTPUT_END CHAT_OUTPUT_END