homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity
Files
4c218a59ec0fcb4650307cb21fbd8beb63d05e01
homelab-public-context/runtime/latest.txt
T

424 lines
28 KiB
Plaintext
Raw Blame History

CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-1116-contract-pipe-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=0ba8e65003a7561a9e01930aa9eec5d2b6a6cd73c9f05d217fca4a399c992fed
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=8314f5500f36a8ab9866f3f778ae569bf18655d73ade7aca6ada77060826a746
SANITIZED_OUTPUT_SHA256=1c8576db6611dd527a417a275078b31f952af42ddef0dd6dd0fa1661455bb847
OUTPUT_BEGIN
WORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1
MODE=read-only
COMPONENT=homelab-backup-1116-contract-pipe-rca
MUTATION_BOUNDARY=NONE;READ_ONLY_APP_RESTORE_CONTRACT_AND_US_NETBIRD_PIPE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP_TARGET_EXECUTION;NO_RESTORE_EXECUTION;NO_GIT_WRITE;NO_CLOUD_WRITE;REMOTE_PROBES_READ_ONLY
ERROR_REGISTER_CHECK=OK
REFERENCE_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
LEGACY_SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7
APP_WORKER_SHA256=1d6c4a04f762e2a33561c109b0c0a4878caaa48627903e7e78fdf5f22d79f196
SCHEDULER_RC=0
SELFTEST_REPLACEMENT486=PASS
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
APP_RESTORE_CONTRACT_SOURCE_BEGIN=1
89-}
90-latest_sealed_state(){
91- local app="$1" d
92- d="$(state_dir "$app")"; [[ -d "$d" ]] || return 0
93- find "$d" -maxdepth 1 -type f -name '*.json' -printf '%T@|%p\n' | LC_ALL=C sort -t'|' -k1,1nr | while IFS='|' read -r _ f; do
94- if jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$f" >/dev/null 2>&1; then echo "$f"; break; fi
95- done
96-}
97:restore_ok(){
98- local app="$1" gen="$2" f
99- f="$(restore_dir "$app")/$gen.json"
100- [[ -s "$f" && ! -L "$f" ]] || return 1
101: jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
102-}
103-remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
104-remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
105-rsync_core(){
106- local src="$1" dst="$2"
107- mkdir -p "$dst"
108- RSYNC_RSH='ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] rsync -a --numeric-ids --delete --info=progress2 --rsync-path='sudo -n rsync' "$CORE:$src/" "$dst/"
109-}
--
344- [[ -f "$ex/$path" && ! -L "$ex/$path" ]] || die RESTORE_COMPONENT_MISSING 70
345- actual="$(sha256sum "$ex/$path"|awk '{print $1}')"; [[ "$actual" == "$sha" ]] || die RESTORE_COMPONENT_SHA_MISMATCH 70
346- done < <(jq -r '.components|to_entries[]|[.value.path,.value.sha256]|@tsv' "$ex/app-generation.json")
347-}
348-reconstruct_cloud_a(){
349- local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
350- logical="$(logical_id "$app")"
351- "$TRANSPORT" verify-sealed "$logical" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null
352: base="$(jq -er '.remote_a_base' "$state")"; expected_msha="$(jq -er '.parts_manifest_sha256' "$state")"
353- manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
354- got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
355- : >"$out"; chmod 0600 "$out"
356- while IFS=$'\t' read -r idx name size sha; do
357- [[ "$idx" =~ ^[0-9]{8}$ && "$name" == part-????????.bin && "$size" =~ ^[1-9][0-9]*$ && "$sha" =~ ^[0-9a-f]{64}$ ]] || die RESTORE_A_MANIFEST_ROW_INVALID 70
358- part="$(dirname "$out")/$name"; rm -f "$part"; rclone cat "${base%/}/payload.parts/$name" >"$part"
359- [[ "$(stat -c %s "$part")" == "$size" ]] || die RESTORE_A_PART_SIZE_MISMATCH 70
360- actual="$(sha256sum "$part"|awk '{print $1}')"; [[ "$actual" == "$sha" ]] || die RESTORE_A_PART_SHA_MISMATCH 70
--
505- scp "${SCP_BASE[@]}" "$ex/immich/db/immich.dump" "$CORE:$remote_dump" >/dev/null
506- scp "${SCP_BASE[@]}" "$ACTIVE_STAGE/postgres-image.txt" "$CORE:$remote_img" >/dev/null
507- if ! ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- validate-immich-db "$gen" "$remote_dump" "$expect" "$remote_img" <"$CORE_HELPER" >/dev/null; then ssh "${SSH_BASE[@]}" "$CORE" rm -f -- "$remote_dump" "$remote_img" >/dev/null 2>&1 || true; die IMMICH_POSTGRES_IMPORT_FAILED 70; fi
508- ssh "${SSH_BASE[@]}" "$CORE" rm -f -- "$remote_dump" "$remote_img" >/dev/null
509- checks="$(jq -n --argjson n "$expect" '[{name:"postgres_import",result:"OK",table_count:$n},{name:"media_manifest",result:"OK"},{name:"config_manifest",result:"OK"}]')" ;;
510- esac
511- printf '%s\n' "$checks"
512-}
513:restore_validate(){
514- local app="$1" gen="$2" logical state work payload ex expected_sha checks out tmp
515- valid_generation "$gen" || die GENERATION_INVALID 64
516- logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
517- [[ -s "$state" && ! -L "$state" ]] || die RESTORE_STATE_MISSING 65
518- jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null || die RESTORE_REQUIRES_SEALED 65
519: if restore_ok "$app" "$gen"; then printf 'RESTORE_VALIDATION_STATUS=OK\nAPP=%s\nGENERATION=%s\nRESULT=ALREADY_VALIDATED\n' "$app" "$gen"; return 0; fi
520- work="$RESTORE_WORK_ROOT/$logical/$gen"; rm -rf --one-file-system "$work" 2>/dev/null || true; mkdir -p "$work/extract"; chmod 0700 "$RESTORE_WORK_ROOT" "$RESTORE_WORK_ROOT/$logical" "$work" 2>/dev/null || true; ACTIVE_STAGE="$work"
521- payload="$work/payload.tar.zst"; reconstruct_cloud_a "$app" "$gen" "$state" "$payload"
522- expected_sha="$(jq -er '.archive_sha256' "$state")"; [[ "$(sha256sum "$payload"|awk '{print $1}')" == "$expected_sha" ]] || die RESTORE_PAYLOAD_SHA_MISMATCH 70
523- zstd -t "$payload" >/dev/null; zstd -dc "$payload" | tar -xf - -C "$work/extract"; ex="$work/extract"
524- verify_binding_components "$ex" "$logical" "$gen"
525- checks="$(semantic_validate "$app" "$gen" "$ex")"
526- out="$(restore_dir "$app")/$gen.json"; mkdir -p "$(dirname "$out")"; chmod 0700 "$RESTORE_STATE_ROOT" "$(dirname "$out")"
527- tmp="${out}.new.$$"
528: jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
529- chmod 0600 "$tmp"; mv -f "$tmp" "$out"
530: jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
531- rm -rf --one-file-system "$work"; ACTIVE_STAGE=""
532- printf 'RESTORE_VALIDATION_STATUS=OK\nAPP=%s\nGENERATION=%s\nEVIDENCE=%s\n' "$app" "$gen" "$out"
533-}
534-backup_one(){
535- local app="$1" logical pending_count state gen payload latest orphan_count
536- logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
537- mapfile -t pending < <(latest_incomplete "$app"); pending_count="${#pending[@]}"; (( pending_count <= 1 )) || die "MULTIPLE_INCOMPLETE_GENERATIONS_$app" 65
538- if (( pending_count == 1 )); then
539- state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
540- "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
541: printf 'STAGE=RESTORE_VALIDATION\n'; restore_validate "$app" "$gen"
542- printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return
543- fi
544- plan "$app" >/dev/null
545- latest="$(latest_sealed_state "$app")"
546: if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
547- if [[ "$app" == scrutiny ]]; then
548- orphan_count="$(find "$STAGE_ROOT/$logical" -maxdepth 1 -type d -name '.capture-app-scrutiny-*' | wc -l)"
549- (( orphan_count == 0 )) || die SCRUTINY_ORPHAN_CAPTURE_REQUIRES_CONTROLLED_RECOVERY 65
550- fi
551- gen="${logical}-pve01-$(date -u +%Y%m%dT%H%M%SZ)"; valid_generation "$gen" || die GENERATED_ID_INVALID 65
552- printf 'RESUME=NO\nAPP=%s\nGENERATION=%s\nSTAGE=CAPTURE\n' "$app" "$gen"; capture_new "$app" "$gen"; payload="$PAYLOAD"
553- printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
554: printf 'STAGE=RESTORE_VALIDATION\n'; restore_validate "$app" "$gen"
555- state="$STATE_ROOT/$logical/$gen.json"; jq -e '.status=="SEALED" and .cloud_quorum==2 and .local_payload_present==false and .seal_written==true' "$state" >/dev/null
556- printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"
557-}
558-plan(){
559- local app="$1" logical free
560- logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
561- [[ "$free" =~ ^[1-9][0-9]+$ ]] || die STAGING_FREE_INVALID 65
562- local source_bytes=0 required=0
--
589- ;;
590- skladchik-reports-monitor) remote_edge probe plan >/dev/null ;;
591- esac
592- printf 'PLAN_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nSOURCE_HOST=%s\nSTAGING_FREE_BYTES=%s\nSOURCE_BYTES=%s\nREQUIRED_STAGING_BYTES=%s\n' "$app" "$logical" "$(source_host "$app")" "$free" "$source_bytes" "$required"
593-}
594-latest_sealed(){
595- local app="$1" f gen
596- f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
597: restore_ok "$app" "$gen" || die "LATEST_RESTORE_NOT_OK_$app" 65
598: printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
599-}
600-recover_scrutiny_orphan(){
601- local gen="$1" started="$2" qstart="$3" qend="$4" expected_config="$5" expected_influx="$6"
602- local completed unitsha ref iid digest repodigests isize scdb idb devices migrations settings influx_tables tsm_files tsm_bytes wal_files wal_bytes series trange cfgsha boltbytes sqlitebytes image_sha meta_sha config_manifest_sha influx_manifest_sha validation_manifest_sha unit_copy_sha comps report actual_config actual_influx logical state restore
603- logical=app-scrutiny
604- valid_generation "$gen" || die GENERATION_INVALID 64
605- [[ "$gen" == app-scrutiny-pve01-* ]] || die SCRUTINY_RECOVERY_GENERATION_INVALID 64
606- [[ "$started" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && "$qstart" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && "$qend" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die SCRUTINY_RECOVERY_TIMESTAMP_INVALID 64
--
639- if [[ -s "$state" && ! -L "$state" ]] && jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null 2>&1; then
640- printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=scrutiny\nGENERATION=%s\n' "$gen"
641- else
642- [[ -s "$PAYLOAD" && ! -L "$PAYLOAD" ]] || die SCRUTINY_RECOVERY_PAYLOAD_MISSING_FOR_TRANSPORT 65
643- printf 'STAGE=TRANSPORT_RESUME\n'
644- "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
645- fi
646- printf 'STAGE=RESTORE_VALIDATION\n'
647: restore_validate scrutiny "$gen"
648- [[ -s "$state" && ! -L "$state" ]] || die SCRUTINY_RECOVERY_SEALED_STATE_MISSING 70
649- jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null || die SCRUTINY_RECOVERY_SEALED_STATE_INVALID 70
650: [[ -s "$restore" && ! -L "$restore" ]] || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_MISSING 70
651: jq -e '.result=="OK" and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true' "$restore" >/dev/null || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_INVALID 70
652- printf 'BACKUP_STATUS=SEALED\nAPP=scrutiny\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$gen"
653-}
654-
655-ACTION="${1:-}"
656-case "$ACTION" in
657- plan) [[ "$#" == 2 ]] || die USAGE_PLAN 64; plan "$2" ;;
658- backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
659: restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
660- recover-scrutiny-orphan) [[ "$#" == 7 ]] || die USAGE_RECOVER_SCRUTINY_ORPHAN 64; recover_scrutiny_orphan "$2" "$3" "$4" "$5" "$6" "$7" ;;
661: latest-sealed) [[ "$#" == 2 ]] || die USAGE_LATEST_SEALED 64; latest_sealed "$2" ;;
662- *) die ACTION_INVALID 64 ;;
663-esac
APP_RESTORE_CONTRACT_SOURCE_END=1
APP_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json
APP_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc
APP_GENERATION=app-skladchik-reports-monitor-pve01-20260921T175029Z
APP_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json
APP_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e
APP_RESTORE_KEYS_BEGIN=1
["app","checks","generation","logical_id","payload_sha256","payload_sha256_verified","restore_source","result","schema","transport_state","transport_state_file","validated_at_utc"]
APP_RESTORE_KEYS_END=1
APP_RESTORE_SAFE_VIEW_BEGIN=1
{"schema":"homelab-app-restore-validation-v1","result":"OK","status":null,"logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","transport_state":"SEALED","transport_state_file":"/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json","restore_source":"Cloud A","payload_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","payload_sha256_verified":true,"payload_decompression_verified":null,"archive_recovery_readback_verified":null,"parts_manifest_verified":null,"bounded_staging":null,"full_local_payload_written":null,"tar_test_ok":null,"required_files_ok":null,"cookie_jar_ok":null,"sqlite_ok":null,"validated_at_utc":"2026-09-21T17:50:46Z"}
APP_RESTORE_SAFE_VIEW_END=1
APP_RESTORE_BASE_CONTRACT_OK=true
US_NETBIRD_STATE_BEGIN=1
US_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26
{"consecutive_failures":14,"failure_backoff_seconds":21600,"generation":"v2-us-netbird-20260922T055316Z-d5c92acdd2e8","last_attempt_epoch":1790056411,"last_success_epoch":1789704670,"last_success_generation":"v2-us-netbird-20260918T041032Z-d7648be4c16b","last_success_green_verified":false,"last_success_manifest_sha256":"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5","last_success_payload_sha256":"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf","last_success_replicas":[{"crypt":"pve01-mail-05-crypt","physical":"pve01-mail-05"},{"crypt":"pve03-mail-04-crypt","physical":"pve03-mail-04"}],"logical_id":"us-netbird","reason":"PIPE_FAILED","result":"FAIL","retry_after_epoch":1790078011,"schema":1,"sealed":false,"updated_at_utc":"2026-09-22T05:53:31Z"}
US_NETBIRD_STATE_END=1
PIPE_TO_ZSTD_SOURCE_BEGIN=1
if not c: return None
c.sort(key=lambda x:(x[0],x[1],x[2]))
return c[0][3]
def run(argv, *, input=None, timeout=300, check=True, env=None):
try:
p=subprocess.run(argv,input=input,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)
except subprocess.TimeoutExpired: raise BackupError('COMMAND_TIMEOUT')
if check and p.returncode!=0: raise BackupError('COMMAND_FAILED',str(p.returncode))
return p
def stream_process_to_file(argv,out_path,timeout=1800,input=None):
out_path=pathlib.Path(out_path)
with open(out_path,'wb') as f:
try:p=subprocess.run(argv,input=input,stdout=f,stderr=subprocess.PIPE,timeout=timeout)
except subprocess.TimeoutExpired: raise BackupError('STREAM_COMMAND_TIMEOUT')
if p.returncode!=0: raise BackupError('STREAM_COMMAND_FAILED',str(p.returncode))
def pipe_to_zstd(argv,out_path,timeout=1800,input=None):
zstd=shutil.which('zstd') or '/usr/bin/zstd'
try:
src=subprocess.Popen(argv,stdin=subprocess.PIPE if input is not None else None,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
z=subprocess.Popen([zstd,'-T0','-3','-q','-o',str(out_path)],stdin=src.stdout,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE)
if src.stdout: src.stdout.close()
if input is not None:
src.stdin.write(input); src.stdin.close()
zerr=z.communicate(timeout=timeout)[1]
try: src.wait(timeout=60)
except subprocess.TimeoutExpired:
src.kill(); src.wait(); raise BackupError('PIPE_SOURCE_TIMEOUT')
serr=src.stderr.read() if src.stderr else b''
except subprocess.TimeoutExpired:
for q in (src,z):
try:q.kill()
except Exception:pass
raise BackupError('PIPE_TIMEOUT')
if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}')
def ensure_dir(p,mode=0o700):
p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p
def atomic_write(path,data,mode=0o600):
path=pathlib.Path(path); ensure_dir(path.parent,0o700)
fd,tmp=tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent))
PIPE_TO_ZSTD_SOURCE_END=1
US_REMOTE_SCRIPT_SOURCE_BEGIN=1
if pre!=post: raise BackupError('SPB_SOURCE_CHANGED_DURING_CAPTURE')
return payload,{'generation':gen,'source_class':'vps_spb','consistency':'STATIC_CONFIG_FILESET_PRE_POST_HASH_STABILITY','tls_recovery':'REISSUE_TLS_ON_RESTORE','hashset_sha256':pre},None
US_REMOTE_SCRIPT=r'''import io,os,sqlite3,sys,tarfile,time,json,hashlib
paths=['/opt/netbird/docker-compose.yml','/opt/netbird/config.yaml']
data_root='/var/lib/docker/volumes/netbird_netbird_data/_data'
le_root='/var/lib/docker/volumes/netbird_netbird_traefik_letsencrypt/_data'
dbs=['events.db','store.db','geonames_20260602.db','idp.db']
non=['GeoLite2-City_20260602.mmdb']
out=sys.stdout.buffer
with tarfile.open(fileobj=out,mode='w|') as tf:
for p in paths:
if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False)
for name in non:
p=os.path.join(data_root,name)
if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False)
ac=os.path.join(le_root,'acme.json')
if os.path.isfile(ac):
st=os.stat(ac)
if (st.st_mode & 0o777)!=0o600: raise SystemExit(31)
json.load(open(ac,'r'))
tf.add(ac,arcname=ac.lstrip('/'),recursive=False)
for name in dbs:
srcp=os.path.join(data_root,name)
src=sqlite3.connect('file:'+srcp+'?mode=ro',uri=True)
if src.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(32)
dst=sqlite3.connect(':memory:'); src.backup(dst)
if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33)
b=dst.serialize(); dst.close(); src.close()
ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b))
'''
def capture_vps_us(t,gen,work):
remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT])
argv=vps_ssh_argv(t,use_agent=False)+[remote]
payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200)
return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None
def _simple_shell_assignments(text):
US_REMOTE_SCRIPT_SOURCE_END=1
VPS_SSH_SOURCE_BEGIN=1
def operator_key_fingerprint():
if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING')
p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30)
s=p.stdout.decode('utf-8','replace')
m=re.search(r'(SHA256:[A-Za-z0-9+/]+)',s)
if not m or m.group(1)!=OPERATOR_KEY_FINGERPRINT: raise BackupError('OPERATOR_KEY_FINGERPRINT_MISMATCH')
return m.group(1)
def scan_hostkey_line(host,port,expected):
p=run([SSH_KEYSCAN,'-T','8','-p',str(port),'-t','ed25519',host],timeout=15)
lines=[x.strip() for x in p.stdout.decode().splitlines() if x.strip() and not x.startswith('#')]
good=[]
for line in lines:
f=line.split()
if len(f)<3 or f[1]!='ssh-ed25519': continue
try: blob=base64.b64decode(f[2])
except Exception: continue
fp='SHA256:'+base64.b64encode(hashlib.sha256(blob).digest()).decode().rstrip('=')
if fp==expected: good.append(line)
if len(good)!=1: raise BackupError('HOSTKEY_PIN_MISMATCH')
return good[0]
def ensure_vps_known_hosts(targets):
lines=[]
for t in targets:
if t['type'] not in ('vps_md','vps_spb','vps_us'): continue
lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint']))
atomic_write(VPS_KNOWN_HOSTS,('\n'.join(lines)+'\n').encode(),0o600)
def vps_ssh_argv(t, *, use_agent=False):
a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15']
if use_agent:
a += ['-o','IdentitiesOnly=no']
else:
a += ['-i',str(OPERATOR_KEY),'-o','IdentitiesOnly=yes','-o','IdentityAgent=none']
a += [t['user']+'@'+t['host']]
return a
def vps_run(t, remote_argv, *, input=None, timeout=300, use_agent=False, check=True):
remote=shlex.join([str(x) for x in remote_argv])
return run(vps_ssh_argv(t,use_agent=use_agent)+[remote],input=input,timeout=timeout,check=check)
def bootstrap_vps_keys(targets):
operator_key_fingerprint()
ensure_vps_known_hosts(targets)
pub=run(['ssh-keygen','-y','-f',str(OPERATOR_KEY)],timeout=30).stdout.strip()+b'\n'
results={}
for t in targets:
if t['type'] not in ('vps_md','vps_spb'): continue
probe=vps_run(t,['true'],timeout=30,use_agent=False,check=False)
if probe.returncode==0:
results[t['logical_id']]='ALREADY_OPERATOR_KEY'; continue
if not os.environ.get('SSH_AUTH_SOCK'): raise BackupError('FORWARDED_AGENT_REQUIRED_FOR_BOOTSTRAP')
script=b"umask 077; mkdir -p \"$HOME/.ssh\"; chmod 700 \"$HOME/.ssh\"; touch \"$HOME/.ssh/authorized_keys\"; chmod 600 \"$HOME/.ssh/authorized_keys\"; IFS= read -r k; if grep -qF -- \"$k\" \"$HOME/.ssh/authorized_keys\"; then printf 'ALREADY\\n'; else printf '%s\\n' \"$k\" >> \"$HOME/.ssh/authorized_keys\"; printf 'ADDED\\n'; fi"
p=vps_run(t,['sh','-c',script.decode()],input=pub,timeout=45,use_agent=True)
tag=p.stdout.decode().strip()
if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT')
vps_run(t,['true'],timeout=30,use_agent=False)
results[t['logical_id']]=tag
# US was already operator-key based; prove it agentless too.
us=[x for x in targets if x['type']=='vps_us'][0]
vps_run(us,['true'],timeout=30,use_agent=False)
results[us['logical_id']]='ALREADY_US_OPERATOR_KEY'
return results
def remote_hashset(t,command):
p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False)
return sha256_bytes(p.stdout)
VPS_SSH_SOURCE_END=1
US_NETBIRD_PID_JOURNAL_BEGIN=1
PID_BEGIN=936114
2026-09-21T20:30:36+03:00 pve01 homelab-backup-v2[936114]: SELFTEST_REPLACEMENT486=PASS
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED
PID_END=936114
PID_BEGIN=2121414
2026-09-22T02:36:58+03:00 pve01 homelab-backup-v2[2121414]: SELFTEST_REPLACEMENT486=PASS
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED
PID_END=2121414
PID_BEGIN=3378768
2026-09-22T08:53:16+03:00 pve01 homelab-backup-v2[3378768]: SELFTEST_REPLACEMENT486=PASS
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_TARGET=us-netbird
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_RESULT=FAIL
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_REASON=PIPE_FAILED
PID_END=3378768
US_NETBIRD_PID_JOURNAL_END=1
US_NETBIRD_WORK_ARTIFACTS_BEGIN=1
f|1790056411.7319950600|792|/var/lib/homelab-backup-v2/state/us-netbird.json
f|1789642259.3981707900|1255|/var/lib/homelab-backup-v2/cold-bootstrap/us-netbird.json
d|1790056411.7288067540|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260922T055316Z-d5c92acdd2e8
d|1790033833.7009700260|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T233658Z-709cf01e1256
d|1790011851.2302881580|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T173036Z-1128898e4cc8
d|1789989471.8069593230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T111736Z-c5a5c5e71776
d|1789955807.6619889330|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T015632Z-dc499030a55f
d|1789932571.7940614350|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T192916Z-ca2592a81a30
d|1789908991.7828657500|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T125616Z-8caeb0b6c7b7
d|1789886227.6266376830|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T063652Z-e6c72515a153
d|1789862827.7065704150|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T000652Z-7ecdcfcafc12
d|1789839091.8314245860|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T173116Z-cfc70719f4e9
d|1789822687.5806853640|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T125752Z-a4efa5c07171
d|1789814653.6399387990|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T104358Z-b2232f971dbc
d|1789810393.6722341230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T093258Z-64f5c2972565
d|1789807993.6075344840|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T085258Z-36f858f91de5
US_NETBIRD_WORK_ARTIFACTS_END=1
US_NETBIRD_READONLY_PROBES_BEGIN=1
US_NETBIRD_READONLY_PROBE_RC=41
PROBE_TARGET_FOUND=true
PROBE_TYPE=vps_us
PROBE_SSH_TRUE_EXCEPTION=BackupError
PROBE_SSH_TRUE_CODE=COMMAND_FAILED
PROBE_SSH_TRUE_DETAIL=255
US_NETBIRD_READONLY_PROBE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
US_NETBIRD_READONLY_PROBES_END=1
US_NETBIRD_PIPE_BASELINE_CLASS=SSH_BASELINE_FAILURE
DECISION=PASS_BACKUP_1117_APP_CONTRACT_AND_PIPE_BASELINE_CLASSIFIED
NEXT_GATE=REPAIR_ONLY_BACKUP_US_NETBIRD_AND_AUDIT_CANONICAL_CLOUDB_SOURCE_OFFHOST
TASK_RESULT=PASS_HOMELAB_BACKUP_1116_CONTRACT_PIPE_RCA
CHANGES_MADE_BY_1117=false
PRODUCT_MUTATION_BY_1117=false
VM_MUTATION_BY_1117=false
CLOUD_MUTATION_BY_1117=false
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
WORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_END=1
OUTPUT_END
CHAT_OUTPUT_END
View Git Blame Copy Permalink