CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-1116-contract-pipe-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=0ba8e65003a7561a9e01930aa9eec5d2b6a6cd73c9f05d217fca4a399c992fed
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=8314f5500f36a8ab9866f3f778ae569bf18655d73ade7aca6ada77060826a746
SANITIZED_OUTPUT_SHA256=1c8576db6611dd527a417a275078b31f952af42ddef0dd6dd0fa1661455bb847
OUTPUT_BEGIN
WORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1
MODE=read-only
COMPONENT=homelab-backup-1116-contract-pipe-rca
MUTATION_BOUNDARY=NONE;READ_ONLY_APP_RESTORE_CONTRACT_AND_US_NETBIRD_PIPE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP_TARGET_EXECUTION;NO_RESTORE_EXECUTION;NO_GIT_WRITE;NO_CLOUD_WRITE;REMOTE_PROBES_READ_ONLY
ERROR_REGISTER_CHECK=OK
REFERENCE_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
LEGACY_SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7
APP_WORKER_SHA256=1d6c4a04f762e2a33561c109b0c0a4878caaa48627903e7e78fdf5f22d79f196
SCHEDULER_RC=0
SELFTEST_REPLACEMENT486=PASS
SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET
APP_RESTORE_CONTRACT_SOURCE_BEGIN=1
89-}
90-latest_sealed_state(){
91-  local app="$1" d
92-  d="$(state_dir "$app")"; [[ -d "$d" ]] || return 0
93-  find "$d" -maxdepth 1 -type f -name '*.json' -printf '%T@|%p\n' | LC_ALL=C sort -t'|' -k1,1nr | while IFS='|' read -r _ f; do
94-    if jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$f" >/dev/null 2>&1; then echo "$f"; break; fi
95-  done
96-}
97:restore_ok(){
98-  local app="$1" gen="$2" f
99-  f="$(restore_dir "$app")/$gen.json"
100-  [[ -s "$f" && ! -L "$f" ]] || return 1
101:  jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
102-}
103-remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
104-remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
105-rsync_core(){
106-  local src="$1" dst="$2"
107-  mkdir -p "$dst"
108-  RSYNC_RSH='ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] rsync -a --numeric-ids --delete --info=progress2 --rsync-path='sudo -n rsync' "$CORE:$src/" "$dst/"
109-}
--
344-    [[ -f "$ex/$path" && ! -L "$ex/$path" ]] || die RESTORE_COMPONENT_MISSING 70
345-    actual="$(sha256sum "$ex/$path"|awk '{print $1}')"; [[ "$actual" == "$sha" ]] || die RESTORE_COMPONENT_SHA_MISMATCH 70
346-  done < <(jq -r '.components|to_entries[]|[.value.path,.value.sha256]|@tsv' "$ex/app-generation.json")
347-}
348-reconstruct_cloud_a(){
349-  local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
350-  logical="$(logical_id "$app")"
351-  "$TRANSPORT" verify-sealed "$logical" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null
352:  base="$(jq -er '.remote_a_base' "$state")"; expected_msha="$(jq -er '.parts_manifest_sha256' "$state")"
353-  manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
354-  got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
355-  : >"$out"; chmod 0600 "$out"
356-  while IFS=$'\t' read -r idx name size sha; do
357-    [[ "$idx" =~ ^[0-9]{8}$ && "$name" == part-????????.bin && "$size" =~ ^[1-9][0-9]*$ && "$sha" =~ ^[0-9a-f]{64}$ ]] || die RESTORE_A_MANIFEST_ROW_INVALID 70
358-    part="$(dirname "$out")/$name"; rm -f "$part"; rclone cat "${base%/}/payload.parts/$name" >"$part"
359-    [[ "$(stat -c %s "$part")" == "$size" ]] || die RESTORE_A_PART_SIZE_MISMATCH 70
360-    actual="$(sha256sum "$part"|awk '{print $1}')"; [[ "$actual" == "$sha" ]] || die RESTORE_A_PART_SHA_MISMATCH 70
--
505-      scp "${SCP_BASE[@]}" "$ex/immich/db/immich.dump" "$CORE:$remote_dump" >/dev/null
506-      scp "${SCP_BASE[@]}" "$ACTIVE_STAGE/postgres-image.txt" "$CORE:$remote_img" >/dev/null
507-      if ! ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- validate-immich-db "$gen" "$remote_dump" "$expect" "$remote_img" <"$CORE_HELPER" >/dev/null; then ssh "${SSH_BASE[@]}" "$CORE" rm -f -- "$remote_dump" "$remote_img" >/dev/null 2>&1 || true; die IMMICH_POSTGRES_IMPORT_FAILED 70; fi
508-      ssh "${SSH_BASE[@]}" "$CORE" rm -f -- "$remote_dump" "$remote_img" >/dev/null
509-      checks="$(jq -n --argjson n "$expect" '[{name:"postgres_import",result:"OK",table_count:$n},{name:"media_manifest",result:"OK"},{name:"config_manifest",result:"OK"}]')" ;;
510-  esac
511-  printf '%s\n' "$checks"
512-}
513:restore_validate(){
514-  local app="$1" gen="$2" logical state work payload ex expected_sha checks out tmp
515-  valid_generation "$gen" || die GENERATION_INVALID 64
516-  logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
517-  [[ -s "$state" && ! -L "$state" ]] || die RESTORE_STATE_MISSING 65
518-  jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null || die RESTORE_REQUIRES_SEALED 65
519:  if restore_ok "$app" "$gen"; then printf 'RESTORE_VALIDATION_STATUS=OK\nAPP=%s\nGENERATION=%s\nRESULT=ALREADY_VALIDATED\n' "$app" "$gen"; return 0; fi
520-  work="$RESTORE_WORK_ROOT/$logical/$gen"; rm -rf --one-file-system "$work" 2>/dev/null || true; mkdir -p "$work/extract"; chmod 0700 "$RESTORE_WORK_ROOT" "$RESTORE_WORK_ROOT/$logical" "$work" 2>/dev/null || true; ACTIVE_STAGE="$work"
521-  payload="$work/payload.tar.zst"; reconstruct_cloud_a "$app" "$gen" "$state" "$payload"
522-  expected_sha="$(jq -er '.archive_sha256' "$state")"; [[ "$(sha256sum "$payload"|awk '{print $1}')" == "$expected_sha" ]] || die RESTORE_PAYLOAD_SHA_MISMATCH 70
523-  zstd -t "$payload" >/dev/null; zstd -dc "$payload" | tar -xf - -C "$work/extract"; ex="$work/extract"
524-  verify_binding_components "$ex" "$logical" "$gen"
525-  checks="$(semantic_validate "$app" "$gen" "$ex")"
526-  out="$(restore_dir "$app")/$gen.json"; mkdir -p "$(dirname "$out")"; chmod 0700 "$RESTORE_STATE_ROOT" "$(dirname "$out")"
527-  tmp="${out}.new.$$"
528:  jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
529-  chmod 0600 "$tmp"; mv -f "$tmp" "$out"
530:  jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
531-  rm -rf --one-file-system "$work"; ACTIVE_STAGE=""
532-  printf 'RESTORE_VALIDATION_STATUS=OK\nAPP=%s\nGENERATION=%s\nEVIDENCE=%s\n' "$app" "$gen" "$out"
533-}
534-backup_one(){
535-  local app="$1" logical pending_count state gen payload latest orphan_count
536-  logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
537-  mapfile -t pending < <(latest_incomplete "$app"); pending_count="${#pending[@]}"; (( pending_count <= 1 )) || die "MULTIPLE_INCOMPLETE_GENERATIONS_$app" 65
538-  if (( pending_count == 1 )); then
539-    state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
540-    "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
541:    printf 'STAGE=RESTORE_VALIDATION\n'; restore_validate "$app" "$gen"
542-    printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return
543-  fi
544-  plan "$app" >/dev/null
545-  latest="$(latest_sealed_state "$app")"
546:  if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
547-  if [[ "$app" == scrutiny ]]; then
548-    orphan_count="$(find "$STAGE_ROOT/$logical" -maxdepth 1 -type d -name '.capture-app-scrutiny-*' | wc -l)"
549-    (( orphan_count == 0 )) || die SCRUTINY_ORPHAN_CAPTURE_REQUIRES_CONTROLLED_RECOVERY 65
550-  fi
551-  gen="${logical}-pve01-$(date -u +%Y%m%dT%H%M%SZ)"; valid_generation "$gen" || die GENERATED_ID_INVALID 65
552-  printf 'RESUME=NO\nAPP=%s\nGENERATION=%s\nSTAGE=CAPTURE\n' "$app" "$gen"; capture_new "$app" "$gen"; payload="$PAYLOAD"
553-  printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
554:  printf 'STAGE=RESTORE_VALIDATION\n'; restore_validate "$app" "$gen"
555-  state="$STATE_ROOT/$logical/$gen.json"; jq -e '.status=="SEALED" and .cloud_quorum==2 and .local_payload_present==false and .seal_written==true' "$state" >/dev/null
556-  printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"
557-}
558-plan(){
559-  local app="$1" logical free
560-  logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
561-  [[ "$free" =~ ^[1-9][0-9]+$ ]] || die STAGING_FREE_INVALID 65
562-  local source_bytes=0 required=0
--
589-      ;;
590-    skladchik-reports-monitor) remote_edge probe plan >/dev/null ;;
591-  esac
592-  printf 'PLAN_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nSOURCE_HOST=%s\nSTAGING_FREE_BYTES=%s\nSOURCE_BYTES=%s\nREQUIRED_STAGING_BYTES=%s\n' "$app" "$logical" "$(source_host "$app")" "$free" "$source_bytes" "$required"
593-}
594-latest_sealed(){
595-  local app="$1" f gen
596-  f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
597:  restore_ok "$app" "$gen" || die "LATEST_RESTORE_NOT_OK_$app" 65
598:  printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
599-}
600-recover_scrutiny_orphan(){
601-  local gen="$1" started="$2" qstart="$3" qend="$4" expected_config="$5" expected_influx="$6"
602-  local completed unitsha ref iid digest repodigests isize scdb idb devices migrations settings influx_tables tsm_files tsm_bytes wal_files wal_bytes series trange cfgsha boltbytes sqlitebytes image_sha meta_sha config_manifest_sha influx_manifest_sha validation_manifest_sha unit_copy_sha comps report actual_config actual_influx logical state restore
603-  logical=app-scrutiny
604-  valid_generation "$gen" || die GENERATION_INVALID 64
605-  [[ "$gen" == app-scrutiny-pve01-* ]] || die SCRUTINY_RECOVERY_GENERATION_INVALID 64
606-  [[ "$started" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && "$qstart" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && "$qend" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die SCRUTINY_RECOVERY_TIMESTAMP_INVALID 64
--
639-  if [[ -s "$state" && ! -L "$state" ]] && jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null 2>&1; then
640-    printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=scrutiny\nGENERATION=%s\n' "$gen"
641-  else
642-    [[ -s "$PAYLOAD" && ! -L "$PAYLOAD" ]] || die SCRUTINY_RECOVERY_PAYLOAD_MISSING_FOR_TRANSPORT 65
643-    printf 'STAGE=TRANSPORT_RESUME\n'
644-    "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
645-  fi
646-  printf 'STAGE=RESTORE_VALIDATION\n'
647:  restore_validate scrutiny "$gen"
648-  [[ -s "$state" && ! -L "$state" ]] || die SCRUTINY_RECOVERY_SEALED_STATE_MISSING 70
649-  jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null || die SCRUTINY_RECOVERY_SEALED_STATE_INVALID 70
650:  [[ -s "$restore" && ! -L "$restore" ]] || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_MISSING 70
651:  jq -e '.result=="OK" and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true' "$restore" >/dev/null || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_INVALID 70
652-  printf 'BACKUP_STATUS=SEALED\nAPP=scrutiny\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$gen"
653-}
654-
655-ACTION="${1:-}"
656-case "$ACTION" in
657-  plan) [[ "$#" == 2 ]] || die USAGE_PLAN 64; plan "$2" ;;
658-  backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
659:  restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
660-  recover-scrutiny-orphan) [[ "$#" == 7 ]] || die USAGE_RECOVER_SCRUTINY_ORPHAN 64; recover_scrutiny_orphan "$2" "$3" "$4" "$5" "$6" "$7" ;;
661:  latest-sealed) [[ "$#" == 2 ]] || die USAGE_LATEST_SEALED 64; latest_sealed "$2" ;;
662-  *) die ACTION_INVALID 64 ;;
663-esac
APP_RESTORE_CONTRACT_SOURCE_END=1
APP_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json
APP_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc
APP_GENERATION=app-skladchik-reports-monitor-pve01-20260921T175029Z
APP_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json
APP_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e
APP_RESTORE_KEYS_BEGIN=1
["app","checks","generation","logical_id","payload_sha256","payload_sha256_verified","restore_source","result","schema","transport_state","transport_state_file","validated_at_utc"]
APP_RESTORE_KEYS_END=1
APP_RESTORE_SAFE_VIEW_BEGIN=1
{"schema":"homelab-app-restore-validation-v1","result":"OK","status":null,"logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","transport_state":"SEALED","transport_state_file":"/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json","restore_source":"Cloud A","payload_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","payload_sha256_verified":true,"payload_decompression_verified":null,"archive_recovery_readback_verified":null,"parts_manifest_verified":null,"bounded_staging":null,"full_local_payload_written":null,"tar_test_ok":null,"required_files_ok":null,"cookie_jar_ok":null,"sqlite_ok":null,"validated_at_utc":"2026-09-21T17:50:46Z"}
APP_RESTORE_SAFE_VIEW_END=1
APP_RESTORE_BASE_CONTRACT_OK=true
US_NETBIRD_STATE_BEGIN=1
US_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26
{"consecutive_failures":14,"failure_backoff_seconds":21600,"generation":"v2-us-netbird-20260922T055316Z-d5c92acdd2e8","last_attempt_epoch":1790056411,"last_success_epoch":1789704670,"last_success_generation":"v2-us-netbird-20260918T041032Z-d7648be4c16b","last_success_green_verified":false,"last_success_manifest_sha256":"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5","last_success_payload_sha256":"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf","last_success_replicas":[{"crypt":"pve01-mail-05-crypt","physical":"pve01-mail-05"},{"crypt":"pve03-mail-04-crypt","physical":"pve03-mail-04"}],"logical_id":"us-netbird","reason":"PIPE_FAILED","result":"FAIL","retry_after_epoch":1790078011,"schema":1,"sealed":false,"updated_at_utc":"2026-09-22T05:53:31Z"}
US_NETBIRD_STATE_END=1
PIPE_TO_ZSTD_SOURCE_BEGIN=1
    if not c: return None
    c.sort(key=lambda x:(x[0],x[1],x[2]))
    return c[0][3]

def run(argv, *, input=None, timeout=300, check=True, env=None):
    try:
        p=subprocess.run(argv,input=input,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)
    except subprocess.TimeoutExpired: raise BackupError('COMMAND_TIMEOUT')
    if check and p.returncode!=0: raise BackupError('COMMAND_FAILED',str(p.returncode))
    return p

def stream_process_to_file(argv,out_path,timeout=1800,input=None):
    out_path=pathlib.Path(out_path)
    with open(out_path,'wb') as f:
        try:p=subprocess.run(argv,input=input,stdout=f,stderr=subprocess.PIPE,timeout=timeout)
        except subprocess.TimeoutExpired: raise BackupError('STREAM_COMMAND_TIMEOUT')
    if p.returncode!=0: raise BackupError('STREAM_COMMAND_FAILED',str(p.returncode))

def pipe_to_zstd(argv,out_path,timeout=1800,input=None):
    zstd=shutil.which('zstd') or '/usr/bin/zstd'
    try:
        src=subprocess.Popen(argv,stdin=subprocess.PIPE if input is not None else None,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        z=subprocess.Popen([zstd,'-T0','-3','-q','-o',str(out_path)],stdin=src.stdout,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE)
        if src.stdout: src.stdout.close()
        if input is not None:
            src.stdin.write(input); src.stdin.close()
        zerr=z.communicate(timeout=timeout)[1]
        try: src.wait(timeout=60)
        except subprocess.TimeoutExpired:
            src.kill(); src.wait(); raise BackupError('PIPE_SOURCE_TIMEOUT')
        serr=src.stderr.read() if src.stderr else b''
    except subprocess.TimeoutExpired:
        for q in (src,z):
            try:q.kill()
            except Exception:pass
        raise BackupError('PIPE_TIMEOUT')
    if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}')

def ensure_dir(p,mode=0o700):
    p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p

def atomic_write(path,data,mode=0o600):
    path=pathlib.Path(path); ensure_dir(path.parent,0o700)
    fd,tmp=tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent))
PIPE_TO_ZSTD_SOURCE_END=1
US_REMOTE_SCRIPT_SOURCE_BEGIN=1
    if pre!=post: raise BackupError('SPB_SOURCE_CHANGED_DURING_CAPTURE')
    return payload,{'generation':gen,'source_class':'vps_spb','consistency':'STATIC_CONFIG_FILESET_PRE_POST_HASH_STABILITY','tls_recovery':'REISSUE_TLS_ON_RESTORE','hashset_sha256':pre},None

US_REMOTE_SCRIPT=r'''import io,os,sqlite3,sys,tarfile,time,json,hashlib
paths=['/opt/netbird/docker-compose.yml','/opt/netbird/config.yaml']
data_root='/var/lib/docker/volumes/netbird_netbird_data/_data'
le_root='/var/lib/docker/volumes/netbird_netbird_traefik_letsencrypt/_data'
dbs=['events.db','store.db','geonames_20260602.db','idp.db']
non=['GeoLite2-City_20260602.mmdb']
out=sys.stdout.buffer
with tarfile.open(fileobj=out,mode='w|') as tf:
  for p in paths:
    if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False)
  for name in non:
    p=os.path.join(data_root,name)
    if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False)
  ac=os.path.join(le_root,'acme.json')
  if os.path.isfile(ac):
    st=os.stat(ac)
    if (st.st_mode & 0o777)!=0o600: raise SystemExit(31)
    json.load(open(ac,'r'))
    tf.add(ac,arcname=ac.lstrip('/'),recursive=False)
  for name in dbs:
    srcp=os.path.join(data_root,name)
    src=sqlite3.connect('file:'+srcp+'?mode=ro',uri=True)
    if src.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(32)
    dst=sqlite3.connect(':memory:'); src.backup(dst)
    if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33)
    b=dst.serialize(); dst.close(); src.close()
    ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b))
'''

def capture_vps_us(t,gen,work):
    remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT])
    argv=vps_ssh_argv(t,use_agent=False)+[remote]
    payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200)
    return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None

def _simple_shell_assignments(text):
US_REMOTE_SCRIPT_SOURCE_END=1
VPS_SSH_SOURCE_BEGIN=1
def operator_key_fingerprint():
    if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING')
    p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30)
    s=p.stdout.decode('utf-8','replace')
    m=re.search(r'(SHA256:[A-Za-z0-9+/]+)',s)
    if not m or m.group(1)!=OPERATOR_KEY_FINGERPRINT: raise BackupError('OPERATOR_KEY_FINGERPRINT_MISMATCH')
    return m.group(1)

def scan_hostkey_line(host,port,expected):
    p=run([SSH_KEYSCAN,'-T','8','-p',str(port),'-t','ed25519',host],timeout=15)
    lines=[x.strip() for x in p.stdout.decode().splitlines() if x.strip() and not x.startswith('#')]
    good=[]
    for line in lines:
        f=line.split()
        if len(f)<3 or f[1]!='ssh-ed25519': continue
        try: blob=base64.b64decode(f[2])
        except Exception: continue
        fp='SHA256:'+base64.b64encode(hashlib.sha256(blob).digest()).decode().rstrip('=')
        if fp==expected: good.append(line)
    if len(good)!=1: raise BackupError('HOSTKEY_PIN_MISMATCH')
    return good[0]

def ensure_vps_known_hosts(targets):
    lines=[]
    for t in targets:
        if t['type'] not in ('vps_md','vps_spb','vps_us'): continue
        lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint']))
    atomic_write(VPS_KNOWN_HOSTS,('\n'.join(lines)+'\n').encode(),0o600)

def vps_ssh_argv(t, *, use_agent=False):
    a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15']
    if use_agent:
        a += ['-o','IdentitiesOnly=no']
    else:
        a += ['-i',str(OPERATOR_KEY),'-o','IdentitiesOnly=yes','-o','IdentityAgent=none']
    a += [t['user']+'@'+t['host']]
    return a

def vps_run(t, remote_argv, *, input=None, timeout=300, use_agent=False, check=True):
    remote=shlex.join([str(x) for x in remote_argv])
    return run(vps_ssh_argv(t,use_agent=use_agent)+[remote],input=input,timeout=timeout,check=check)

def bootstrap_vps_keys(targets):
    operator_key_fingerprint()
    ensure_vps_known_hosts(targets)
    pub=run(['ssh-keygen','-y','-f',str(OPERATOR_KEY)],timeout=30).stdout.strip()+b'\n'
    results={}
    for t in targets:
        if t['type'] not in ('vps_md','vps_spb'): continue
        probe=vps_run(t,['true'],timeout=30,use_agent=False,check=False)
        if probe.returncode==0:
            results[t['logical_id']]='ALREADY_OPERATOR_KEY'; continue
        if not os.environ.get('SSH_AUTH_SOCK'): raise BackupError('FORWARDED_AGENT_REQUIRED_FOR_BOOTSTRAP')
        script=b"umask 077; mkdir -p \"$HOME/.ssh\"; chmod 700 \"$HOME/.ssh\"; touch \"$HOME/.ssh/authorized_keys\"; chmod 600 \"$HOME/.ssh/authorized_keys\"; IFS= read -r k; if grep -qF -- \"$k\" \"$HOME/.ssh/authorized_keys\"; then printf 'ALREADY\\n'; else printf '%s\\n' \"$k\" >> \"$HOME/.ssh/authorized_keys\"; printf 'ADDED\\n'; fi"
        p=vps_run(t,['sh','-c',script.decode()],input=pub,timeout=45,use_agent=True)
        tag=p.stdout.decode().strip()
        if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT')
        vps_run(t,['true'],timeout=30,use_agent=False)
        results[t['logical_id']]=tag
    # US was already operator-key based; prove it agentless too.
    us=[x for x in targets if x['type']=='vps_us'][0]
    vps_run(us,['true'],timeout=30,use_agent=False)
    results[us['logical_id']]='ALREADY_US_OPERATOR_KEY'
    return results

def remote_hashset(t,command):
    p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False)
    return sha256_bytes(p.stdout)
VPS_SSH_SOURCE_END=1
US_NETBIRD_PID_JOURNAL_BEGIN=1
PID_BEGIN=936114
2026-09-21T20:30:36+03:00 pve01 homelab-backup-v2[936114]: SELFTEST_REPLACEMENT486=PASS
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL
2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED
PID_END=936114
PID_BEGIN=2121414
2026-09-22T02:36:58+03:00 pve01 homelab-backup-v2[2121414]: SELFTEST_REPLACEMENT486=PASS
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL
2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED
PID_END=2121414
PID_BEGIN=3378768
2026-09-22T08:53:16+03:00 pve01 homelab-backup-v2[3378768]: SELFTEST_REPLACEMENT486=PASS
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_TARGET=us-netbird
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_RESULT=FAIL
2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_REASON=PIPE_FAILED
PID_END=3378768
US_NETBIRD_PID_JOURNAL_END=1
US_NETBIRD_WORK_ARTIFACTS_BEGIN=1
f|1790056411.7319950600|792|/var/lib/homelab-backup-v2/state/us-netbird.json
f|1789642259.3981707900|1255|/var/lib/homelab-backup-v2/cold-bootstrap/us-netbird.json
d|1790056411.7288067540|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260922T055316Z-d5c92acdd2e8
d|1790033833.7009700260|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T233658Z-709cf01e1256
d|1790011851.2302881580|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T173036Z-1128898e4cc8
d|1789989471.8069593230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T111736Z-c5a5c5e71776
d|1789955807.6619889330|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T015632Z-dc499030a55f
d|1789932571.7940614350|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T192916Z-ca2592a81a30
d|1789908991.7828657500|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T125616Z-8caeb0b6c7b7
d|1789886227.6266376830|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T063652Z-e6c72515a153
d|1789862827.7065704150|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T000652Z-7ecdcfcafc12
d|1789839091.8314245860|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T173116Z-cfc70719f4e9
d|1789822687.5806853640|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T125752Z-a4efa5c07171
d|1789814653.6399387990|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T104358Z-b2232f971dbc
d|1789810393.6722341230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T093258Z-64f5c2972565
d|1789807993.6075344840|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T085258Z-36f858f91de5
US_NETBIRD_WORK_ARTIFACTS_END=1
US_NETBIRD_READONLY_PROBES_BEGIN=1
US_NETBIRD_READONLY_PROBE_RC=41
PROBE_TARGET_FOUND=true
PROBE_TYPE=vps_us
PROBE_SSH_TRUE_EXCEPTION=BackupError
PROBE_SSH_TRUE_CODE=COMMAND_FAILED
PROBE_SSH_TRUE_DETAIL=255
US_NETBIRD_READONLY_PROBE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
US_NETBIRD_READONLY_PROBES_END=1
US_NETBIRD_PIPE_BASELINE_CLASS=SSH_BASELINE_FAILURE
DECISION=PASS_BACKUP_1117_APP_CONTRACT_AND_PIPE_BASELINE_CLASSIFIED
NEXT_GATE=REPAIR_ONLY_BACKUP_US_NETBIRD_AND_AUDIT_CANONICAL_CLOUDB_SOURCE_OFFHOST
TASK_RESULT=PASS_HOMELAB_BACKUP_1116_CONTRACT_PIPE_RCA
CHANGES_MADE_BY_1117=false
PRODUCT_MUTATION_BY_1117=false
VM_MUTATION_BY_1117=false
CLOUD_MUTATION_BY_1117=false
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
WORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_END=1

OUTPUT_END
CHAT_OUTPUT_END
