diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1.json b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1.json new file mode 100644 index 00000000..e4af0c69 --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-backup-1116-contract-pipe-rca", + "started_at_utc": "2026-09-22T06:05:29Z", + "finished_at_utc": "2026-09-22T06:05:29Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "0ba8e65003a7561a9e01930aa9eec5d2b6a6cd73c9f05d217fca4a399c992fed", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "8314f5500f36a8ab9866f3f778ae569bf18655d73ade7aca6ada77060826a746", + "sanitized_output_sha256": "1c8576db6611dd527a417a275078b31f952af42ddef0dd6dd0fa1661455bb847", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1\nMODE=read-only\nCOMPONENT=homelab-backup-1116-contract-pipe-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_APP_RESTORE_CONTRACT_AND_US_NETBIRD_PIPE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP_TARGET_EXECUTION;NO_RESTORE_EXECUTION;NO_GIT_WRITE;NO_CLOUD_WRITE;REMOTE_PROBES_READ_ONLY\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nLEGACY_SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7\nAPP_WORKER_SHA256=1d6c4a04f762e2a33561c109b0c0a4878caaa48627903e7e78fdf5f22d79f196\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nAPP_RESTORE_CONTRACT_SOURCE_BEGIN=1\n89-}\n90-latest_sealed_state(){\n91- local app=\"$1\" d\n92- d=\"$(state_dir \"$app\")\"; [[ -d \"$d\" ]] || return 0\n93- find \"$d\" -maxdepth 1 -type f -name '*.json' -printf '%T@|%p\\n' | LC_ALL=C sort -t'|' -k1,1nr | while IFS='|' read -r _ f; do\n94- if jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$f\" >/dev/null 2>&1; then echo \"$f\"; break; fi\n95- done\n96-}\n97:restore_ok(){\n98- local app=\"$1\" gen=\"$2\" f\n99- f=\"$(restore_dir \"$app\")/$gen.json\"\n100- [[ -s \"$f\" && ! -L \"$f\" ]] || return 1\n101: jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\n102-}\n103-remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\n104-remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\n105-rsync_core(){\n106- local src=\"$1\" dst=\"$2\"\n107- mkdir -p \"$dst\"\n108- RSYNC_RSH='ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] rsync -a --numeric-ids --delete --info=progress2 --rsync-path='sudo -n rsync' \"$CORE:$src/\" \"$dst/\"\n109-}\n--\n344- [[ -f \"$ex/$path\" && ! -L \"$ex/$path\" ]] || die RESTORE_COMPONENT_MISSING 70\n345- actual=\"$(sha256sum \"$ex/$path\"|awk '{print $1}')\"; [[ \"$actual\" == \"$sha\" ]] || die RESTORE_COMPONENT_SHA_MISMATCH 70\n346- done < <(jq -r '.components|to_entries[]|[.value.path,.value.sha256]|@tsv' \"$ex/app-generation.json\")\n347-}\n348-reconstruct_cloud_a(){\n349- local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\n350- logical=\"$(logical_id \"$app\")\"\n351- \"$TRANSPORT\" verify-sealed \"$logical\" \"$gen\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\" >/dev/null\n352: base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"\n353- manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\n354- got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\n355- : >\"$out\"; chmod 0600 \"$out\"\n356- while IFS=$'\\t' read -r idx name size sha; do\n357- [[ \"$idx\" =~ ^[0-9]{8}$ && \"$name\" == part-????????.bin && \"$size\" =~ ^[1-9][0-9]*$ && \"$sha\" =~ ^[0-9a-f]{64}$ ]] || die RESTORE_A_MANIFEST_ROW_INVALID 70\n358- part=\"$(dirname \"$out\")/$name\"; rm -f \"$part\"; rclone cat \"${base%/}/payload.parts/$name\" >\"$part\"\n359- [[ \"$(stat -c %s \"$part\")\" == \"$size\" ]] || die RESTORE_A_PART_SIZE_MISMATCH 70\n360- actual=\"$(sha256sum \"$part\"|awk '{print $1}')\"; [[ \"$actual\" == \"$sha\" ]] || die RESTORE_A_PART_SHA_MISMATCH 70\n--\n505- scp \"${SCP_BASE[@]}\" \"$ex/immich/db/immich.dump\" \"$CORE:$remote_dump\" >/dev/null\n506- scp \"${SCP_BASE[@]}\" \"$ACTIVE_STAGE/postgres-image.txt\" \"$CORE:$remote_img\" >/dev/null\n507- if ! ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- validate-immich-db \"$gen\" \"$remote_dump\" \"$expect\" \"$remote_img\" <\"$CORE_HELPER\" >/dev/null; then ssh \"${SSH_BASE[@]}\" \"$CORE\" rm -f -- \"$remote_dump\" \"$remote_img\" >/dev/null 2>&1 || true; die IMMICH_POSTGRES_IMPORT_FAILED 70; fi\n508- ssh \"${SSH_BASE[@]}\" \"$CORE\" rm -f -- \"$remote_dump\" \"$remote_img\" >/dev/null\n509- checks=\"$(jq -n --argjson n \"$expect\" '[{name:\"postgres_import\",result:\"OK\",table_count:$n},{name:\"media_manifest\",result:\"OK\"},{name:\"config_manifest\",result:\"OK\"}]')\" ;;\n510- esac\n511- printf '%s\\n' \"$checks\"\n512-}\n513:restore_validate(){\n514- local app=\"$1\" gen=\"$2\" logical state work payload ex expected_sha checks out tmp\n515- valid_generation \"$gen\" || die GENERATION_INVALID 64\n516- logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\n517- [[ -s \"$state\" && ! -L \"$state\" ]] || die RESTORE_STATE_MISSING 65\n518- jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$state\" >/dev/null || die RESTORE_REQUIRES_SEALED 65\n519: if restore_ok \"$app\" \"$gen\"; then printf 'RESTORE_VALIDATION_STATUS=OK\\nAPP=%s\\nGENERATION=%s\\nRESULT=ALREADY_VALIDATED\\n' \"$app\" \"$gen\"; return 0; fi\n520- work=\"$RESTORE_WORK_ROOT/$logical/$gen\"; rm -rf --one-file-system \"$work\" 2>/dev/null || true; mkdir -p \"$work/extract\"; chmod 0700 \"$RESTORE_WORK_ROOT\" \"$RESTORE_WORK_ROOT/$logical\" \"$work\" 2>/dev/null || true; ACTIVE_STAGE=\"$work\"\n521- payload=\"$work/payload.tar.zst\"; reconstruct_cloud_a \"$app\" \"$gen\" \"$state\" \"$payload\"\n522- expected_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; [[ \"$(sha256sum \"$payload\"|awk '{print $1}')\" == \"$expected_sha\" ]] || die RESTORE_PAYLOAD_SHA_MISMATCH 70\n523- zstd -t \"$payload\" >/dev/null; zstd -dc \"$payload\" | tar -xf - -C \"$work/extract\"; ex=\"$work/extract\"\n524- verify_binding_components \"$ex\" \"$logical\" \"$gen\"\n525- checks=\"$(semantic_validate \"$app\" \"$gen\" \"$ex\")\"\n526- out=\"$(restore_dir \"$app\")/$gen.json\"; mkdir -p \"$(dirname \"$out\")\"; chmod 0700 \"$RESTORE_STATE_ROOT\" \"$(dirname \"$out\")\"\n527- tmp=\"${out}.new.$$\"\n528: jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\n529- chmod 0600 \"$tmp\"; mv -f \"$tmp\" \"$out\"\n530: jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\n531- rm -rf --one-file-system \"$work\"; ACTIVE_STAGE=\"\"\n532- printf 'RESTORE_VALIDATION_STATUS=OK\\nAPP=%s\\nGENERATION=%s\\nEVIDENCE=%s\\n' \"$app\" \"$gen\" \"$out\"\n533-}\n534-backup_one(){\n535- local app=\"$1\" logical pending_count state gen payload latest orphan_count\n536- logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\n537- mapfile -t pending < <(latest_incomplete \"$app\"); pending_count=\"${#pending[@]}\"; (( pending_count <= 1 )) || die \"MULTIPLE_INCOMPLETE_GENERATIONS_$app\" 65\n538- if (( pending_count == 1 )); then\n539- state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\n540- \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\n541: printf 'STAGE=RESTORE_VALIDATION\\n'; restore_validate \"$app\" \"$gen\"\n542- printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return\n543- fi\n544- plan \"$app\" >/dev/null\n545- latest=\"$(latest_sealed_state \"$app\")\"\n546: if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\n547- if [[ \"$app\" == scrutiny ]]; then\n548- orphan_count=\"$(find \"$STAGE_ROOT/$logical\" -maxdepth 1 -type d -name '.capture-app-scrutiny-*' | wc -l)\"\n549- (( orphan_count == 0 )) || die SCRUTINY_ORPHAN_CAPTURE_REQUIRES_CONTROLLED_RECOVERY 65\n550- fi\n551- gen=\"${logical}-pve01-$(date -u +%Y%m%dT%H%M%SZ)\"; valid_generation \"$gen\" || die GENERATED_ID_INVALID 65\n552- printf 'RESUME=NO\\nAPP=%s\\nGENERATION=%s\\nSTAGE=CAPTURE\\n' \"$app\" \"$gen\"; capture_new \"$app\" \"$gen\"; payload=\"$PAYLOAD\"\n553- printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\n554: printf 'STAGE=RESTORE_VALIDATION\\n'; restore_validate \"$app\" \"$gen\"\n555- state=\"$STATE_ROOT/$logical/$gen.json\"; jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .local_payload_present==false and .seal_written==true' \"$state\" >/dev/null\n556- printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"\n557-}\n558-plan(){\n559- local app=\"$1\" logical free\n560- logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\n561- [[ \"$free\" =~ ^[1-9][0-9]+$ ]] || die STAGING_FREE_INVALID 65\n562- local source_bytes=0 required=0\n--\n589- ;;\n590- skladchik-reports-monitor) remote_edge probe plan >/dev/null ;;\n591- esac\n592- printf 'PLAN_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nSOURCE_HOST=%s\\nSTAGING_FREE_BYTES=%s\\nSOURCE_BYTES=%s\\nREQUIRED_STAGING_BYTES=%s\\n' \"$app\" \"$logical\" \"$(source_host \"$app\")\" \"$free\" \"$source_bytes\" \"$required\"\n593-}\n594-latest_sealed(){\n595- local app=\"$1\" f gen\n596- f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\n597: restore_ok \"$app\" \"$gen\" || die \"LATEST_RESTORE_NOT_OK_$app\" 65\n598: printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\n599-}\n600-recover_scrutiny_orphan(){\n601- local gen=\"$1\" started=\"$2\" qstart=\"$3\" qend=\"$4\" expected_config=\"$5\" expected_influx=\"$6\"\n602- local completed unitsha ref iid digest repodigests isize scdb idb devices migrations settings influx_tables tsm_files tsm_bytes wal_files wal_bytes series trange cfgsha boltbytes sqlitebytes image_sha meta_sha config_manifest_sha influx_manifest_sha validation_manifest_sha unit_copy_sha comps report actual_config actual_influx logical state restore\n603- logical=app-scrutiny\n604- valid_generation \"$gen\" || die GENERATION_INVALID 64\n605- [[ \"$gen\" == app-scrutiny-pve01-* ]] || die SCRUTINY_RECOVERY_GENERATION_INVALID 64\n606- [[ \"$started\" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && \"$qstart\" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && \"$qend\" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die SCRUTINY_RECOVERY_TIMESTAMP_INVALID 64\n--\n639- if [[ -s \"$state\" && ! -L \"$state\" ]] && jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$state\" >/dev/null 2>&1; then\n640- printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=scrutiny\\nGENERATION=%s\\n' \"$gen\"\n641- else\n642- [[ -s \"$PAYLOAD\" && ! -L \"$PAYLOAD\" ]] || die SCRUTINY_RECOVERY_PAYLOAD_MISSING_FOR_TRANSPORT 65\n643- printf 'STAGE=TRANSPORT_RESUME\\n'\n644- \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\n645- fi\n646- printf 'STAGE=RESTORE_VALIDATION\\n'\n647: restore_validate scrutiny \"$gen\"\n648- [[ -s \"$state\" && ! -L \"$state\" ]] || die SCRUTINY_RECOVERY_SEALED_STATE_MISSING 70\n649- jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$state\" >/dev/null || die SCRUTINY_RECOVERY_SEALED_STATE_INVALID 70\n650: [[ -s \"$restore\" && ! -L \"$restore\" ]] || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_MISSING 70\n651: jq -e '.result==\"OK\" and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true' \"$restore\" >/dev/null || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_INVALID 70\n652- printf 'BACKUP_STATUS=SEALED\\nAPP=scrutiny\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$gen\"\n653-}\n654-\n655-ACTION=\"${1:-}\"\n656-case \"$ACTION\" in\n657- plan) [[ \"$#\" == 2 ]] || die USAGE_PLAN 64; plan \"$2\" ;;\n658- backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\n659: restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\n660- recover-scrutiny-orphan) [[ \"$#\" == 7 ]] || die USAGE_RECOVER_SCRUTINY_ORPHAN 64; recover_scrutiny_orphan \"$2\" \"$3\" \"$4\" \"$5\" \"$6\" \"$7\" ;;\n661: latest-sealed) [[ \"$#\" == 2 ]] || die USAGE_LATEST_SEALED 64; latest_sealed \"$2\" ;;\n662- *) die ACTION_INVALID 64 ;;\n663-esac\nAPP_RESTORE_CONTRACT_SOURCE_END=1\nAPP_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json\nAPP_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc\nAPP_GENERATION=app-skladchik-reports-monitor-pve01-20260921T175029Z\nAPP_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json\nAPP_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e\nAPP_RESTORE_KEYS_BEGIN=1\n[\"app\",\"checks\",\"generation\",\"logical_id\",\"payload_sha256\",\"payload_sha256_verified\",\"restore_source\",\"result\",\"schema\",\"transport_state\",\"transport_state_file\",\"validated_at_utc\"]\nAPP_RESTORE_KEYS_END=1\nAPP_RESTORE_SAFE_VIEW_BEGIN=1\n{\"schema\":\"homelab-app-restore-validation-v1\",\"result\":\"OK\",\"status\":null,\"logical_id\":\"app-skladchik-reports-monitor\",\"generation\":\"app-skladchik-reports-monitor-pve01-20260921T175029Z\",\"transport_state\":\"SEALED\",\"transport_state_file\":\"/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json\",\"restore_source\":\"Cloud A\",\"payload_sha256\":\"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc\",\"payload_sha256_verified\":true,\"payload_decompression_verified\":null,\"archive_recovery_readback_verified\":null,\"parts_manifest_verified\":null,\"bounded_staging\":null,\"full_local_payload_written\":null,\"tar_test_ok\":null,\"required_files_ok\":null,\"cookie_jar_ok\":null,\"sqlite_ok\":null,\"validated_at_utc\":\"2026-09-21T17:50:46Z\"}\nAPP_RESTORE_SAFE_VIEW_END=1\nAPP_RESTORE_BASE_CONTRACT_OK=true\nUS_NETBIRD_STATE_BEGIN=1\nUS_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26\n{\"consecutive_failures\":14,\"failure_backoff_seconds\":21600,\"generation\":\"v2-us-netbird-20260922T055316Z-d5c92acdd2e8\",\"last_attempt_epoch\":1790056411,\"last_success_epoch\":1789704670,\"last_success_generation\":\"v2-us-netbird-20260918T041032Z-d7648be4c16b\",\"last_success_green_verified\":false,\"last_success_manifest_sha256\":\"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5\",\"last_success_payload_sha256\":\"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf\",\"last_success_replicas\":[{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"},{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"}],\"logical_id\":\"us-netbird\",\"reason\":\"PIPE_FAILED\",\"result\":\"FAIL\",\"retry_after_epoch\":1790078011,\"schema\":1,\"sealed\":false,\"updated_at_utc\":\"2026-09-22T05:53:31Z\"}\nUS_NETBIRD_STATE_END=1\nPIPE_TO_ZSTD_SOURCE_BEGIN=1\n if not c: return None\n c.sort(key=lambda x:(x[0],x[1],x[2]))\n return c[0][3]\n\ndef run(argv, *, input=None, timeout=300, check=True, env=None):\n try:\n p=subprocess.run(argv,input=input,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)\n except subprocess.TimeoutExpired: raise BackupError('COMMAND_TIMEOUT')\n if check and p.returncode!=0: raise BackupError('COMMAND_FAILED',str(p.returncode))\n return p\n\ndef stream_process_to_file(argv,out_path,timeout=1800,input=None):\n out_path=pathlib.Path(out_path)\n with open(out_path,'wb') as f:\n try:p=subprocess.run(argv,input=input,stdout=f,stderr=subprocess.PIPE,timeout=timeout)\n except subprocess.TimeoutExpired: raise BackupError('STREAM_COMMAND_TIMEOUT')\n if p.returncode!=0: raise BackupError('STREAM_COMMAND_FAILED',str(p.returncode))\n\ndef pipe_to_zstd(argv,out_path,timeout=1800,input=None):\n zstd=shutil.which('zstd') or '/usr/bin/zstd'\n try:\n src=subprocess.Popen(argv,stdin=subprocess.PIPE if input is not None else None,stdout=subprocess.PIPE,stderr=subprocess.PIPE)\n z=subprocess.Popen([zstd,'-T0','-3','-q','-o',str(out_path)],stdin=src.stdout,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE)\n if src.stdout: src.stdout.close()\n if input is not None:\n src.stdin.write(input); src.stdin.close()\n zerr=z.communicate(timeout=timeout)[1]\n try: src.wait(timeout=60)\n except subprocess.TimeoutExpired:\n src.kill(); src.wait(); raise BackupError('PIPE_SOURCE_TIMEOUT')\n serr=src.stderr.read() if src.stderr else b''\n except subprocess.TimeoutExpired:\n for q in (src,z):\n try:q.kill()\n except Exception:pass\n raise BackupError('PIPE_TIMEOUT')\n if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}')\n\ndef ensure_dir(p,mode=0o700):\n p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p\n\ndef atomic_write(path,data,mode=0o600):\n path=pathlib.Path(path); ensure_dir(path.parent,0o700)\n fd,tmp=tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent))\nPIPE_TO_ZSTD_SOURCE_END=1\nUS_REMOTE_SCRIPT_SOURCE_BEGIN=1\n if pre!=post: raise BackupError('SPB_SOURCE_CHANGED_DURING_CAPTURE')\n return payload,{'generation':gen,'source_class':'vps_spb','consistency':'STATIC_CONFIG_FILESET_PRE_POST_HASH_STABILITY','tls_recovery':'REISSUE_TLS_ON_RESTORE','hashset_sha256':pre},None\n\nUS_REMOTE_SCRIPT=r'''import io,os,sqlite3,sys,tarfile,time,json,hashlib\npaths=['/opt/netbird/docker-compose.yml','/opt/netbird/config.yaml']\ndata_root='/var/lib/docker/volumes/netbird_netbird_data/_data'\nle_root='/var/lib/docker/volumes/netbird_netbird_traefik_letsencrypt/_data'\ndbs=['events.db','store.db','geonames_20260602.db','idp.db']\nnon=['GeoLite2-City_20260602.mmdb']\nout=sys.stdout.buffer\nwith tarfile.open(fileobj=out,mode='w|') as tf:\n for p in paths:\n if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False)\n for name in non:\n p=os.path.join(data_root,name)\n if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False)\n ac=os.path.join(le_root,'acme.json')\n if os.path.isfile(ac):\n st=os.stat(ac)\n if (st.st_mode & 0o777)!=0o600: raise SystemExit(31)\n json.load(open(ac,'r'))\n tf.add(ac,arcname=ac.lstrip('/'),recursive=False)\n for name in dbs:\n srcp=os.path.join(data_root,name)\n src=sqlite3.connect('file:'+srcp+'?mode=ro',uri=True)\n if src.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(32)\n dst=sqlite3.connect(':memory:'); src.backup(dst)\n if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33)\n b=dst.serialize(); dst.close(); src.close()\n ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b))\n'''\n\ndef capture_vps_us(t,gen,work):\n remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT])\n argv=vps_ssh_argv(t,use_agent=False)+[remote]\n payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200)\n return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None\n\ndef _simple_shell_assignments(text):\nUS_REMOTE_SCRIPT_SOURCE_END=1\nVPS_SSH_SOURCE_BEGIN=1\ndef operator_key_fingerprint():\n if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING')\n p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30)\n s=p.stdout.decode('utf-8','replace')\n m=re.search(r'(SHA256:[A-Za-z0-9+/]+)',s)\n if not m or m.group(1)!=OPERATOR_KEY_FINGERPRINT: raise BackupError('OPERATOR_KEY_FINGERPRINT_MISMATCH')\n return m.group(1)\n\ndef scan_hostkey_line(host,port,expected):\n p=run([SSH_KEYSCAN,'-T','8','-p',str(port),'-t','ed25519',host],timeout=15)\n lines=[x.strip() for x in p.stdout.decode().splitlines() if x.strip() and not x.startswith('#')]\n good=[]\n for line in lines:\n f=line.split()\n if len(f)<3 or f[1]!='ssh-ed25519': continue\n try: blob=base64.b64decode(f[2])\n except Exception: continue\n fp='SHA256:'+base64.b64encode(hashlib.sha256(blob).digest()).decode().rstrip('=')\n if fp==expected: good.append(line)\n if len(good)!=1: raise BackupError('HOSTKEY_PIN_MISMATCH')\n return good[0]\n\ndef ensure_vps_known_hosts(targets):\n lines=[]\n for t in targets:\n if t['type'] not in ('vps_md','vps_spb','vps_us'): continue\n lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint']))\n atomic_write(VPS_KNOWN_HOSTS,('\\n'.join(lines)+'\\n').encode(),0o600)\n\ndef vps_ssh_argv(t, *, use_agent=False):\n a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15']\n if use_agent:\n a += ['-o','IdentitiesOnly=no']\n else:\n a += ['-i',str(OPERATOR_KEY),'-o','IdentitiesOnly=yes','-o','IdentityAgent=none']\n a += [t['user']+'@'+t['host']]\n return a\n\ndef vps_run(t, remote_argv, *, input=None, timeout=300, use_agent=False, check=True):\n remote=shlex.join([str(x) for x in remote_argv])\n return run(vps_ssh_argv(t,use_agent=use_agent)+[remote],input=input,timeout=timeout,check=check)\n\ndef bootstrap_vps_keys(targets):\n operator_key_fingerprint()\n ensure_vps_known_hosts(targets)\n pub=run(['ssh-keygen','-y','-f',str(OPERATOR_KEY)],timeout=30).stdout.strip()+b'\\n'\n results={}\n for t in targets:\n if t['type'] not in ('vps_md','vps_spb'): continue\n probe=vps_run(t,['true'],timeout=30,use_agent=False,check=False)\n if probe.returncode==0:\n results[t['logical_id']]='ALREADY_OPERATOR_KEY'; continue\n if not os.environ.get('SSH_AUTH_SOCK'): raise BackupError('FORWARDED_AGENT_REQUIRED_FOR_BOOTSTRAP')\n script=b\"umask 077; mkdir -p \\\"$HOME/.ssh\\\"; chmod 700 \\\"$HOME/.ssh\\\"; touch \\\"$HOME/.ssh/authorized_keys\\\"; chmod 600 \\\"$HOME/.ssh/authorized_keys\\\"; IFS= read -r k; if grep -qF -- \\\"$k\\\" \\\"$HOME/.ssh/authorized_keys\\\"; then printf 'ALREADY\\\\n'; else printf '%s\\\\n' \\\"$k\\\" >> \\\"$HOME/.ssh/authorized_keys\\\"; printf 'ADDED\\\\n'; fi\"\n p=vps_run(t,['sh','-c',script.decode()],input=pub,timeout=45,use_agent=True)\n tag=p.stdout.decode().strip()\n if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT')\n vps_run(t,['true'],timeout=30,use_agent=False)\n results[t['logical_id']]=tag\n # US was already operator-key based; prove it agentless too.\n us=[x for x in targets if x['type']=='vps_us'][0]\n vps_run(us,['true'],timeout=30,use_agent=False)\n results[us['logical_id']]='ALREADY_US_OPERATOR_KEY'\n return results\n\ndef remote_hashset(t,command):\n p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False)\n return sha256_bytes(p.stdout)\nVPS_SSH_SOURCE_END=1\nUS_NETBIRD_PID_JOURNAL_BEGIN=1\nPID_BEGIN=936114\n2026-09-21T20:30:36+03:00 pve01 homelab-backup-v2[936114]: SELFTEST_REPLACEMENT486=PASS\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED\nPID_END=936114\nPID_BEGIN=2121414\n2026-09-22T02:36:58+03:00 pve01 homelab-backup-v2[2121414]: SELFTEST_REPLACEMENT486=PASS\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED\nPID_END=2121414\nPID_BEGIN=3378768\n2026-09-22T08:53:16+03:00 pve01 homelab-backup-v2[3378768]: SELFTEST_REPLACEMENT486=PASS\n2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_TARGET=us-netbird\n2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_RESULT=FAIL\n2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_REASON=PIPE_FAILED\nPID_END=3378768\nUS_NETBIRD_PID_JOURNAL_END=1\nUS_NETBIRD_WORK_ARTIFACTS_BEGIN=1\nf|1790056411.7319950600|792|/var/lib/homelab-backup-v2/state/us-netbird.json\nf|1789642259.3981707900|1255|/var/lib/homelab-backup-v2/cold-bootstrap/us-netbird.json\nd|1790056411.7288067540|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260922T055316Z-d5c92acdd2e8\nd|1790033833.7009700260|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T233658Z-709cf01e1256\nd|1790011851.2302881580|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T173036Z-1128898e4cc8\nd|1789989471.8069593230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T111736Z-c5a5c5e71776\nd|1789955807.6619889330|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T015632Z-dc499030a55f\nd|1789932571.7940614350|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T192916Z-ca2592a81a30\nd|1789908991.7828657500|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T125616Z-8caeb0b6c7b7\nd|1789886227.6266376830|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T063652Z-e6c72515a153\nd|1789862827.7065704150|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T000652Z-7ecdcfcafc12\nd|1789839091.8314245860|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T173116Z-cfc70719f4e9\nd|1789822687.5806853640|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T125752Z-a4efa5c07171\nd|1789814653.6399387990|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T104358Z-b2232f971dbc\nd|1789810393.6722341230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T093258Z-64f5c2972565\nd|1789807993.6075344840|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T085258Z-36f858f91de5\nUS_NETBIRD_WORK_ARTIFACTS_END=1\nUS_NETBIRD_READONLY_PROBES_BEGIN=1\nUS_NETBIRD_READONLY_PROBE_RC=41\nPROBE_TARGET_FOUND=true\nPROBE_TYPE=vps_us\nPROBE_SSH_TRUE_EXCEPTION=BackupError\nPROBE_SSH_TRUE_CODE=COMMAND_FAILED\nPROBE_SSH_TRUE_DETAIL=255\nUS_NETBIRD_READONLY_PROBE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nUS_NETBIRD_READONLY_PROBES_END=1\nUS_NETBIRD_PIPE_BASELINE_CLASS=SSH_BASELINE_FAILURE\nDECISION=PASS_BACKUP_1117_APP_CONTRACT_AND_PIPE_BASELINE_CLASSIFIED\nNEXT_GATE=REPAIR_ONLY_BACKUP_US_NETBIRD_AND_AUDIT_CANONICAL_CLOUDB_SOURCE_OFFHOST\nTASK_RESULT=PASS_HOMELAB_BACKUP_1116_CONTRACT_PIPE_RCA\nCHANGES_MADE_BY_1117=false\nPRODUCT_MUTATION_BY_1117=false\nVM_MUTATION_BY_1117=false\nCLOUD_MUTATION_BY_1117=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1.txt b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1.txt new file mode 100644 index 00000000..790521b2 --- /dev/null +++ b/runtime/history/SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1.txt @@ -0,0 +1,423 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-backup-1116-contract-pipe-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=0ba8e65003a7561a9e01930aa9eec5d2b6a6cd73c9f05d217fca4a399c992fed +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=8314f5500f36a8ab9866f3f778ae569bf18655d73ade7aca6ada77060826a746 +SANITIZED_OUTPUT_SHA256=1c8576db6611dd527a417a275078b31f952af42ddef0dd6dd0fa1661455bb847 +OUTPUT_BEGIN +WORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1 +MODE=read-only +COMPONENT=homelab-backup-1116-contract-pipe-rca +MUTATION_BOUNDARY=NONE;READ_ONLY_APP_RESTORE_CONTRACT_AND_US_NETBIRD_PIPE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP_TARGET_EXECUTION;NO_RESTORE_EXECUTION;NO_GIT_WRITE;NO_CLOUD_WRITE;REMOTE_PROBES_READ_ONLY +ERROR_REGISTER_CHECK=OK +REFERENCE_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 +LEGACY_SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7 +APP_WORKER_SHA256=1d6c4a04f762e2a33561c109b0c0a4878caaa48627903e7e78fdf5f22d79f196 +SCHEDULER_RC=0 +SELFTEST_REPLACEMENT486=PASS +SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET +APP_RESTORE_CONTRACT_SOURCE_BEGIN=1 +89-} +90-latest_sealed_state(){ +91- local app="$1" d +92- d="$(state_dir "$app")"; [[ -d "$d" ]] || return 0 +93- find "$d" -maxdepth 1 -type f -name '*.json' -printf '%T@|%p\n' | LC_ALL=C sort -t'|' -k1,1nr | while IFS='|' read -r _ f; do +94- if jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$f" >/dev/null 2>&1; then echo "$f"; break; fi +95- done +96-} +97:restore_ok(){ +98- local app="$1" gen="$2" f +99- f="$(restore_dir "$app")/$gen.json" +100- [[ -s "$f" && ! -L "$f" ]] || return 1 +101: jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +102-} +103-remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +104-remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +105-rsync_core(){ +106- local src="$1" dst="$2" +107- mkdir -p "$dst" +108- RSYNC_RSH='ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] rsync -a --numeric-ids --delete --info=progress2 --rsync-path='sudo -n rsync' "$CORE:$src/" "$dst/" +109-} +-- +344- [[ -f "$ex/$path" && ! -L "$ex/$path" ]] || die RESTORE_COMPONENT_MISSING 70 +345- actual="$(sha256sum "$ex/$path"|awk '{print $1}')"; [[ "$actual" == "$sha" ]] || die RESTORE_COMPONENT_SHA_MISMATCH 70 +346- done < <(jq -r '.components|to_entries[]|[.value.path,.value.sha256]|@tsv' "$ex/app-generation.json") +347-} +348-reconstruct_cloud_a(){ +349- local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +350- logical="$(logical_id "$app")" +351- "$TRANSPORT" verify-sealed "$logical" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null +352: base="$(jq -er '.remote_a_base' "$state")"; expected_msha="$(jq -er '.parts_manifest_sha256' "$state")" +353- manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +354- got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +355- : >"$out"; chmod 0600 "$out" +356- while IFS=$'\t' read -r idx name size sha; do +357- [[ "$idx" =~ ^[0-9]{8}$ && "$name" == part-????????.bin && "$size" =~ ^[1-9][0-9]*$ && "$sha" =~ ^[0-9a-f]{64}$ ]] || die RESTORE_A_MANIFEST_ROW_INVALID 70 +358- part="$(dirname "$out")/$name"; rm -f "$part"; rclone cat "${base%/}/payload.parts/$name" >"$part" +359- [[ "$(stat -c %s "$part")" == "$size" ]] || die RESTORE_A_PART_SIZE_MISMATCH 70 +360- actual="$(sha256sum "$part"|awk '{print $1}')"; [[ "$actual" == "$sha" ]] || die RESTORE_A_PART_SHA_MISMATCH 70 +-- +505- scp "${SCP_BASE[@]}" "$ex/immich/db/immich.dump" "$CORE:$remote_dump" >/dev/null +506- scp "${SCP_BASE[@]}" "$ACTIVE_STAGE/postgres-image.txt" "$CORE:$remote_img" >/dev/null +507- if ! ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- validate-immich-db "$gen" "$remote_dump" "$expect" "$remote_img" <"$CORE_HELPER" >/dev/null; then ssh "${SSH_BASE[@]}" "$CORE" rm -f -- "$remote_dump" "$remote_img" >/dev/null 2>&1 || true; die IMMICH_POSTGRES_IMPORT_FAILED 70; fi +508- ssh "${SSH_BASE[@]}" "$CORE" rm -f -- "$remote_dump" "$remote_img" >/dev/null +509- checks="$(jq -n --argjson n "$expect" '[{name:"postgres_import",result:"OK",table_count:$n},{name:"media_manifest",result:"OK"},{name:"config_manifest",result:"OK"}]')" ;; +510- esac +511- printf '%s\n' "$checks" +512-} +513:restore_validate(){ +514- local app="$1" gen="$2" logical state work payload ex expected_sha checks out tmp +515- valid_generation "$gen" || die GENERATION_INVALID 64 +516- logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +517- [[ -s "$state" && ! -L "$state" ]] || die RESTORE_STATE_MISSING 65 +518- jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null || die RESTORE_REQUIRES_SEALED 65 +519: if restore_ok "$app" "$gen"; then printf 'RESTORE_VALIDATION_STATUS=OK\nAPP=%s\nGENERATION=%s\nRESULT=ALREADY_VALIDATED\n' "$app" "$gen"; return 0; fi +520- work="$RESTORE_WORK_ROOT/$logical/$gen"; rm -rf --one-file-system "$work" 2>/dev/null || true; mkdir -p "$work/extract"; chmod 0700 "$RESTORE_WORK_ROOT" "$RESTORE_WORK_ROOT/$logical" "$work" 2>/dev/null || true; ACTIVE_STAGE="$work" +521- payload="$work/payload.tar.zst"; reconstruct_cloud_a "$app" "$gen" "$state" "$payload" +522- expected_sha="$(jq -er '.archive_sha256' "$state")"; [[ "$(sha256sum "$payload"|awk '{print $1}')" == "$expected_sha" ]] || die RESTORE_PAYLOAD_SHA_MISMATCH 70 +523- zstd -t "$payload" >/dev/null; zstd -dc "$payload" | tar -xf - -C "$work/extract"; ex="$work/extract" +524- verify_binding_components "$ex" "$logical" "$gen" +525- checks="$(semantic_validate "$app" "$gen" "$ex")" +526- out="$(restore_dir "$app")/$gen.json"; mkdir -p "$(dirname "$out")"; chmod 0700 "$RESTORE_STATE_ROOT" "$(dirname "$out")" +527- tmp="${out}.new.$$" +528: jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +529- chmod 0600 "$tmp"; mv -f "$tmp" "$out" +530: jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +531- rm -rf --one-file-system "$work"; ACTIVE_STAGE="" +532- printf 'RESTORE_VALIDATION_STATUS=OK\nAPP=%s\nGENERATION=%s\nEVIDENCE=%s\n' "$app" "$gen" "$out" +533-} +534-backup_one(){ +535- local app="$1" logical pending_count state gen payload latest orphan_count +536- logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +537- mapfile -t pending < <(latest_incomplete "$app"); pending_count="${#pending[@]}"; (( pending_count <= 1 )) || die "MULTIPLE_INCOMPLETE_GENERATIONS_$app" 65 +538- if (( pending_count == 1 )); then +539- state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +540- "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +541: printf 'STAGE=RESTORE_VALIDATION\n'; restore_validate "$app" "$gen" +542- printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return +543- fi +544- plan "$app" >/dev/null +545- latest="$(latest_sealed_state "$app")" +546: if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +547- if [[ "$app" == scrutiny ]]; then +548- orphan_count="$(find "$STAGE_ROOT/$logical" -maxdepth 1 -type d -name '.capture-app-scrutiny-*' | wc -l)" +549- (( orphan_count == 0 )) || die SCRUTINY_ORPHAN_CAPTURE_REQUIRES_CONTROLLED_RECOVERY 65 +550- fi +551- gen="${logical}-pve01-$(date -u +%Y%m%dT%H%M%SZ)"; valid_generation "$gen" || die GENERATED_ID_INVALID 65 +552- printf 'RESUME=NO\nAPP=%s\nGENERATION=%s\nSTAGE=CAPTURE\n' "$app" "$gen"; capture_new "$app" "$gen"; payload="$PAYLOAD" +553- printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +554: printf 'STAGE=RESTORE_VALIDATION\n'; restore_validate "$app" "$gen" +555- state="$STATE_ROOT/$logical/$gen.json"; jq -e '.status=="SEALED" and .cloud_quorum==2 and .local_payload_present==false and .seal_written==true' "$state" >/dev/null +556- printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen" +557-} +558-plan(){ +559- local app="$1" logical free +560- logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +561- [[ "$free" =~ ^[1-9][0-9]+$ ]] || die STAGING_FREE_INVALID 65 +562- local source_bytes=0 required=0 +-- +589- ;; +590- skladchik-reports-monitor) remote_edge probe plan >/dev/null ;; +591- esac +592- printf 'PLAN_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nSOURCE_HOST=%s\nSTAGING_FREE_BYTES=%s\nSOURCE_BYTES=%s\nREQUIRED_STAGING_BYTES=%s\n' "$app" "$logical" "$(source_host "$app")" "$free" "$source_bytes" "$required" +593-} +594-latest_sealed(){ +595- local app="$1" f gen +596- f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +597: restore_ok "$app" "$gen" || die "LATEST_RESTORE_NOT_OK_$app" 65 +598: printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +599-} +600-recover_scrutiny_orphan(){ +601- local gen="$1" started="$2" qstart="$3" qend="$4" expected_config="$5" expected_influx="$6" +602- local completed unitsha ref iid digest repodigests isize scdb idb devices migrations settings influx_tables tsm_files tsm_bytes wal_files wal_bytes series trange cfgsha boltbytes sqlitebytes image_sha meta_sha config_manifest_sha influx_manifest_sha validation_manifest_sha unit_copy_sha comps report actual_config actual_influx logical state restore +603- logical=app-scrutiny +604- valid_generation "$gen" || die GENERATION_INVALID 64 +605- [[ "$gen" == app-scrutiny-pve01-* ]] || die SCRUTINY_RECOVERY_GENERATION_INVALID 64 +606- [[ "$started" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && "$qstart" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && "$qend" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die SCRUTINY_RECOVERY_TIMESTAMP_INVALID 64 +-- +639- if [[ -s "$state" && ! -L "$state" ]] && jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null 2>&1; then +640- printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=scrutiny\nGENERATION=%s\n' "$gen" +641- else +642- [[ -s "$PAYLOAD" && ! -L "$PAYLOAD" ]] || die SCRUTINY_RECOVERY_PAYLOAD_MISSING_FOR_TRANSPORT 65 +643- printf 'STAGE=TRANSPORT_RESUME\n' +644- "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +645- fi +646- printf 'STAGE=RESTORE_VALIDATION\n' +647: restore_validate scrutiny "$gen" +648- [[ -s "$state" && ! -L "$state" ]] || die SCRUTINY_RECOVERY_SEALED_STATE_MISSING 70 +649- jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null || die SCRUTINY_RECOVERY_SEALED_STATE_INVALID 70 +650: [[ -s "$restore" && ! -L "$restore" ]] || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_MISSING 70 +651: jq -e '.result=="OK" and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true' "$restore" >/dev/null || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_INVALID 70 +652- printf 'BACKUP_STATUS=SEALED\nAPP=scrutiny\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$gen" +653-} +654- +655-ACTION="${1:-}" +656-case "$ACTION" in +657- plan) [[ "$#" == 2 ]] || die USAGE_PLAN 64; plan "$2" ;; +658- backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +659: restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +660- recover-scrutiny-orphan) [[ "$#" == 7 ]] || die USAGE_RECOVER_SCRUTINY_ORPHAN 64; recover_scrutiny_orphan "$2" "$3" "$4" "$5" "$6" "$7" ;; +661: latest-sealed) [[ "$#" == 2 ]] || die USAGE_LATEST_SEALED 64; latest_sealed "$2" ;; +662- *) die ACTION_INVALID 64 ;; +663-esac +APP_RESTORE_CONTRACT_SOURCE_END=1 +APP_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json +APP_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc +APP_GENERATION=app-skladchik-reports-monitor-pve01-20260921T175029Z +APP_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json +APP_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e +APP_RESTORE_KEYS_BEGIN=1 +["app","checks","generation","logical_id","payload_sha256","payload_sha256_verified","restore_source","result","schema","transport_state","transport_state_file","validated_at_utc"] +APP_RESTORE_KEYS_END=1 +APP_RESTORE_SAFE_VIEW_BEGIN=1 +{"schema":"homelab-app-restore-validation-v1","result":"OK","status":null,"logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","transport_state":"SEALED","transport_state_file":"/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json","restore_source":"Cloud A","payload_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","payload_sha256_verified":true,"payload_decompression_verified":null,"archive_recovery_readback_verified":null,"parts_manifest_verified":null,"bounded_staging":null,"full_local_payload_written":null,"tar_test_ok":null,"required_files_ok":null,"cookie_jar_ok":null,"sqlite_ok":null,"validated_at_utc":"2026-09-21T17:50:46Z"} +APP_RESTORE_SAFE_VIEW_END=1 +APP_RESTORE_BASE_CONTRACT_OK=true +US_NETBIRD_STATE_BEGIN=1 +US_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26 +{"consecutive_failures":14,"failure_backoff_seconds":21600,"generation":"v2-us-netbird-20260922T055316Z-d5c92acdd2e8","last_attempt_epoch":1790056411,"last_success_epoch":1789704670,"last_success_generation":"v2-us-netbird-20260918T041032Z-d7648be4c16b","last_success_green_verified":false,"last_success_manifest_sha256":"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5","last_success_payload_sha256":"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf","last_success_replicas":[{"crypt":"pve01-mail-05-crypt","physical":"pve01-mail-05"},{"crypt":"pve03-mail-04-crypt","physical":"pve03-mail-04"}],"logical_id":"us-netbird","reason":"PIPE_FAILED","result":"FAIL","retry_after_epoch":1790078011,"schema":1,"sealed":false,"updated_at_utc":"2026-09-22T05:53:31Z"} +US_NETBIRD_STATE_END=1 +PIPE_TO_ZSTD_SOURCE_BEGIN=1 + if not c: return None + c.sort(key=lambda x:(x[0],x[1],x[2])) + return c[0][3] + +def run(argv, *, input=None, timeout=300, check=True, env=None): + try: + p=subprocess.run(argv,input=input,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env) + except subprocess.TimeoutExpired: raise BackupError('COMMAND_TIMEOUT') + if check and p.returncode!=0: raise BackupError('COMMAND_FAILED',str(p.returncode)) + return p + +def stream_process_to_file(argv,out_path,timeout=1800,input=None): + out_path=pathlib.Path(out_path) + with open(out_path,'wb') as f: + try:p=subprocess.run(argv,input=input,stdout=f,stderr=subprocess.PIPE,timeout=timeout) + except subprocess.TimeoutExpired: raise BackupError('STREAM_COMMAND_TIMEOUT') + if p.returncode!=0: raise BackupError('STREAM_COMMAND_FAILED',str(p.returncode)) + +def pipe_to_zstd(argv,out_path,timeout=1800,input=None): + zstd=shutil.which('zstd') or '/usr/bin/zstd' + try: + src=subprocess.Popen(argv,stdin=subprocess.PIPE if input is not None else None,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + z=subprocess.Popen([zstd,'-T0','-3','-q','-o',str(out_path)],stdin=src.stdout,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE) + if src.stdout: src.stdout.close() + if input is not None: + src.stdin.write(input); src.stdin.close() + zerr=z.communicate(timeout=timeout)[1] + try: src.wait(timeout=60) + except subprocess.TimeoutExpired: + src.kill(); src.wait(); raise BackupError('PIPE_SOURCE_TIMEOUT') + serr=src.stderr.read() if src.stderr else b'' + except subprocess.TimeoutExpired: + for q in (src,z): + try:q.kill() + except Exception:pass + raise BackupError('PIPE_TIMEOUT') + if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}') + +def ensure_dir(p,mode=0o700): + p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p + +def atomic_write(path,data,mode=0o600): + path=pathlib.Path(path); ensure_dir(path.parent,0o700) + fd,tmp=tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent)) +PIPE_TO_ZSTD_SOURCE_END=1 +US_REMOTE_SCRIPT_SOURCE_BEGIN=1 + if pre!=post: raise BackupError('SPB_SOURCE_CHANGED_DURING_CAPTURE') + return payload,{'generation':gen,'source_class':'vps_spb','consistency':'STATIC_CONFIG_FILESET_PRE_POST_HASH_STABILITY','tls_recovery':'REISSUE_TLS_ON_RESTORE','hashset_sha256':pre},None + +US_REMOTE_SCRIPT=r'''import io,os,sqlite3,sys,tarfile,time,json,hashlib +paths=['/opt/netbird/docker-compose.yml','/opt/netbird/config.yaml'] +data_root='/var/lib/docker/volumes/netbird_netbird_data/_data' +le_root='/var/lib/docker/volumes/netbird_netbird_traefik_letsencrypt/_data' +dbs=['events.db','store.db','geonames_20260602.db','idp.db'] +non=['GeoLite2-City_20260602.mmdb'] +out=sys.stdout.buffer +with tarfile.open(fileobj=out,mode='w|') as tf: + for p in paths: + if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False) + for name in non: + p=os.path.join(data_root,name) + if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False) + ac=os.path.join(le_root,'acme.json') + if os.path.isfile(ac): + st=os.stat(ac) + if (st.st_mode & 0o777)!=0o600: raise SystemExit(31) + json.load(open(ac,'r')) + tf.add(ac,arcname=ac.lstrip('/'),recursive=False) + for name in dbs: + srcp=os.path.join(data_root,name) + src=sqlite3.connect('file:'+srcp+'?mode=ro',uri=True) + if src.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(32) + dst=sqlite3.connect(':memory:'); src.backup(dst) + if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33) + b=dst.serialize(); dst.close(); src.close() + ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b)) +''' + +def capture_vps_us(t,gen,work): + remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT]) + argv=vps_ssh_argv(t,use_agent=False)+[remote] + payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200) + return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None + +def _simple_shell_assignments(text): +US_REMOTE_SCRIPT_SOURCE_END=1 +VPS_SSH_SOURCE_BEGIN=1 +def operator_key_fingerprint(): + if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING') + p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30) + s=p.stdout.decode('utf-8','replace') + m=re.search(r'(SHA256:[A-Za-z0-9+/]+)',s) + if not m or m.group(1)!=OPERATOR_KEY_FINGERPRINT: raise BackupError('OPERATOR_KEY_FINGERPRINT_MISMATCH') + return m.group(1) + +def scan_hostkey_line(host,port,expected): + p=run([SSH_KEYSCAN,'-T','8','-p',str(port),'-t','ed25519',host],timeout=15) + lines=[x.strip() for x in p.stdout.decode().splitlines() if x.strip() and not x.startswith('#')] + good=[] + for line in lines: + f=line.split() + if len(f)<3 or f[1]!='ssh-ed25519': continue + try: blob=base64.b64decode(f[2]) + except Exception: continue + fp='SHA256:'+base64.b64encode(hashlib.sha256(blob).digest()).decode().rstrip('=') + if fp==expected: good.append(line) + if len(good)!=1: raise BackupError('HOSTKEY_PIN_MISMATCH') + return good[0] + +def ensure_vps_known_hosts(targets): + lines=[] + for t in targets: + if t['type'] not in ('vps_md','vps_spb','vps_us'): continue + lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint'])) + atomic_write(VPS_KNOWN_HOSTS,('\n'.join(lines)+'\n').encode(),0o600) + +def vps_ssh_argv(t, *, use_agent=False): + a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15'] + if use_agent: + a += ['-o','IdentitiesOnly=no'] + else: + a += ['-i',str(OPERATOR_KEY),'-o','IdentitiesOnly=yes','-o','IdentityAgent=none'] + a += [t['user']+'@'+t['host']] + return a + +def vps_run(t, remote_argv, *, input=None, timeout=300, use_agent=False, check=True): + remote=shlex.join([str(x) for x in remote_argv]) + return run(vps_ssh_argv(t,use_agent=use_agent)+[remote],input=input,timeout=timeout,check=check) + +def bootstrap_vps_keys(targets): + operator_key_fingerprint() + ensure_vps_known_hosts(targets) + pub=run(['ssh-keygen','-y','-f',str(OPERATOR_KEY)],timeout=30).stdout.strip()+b'\n' + results={} + for t in targets: + if t['type'] not in ('vps_md','vps_spb'): continue + probe=vps_run(t,['true'],timeout=30,use_agent=False,check=False) + if probe.returncode==0: + results[t['logical_id']]='ALREADY_OPERATOR_KEY'; continue + if not os.environ.get('SSH_AUTH_SOCK'): raise BackupError('FORWARDED_AGENT_REQUIRED_FOR_BOOTSTRAP') + script=b"umask 077; mkdir -p \"$HOME/.ssh\"; chmod 700 \"$HOME/.ssh\"; touch \"$HOME/.ssh/authorized_keys\"; chmod 600 \"$HOME/.ssh/authorized_keys\"; IFS= read -r k; if grep -qF -- \"$k\" \"$HOME/.ssh/authorized_keys\"; then printf 'ALREADY\\n'; else printf '%s\\n' \"$k\" >> \"$HOME/.ssh/authorized_keys\"; printf 'ADDED\\n'; fi" + p=vps_run(t,['sh','-c',script.decode()],input=pub,timeout=45,use_agent=True) + tag=p.stdout.decode().strip() + if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT') + vps_run(t,['true'],timeout=30,use_agent=False) + results[t['logical_id']]=tag + # US was already operator-key based; prove it agentless too. + us=[x for x in targets if x['type']=='vps_us'][0] + vps_run(us,['true'],timeout=30,use_agent=False) + results[us['logical_id']]='ALREADY_US_OPERATOR_KEY' + return results + +def remote_hashset(t,command): + p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False) + return sha256_bytes(p.stdout) +VPS_SSH_SOURCE_END=1 +US_NETBIRD_PID_JOURNAL_BEGIN=1 +PID_BEGIN=936114 +2026-09-21T20:30:36+03:00 pve01 homelab-backup-v2[936114]: SELFTEST_REPLACEMENT486=PASS +2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird +2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL +2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED +PID_END=936114 +PID_BEGIN=2121414 +2026-09-22T02:36:58+03:00 pve01 homelab-backup-v2[2121414]: SELFTEST_REPLACEMENT486=PASS +2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird +2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL +2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED +PID_END=2121414 +PID_BEGIN=3378768 +2026-09-22T08:53:16+03:00 pve01 homelab-backup-v2[3378768]: SELFTEST_REPLACEMENT486=PASS +2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_TARGET=us-netbird +2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_RESULT=FAIL +2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_REASON=PIPE_FAILED +PID_END=3378768 +US_NETBIRD_PID_JOURNAL_END=1 +US_NETBIRD_WORK_ARTIFACTS_BEGIN=1 +f|1790056411.7319950600|792|/var/lib/homelab-backup-v2/state/us-netbird.json +f|1789642259.3981707900|1255|/var/lib/homelab-backup-v2/cold-bootstrap/us-netbird.json +d|1790056411.7288067540|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260922T055316Z-d5c92acdd2e8 +d|1790033833.7009700260|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T233658Z-709cf01e1256 +d|1790011851.2302881580|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T173036Z-1128898e4cc8 +d|1789989471.8069593230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T111736Z-c5a5c5e71776 +d|1789955807.6619889330|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T015632Z-dc499030a55f +d|1789932571.7940614350|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T192916Z-ca2592a81a30 +d|1789908991.7828657500|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T125616Z-8caeb0b6c7b7 +d|1789886227.6266376830|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T063652Z-e6c72515a153 +d|1789862827.7065704150|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T000652Z-7ecdcfcafc12 +d|1789839091.8314245860|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T173116Z-cfc70719f4e9 +d|1789822687.5806853640|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T125752Z-a4efa5c07171 +d|1789814653.6399387990|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T104358Z-b2232f971dbc +d|1789810393.6722341230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T093258Z-64f5c2972565 +d|1789807993.6075344840|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T085258Z-36f858f91de5 +US_NETBIRD_WORK_ARTIFACTS_END=1 +US_NETBIRD_READONLY_PROBES_BEGIN=1 +US_NETBIRD_READONLY_PROBE_RC=41 +PROBE_TARGET_FOUND=true +PROBE_TYPE=vps_us +PROBE_SSH_TRUE_EXCEPTION=BackupError +PROBE_SSH_TRUE_CODE=COMMAND_FAILED +PROBE_SSH_TRUE_DETAIL=255 +US_NETBIRD_READONLY_PROBE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +US_NETBIRD_READONLY_PROBES_END=1 +US_NETBIRD_PIPE_BASELINE_CLASS=SSH_BASELINE_FAILURE +DECISION=PASS_BACKUP_1117_APP_CONTRACT_AND_PIPE_BASELINE_CLASSIFIED +NEXT_GATE=REPAIR_ONLY_BACKUP_US_NETBIRD_AND_AUDIT_CANONICAL_CLOUDB_SOURCE_OFFHOST +TASK_RESULT=PASS_HOMELAB_BACKUP_1116_CONTRACT_PIPE_RCA +CHANGES_MADE_BY_1117=false +PRODUCT_MUTATION_BY_1117=false +VM_MUTATION_BY_1117=false +CLOUD_MUTATION_BY_1117=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 28668f77..e4af0c69 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1", - "status": "FAIL", - "rc": 3, + "command_id": "SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1", + "status": "OK", + "rc": 0, "host": "pve01", "mode": "read-only", - "component": "homelab-backup-1115-root-rca", - "started_at_utc": "2026-09-22T05:57:35Z", - "finished_at_utc": "2026-09-22T05:57:35Z", + "component": "homelab-backup-1116-contract-pipe-rca", + "started_at_utc": "2026-09-22T06:05:29Z", + "finished_at_utc": "2026-09-22T06:05:29Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "a475f4109c57c07002ec72ba49450725574b1c08d698b9878c58257e7cc704c8", + "command_sha256": "0ba8e65003a7561a9e01930aa9eec5d2b6a6cd73c9f05d217fca4a399c992fed", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -21,7 +21,7 @@ "result_contract_valid": true, "result_contract_status": null, "result_contract_error": null, - "command_rc": 3, + "command_rc": 0, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "17278f059b28a5b338724f4e22fad8aa2db50e23143eeee6635c075bf93a9691", - "sanitized_output_sha256": "17278f059b28a5b338724f4e22fad8aa2db50e23143eeee6635c075bf93a9691", + "raw_evidence_sha256": "8314f5500f36a8ab9866f3f778ae569bf18655d73ade7aca6ada77060826a746", + "sanitized_output_sha256": "1c8576db6611dd527a417a275078b31f952af42ddef0dd6dd0fa1661455bb847", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_BACKUP_1115_ROOT_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1\nMODE=read-only\nCOMPONENT=homelab-backup-1115-root-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_FAILURE_AND_CANONICAL_SKLADCHIK_EVIDENCE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE;NO_REMOTE_MUTATION\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nAUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nUS_NETBIRD_STATE_SAFE_BEGIN=1\nUS_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26\n{\"schema\":1,\"logical_id\":\"us-netbird\",\"result\":\"FAIL\",\"generation\":\"v2-us-netbird-20260922T055316Z-d5c92acdd2e8\",\"updated_at_utc\":\"2026-09-22T05:53:31Z\",\"last_success_epoch\":1789704670,\"last_attempt_epoch\":1790056411,\"retry_after_epoch\":1790078011,\"consecutive_failures\":14,\"replica_count\":null}\n{\"path\":\"consecutive_failures\",\"value\":14}\n{\"path\":\"failure_backoff_seconds\",\"value\":21600}\n{\"path\":\"last_attempt_epoch\",\"value\":1790056411}\n{\"path\":\"last_success_epoch\",\"value\":1789704670}\n{\"path\":\"last_success_generation\",\"value\":\"v2-us-netbird-20260918T041032Z-d7648be4c16b\"}\n{\"path\":\"last_success_manifest_sha256\",\"value\":\"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5\"}\n{\"path\":\"last_success_payload_sha256\",\"value\":\"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf\"}\n{\"path\":\"reason\",\"value\":\"PIPE_FAILED\"}\n{\"path\":\"result\",\"value\":\"FAIL\"}\n{\"path\":\"retry_after_epoch\",\"value\":1790078011}\nUS_NETBIRD_STATE_SAFE_END=1\nUS_NETBIRD_SOURCE_HITS_BEGIN=1\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7\n155- for node in ('pve01','pve02','pve03'):\n156- out.append({'logical_id':'node-'+node,'scope':'node','type':'pve_node_bundle','node':node,'due_seconds':7*d,'enabled':True,'green1':node=='pve01'})\n157- out += [\n158- {'logical_id':'md-relay','scope':'vps','type':'vps_md','host':'[PRIVATE_IP]','port':22,'user':'ops','hostkey_fingerprint':'SHA256:DXJGwunC5tEVvnC3nRxyayTN8FXmyJcVgf+Oa6udQsY','due_seconds':d,'enabled':True,'green1':True},\n159- {'logical_id':'spb-ingress','scope':'vps','type':'vps_spb','host':'[PRIVATE_IP]','port':22,'user':'edgeadmin','hostkey_fingerprint':'SHA256:Ef+HsZb8lIXoadvYBXuGKIL3H7NyJMmf9EYCoJRoSv8','due_seconds':d,'enabled':True,'green1':True},\n160: {'logical_id':'us-netbird','scope':'vps','type':'vps_us','host':'[PRIVATE_IP]','port':22,'user':'root','hostkey_fingerprint':'SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE','due_seconds':d,'enabled':True,'green1':True},\n161- {'logical_id':'router-netcraze','scope':'router','type':'router','due_seconds':d,'enabled':True,'green1':True},\n162- ]\n163- sites=[\n164- ('codevipe',160,'pve02','/var/www/forums/codevipe/public','xf_codevipe','single_transaction'),\n165- ('dsmods',160,'pve02','/var/www/forums/dsmods/public','xf_dsmods','single_transaction'),\n--\n216- except subprocess.TimeoutExpired:\n217- for q in (src,z):\n218- try:q.kill()\n219- except Exception:pass\n220- raise BackupError('PIPE_TIMEOUT')\n221: if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}')\n222-\n223-def ensure_dir(p,mode=0o700):\n224- p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p\n225-\n226-def atomic_write(path,data,mode=0o600):\n--\n248- try:\n249- j=json.loads(p.read_text()); d[j['logical_id']]=j\n250- except Exception: continue\n251- return d\n252-\n253:def write_state(logical_id,result,**kw):\n254- if result not in RESULT_CLASSES: raise BackupError('RESULT_CLASS_INVALID')\n255- j={'schema':1,'logical_id':logical_id,'result':result,'updated_at_utc':utcnow(),**kw}\n256- if result=='SUCCESS': j['last_success_epoch']=epoch()\n257- atomic_write(STATE_ROOT/'state'/(logical_id+'.json'),canonical_json(j)+b'\\n',0o600)\n258- return j\n--\n384- if not names: raise BackupError('TAR_ZST_EMPTY')\n385- if required_member and required_member not in names: raise BackupError('REQUIRED_MEMBER_MISSING')\n386-\n387-def validate_payload(t,path):\n388- typ=t['type']\n389: if typ in ('pve_node_bundle','vps_md','vps_spb','vps_us','router'):\n390- validate_tar_zst(path)\n391- elif typ=='pve_guest':\n392- if t['guest_type']=='lxc': validate_tar_zst(path)\n393- else:\n394- # Green set is intentionally CT-only. Scheduled QEMU backups still get dual remote byte readback.\n--\n405-def cluster_inventory():\n406- p=run(['pvesh','get','/cluster/resources','--type','vm','--output-format','json'],timeout=60)\n407- try:return json.loads(p.stdout)\n408- except Exception: raise BackupError('PVE_INVENTORY_JSON')\n409-\n410:def ssh_remote(node,argv,input=None,timeout=1800,check=True):\n411- cmd='PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec '+shlex.join([str(x) for x in argv])\n412- if node=='pve01': return run([str(x) for x in argv],input=input,timeout=timeout,check=check)\n413- return run([SSH,'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node,cmd],input=input,timeout=timeout,check=check)\n414-\n415-def _pve_dumpdir_spec(guest_type):\n--\n471- if i+1>=len(argv) or argv[i+1]!=str(scratch): raise BackupError('PROVEN_DUMP_ARGV_TMPDIR_BINDING')\n472- elif '--tmpdir='+str(scratch) not in argv: raise BackupError('PROVEN_DUMP_ARGV_NO_TMPDIR')\n473- return argv,job\n474-\n475-def _default_unprivileged_lxc(node,vmid):\n476: q=ssh_remote(node,['pct','config',str(vmid)],timeout=30)\n477- lines=[x.strip() for x in q.stdout.decode('utf-8','replace').splitlines()]\n478- if 'unprivileged: 1' not in lines: raise BackupError('LXC_NOT_UNPRIVILEGED')\n479- if any(x.startswith('lxc.idmap:') or x.startswith('lxc.idmap =') for x in lines): raise BackupError('LXC_CUSTOM_IDMAP_UNSUPPORTED')\n480-\n481-def mapped_scratch_probe(node,scratch):\n482- tool=shutil.which('lxc-usernsexec') or '/usr/bin/lxc-usernsexec'\n483- pos=[tool,'-m','u:0:100000:65536','-m','g:0:100000:65536','--','/usr/bin/test','-x',str(scratch)]\n484: q=ssh_remote(node,pos,timeout=30,check=False)\n485- if q.returncode!=0: raise BackupError('MAPPED_UID_ACCESS_PROBE_FAILED')\n486-\n487-def _stream_argv_to_local(node,argv,dest,timeout=7200):\n488- dest=pathlib.Path(dest)\n489- if node=='pve01': cmd=argv\n--\n509- if typ!=t['guest_type']: raise BackupError('PVE_GUEST_TYPE_DRIFT')\n510- if typ=='lxc':\n511- _require_exact_source(P270,P270_SHA,'P270_IDENTITY');_require_exact_source(P282,P282_SHA,'P282_IDENTITY')\n512- _default_unprivileged_lxc(node,vmid)\n513- scratch=f'/var/tmp/homelab-vzdump-v2-{gen}'\n514: ssh_remote(node,['install','-d','-m','710','-o','0','-g','100000',scratch],timeout=30)\n515: q=ssh_remote(node,['stat','-c','%a:%u:%g',scratch],timeout=30)\n516- if q.stdout.decode().strip()!='710:0:100000': raise BackupError('PVE_MAPPED_SCRATCH_ACCESS_CONTRACT')\n517- try:\n518- mapped_scratch_probe(node,scratch)\n519- argv,job=proven_lxc_dump_argv(t,row,node,scratch)\n520- local=work/'payload.bin'\n521- _stream_argv_to_local(node,argv,local,timeout=7200)\n522- if local.stat().st_size>int(job['archive_cap_bytes']): raise BackupError('VZDUMP_ARCHIVE_CAP_EXCEEDED')\n523- except Exception:\n524: ssh_remote(node,['rm','-rf','--',scratch],timeout=60,check=False);raise\n525- cleanup={'kind':'pve_source_dir','node':node,'path':scratch}\n526- return local,{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status'),'producer_contract':'proven-270-282-stream'},cleanup\n527- # QEMU path remains unchanged: bounded dedicated dumpdir with later cloud readback/reconstruction.\n528- srcdir=f'/var/tmp/homelab-vzdump-v2-{gen}'\n529: ssh_remote(node,['install','-d','-m','700','-o','0','-g','0',srcdir],timeout=30)\n530- try:\n531: ssh_remote(node,['vzdump',str(vmid),'--mode','snapshot','--compress','zstd','--dumpdir',srcdir],timeout=7200)\n532: p=ssh_remote(node,['find',srcdir,'-maxdepth','1','-type','f','-printf','%p\\n'],timeout=30)\n533- files=[x for x in p.stdout.decode().splitlines() if x.strip()]\n534- if len(files)!=1: raise BackupError('VZDUMP_OUTPUT_NOT_UNIQUE')\n535- src=files[0];local=work/'payload.bin'\n536- if node=='pve01': local=pathlib.Path(src)\n537- else: run([SCP,'-q','-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node+':'+src,str(local)],timeout=7200)\n538- if not pathlib.Path(local).is_file() or pathlib.Path(local).stat().st_size<=0: raise BackupError('VZDUMP_PAYLOAD_MISSING')\n539- except Exception:\n540: ssh_remote(node,['rm','-rf','--',srcdir],timeout=60,check=False);raise\n541- return pathlib.Path(local),{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status')},{'kind':'pve_source_dir','node':node,'path':srcdir}\n542-\n543-def operator_key_fingerprint():\n544- if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING')\n545- p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30)\n--\n563- return good[0]\n564-\n565-def ensure_vps_known_hosts(targets):\n566- lines=[]\n567- for t in targets:\n568: if t['type'] not in ('vps_md','vps_spb','vps_us'): continue\n569- lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint']))\n570- atomic_write(VPS_KNOWN_HOSTS,('\\n'.join(lines)+'\\n').encode(),0o600)\n571-\n572-def vps_ssh_argv(t, *, use_agent=False):\n573- a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15']\n--\n598- tag=p.stdout.decode().strip()\n599- if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT')\n600- vps_run(t,['true'],timeout=30,use_agent=False)\n601- results[t['logical_id']]=tag\n602- # US was already operator-key based; prove it agentless too.\n603: us=[x for x in targets if x['type']=='vps_us'][0]\n604- vps_run(us,['true'],timeout=30,use_agent=False)\n605- results[us['logical_id']]='ALREADY_US_OPERATOR_KEY'\n606- return results\n607-\n608-def remote_hashset(t,command):\n609- p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False)\n610- return sha256_bytes(p.stdout)\n611-\n612:def capture_vps_md(t,gen,work):\n613- files=['etc/netbird/install.conf','var/lib/netbird/state.json','var/lib/netbird/service.json','var/lib/netbird/active_profile.json','var/lib/netbird/default.json']\n614- cmd=\"sudo -n sha256sum -- \"+' '.join(shlex.quote('/'+x) for x in files)\n615- pre=remote_hashset(t,cmd)\n616- remote=shlex.join(['sudo','-n','tar','--numeric-owner','-C','/','-cf','-']+files)\n617- argv=vps_ssh_argv(t,use_agent=False)+[remote]\n618- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=600)\n619- post=remote_hashset(t,cmd)\n620- if pre!=post: raise BackupError('MD_SOURCE_CHANGED_DURING_CAPTURE')\n621- return payload,{'generation':gen,'source_class':'vps_md','consistency':'FILESET_PRE_POST_HASH_STABILITY','hashset_sha256':pre},None\n622-\n623:def capture_vps_spb(t,gen,work):\n624- hcmd=\"find /etc/nginx -xdev -type f -print0 | sort -z | xargs -0 sha256sum\"\n625- pre=remote_hashset(t,hcmd)\n626- remote=shlex.join(['tar','--numeric-owner','-C','/','-cf','-','etc/nginx'])\n627- argv=vps_ssh_argv(t,use_agent=False)+[remote]\n628- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=900)\n--\n657- if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33)\n658- b=dst.serialize(); dst.close(); src.close()\n659- ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b))\n660-'''\n661-\n662:def capture_vps_us(t,gen,work):\n663- remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT])\n664- argv=vps_ssh_argv(t,use_agent=False)+[remote]\n665- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200)\n666: return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None\n667-\n668-def _simple_shell_assignments(text):\n669- vals={}\n670- for line in text.splitlines():\n671- m=re.match(r'^\\s*(?:export\\s+)?([A-Z_][A-Z0-9_]*)=(.*)$',line)\n--\n801-\n802-def qga_exec(node,vmid,guest_argv,input_bytes=None,timeout=600):\n803- args=['qm','guest','exec',str(vmid),'--synchronous','1','--timeout',str(int(timeout))]\n804- if input_bytes is not None: args += ['--pass-stdin','1']\n805- args += ['--']+[str(x) for x in guest_argv]\n806: p=ssh_remote(node,args,input=input_bytes,timeout=timeout+60)\n807- try:j=json.loads(p.stdout)\n808- except Exception: raise BackupError('QGA_JSON')\n809- rc=j.get('exitcode',j.get('exit-code'))\n810- if rc is None: rc=0 if j.get('exited') else None\n811- if rc not in (0,None): raise BackupError('QGA_GUEST_COMMAND_FAILED',str(rc))\n--\n854-\n855-def capture_node_bundle(t,gen,work):\n856- node=t['node']; raw=node_bundle_paths(node); paths=[]\n857- for rel in raw:\n858- if node=='pve01': ok=(pathlib.Path('/')/rel).exists()\n859: else: ok=(ssh_remote(node,['test','-e','/'+rel],timeout=20,check=False).returncode==0)\n860- if ok: paths.append(rel)\n861- required={'etc/network/interfaces','etc/hosts','etc/hostname','etc/pve/nodes/'+node}\n862- if not required.issubset(set(paths)): raise BackupError('NODE_BUNDLE_REQUIRED_PATH_MISSING')\n863- payload=work/'payload.tar.zst'\n864- argv=['tar','--numeric-owner','--xattrs','--acls','-C','/','-cf','-']+paths\n--\n870-\n871-def capture_target(t,gen,work):\n872- typ=t['type']\n873- if typ=='pve_guest': return capture_pve_guest(t,gen,work)\n874- if typ=='pve_node_bundle': return capture_node_bundle(t,gen,work)\n875: if typ=='vps_md': return capture_vps_md(t,gen,work)\n876: if typ=='vps_spb': return capture_vps_spb(t,gen,work)\n877: if typ=='vps_us': return capture_vps_us(t,gen,work)\n878- if typ=='router': return capture_router(t,gen,work)\n879- if typ=='xenforo': return capture_xenforo(t,gen,work)\n880- raise BackupError('CAPTURE_TYPE_UNKNOWN')\n881-\n882-def cleanup_source(cleanup):\n883- if not cleanup: return\n884- if cleanup.get('kind')=='pve_source_dir':\n885: ssh_remote(cleanup['node'],['rm','-rf','--',cleanup['path']],timeout=60)\n886-\n887-def control_verify(t):\n888- engine=pathlib.Path('/usr/local/sbin/homelab-control-backup')\n889- if sha256_file(engine)!='0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57': raise BackupError('CONTROL_ENGINE_IDENTITY')\n890- root=pathlib.Path('/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2')\n--\n895- if j.get('status')=='SEALED' and j.get('generation'): rows.append((p.stat().st_mtime,j['generation']))\n896- except Exception: continue\n897- if not rows: raise BackupError('CONTROL_SEALED_GENERATION_MISSING')\n898- gen=sorted(rows)[-1][1]\n899- p=run([str(engine),'verify',gen],timeout=1800)\n900: write_state(t['logical_id'],'SUCCESS',generation=gen,verification='existing_control_engine_verify',last_success_epoch=epoch())\n901- return {'logical_id':t['logical_id'],'generation':gen,'result':'SUCCESS','payload_bytes':0,'replicas':2}\n902-\n903:def backup_target(t,green_verify=False):\n904- if t['type']=='control': return control_verify(t)\n905- gen=generation_id(t['logical_id']); work=ensure_dir(WORK_ROOT/gen,0o700)\n906- payload=None; cleanup=None; sealed=False\n907- try:\n908- payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen\n--\n916- ha=reconstruct_replica(manifest,manifest['replicas'][0],a); validate_payload(t,a)\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py SHA256=2b5eb4fc0f6f466a0d3e5a007af987092b6c17baa3ff9241a6afc2c2e97df1d5\n2-from __future__ import annotations\n3-import errno,fcntl,importlib.util,os,sys\n4-from pathlib import Path\n5-ROOT=Path(__file__).resolve().parent\n6-CT_ID='guest-pve01-lxc-112'\n7:LEGACY_IDS=('node-pve01','md-relay','spb-ingress','us-netbird','router-netcraze','xf-codevipe','xf-zakrutim','control-bootstrap-v2')\n8-LOCK=Path('/run/homelab-backup-heavy.lock')\n9-\n10-def _load(name,path):\n11- spec=importlib.util.spec_from_file_location(name,str(path))\n12- if spec is None or spec.loader is None: raise RuntimeError('MODULE_SPEC')\n--\n14-\n15-legacy=_load('_backupv2_legacy486',ROOT/'legacy_engine.py')\n16-ct112=_load('_backupv2_ct112_486',ROOT/'ct112_handler.py')\n17-driver=_load('_backupv2_driver454_479',ROOT/'representative_driver.py')\n18-_legacy_load_targets=legacy.load_targets\n19:_legacy_backup_target=legacy.backup_target\n20-\n21-def canonical_targets():\n22- rows=_legacy_load_targets()\n23- if not isinstance(rows,list) or len(rows)!=28: raise legacy.BackupError('TARGET_REGISTRY_COUNT')\n24- out=[dict(x) for x in rows]\n--\n35- 'guest_type':'lxc','due_seconds':due,'enabled':True,'green1':True}\n36-\n37-def load_targets():\n38- base=canonical_targets();return base+[_ct_descriptor(base)]\n39-\n40:def backup_target(t,green_verify=False):\n41- if not isinstance(t,dict) or t.get('logical_id')!=CT_ID:\n42: return _legacy_backup_target(t,green_verify)\n43- try:\n44: r=ct112.run_target(t,state_writer=legacy.write_state)\n45- return {'logical_id':CT_ID,'generation':r['generation'],'result':'SUCCESS','payload_bytes':r['payload_bytes'],\n46- 'replicas':2,'manifest_sha256':r['manifest_sha256'],'payload_sha256':r['payload_sha256']}\n47- except BaseException:\n48: try: legacy.write_state(CT_ID,'FAIL',reason='CT112_PIPELINE_FAILED')\n49- except BaseException: pass\n50- raise legacy.BackupError('CT112_PIPELINE_FAILED') from None\n51-\n52-legacy.load_targets=load_targets\n53:legacy.backup_target=backup_target\n54-\n55-def _legacy_reused_boundary_selftest():\n56- if legacy.PART_SIZE!=536870912: raise RuntimeError('LEGACY_PART_SIZE')\n57- if len(legacy.REMOTES)!=13 or len({p for _,p in legacy.REMOTES})!=13: raise RuntimeError('LEGACY_REMOTES')\n58- d=legacy.default_targets()\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py SHA256=ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793\n57- if gp==legacy_path:\n58- found[id(g)]=g\n59- if len(found)!=1:\n60- raise RuntimeError('LEGACY_PROVENANCE_CARDINALITY')\n61- g=next(iter(found.values()))\n62: required=('capture_pve_guest','backup_target','load_targets','cluster_inventory','ssh_remote',\n63- '_default_unprivileged_lxc','mapped_scratch_probe','_stream_argv_to_local','run',\n64: 'choose_due_target','write_state','epoch')\n65- if any(not callable(g.get(n)) for n in required):\n66- raise RuntimeError('LEGACY_PROVENANCE_API')\n67- original=g['capture_pve_guest']\n68- proxy=_GlobalsProxy(g)\n69- if not isinstance(g.get('SCP'),str) or not g['SCP'].startswith('/'):\n--\n73- raise RuntimeError('QEMU_ADAPTER_BIND_RESULT')\n74- patched=lxc_adapter.bind(proxy,qemu_patched)\n75- if not callable(patched):\n76- raise RuntimeError('LXC_ADAPTER_BIND_RESULT')\n77- g['capture_pve_guest']=patched\n78: scheduler_wired=scheduler_adapter.bind(proxy,g['choose_due_target'],g['write_state'])\n79: if g.get('choose_due_target') is not scheduler_wired['choose_due_target'] or g.get('write_state') is not scheduler_wired['write_state']:\n80- raise RuntimeError('SCHEDULER_ADAPTER_BIND_RESULT')\n81- return {'legacy_globals':g,'legacy':proxy,'original_capture':original,'qemu_patched_capture':qemu_patched,'patched_capture':patched,'scheduler_wired':scheduler_wired}\n82-\n83-def _load():\n84- _verify_base()\n--\n105- wrapper,lxc_adapter,qemu_adapter,scheduler_adapter,wired=_load()\n106- if wired['legacy_globals'].get('capture_pve_guest') is not wired['patched_capture']:\n107- raise RuntimeError('PATCH_BINDING')\n108- if wired['legacy_globals'].get('choose_due_target') is not wired['scheduler_wired']['choose_due_target']:\n109- raise RuntimeError('SCHEDULER_CHOOSE_BINDING')\n110: if wired['legacy_globals'].get('write_state') is not wired['scheduler_wired']['write_state']:\n111- raise RuntimeError('SCHEDULER_WRITE_BINDING')\n112- if [scheduler_adapter.backoff_seconds(n) for n in range(1,7)]!=[1800,3600,7200,14400,21600,21600]:\n113- raise RuntimeError('SCHEDULER_BACKOFF_CONTRACT')\n114: if not callable(wrapper.backup_target) or not callable(wrapper.main) or not callable(wrapper.load_targets):\n115- raise RuntimeError('WRAPPER_PUBLIC_API')\n116- print('SELFTEST_OVERLAY504=PASS',flush=True)\n117- return 0\n118-\n119-def main(argv=None):\n--\n131- try:\n132- with tempfile.TemporaryDirectory(prefix='homelab-scheduler-proof-',dir='/run') as td:\n133- g['STATE_ROOT']=pathlib.Path(td); (g['STATE_ROOT']/'state').mkdir(mode=0o700)\n134- g['epoch']=lambda: now[0]\n135- common={'payload_sha256':'a'*64,'manifest_sha256':'b'*64,'replicas':['proof-r1','proof-r2']}\n136: legacy.write_state('proof-a','SUCCESS',generation='proof-a-good',**common)\n137: legacy.write_state('proof-b','SUCCESS',generation='proof-b-good',**common)\n138- now[0]=100000\n139: legacy.write_state('proof-a','FAIL',generation='proof-a-bad',reason='PROOF_FAILURE',sealed=False)\n140- state=legacy.load_state_all(); a=state['proof-a']\n141- targets=[{'logical_id':'proof-a','enabled':True,'due_seconds':100},{'logical_id':'proof-b','enabled':True,'due_seconds':100}]\n142- first=legacy.choose_due_target(targets,state,now[0])\n143- if not isinstance(first,dict) or first.get('logical_id')!='proof-b': raise RuntimeError('SCHEDULER_PROOF_PEER_NOT_SELECTED')\n144: retry=int(a.get('retry_after_epoch',0)); preserved=int(a.get('last_success_epoch',0))\n145: if preserved!=1000 or a.get('last_success_generation')!='proof-a-good' or int(a.get('consecutive_failures',0))!=1 or retry!=101800:\n146- raise RuntimeError('SCHEDULER_PROOF_FAILURE_STATE')\n147- now[0]=retry+1\n148- second=legacy.choose_due_target(targets,legacy.load_state_all(),now[0])\n149- if not isinstance(second,dict) or second.get('logical_id')!='proof-a': raise RuntimeError('SCHEDULER_PROOF_RETRY_NOT_REELIGIBLE')\n150: legacy.write_state('proof-a','SUCCESS',generation='proof-a-recovered',**common)\n151- recovered=legacy.load_state_all()['proof-a']\n152: if int(recovered.get('consecutive_failures',-1))!=0 or int(recovered.get('retry_after_epoch',-1))!=0:\n153- raise RuntimeError('SCHEDULER_PROOF_SUCCESS_RESET')\n154: proof={'failed_target':'proof-a','peer_selected_during_backoff':first['logical_id'],'retry_after_epoch':retry,'last_success_preserved':preserved==1000,'retry_target_after_backoff':second['logical_id'],'success_resets_failures':True,'production_state_root_used':False}\n155- print('SCHEDULER_FAILURE_ISOLATION_PROOF='+json.dumps(proof,separators=(',',':'),sort_keys=True),flush=True)\n156- return 0\n157- finally:\n158- g['STATE_ROOT']=old_root; g['epoch']=old_epoch\n159- if argv==['scheduler-select-live']:\n--\n174- if len(argv)==2 and argv[0]=='verify-target':\n175- t=_target(wrapper,argv[1])\n176- if t.get('logical_id')=='guest-pve01-lxc-112':\n177- raise RuntimeError('VERIFY_SYNTHETIC_CT112_FORBIDDEN')\n178- try:\n179: out=wrapper.backup_target(t,True)\n180- except legacy.BackupError as e:\n181- print('VERIFY_TARGET_RESULT=FAIL',flush=True)\n182- print('VERIFY_TARGET_REASON='+str(e.code),flush=True)\n183- return 2\n184- print('VERIFY_TARGET_RESULT='+json.dumps(out,separators=(',',':'),sort_keys=True),flush=True)\nSOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/scheduler_adapter.py SHA256=a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96\n14- n=int(v)\n15- except Exception:\n16- return default\n17- return n if n>=0 else default\n18-\n19:def backoff_seconds(consecutive_failures:int)->int:\n20: n=max(1,_as_nonneg_int(consecutive_failures,1))\n21- return min(MAX_BACKOFF,MIN_BACKOFF*(2**min(n-1,4)))\n22-\n23-def _read_prior(legacy,logical_id):\n24- root=pathlib.Path(legacy.STATE_ROOT)/'state'\n25- p=root/(str(logical_id)+'.json')\n--\n60- if not isinstance(t,dict) or t.get('enabled') is not True: continue\n61- lid=t.get('logical_id'); seconds=_as_nonneg_int(t.get('due_seconds'),0)\n62- _need(isinstance(lid,str) and lid and seconds>0,'SCHEDULER_TARGET_SCHEMA')\n63- s=state.get(lid,{})\n64- if not isinstance(s,dict): s={}\n65: retry_after=_as_nonneg_int(s.get('retry_after_epoch'),0)\n66: if retry_after>now: continue\n67- last_success=_as_nonneg_int(s.get('last_success_epoch'),0)\n68- age=(now-last_success) if last_success and now>=last_success else (10**12 if last_success==0 else 0)\n69- if age/dev/null 2>&1; then echo \"$f\"; break; fi\n95- done\n96-}\n97:restore_ok(){\n98- local app=\"$1\" gen=\"$2\" f\n99- f=\"$(restore_dir \"$app\")/$gen.json\"\n100- [[ -s \"$f\" && ! -L \"$f\" ]] || return 1\n101: jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\n102-}\n103-remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\n104-remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\n105-rsync_core(){\n106- local src=\"$1\" dst=\"$2\"\n107- mkdir -p \"$dst\"\n108- RSYNC_RSH='ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] rsync -a --numeric-ids --delete --info=progress2 --rsync-path='sudo -n rsync' \"$CORE:$src/\" \"$dst/\"\n109-}\n--\n344- [[ -f \"$ex/$path\" && ! -L \"$ex/$path\" ]] || die RESTORE_COMPONENT_MISSING 70\n345- actual=\"$(sha256sum \"$ex/$path\"|awk '{print $1}')\"; [[ \"$actual\" == \"$sha\" ]] || die RESTORE_COMPONENT_SHA_MISMATCH 70\n346- done < <(jq -r '.components|to_entries[]|[.value.path,.value.sha256]|@tsv' \"$ex/app-generation.json\")\n347-}\n348-reconstruct_cloud_a(){\n349- local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\n350- logical=\"$(logical_id \"$app\")\"\n351- \"$TRANSPORT\" verify-sealed \"$logical\" \"$gen\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\" >/dev/null\n352: base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"\n353- manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\n354- got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\n355- : >\"$out\"; chmod 0600 \"$out\"\n356- while IFS=$'\\t' read -r idx name size sha; do\n357- [[ \"$idx\" =~ ^[0-9]{8}$ && \"$name\" == part-????????.bin && \"$size\" =~ ^[1-9][0-9]*$ && \"$sha\" =~ ^[0-9a-f]{64}$ ]] || die RESTORE_A_MANIFEST_ROW_INVALID 70\n358- part=\"$(dirname \"$out\")/$name\"; rm -f \"$part\"; rclone cat \"${base%/}/payload.parts/$name\" >\"$part\"\n359- [[ \"$(stat -c %s \"$part\")\" == \"$size\" ]] || die RESTORE_A_PART_SIZE_MISMATCH 70\n360- actual=\"$(sha256sum \"$part\"|awk '{print $1}')\"; [[ \"$actual\" == \"$sha\" ]] || die RESTORE_A_PART_SHA_MISMATCH 70\n--\n505- scp \"${SCP_BASE[@]}\" \"$ex/immich/db/immich.dump\" \"$CORE:$remote_dump\" >/dev/null\n506- scp \"${SCP_BASE[@]}\" \"$ACTIVE_STAGE/postgres-image.txt\" \"$CORE:$remote_img\" >/dev/null\n507- if ! ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- validate-immich-db \"$gen\" \"$remote_dump\" \"$expect\" \"$remote_img\" <\"$CORE_HELPER\" >/dev/null; then ssh \"${SSH_BASE[@]}\" \"$CORE\" rm -f -- \"$remote_dump\" \"$remote_img\" >/dev/null 2>&1 || true; die IMMICH_POSTGRES_IMPORT_FAILED 70; fi\n508- ssh \"${SSH_BASE[@]}\" \"$CORE\" rm -f -- \"$remote_dump\" \"$remote_img\" >/dev/null\n509- checks=\"$(jq -n --argjson n \"$expect\" '[{name:\"postgres_import\",result:\"OK\",table_count:$n},{name:\"media_manifest\",result:\"OK\"},{name:\"config_manifest\",result:\"OK\"}]')\" ;;\n510- esac\n511- printf '%s\\n' \"$checks\"\n512-}\n513:restore_validate(){\n514- local app=\"$1\" gen=\"$2\" logical state work payload ex expected_sha checks out tmp\n515- valid_generation \"$gen\" || die GENERATION_INVALID 64\n516- logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\n517- [[ -s \"$state\" && ! -L \"$state\" ]] || die RESTORE_STATE_MISSING 65\n518- jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$state\" >/dev/null || die RESTORE_REQUIRES_SEALED 65\n519: if restore_ok \"$app\" \"$gen\"; then printf 'RESTORE_VALIDATION_STATUS=OK\\nAPP=%s\\nGENERATION=%s\\nRESULT=ALREADY_VALIDATED\\n' \"$app\" \"$gen\"; return 0; fi\n520- work=\"$RESTORE_WORK_ROOT/$logical/$gen\"; rm -rf --one-file-system \"$work\" 2>/dev/null || true; mkdir -p \"$work/extract\"; chmod 0700 \"$RESTORE_WORK_ROOT\" \"$RESTORE_WORK_ROOT/$logical\" \"$work\" 2>/dev/null || true; ACTIVE_STAGE=\"$work\"\n521- payload=\"$work/payload.tar.zst\"; reconstruct_cloud_a \"$app\" \"$gen\" \"$state\" \"$payload\"\n522- expected_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; [[ \"$(sha256sum \"$payload\"|awk '{print $1}')\" == \"$expected_sha\" ]] || die RESTORE_PAYLOAD_SHA_MISMATCH 70\n523- zstd -t \"$payload\" >/dev/null; zstd -dc \"$payload\" | tar -xf - -C \"$work/extract\"; ex=\"$work/extract\"\n524- verify_binding_components \"$ex\" \"$logical\" \"$gen\"\n525- checks=\"$(semantic_validate \"$app\" \"$gen\" \"$ex\")\"\n526- out=\"$(restore_dir \"$app\")/$gen.json\"; mkdir -p \"$(dirname \"$out\")\"; chmod 0700 \"$RESTORE_STATE_ROOT\" \"$(dirname \"$out\")\"\n527- tmp=\"${out}.new.$$\"\n528: jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\n529- chmod 0600 \"$tmp\"; mv -f \"$tmp\" \"$out\"\n530: jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\n531- rm -rf --one-file-system \"$work\"; ACTIVE_STAGE=\"\"\n532- printf 'RESTORE_VALIDATION_STATUS=OK\\nAPP=%s\\nGENERATION=%s\\nEVIDENCE=%s\\n' \"$app\" \"$gen\" \"$out\"\n533-}\n534-backup_one(){\n535- local app=\"$1\" logical pending_count state gen payload latest orphan_count\n536- logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\n537- mapfile -t pending < <(latest_incomplete \"$app\"); pending_count=\"${#pending[@]}\"; (( pending_count <= 1 )) || die \"MULTIPLE_INCOMPLETE_GENERATIONS_$app\" 65\n538- if (( pending_count == 1 )); then\n539- state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\n540- \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\n541: printf 'STAGE=RESTORE_VALIDATION\\n'; restore_validate \"$app\" \"$gen\"\n542- printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return\n543- fi\n544- plan \"$app\" >/dev/null\n545- latest=\"$(latest_sealed_state \"$app\")\"\n546: if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\n547- if [[ \"$app\" == scrutiny ]]; then\n548- orphan_count=\"$(find \"$STAGE_ROOT/$logical\" -maxdepth 1 -type d -name '.capture-app-scrutiny-*' | wc -l)\"\n549- (( orphan_count == 0 )) || die SCRUTINY_ORPHAN_CAPTURE_REQUIRES_CONTROLLED_RECOVERY 65\n550- fi\n551- gen=\"${logical}-pve01-$(date -u +%Y%m%dT%H%M%SZ)\"; valid_generation \"$gen\" || die GENERATED_ID_INVALID 65\n552- printf 'RESUME=NO\\nAPP=%s\\nGENERATION=%s\\nSTAGE=CAPTURE\\n' \"$app\" \"$gen\"; capture_new \"$app\" \"$gen\"; payload=\"$PAYLOAD\"\n553- printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\n554: printf 'STAGE=RESTORE_VALIDATION\\n'; restore_validate \"$app\" \"$gen\"\n555- state=\"$STATE_ROOT/$logical/$gen.json\"; jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .local_payload_present==false and .seal_written==true' \"$state\" >/dev/null\n556- printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"\n557-}\n558-plan(){\n559- local app=\"$1\" logical free\n560- logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\n561- [[ \"$free\" =~ ^[1-9][0-9]+$ ]] || die STAGING_FREE_INVALID 65\n562- local source_bytes=0 required=0\n--\n589- ;;\n590- skladchik-reports-monitor) remote_edge probe plan >/dev/null ;;\n591- esac\n592- printf 'PLAN_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nSOURCE_HOST=%s\\nSTAGING_FREE_BYTES=%s\\nSOURCE_BYTES=%s\\nREQUIRED_STAGING_BYTES=%s\\n' \"$app\" \"$logical\" \"$(source_host \"$app\")\" \"$free\" \"$source_bytes\" \"$required\"\n593-}\n594-latest_sealed(){\n595- local app=\"$1\" f gen\n596- f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\n597: restore_ok \"$app\" \"$gen\" || die \"LATEST_RESTORE_NOT_OK_$app\" 65\n598: printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\n599-}\n600-recover_scrutiny_orphan(){\n601- local gen=\"$1\" started=\"$2\" qstart=\"$3\" qend=\"$4\" expected_config=\"$5\" expected_influx=\"$6\"\n602- local completed unitsha ref iid digest repodigests isize scdb idb devices migrations settings influx_tables tsm_files tsm_bytes wal_files wal_bytes series trange cfgsha boltbytes sqlitebytes image_sha meta_sha config_manifest_sha influx_manifest_sha validation_manifest_sha unit_copy_sha comps report actual_config actual_influx logical state restore\n603- logical=app-scrutiny\n604- valid_generation \"$gen\" || die GENERATION_INVALID 64\n605- [[ \"$gen\" == app-scrutiny-pve01-* ]] || die SCRUTINY_RECOVERY_GENERATION_INVALID 64\n606- [[ \"$started\" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && \"$qstart\" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && \"$qend\" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die SCRUTINY_RECOVERY_TIMESTAMP_INVALID 64\n--\n639- if [[ -s \"$state\" && ! -L \"$state\" ]] && jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$state\" >/dev/null 2>&1; then\n640- printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=scrutiny\\nGENERATION=%s\\n' \"$gen\"\n641- else\n642- [[ -s \"$PAYLOAD\" && ! -L \"$PAYLOAD\" ]] || die SCRUTINY_RECOVERY_PAYLOAD_MISSING_FOR_TRANSPORT 65\n643- printf 'STAGE=TRANSPORT_RESUME\\n'\n644- \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\n645- fi\n646- printf 'STAGE=RESTORE_VALIDATION\\n'\n647: restore_validate scrutiny \"$gen\"\n648- [[ -s \"$state\" && ! -L \"$state\" ]] || die SCRUTINY_RECOVERY_SEALED_STATE_MISSING 70\n649- jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$state\" >/dev/null || die SCRUTINY_RECOVERY_SEALED_STATE_INVALID 70\n650: [[ -s \"$restore\" && ! -L \"$restore\" ]] || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_MISSING 70\n651: jq -e '.result==\"OK\" and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true' \"$restore\" >/dev/null || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_INVALID 70\n652- printf 'BACKUP_STATUS=SEALED\\nAPP=scrutiny\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$gen\"\n653-}\n654-\n655-ACTION=\"${1:-}\"\n656-case \"$ACTION\" in\n657- plan) [[ \"$#\" == 2 ]] || die USAGE_PLAN 64; plan \"$2\" ;;\n658- backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\n659: restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\n660- recover-scrutiny-orphan) [[ \"$#\" == 7 ]] || die USAGE_RECOVER_SCRUTINY_ORPHAN 64; recover_scrutiny_orphan \"$2\" \"$3\" \"$4\" \"$5\" \"$6\" \"$7\" ;;\n661: latest-sealed) [[ \"$#\" == 2 ]] || die USAGE_LATEST_SEALED 64; latest_sealed \"$2\" ;;\n662- *) die ACTION_INVALID 64 ;;\n663-esac\nAPP_RESTORE_CONTRACT_SOURCE_END=1\nAPP_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json\nAPP_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc\nAPP_GENERATION=app-skladchik-reports-monitor-pve01-20260921T175029Z\nAPP_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json\nAPP_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e\nAPP_RESTORE_KEYS_BEGIN=1\n[\"app\",\"checks\",\"generation\",\"logical_id\",\"payload_sha256\",\"payload_sha256_verified\",\"restore_source\",\"result\",\"schema\",\"transport_state\",\"transport_state_file\",\"validated_at_utc\"]\nAPP_RESTORE_KEYS_END=1\nAPP_RESTORE_SAFE_VIEW_BEGIN=1\n{\"schema\":\"homelab-app-restore-validation-v1\",\"result\":\"OK\",\"status\":null,\"logical_id\":\"app-skladchik-reports-monitor\",\"generation\":\"app-skladchik-reports-monitor-pve01-20260921T175029Z\",\"transport_state\":\"SEALED\",\"transport_state_file\":\"/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json\",\"restore_source\":\"Cloud A\",\"payload_sha256\":\"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc\",\"payload_sha256_verified\":true,\"payload_decompression_verified\":null,\"archive_recovery_readback_verified\":null,\"parts_manifest_verified\":null,\"bounded_staging\":null,\"full_local_payload_written\":null,\"tar_test_ok\":null,\"required_files_ok\":null,\"cookie_jar_ok\":null,\"sqlite_ok\":null,\"validated_at_utc\":\"2026-09-21T17:50:46Z\"}\nAPP_RESTORE_SAFE_VIEW_END=1\nAPP_RESTORE_BASE_CONTRACT_OK=true\nUS_NETBIRD_STATE_BEGIN=1\nUS_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26\n{\"consecutive_failures\":14,\"failure_backoff_seconds\":21600,\"generation\":\"v2-us-netbird-20260922T055316Z-d5c92acdd2e8\",\"last_attempt_epoch\":1790056411,\"last_success_epoch\":1789704670,\"last_success_generation\":\"v2-us-netbird-20260918T041032Z-d7648be4c16b\",\"last_success_green_verified\":false,\"last_success_manifest_sha256\":\"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5\",\"last_success_payload_sha256\":\"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf\",\"last_success_replicas\":[{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"},{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"}],\"logical_id\":\"us-netbird\",\"reason\":\"PIPE_FAILED\",\"result\":\"FAIL\",\"retry_after_epoch\":1790078011,\"schema\":1,\"sealed\":false,\"updated_at_utc\":\"2026-09-22T05:53:31Z\"}\nUS_NETBIRD_STATE_END=1\nPIPE_TO_ZSTD_SOURCE_BEGIN=1\n if not c: return None\n c.sort(key=lambda x:(x[0],x[1],x[2]))\n return c[0][3]\n\ndef run(argv, *, input=None, timeout=300, check=True, env=None):\n try:\n p=subprocess.run(argv,input=input,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)\n except subprocess.TimeoutExpired: raise BackupError('COMMAND_TIMEOUT')\n if check and p.returncode!=0: raise BackupError('COMMAND_FAILED',str(p.returncode))\n return p\n\ndef stream_process_to_file(argv,out_path,timeout=1800,input=None):\n out_path=pathlib.Path(out_path)\n with open(out_path,'wb') as f:\n try:p=subprocess.run(argv,input=input,stdout=f,stderr=subprocess.PIPE,timeout=timeout)\n except subprocess.TimeoutExpired: raise BackupError('STREAM_COMMAND_TIMEOUT')\n if p.returncode!=0: raise BackupError('STREAM_COMMAND_FAILED',str(p.returncode))\n\ndef pipe_to_zstd(argv,out_path,timeout=1800,input=None):\n zstd=shutil.which('zstd') or '/usr/bin/zstd'\n try:\n src=subprocess.Popen(argv,stdin=subprocess.PIPE if input is not None else None,stdout=subprocess.PIPE,stderr=subprocess.PIPE)\n z=subprocess.Popen([zstd,'-T0','-3','-q','-o',str(out_path)],stdin=src.stdout,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE)\n if src.stdout: src.stdout.close()\n if input is not None:\n src.stdin.write(input); src.stdin.close()\n zerr=z.communicate(timeout=timeout)[1]\n try: src.wait(timeout=60)\n except subprocess.TimeoutExpired:\n src.kill(); src.wait(); raise BackupError('PIPE_SOURCE_TIMEOUT')\n serr=src.stderr.read() if src.stderr else b''\n except subprocess.TimeoutExpired:\n for q in (src,z):\n try:q.kill()\n except Exception:pass\n raise BackupError('PIPE_TIMEOUT')\n if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}')\n\ndef ensure_dir(p,mode=0o700):\n p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p\n\ndef atomic_write(path,data,mode=0o600):\n path=pathlib.Path(path); ensure_dir(path.parent,0o700)\n fd,tmp=tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent))\nPIPE_TO_ZSTD_SOURCE_END=1\nUS_REMOTE_SCRIPT_SOURCE_BEGIN=1\n if pre!=post: raise BackupError('SPB_SOURCE_CHANGED_DURING_CAPTURE')\n return payload,{'generation':gen,'source_class':'vps_spb','consistency':'STATIC_CONFIG_FILESET_PRE_POST_HASH_STABILITY','tls_recovery':'REISSUE_TLS_ON_RESTORE','hashset_sha256':pre},None\n\nUS_REMOTE_SCRIPT=r'''import io,os,sqlite3,sys,tarfile,time,json,hashlib\npaths=['/opt/netbird/docker-compose.yml','/opt/netbird/config.yaml']\ndata_root='/var/lib/docker/volumes/netbird_netbird_data/_data'\nle_root='/var/lib/docker/volumes/netbird_netbird_traefik_letsencrypt/_data'\ndbs=['events.db','store.db','geonames_20260602.db','idp.db']\nnon=['GeoLite2-City_20260602.mmdb']\nout=sys.stdout.buffer\nwith tarfile.open(fileobj=out,mode='w|') as tf:\n for p in paths:\n if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False)\n for name in non:\n p=os.path.join(data_root,name)\n if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False)\n ac=os.path.join(le_root,'acme.json')\n if os.path.isfile(ac):\n st=os.stat(ac)\n if (st.st_mode & 0o777)!=0o600: raise SystemExit(31)\n json.load(open(ac,'r'))\n tf.add(ac,arcname=ac.lstrip('/'),recursive=False)\n for name in dbs:\n srcp=os.path.join(data_root,name)\n src=sqlite3.connect('file:'+srcp+'?mode=ro',uri=True)\n if src.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(32)\n dst=sqlite3.connect(':memory:'); src.backup(dst)\n if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33)\n b=dst.serialize(); dst.close(); src.close()\n ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b))\n'''\n\ndef capture_vps_us(t,gen,work):\n remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT])\n argv=vps_ssh_argv(t,use_agent=False)+[remote]\n payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200)\n return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None\n\ndef _simple_shell_assignments(text):\nUS_REMOTE_SCRIPT_SOURCE_END=1\nVPS_SSH_SOURCE_BEGIN=1\ndef operator_key_fingerprint():\n if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING')\n p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30)\n s=p.stdout.decode('utf-8','replace')\n m=re.search(r'(SHA256:[A-Za-z0-9+/]+)',s)\n if not m or m.group(1)!=OPERATOR_KEY_FINGERPRINT: raise BackupError('OPERATOR_KEY_FINGERPRINT_MISMATCH')\n return m.group(1)\n\ndef scan_hostkey_line(host,port,expected):\n p=run([SSH_KEYSCAN,'-T','8','-p',str(port),'-t','ed25519',host],timeout=15)\n lines=[x.strip() for x in p.stdout.decode().splitlines() if x.strip() and not x.startswith('#')]\n good=[]\n for line in lines:\n f=line.split()\n if len(f)<3 or f[1]!='ssh-ed25519': continue\n try: blob=base64.b64decode(f[2])\n except Exception: continue\n fp='SHA256:'+base64.b64encode(hashlib.sha256(blob).digest()).decode().rstrip('=')\n if fp==expected: good.append(line)\n if len(good)!=1: raise BackupError('HOSTKEY_PIN_MISMATCH')\n return good[0]\n\ndef ensure_vps_known_hosts(targets):\n lines=[]\n for t in targets:\n if t['type'] not in ('vps_md','vps_spb','vps_us'): continue\n lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint']))\n atomic_write(VPS_KNOWN_HOSTS,('\\n'.join(lines)+'\\n').encode(),0o600)\n\ndef vps_ssh_argv(t, *, use_agent=False):\n a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15']\n if use_agent:\n a += ['-o','IdentitiesOnly=no']\n else:\n a += ['-i',str(OPERATOR_KEY),'-o','IdentitiesOnly=yes','-o','IdentityAgent=none']\n a += [t['user']+'@'+t['host']]\n return a\n\ndef vps_run(t, remote_argv, *, input=None, timeout=300, use_agent=False, check=True):\n remote=shlex.join([str(x) for x in remote_argv])\n return run(vps_ssh_argv(t,use_agent=use_agent)+[remote],input=input,timeout=timeout,check=check)\n\ndef bootstrap_vps_keys(targets):\n operator_key_fingerprint()\n ensure_vps_known_hosts(targets)\n pub=run(['ssh-keygen','-y','-f',str(OPERATOR_KEY)],timeout=30).stdout.strip()+b'\\n'\n results={}\n for t in targets:\n if t['type'] not in ('vps_md','vps_spb'): continue\n probe=vps_run(t,['true'],timeout=30,use_agent=False,check=False)\n if probe.returncode==0:\n results[t['logical_id']]='ALREADY_OPERATOR_KEY'; continue\n if not os.environ.get('SSH_AUTH_SOCK'): raise BackupError('FORWARDED_AGENT_REQUIRED_FOR_BOOTSTRAP')\n script=b\"umask 077; mkdir -p \\\"$HOME/.ssh\\\"; chmod 700 \\\"$HOME/.ssh\\\"; touch \\\"$HOME/.ssh/authorized_keys\\\"; chmod 600 \\\"$HOME/.ssh/authorized_keys\\\"; IFS= read -r k; if grep -qF -- \\\"$k\\\" \\\"$HOME/.ssh/authorized_keys\\\"; then printf 'ALREADY\\\\n'; else printf '%s\\\\n' \\\"$k\\\" >> \\\"$HOME/.ssh/authorized_keys\\\"; printf 'ADDED\\\\n'; fi\"\n p=vps_run(t,['sh','-c',script.decode()],input=pub,timeout=45,use_agent=True)\n tag=p.stdout.decode().strip()\n if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT')\n vps_run(t,['true'],timeout=30,use_agent=False)\n results[t['logical_id']]=tag\n # US was already operator-key based; prove it agentless too.\n us=[x for x in targets if x['type']=='vps_us'][0]\n vps_run(us,['true'],timeout=30,use_agent=False)\n results[us['logical_id']]='ALREADY_US_OPERATOR_KEY'\n return results\n\ndef remote_hashset(t,command):\n p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False)\n return sha256_bytes(p.stdout)\nVPS_SSH_SOURCE_END=1\nUS_NETBIRD_PID_JOURNAL_BEGIN=1\nPID_BEGIN=936114\n2026-09-21T20:30:36+03:00 pve01 homelab-backup-v2[936114]: SELFTEST_REPLACEMENT486=PASS\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED\nPID_END=936114\nPID_BEGIN=2121414\n2026-09-22T02:36:58+03:00 pve01 homelab-backup-v2[2121414]: SELFTEST_REPLACEMENT486=PASS\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED\nPID_END=2121414\nPID_BEGIN=3378768\n2026-09-22T08:53:16+03:00 pve01 homelab-backup-v2[3378768]: SELFTEST_REPLACEMENT486=PASS\n2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_TARGET=us-netbird\n2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_RESULT=FAIL\n2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_REASON=PIPE_FAILED\nPID_END=3378768\nUS_NETBIRD_PID_JOURNAL_END=1\nUS_NETBIRD_WORK_ARTIFACTS_BEGIN=1\nf|1790056411.7319950600|792|/var/lib/homelab-backup-v2/state/us-netbird.json\nf|1789642259.3981707900|1255|/var/lib/homelab-backup-v2/cold-bootstrap/us-netbird.json\nd|1790056411.7288067540|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260922T055316Z-d5c92acdd2e8\nd|1790033833.7009700260|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T233658Z-709cf01e1256\nd|1790011851.2302881580|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T173036Z-1128898e4cc8\nd|1789989471.8069593230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T111736Z-c5a5c5e71776\nd|1789955807.6619889330|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T015632Z-dc499030a55f\nd|1789932571.7940614350|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T192916Z-ca2592a81a30\nd|1789908991.7828657500|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T125616Z-8caeb0b6c7b7\nd|1789886227.6266376830|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T063652Z-e6c72515a153\nd|1789862827.7065704150|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T000652Z-7ecdcfcafc12\nd|1789839091.8314245860|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T173116Z-cfc70719f4e9\nd|1789822687.5806853640|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T125752Z-a4efa5c07171\nd|1789814653.6399387990|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T104358Z-b2232f971dbc\nd|1789810393.6722341230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T093258Z-64f5c2972565\nd|1789807993.6075344840|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T085258Z-36f858f91de5\nUS_NETBIRD_WORK_ARTIFACTS_END=1\nUS_NETBIRD_READONLY_PROBES_BEGIN=1\nUS_NETBIRD_READONLY_PROBE_RC=41\nPROBE_TARGET_FOUND=true\nPROBE_TYPE=vps_us\nPROBE_SSH_TRUE_EXCEPTION=BackupError\nPROBE_SSH_TRUE_CODE=COMMAND_FAILED\nPROBE_SSH_TRUE_DETAIL=255\nUS_NETBIRD_READONLY_PROBE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nUS_NETBIRD_READONLY_PROBES_END=1\nUS_NETBIRD_PIPE_BASELINE_CLASS=SSH_BASELINE_FAILURE\nDECISION=PASS_BACKUP_1117_APP_CONTRACT_AND_PIPE_BASELINE_CLASSIFIED\nNEXT_GATE=REPAIR_ONLY_BACKUP_US_NETBIRD_AND_AUDIT_CANONICAL_CLOUDB_SOURCE_OFFHOST\nTASK_RESULT=PASS_HOMELAB_BACKUP_1116_CONTRACT_PIPE_RCA\nCHANGES_MADE_BY_1117=false\nPRODUCT_MUTATION_BY_1117=false\nVM_MUTATION_BY_1117=false\nCLOUD_MUTATION_BY_1117=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index a369975d..790521b2 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1 -STATUS=FAIL -RC=3 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1 +STATUS=OK +RC=0 HOST=pve01 MODE=read-only -COMPONENT=homelab-backup-1115-root-rca +COMPONENT=homelab-backup-1116-contract-pipe-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=a475f4109c57c07002ec72ba49450725574b1c08d698b9878c58257e7cc704c8 +COMMAND_SHA256=0ba8e65003a7561a9e01930aa9eec5d2b6a6cd73c9f05d217fca4a399c992fed DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=3 +COMMAND_RC=0 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,599 +24,400 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=17278f059b28a5b338724f4e22fad8aa2db50e23143eeee6635c075bf93a9691 -SANITIZED_OUTPUT_SHA256=17278f059b28a5b338724f4e22fad8aa2db50e23143eeee6635c075bf93a9691 +RAW_EVIDENCE_SHA256=8314f5500f36a8ab9866f3f778ae569bf18655d73ade7aca6ada77060826a746 +SANITIZED_OUTPUT_SHA256=1c8576db6611dd527a417a275078b31f952af42ddef0dd6dd0fa1661455bb847 OUTPUT_BEGIN -WORKERS2_BACKUP_1115_ROOT_RCA_BEGIN=1 -COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1 +WORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1 MODE=read-only -COMPONENT=homelab-backup-1115-root-rca -MUTATION_BOUNDARY=NONE;READ_ONLY_US_NETBIRD_FAILURE_AND_CANONICAL_SKLADCHIK_EVIDENCE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP;NO_RESTORE_EXECUTION;NO_GIT_FETCH;NO_GIT_WRITE;NO_CLOUD_WRITE;NO_REMOTE_MUTATION +COMPONENT=homelab-backup-1116-contract-pipe-rca +MUTATION_BOUNDARY=NONE;READ_ONLY_APP_RESTORE_CONTRACT_AND_US_NETBIRD_PIPE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP_TARGET_EXECUTION;NO_RESTORE_EXECUTION;NO_GIT_WRITE;NO_CLOUD_WRITE;REMOTE_PROBES_READ_ONLY ERROR_REGISTER_CHECK=OK REFERENCE_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 -AUDIT_SHA256=5777bbb204708ffcebba0753506b819833b76370ecda59b4574e1aff3b9e4593 +LEGACY_SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7 +APP_WORKER_SHA256=1d6c4a04f762e2a33561c109b0c0a4878caaa48627903e7e78fdf5f22d79f196 SCHEDULER_RC=0 SELFTEST_REPLACEMENT486=PASS SCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET -US_NETBIRD_STATE_SAFE_BEGIN=1 +APP_RESTORE_CONTRACT_SOURCE_BEGIN=1 +89-} +90-latest_sealed_state(){ +91- local app="$1" d +92- d="$(state_dir "$app")"; [[ -d "$d" ]] || return 0 +93- find "$d" -maxdepth 1 -type f -name '*.json' -printf '%T@|%p\n' | LC_ALL=C sort -t'|' -k1,1nr | while IFS='|' read -r _ f; do +94- if jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$f" >/dev/null 2>&1; then echo "$f"; break; fi +95- done +96-} +97:restore_ok(){ +98- local app="$1" gen="$2" f +99- f="$(restore_dir "$app")/$gen.json" +100- [[ -s "$f" && ! -L "$f" ]] || return 1 +101: jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +102-} +103-remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +104-remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +105-rsync_core(){ +106- local src="$1" dst="$2" +107- mkdir -p "$dst" +108- RSYNC_RSH='ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] rsync -a --numeric-ids --delete --info=progress2 --rsync-path='sudo -n rsync' "$CORE:$src/" "$dst/" +109-} +-- +344- [[ -f "$ex/$path" && ! -L "$ex/$path" ]] || die RESTORE_COMPONENT_MISSING 70 +345- actual="$(sha256sum "$ex/$path"|awk '{print $1}')"; [[ "$actual" == "$sha" ]] || die RESTORE_COMPONENT_SHA_MISMATCH 70 +346- done < <(jq -r '.components|to_entries[]|[.value.path,.value.sha256]|@tsv' "$ex/app-generation.json") +347-} +348-reconstruct_cloud_a(){ +349- local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +350- logical="$(logical_id "$app")" +351- "$TRANSPORT" verify-sealed "$logical" "$gen" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" >/dev/null +352: base="$(jq -er '.remote_a_base' "$state")"; expected_msha="$(jq -er '.parts_manifest_sha256' "$state")" +353- manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +354- got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +355- : >"$out"; chmod 0600 "$out" +356- while IFS=$'\t' read -r idx name size sha; do +357- [[ "$idx" =~ ^[0-9]{8}$ && "$name" == part-????????.bin && "$size" =~ ^[1-9][0-9]*$ && "$sha" =~ ^[0-9a-f]{64}$ ]] || die RESTORE_A_MANIFEST_ROW_INVALID 70 +358- part="$(dirname "$out")/$name"; rm -f "$part"; rclone cat "${base%/}/payload.parts/$name" >"$part" +359- [[ "$(stat -c %s "$part")" == "$size" ]] || die RESTORE_A_PART_SIZE_MISMATCH 70 +360- actual="$(sha256sum "$part"|awk '{print $1}')"; [[ "$actual" == "$sha" ]] || die RESTORE_A_PART_SHA_MISMATCH 70 +-- +505- scp "${SCP_BASE[@]}" "$ex/immich/db/immich.dump" "$CORE:$remote_dump" >/dev/null +506- scp "${SCP_BASE[@]}" "$ACTIVE_STAGE/postgres-image.txt" "$CORE:$remote_img" >/dev/null +507- if ! ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- validate-immich-db "$gen" "$remote_dump" "$expect" "$remote_img" <"$CORE_HELPER" >/dev/null; then ssh "${SSH_BASE[@]}" "$CORE" rm -f -- "$remote_dump" "$remote_img" >/dev/null 2>&1 || true; die IMMICH_POSTGRES_IMPORT_FAILED 70; fi +508- ssh "${SSH_BASE[@]}" "$CORE" rm -f -- "$remote_dump" "$remote_img" >/dev/null +509- checks="$(jq -n --argjson n "$expect" '[{name:"postgres_import",result:"OK",table_count:$n},{name:"media_manifest",result:"OK"},{name:"config_manifest",result:"OK"}]')" ;; +510- esac +511- printf '%s\n' "$checks" +512-} +513:restore_validate(){ +514- local app="$1" gen="$2" logical state work payload ex expected_sha checks out tmp +515- valid_generation "$gen" || die GENERATION_INVALID 64 +516- logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +517- [[ -s "$state" && ! -L "$state" ]] || die RESTORE_STATE_MISSING 65 +518- jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null || die RESTORE_REQUIRES_SEALED 65 +519: if restore_ok "$app" "$gen"; then printf 'RESTORE_VALIDATION_STATUS=OK\nAPP=%s\nGENERATION=%s\nRESULT=ALREADY_VALIDATED\n' "$app" "$gen"; return 0; fi +520- work="$RESTORE_WORK_ROOT/$logical/$gen"; rm -rf --one-file-system "$work" 2>/dev/null || true; mkdir -p "$work/extract"; chmod 0700 "$RESTORE_WORK_ROOT" "$RESTORE_WORK_ROOT/$logical" "$work" 2>/dev/null || true; ACTIVE_STAGE="$work" +521- payload="$work/payload.tar.zst"; reconstruct_cloud_a "$app" "$gen" "$state" "$payload" +522- expected_sha="$(jq -er '.archive_sha256' "$state")"; [[ "$(sha256sum "$payload"|awk '{print $1}')" == "$expected_sha" ]] || die RESTORE_PAYLOAD_SHA_MISMATCH 70 +523- zstd -t "$payload" >/dev/null; zstd -dc "$payload" | tar -xf - -C "$work/extract"; ex="$work/extract" +524- verify_binding_components "$ex" "$logical" "$gen" +525- checks="$(semantic_validate "$app" "$gen" "$ex")" +526- out="$(restore_dir "$app")/$gen.json"; mkdir -p "$(dirname "$out")"; chmod 0700 "$RESTORE_STATE_ROOT" "$(dirname "$out")" +527- tmp="${out}.new.$$" +528: jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +529- chmod 0600 "$tmp"; mv -f "$tmp" "$out" +530: jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +531- rm -rf --one-file-system "$work"; ACTIVE_STAGE="" +532- printf 'RESTORE_VALIDATION_STATUS=OK\nAPP=%s\nGENERATION=%s\nEVIDENCE=%s\n' "$app" "$gen" "$out" +533-} +534-backup_one(){ +535- local app="$1" logical pending_count state gen payload latest orphan_count +536- logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +537- mapfile -t pending < <(latest_incomplete "$app"); pending_count="${#pending[@]}"; (( pending_count <= 1 )) || die "MULTIPLE_INCOMPLETE_GENERATIONS_$app" 65 +538- if (( pending_count == 1 )); then +539- state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +540- "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +541: printf 'STAGE=RESTORE_VALIDATION\n'; restore_validate "$app" "$gen" +542- printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return +543- fi +544- plan "$app" >/dev/null +545- latest="$(latest_sealed_state "$app")" +546: if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +547- if [[ "$app" == scrutiny ]]; then +548- orphan_count="$(find "$STAGE_ROOT/$logical" -maxdepth 1 -type d -name '.capture-app-scrutiny-*' | wc -l)" +549- (( orphan_count == 0 )) || die SCRUTINY_ORPHAN_CAPTURE_REQUIRES_CONTROLLED_RECOVERY 65 +550- fi +551- gen="${logical}-pve01-$(date -u +%Y%m%dT%H%M%SZ)"; valid_generation "$gen" || die GENERATED_ID_INVALID 65 +552- printf 'RESUME=NO\nAPP=%s\nGENERATION=%s\nSTAGE=CAPTURE\n' "$app" "$gen"; capture_new "$app" "$gen"; payload="$PAYLOAD" +553- printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +554: printf 'STAGE=RESTORE_VALIDATION\n'; restore_validate "$app" "$gen" +555- state="$STATE_ROOT/$logical/$gen.json"; jq -e '.status=="SEALED" and .cloud_quorum==2 and .local_payload_present==false and .seal_written==true' "$state" >/dev/null +556- printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen" +557-} +558-plan(){ +559- local app="$1" logical free +560- logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +561- [[ "$free" =~ ^[1-9][0-9]+$ ]] || die STAGING_FREE_INVALID 65 +562- local source_bytes=0 required=0 +-- +589- ;; +590- skladchik-reports-monitor) remote_edge probe plan >/dev/null ;; +591- esac +592- printf 'PLAN_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nSOURCE_HOST=%s\nSTAGING_FREE_BYTES=%s\nSOURCE_BYTES=%s\nREQUIRED_STAGING_BYTES=%s\n' "$app" "$logical" "$(source_host "$app")" "$free" "$source_bytes" "$required" +593-} +594-latest_sealed(){ +595- local app="$1" f gen +596- f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +597: restore_ok "$app" "$gen" || die "LATEST_RESTORE_NOT_OK_$app" 65 +598: printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +599-} +600-recover_scrutiny_orphan(){ +601- local gen="$1" started="$2" qstart="$3" qend="$4" expected_config="$5" expected_influx="$6" +602- local completed unitsha ref iid digest repodigests isize scdb idb devices migrations settings influx_tables tsm_files tsm_bytes wal_files wal_bytes series trange cfgsha boltbytes sqlitebytes image_sha meta_sha config_manifest_sha influx_manifest_sha validation_manifest_sha unit_copy_sha comps report actual_config actual_influx logical state restore +603- logical=app-scrutiny +604- valid_generation "$gen" || die GENERATION_INVALID 64 +605- [[ "$gen" == app-scrutiny-pve01-* ]] || die SCRUTINY_RECOVERY_GENERATION_INVALID 64 +606- [[ "$started" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && "$qstart" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && "$qend" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die SCRUTINY_RECOVERY_TIMESTAMP_INVALID 64 +-- +639- if [[ -s "$state" && ! -L "$state" ]] && jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null 2>&1; then +640- printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=scrutiny\nGENERATION=%s\n' "$gen" +641- else +642- [[ -s "$PAYLOAD" && ! -L "$PAYLOAD" ]] || die SCRUTINY_RECOVERY_PAYLOAD_MISSING_FOR_TRANSPORT 65 +643- printf 'STAGE=TRANSPORT_RESUME\n' +644- "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +645- fi +646- printf 'STAGE=RESTORE_VALIDATION\n' +647: restore_validate scrutiny "$gen" +648- [[ -s "$state" && ! -L "$state" ]] || die SCRUTINY_RECOVERY_SEALED_STATE_MISSING 70 +649- jq -e '.status=="SEALED" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' "$state" >/dev/null || die SCRUTINY_RECOVERY_SEALED_STATE_INVALID 70 +650: [[ -s "$restore" && ! -L "$restore" ]] || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_MISSING 70 +651: jq -e '.result=="OK" and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true' "$restore" >/dev/null || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_INVALID 70 +652- printf 'BACKUP_STATUS=SEALED\nAPP=scrutiny\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$gen" +653-} +654- +655-ACTION="${1:-}" +656-case "$ACTION" in +657- plan) [[ "$#" == 2 ]] || die USAGE_PLAN 64; plan "$2" ;; +658- backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +659: restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +660- recover-scrutiny-orphan) [[ "$#" == 7 ]] || die USAGE_RECOVER_SCRUTINY_ORPHAN 64; recover_scrutiny_orphan "$2" "$3" "$4" "$5" "$6" "$7" ;; +661: latest-sealed) [[ "$#" == 2 ]] || die USAGE_LATEST_SEALED 64; latest_sealed "$2" ;; +662- *) die ACTION_INVALID 64 ;; +663-esac +APP_RESTORE_CONTRACT_SOURCE_END=1 +APP_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json +APP_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc +APP_GENERATION=app-skladchik-reports-monitor-pve01-20260921T175029Z +APP_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json +APP_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e +APP_RESTORE_KEYS_BEGIN=1 +["app","checks","generation","logical_id","payload_sha256","payload_sha256_verified","restore_source","result","schema","transport_state","transport_state_file","validated_at_utc"] +APP_RESTORE_KEYS_END=1 +APP_RESTORE_SAFE_VIEW_BEGIN=1 +{"schema":"homelab-app-restore-validation-v1","result":"OK","status":null,"logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","transport_state":"SEALED","transport_state_file":"/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json","restore_source":"Cloud A","payload_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","payload_sha256_verified":true,"payload_decompression_verified":null,"archive_recovery_readback_verified":null,"parts_manifest_verified":null,"bounded_staging":null,"full_local_payload_written":null,"tar_test_ok":null,"required_files_ok":null,"cookie_jar_ok":null,"sqlite_ok":null,"validated_at_utc":"2026-09-21T17:50:46Z"} +APP_RESTORE_SAFE_VIEW_END=1 +APP_RESTORE_BASE_CONTRACT_OK=true +US_NETBIRD_STATE_BEGIN=1 US_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26 -{"schema":1,"logical_id":"us-netbird","result":"FAIL","generation":"v2-us-netbird-20260922T055316Z-d5c92acdd2e8","updated_at_utc":"2026-09-22T05:53:31Z","last_success_epoch":1789704670,"last_attempt_epoch":1790056411,"retry_after_epoch":1790078011,"consecutive_failures":14,"replica_count":null} -{"path":"consecutive_failures","value":14} -{"path":"failure_backoff_seconds","value":21600} -{"path":"last_attempt_epoch","value":1790056411} -{"path":"last_success_epoch","value":1789704670} -{"path":"last_success_generation","value":"v2-us-netbird-20260918T041032Z-d7648be4c16b"} -{"path":"last_success_manifest_sha256","value":"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5"} -{"path":"last_success_payload_sha256","value":"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf"} -{"path":"reason","value":"PIPE_FAILED"} -{"path":"result","value":"FAIL"} -{"path":"retry_after_epoch","value":1790078011} -US_NETBIRD_STATE_SAFE_END=1 -US_NETBIRD_SOURCE_HITS_BEGIN=1 -SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/legacy_engine.py SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7 -155- for node in ('pve01','pve02','pve03'): -156- out.append({'logical_id':'node-'+node,'scope':'node','type':'pve_node_bundle','node':node,'due_seconds':7*d,'enabled':True,'green1':node=='pve01'}) -157- out += [ -158- {'logical_id':'md-relay','scope':'vps','type':'vps_md','host':'[PRIVATE_IP]','port':22,'user':'ops','hostkey_fingerprint':'SHA256:DXJGwunC5tEVvnC3nRxyayTN8FXmyJcVgf+Oa6udQsY','due_seconds':d,'enabled':True,'green1':True}, -159- {'logical_id':'spb-ingress','scope':'vps','type':'vps_spb','host':'[PRIVATE_IP]','port':22,'user':'edgeadmin','hostkey_fingerprint':'SHA256:Ef+HsZb8lIXoadvYBXuGKIL3H7NyJMmf9EYCoJRoSv8','due_seconds':d,'enabled':True,'green1':True}, -160: {'logical_id':'us-netbird','scope':'vps','type':'vps_us','host':'[PRIVATE_IP]','port':22,'user':'root','hostkey_fingerprint':'SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE','due_seconds':d,'enabled':True,'green1':True}, -161- {'logical_id':'router-netcraze','scope':'router','type':'router','due_seconds':d,'enabled':True,'green1':True}, -162- ] -163- sites=[ -164- ('codevipe',160,'pve02','/var/www/forums/codevipe/public','xf_codevipe','single_transaction'), -165- ('dsmods',160,'pve02','/var/www/forums/dsmods/public','xf_dsmods','single_transaction'), --- -216- except subprocess.TimeoutExpired: -217- for q in (src,z): -218- try:q.kill() -219- except Exception:pass -220- raise BackupError('PIPE_TIMEOUT') -221: if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}') -222- -223-def ensure_dir(p,mode=0o700): -224- p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p -225- -226-def atomic_write(path,data,mode=0o600): --- -248- try: -249- j=json.loads(p.read_text()); d[j['logical_id']]=j -250- except Exception: continue -251- return d -252- -253:def write_state(logical_id,result,**kw): -254- if result not in RESULT_CLASSES: raise BackupError('RESULT_CLASS_INVALID') -255- j={'schema':1,'logical_id':logical_id,'result':result,'updated_at_utc':utcnow(),**kw} -256- if result=='SUCCESS': j['last_success_epoch']=epoch() -257- atomic_write(STATE_ROOT/'state'/(logical_id+'.json'),canonical_json(j)+b'\n',0o600) -258- return j --- -384- if not names: raise BackupError('TAR_ZST_EMPTY') -385- if required_member and required_member not in names: raise BackupError('REQUIRED_MEMBER_MISSING') -386- -387-def validate_payload(t,path): -388- typ=t['type'] -389: if typ in ('pve_node_bundle','vps_md','vps_spb','vps_us','router'): -390- validate_tar_zst(path) -391- elif typ=='pve_guest': -392- if t['guest_type']=='lxc': validate_tar_zst(path) -393- else: -394- # Green set is intentionally CT-only. Scheduled QEMU backups still get dual remote byte readback. --- -405-def cluster_inventory(): -406- p=run(['pvesh','get','/cluster/resources','--type','vm','--output-format','json'],timeout=60) -407- try:return json.loads(p.stdout) -408- except Exception: raise BackupError('PVE_INVENTORY_JSON') -409- -410:def ssh_remote(node,argv,input=None,timeout=1800,check=True): -411- cmd='PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec '+shlex.join([str(x) for x in argv]) -412- if node=='pve01': return run([str(x) for x in argv],input=input,timeout=timeout,check=check) -413- return run([SSH,'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node,cmd],input=input,timeout=timeout,check=check) -414- -415-def _pve_dumpdir_spec(guest_type): --- -471- if i+1>=len(argv) or argv[i+1]!=str(scratch): raise BackupError('PROVEN_DUMP_ARGV_TMPDIR_BINDING') -472- elif '--tmpdir='+str(scratch) not in argv: raise BackupError('PROVEN_DUMP_ARGV_NO_TMPDIR') -473- return argv,job -474- -475-def _default_unprivileged_lxc(node,vmid): -476: q=ssh_remote(node,['pct','config',str(vmid)],timeout=30) -477- lines=[x.strip() for x in q.stdout.decode('utf-8','replace').splitlines()] -478- if 'unprivileged: 1' not in lines: raise BackupError('LXC_NOT_UNPRIVILEGED') -479- if any(x.startswith('lxc.idmap:') or x.startswith('lxc.idmap =') for x in lines): raise BackupError('LXC_CUSTOM_IDMAP_UNSUPPORTED') -480- -481-def mapped_scratch_probe(node,scratch): -482- tool=shutil.which('lxc-usernsexec') or '/usr/bin/lxc-usernsexec' -483- pos=[tool,'-m','u:0:100000:65536','-m','g:0:100000:65536','--','/usr/bin/test','-x',str(scratch)] -484: q=ssh_remote(node,pos,timeout=30,check=False) -485- if q.returncode!=0: raise BackupError('MAPPED_UID_ACCESS_PROBE_FAILED') -486- -487-def _stream_argv_to_local(node,argv,dest,timeout=7200): -488- dest=pathlib.Path(dest) -489- if node=='pve01': cmd=argv --- -509- if typ!=t['guest_type']: raise BackupError('PVE_GUEST_TYPE_DRIFT') -510- if typ=='lxc': -511- _require_exact_source(P270,P270_SHA,'P270_IDENTITY');_require_exact_source(P282,P282_SHA,'P282_IDENTITY') -512- _default_unprivileged_lxc(node,vmid) -513- scratch=f'/var/tmp/homelab-vzdump-v2-{gen}' -514: ssh_remote(node,['install','-d','-m','710','-o','0','-g','100000',scratch],timeout=30) -515: q=ssh_remote(node,['stat','-c','%a:%u:%g',scratch],timeout=30) -516- if q.stdout.decode().strip()!='710:0:100000': raise BackupError('PVE_MAPPED_SCRATCH_ACCESS_CONTRACT') -517- try: -518- mapped_scratch_probe(node,scratch) -519- argv,job=proven_lxc_dump_argv(t,row,node,scratch) -520- local=work/'payload.bin' -521- _stream_argv_to_local(node,argv,local,timeout=7200) -522- if local.stat().st_size>int(job['archive_cap_bytes']): raise BackupError('VZDUMP_ARCHIVE_CAP_EXCEEDED') -523- except Exception: -524: ssh_remote(node,['rm','-rf','--',scratch],timeout=60,check=False);raise -525- cleanup={'kind':'pve_source_dir','node':node,'path':scratch} -526- return local,{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status'),'producer_contract':'proven-270-282-stream'},cleanup -527- # QEMU path remains unchanged: bounded dedicated dumpdir with later cloud readback/reconstruction. -528- srcdir=f'/var/tmp/homelab-vzdump-v2-{gen}' -529: ssh_remote(node,['install','-d','-m','700','-o','0','-g','0',srcdir],timeout=30) -530- try: -531: ssh_remote(node,['vzdump',str(vmid),'--mode','snapshot','--compress','zstd','--dumpdir',srcdir],timeout=7200) -532: p=ssh_remote(node,['find',srcdir,'-maxdepth','1','-type','f','-printf','%p\n'],timeout=30) -533- files=[x for x in p.stdout.decode().splitlines() if x.strip()] -534- if len(files)!=1: raise BackupError('VZDUMP_OUTPUT_NOT_UNIQUE') -535- src=files[0];local=work/'payload.bin' -536- if node=='pve01': local=pathlib.Path(src) -537- else: run([SCP,'-q','-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','root@'+node+':'+src,str(local)],timeout=7200) -538- if not pathlib.Path(local).is_file() or pathlib.Path(local).stat().st_size<=0: raise BackupError('VZDUMP_PAYLOAD_MISSING') -539- except Exception: -540: ssh_remote(node,['rm','-rf','--',srcdir],timeout=60,check=False);raise -541- return pathlib.Path(local),{'generation':gen,'source_class':'pve_guest','vmid':vmid,'guest_type':typ,'node':node,'status':row.get('status')},{'kind':'pve_source_dir','node':node,'path':srcdir} -542- -543-def operator_key_fingerprint(): -544- if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING') -545- p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30) --- -563- return good[0] -564- -565-def ensure_vps_known_hosts(targets): -566- lines=[] -567- for t in targets: -568: if t['type'] not in ('vps_md','vps_spb','vps_us'): continue -569- lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint'])) -570- atomic_write(VPS_KNOWN_HOSTS,('\n'.join(lines)+'\n').encode(),0o600) -571- -572-def vps_ssh_argv(t, *, use_agent=False): -573- a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15'] --- -598- tag=p.stdout.decode().strip() -599- if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT') -600- vps_run(t,['true'],timeout=30,use_agent=False) -601- results[t['logical_id']]=tag -602- # US was already operator-key based; prove it agentless too. -603: us=[x for x in targets if x['type']=='vps_us'][0] -604- vps_run(us,['true'],timeout=30,use_agent=False) -605- results[us['logical_id']]='ALREADY_US_OPERATOR_KEY' -606- return results -607- -608-def remote_hashset(t,command): -609- p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False) -610- return sha256_bytes(p.stdout) -611- -612:def capture_vps_md(t,gen,work): -613- files=['etc/netbird/install.conf','var/lib/netbird/state.json','var/lib/netbird/service.json','var/lib/netbird/active_profile.json','var/lib/netbird/default.json'] -614- cmd="sudo -n sha256sum -- "+' '.join(shlex.quote('/'+x) for x in files) -615- pre=remote_hashset(t,cmd) -616- remote=shlex.join(['sudo','-n','tar','--numeric-owner','-C','/','-cf','-']+files) -617- argv=vps_ssh_argv(t,use_agent=False)+[remote] -618- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=600) -619- post=remote_hashset(t,cmd) -620- if pre!=post: raise BackupError('MD_SOURCE_CHANGED_DURING_CAPTURE') -621- return payload,{'generation':gen,'source_class':'vps_md','consistency':'FILESET_PRE_POST_HASH_STABILITY','hashset_sha256':pre},None -622- -623:def capture_vps_spb(t,gen,work): -624- hcmd="find /etc/nginx -xdev -type f -print0 | sort -z | xargs -0 sha256sum" -625- pre=remote_hashset(t,hcmd) -626- remote=shlex.join(['tar','--numeric-owner','-C','/','-cf','-','etc/nginx']) -627- argv=vps_ssh_argv(t,use_agent=False)+[remote] -628- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=900) --- -657- if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33) -658- b=dst.serialize(); dst.close(); src.close() -659- ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b)) -660-''' -661- -662:def capture_vps_us(t,gen,work): -663- remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT]) -664- argv=vps_ssh_argv(t,use_agent=False)+[remote] -665- payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200) -666: return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None -667- -668-def _simple_shell_assignments(text): -669- vals={} -670- for line in text.splitlines(): -671- m=re.match(r'^\s*(?:export\s+)?([A-Z_][A-Z0-9_]*)=(.*)$',line) --- -801- -802-def qga_exec(node,vmid,guest_argv,input_bytes=None,timeout=600): -803- args=['qm','guest','exec',str(vmid),'--synchronous','1','--timeout',str(int(timeout))] -804- if input_bytes is not None: args += ['--pass-stdin','1'] -805- args += ['--']+[str(x) for x in guest_argv] -806: p=ssh_remote(node,args,input=input_bytes,timeout=timeout+60) -807- try:j=json.loads(p.stdout) -808- except Exception: raise BackupError('QGA_JSON') -809- rc=j.get('exitcode',j.get('exit-code')) -810- if rc is None: rc=0 if j.get('exited') else None -811- if rc not in (0,None): raise BackupError('QGA_GUEST_COMMAND_FAILED',str(rc)) --- -854- -855-def capture_node_bundle(t,gen,work): -856- node=t['node']; raw=node_bundle_paths(node); paths=[] -857- for rel in raw: -858- if node=='pve01': ok=(pathlib.Path('/')/rel).exists() -859: else: ok=(ssh_remote(node,['test','-e','/'+rel],timeout=20,check=False).returncode==0) -860- if ok: paths.append(rel) -861- required={'etc/network/interfaces','etc/hosts','etc/hostname','etc/pve/nodes/'+node} -862- if not required.issubset(set(paths)): raise BackupError('NODE_BUNDLE_REQUIRED_PATH_MISSING') -863- payload=work/'payload.tar.zst' -864- argv=['tar','--numeric-owner','--xattrs','--acls','-C','/','-cf','-']+paths --- -870- -871-def capture_target(t,gen,work): -872- typ=t['type'] -873- if typ=='pve_guest': return capture_pve_guest(t,gen,work) -874- if typ=='pve_node_bundle': return capture_node_bundle(t,gen,work) -875: if typ=='vps_md': return capture_vps_md(t,gen,work) -876: if typ=='vps_spb': return capture_vps_spb(t,gen,work) -877: if typ=='vps_us': return capture_vps_us(t,gen,work) -878- if typ=='router': return capture_router(t,gen,work) -879- if typ=='xenforo': return capture_xenforo(t,gen,work) -880- raise BackupError('CAPTURE_TYPE_UNKNOWN') -881- -882-def cleanup_source(cleanup): -883- if not cleanup: return -884- if cleanup.get('kind')=='pve_source_dir': -885: ssh_remote(cleanup['node'],['rm','-rf','--',cleanup['path']],timeout=60) -886- -887-def control_verify(t): -888- engine=pathlib.Path('/usr/local/sbin/homelab-control-backup') -889- if sha256_file(engine)!='0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57': raise BackupError('CONTROL_ENGINE_IDENTITY') -890- root=pathlib.Path('/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2') --- -895- if j.get('status')=='SEALED' and j.get('generation'): rows.append((p.stat().st_mtime,j['generation'])) -896- except Exception: continue -897- if not rows: raise BackupError('CONTROL_SEALED_GENERATION_MISSING') -898- gen=sorted(rows)[-1][1] -899- p=run([str(engine),'verify',gen],timeout=1800) -900: write_state(t['logical_id'],'SUCCESS',generation=gen,verification='existing_control_engine_verify',last_success_epoch=epoch()) -901- return {'logical_id':t['logical_id'],'generation':gen,'result':'SUCCESS','payload_bytes':0,'replicas':2} -902- -903:def backup_target(t,green_verify=False): -904- if t['type']=='control': return control_verify(t) -905- gen=generation_id(t['logical_id']); work=ensure_dir(WORK_ROOT/gen,0o700) -906- payload=None; cleanup=None; sealed=False -907- try: -908- payload,meta,cleanup=capture_target(t,gen,work); meta['generation']=gen --- -916- ha=reconstruct_replica(manifest,manifest['replicas'][0],a); validate_payload(t,a) -SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-accepted-486r1/wrapper.py SHA256=2b5eb4fc0f6f466a0d3e5a007af987092b6c17baa3ff9241a6afc2c2e97df1d5 -2-from __future__ import annotations -3-import errno,fcntl,importlib.util,os,sys -4-from pathlib import Path -5-ROOT=Path(__file__).resolve().parent -6-CT_ID='guest-pve01-lxc-112' -7:LEGACY_IDS=('node-pve01','md-relay','spb-ingress','us-netbird','router-netcraze','xf-codevipe','xf-zakrutim','control-bootstrap-v2') -8-LOCK=Path('/run/homelab-backup-heavy.lock') -9- -10-def _load(name,path): -11- spec=importlib.util.spec_from_file_location(name,str(path)) -12- if spec is None or spec.loader is None: raise RuntimeError('MODULE_SPEC') --- -14- -15-legacy=_load('_backupv2_legacy486',ROOT/'legacy_engine.py') -16-ct112=_load('_backupv2_ct112_486',ROOT/'ct112_handler.py') -17-driver=_load('_backupv2_driver454_479',ROOT/'representative_driver.py') -18-_legacy_load_targets=legacy.load_targets -19:_legacy_backup_target=legacy.backup_target -20- -21-def canonical_targets(): -22- rows=_legacy_load_targets() -23- if not isinstance(rows,list) or len(rows)!=28: raise legacy.BackupError('TARGET_REGISTRY_COUNT') -24- out=[dict(x) for x in rows] --- -35- 'guest_type':'lxc','due_seconds':due,'enabled':True,'green1':True} -36- -37-def load_targets(): -38- base=canonical_targets();return base+[_ct_descriptor(base)] -39- -40:def backup_target(t,green_verify=False): -41- if not isinstance(t,dict) or t.get('logical_id')!=CT_ID: -42: return _legacy_backup_target(t,green_verify) -43- try: -44: r=ct112.run_target(t,state_writer=legacy.write_state) -45- return {'logical_id':CT_ID,'generation':r['generation'],'result':'SUCCESS','payload_bytes':r['payload_bytes'], -46- 'replicas':2,'manifest_sha256':r['manifest_sha256'],'payload_sha256':r['payload_sha256']} -47- except BaseException: -48: try: legacy.write_state(CT_ID,'FAIL',reason='CT112_PIPELINE_FAILED') -49- except BaseException: pass -50- raise legacy.BackupError('CT112_PIPELINE_FAILED') from None -51- -52-legacy.load_targets=load_targets -53:legacy.backup_target=backup_target -54- -55-def _legacy_reused_boundary_selftest(): -56- if legacy.PART_SIZE!=536870912: raise RuntimeError('LEGACY_PART_SIZE') -57- if len(legacy.REMOTES)!=13 or len({p for _,p in legacy.REMOTES})!=13: raise RuntimeError('LEGACY_REMOTES') -58- d=legacy.default_targets() -SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/entry504.py SHA256=ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793 -57- if gp==legacy_path: -58- found[id(g)]=g -59- if len(found)!=1: -60- raise RuntimeError('LEGACY_PROVENANCE_CARDINALITY') -61- g=next(iter(found.values())) -62: required=('capture_pve_guest','backup_target','load_targets','cluster_inventory','ssh_remote', -63- '_default_unprivileged_lxc','mapped_scratch_probe','_stream_argv_to_local','run', -64: 'choose_due_target','write_state','epoch') -65- if any(not callable(g.get(n)) for n in required): -66- raise RuntimeError('LEGACY_PROVENANCE_API') -67- original=g['capture_pve_guest'] -68- proxy=_GlobalsProxy(g) -69- if not isinstance(g.get('SCP'),str) or not g['SCP'].startswith('/'): --- -73- raise RuntimeError('QEMU_ADAPTER_BIND_RESULT') -74- patched=lxc_adapter.bind(proxy,qemu_patched) -75- if not callable(patched): -76- raise RuntimeError('LXC_ADAPTER_BIND_RESULT') -77- g['capture_pve_guest']=patched -78: scheduler_wired=scheduler_adapter.bind(proxy,g['choose_due_target'],g['write_state']) -79: if g.get('choose_due_target') is not scheduler_wired['choose_due_target'] or g.get('write_state') is not scheduler_wired['write_state']: -80- raise RuntimeError('SCHEDULER_ADAPTER_BIND_RESULT') -81- return {'legacy_globals':g,'legacy':proxy,'original_capture':original,'qemu_patched_capture':qemu_patched,'patched_capture':patched,'scheduler_wired':scheduler_wired} -82- -83-def _load(): -84- _verify_base() --- -105- wrapper,lxc_adapter,qemu_adapter,scheduler_adapter,wired=_load() -106- if wired['legacy_globals'].get('capture_pve_guest') is not wired['patched_capture']: -107- raise RuntimeError('PATCH_BINDING') -108- if wired['legacy_globals'].get('choose_due_target') is not wired['scheduler_wired']['choose_due_target']: -109- raise RuntimeError('SCHEDULER_CHOOSE_BINDING') -110: if wired['legacy_globals'].get('write_state') is not wired['scheduler_wired']['write_state']: -111- raise RuntimeError('SCHEDULER_WRITE_BINDING') -112- if [scheduler_adapter.backoff_seconds(n) for n in range(1,7)]!=[1800,3600,7200,14400,21600,21600]: -113- raise RuntimeError('SCHEDULER_BACKOFF_CONTRACT') -114: if not callable(wrapper.backup_target) or not callable(wrapper.main) or not callable(wrapper.load_targets): -115- raise RuntimeError('WRAPPER_PUBLIC_API') -116- print('SELFTEST_OVERLAY504=PASS',flush=True) -117- return 0 -118- -119-def main(argv=None): --- -131- try: -132- with tempfile.TemporaryDirectory(prefix='homelab-scheduler-proof-',dir='/run') as td: -133- g['STATE_ROOT']=pathlib.Path(td); (g['STATE_ROOT']/'state').mkdir(mode=0o700) -134- g['epoch']=lambda: now[0] -135- common={'payload_sha256':'a'*64,'manifest_sha256':'b'*64,'replicas':['proof-r1','proof-r2']} -136: legacy.write_state('proof-a','SUCCESS',generation='proof-a-good',**common) -137: legacy.write_state('proof-b','SUCCESS',generation='proof-b-good',**common) -138- now[0]=100000 -139: legacy.write_state('proof-a','FAIL',generation='proof-a-bad',reason='PROOF_FAILURE',sealed=False) -140- state=legacy.load_state_all(); a=state['proof-a'] -141- targets=[{'logical_id':'proof-a','enabled':True,'due_seconds':100},{'logical_id':'proof-b','enabled':True,'due_seconds':100}] -142- first=legacy.choose_due_target(targets,state,now[0]) -143- if not isinstance(first,dict) or first.get('logical_id')!='proof-b': raise RuntimeError('SCHEDULER_PROOF_PEER_NOT_SELECTED') -144: retry=int(a.get('retry_after_epoch',0)); preserved=int(a.get('last_success_epoch',0)) -145: if preserved!=1000 or a.get('last_success_generation')!='proof-a-good' or int(a.get('consecutive_failures',0))!=1 or retry!=101800: -146- raise RuntimeError('SCHEDULER_PROOF_FAILURE_STATE') -147- now[0]=retry+1 -148- second=legacy.choose_due_target(targets,legacy.load_state_all(),now[0]) -149- if not isinstance(second,dict) or second.get('logical_id')!='proof-a': raise RuntimeError('SCHEDULER_PROOF_RETRY_NOT_REELIGIBLE') -150: legacy.write_state('proof-a','SUCCESS',generation='proof-a-recovered',**common) -151- recovered=legacy.load_state_all()['proof-a'] -152: if int(recovered.get('consecutive_failures',-1))!=0 or int(recovered.get('retry_after_epoch',-1))!=0: -153- raise RuntimeError('SCHEDULER_PROOF_SUCCESS_RESET') -154: proof={'failed_target':'proof-a','peer_selected_during_backoff':first['logical_id'],'retry_after_epoch':retry,'last_success_preserved':preserved==1000,'retry_target_after_backoff':second['logical_id'],'success_resets_failures':True,'production_state_root_used':False} -155- print('SCHEDULER_FAILURE_ISOLATION_PROOF='+json.dumps(proof,separators=(',',':'),sort_keys=True),flush=True) -156- return 0 -157- finally: -158- g['STATE_ROOT']=old_root; g['epoch']=old_epoch -159- if argv==['scheduler-select-live']: --- -174- if len(argv)==2 and argv[0]=='verify-target': -175- t=_target(wrapper,argv[1]) -176- if t.get('logical_id')=='guest-pve01-lxc-112': -177- raise RuntimeError('VERIFY_SYNTHETIC_CT112_FORBIDDEN') -178- try: -179: out=wrapper.backup_target(t,True) -180- except legacy.BackupError as e: -181- print('VERIFY_TARGET_RESULT=FAIL',flush=True) -182- print('VERIFY_TARGET_REASON='+str(e.code),flush=True) -183- return 2 -184- print('VERIFY_TARGET_RESULT='+json.dumps(out,separators=(',',':'),sort_keys=True),flush=True) -SOURCE_FILE=/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b/scheduler_adapter.py SHA256=a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96 -14- n=int(v) -15- except Exception: -16- return default -17- return n if n>=0 else default -18- -19:def backoff_seconds(consecutive_failures:int)->int: -20: n=max(1,_as_nonneg_int(consecutive_failures,1)) -21- return min(MAX_BACKOFF,MIN_BACKOFF*(2**min(n-1,4))) -22- -23-def _read_prior(legacy,logical_id): -24- root=pathlib.Path(legacy.STATE_ROOT)/'state' -25- p=root/(str(logical_id)+'.json') --- -60- if not isinstance(t,dict) or t.get('enabled') is not True: continue -61- lid=t.get('logical_id'); seconds=_as_nonneg_int(t.get('due_seconds'),0) -62- _need(isinstance(lid,str) and lid and seconds>0,'SCHEDULER_TARGET_SCHEMA') -63- s=state.get(lid,{}) -64- if not isinstance(s,dict): s={} -65: retry_after=_as_nonneg_int(s.get('retry_after_epoch'),0) -66: if retry_after>now: continue -67- last_success=_as_nonneg_int(s.get('last_success_epoch'),0) -68- age=(now-last_success) if last_success and now>=last_success else (10**12 if last_success==0 else 0) -69- if age> \"$HOME/.ssh/authorized_keys\"; printf 'ADDED\\n'; fi" + p=vps_run(t,['sh','-c',script.decode()],input=pub,timeout=45,use_agent=True) + tag=p.stdout.decode().strip() + if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT') + vps_run(t,['true'],timeout=30,use_agent=False) + results[t['logical_id']]=tag + # US was already operator-key based; prove it agentless too. + us=[x for x in targets if x['type']=='vps_us'][0] + vps_run(us,['true'],timeout=30,use_agent=False) + results[us['logical_id']]='ALREADY_US_OPERATOR_KEY' + return results + +def remote_hashset(t,command): + p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False) + return sha256_bytes(p.stdout) +VPS_SSH_SOURCE_END=1 +US_NETBIRD_PID_JOURNAL_BEGIN=1 +PID_BEGIN=936114 +2026-09-21T20:30:36+03:00 pve01 homelab-backup-v2[936114]: SELFTEST_REPLACEMENT486=PASS 2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird 2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL 2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED -2026-09-21T20:30:51+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. -2026-09-21T20:30:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. -2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_TARGET=md-relay -2026-09-21T21:03:57+03:00 pve01 homelab-backup-v2[1046231]: SCHEDULE_RESULT=SUCCESS -2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_TARGET=spb-ingress -2026-09-21T21:37:49+03:00 pve01 homelab-backup-v2[1149168]: SCHEDULE_RESULT=SUCCESS -2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_TARGET=router-netcraze -2026-09-21T22:17:41+03:00 pve01 homelab-backup-v2[1272743]: SCHEDULE_RESULT=SUCCESS -2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_TARGET=xf-codevipe -2026-09-21T23:02:25+03:00 pve01 homelab-backup-v2[1384336]: SCHEDULE_RESULT=SUCCESS -2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_TARGET=xf-zakrutim -2026-09-21T23:25:05+03:00 pve01 homelab-backup-v2[1454793]: SCHEDULE_RESULT=SUCCESS -2026-09-21T23:27:32+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_TARGET=ct110 -2026-09-21T23:27:32+03:00 pve01 homelab-backup-v2[1495880]: LXC_EXECUTION_DIAG={"archive_bytes":546400504,"exception_class":null,"exception_message_sha256":null,"executor":"local-proven-pve270","helper_rc":0,"node":"pve01","producer_rc":0,"stderr_bytes_observed":1405,"stderr_bytes_retained":1405,"stderr_stream_sha256":"35adf6e0c931a6c547164a0716f388ca307d8c4aac70af8300f821f47bd0f01c","stderr_truncated":false} -2026-09-21T23:28:28+03:00 pve01 homelab-backup-v2[1495880]: SCHEDULE_RESULT=SUCCESS -2026-09-21T23:30:13+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_TARGET=ct111 -2026-09-21T23:30:13+03:00 pve01 homelab-backup-v2[1511723]: LXC_EXECUTION_DIAG={"executor":"remote-bounded-ssh-cwd-root","node":"pve02","producer_rc":0,"stderr_bytes_observed":1035,"stderr_bytes_retained":1035,"stderr_stream_sha256":"ad5f87b168930991fbbf7aff71f42683f2bd8c3a7a0e678d12a09b665f4dd5af","stderr_truncated":false} -2026-09-21T23:30:55+03:00 pve01 homelab-backup-v2[1511723]: SCHEDULE_RESULT=SUCCESS -2026-09-22T00:04:14+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_TARGET=vm210 -2026-09-22T00:05:17+03:00 pve01 homelab-backup-v2[1627342]: SCHEDULE_RESULT=SUCCESS -2026-09-22T02:02:23+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_TARGET=vm170 -2026-09-22T02:12:47+03:00 pve01 homelab-backup-v2[1999915]: SCHEDULE_RESULT=SUCCESS +PID_END=936114 +PID_BEGIN=2121414 +2026-09-22T02:36:58+03:00 pve01 homelab-backup-v2[2121414]: SELFTEST_REPLACEMENT486=PASS 2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird 2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL 2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED -2026-09-22T02:37:13+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. -2026-09-22T02:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. -2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_TARGET=control-bootstrap-v2 -2026-09-22T03:46:31+03:00 pve01 homelab-backup-v2[2408783]: SCHEDULE_RESULT=SUCCESS -2026-09-22T04:29:13+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_TARGET=vm9130 -2026-09-22T04:44:11+03:00 pve01 homelab-backup-v2[2510842]: SCHEDULE_RESULT=SUCCESS -2026-09-22T06:31:51+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_TARGET=vm211 -2026-09-22T06:32:52+03:00 pve01 homelab-backup-v2[2931093]: SCHEDULE_RESULT=SUCCESS -2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_TARGET=node-pve01 -2026-09-22T07:09:46+03:00 pve01 homelab-backup-v2[3046566]: SCHEDULE_RESULT=SUCCESS +PID_END=2121414 +PID_BEGIN=3378768 +2026-09-22T08:53:16+03:00 pve01 homelab-backup-v2[3378768]: SELFTEST_REPLACEMENT486=PASS 2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_TARGET=us-netbird 2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_RESULT=FAIL 2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_REASON=PIPE_FAILED -2026-09-22T08:53:31+03:00 pve01 systemd[1]: homelab-backup-v2.service: Failed with result 'exit-code'. -2026-09-22T08:53:31+03:00 pve01 systemd[1]: Failed to start homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. -US_NETBIRD_RECENT_JOURNAL_END=1 -BACKUP_V2_TIMER_STATE_BEGIN=1 -MainPID=0 -Result=exit-code -ExecMainStartTimestamp=Tue 2026-09-22 08:53:16 MSK -ExecMainExitTimestamp=Tue 2026-09-22 08:53:31 MSK -ExecMainStatus=2 -ActiveState=failed -SubState=failed -Unit=homelab-backup-v2.service -NextElapseUSecRealtime= -LastTriggerUSec=Tue 2026-09-22 08:53:16 MSK -LoadState=loaded -ActiveState=active -SubState=waiting -UnitFileState=enabled -BACKUP_V2_TIMER_STATE_END=1 -CANONICAL_SKLADCHIK_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json -CANONICAL_SKLADCHIK_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc -CANONICAL_SKLADCHIK_STATE_BEGIN=1 -{"schema":"homelab-cloud-quorum-payload-v1","status":"SEALED","logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","archive_size_bytes":178950,"archive_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","part_count":1,"parts_manifest_sha256":"664ef74ec0f30cd3e7fef848794a8813e390f55295c5c6874c1f30867708f740","cloud_quorum":2,"cloud_a_verified":true,"cloud_b_verified":true,"part_hashes_verified":true,"seal_written":true,"cloud_a_seal_verified":true,"cloud_b_seal_verified":true,"local_payload_present":false,"local_cleanup":"OK","sealed_at_utc":"2026-09-21T17:50:41Z","updated_at_utc":"2026-09-21T17:50:43Z","remote_a_base":"pve01-mail-01-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z","remote_b_base":"pve01-mail-02-crypt:homelab-backups/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z"} -CANONICAL_SKLADCHIK_STATE_END=1 -CANONICAL_SKLADCHIK_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json -CANONICAL_SKLADCHIK_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e -CANONICAL_SKLADCHIK_RESTORE_BEGIN=1 -{"result":"OK","logical_id":"app-skladchik-reports-monitor","generation":"app-skladchik-reports-monitor-pve01-20260921T175029Z","restore_source":"Cloud A","transport_state":"SEALED","payload_sha256":"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc","payload_sha256_verified":true,"payload_decompression_verified":null,"archive_recovery_readback_verified":null,"parts_manifest_verified":null,"bounded_staging":null,"full_local_payload_written":null,"validation_level":null,"validated_at_utc":"2026-09-21T17:50:46Z"} -CANONICAL_SKLADCHIK_RESTORE_END=1 -CANONICAL_SKLADCHIK_STRICT_STATE=true -CANONICAL_SKLADCHIK_STRICT_RESTORE=false -CANONICAL_CLOUDB_MANIFEST_READ_RC=0 -CANONICAL_CLOUDB_MANIFEST_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -CANONICAL_CLOUDB_MANIFEST_SHA256=664ef74ec0f30cd3e7fef848794a8813e390f55295c5c6874c1f30867708f740 -CANONICAL_CLOUDB_MANIFEST_ROWS=1 -CANONICAL_CLOUDB_MANIFEST_TOTAL=178950 -CANONICAL_CLOUDB_MANIFEST_MATCH=true -AUDIT_LEGACY_CLOUDB_CONTRACT_BEGIN=1 - except Exception: pass - if lrc!=0: issues.append({"severity":"CRITICAL","code":"CLOUD_UNREADABLE","logical_id":"cloud-"+label,"root_cause":"RCLONE_LSF_FAILED"}) - cloud[label]=rec - -try: dr=json.loads(DRSEAL.read_text(encoding="utf-8")) -except Exception: dr=None -if not isinstance(dr,dict) or dr.get("status")!="SEALED": - issues.append({"severity":"CRITICAL","code":"DR_SEAL_MISSING","logical_id":"-","root_cause":"SITE_DR_FINAL_SEAL_NOT_SEALED"}) - -cloudb_ok=False -if CLOUDB_HEALTH.is_file(): - txt=CLOUDB_HEALTH.read_text(encoding="utf-8",errors="replace") - kv={} - for line in txt.splitlines(): - if "=" in line: - k,v=line.split("=",1); kv[k]=v - cloudb_ok=(kv.get("STATUS")=="OK" and kv.get("CLOUD_2_STATUS")=="OK" and kv.get("SHA256_OK")=="yes" and kv.get("TAR_OK")=="yes" and kv.get("REQUIRED_FILES_OK")=="yes") -if not cloudb_ok: - issues.append({"severity":"CRITICAL","code":"CLOUD_B_RESTORE_DRILL_BAD","logical_id":"app-skladchik-reports-monitor","root_cause":"LATEST_CLOUD_B_DRILL_NOT_OK"}) - -source={"local_head":None,"remote_head":None,"pushurl_disabled":False} -hrc,ho,_=run(["git","-C",WT,"rev-parse","HEAD"],10) -if hrc==0: source["local_head"]=ho.strip() -frc,fo,_=run(["git","-C",WT,"config","--get","remote.origin.url"],10) -prc,po,_=run(["git","-C",WT,"config","--get","remote.origin.pushurl"],10) -source["pushurl_disabled"]=(prc==0 and po.strip()=="DISABLED") -if frc==0 and fo.strip(): - lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30) - if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0] -if source["remote_head"]!=source["local_head"]: - issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"}) - -status="GREEN" if not issues else "RED" -for t in targets: - print("TARGET|"+"|".join(f"{k}={v}" for k,v in t.items() if v is not None)) -for i in issues: -AUDIT_LEGACY_CLOUDB_CONTRACT_END=1 -SOURCE_LOCAL_HEAD=4e59d5afaa670899c56f87a84bc887175f299614 -SOURCE_CURRENT_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation -SOURCE_REMOTE_HEAD_COMMIT=326622a4864fa9b5f43b69981733236bb42c4d7b -SOURCE_REMOTE_BRANCH_HEAD=ABSENT -SOURCE_REMOTE_HEAD_RC=0 -SOURCE_REMOTE_BRANCH_RC=0 -SOURCE_OFFHOST_CLASS=REMOTE_REACHABLE_EXPECTED_BRANCH_ABSENT -US_NETBIRD_FAILURE_CLASS=STATE_FAILURE_DETAIL_PRESENT -DECISION=HOLD_BACKUP_1116_ROOT_CAUSE_INCOMPLETE -NEXT_GATE=RECONCILE_ONLY_BACKUP_1116_FAILURE -TASK_RESULT=HOLD_HOMELAB_BACKUP_1115_ROOT_RCA -CHANGES_MADE_BY_1116=false -PRODUCT_MUTATION_BY_1116=false -VM_MUTATION_BY_1116=false -CLOUD_MUTATION_BY_1116=false -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1115-ROOT-RCA-1116R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} -WORKERS2_BACKUP_1115_ROOT_RCA_END=1 +PID_END=3378768 +US_NETBIRD_PID_JOURNAL_END=1 +US_NETBIRD_WORK_ARTIFACTS_BEGIN=1 +f|1790056411.7319950600|792|/var/lib/homelab-backup-v2/state/us-netbird.json +f|1789642259.3981707900|1255|/var/lib/homelab-backup-v2/cold-bootstrap/us-netbird.json +d|1790056411.7288067540|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260922T055316Z-d5c92acdd2e8 +d|1790033833.7009700260|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T233658Z-709cf01e1256 +d|1790011851.2302881580|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T173036Z-1128898e4cc8 +d|1789989471.8069593230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T111736Z-c5a5c5e71776 +d|1789955807.6619889330|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T015632Z-dc499030a55f +d|1789932571.7940614350|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T192916Z-ca2592a81a30 +d|1789908991.7828657500|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T125616Z-8caeb0b6c7b7 +d|1789886227.6266376830|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T063652Z-e6c72515a153 +d|1789862827.7065704150|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T000652Z-7ecdcfcafc12 +d|1789839091.8314245860|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T173116Z-cfc70719f4e9 +d|1789822687.5806853640|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T125752Z-a4efa5c07171 +d|1789814653.6399387990|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T104358Z-b2232f971dbc +d|1789810393.6722341230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T093258Z-64f5c2972565 +d|1789807993.6075344840|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T085258Z-36f858f91de5 +US_NETBIRD_WORK_ARTIFACTS_END=1 +US_NETBIRD_READONLY_PROBES_BEGIN=1 +US_NETBIRD_READONLY_PROBE_RC=41 +PROBE_TARGET_FOUND=true +PROBE_TYPE=vps_us +PROBE_SSH_TRUE_EXCEPTION=BackupError +PROBE_SSH_TRUE_CODE=COMMAND_FAILED +PROBE_SSH_TRUE_DETAIL=255 +US_NETBIRD_READONLY_PROBE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +US_NETBIRD_READONLY_PROBES_END=1 +US_NETBIRD_PIPE_BASELINE_CLASS=SSH_BASELINE_FAILURE +DECISION=PASS_BACKUP_1117_APP_CONTRACT_AND_PIPE_BASELINE_CLASSIFIED +NEXT_GATE=REPAIR_ONLY_BACKUP_US_NETBIRD_AND_AUDIT_CANONICAL_CLOUDB_SOURCE_OFFHOST +TASK_RESULT=PASS_HOMELAB_BACKUP_1116_CONTRACT_PIPE_RCA +CHANGES_MADE_BY_1117=false +PRODUCT_MUTATION_BY_1117=false +VM_MUTATION_BY_1117=false +CLOUD_MUTATION_BY_1117=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END