homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity
Files
4c218a59ec0fcb4650307cb21fbd8beb63d05e01
homelab-public-context/runtime/latest.json
T

39 lines
30 KiB
JSON
Raw Blame History

{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "read-only",
"component": "homelab-backup-1116-contract-pipe-rca",
"started_at_utc": "2026-09-22T06:05:29Z",
"finished_at_utc": "2026-09-22T06:05:29Z",
"reference_register_checked": true,
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
"error_register_checked": true,
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
"command_sha256": "0ba8e65003a7561a9e01930aa9eec5d2b6a6cd73c9f05d217fca4a399c992fed",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
"change_declared": false,
"result_contract_valid": true,
"result_contract_status": null,
"result_contract_error": null,
"command_rc": 0,
"changes_made": false,
"rollback_started": false,
"rollback_restored": null,
"mutation_outcome": "NO_MUTATION",
"sanitized": true,
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "8314f5500f36a8ab9866f3f778ae569bf18655d73ade7aca6ada77060826a746",
"sanitized_output_sha256": "1c8576db6611dd527a417a275078b31f952af42ddef0dd6dd0fa1661455bb847",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "WORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1\nMODE=read-only\nCOMPONENT=homelab-backup-1116-contract-pipe-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_APP_RESTORE_CONTRACT_AND_US_NETBIRD_PIPE_RCA;NO_SYSTEMCTL_ACTION;NO_BACKUP_TARGET_EXECUTION;NO_RESTORE_EXECUTION;NO_GIT_WRITE;NO_CLOUD_WRITE;REMOTE_PROBES_READ_ONLY\nERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nLEGACY_SHA256=b6e79f087b9f1d21dac4f77a7b46b743299bbb85bc716e80d2ea67c2e038bcd7\nAPP_WORKER_SHA256=1d6c4a04f762e2a33561c109b0c0a4878caaa48627903e7e78fdf5f22d79f196\nSCHEDULER_RC=0\nSELFTEST_REPLACEMENT486=PASS\nSCHEDULER_LIVE_SELECTION=DEFER_NO_DUE_TARGET\nAPP_RESTORE_CONTRACT_SOURCE_BEGIN=1\n89-}\n90-latest_sealed_state(){\n91- local app=\"$1\" d\n92- d=\"$(state_dir \"$app\")\"; [[ -d \"$d\" ]] || return 0\n93- find \"$d\" -maxdepth 1 -type f -name '*.json' -printf '%T@|%p\\n' | LC_ALL=C sort -t'|' -k1,1nr | while IFS='|' read -r _ f; do\n94- if jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$f\" >/dev/null 2>&1; then echo \"$f\"; break; fi\n95- done\n96-}\n97:restore_ok(){\n98- local app=\"$1\" gen=\"$2\" f\n99- f=\"$(restore_dir \"$app\")/$gen.json\"\n100- [[ -s \"$f\" && ! -L \"$f\" ]] || return 1\n101: jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\n102-}\n103-remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\n104-remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\n105-rsync_core(){\n106- local src=\"$1\" dst=\"$2\"\n107- mkdir -p \"$dst\"\n108- RSYNC_RSH='ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] rsync -a --numeric-ids --delete --info=progress2 --rsync-path='sudo -n rsync' \"$CORE:$src/\" \"$dst/\"\n109-}\n--\n344- [[ -f \"$ex/$path\" && ! -L \"$ex/$path\" ]] || die RESTORE_COMPONENT_MISSING 70\n345- actual=\"$(sha256sum \"$ex/$path\"|awk '{print $1}')\"; [[ \"$actual\" == \"$sha\" ]] || die RESTORE_COMPONENT_SHA_MISMATCH 70\n346- done < <(jq -r '.components|to_entries[]|[.value.path,.value.sha256]|@tsv' \"$ex/app-generation.json\")\n347-}\n348-reconstruct_cloud_a(){\n349- local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\n350- logical=\"$(logical_id \"$app\")\"\n351- \"$TRANSPORT\" verify-sealed \"$logical\" \"$gen\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\" >/dev/null\n352: base=\"$(jq -er '.remote_a_base' \"$state\")\"; expected_msha=\"$(jq -er '.parts_manifest_sha256' \"$state\")\"\n353- manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\n354- got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\n355- : >\"$out\"; chmod 0600 \"$out\"\n356- while IFS=$'\\t' read -r idx name size sha; do\n357- [[ \"$idx\" =~ ^[0-9]{8}$ && \"$name\" == part-????????.bin && \"$size\" =~ ^[1-9][0-9]*$ && \"$sha\" =~ ^[0-9a-f]{64}$ ]] || die RESTORE_A_MANIFEST_ROW_INVALID 70\n358- part=\"$(dirname \"$out\")/$name\"; rm -f \"$part\"; rclone cat \"${base%/}/payload.parts/$name\" >\"$part\"\n359- [[ \"$(stat -c %s \"$part\")\" == \"$size\" ]] || die RESTORE_A_PART_SIZE_MISMATCH 70\n360- actual=\"$(sha256sum \"$part\"|awk '{print $1}')\"; [[ \"$actual\" == \"$sha\" ]] || die RESTORE_A_PART_SHA_MISMATCH 70\n--\n505- scp \"${SCP_BASE[@]}\" \"$ex/immich/db/immich.dump\" \"$CORE:$remote_dump\" >/dev/null\n506- scp \"${SCP_BASE[@]}\" \"$ACTIVE_STAGE/postgres-image.txt\" \"$CORE:$remote_img\" >/dev/null\n507- if ! ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- validate-immich-db \"$gen\" \"$remote_dump\" \"$expect\" \"$remote_img\" <\"$CORE_HELPER\" >/dev/null; then ssh \"${SSH_BASE[@]}\" \"$CORE\" rm -f -- \"$remote_dump\" \"$remote_img\" >/dev/null 2>&1 || true; die IMMICH_POSTGRES_IMPORT_FAILED 70; fi\n508- ssh \"${SSH_BASE[@]}\" \"$CORE\" rm -f -- \"$remote_dump\" \"$remote_img\" >/dev/null\n509- checks=\"$(jq -n --argjson n \"$expect\" '[{name:\"postgres_import\",result:\"OK\",table_count:$n},{name:\"media_manifest\",result:\"OK\"},{name:\"config_manifest\",result:\"OK\"}]')\" ;;\n510- esac\n511- printf '%s\\n' \"$checks\"\n512-}\n513:restore_validate(){\n514- local app=\"$1\" gen=\"$2\" logical state work payload ex expected_sha checks out tmp\n515- valid_generation \"$gen\" || die GENERATION_INVALID 64\n516- logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\n517- [[ -s \"$state\" && ! -L \"$state\" ]] || die RESTORE_STATE_MISSING 65\n518- jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$state\" >/dev/null || die RESTORE_REQUIRES_SEALED 65\n519: if restore_ok \"$app\" \"$gen\"; then printf 'RESTORE_VALIDATION_STATUS=OK\\nAPP=%s\\nGENERATION=%s\\nRESULT=ALREADY_VALIDATED\\n' \"$app\" \"$gen\"; return 0; fi\n520- work=\"$RESTORE_WORK_ROOT/$logical/$gen\"; rm -rf --one-file-system \"$work\" 2>/dev/null || true; mkdir -p \"$work/extract\"; chmod 0700 \"$RESTORE_WORK_ROOT\" \"$RESTORE_WORK_ROOT/$logical\" \"$work\" 2>/dev/null || true; ACTIVE_STAGE=\"$work\"\n521- payload=\"$work/payload.tar.zst\"; reconstruct_cloud_a \"$app\" \"$gen\" \"$state\" \"$payload\"\n522- expected_sha=\"$(jq -er '.archive_sha256' \"$state\")\"; [[ \"$(sha256sum \"$payload\"|awk '{print $1}')\" == \"$expected_sha\" ]] || die RESTORE_PAYLOAD_SHA_MISMATCH 70\n523- zstd -t \"$payload\" >/dev/null; zstd -dc \"$payload\" | tar -xf - -C \"$work/extract\"; ex=\"$work/extract\"\n524- verify_binding_components \"$ex\" \"$logical\" \"$gen\"\n525- checks=\"$(semantic_validate \"$app\" \"$gen\" \"$ex\")\"\n526- out=\"$(restore_dir \"$app\")/$gen.json\"; mkdir -p \"$(dirname \"$out\")\"; chmod 0700 \"$RESTORE_STATE_ROOT\" \"$(dirname \"$out\")\"\n527- tmp=\"${out}.new.$$\"\n528: jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\n529- chmod 0600 \"$tmp\"; mv -f \"$tmp\" \"$out\"\n530: jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\n531- rm -rf --one-file-system \"$work\"; ACTIVE_STAGE=\"\"\n532- printf 'RESTORE_VALIDATION_STATUS=OK\\nAPP=%s\\nGENERATION=%s\\nEVIDENCE=%s\\n' \"$app\" \"$gen\" \"$out\"\n533-}\n534-backup_one(){\n535- local app=\"$1\" logical pending_count state gen payload latest orphan_count\n536- logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\n537- mapfile -t pending < <(latest_incomplete \"$app\"); pending_count=\"${#pending[@]}\"; (( pending_count <= 1 )) || die \"MULTIPLE_INCOMPLETE_GENERATIONS_$app\" 65\n538- if (( pending_count == 1 )); then\n539- state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\n540- \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\n541: printf 'STAGE=RESTORE_VALIDATION\\n'; restore_validate \"$app\" \"$gen\"\n542- printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return\n543- fi\n544- plan \"$app\" >/dev/null\n545- latest=\"$(latest_sealed_state \"$app\")\"\n546: if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\n547- if [[ \"$app\" == scrutiny ]]; then\n548- orphan_count=\"$(find \"$STAGE_ROOT/$logical\" -maxdepth 1 -type d -name '.capture-app-scrutiny-*' | wc -l)\"\n549- (( orphan_count == 0 )) || die SCRUTINY_ORPHAN_CAPTURE_REQUIRES_CONTROLLED_RECOVERY 65\n550- fi\n551- gen=\"${logical}-pve01-$(date -u +%Y%m%dT%H%M%SZ)\"; valid_generation \"$gen\" || die GENERATED_ID_INVALID 65\n552- printf 'RESUME=NO\\nAPP=%s\\nGENERATION=%s\\nSTAGE=CAPTURE\\n' \"$app\" \"$gen\"; capture_new \"$app\" \"$gen\"; payload=\"$PAYLOAD\"\n553- printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\n554: printf 'STAGE=RESTORE_VALIDATION\\n'; restore_validate \"$app\" \"$gen\"\n555- state=\"$STATE_ROOT/$logical/$gen.json\"; jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .local_payload_present==false and .seal_written==true' \"$state\" >/dev/null\n556- printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"\n557-}\n558-plan(){\n559- local app=\"$1\" logical free\n560- logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\n561- [[ \"$free\" =~ ^[1-9][0-9]+$ ]] || die STAGING_FREE_INVALID 65\n562- local source_bytes=0 required=0\n--\n589- ;;\n590- skladchik-reports-monitor) remote_edge probe plan >/dev/null ;;\n591- esac\n592- printf 'PLAN_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nSOURCE_HOST=%s\\nSTAGING_FREE_BYTES=%s\\nSOURCE_BYTES=%s\\nREQUIRED_STAGING_BYTES=%s\\n' \"$app\" \"$logical\" \"$(source_host \"$app\")\" \"$free\" \"$source_bytes\" \"$required\"\n593-}\n594-latest_sealed(){\n595- local app=\"$1\" f gen\n596- f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\n597: restore_ok \"$app\" \"$gen\" || die \"LATEST_RESTORE_NOT_OK_$app\" 65\n598: printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\n599-}\n600-recover_scrutiny_orphan(){\n601- local gen=\"$1\" started=\"$2\" qstart=\"$3\" qend=\"$4\" expected_config=\"$5\" expected_influx=\"$6\"\n602- local completed unitsha ref iid digest repodigests isize scdb idb devices migrations settings influx_tables tsm_files tsm_bytes wal_files wal_bytes series trange cfgsha boltbytes sqlitebytes image_sha meta_sha config_manifest_sha influx_manifest_sha validation_manifest_sha unit_copy_sha comps report actual_config actual_influx logical state restore\n603- logical=app-scrutiny\n604- valid_generation \"$gen\" || die GENERATION_INVALID 64\n605- [[ \"$gen\" == app-scrutiny-pve01-* ]] || die SCRUTINY_RECOVERY_GENERATION_INVALID 64\n606- [[ \"$started\" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && \"$qstart\" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ && \"$qend\" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die SCRUTINY_RECOVERY_TIMESTAMP_INVALID 64\n--\n639- if [[ -s \"$state\" && ! -L \"$state\" ]] && jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$state\" >/dev/null 2>&1; then\n640- printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=scrutiny\\nGENERATION=%s\\n' \"$gen\"\n641- else\n642- [[ -s \"$PAYLOAD\" && ! -L \"$PAYLOAD\" ]] || die SCRUTINY_RECOVERY_PAYLOAD_MISSING_FOR_TRANSPORT 65\n643- printf 'STAGE=TRANSPORT_RESUME\\n'\n644- \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\n645- fi\n646- printf 'STAGE=RESTORE_VALIDATION\\n'\n647: restore_validate scrutiny \"$gen\"\n648- [[ -s \"$state\" && ! -L \"$state\" ]] || die SCRUTINY_RECOVERY_SEALED_STATE_MISSING 70\n649- jq -e '.status==\"SEALED\" and .cloud_quorum==2 and .seal_written==true and .local_payload_present==false' \"$state\" >/dev/null || die SCRUTINY_RECOVERY_SEALED_STATE_INVALID 70\n650: [[ -s \"$restore\" && ! -L \"$restore\" ]] || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_MISSING 70\n651: jq -e '.result==\"OK\" and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true' \"$restore\" >/dev/null || die SCRUTINY_RECOVERY_RESTORE_EVIDENCE_INVALID 70\n652- printf 'BACKUP_STATUS=SEALED\\nAPP=scrutiny\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$gen\"\n653-}\n654-\n655-ACTION=\"${1:-}\"\n656-case \"$ACTION\" in\n657- plan) [[ \"$#\" == 2 ]] || die USAGE_PLAN 64; plan \"$2\" ;;\n658- backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\n659: restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\n660- recover-scrutiny-orphan) [[ \"$#\" == 7 ]] || die USAGE_RECOVER_SCRUTINY_ORPHAN 64; recover_scrutiny_orphan \"$2\" \"$3\" \"$4\" \"$5\" \"$6\" \"$7\" ;;\n661: latest-sealed) [[ \"$#\" == 2 ]] || die USAGE_LATEST_SEALED 64; latest_sealed \"$2\" ;;\n662- *) die ACTION_INVALID 64 ;;\n663-esac\nAPP_RESTORE_CONTRACT_SOURCE_END=1\nAPP_STATE_FILE=/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json\nAPP_STATE_SHA256=5a0e9a386c2bf83ebed141ef2687dc687427ac186943029d69a6c075087592cc\nAPP_GENERATION=app-skladchik-reports-monitor-pve01-20260921T175029Z\nAPP_RESTORE_FILE=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json\nAPP_RESTORE_SHA256=b33104501140837eac006f04a8256d19d2bf027d9f06fff7dba1d17199edce8e\nAPP_RESTORE_KEYS_BEGIN=1\n[\"app\",\"checks\",\"generation\",\"logical_id\",\"payload_sha256\",\"payload_sha256_verified\",\"restore_source\",\"result\",\"schema\",\"transport_state\",\"transport_state_file\",\"validated_at_utc\"]\nAPP_RESTORE_KEYS_END=1\nAPP_RESTORE_SAFE_VIEW_BEGIN=1\n{\"schema\":\"homelab-app-restore-validation-v1\",\"result\":\"OK\",\"status\":null,\"logical_id\":\"app-skladchik-reports-monitor\",\"generation\":\"app-skladchik-reports-monitor-pve01-20260921T175029Z\",\"transport_state\":\"SEALED\",\"transport_state_file\":\"/var/lib/homelab-backup/cloud-quorum/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260921T175029Z.json\",\"restore_source\":\"Cloud A\",\"payload_sha256\":\"8034a3188e933ff7b316e369869561b8a14ef7a1688ab497477035346b43b3dc\",\"payload_sha256_verified\":true,\"payload_decompression_verified\":null,\"archive_recovery_readback_verified\":null,\"parts_manifest_verified\":null,\"bounded_staging\":null,\"full_local_payload_written\":null,\"tar_test_ok\":null,\"required_files_ok\":null,\"cookie_jar_ok\":null,\"sqlite_ok\":null,\"validated_at_utc\":\"2026-09-21T17:50:46Z\"}\nAPP_RESTORE_SAFE_VIEW_END=1\nAPP_RESTORE_BASE_CONTRACT_OK=true\nUS_NETBIRD_STATE_BEGIN=1\nUS_NETBIRD_STATE_SHA256=12d13c0d07056469b1ad4b9328ce29737d1ab76698a4d6115ca1c5648b95ba26\n{\"consecutive_failures\":14,\"failure_backoff_seconds\":21600,\"generation\":\"v2-us-netbird-20260922T055316Z-d5c92acdd2e8\",\"last_attempt_epoch\":1790056411,\"last_success_epoch\":1789704670,\"last_success_generation\":\"v2-us-netbird-20260918T041032Z-d7648be4c16b\",\"last_success_green_verified\":false,\"last_success_manifest_sha256\":\"f82c761cafe8aa43d90d0b4b6124933799b5087d34daf880791d850d0d15cee5\",\"last_success_payload_sha256\":\"e41e3e012442ef41da1c667d7f5fbe4fd52786b0bb5ffca195660750c773efbf\",\"last_success_replicas\":[{\"crypt\":\"pve01-mail-05-crypt\",\"physical\":\"pve01-mail-05\"},{\"crypt\":\"pve03-mail-04-crypt\",\"physical\":\"pve03-mail-04\"}],\"logical_id\":\"us-netbird\",\"reason\":\"PIPE_FAILED\",\"result\":\"FAIL\",\"retry_after_epoch\":1790078011,\"schema\":1,\"sealed\":false,\"updated_at_utc\":\"2026-09-22T05:53:31Z\"}\nUS_NETBIRD_STATE_END=1\nPIPE_TO_ZSTD_SOURCE_BEGIN=1\n if not c: return None\n c.sort(key=lambda x:(x[0],x[1],x[2]))\n return c[0][3]\n\ndef run(argv, *, input=None, timeout=300, check=True, env=None):\n try:\n p=subprocess.run(argv,input=input,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=timeout,env=env)\n except subprocess.TimeoutExpired: raise BackupError('COMMAND_TIMEOUT')\n if check and p.returncode!=0: raise BackupError('COMMAND_FAILED',str(p.returncode))\n return p\n\ndef stream_process_to_file(argv,out_path,timeout=1800,input=None):\n out_path=pathlib.Path(out_path)\n with open(out_path,'wb') as f:\n try:p=subprocess.run(argv,input=input,stdout=f,stderr=subprocess.PIPE,timeout=timeout)\n except subprocess.TimeoutExpired: raise BackupError('STREAM_COMMAND_TIMEOUT')\n if p.returncode!=0: raise BackupError('STREAM_COMMAND_FAILED',str(p.returncode))\n\ndef pipe_to_zstd(argv,out_path,timeout=1800,input=None):\n zstd=shutil.which('zstd') or '/usr/bin/zstd'\n try:\n src=subprocess.Popen(argv,stdin=subprocess.PIPE if input is not None else None,stdout=subprocess.PIPE,stderr=subprocess.PIPE)\n z=subprocess.Popen([zstd,'-T0','-3','-q','-o',str(out_path)],stdin=src.stdout,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE)\n if src.stdout: src.stdout.close()\n if input is not None:\n src.stdin.write(input); src.stdin.close()\n zerr=z.communicate(timeout=timeout)[1]\n try: src.wait(timeout=60)\n except subprocess.TimeoutExpired:\n src.kill(); src.wait(); raise BackupError('PIPE_SOURCE_TIMEOUT')\n serr=src.stderr.read() if src.stderr else b''\n except subprocess.TimeoutExpired:\n for q in (src,z):\n try:q.kill()\n except Exception:pass\n raise BackupError('PIPE_TIMEOUT')\n if src.returncode!=0 or z.returncode!=0: raise BackupError('PIPE_FAILED',f'{src.returncode}_{z.returncode}')\n\ndef ensure_dir(p,mode=0o700):\n p=pathlib.Path(p); p.mkdir(parents=True,exist_ok=True); os.chmod(p,mode); return p\n\ndef atomic_write(path,data,mode=0o600):\n path=pathlib.Path(path); ensure_dir(path.parent,0o700)\n fd,tmp=tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent))\nPIPE_TO_ZSTD_SOURCE_END=1\nUS_REMOTE_SCRIPT_SOURCE_BEGIN=1\n if pre!=post: raise BackupError('SPB_SOURCE_CHANGED_DURING_CAPTURE')\n return payload,{'generation':gen,'source_class':'vps_spb','consistency':'STATIC_CONFIG_FILESET_PRE_POST_HASH_STABILITY','tls_recovery':'REISSUE_TLS_ON_RESTORE','hashset_sha256':pre},None\n\nUS_REMOTE_SCRIPT=r'''import io,os,sqlite3,sys,tarfile,time,json,hashlib\npaths=['/opt/netbird/docker-compose.yml','/opt/netbird/config.yaml']\ndata_root='/var/lib/docker/volumes/netbird_netbird_data/_data'\nle_root='/var/lib/docker/volumes/netbird_netbird_traefik_letsencrypt/_data'\ndbs=['events.db','store.db','geonames_20260602.db','idp.db']\nnon=['GeoLite2-City_20260602.mmdb']\nout=sys.stdout.buffer\nwith tarfile.open(fileobj=out,mode='w|') as tf:\n for p in paths:\n if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False)\n for name in non:\n p=os.path.join(data_root,name)\n if os.path.isfile(p): tf.add(p,arcname=p.lstrip('/'),recursive=False)\n ac=os.path.join(le_root,'acme.json')\n if os.path.isfile(ac):\n st=os.stat(ac)\n if (st.st_mode & 0o777)!=0o600: raise SystemExit(31)\n json.load(open(ac,'r'))\n tf.add(ac,arcname=ac.lstrip('/'),recursive=False)\n for name in dbs:\n srcp=os.path.join(data_root,name)\n src=sqlite3.connect('file:'+srcp+'?mode=ro',uri=True)\n if src.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(32)\n dst=sqlite3.connect(':memory:'); src.backup(dst)\n if dst.execute('pragma quick_check').fetchone()[0]!='ok': raise SystemExit(33)\n b=dst.serialize(); dst.close(); src.close()\n ti=tarfile.TarInfo(name=('sqlite-backups/'+name)); ti.size=len(b); ti.mode=0o600; ti.mtime=int(time.time()); tf.addfile(ti,io.BytesIO(b))\n'''\n\ndef capture_vps_us(t,gen,work):\n remote=shlex.join(['python3','-I','-B','-c',US_REMOTE_SCRIPT])\n argv=vps_ssh_argv(t,use_agent=False)+[remote]\n payload=work/'payload.tar.zst'; pipe_to_zstd(argv,payload,timeout=1200)\n return payload,{'generation':gen,'source_class':'vps_us','consistency':'SQLITE_ONLINE_BACKUP_PLUS_NONDB_FILES','sqlite_count':4},None\n\ndef _simple_shell_assignments(text):\nUS_REMOTE_SCRIPT_SOURCE_END=1\nVPS_SSH_SOURCE_BEGIN=1\ndef operator_key_fingerprint():\n if not OPERATOR_KEY.is_file() or OPERATOR_KEY.is_symlink(): raise BackupError('OPERATOR_KEY_MISSING')\n p=run(['ssh-keygen','-lf',str(OPERATOR_KEY),'-E','sha256'],timeout=30)\n s=p.stdout.decode('utf-8','replace')\n m=re.search(r'(SHA256:[A-Za-z0-9+/]+)',s)\n if not m or m.group(1)!=OPERATOR_KEY_FINGERPRINT: raise BackupError('OPERATOR_KEY_FINGERPRINT_MISMATCH')\n return m.group(1)\n\ndef scan_hostkey_line(host,port,expected):\n p=run([SSH_KEYSCAN,'-T','8','-p',str(port),'-t','ed25519',host],timeout=15)\n lines=[x.strip() for x in p.stdout.decode().splitlines() if x.strip() and not x.startswith('#')]\n good=[]\n for line in lines:\n f=line.split()\n if len(f)<3 or f[1]!='ssh-ed25519': continue\n try: blob=base64.b64decode(f[2])\n except Exception: continue\n fp='SHA256:'+base64.b64encode(hashlib.sha256(blob).digest()).decode().rstrip('=')\n if fp==expected: good.append(line)\n if len(good)!=1: raise BackupError('HOSTKEY_PIN_MISMATCH')\n return good[0]\n\ndef ensure_vps_known_hosts(targets):\n lines=[]\n for t in targets:\n if t['type'] not in ('vps_md','vps_spb','vps_us'): continue\n lines.append(scan_hostkey_line(t['host'],t['port'],t['hostkey_fingerprint']))\n atomic_write(VPS_KNOWN_HOSTS,('\\n'.join(lines)+'\\n').encode(),0o600)\n\ndef vps_ssh_argv(t, *, use_agent=False):\n a=[SSH,'-p',str(t['port']),'-o','BatchMode=yes','-o','StrictHostKeyChecking=yes','-o','UserKnownHostsFile='+str(VPS_KNOWN_HOSTS),'-o','ConnectTimeout=15']\n if use_agent:\n a += ['-o','IdentitiesOnly=no']\n else:\n a += ['-i',str(OPERATOR_KEY),'-o','IdentitiesOnly=yes','-o','IdentityAgent=none']\n a += [t['user']+'@'+t['host']]\n return a\n\ndef vps_run(t, remote_argv, *, input=None, timeout=300, use_agent=False, check=True):\n remote=shlex.join([str(x) for x in remote_argv])\n return run(vps_ssh_argv(t,use_agent=use_agent)+[remote],input=input,timeout=timeout,check=check)\n\ndef bootstrap_vps_keys(targets):\n operator_key_fingerprint()\n ensure_vps_known_hosts(targets)\n pub=run(['ssh-keygen','-y','-f',str(OPERATOR_KEY)],timeout=30).stdout.strip()+b'\\n'\n results={}\n for t in targets:\n if t['type'] not in ('vps_md','vps_spb'): continue\n probe=vps_run(t,['true'],timeout=30,use_agent=False,check=False)\n if probe.returncode==0:\n results[t['logical_id']]='ALREADY_OPERATOR_KEY'; continue\n if not os.environ.get('SSH_AUTH_SOCK'): raise BackupError('FORWARDED_AGENT_REQUIRED_FOR_BOOTSTRAP')\n script=b\"umask 077; mkdir -p \\\"$HOME/.ssh\\\"; chmod 700 \\\"$HOME/.ssh\\\"; touch \\\"$HOME/.ssh/authorized_keys\\\"; chmod 600 \\\"$HOME/.ssh/authorized_keys\\\"; IFS= read -r k; if grep -qF -- \\\"$k\\\" \\\"$HOME/.ssh/authorized_keys\\\"; then printf 'ALREADY\\\\n'; else printf '%s\\\\n' \\\"$k\\\" >> \\\"$HOME/.ssh/authorized_keys\\\"; printf 'ADDED\\\\n'; fi\"\n p=vps_run(t,['sh','-c',script.decode()],input=pub,timeout=45,use_agent=True)\n tag=p.stdout.decode().strip()\n if tag not in ('ADDED','ALREADY'): raise BackupError('VPS_AUTHORIZED_KEY_BOOTSTRAP_RESULT')\n vps_run(t,['true'],timeout=30,use_agent=False)\n results[t['logical_id']]=tag\n # US was already operator-key based; prove it agentless too.\n us=[x for x in targets if x['type']=='vps_us'][0]\n vps_run(us,['true'],timeout=30,use_agent=False)\n results[us['logical_id']]='ALREADY_US_OPERATOR_KEY'\n return results\n\ndef remote_hashset(t,command):\n p=vps_run(t,['sh','-c',command],timeout=120,use_agent=False)\n return sha256_bytes(p.stdout)\nVPS_SSH_SOURCE_END=1\nUS_NETBIRD_PID_JOURNAL_BEGIN=1\nPID_BEGIN=936114\n2026-09-21T20:30:36+03:00 pve01 homelab-backup-v2[936114]: SELFTEST_REPLACEMENT486=PASS\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_TARGET=us-netbird\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_RESULT=FAIL\n2026-09-21T20:30:51+03:00 pve01 homelab-backup-v2[936114]: SCHEDULE_REASON=PIPE_FAILED\nPID_END=936114\nPID_BEGIN=2121414\n2026-09-22T02:36:58+03:00 pve01 homelab-backup-v2[2121414]: SELFTEST_REPLACEMENT486=PASS\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_TARGET=us-netbird\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_RESULT=FAIL\n2026-09-22T02:37:13+03:00 pve01 homelab-backup-v2[2121414]: SCHEDULE_REASON=PIPE_FAILED\nPID_END=2121414\nPID_BEGIN=3378768\n2026-09-22T08:53:16+03:00 pve01 homelab-backup-v2[3378768]: SELFTEST_REPLACEMENT486=PASS\n2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_TARGET=us-netbird\n2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_RESULT=FAIL\n2026-09-22T08:53:31+03:00 pve01 homelab-backup-v2[3378768]: SCHEDULE_REASON=PIPE_FAILED\nPID_END=3378768\nUS_NETBIRD_PID_JOURNAL_END=1\nUS_NETBIRD_WORK_ARTIFACTS_BEGIN=1\nf|1790056411.7319950600|792|/var/lib/homelab-backup-v2/state/us-netbird.json\nf|1789642259.3981707900|1255|/var/lib/homelab-backup-v2/cold-bootstrap/us-netbird.json\nd|1790056411.7288067540|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260922T055316Z-d5c92acdd2e8\nd|1790033833.7009700260|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T233658Z-709cf01e1256\nd|1790011851.2302881580|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T173036Z-1128898e4cc8\nd|1789989471.8069593230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T111736Z-c5a5c5e71776\nd|1789955807.6619889330|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260921T015632Z-dc499030a55f\nd|1789932571.7940614350|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T192916Z-ca2592a81a30\nd|1789908991.7828657500|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T125616Z-8caeb0b6c7b7\nd|1789886227.6266376830|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T063652Z-e6c72515a153\nd|1789862827.7065704150|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260920T000652Z-7ecdcfcafc12\nd|1789839091.8314245860|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T173116Z-cfc70719f4e9\nd|1789822687.5806853640|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T125752Z-a4efa5c07171\nd|1789814653.6399387990|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T104358Z-b2232f971dbc\nd|1789810393.6722341230|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T093258Z-64f5c2972565\nd|1789807993.6075344840|4096|/mnt/staging/homelab-backup-v2/v2-us-netbird-20260919T085258Z-36f858f91de5\nUS_NETBIRD_WORK_ARTIFACTS_END=1\nUS_NETBIRD_READONLY_PROBES_BEGIN=1\nUS_NETBIRD_READONLY_PROBE_RC=41\nPROBE_TARGET_FOUND=true\nPROBE_TYPE=vps_us\nPROBE_SSH_TRUE_EXCEPTION=BackupError\nPROBE_SSH_TRUE_CODE=COMMAND_FAILED\nPROBE_SSH_TRUE_DETAIL=255\nUS_NETBIRD_READONLY_PROBE_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nUS_NETBIRD_READONLY_PROBES_END=1\nUS_NETBIRD_PIPE_BASELINE_CLASS=SSH_BASELINE_FAILURE\nDECISION=PASS_BACKUP_1117_APP_CONTRACT_AND_PIPE_BASELINE_CLASSIFIED\nNEXT_GATE=REPAIR_ONLY_BACKUP_US_NETBIRD_AND_AUDIT_CANONICAL_CLOUDB_SOURCE_OFFHOST\nTASK_RESULT=PASS_HOMELAB_BACKUP_1116_CONTRACT_PIPE_RCA\nCHANGES_MADE_BY_1117=false\nPRODUCT_MUTATION_BY_1117=false\nVM_MUTATION_BY_1117=false\nCLOUD_MUTATION_BY_1117=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260922-HOMELAB-BACKUP-1116-CONTRACT-PIPE-RCA-1117R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_1116_CONTRACT_PIPE_RCA_END=1\n"
}
View Git Blame Copy Permalink