homelab-runtime-publisher
2760 lines
234 KiB
Plaintext
2760 lines
234 KiB
Plaintext
CHAT_OUTPUT_BEGIN
|
|
COMMAND_ID=SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1
|
|
STATUS=OK
|
|
RC=0
|
|
HOST=pve01
|
|
MODE=read-only
|
|
COMPONENT=skladchik-durability-queue-worker-rca
|
|
REFERENCE_REGISTER_CHECK=OK
|
|
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
|
ERROR_REGISTER_CHECK=OK
|
|
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
|
COMMAND_SHA256=9786047247e48e71139ff52c618c59da24ce317cd7093ee00b699a69b178af52
|
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
|
EXECUTION_STARTED=true
|
|
CHANGE_DECLARED=false
|
|
RESULT_CONTRACT_VALID=true
|
|
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
|
RESULT_CONTRACT_ERROR=NONE
|
|
COMMAND_RC=0
|
|
CHANGES_MADE=false
|
|
ROLLBACK_STARTED=false
|
|
ROLLBACK_RESTORED=null
|
|
MUTATION_OUTCOME=NO_MUTATION
|
|
SANITIZED=yes
|
|
SECRETS_INCLUDED=no
|
|
PRIVATE_ADDRESSES_INCLUDED=no
|
|
RAW_EVIDENCE_SHA256=df86e4a999ff180077eddd8c50c0647c83310a5062be61a88ccaab005bb577ea
|
|
SANITIZED_OUTPUT_SHA256=3aa1864388e614772490c9c29217a4ed56c483df64b4b67276683b5c5078e47e
|
|
OUTPUT_BEGIN
|
|
SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA=V1
|
|
NO_MAIL_SEND=true
|
|
NO_SERVICE_RESTART=true
|
|
NO_TIMER_MUTATION=true
|
|
NO_NETWORK_MUTATION=true
|
|
NO_SOURCE_MUTATION=true
|
|
NO_BACKUP_MUTATION=true
|
|
NO_GIT_PUSH=true
|
|
MODE=read-only
|
|
HOST_PVE01=true
|
|
QUEUE_VM170_QM_RC=0
|
|
QUEUE_VM170_STDERR_PRESENT=false
|
|
QUEUE_VM170_JSON=true
|
|
QUEUE_VM170_GUEST_EXITCODE=0
|
|
VM170_MONITOR_SHA=19123e359f2f7b9a2d373fd7e4464dd426d31b91cabc567c252e70cad40d77c8
|
|
VM170_TIMER_ACTIVE=active
|
|
VM170_TIMER_ENABLED=enabled
|
|
VM170_SERVICE_RESULT=success
|
|
VM170_HEALTH_STATUS=OK
|
|
VM170_HEALTH_TS=2026-10-05T06:49:35Z
|
|
VM170_HEALTH_REPORTS=22
|
|
VM170_HEALTH_OPEN=13
|
|
VM170_HEALTH_RESOLVED=9
|
|
VM170_BASELINE_OK=true
|
|
SUPERVISOR_SHA=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2
|
|
SUPERVISOR_TOP_CONST_1_NAME=BACKUP_HEALTH_AGE_MAX
|
|
SUPERVISOR_TOP_CONST_1_EXPR_SHA=056f62a1cb138698ac25abded3e6493c5519d37a485bb3900a74bad1806f70b3
|
|
SUPERVISOR_TOP_CONST_2_NAME=RESTORE_HEALTH_AGE_MAX
|
|
SUPERVISOR_TOP_CONST_2_EXPR_SHA=ce6a9ab58cc2e0db3f07ff3ef7d288b29b6611afff056f5b818c706075d9dbd5
|
|
SUPERVISOR_TOP_CONST_3_NAME=APP_FRESHNESS_PROBE
|
|
SUPERVISOR_TOP_CONST_3_EXPR_SHA=d6593f7b36fcf7550285e4af7a5f5766c13c2ee0e0a992b808caaed52e54693a
|
|
SUPERVISOR_TOP_CONST_4_NAME=LEGACY_BACKUP_PROBLEMS
|
|
SUPERVISOR_TOP_CONST_4_EXPR_SHA=d84baf639fd4e63d9c728a793f4dc6c457b6bee5737789745170d2150d858ddd
|
|
SUPERVISOR_TOP_CONST_COUNT=4
|
|
SUPERVISOR_TEST_RC=0
|
|
SUPERVISOR_TEST_STDOUT_SHA=9b4438f36f4a40de86d41fd480c6900c28603079d96011398674df7637868c5e
|
|
SUPERVISOR_TEST_OUTPUT_BEGIN
|
|
SUPERVISOR_STATUS=OK
|
|
SUPERVISOR_PROBLEMS=none
|
|
MONITOR=OK reports=22 open=13 resolved=9
|
|
CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z
|
|
REGISTRY_SOURCE=<IP>
|
|
<EMAIL>>
|
|
NETBIRD_ROUTE=<IP> dev wt0 table 7120 src <IP> uid 0
|
|
SUPERVISOR_TEST_OUTPUT_END
|
|
SUPTEST_SUPERVISOR_STATUS=OK
|
|
SUPTEST_SUPERVISOR_PROBLEMS=none
|
|
SUPTEST_MONITOR=OK reports=22 open=13 resolved=9
|
|
SUPTEST_CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z
|
|
SUPTEST_REGISTRY_SOURCE=[PRIVATE_IP]
|
|
SUPTEST_WORKER_TARGET=debian@[PRIVATE_IP]
|
|
SUPTEST_NETBIRD_ROUTE=1.1.1.1 dev wt0 table 7120 src 100.125.187.138 uid 0
|
|
QUEUE_PRESENT=true
|
|
QUEUE_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650
|
|
QUEUE_SHEBANG=#!/usr/bin/env bash
|
|
QUEUE_BASH_N_RC=0
|
|
QUEUE_FUNCTIONS=backup_list,die,dispatch_one,dispatch_plan,freshness_decision,freshness_plan,lock_heavy,migrate_list,plan,verify_all
|
|
QUEUE_ACTIONS=BACKUP_RESUME,BACKUP_UNVERIFIED,SEALED_PROVEN,SKIP_FRESH
|
|
QUEUE_VAR_NAMES=DUE_SECONDS,FRESHNESS_SECONDS,HEAVY_LOCK,PROBE,QUEUE_LOCK,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,WORKER,age,decision,freshness,generation,now_epoch,out,sealed_epoch
|
|
QUEUE_COMMAND_BASENAMES=flock,homelab-app-cloud-quorum-worker,sha256sum
|
|
QUEUE_INTERESTING_LINE_COUNT=32
|
|
QUEUE_LINE_1_NO=4
|
|
QUEUE_LINE_1_TEXT=WORKER=/usr/local/libexec/homelab-app-cloud-quorum-worker
|
|
QUEUE_LINE_2_NO=5
|
|
QUEUE_LINE_2_TEXT=PROBE=/usr/local/libexec/homelab-app-cloud-quorum-freshness-probe
|
|
QUEUE_LINE_3_NO=6
|
|
QUEUE_LINE_3_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}"
|
|
QUEUE_LINE_4_NO=8
|
|
QUEUE_LINE_4_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}"
|
|
QUEUE_LINE_5_NO=9
|
|
QUEUE_LINE_5_TEXT=QUEUE_LOCK=/run/lock/homelab-app-cloud-quorum-queue.lock
|
|
QUEUE_LINE_6_NO=10
|
|
QUEUE_LINE_6_TEXT=HEAVY_LOCK=/run/homelab-backup-heavy.lock
|
|
QUEUE_LINE_7_NO=21
|
|
QUEUE_LINE_7_TEXT=[[ -x "$WORKER" && ! -L "$WORKER" ]] || die WORKER_MISSING 69
|
|
QUEUE_LINE_8_NO=29
|
|
QUEUE_LINE_8_TEXT= for a in "${ALL_APPS[@]}"; do "$WORKER" plan "$a"; done
|
|
QUEUE_LINE_9_NO=32
|
|
QUEUE_LINE_9_TEXT= local app="$1" now_epoch="$2" out decision generation sealed_epoch age
|
|
QUEUE_LINE_10_NO=38
|
|
QUEUE_LINE_10_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
|
|
QUEUE_LINE_11_NO=39
|
|
QUEUE_LINE_11_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_RESUME GENERATION=%s\n' "$app" "$generation"
|
|
QUEUE_LINE_12_NO=42
|
|
QUEUE_LINE_12_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
|
|
QUEUE_LINE_13_NO=50
|
|
QUEUE_LINE_13_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS"
|
|
QUEUE_LINE_14_NO=52
|
|
QUEUE_LINE_14_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS"
|
|
QUEUE_LINE_15_NO=54
|
|
QUEUE_LINE_15_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS"
|
|
QUEUE_LINE_16_NO=56
|
|
QUEUE_LINE_16_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\n' "$app" "$generation" "$age"
|
|
QUEUE_LINE_17_NO=84
|
|
QUEUE_LINE_17_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"; printf '%s\n' "$out"
|
|
QUEUE_LINE_18_NO=85
|
|
QUEUE_LINE_18_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null
|
|
QUEUE_LINE_19_NO=93
|
|
QUEUE_LINE_19_TEXT= local a="$1" freshness decision now_epoch out generation
|
|
QUEUE_LINE_20_NO=112
|
|
QUEUE_LINE_20_TEXT= generation="$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$freshness" | tail -1)"
|
|
QUEUE_LINE_21_NO=113
|
|
QUEUE_LINE_21_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SKIP_FRESH GENERATION=%s\n' "$a" "$generation"
|
|
QUEUE_LINE_22_NO=120
|
|
QUEUE_LINE_22_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"
|
|
QUEUE_LINE_23_NO=122
|
|
QUEUE_LINE_23_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null
|
|
QUEUE_LINE_24_NO=123
|
|
QUEUE_LINE_24_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)"
|
|
QUEUE_LINE_25_NO=124
|
|
QUEUE_LINE_25_TEXT= [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$a" 65
|
|
QUEUE_LINE_26_NO=125
|
|
QUEUE_LINE_26_TEXT= printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$a" "$generation"
|
|
QUEUE_LINE_27_NO=126
|
|
QUEUE_LINE_27_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\n' "$a" "$generation"
|
|
QUEUE_LINE_28_NO=133
|
|
QUEUE_LINE_28_TEXT= if out="$(exec 8>&-; exec 9>&-; "$WORKER" latest-sealed "$a" 2>/dev/null)"; then
|
|
QUEUE_LINE_29_NO=138
|
|
QUEUE_LINE_29_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"; printf '%s\n' "$out"
|
|
QUEUE_LINE_30_NO=139
|
|
QUEUE_LINE_30_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null
|
|
QUEUE_LINE_31_NO=145
|
|
QUEUE_LINE_31_TEXT= for a in "${ALL_APPS[@]}"; do out="$("$WORKER" latest-sealed "$a")"; printf '%s\n' "$out"; done
|
|
QUEUE_LINE_32_NO=153
|
|
QUEUE_LINE_32_TEXT= all) [[ "$#" == 1 ]] || die USAGE_ALL 64; [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die DISPATCHER_MISSING 69; /usr/local/sbin/homelab-backup-dispatcher run APP_TIMER ;;
|
|
QUEUE_SERVICE_SHOW_RC=0
|
|
QUEUE_SERVICE_LOADSTATE=loaded
|
|
QUEUE_SERVICE_ACTIVESTATE=inactive
|
|
QUEUE_SERVICE_UNITFILESTATE=static
|
|
QUEUE_SERVICE_RESULT=success
|
|
QUEUE_SERVICE_EXECMAINSTATUS=0
|
|
QUEUE_SERVICE_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.service
|
|
QUEUE_SERVICE_FRAGMENT_SHA=afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a
|
|
QUEUE_SERVICE_EXECSTART_SHA=22094bb14d8dc5f7399744e183b096583a9c9625527a2f00981f2ec5cd8e1713
|
|
QUEUE_SERVICE_EXEC_BASENAMES=homelab-backup-v2-app-scheduled
|
|
QUEUE_SERVICE_EXEC_ACTIONS=NONE
|
|
QUEUE_TIMER_SHOW_RC=0
|
|
QUEUE_TIMER_LOADSTATE=loaded
|
|
QUEUE_TIMER_ACTIVESTATE=active
|
|
QUEUE_TIMER_UNITFILESTATE=enabled
|
|
QUEUE_TIMER_RESULT=success
|
|
QUEUE_TIMER_EXECMAINSTATUS=
|
|
QUEUE_TIMER_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.timer
|
|
QUEUE_TIMER_FRAGMENT_SHA=af2eedce3df53029b591d31715a3f3e6688ddb997b4823f487e70c339ce88078
|
|
QUEUE_TIMER_EXECSTART_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
|
QUEUE_TIMER_EXEC_BASENAMES=NONE
|
|
QUEUE_TIMER_EXEC_ACTIONS=NONE
|
|
WORKER_CANDIDATE_COUNT=173
|
|
WORKER_1_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_1_PATH_HINT=//root/BACKUP_closeout_evidence_20260825T165406Z/source/homelab-app-cloud-quorum-worker
|
|
WORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913
|
|
WORKER_1_EXECUTABLE=false
|
|
WORKER_1_PRESENT=true
|
|
WORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913
|
|
WORKER_1_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_1_BASH_N_RC=0
|
|
WORKER_1_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_1_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor
|
|
WORKER_1_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_1_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_1_INTERESTING_LINE_COUNT=57
|
|
WORKER_1_LINE_1_NO=4
|
|
WORKER_1_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_1_LINE_2_NO=6
|
|
WORKER_1_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_1_LINE_3_NO=7
|
|
WORKER_1_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_1_LINE_4_NO=8
|
|
WORKER_1_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_1_LINE_5_NO=9
|
|
WORKER_1_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_1_LINE_6_NO=10
|
|
WORKER_1_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_1_LINE_7_NO=12
|
|
WORKER_1_LINE_7_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}"
|
|
WORKER_1_LINE_8_NO=14
|
|
WORKER_1_LINE_8_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}"
|
|
WORKER_1_LINE_9_NO=75
|
|
WORKER_1_LINE_9_TEXT=logical_id(){
|
|
WORKER_1_LINE_10_NO=82
|
|
WORKER_1_LINE_10_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_1_LINE_11_NO=83
|
|
WORKER_1_LINE_11_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_1_LINE_12_NO=101
|
|
WORKER_1_LINE_12_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_1_LINE_13_NO=103
|
|
WORKER_1_LINE_13_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_1_LINE_14_NO=104
|
|
WORKER_1_LINE_14_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_1_LINE_15_NO=115
|
|
WORKER_1_LINE_15_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_1_LINE_16_NO=117
|
|
WORKER_1_LINE_16_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_1_LINE_17_NO=118
|
|
WORKER_1_LINE_17_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_1_LINE_18_NO=119
|
|
WORKER_1_LINE_18_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_1_LINE_19_NO=126
|
|
WORKER_1_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_1_LINE_20_NO=141
|
|
WORKER_1_LINE_20_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_1_LINE_21_NO=160
|
|
WORKER_1_LINE_21_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_1_LINE_22_NO=191
|
|
WORKER_1_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_1_LINE_23_NO=202
|
|
WORKER_1_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_1_LINE_24_NO=218
|
|
WORKER_1_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_1_LINE_25_NO=239
|
|
WORKER_1_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_1_LINE_26_NO=274
|
|
WORKER_1_LINE_26_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_1_LINE_27_NO=275
|
|
WORKER_1_LINE_27_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_1_LINE_28_NO=293
|
|
WORKER_1_LINE_28_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_1_LINE_29_NO=341
|
|
WORKER_1_LINE_29_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_1_LINE_30_NO=346
|
|
WORKER_1_LINE_30_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_1_LINE_31_NO=349
|
|
WORKER_1_LINE_31_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_1_LINE_32_NO=350
|
|
WORKER_1_LINE_32_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_1_LINE_33_NO=353
|
|
WORKER_1_LINE_33_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_1_LINE_34_NO=354
|
|
WORKER_1_LINE_34_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_1_LINE_35_NO=362
|
|
WORKER_1_LINE_35_TEXT= done <"$manifest"
|
|
WORKER_1_LINE_36_NO=431
|
|
WORKER_1_LINE_36_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_1_LINE_37_NO=432
|
|
WORKER_1_LINE_37_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_1_LINE_38_NO=466
|
|
WORKER_1_LINE_38_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_1_LINE_39_NO=471
|
|
WORKER_1_LINE_39_TEXT= if "manifest.json" not in names:
|
|
WORKER_1_LINE_40_NO=473
|
|
WORKER_1_LINE_40_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_1_LINE_41_NO=475
|
|
WORKER_1_LINE_41_TEXT= for row in manifest:
|
|
WORKER_1_LINE_42_NO=516
|
|
WORKER_1_LINE_42_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_1_LINE_43_NO=528
|
|
WORKER_1_LINE_43_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_1_LINE_44_NO=530
|
|
WORKER_1_LINE_44_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_1_LINE_45_NO=536
|
|
WORKER_1_LINE_45_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_1_LINE_46_NO=539
|
|
WORKER_1_LINE_46_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_1_LINE_47_NO=540
|
|
WORKER_1_LINE_47_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_1_LINE_48_NO=546
|
|
WORKER_1_LINE_48_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_1_LINE_49_NO=553
|
|
WORKER_1_LINE_49_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_1_LINE_50_NO=560
|
|
WORKER_1_LINE_50_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_1_LINE_51_NO=596
|
|
WORKER_1_LINE_51_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_1_LINE_52_NO=598
|
|
WORKER_1_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_1_LINE_53_NO=644
|
|
WORKER_1_LINE_53_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_1_LINE_54_NO=655
|
|
WORKER_1_LINE_54_TEXT=ACTION="${1:-}"
|
|
WORKER_1_LINE_55_NO=656
|
|
WORKER_1_LINE_55_TEXT=case "$ACTION" in
|
|
WORKER_1_LINE_56_NO=658
|
|
WORKER_1_LINE_56_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_1_LINE_57_NO=659
|
|
WORKER_1_LINE_57_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_2_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_2_PATH_HINT=root/backup-registry-sha-pins-20260904T062507Z/usr/local/libexec/homelab-app-cloud-quorum-worker
|
|
WORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913
|
|
WORKER_2_EXECUTABLE=true
|
|
WORKER_2_PRESENT=true
|
|
WORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913
|
|
WORKER_2_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_2_BASH_N_RC=0
|
|
WORKER_2_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_2_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor
|
|
WORKER_2_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_2_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_2_INTERESTING_LINE_COUNT=57
|
|
WORKER_2_LINE_1_NO=4
|
|
WORKER_2_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_2_LINE_2_NO=6
|
|
WORKER_2_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_2_LINE_3_NO=7
|
|
WORKER_2_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_2_LINE_4_NO=8
|
|
WORKER_2_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_2_LINE_5_NO=9
|
|
WORKER_2_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_2_LINE_6_NO=10
|
|
WORKER_2_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_2_LINE_7_NO=12
|
|
WORKER_2_LINE_7_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}"
|
|
WORKER_2_LINE_8_NO=14
|
|
WORKER_2_LINE_8_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}"
|
|
WORKER_2_LINE_9_NO=75
|
|
WORKER_2_LINE_9_TEXT=logical_id(){
|
|
WORKER_2_LINE_10_NO=82
|
|
WORKER_2_LINE_10_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_2_LINE_11_NO=83
|
|
WORKER_2_LINE_11_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_2_LINE_12_NO=101
|
|
WORKER_2_LINE_12_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_2_LINE_13_NO=103
|
|
WORKER_2_LINE_13_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_2_LINE_14_NO=104
|
|
WORKER_2_LINE_14_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_2_LINE_15_NO=115
|
|
WORKER_2_LINE_15_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_2_LINE_16_NO=117
|
|
WORKER_2_LINE_16_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_2_LINE_17_NO=118
|
|
WORKER_2_LINE_17_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_2_LINE_18_NO=119
|
|
WORKER_2_LINE_18_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_2_LINE_19_NO=126
|
|
WORKER_2_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_2_LINE_20_NO=141
|
|
WORKER_2_LINE_20_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_2_LINE_21_NO=160
|
|
WORKER_2_LINE_21_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_2_LINE_22_NO=191
|
|
WORKER_2_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_2_LINE_23_NO=202
|
|
WORKER_2_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_2_LINE_24_NO=218
|
|
WORKER_2_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_2_LINE_25_NO=239
|
|
WORKER_2_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_2_LINE_26_NO=274
|
|
WORKER_2_LINE_26_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_2_LINE_27_NO=275
|
|
WORKER_2_LINE_27_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_2_LINE_28_NO=293
|
|
WORKER_2_LINE_28_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_2_LINE_29_NO=341
|
|
WORKER_2_LINE_29_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_2_LINE_30_NO=346
|
|
WORKER_2_LINE_30_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_2_LINE_31_NO=349
|
|
WORKER_2_LINE_31_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_2_LINE_32_NO=350
|
|
WORKER_2_LINE_32_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_2_LINE_33_NO=353
|
|
WORKER_2_LINE_33_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_2_LINE_34_NO=354
|
|
WORKER_2_LINE_34_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_2_LINE_35_NO=362
|
|
WORKER_2_LINE_35_TEXT= done <"$manifest"
|
|
WORKER_2_LINE_36_NO=431
|
|
WORKER_2_LINE_36_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_2_LINE_37_NO=432
|
|
WORKER_2_LINE_37_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_2_LINE_38_NO=466
|
|
WORKER_2_LINE_38_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_2_LINE_39_NO=471
|
|
WORKER_2_LINE_39_TEXT= if "manifest.json" not in names:
|
|
WORKER_2_LINE_40_NO=473
|
|
WORKER_2_LINE_40_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_2_LINE_41_NO=475
|
|
WORKER_2_LINE_41_TEXT= for row in manifest:
|
|
WORKER_2_LINE_42_NO=516
|
|
WORKER_2_LINE_42_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_2_LINE_43_NO=528
|
|
WORKER_2_LINE_43_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_2_LINE_44_NO=530
|
|
WORKER_2_LINE_44_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_2_LINE_45_NO=536
|
|
WORKER_2_LINE_45_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_2_LINE_46_NO=539
|
|
WORKER_2_LINE_46_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_2_LINE_47_NO=540
|
|
WORKER_2_LINE_47_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_2_LINE_48_NO=546
|
|
WORKER_2_LINE_48_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_2_LINE_49_NO=553
|
|
WORKER_2_LINE_49_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_2_LINE_50_NO=560
|
|
WORKER_2_LINE_50_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_2_LINE_51_NO=596
|
|
WORKER_2_LINE_51_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_2_LINE_52_NO=598
|
|
WORKER_2_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_2_LINE_53_NO=644
|
|
WORKER_2_LINE_53_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_2_LINE_54_NO=655
|
|
WORKER_2_LINE_54_TEXT=ACTION="${1:-}"
|
|
WORKER_2_LINE_55_NO=656
|
|
WORKER_2_LINE_55_TEXT=case "$ACTION" in
|
|
WORKER_2_LINE_56_NO=658
|
|
WORKER_2_LINE_56_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_2_LINE_57_NO=659
|
|
WORKER_2_LINE_57_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_3_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_3_PATH_HINT=root/homelab-ops-cr-2026-0024/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker
|
|
WORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309
|
|
WORKER_3_EXECUTABLE=true
|
|
WORKER_3_PRESENT=true
|
|
WORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309
|
|
WORKER_3_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_3_BASH_N_RC=0
|
|
WORKER_3_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_3_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_3_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work
|
|
WORKER_3_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd
|
|
WORKER_3_INTERESTING_LINE_COUNT=52
|
|
WORKER_3_LINE_1_NO=4
|
|
WORKER_3_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_3_LINE_2_NO=6
|
|
WORKER_3_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_3_LINE_3_NO=7
|
|
WORKER_3_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_3_LINE_4_NO=8
|
|
WORKER_3_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_3_LINE_5_NO=9
|
|
WORKER_3_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_3_LINE_6_NO=10
|
|
WORKER_3_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_3_LINE_7_NO=45
|
|
WORKER_3_LINE_7_TEXT=logical_id(){
|
|
WORKER_3_LINE_8_NO=50
|
|
WORKER_3_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_3_LINE_9_NO=64
|
|
WORKER_3_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_3_LINE_10_NO=65
|
|
WORKER_3_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_3_LINE_11_NO=83
|
|
WORKER_3_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_3_LINE_12_NO=85
|
|
WORKER_3_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_3_LINE_13_NO=86
|
|
WORKER_3_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_3_LINE_14_NO=97
|
|
WORKER_3_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_3_LINE_15_NO=99
|
|
WORKER_3_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_3_LINE_16_NO=100
|
|
WORKER_3_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_3_LINE_17_NO=101
|
|
WORKER_3_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_3_LINE_18_NO=108
|
|
WORKER_3_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_3_LINE_19_NO=123
|
|
WORKER_3_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_3_LINE_20_NO=142
|
|
WORKER_3_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_3_LINE_21_NO=150
|
|
WORKER_3_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_3_LINE_22_NO=161
|
|
WORKER_3_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_3_LINE_23_NO=192
|
|
WORKER_3_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_3_LINE_24_NO=203
|
|
WORKER_3_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_3_LINE_25_NO=219
|
|
WORKER_3_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_3_LINE_26_NO=240
|
|
WORKER_3_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_3_LINE_27_NO=256
|
|
WORKER_3_LINE_27_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_3_LINE_28_NO=261
|
|
WORKER_3_LINE_28_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_3_LINE_29_NO=264
|
|
WORKER_3_LINE_29_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_3_LINE_30_NO=265
|
|
WORKER_3_LINE_30_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_3_LINE_31_NO=268
|
|
WORKER_3_LINE_31_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_3_LINE_32_NO=269
|
|
WORKER_3_LINE_32_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_3_LINE_33_NO=277
|
|
WORKER_3_LINE_33_TEXT= done <"$manifest"
|
|
WORKER_3_LINE_34_NO=344
|
|
WORKER_3_LINE_34_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_3_LINE_35_NO=349
|
|
WORKER_3_LINE_35_TEXT= if "manifest.json" not in names:
|
|
WORKER_3_LINE_36_NO=351
|
|
WORKER_3_LINE_36_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_3_LINE_37_NO=353
|
|
WORKER_3_LINE_37_TEXT= for row in manifest:
|
|
WORKER_3_LINE_38_NO=394
|
|
WORKER_3_LINE_38_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_3_LINE_39_NO=406
|
|
WORKER_3_LINE_39_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_3_LINE_40_NO=408
|
|
WORKER_3_LINE_40_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_3_LINE_41_NO=414
|
|
WORKER_3_LINE_41_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_3_LINE_42_NO=417
|
|
WORKER_3_LINE_42_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_3_LINE_43_NO=418
|
|
WORKER_3_LINE_43_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_3_LINE_44_NO=424
|
|
WORKER_3_LINE_44_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_3_LINE_45_NO=427
|
|
WORKER_3_LINE_45_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_3_LINE_46_NO=434
|
|
WORKER_3_LINE_46_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_3_LINE_47_NO=459
|
|
WORKER_3_LINE_47_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_3_LINE_48_NO=461
|
|
WORKER_3_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_3_LINE_49_NO=463
|
|
WORKER_3_LINE_49_TEXT=ACTION="${1:-}"
|
|
WORKER_3_LINE_50_NO=464
|
|
WORKER_3_LINE_50_TEXT=case "$ACTION" in
|
|
WORKER_3_LINE_51_NO=466
|
|
WORKER_3_LINE_51_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_3_LINE_52_NO=467
|
|
WORKER_3_LINE_52_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_4_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_4_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker
|
|
WORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309
|
|
WORKER_4_EXECUTABLE=true
|
|
WORKER_4_PRESENT=true
|
|
WORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309
|
|
WORKER_4_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_4_BASH_N_RC=0
|
|
WORKER_4_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_4_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_4_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work
|
|
WORKER_4_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd
|
|
WORKER_4_INTERESTING_LINE_COUNT=52
|
|
WORKER_4_LINE_1_NO=4
|
|
WORKER_4_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_4_LINE_2_NO=6
|
|
WORKER_4_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_4_LINE_3_NO=7
|
|
WORKER_4_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_4_LINE_4_NO=8
|
|
WORKER_4_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_4_LINE_5_NO=9
|
|
WORKER_4_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_4_LINE_6_NO=10
|
|
WORKER_4_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_4_LINE_7_NO=45
|
|
WORKER_4_LINE_7_TEXT=logical_id(){
|
|
WORKER_4_LINE_8_NO=50
|
|
WORKER_4_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_4_LINE_9_NO=64
|
|
WORKER_4_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_4_LINE_10_NO=65
|
|
WORKER_4_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_4_LINE_11_NO=83
|
|
WORKER_4_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_4_LINE_12_NO=85
|
|
WORKER_4_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_4_LINE_13_NO=86
|
|
WORKER_4_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_4_LINE_14_NO=97
|
|
WORKER_4_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_4_LINE_15_NO=99
|
|
WORKER_4_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_4_LINE_16_NO=100
|
|
WORKER_4_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_4_LINE_17_NO=101
|
|
WORKER_4_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_4_LINE_18_NO=108
|
|
WORKER_4_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_4_LINE_19_NO=123
|
|
WORKER_4_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_4_LINE_20_NO=142
|
|
WORKER_4_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_4_LINE_21_NO=150
|
|
WORKER_4_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_4_LINE_22_NO=161
|
|
WORKER_4_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_4_LINE_23_NO=192
|
|
WORKER_4_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_4_LINE_24_NO=203
|
|
WORKER_4_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_4_LINE_25_NO=219
|
|
WORKER_4_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_4_LINE_26_NO=240
|
|
WORKER_4_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_4_LINE_27_NO=256
|
|
WORKER_4_LINE_27_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_4_LINE_28_NO=261
|
|
WORKER_4_LINE_28_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_4_LINE_29_NO=264
|
|
WORKER_4_LINE_29_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_4_LINE_30_NO=265
|
|
WORKER_4_LINE_30_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_4_LINE_31_NO=268
|
|
WORKER_4_LINE_31_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_4_LINE_32_NO=269
|
|
WORKER_4_LINE_32_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_4_LINE_33_NO=277
|
|
WORKER_4_LINE_33_TEXT= done <"$manifest"
|
|
WORKER_4_LINE_34_NO=344
|
|
WORKER_4_LINE_34_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_4_LINE_35_NO=349
|
|
WORKER_4_LINE_35_TEXT= if "manifest.json" not in names:
|
|
WORKER_4_LINE_36_NO=351
|
|
WORKER_4_LINE_36_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_4_LINE_37_NO=353
|
|
WORKER_4_LINE_37_TEXT= for row in manifest:
|
|
WORKER_4_LINE_38_NO=394
|
|
WORKER_4_LINE_38_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_4_LINE_39_NO=406
|
|
WORKER_4_LINE_39_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_4_LINE_40_NO=408
|
|
WORKER_4_LINE_40_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_4_LINE_41_NO=414
|
|
WORKER_4_LINE_41_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_4_LINE_42_NO=417
|
|
WORKER_4_LINE_42_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_4_LINE_43_NO=418
|
|
WORKER_4_LINE_43_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_4_LINE_44_NO=424
|
|
WORKER_4_LINE_44_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_4_LINE_45_NO=427
|
|
WORKER_4_LINE_45_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_4_LINE_46_NO=434
|
|
WORKER_4_LINE_46_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_4_LINE_47_NO=459
|
|
WORKER_4_LINE_47_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_4_LINE_48_NO=461
|
|
WORKER_4_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_4_LINE_49_NO=463
|
|
WORKER_4_LINE_49_TEXT=ACTION="${1:-}"
|
|
WORKER_4_LINE_50_NO=464
|
|
WORKER_4_LINE_50_TEXT=case "$ACTION" in
|
|
WORKER_4_LINE_51_NO=466
|
|
WORKER_4_LINE_51_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_4_LINE_52_NO=467
|
|
WORKER_4_LINE_52_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_5_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_5_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-scrutiny-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker
|
|
WORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_5_EXECUTABLE=true
|
|
WORKER_5_PRESENT=true
|
|
WORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_5_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_5_BASH_N_RC=0
|
|
WORKER_5_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_5_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_5_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_5_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_5_INTERESTING_LINE_COUNT=58
|
|
WORKER_5_LINE_1_NO=4
|
|
WORKER_5_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_5_LINE_2_NO=6
|
|
WORKER_5_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_5_LINE_3_NO=7
|
|
WORKER_5_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_5_LINE_4_NO=8
|
|
WORKER_5_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_5_LINE_5_NO=9
|
|
WORKER_5_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_5_LINE_6_NO=10
|
|
WORKER_5_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_5_LINE_7_NO=69
|
|
WORKER_5_LINE_7_TEXT=logical_id(){
|
|
WORKER_5_LINE_8_NO=74
|
|
WORKER_5_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_5_LINE_9_NO=90
|
|
WORKER_5_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_5_LINE_10_NO=91
|
|
WORKER_5_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_5_LINE_11_NO=109
|
|
WORKER_5_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_5_LINE_12_NO=111
|
|
WORKER_5_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_5_LINE_13_NO=112
|
|
WORKER_5_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_5_LINE_14_NO=123
|
|
WORKER_5_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_5_LINE_15_NO=125
|
|
WORKER_5_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_5_LINE_16_NO=126
|
|
WORKER_5_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_5_LINE_17_NO=127
|
|
WORKER_5_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_5_LINE_18_NO=134
|
|
WORKER_5_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_5_LINE_19_NO=149
|
|
WORKER_5_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_5_LINE_20_NO=168
|
|
WORKER_5_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_5_LINE_21_NO=176
|
|
WORKER_5_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_5_LINE_22_NO=187
|
|
WORKER_5_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_5_LINE_23_NO=218
|
|
WORKER_5_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_5_LINE_24_NO=229
|
|
WORKER_5_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_5_LINE_25_NO=245
|
|
WORKER_5_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_5_LINE_26_NO=266
|
|
WORKER_5_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_5_LINE_27_NO=301
|
|
WORKER_5_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_5_LINE_28_NO=302
|
|
WORKER_5_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_5_LINE_29_NO=320
|
|
WORKER_5_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_5_LINE_30_NO=368
|
|
WORKER_5_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_5_LINE_31_NO=373
|
|
WORKER_5_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_5_LINE_32_NO=376
|
|
WORKER_5_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_5_LINE_33_NO=377
|
|
WORKER_5_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_5_LINE_34_NO=380
|
|
WORKER_5_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_5_LINE_35_NO=381
|
|
WORKER_5_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_5_LINE_36_NO=389
|
|
WORKER_5_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_5_LINE_37_NO=464
|
|
WORKER_5_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_5_LINE_38_NO=465
|
|
WORKER_5_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_5_LINE_39_NO=499
|
|
WORKER_5_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_5_LINE_40_NO=504
|
|
WORKER_5_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_5_LINE_41_NO=506
|
|
WORKER_5_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_5_LINE_42_NO=508
|
|
WORKER_5_LINE_42_TEXT= for row in manifest:
|
|
WORKER_5_LINE_43_NO=549
|
|
WORKER_5_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_5_LINE_44_NO=561
|
|
WORKER_5_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_5_LINE_45_NO=563
|
|
WORKER_5_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_5_LINE_46_NO=569
|
|
WORKER_5_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_5_LINE_47_NO=572
|
|
WORKER_5_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_5_LINE_48_NO=573
|
|
WORKER_5_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_5_LINE_49_NO=579
|
|
WORKER_5_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_5_LINE_50_NO=586
|
|
WORKER_5_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_5_LINE_51_NO=593
|
|
WORKER_5_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_5_LINE_52_NO=630
|
|
WORKER_5_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_5_LINE_53_NO=632
|
|
WORKER_5_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_5_LINE_54_NO=678
|
|
WORKER_5_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_5_LINE_55_NO=689
|
|
WORKER_5_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_5_LINE_56_NO=690
|
|
WORKER_5_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_5_LINE_57_NO=692
|
|
WORKER_5_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_5_LINE_58_NO=693
|
|
WORKER_5_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_6_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_6_PATH_HINT=homelab-ops-cr-2026-0029/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_6_EXECUTABLE=true
|
|
WORKER_6_PRESENT=true
|
|
WORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_6_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_6_BASH_N_RC=0
|
|
WORKER_6_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_6_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_6_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_6_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_6_INTERESTING_LINE_COUNT=58
|
|
WORKER_6_LINE_1_NO=4
|
|
WORKER_6_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_6_LINE_2_NO=6
|
|
WORKER_6_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_6_LINE_3_NO=7
|
|
WORKER_6_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_6_LINE_4_NO=8
|
|
WORKER_6_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_6_LINE_5_NO=9
|
|
WORKER_6_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_6_LINE_6_NO=10
|
|
WORKER_6_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_6_LINE_7_NO=69
|
|
WORKER_6_LINE_7_TEXT=logical_id(){
|
|
WORKER_6_LINE_8_NO=74
|
|
WORKER_6_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_6_LINE_9_NO=90
|
|
WORKER_6_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_6_LINE_10_NO=91
|
|
WORKER_6_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_6_LINE_11_NO=109
|
|
WORKER_6_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_6_LINE_12_NO=111
|
|
WORKER_6_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_6_LINE_13_NO=112
|
|
WORKER_6_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_6_LINE_14_NO=123
|
|
WORKER_6_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_6_LINE_15_NO=125
|
|
WORKER_6_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_6_LINE_16_NO=126
|
|
WORKER_6_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_6_LINE_17_NO=127
|
|
WORKER_6_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_6_LINE_18_NO=134
|
|
WORKER_6_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_6_LINE_19_NO=149
|
|
WORKER_6_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_6_LINE_20_NO=168
|
|
WORKER_6_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_6_LINE_21_NO=176
|
|
WORKER_6_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_6_LINE_22_NO=187
|
|
WORKER_6_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_6_LINE_23_NO=218
|
|
WORKER_6_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_6_LINE_24_NO=229
|
|
WORKER_6_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_6_LINE_25_NO=245
|
|
WORKER_6_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_6_LINE_26_NO=266
|
|
WORKER_6_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_6_LINE_27_NO=301
|
|
WORKER_6_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_6_LINE_28_NO=302
|
|
WORKER_6_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_6_LINE_29_NO=320
|
|
WORKER_6_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_6_LINE_30_NO=368
|
|
WORKER_6_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_6_LINE_31_NO=373
|
|
WORKER_6_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_6_LINE_32_NO=376
|
|
WORKER_6_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_6_LINE_33_NO=377
|
|
WORKER_6_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_6_LINE_34_NO=380
|
|
WORKER_6_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_6_LINE_35_NO=381
|
|
WORKER_6_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_6_LINE_36_NO=389
|
|
WORKER_6_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_6_LINE_37_NO=464
|
|
WORKER_6_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_6_LINE_38_NO=465
|
|
WORKER_6_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_6_LINE_39_NO=499
|
|
WORKER_6_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_6_LINE_40_NO=504
|
|
WORKER_6_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_6_LINE_41_NO=506
|
|
WORKER_6_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_6_LINE_42_NO=508
|
|
WORKER_6_LINE_42_TEXT= for row in manifest:
|
|
WORKER_6_LINE_43_NO=549
|
|
WORKER_6_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_6_LINE_44_NO=561
|
|
WORKER_6_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_6_LINE_45_NO=563
|
|
WORKER_6_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_6_LINE_46_NO=569
|
|
WORKER_6_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_6_LINE_47_NO=572
|
|
WORKER_6_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_6_LINE_48_NO=573
|
|
WORKER_6_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_6_LINE_49_NO=579
|
|
WORKER_6_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_6_LINE_50_NO=586
|
|
WORKER_6_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_6_LINE_51_NO=593
|
|
WORKER_6_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_6_LINE_52_NO=630
|
|
WORKER_6_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_6_LINE_53_NO=632
|
|
WORKER_6_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_6_LINE_54_NO=678
|
|
WORKER_6_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_6_LINE_55_NO=689
|
|
WORKER_6_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_6_LINE_56_NO=690
|
|
WORKER_6_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_6_LINE_57_NO=692
|
|
WORKER_6_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_6_LINE_58_NO=693
|
|
WORKER_6_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_7_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_7_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_7_EXECUTABLE=true
|
|
WORKER_7_PRESENT=true
|
|
WORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_7_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_7_BASH_N_RC=0
|
|
WORKER_7_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_7_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_7_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_7_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_7_INTERESTING_LINE_COUNT=58
|
|
WORKER_7_LINE_1_NO=4
|
|
WORKER_7_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_7_LINE_2_NO=6
|
|
WORKER_7_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_7_LINE_3_NO=7
|
|
WORKER_7_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_7_LINE_4_NO=8
|
|
WORKER_7_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_7_LINE_5_NO=9
|
|
WORKER_7_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_7_LINE_6_NO=10
|
|
WORKER_7_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_7_LINE_7_NO=69
|
|
WORKER_7_LINE_7_TEXT=logical_id(){
|
|
WORKER_7_LINE_8_NO=74
|
|
WORKER_7_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_7_LINE_9_NO=90
|
|
WORKER_7_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_7_LINE_10_NO=91
|
|
WORKER_7_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_7_LINE_11_NO=109
|
|
WORKER_7_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_7_LINE_12_NO=111
|
|
WORKER_7_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_7_LINE_13_NO=112
|
|
WORKER_7_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_7_LINE_14_NO=123
|
|
WORKER_7_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_7_LINE_15_NO=125
|
|
WORKER_7_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_7_LINE_16_NO=126
|
|
WORKER_7_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_7_LINE_17_NO=127
|
|
WORKER_7_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_7_LINE_18_NO=134
|
|
WORKER_7_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_7_LINE_19_NO=149
|
|
WORKER_7_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_7_LINE_20_NO=168
|
|
WORKER_7_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_7_LINE_21_NO=176
|
|
WORKER_7_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_7_LINE_22_NO=187
|
|
WORKER_7_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_7_LINE_23_NO=218
|
|
WORKER_7_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_7_LINE_24_NO=229
|
|
WORKER_7_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_7_LINE_25_NO=245
|
|
WORKER_7_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_7_LINE_26_NO=266
|
|
WORKER_7_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_7_LINE_27_NO=301
|
|
WORKER_7_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_7_LINE_28_NO=302
|
|
WORKER_7_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_7_LINE_29_NO=320
|
|
WORKER_7_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_7_LINE_30_NO=368
|
|
WORKER_7_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_7_LINE_31_NO=373
|
|
WORKER_7_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_7_LINE_32_NO=376
|
|
WORKER_7_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_7_LINE_33_NO=377
|
|
WORKER_7_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_7_LINE_34_NO=380
|
|
WORKER_7_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_7_LINE_35_NO=381
|
|
WORKER_7_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_7_LINE_36_NO=389
|
|
WORKER_7_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_7_LINE_37_NO=464
|
|
WORKER_7_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_7_LINE_38_NO=465
|
|
WORKER_7_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_7_LINE_39_NO=499
|
|
WORKER_7_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_7_LINE_40_NO=504
|
|
WORKER_7_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_7_LINE_41_NO=506
|
|
WORKER_7_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_7_LINE_42_NO=508
|
|
WORKER_7_LINE_42_TEXT= for row in manifest:
|
|
WORKER_7_LINE_43_NO=549
|
|
WORKER_7_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_7_LINE_44_NO=561
|
|
WORKER_7_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_7_LINE_45_NO=563
|
|
WORKER_7_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_7_LINE_46_NO=569
|
|
WORKER_7_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_7_LINE_47_NO=572
|
|
WORKER_7_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_7_LINE_48_NO=573
|
|
WORKER_7_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_7_LINE_49_NO=579
|
|
WORKER_7_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_7_LINE_50_NO=586
|
|
WORKER_7_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_7_LINE_51_NO=593
|
|
WORKER_7_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_7_LINE_52_NO=630
|
|
WORKER_7_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_7_LINE_53_NO=632
|
|
WORKER_7_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_7_LINE_54_NO=678
|
|
WORKER_7_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_7_LINE_55_NO=689
|
|
WORKER_7_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_7_LINE_56_NO=690
|
|
WORKER_7_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_7_LINE_57_NO=692
|
|
WORKER_7_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_7_LINE_58_NO=693
|
|
WORKER_7_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_8_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_8_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_8_EXECUTABLE=true
|
|
WORKER_8_PRESENT=true
|
|
WORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_8_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_8_BASH_N_RC=0
|
|
WORKER_8_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_8_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_8_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_8_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_8_INTERESTING_LINE_COUNT=58
|
|
WORKER_8_LINE_1_NO=4
|
|
WORKER_8_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_8_LINE_2_NO=6
|
|
WORKER_8_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_8_LINE_3_NO=7
|
|
WORKER_8_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_8_LINE_4_NO=8
|
|
WORKER_8_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_8_LINE_5_NO=9
|
|
WORKER_8_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_8_LINE_6_NO=10
|
|
WORKER_8_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_8_LINE_7_NO=69
|
|
WORKER_8_LINE_7_TEXT=logical_id(){
|
|
WORKER_8_LINE_8_NO=74
|
|
WORKER_8_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_8_LINE_9_NO=90
|
|
WORKER_8_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_8_LINE_10_NO=91
|
|
WORKER_8_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_8_LINE_11_NO=109
|
|
WORKER_8_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_8_LINE_12_NO=111
|
|
WORKER_8_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_8_LINE_13_NO=112
|
|
WORKER_8_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_8_LINE_14_NO=123
|
|
WORKER_8_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_8_LINE_15_NO=125
|
|
WORKER_8_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_8_LINE_16_NO=126
|
|
WORKER_8_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_8_LINE_17_NO=127
|
|
WORKER_8_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_8_LINE_18_NO=134
|
|
WORKER_8_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_8_LINE_19_NO=149
|
|
WORKER_8_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_8_LINE_20_NO=168
|
|
WORKER_8_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_8_LINE_21_NO=176
|
|
WORKER_8_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_8_LINE_22_NO=187
|
|
WORKER_8_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_8_LINE_23_NO=218
|
|
WORKER_8_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_8_LINE_24_NO=229
|
|
WORKER_8_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_8_LINE_25_NO=245
|
|
WORKER_8_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_8_LINE_26_NO=266
|
|
WORKER_8_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_8_LINE_27_NO=301
|
|
WORKER_8_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_8_LINE_28_NO=302
|
|
WORKER_8_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_8_LINE_29_NO=320
|
|
WORKER_8_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_8_LINE_30_NO=368
|
|
WORKER_8_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_8_LINE_31_NO=373
|
|
WORKER_8_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_8_LINE_32_NO=376
|
|
WORKER_8_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_8_LINE_33_NO=377
|
|
WORKER_8_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_8_LINE_34_NO=380
|
|
WORKER_8_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_8_LINE_35_NO=381
|
|
WORKER_8_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_8_LINE_36_NO=389
|
|
WORKER_8_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_8_LINE_37_NO=464
|
|
WORKER_8_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_8_LINE_38_NO=465
|
|
WORKER_8_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_8_LINE_39_NO=499
|
|
WORKER_8_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_8_LINE_40_NO=504
|
|
WORKER_8_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_8_LINE_41_NO=506
|
|
WORKER_8_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_8_LINE_42_NO=508
|
|
WORKER_8_LINE_42_TEXT= for row in manifest:
|
|
WORKER_8_LINE_43_NO=549
|
|
WORKER_8_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_8_LINE_44_NO=561
|
|
WORKER_8_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_8_LINE_45_NO=563
|
|
WORKER_8_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_8_LINE_46_NO=569
|
|
WORKER_8_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_8_LINE_47_NO=572
|
|
WORKER_8_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_8_LINE_48_NO=573
|
|
WORKER_8_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_8_LINE_49_NO=579
|
|
WORKER_8_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_8_LINE_50_NO=586
|
|
WORKER_8_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_8_LINE_51_NO=593
|
|
WORKER_8_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_8_LINE_52_NO=630
|
|
WORKER_8_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_8_LINE_53_NO=632
|
|
WORKER_8_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_8_LINE_54_NO=678
|
|
WORKER_8_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_8_LINE_55_NO=689
|
|
WORKER_8_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_8_LINE_56_NO=690
|
|
WORKER_8_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_8_LINE_57_NO=692
|
|
WORKER_8_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_8_LINE_58_NO=693
|
|
WORKER_8_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_9_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_9_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_9_EXECUTABLE=true
|
|
WORKER_9_PRESENT=true
|
|
WORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_9_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_9_BASH_N_RC=0
|
|
WORKER_9_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_9_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_9_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_9_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_9_INTERESTING_LINE_COUNT=58
|
|
WORKER_9_LINE_1_NO=4
|
|
WORKER_9_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_9_LINE_2_NO=6
|
|
WORKER_9_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_9_LINE_3_NO=7
|
|
WORKER_9_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_9_LINE_4_NO=8
|
|
WORKER_9_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_9_LINE_5_NO=9
|
|
WORKER_9_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_9_LINE_6_NO=10
|
|
WORKER_9_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_9_LINE_7_NO=69
|
|
WORKER_9_LINE_7_TEXT=logical_id(){
|
|
WORKER_9_LINE_8_NO=74
|
|
WORKER_9_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_9_LINE_9_NO=90
|
|
WORKER_9_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_9_LINE_10_NO=91
|
|
WORKER_9_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_9_LINE_11_NO=109
|
|
WORKER_9_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_9_LINE_12_NO=111
|
|
WORKER_9_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_9_LINE_13_NO=112
|
|
WORKER_9_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_9_LINE_14_NO=123
|
|
WORKER_9_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_9_LINE_15_NO=125
|
|
WORKER_9_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_9_LINE_16_NO=126
|
|
WORKER_9_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_9_LINE_17_NO=127
|
|
WORKER_9_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_9_LINE_18_NO=134
|
|
WORKER_9_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_9_LINE_19_NO=149
|
|
WORKER_9_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_9_LINE_20_NO=168
|
|
WORKER_9_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_9_LINE_21_NO=176
|
|
WORKER_9_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_9_LINE_22_NO=187
|
|
WORKER_9_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_9_LINE_23_NO=218
|
|
WORKER_9_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_9_LINE_24_NO=229
|
|
WORKER_9_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_9_LINE_25_NO=245
|
|
WORKER_9_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_9_LINE_26_NO=266
|
|
WORKER_9_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_9_LINE_27_NO=301
|
|
WORKER_9_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_9_LINE_28_NO=302
|
|
WORKER_9_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_9_LINE_29_NO=320
|
|
WORKER_9_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_9_LINE_30_NO=368
|
|
WORKER_9_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_9_LINE_31_NO=373
|
|
WORKER_9_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_9_LINE_32_NO=376
|
|
WORKER_9_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_9_LINE_33_NO=377
|
|
WORKER_9_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_9_LINE_34_NO=380
|
|
WORKER_9_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_9_LINE_35_NO=381
|
|
WORKER_9_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_9_LINE_36_NO=389
|
|
WORKER_9_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_9_LINE_37_NO=464
|
|
WORKER_9_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_9_LINE_38_NO=465
|
|
WORKER_9_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_9_LINE_39_NO=499
|
|
WORKER_9_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_9_LINE_40_NO=504
|
|
WORKER_9_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_9_LINE_41_NO=506
|
|
WORKER_9_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_9_LINE_42_NO=508
|
|
WORKER_9_LINE_42_TEXT= for row in manifest:
|
|
WORKER_9_LINE_43_NO=549
|
|
WORKER_9_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_9_LINE_44_NO=561
|
|
WORKER_9_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_9_LINE_45_NO=563
|
|
WORKER_9_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_9_LINE_46_NO=569
|
|
WORKER_9_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_9_LINE_47_NO=572
|
|
WORKER_9_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_9_LINE_48_NO=573
|
|
WORKER_9_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_9_LINE_49_NO=579
|
|
WORKER_9_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_9_LINE_50_NO=586
|
|
WORKER_9_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_9_LINE_51_NO=593
|
|
WORKER_9_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_9_LINE_52_NO=630
|
|
WORKER_9_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_9_LINE_53_NO=632
|
|
WORKER_9_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_9_LINE_54_NO=678
|
|
WORKER_9_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_9_LINE_55_NO=689
|
|
WORKER_9_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_9_LINE_56_NO=690
|
|
WORKER_9_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_9_LINE_57_NO=692
|
|
WORKER_9_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_9_LINE_58_NO=693
|
|
WORKER_9_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_10_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_10_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_10_EXECUTABLE=true
|
|
WORKER_10_PRESENT=true
|
|
WORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_10_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_10_BASH_N_RC=0
|
|
WORKER_10_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_10_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_10_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_10_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_10_INTERESTING_LINE_COUNT=58
|
|
WORKER_10_LINE_1_NO=4
|
|
WORKER_10_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_10_LINE_2_NO=6
|
|
WORKER_10_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_10_LINE_3_NO=7
|
|
WORKER_10_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_10_LINE_4_NO=8
|
|
WORKER_10_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_10_LINE_5_NO=9
|
|
WORKER_10_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_10_LINE_6_NO=10
|
|
WORKER_10_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_10_LINE_7_NO=69
|
|
WORKER_10_LINE_7_TEXT=logical_id(){
|
|
WORKER_10_LINE_8_NO=74
|
|
WORKER_10_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_10_LINE_9_NO=90
|
|
WORKER_10_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_10_LINE_10_NO=91
|
|
WORKER_10_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_10_LINE_11_NO=109
|
|
WORKER_10_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_10_LINE_12_NO=111
|
|
WORKER_10_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_10_LINE_13_NO=112
|
|
WORKER_10_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_10_LINE_14_NO=123
|
|
WORKER_10_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_10_LINE_15_NO=125
|
|
WORKER_10_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_10_LINE_16_NO=126
|
|
WORKER_10_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_10_LINE_17_NO=127
|
|
WORKER_10_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_10_LINE_18_NO=134
|
|
WORKER_10_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_10_LINE_19_NO=149
|
|
WORKER_10_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_10_LINE_20_NO=168
|
|
WORKER_10_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_10_LINE_21_NO=176
|
|
WORKER_10_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_10_LINE_22_NO=187
|
|
WORKER_10_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_10_LINE_23_NO=218
|
|
WORKER_10_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_10_LINE_24_NO=229
|
|
WORKER_10_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_10_LINE_25_NO=245
|
|
WORKER_10_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_10_LINE_26_NO=266
|
|
WORKER_10_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_10_LINE_27_NO=301
|
|
WORKER_10_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_10_LINE_28_NO=302
|
|
WORKER_10_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_10_LINE_29_NO=320
|
|
WORKER_10_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_10_LINE_30_NO=368
|
|
WORKER_10_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_10_LINE_31_NO=373
|
|
WORKER_10_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_10_LINE_32_NO=376
|
|
WORKER_10_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_10_LINE_33_NO=377
|
|
WORKER_10_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_10_LINE_34_NO=380
|
|
WORKER_10_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_10_LINE_35_NO=381
|
|
WORKER_10_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_10_LINE_36_NO=389
|
|
WORKER_10_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_10_LINE_37_NO=464
|
|
WORKER_10_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_10_LINE_38_NO=465
|
|
WORKER_10_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_10_LINE_39_NO=499
|
|
WORKER_10_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_10_LINE_40_NO=504
|
|
WORKER_10_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_10_LINE_41_NO=506
|
|
WORKER_10_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_10_LINE_42_NO=508
|
|
WORKER_10_LINE_42_TEXT= for row in manifest:
|
|
WORKER_10_LINE_43_NO=549
|
|
WORKER_10_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_10_LINE_44_NO=561
|
|
WORKER_10_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_10_LINE_45_NO=563
|
|
WORKER_10_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_10_LINE_46_NO=569
|
|
WORKER_10_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_10_LINE_47_NO=572
|
|
WORKER_10_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_10_LINE_48_NO=573
|
|
WORKER_10_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_10_LINE_49_NO=579
|
|
WORKER_10_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_10_LINE_50_NO=586
|
|
WORKER_10_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_10_LINE_51_NO=593
|
|
WORKER_10_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_10_LINE_52_NO=630
|
|
WORKER_10_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_10_LINE_53_NO=632
|
|
WORKER_10_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_10_LINE_54_NO=678
|
|
WORKER_10_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_10_LINE_55_NO=689
|
|
WORKER_10_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_10_LINE_56_NO=690
|
|
WORKER_10_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_10_LINE_57_NO=692
|
|
WORKER_10_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_10_LINE_58_NO=693
|
|
WORKER_10_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_11_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_11_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_11_EXECUTABLE=true
|
|
WORKER_11_PRESENT=true
|
|
WORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_11_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_11_BASH_N_RC=0
|
|
WORKER_11_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_11_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_11_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_11_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_11_INTERESTING_LINE_COUNT=58
|
|
WORKER_11_LINE_1_NO=4
|
|
WORKER_11_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_11_LINE_2_NO=6
|
|
WORKER_11_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_11_LINE_3_NO=7
|
|
WORKER_11_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_11_LINE_4_NO=8
|
|
WORKER_11_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_11_LINE_5_NO=9
|
|
WORKER_11_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_11_LINE_6_NO=10
|
|
WORKER_11_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_11_LINE_7_NO=69
|
|
WORKER_11_LINE_7_TEXT=logical_id(){
|
|
WORKER_11_LINE_8_NO=74
|
|
WORKER_11_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_11_LINE_9_NO=90
|
|
WORKER_11_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_11_LINE_10_NO=91
|
|
WORKER_11_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_11_LINE_11_NO=109
|
|
WORKER_11_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_11_LINE_12_NO=111
|
|
WORKER_11_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_11_LINE_13_NO=112
|
|
WORKER_11_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_11_LINE_14_NO=123
|
|
WORKER_11_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_11_LINE_15_NO=125
|
|
WORKER_11_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_11_LINE_16_NO=126
|
|
WORKER_11_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_11_LINE_17_NO=127
|
|
WORKER_11_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_11_LINE_18_NO=134
|
|
WORKER_11_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_11_LINE_19_NO=149
|
|
WORKER_11_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_11_LINE_20_NO=168
|
|
WORKER_11_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_11_LINE_21_NO=176
|
|
WORKER_11_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_11_LINE_22_NO=187
|
|
WORKER_11_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_11_LINE_23_NO=218
|
|
WORKER_11_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_11_LINE_24_NO=229
|
|
WORKER_11_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_11_LINE_25_NO=245
|
|
WORKER_11_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_11_LINE_26_NO=266
|
|
WORKER_11_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_11_LINE_27_NO=301
|
|
WORKER_11_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_11_LINE_28_NO=302
|
|
WORKER_11_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_11_LINE_29_NO=320
|
|
WORKER_11_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_11_LINE_30_NO=368
|
|
WORKER_11_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_11_LINE_31_NO=373
|
|
WORKER_11_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_11_LINE_32_NO=376
|
|
WORKER_11_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_11_LINE_33_NO=377
|
|
WORKER_11_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_11_LINE_34_NO=380
|
|
WORKER_11_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_11_LINE_35_NO=381
|
|
WORKER_11_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_11_LINE_36_NO=389
|
|
WORKER_11_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_11_LINE_37_NO=464
|
|
WORKER_11_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_11_LINE_38_NO=465
|
|
WORKER_11_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_11_LINE_39_NO=499
|
|
WORKER_11_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_11_LINE_40_NO=504
|
|
WORKER_11_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_11_LINE_41_NO=506
|
|
WORKER_11_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_11_LINE_42_NO=508
|
|
WORKER_11_LINE_42_TEXT= for row in manifest:
|
|
WORKER_11_LINE_43_NO=549
|
|
WORKER_11_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_11_LINE_44_NO=561
|
|
WORKER_11_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_11_LINE_45_NO=563
|
|
WORKER_11_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_11_LINE_46_NO=569
|
|
WORKER_11_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_11_LINE_47_NO=572
|
|
WORKER_11_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_11_LINE_48_NO=573
|
|
WORKER_11_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_11_LINE_49_NO=579
|
|
WORKER_11_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_11_LINE_50_NO=586
|
|
WORKER_11_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_11_LINE_51_NO=593
|
|
WORKER_11_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_11_LINE_52_NO=630
|
|
WORKER_11_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_11_LINE_53_NO=632
|
|
WORKER_11_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_11_LINE_54_NO=678
|
|
WORKER_11_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_11_LINE_55_NO=689
|
|
WORKER_11_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_11_LINE_56_NO=690
|
|
WORKER_11_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_11_LINE_57_NO=692
|
|
WORKER_11_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_11_LINE_58_NO=693
|
|
WORKER_11_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_12_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_12_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_12_EXECUTABLE=true
|
|
WORKER_12_PRESENT=true
|
|
WORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_12_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_12_BASH_N_RC=0
|
|
WORKER_12_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_12_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_12_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_12_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_12_INTERESTING_LINE_COUNT=58
|
|
WORKER_12_LINE_1_NO=4
|
|
WORKER_12_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_12_LINE_2_NO=6
|
|
WORKER_12_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_12_LINE_3_NO=7
|
|
WORKER_12_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_12_LINE_4_NO=8
|
|
WORKER_12_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_12_LINE_5_NO=9
|
|
WORKER_12_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_12_LINE_6_NO=10
|
|
WORKER_12_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_12_LINE_7_NO=69
|
|
WORKER_12_LINE_7_TEXT=logical_id(){
|
|
WORKER_12_LINE_8_NO=74
|
|
WORKER_12_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_12_LINE_9_NO=90
|
|
WORKER_12_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_12_LINE_10_NO=91
|
|
WORKER_12_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_12_LINE_11_NO=109
|
|
WORKER_12_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_12_LINE_12_NO=111
|
|
WORKER_12_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_12_LINE_13_NO=112
|
|
WORKER_12_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_12_LINE_14_NO=123
|
|
WORKER_12_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_12_LINE_15_NO=125
|
|
WORKER_12_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_12_LINE_16_NO=126
|
|
WORKER_12_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_12_LINE_17_NO=127
|
|
WORKER_12_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_12_LINE_18_NO=134
|
|
WORKER_12_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_12_LINE_19_NO=149
|
|
WORKER_12_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_12_LINE_20_NO=168
|
|
WORKER_12_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_12_LINE_21_NO=176
|
|
WORKER_12_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_12_LINE_22_NO=187
|
|
WORKER_12_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_12_LINE_23_NO=218
|
|
WORKER_12_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_12_LINE_24_NO=229
|
|
WORKER_12_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_12_LINE_25_NO=245
|
|
WORKER_12_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_12_LINE_26_NO=266
|
|
WORKER_12_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_12_LINE_27_NO=301
|
|
WORKER_12_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_12_LINE_28_NO=302
|
|
WORKER_12_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_12_LINE_29_NO=320
|
|
WORKER_12_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_12_LINE_30_NO=368
|
|
WORKER_12_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_12_LINE_31_NO=373
|
|
WORKER_12_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_12_LINE_32_NO=376
|
|
WORKER_12_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_12_LINE_33_NO=377
|
|
WORKER_12_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_12_LINE_34_NO=380
|
|
WORKER_12_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_12_LINE_35_NO=381
|
|
WORKER_12_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_12_LINE_36_NO=389
|
|
WORKER_12_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_12_LINE_37_NO=464
|
|
WORKER_12_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_12_LINE_38_NO=465
|
|
WORKER_12_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_12_LINE_39_NO=499
|
|
WORKER_12_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_12_LINE_40_NO=504
|
|
WORKER_12_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_12_LINE_41_NO=506
|
|
WORKER_12_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_12_LINE_42_NO=508
|
|
WORKER_12_LINE_42_TEXT= for row in manifest:
|
|
WORKER_12_LINE_43_NO=549
|
|
WORKER_12_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_12_LINE_44_NO=561
|
|
WORKER_12_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_12_LINE_45_NO=563
|
|
WORKER_12_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_12_LINE_46_NO=569
|
|
WORKER_12_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_12_LINE_47_NO=572
|
|
WORKER_12_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_12_LINE_48_NO=573
|
|
WORKER_12_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_12_LINE_49_NO=579
|
|
WORKER_12_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_12_LINE_50_NO=586
|
|
WORKER_12_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_12_LINE_51_NO=593
|
|
WORKER_12_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_12_LINE_52_NO=630
|
|
WORKER_12_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_12_LINE_53_NO=632
|
|
WORKER_12_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_12_LINE_54_NO=678
|
|
WORKER_12_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_12_LINE_55_NO=689
|
|
WORKER_12_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_12_LINE_56_NO=690
|
|
WORKER_12_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_12_LINE_57_NO=692
|
|
WORKER_12_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_12_LINE_58_NO=693
|
|
WORKER_12_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_13_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_13_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_13_EXECUTABLE=true
|
|
WORKER_13_PRESENT=true
|
|
WORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_13_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_13_BASH_N_RC=0
|
|
WORKER_13_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_13_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_13_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_13_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_13_INTERESTING_LINE_COUNT=58
|
|
WORKER_13_LINE_1_NO=4
|
|
WORKER_13_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_13_LINE_2_NO=6
|
|
WORKER_13_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_13_LINE_3_NO=7
|
|
WORKER_13_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_13_LINE_4_NO=8
|
|
WORKER_13_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_13_LINE_5_NO=9
|
|
WORKER_13_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_13_LINE_6_NO=10
|
|
WORKER_13_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_13_LINE_7_NO=69
|
|
WORKER_13_LINE_7_TEXT=logical_id(){
|
|
WORKER_13_LINE_8_NO=74
|
|
WORKER_13_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_13_LINE_9_NO=90
|
|
WORKER_13_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_13_LINE_10_NO=91
|
|
WORKER_13_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_13_LINE_11_NO=109
|
|
WORKER_13_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_13_LINE_12_NO=111
|
|
WORKER_13_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_13_LINE_13_NO=112
|
|
WORKER_13_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_13_LINE_14_NO=123
|
|
WORKER_13_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_13_LINE_15_NO=125
|
|
WORKER_13_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_13_LINE_16_NO=126
|
|
WORKER_13_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_13_LINE_17_NO=127
|
|
WORKER_13_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_13_LINE_18_NO=134
|
|
WORKER_13_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_13_LINE_19_NO=149
|
|
WORKER_13_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_13_LINE_20_NO=168
|
|
WORKER_13_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_13_LINE_21_NO=176
|
|
WORKER_13_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_13_LINE_22_NO=187
|
|
WORKER_13_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_13_LINE_23_NO=218
|
|
WORKER_13_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_13_LINE_24_NO=229
|
|
WORKER_13_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_13_LINE_25_NO=245
|
|
WORKER_13_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_13_LINE_26_NO=266
|
|
WORKER_13_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_13_LINE_27_NO=301
|
|
WORKER_13_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_13_LINE_28_NO=302
|
|
WORKER_13_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_13_LINE_29_NO=320
|
|
WORKER_13_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_13_LINE_30_NO=368
|
|
WORKER_13_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_13_LINE_31_NO=373
|
|
WORKER_13_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_13_LINE_32_NO=376
|
|
WORKER_13_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_13_LINE_33_NO=377
|
|
WORKER_13_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_13_LINE_34_NO=380
|
|
WORKER_13_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_13_LINE_35_NO=381
|
|
WORKER_13_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_13_LINE_36_NO=389
|
|
WORKER_13_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_13_LINE_37_NO=464
|
|
WORKER_13_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_13_LINE_38_NO=465
|
|
WORKER_13_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_13_LINE_39_NO=499
|
|
WORKER_13_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_13_LINE_40_NO=504
|
|
WORKER_13_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_13_LINE_41_NO=506
|
|
WORKER_13_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_13_LINE_42_NO=508
|
|
WORKER_13_LINE_42_TEXT= for row in manifest:
|
|
WORKER_13_LINE_43_NO=549
|
|
WORKER_13_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_13_LINE_44_NO=561
|
|
WORKER_13_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_13_LINE_45_NO=563
|
|
WORKER_13_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_13_LINE_46_NO=569
|
|
WORKER_13_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_13_LINE_47_NO=572
|
|
WORKER_13_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_13_LINE_48_NO=573
|
|
WORKER_13_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_13_LINE_49_NO=579
|
|
WORKER_13_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_13_LINE_50_NO=586
|
|
WORKER_13_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_13_LINE_51_NO=593
|
|
WORKER_13_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_13_LINE_52_NO=630
|
|
WORKER_13_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_13_LINE_53_NO=632
|
|
WORKER_13_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_13_LINE_54_NO=678
|
|
WORKER_13_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_13_LINE_55_NO=689
|
|
WORKER_13_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_13_LINE_56_NO=690
|
|
WORKER_13_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_13_LINE_57_NO=692
|
|
WORKER_13_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_13_LINE_58_NO=693
|
|
WORKER_13_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_14_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_14_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_14_EXECUTABLE=true
|
|
WORKER_14_PRESENT=true
|
|
WORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_14_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_14_BASH_N_RC=0
|
|
WORKER_14_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_14_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_14_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_14_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_14_INTERESTING_LINE_COUNT=58
|
|
WORKER_14_LINE_1_NO=4
|
|
WORKER_14_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_14_LINE_2_NO=6
|
|
WORKER_14_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_14_LINE_3_NO=7
|
|
WORKER_14_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_14_LINE_4_NO=8
|
|
WORKER_14_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_14_LINE_5_NO=9
|
|
WORKER_14_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_14_LINE_6_NO=10
|
|
WORKER_14_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_14_LINE_7_NO=69
|
|
WORKER_14_LINE_7_TEXT=logical_id(){
|
|
WORKER_14_LINE_8_NO=74
|
|
WORKER_14_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_14_LINE_9_NO=90
|
|
WORKER_14_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_14_LINE_10_NO=91
|
|
WORKER_14_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_14_LINE_11_NO=109
|
|
WORKER_14_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_14_LINE_12_NO=111
|
|
WORKER_14_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_14_LINE_13_NO=112
|
|
WORKER_14_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_14_LINE_14_NO=123
|
|
WORKER_14_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_14_LINE_15_NO=125
|
|
WORKER_14_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_14_LINE_16_NO=126
|
|
WORKER_14_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_14_LINE_17_NO=127
|
|
WORKER_14_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_14_LINE_18_NO=134
|
|
WORKER_14_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_14_LINE_19_NO=149
|
|
WORKER_14_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_14_LINE_20_NO=168
|
|
WORKER_14_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_14_LINE_21_NO=176
|
|
WORKER_14_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_14_LINE_22_NO=187
|
|
WORKER_14_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_14_LINE_23_NO=218
|
|
WORKER_14_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_14_LINE_24_NO=229
|
|
WORKER_14_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_14_LINE_25_NO=245
|
|
WORKER_14_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_14_LINE_26_NO=266
|
|
WORKER_14_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_14_LINE_27_NO=301
|
|
WORKER_14_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_14_LINE_28_NO=302
|
|
WORKER_14_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_14_LINE_29_NO=320
|
|
WORKER_14_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_14_LINE_30_NO=368
|
|
WORKER_14_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_14_LINE_31_NO=373
|
|
WORKER_14_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_14_LINE_32_NO=376
|
|
WORKER_14_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_14_LINE_33_NO=377
|
|
WORKER_14_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_14_LINE_34_NO=380
|
|
WORKER_14_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_14_LINE_35_NO=381
|
|
WORKER_14_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_14_LINE_36_NO=389
|
|
WORKER_14_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_14_LINE_37_NO=464
|
|
WORKER_14_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_14_LINE_38_NO=465
|
|
WORKER_14_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_14_LINE_39_NO=499
|
|
WORKER_14_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_14_LINE_40_NO=504
|
|
WORKER_14_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_14_LINE_41_NO=506
|
|
WORKER_14_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_14_LINE_42_NO=508
|
|
WORKER_14_LINE_42_TEXT= for row in manifest:
|
|
WORKER_14_LINE_43_NO=549
|
|
WORKER_14_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_14_LINE_44_NO=561
|
|
WORKER_14_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_14_LINE_45_NO=563
|
|
WORKER_14_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_14_LINE_46_NO=569
|
|
WORKER_14_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_14_LINE_47_NO=572
|
|
WORKER_14_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_14_LINE_48_NO=573
|
|
WORKER_14_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_14_LINE_49_NO=579
|
|
WORKER_14_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_14_LINE_50_NO=586
|
|
WORKER_14_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_14_LINE_51_NO=593
|
|
WORKER_14_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_14_LINE_52_NO=630
|
|
WORKER_14_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_14_LINE_53_NO=632
|
|
WORKER_14_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_14_LINE_54_NO=678
|
|
WORKER_14_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_14_LINE_55_NO=689
|
|
WORKER_14_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_14_LINE_56_NO=690
|
|
WORKER_14_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_14_LINE_57_NO=692
|
|
WORKER_14_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_14_LINE_58_NO=693
|
|
WORKER_14_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_15_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_15_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_15_EXECUTABLE=true
|
|
WORKER_15_PRESENT=true
|
|
WORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_15_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_15_BASH_N_RC=0
|
|
WORKER_15_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_15_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_15_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_15_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_15_INTERESTING_LINE_COUNT=58
|
|
WORKER_15_LINE_1_NO=4
|
|
WORKER_15_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_15_LINE_2_NO=6
|
|
WORKER_15_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_15_LINE_3_NO=7
|
|
WORKER_15_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_15_LINE_4_NO=8
|
|
WORKER_15_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_15_LINE_5_NO=9
|
|
WORKER_15_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_15_LINE_6_NO=10
|
|
WORKER_15_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_15_LINE_7_NO=69
|
|
WORKER_15_LINE_7_TEXT=logical_id(){
|
|
WORKER_15_LINE_8_NO=74
|
|
WORKER_15_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_15_LINE_9_NO=90
|
|
WORKER_15_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_15_LINE_10_NO=91
|
|
WORKER_15_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_15_LINE_11_NO=109
|
|
WORKER_15_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_15_LINE_12_NO=111
|
|
WORKER_15_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_15_LINE_13_NO=112
|
|
WORKER_15_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_15_LINE_14_NO=123
|
|
WORKER_15_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_15_LINE_15_NO=125
|
|
WORKER_15_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_15_LINE_16_NO=126
|
|
WORKER_15_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_15_LINE_17_NO=127
|
|
WORKER_15_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_15_LINE_18_NO=134
|
|
WORKER_15_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_15_LINE_19_NO=149
|
|
WORKER_15_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_15_LINE_20_NO=168
|
|
WORKER_15_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_15_LINE_21_NO=176
|
|
WORKER_15_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_15_LINE_22_NO=187
|
|
WORKER_15_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_15_LINE_23_NO=218
|
|
WORKER_15_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_15_LINE_24_NO=229
|
|
WORKER_15_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_15_LINE_25_NO=245
|
|
WORKER_15_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_15_LINE_26_NO=266
|
|
WORKER_15_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_15_LINE_27_NO=301
|
|
WORKER_15_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_15_LINE_28_NO=302
|
|
WORKER_15_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_15_LINE_29_NO=320
|
|
WORKER_15_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_15_LINE_30_NO=368
|
|
WORKER_15_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_15_LINE_31_NO=373
|
|
WORKER_15_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_15_LINE_32_NO=376
|
|
WORKER_15_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_15_LINE_33_NO=377
|
|
WORKER_15_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_15_LINE_34_NO=380
|
|
WORKER_15_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_15_LINE_35_NO=381
|
|
WORKER_15_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_15_LINE_36_NO=389
|
|
WORKER_15_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_15_LINE_37_NO=464
|
|
WORKER_15_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_15_LINE_38_NO=465
|
|
WORKER_15_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_15_LINE_39_NO=499
|
|
WORKER_15_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_15_LINE_40_NO=504
|
|
WORKER_15_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_15_LINE_41_NO=506
|
|
WORKER_15_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_15_LINE_42_NO=508
|
|
WORKER_15_LINE_42_TEXT= for row in manifest:
|
|
WORKER_15_LINE_43_NO=549
|
|
WORKER_15_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_15_LINE_44_NO=561
|
|
WORKER_15_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_15_LINE_45_NO=563
|
|
WORKER_15_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_15_LINE_46_NO=569
|
|
WORKER_15_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_15_LINE_47_NO=572
|
|
WORKER_15_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_15_LINE_48_NO=573
|
|
WORKER_15_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_15_LINE_49_NO=579
|
|
WORKER_15_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_15_LINE_50_NO=586
|
|
WORKER_15_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_15_LINE_51_NO=593
|
|
WORKER_15_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_15_LINE_52_NO=630
|
|
WORKER_15_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_15_LINE_53_NO=632
|
|
WORKER_15_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_15_LINE_54_NO=678
|
|
WORKER_15_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_15_LINE_55_NO=689
|
|
WORKER_15_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_15_LINE_56_NO=690
|
|
WORKER_15_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_15_LINE_57_NO=692
|
|
WORKER_15_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_15_LINE_58_NO=693
|
|
WORKER_15_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_16_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_16_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_16_EXECUTABLE=true
|
|
WORKER_16_PRESENT=true
|
|
WORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_16_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_16_BASH_N_RC=0
|
|
WORKER_16_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_16_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_16_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_16_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_16_INTERESTING_LINE_COUNT=58
|
|
WORKER_16_LINE_1_NO=4
|
|
WORKER_16_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_16_LINE_2_NO=6
|
|
WORKER_16_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_16_LINE_3_NO=7
|
|
WORKER_16_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_16_LINE_4_NO=8
|
|
WORKER_16_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_16_LINE_5_NO=9
|
|
WORKER_16_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_16_LINE_6_NO=10
|
|
WORKER_16_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_16_LINE_7_NO=69
|
|
WORKER_16_LINE_7_TEXT=logical_id(){
|
|
WORKER_16_LINE_8_NO=74
|
|
WORKER_16_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_16_LINE_9_NO=90
|
|
WORKER_16_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_16_LINE_10_NO=91
|
|
WORKER_16_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_16_LINE_11_NO=109
|
|
WORKER_16_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_16_LINE_12_NO=111
|
|
WORKER_16_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_16_LINE_13_NO=112
|
|
WORKER_16_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_16_LINE_14_NO=123
|
|
WORKER_16_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_16_LINE_15_NO=125
|
|
WORKER_16_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_16_LINE_16_NO=126
|
|
WORKER_16_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_16_LINE_17_NO=127
|
|
WORKER_16_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_16_LINE_18_NO=134
|
|
WORKER_16_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_16_LINE_19_NO=149
|
|
WORKER_16_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_16_LINE_20_NO=168
|
|
WORKER_16_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_16_LINE_21_NO=176
|
|
WORKER_16_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_16_LINE_22_NO=187
|
|
WORKER_16_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_16_LINE_23_NO=218
|
|
WORKER_16_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_16_LINE_24_NO=229
|
|
WORKER_16_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_16_LINE_25_NO=245
|
|
WORKER_16_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_16_LINE_26_NO=266
|
|
WORKER_16_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_16_LINE_27_NO=301
|
|
WORKER_16_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_16_LINE_28_NO=302
|
|
WORKER_16_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_16_LINE_29_NO=320
|
|
WORKER_16_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_16_LINE_30_NO=368
|
|
WORKER_16_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_16_LINE_31_NO=373
|
|
WORKER_16_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_16_LINE_32_NO=376
|
|
WORKER_16_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_16_LINE_33_NO=377
|
|
WORKER_16_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_16_LINE_34_NO=380
|
|
WORKER_16_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_16_LINE_35_NO=381
|
|
WORKER_16_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_16_LINE_36_NO=389
|
|
WORKER_16_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_16_LINE_37_NO=464
|
|
WORKER_16_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_16_LINE_38_NO=465
|
|
WORKER_16_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_16_LINE_39_NO=499
|
|
WORKER_16_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_16_LINE_40_NO=504
|
|
WORKER_16_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_16_LINE_41_NO=506
|
|
WORKER_16_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_16_LINE_42_NO=508
|
|
WORKER_16_LINE_42_TEXT= for row in manifest:
|
|
WORKER_16_LINE_43_NO=549
|
|
WORKER_16_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_16_LINE_44_NO=561
|
|
WORKER_16_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_16_LINE_45_NO=563
|
|
WORKER_16_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_16_LINE_46_NO=569
|
|
WORKER_16_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_16_LINE_47_NO=572
|
|
WORKER_16_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_16_LINE_48_NO=573
|
|
WORKER_16_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_16_LINE_49_NO=579
|
|
WORKER_16_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_16_LINE_50_NO=586
|
|
WORKER_16_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_16_LINE_51_NO=593
|
|
WORKER_16_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_16_LINE_52_NO=630
|
|
WORKER_16_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_16_LINE_53_NO=632
|
|
WORKER_16_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_16_LINE_54_NO=678
|
|
WORKER_16_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_16_LINE_55_NO=689
|
|
WORKER_16_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_16_LINE_56_NO=690
|
|
WORKER_16_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_16_LINE_57_NO=692
|
|
WORKER_16_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_16_LINE_58_NO=693
|
|
WORKER_16_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_17_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_17_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_17_EXECUTABLE=true
|
|
WORKER_17_PRESENT=true
|
|
WORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_17_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_17_BASH_N_RC=0
|
|
WORKER_17_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_17_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_17_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_17_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_17_INTERESTING_LINE_COUNT=58
|
|
WORKER_17_LINE_1_NO=4
|
|
WORKER_17_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_17_LINE_2_NO=6
|
|
WORKER_17_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_17_LINE_3_NO=7
|
|
WORKER_17_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_17_LINE_4_NO=8
|
|
WORKER_17_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_17_LINE_5_NO=9
|
|
WORKER_17_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_17_LINE_6_NO=10
|
|
WORKER_17_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_17_LINE_7_NO=69
|
|
WORKER_17_LINE_7_TEXT=logical_id(){
|
|
WORKER_17_LINE_8_NO=74
|
|
WORKER_17_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_17_LINE_9_NO=90
|
|
WORKER_17_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_17_LINE_10_NO=91
|
|
WORKER_17_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_17_LINE_11_NO=109
|
|
WORKER_17_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_17_LINE_12_NO=111
|
|
WORKER_17_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_17_LINE_13_NO=112
|
|
WORKER_17_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_17_LINE_14_NO=123
|
|
WORKER_17_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_17_LINE_15_NO=125
|
|
WORKER_17_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_17_LINE_16_NO=126
|
|
WORKER_17_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_17_LINE_17_NO=127
|
|
WORKER_17_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_17_LINE_18_NO=134
|
|
WORKER_17_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_17_LINE_19_NO=149
|
|
WORKER_17_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_17_LINE_20_NO=168
|
|
WORKER_17_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_17_LINE_21_NO=176
|
|
WORKER_17_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_17_LINE_22_NO=187
|
|
WORKER_17_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_17_LINE_23_NO=218
|
|
WORKER_17_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_17_LINE_24_NO=229
|
|
WORKER_17_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_17_LINE_25_NO=245
|
|
WORKER_17_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_17_LINE_26_NO=266
|
|
WORKER_17_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_17_LINE_27_NO=301
|
|
WORKER_17_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_17_LINE_28_NO=302
|
|
WORKER_17_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_17_LINE_29_NO=320
|
|
WORKER_17_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_17_LINE_30_NO=368
|
|
WORKER_17_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_17_LINE_31_NO=373
|
|
WORKER_17_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_17_LINE_32_NO=376
|
|
WORKER_17_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_17_LINE_33_NO=377
|
|
WORKER_17_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_17_LINE_34_NO=380
|
|
WORKER_17_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_17_LINE_35_NO=381
|
|
WORKER_17_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_17_LINE_36_NO=389
|
|
WORKER_17_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_17_LINE_37_NO=464
|
|
WORKER_17_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_17_LINE_38_NO=465
|
|
WORKER_17_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_17_LINE_39_NO=499
|
|
WORKER_17_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_17_LINE_40_NO=504
|
|
WORKER_17_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_17_LINE_41_NO=506
|
|
WORKER_17_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_17_LINE_42_NO=508
|
|
WORKER_17_LINE_42_TEXT= for row in manifest:
|
|
WORKER_17_LINE_43_NO=549
|
|
WORKER_17_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_17_LINE_44_NO=561
|
|
WORKER_17_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_17_LINE_45_NO=563
|
|
WORKER_17_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_17_LINE_46_NO=569
|
|
WORKER_17_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_17_LINE_47_NO=572
|
|
WORKER_17_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_17_LINE_48_NO=573
|
|
WORKER_17_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_17_LINE_49_NO=579
|
|
WORKER_17_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_17_LINE_50_NO=586
|
|
WORKER_17_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_17_LINE_51_NO=593
|
|
WORKER_17_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_17_LINE_52_NO=630
|
|
WORKER_17_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_17_LINE_53_NO=632
|
|
WORKER_17_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_17_LINE_54_NO=678
|
|
WORKER_17_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_17_LINE_55_NO=689
|
|
WORKER_17_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_17_LINE_56_NO=690
|
|
WORKER_17_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_17_LINE_57_NO=692
|
|
WORKER_17_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_17_LINE_58_NO=693
|
|
WORKER_17_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_18_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_18_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_18_EXECUTABLE=true
|
|
WORKER_18_PRESENT=true
|
|
WORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_18_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_18_BASH_N_RC=0
|
|
WORKER_18_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_18_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_18_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_18_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_18_INTERESTING_LINE_COUNT=58
|
|
WORKER_18_LINE_1_NO=4
|
|
WORKER_18_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_18_LINE_2_NO=6
|
|
WORKER_18_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_18_LINE_3_NO=7
|
|
WORKER_18_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_18_LINE_4_NO=8
|
|
WORKER_18_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_18_LINE_5_NO=9
|
|
WORKER_18_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_18_LINE_6_NO=10
|
|
WORKER_18_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_18_LINE_7_NO=69
|
|
WORKER_18_LINE_7_TEXT=logical_id(){
|
|
WORKER_18_LINE_8_NO=74
|
|
WORKER_18_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_18_LINE_9_NO=90
|
|
WORKER_18_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_18_LINE_10_NO=91
|
|
WORKER_18_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_18_LINE_11_NO=109
|
|
WORKER_18_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_18_LINE_12_NO=111
|
|
WORKER_18_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_18_LINE_13_NO=112
|
|
WORKER_18_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_18_LINE_14_NO=123
|
|
WORKER_18_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_18_LINE_15_NO=125
|
|
WORKER_18_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_18_LINE_16_NO=126
|
|
WORKER_18_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_18_LINE_17_NO=127
|
|
WORKER_18_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_18_LINE_18_NO=134
|
|
WORKER_18_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_18_LINE_19_NO=149
|
|
WORKER_18_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_18_LINE_20_NO=168
|
|
WORKER_18_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_18_LINE_21_NO=176
|
|
WORKER_18_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_18_LINE_22_NO=187
|
|
WORKER_18_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_18_LINE_23_NO=218
|
|
WORKER_18_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_18_LINE_24_NO=229
|
|
WORKER_18_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_18_LINE_25_NO=245
|
|
WORKER_18_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_18_LINE_26_NO=266
|
|
WORKER_18_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_18_LINE_27_NO=301
|
|
WORKER_18_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_18_LINE_28_NO=302
|
|
WORKER_18_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_18_LINE_29_NO=320
|
|
WORKER_18_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_18_LINE_30_NO=368
|
|
WORKER_18_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_18_LINE_31_NO=373
|
|
WORKER_18_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_18_LINE_32_NO=376
|
|
WORKER_18_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_18_LINE_33_NO=377
|
|
WORKER_18_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_18_LINE_34_NO=380
|
|
WORKER_18_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_18_LINE_35_NO=381
|
|
WORKER_18_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_18_LINE_36_NO=389
|
|
WORKER_18_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_18_LINE_37_NO=464
|
|
WORKER_18_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_18_LINE_38_NO=465
|
|
WORKER_18_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_18_LINE_39_NO=499
|
|
WORKER_18_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_18_LINE_40_NO=504
|
|
WORKER_18_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_18_LINE_41_NO=506
|
|
WORKER_18_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_18_LINE_42_NO=508
|
|
WORKER_18_LINE_42_TEXT= for row in manifest:
|
|
WORKER_18_LINE_43_NO=549
|
|
WORKER_18_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_18_LINE_44_NO=561
|
|
WORKER_18_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_18_LINE_45_NO=563
|
|
WORKER_18_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_18_LINE_46_NO=569
|
|
WORKER_18_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_18_LINE_47_NO=572
|
|
WORKER_18_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_18_LINE_48_NO=573
|
|
WORKER_18_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_18_LINE_49_NO=579
|
|
WORKER_18_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_18_LINE_50_NO=586
|
|
WORKER_18_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_18_LINE_51_NO=593
|
|
WORKER_18_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_18_LINE_52_NO=630
|
|
WORKER_18_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_18_LINE_53_NO=632
|
|
WORKER_18_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_18_LINE_54_NO=678
|
|
WORKER_18_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_18_LINE_55_NO=689
|
|
WORKER_18_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_18_LINE_56_NO=690
|
|
WORKER_18_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_18_LINE_57_NO=692
|
|
WORKER_18_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_18_LINE_58_NO=693
|
|
WORKER_18_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_19_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_19_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_19_EXECUTABLE=true
|
|
WORKER_19_PRESENT=true
|
|
WORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_19_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_19_BASH_N_RC=0
|
|
WORKER_19_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_19_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_19_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_19_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_19_INTERESTING_LINE_COUNT=58
|
|
WORKER_19_LINE_1_NO=4
|
|
WORKER_19_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_19_LINE_2_NO=6
|
|
WORKER_19_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_19_LINE_3_NO=7
|
|
WORKER_19_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_19_LINE_4_NO=8
|
|
WORKER_19_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_19_LINE_5_NO=9
|
|
WORKER_19_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_19_LINE_6_NO=10
|
|
WORKER_19_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_19_LINE_7_NO=69
|
|
WORKER_19_LINE_7_TEXT=logical_id(){
|
|
WORKER_19_LINE_8_NO=74
|
|
WORKER_19_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_19_LINE_9_NO=90
|
|
WORKER_19_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_19_LINE_10_NO=91
|
|
WORKER_19_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_19_LINE_11_NO=109
|
|
WORKER_19_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_19_LINE_12_NO=111
|
|
WORKER_19_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_19_LINE_13_NO=112
|
|
WORKER_19_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_19_LINE_14_NO=123
|
|
WORKER_19_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_19_LINE_15_NO=125
|
|
WORKER_19_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_19_LINE_16_NO=126
|
|
WORKER_19_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_19_LINE_17_NO=127
|
|
WORKER_19_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_19_LINE_18_NO=134
|
|
WORKER_19_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_19_LINE_19_NO=149
|
|
WORKER_19_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_19_LINE_20_NO=168
|
|
WORKER_19_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_19_LINE_21_NO=176
|
|
WORKER_19_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_19_LINE_22_NO=187
|
|
WORKER_19_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_19_LINE_23_NO=218
|
|
WORKER_19_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_19_LINE_24_NO=229
|
|
WORKER_19_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_19_LINE_25_NO=245
|
|
WORKER_19_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_19_LINE_26_NO=266
|
|
WORKER_19_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_19_LINE_27_NO=301
|
|
WORKER_19_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_19_LINE_28_NO=302
|
|
WORKER_19_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_19_LINE_29_NO=320
|
|
WORKER_19_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_19_LINE_30_NO=368
|
|
WORKER_19_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_19_LINE_31_NO=373
|
|
WORKER_19_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_19_LINE_32_NO=376
|
|
WORKER_19_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_19_LINE_33_NO=377
|
|
WORKER_19_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_19_LINE_34_NO=380
|
|
WORKER_19_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_19_LINE_35_NO=381
|
|
WORKER_19_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_19_LINE_36_NO=389
|
|
WORKER_19_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_19_LINE_37_NO=464
|
|
WORKER_19_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_19_LINE_38_NO=465
|
|
WORKER_19_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_19_LINE_39_NO=499
|
|
WORKER_19_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_19_LINE_40_NO=504
|
|
WORKER_19_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_19_LINE_41_NO=506
|
|
WORKER_19_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_19_LINE_42_NO=508
|
|
WORKER_19_LINE_42_TEXT= for row in manifest:
|
|
WORKER_19_LINE_43_NO=549
|
|
WORKER_19_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_19_LINE_44_NO=561
|
|
WORKER_19_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_19_LINE_45_NO=563
|
|
WORKER_19_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_19_LINE_46_NO=569
|
|
WORKER_19_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_19_LINE_47_NO=572
|
|
WORKER_19_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_19_LINE_48_NO=573
|
|
WORKER_19_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_19_LINE_49_NO=579
|
|
WORKER_19_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_19_LINE_50_NO=586
|
|
WORKER_19_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_19_LINE_51_NO=593
|
|
WORKER_19_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_19_LINE_52_NO=630
|
|
WORKER_19_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_19_LINE_53_NO=632
|
|
WORKER_19_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_19_LINE_54_NO=678
|
|
WORKER_19_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_19_LINE_55_NO=689
|
|
WORKER_19_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_19_LINE_56_NO=690
|
|
WORKER_19_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_19_LINE_57_NO=692
|
|
WORKER_19_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_19_LINE_58_NO=693
|
|
WORKER_19_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
WORKER_20_BASENAME=homelab-app-cloud-quorum-worker
|
|
WORKER_20_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker
|
|
WORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_20_EXECUTABLE=true
|
|
WORKER_20_PRESENT=true
|
|
WORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72
|
|
WORKER_20_SHEBANG=#!/usr/bin/env bash
|
|
WORKER_20_BASH_N_RC=0
|
|
WORKER_20_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding
|
|
WORKER_20_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor
|
|
WORKER_20_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work
|
|
WORKER_20_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd
|
|
WORKER_20_INTERESTING_LINE_COUNT=58
|
|
WORKER_20_LINE_1_NO=4
|
|
WORKER_20_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload
|
|
WORKER_20_LINE_2_NO=6
|
|
WORKER_20_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper
|
|
WORKER_20_LINE_3_NO=7
|
|
WORKER_20_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper
|
|
WORKER_20_LINE_4_NO=8
|
|
WORKER_20_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum
|
|
WORKER_20_LINE_5_NO=9
|
|
WORKER_20_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation
|
|
WORKER_20_LINE_6_NO=10
|
|
WORKER_20_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum
|
|
WORKER_20_LINE_7_NO=69
|
|
WORKER_20_LINE_7_TEXT=logical_id(){
|
|
WORKER_20_LINE_8_NO=74
|
|
WORKER_20_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;
|
|
WORKER_20_LINE_9_NO=90
|
|
WORKER_20_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_20_LINE_10_NO=91
|
|
WORKER_20_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; }
|
|
WORKER_20_LINE_11_NO=109
|
|
WORKER_20_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null
|
|
WORKER_20_LINE_12_NO=111
|
|
WORKER_20_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; }
|
|
WORKER_20_LINE_13_NO=112
|
|
WORKER_20_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; }
|
|
WORKER_20_LINE_14_NO=123
|
|
WORKER_20_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3"
|
|
WORKER_20_LINE_15_NO=125
|
|
WORKER_20_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70
|
|
WORKER_20_LINE_16_NO=126
|
|
WORKER_20_LINE_16_TEXT= if [[ -s "$manifest" ]]; then
|
|
WORKER_20_LINE_17_NO=127
|
|
WORKER_20_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70
|
|
WORKER_20_LINE_18_NO=134
|
|
WORKER_20_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file"
|
|
WORKER_20_LINE_19_NO=149
|
|
WORKER_20_LINE_19_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_20_LINE_20_NO=168
|
|
WORKER_20_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_20_LINE_21_NO=176
|
|
WORKER_20_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'"
|
|
WORKER_20_LINE_22_NO=187
|
|
WORKER_20_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps"
|
|
WORKER_20_LINE_23_NO=218
|
|
WORKER_20_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_20_LINE_24_NO=229
|
|
WORKER_20_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps"
|
|
WORKER_20_LINE_25_NO=245
|
|
WORKER_20_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_20_LINE_26_NO=266
|
|
WORKER_20_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_20_LINE_27_NO=301
|
|
WORKER_20_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_20_LINE_28_NO=302
|
|
WORKER_20_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_20_LINE_29_NO=320
|
|
WORKER_20_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps"
|
|
WORKER_20_LINE_30_NO=368
|
|
WORKER_20_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70
|
|
WORKER_20_LINE_31_NO=373
|
|
WORKER_20_LINE_31_TEXT= done < <(jq -r '.<EMAIL>' "$ex/app-generation.json")
|
|
WORKER_20_LINE_32_NO=376
|
|
WORKER_20_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual
|
|
WORKER_20_LINE_33_NO=377
|
|
WORKER_20_LINE_33_TEXT= logical="$(logical_id "$app")"
|
|
WORKER_20_LINE_34_NO=380
|
|
WORKER_20_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest"
|
|
WORKER_20_LINE_35_NO=381
|
|
WORKER_20_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70
|
|
WORKER_20_LINE_36_NO=389
|
|
WORKER_20_LINE_36_TEXT= done <"$manifest"
|
|
WORKER_20_LINE_37_NO=464
|
|
WORKER_20_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70)
|
|
WORKER_20_LINE_38_NO=465
|
|
WORKER_20_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_20_LINE_39_NO=499
|
|
WORKER_20_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"}
|
|
WORKER_20_LINE_40_NO=504
|
|
WORKER_20_LINE_40_TEXT= if "manifest.json" not in names:
|
|
WORKER_20_LINE_41_NO=506
|
|
WORKER_20_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json"))
|
|
WORKER_20_LINE_42_NO=508
|
|
WORKER_20_LINE_42_TEXT= for row in manifest:
|
|
WORKER_20_LINE_43_NO=549
|
|
WORKER_20_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json"
|
|
WORKER_20_LINE_44_NO=561
|
|
WORKER_20_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp"
|
|
WORKER_20_LINE_45_NO=563
|
|
WORKER_20_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null
|
|
WORKER_20_LINE_46_NO=569
|
|
WORKER_20_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical"
|
|
WORKER_20_LINE_47_NO=572
|
|
WORKER_20_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen"
|
|
WORKER_20_LINE_48_NO=573
|
|
WORKER_20_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_20_LINE_49_NO=579
|
|
WORKER_20_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi
|
|
WORKER_20_LINE_50_NO=586
|
|
WORKER_20_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_20_LINE_51_NO=593
|
|
WORKER_20_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')"
|
|
WORKER_20_LINE_52_NO=630
|
|
WORKER_20_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")"
|
|
WORKER_20_LINE_53_NO=632
|
|
WORKER_20_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json"
|
|
WORKER_20_LINE_54_NO=678
|
|
WORKER_20_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT"
|
|
WORKER_20_LINE_55_NO=689
|
|
WORKER_20_LINE_55_TEXT=ACTION="${1:-}"
|
|
WORKER_20_LINE_56_NO=690
|
|
WORKER_20_LINE_56_TEXT=case "$ACTION" in
|
|
WORKER_20_LINE_57_NO=692
|
|
WORKER_20_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;;
|
|
WORKER_20_LINE_58_NO=693
|
|
WORKER_20_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;;
|
|
TARGETS_PRESENT=true
|
|
TARGETS_SHA=2f526330152d49461ff66f50adab57d1a195ea5d1b4c292945d523411b8a68a8
|
|
TARGET_OBJECT_FOUND=true
|
|
TARGET_KEYS=id,kind,lifecycle,logical_id,migrate,owner,source_host
|
|
TARGET_ID=skladchik-reports-monitor
|
|
TARGET_KIND=APP
|
|
TARGET_LIFECYCLE=ACTIVE
|
|
TARGET_LOGICAL_ID=app-skladchik-reports-monitor
|
|
TARGET_MIGRATE=True
|
|
TARGET_OWNER=pve01
|
|
TARGET_SOURCE_HOST_BASENAME=[PRIVATE_IP]
|
|
TARGET_SOURCE_HOST_SHA=3eb2319e39eac307b3c7bd11f67663c0a0c814fd603989f5965083f9d1c80fb0
|
|
QUEUE_APPID_LITERAL_COUNT=0
|
|
QUEUE_TARGETS_BASENAME_LITERAL_COUNT=0
|
|
BACKUP_AUDIT_RC=0
|
|
BACKUP_AUDIT_RELEVANT_COUNT=2
|
|
BACKUP_AUDIT_RELEVANT_1=TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=22759|generation=app-skladchik-reports-monitor-pve01-20261005T003035Z
|
|
BACKUP_AUDIT_RELEVANT_2=TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=32145|legacy_age_seconds=1627370|replica_count=2|state_id=vm170|generation=v2-vm170-20261004T214054Z-70a537ab4e91
|
|
TARGETED_SEAL_DESIGN_READY=true
|
|
MAIL_SENT_BY_COMMAND=false
|
|
CHANGES_MADE=false
|
|
ROLLBACK_STARTED=false
|
|
ROLLBACK_RESTORED=null
|
|
MUTATION_OUTCOME=NO_MUTATION
|
|
DECISION=PASS_QUEUE_WORKER_RCA
|
|
NEXT_GATE=DESIGN_ATOMIC_SOURCE_BACKUP_SEAL
|
|
TASK_RESULT=SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA
|
|
|
|
OUTPUT_END
|
|
CHAT_OUTPUT_END
|