diff --git a/runtime/history/SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1.json b/runtime/history/SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1.json new file mode 100644 index 00000000..88b54e4f --- /dev/null +++ b/runtime/history/SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "skladchik-durability-queue-worker-rca", + "started_at_utc": "2026-10-05T06:49:57Z", + "finished_at_utc": "2026-10-05T06:50:09Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "9786047247e48e71139ff52c618c59da24ce317cd7093ee00b699a69b178af52", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "df86e4a999ff180077eddd8c50c0647c83310a5062be61a88ccaab005bb577ea", + "sanitized_output_sha256": "3aa1864388e614772490c9c29217a4ed56c483df64b4b67276683b5c5078e47e", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA=V1\nNO_MAIL_SEND=true\nNO_SERVICE_RESTART=true\nNO_TIMER_MUTATION=true\nNO_NETWORK_MUTATION=true\nNO_SOURCE_MUTATION=true\nNO_BACKUP_MUTATION=true\nNO_GIT_PUSH=true\nMODE=read-only\nHOST_PVE01=true\nQUEUE_VM170_QM_RC=0\nQUEUE_VM170_STDERR_PRESENT=false\nQUEUE_VM170_JSON=true\nQUEUE_VM170_GUEST_EXITCODE=0\nVM170_MONITOR_SHA=19123e359f2f7b9a2d373fd7e4464dd426d31b91cabc567c252e70cad40d77c8\nVM170_TIMER_ACTIVE=active\nVM170_TIMER_ENABLED=enabled\nVM170_SERVICE_RESULT=success\nVM170_HEALTH_STATUS=OK\nVM170_HEALTH_TS=2026-10-05T06:49:35Z\nVM170_HEALTH_REPORTS=22\nVM170_HEALTH_OPEN=13\nVM170_HEALTH_RESOLVED=9\nVM170_BASELINE_OK=true\nSUPERVISOR_SHA=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2\nSUPERVISOR_TOP_CONST_1_NAME=BACKUP_HEALTH_AGE_MAX\nSUPERVISOR_TOP_CONST_1_EXPR_SHA=056f62a1cb138698ac25abded3e6493c5519d37a485bb3900a74bad1806f70b3\nSUPERVISOR_TOP_CONST_2_NAME=RESTORE_HEALTH_AGE_MAX\nSUPERVISOR_TOP_CONST_2_EXPR_SHA=ce6a9ab58cc2e0db3f07ff3ef7d288b29b6611afff056f5b818c706075d9dbd5\nSUPERVISOR_TOP_CONST_3_NAME=APP_FRESHNESS_PROBE\nSUPERVISOR_TOP_CONST_3_EXPR_SHA=d6593f7b36fcf7550285e4af7a5f5766c13c2ee0e0a992b808caaed52e54693a\nSUPERVISOR_TOP_CONST_4_NAME=LEGACY_BACKUP_PROBLEMS\nSUPERVISOR_TOP_CONST_4_EXPR_SHA=d84baf639fd4e63d9c728a793f4dc6c457b6bee5737789745170d2150d858ddd\nSUPERVISOR_TOP_CONST_COUNT=4\nSUPERVISOR_TEST_RC=0\nSUPERVISOR_TEST_STDOUT_SHA=9b4438f36f4a40de86d41fd480c6900c28603079d96011398674df7637868c5e\nSUPERVISOR_TEST_OUTPUT_BEGIN\nSUPERVISOR_STATUS=OK\nSUPERVISOR_PROBLEMS=none\nMONITOR=OK reports=22 open=13 resolved=9\nCANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z\nREGISTRY_SOURCE=\n>\nNETBIRD_ROUTE= dev wt0 table 7120 src uid 0\nSUPERVISOR_TEST_OUTPUT_END\nSUPTEST_SUPERVISOR_STATUS=OK\nSUPTEST_SUPERVISOR_PROBLEMS=none\nSUPTEST_MONITOR=OK reports=22 open=13 resolved=9\nSUPTEST_CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z\nSUPTEST_REGISTRY_SOURCE=[PRIVATE_IP]\nSUPTEST_WORKER_TARGET=debian@[PRIVATE_IP]\nSUPTEST_NETBIRD_ROUTE=1.1.1.1 dev wt0 table 7120 src 100.125.187.138 uid 0\nQUEUE_PRESENT=true\nQUEUE_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\nQUEUE_SHEBANG=#!/usr/bin/env bash\nQUEUE_BASH_N_RC=0\nQUEUE_FUNCTIONS=backup_list,die,dispatch_one,dispatch_plan,freshness_decision,freshness_plan,lock_heavy,migrate_list,plan,verify_all\nQUEUE_ACTIONS=BACKUP_RESUME,BACKUP_UNVERIFIED,SEALED_PROVEN,SKIP_FRESH\nQUEUE_VAR_NAMES=DUE_SECONDS,FRESHNESS_SECONDS,HEAVY_LOCK,PROBE,QUEUE_LOCK,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,WORKER,age,decision,freshness,generation,now_epoch,out,sealed_epoch\nQUEUE_COMMAND_BASENAMES=flock,homelab-app-cloud-quorum-worker,sha256sum\nQUEUE_INTERESTING_LINE_COUNT=32\nQUEUE_LINE_1_NO=4\nQUEUE_LINE_1_TEXT=WORKER=/usr/local/libexec/homelab-app-cloud-quorum-worker\nQUEUE_LINE_2_NO=5\nQUEUE_LINE_2_TEXT=PROBE=/usr/local/libexec/homelab-app-cloud-quorum-freshness-probe\nQUEUE_LINE_3_NO=6\nQUEUE_LINE_3_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nQUEUE_LINE_4_NO=8\nQUEUE_LINE_4_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nQUEUE_LINE_5_NO=9\nQUEUE_LINE_5_TEXT=QUEUE_LOCK=/run/lock/homelab-app-cloud-quorum-queue.lock\nQUEUE_LINE_6_NO=10\nQUEUE_LINE_6_TEXT=HEAVY_LOCK=/run/homelab-backup-heavy.lock\nQUEUE_LINE_7_NO=21\nQUEUE_LINE_7_TEXT=[[ -x \"$WORKER\" && ! -L \"$WORKER\" ]] || die WORKER_MISSING 69\nQUEUE_LINE_8_NO=29\nQUEUE_LINE_8_TEXT= for a in \"${ALL_APPS[@]}\"; do \"$WORKER\" plan \"$a\"; done\nQUEUE_LINE_9_NO=32\nQUEUE_LINE_9_TEXT= local app=\"$1\" now_epoch=\"$2\" out decision generation sealed_epoch age\nQUEUE_LINE_10_NO=38\nQUEUE_LINE_10_TEXT= generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nQUEUE_LINE_11_NO=39\nQUEUE_LINE_11_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_RESUME GENERATION=%s\\n' \"$app\" \"$generation\"\nQUEUE_LINE_12_NO=42\nQUEUE_LINE_12_TEXT= generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nQUEUE_LINE_13_NO=50\nQUEUE_LINE_13_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\"\nQUEUE_LINE_14_NO=52\nQUEUE_LINE_14_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\"\nQUEUE_LINE_15_NO=54\nQUEUE_LINE_15_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\"\nQUEUE_LINE_16_NO=56\nQUEUE_LINE_16_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\"\nQUEUE_LINE_17_NO=84\nQUEUE_LINE_17_TEXT= out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"; printf '%s\\n' \"$out\"\nQUEUE_LINE_18_NO=85\nQUEUE_LINE_18_TEXT= \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\nQUEUE_LINE_19_NO=93\nQUEUE_LINE_19_TEXT= local a=\"$1\" freshness decision now_epoch out generation\nQUEUE_LINE_20_NO=112\nQUEUE_LINE_20_TEXT= generation=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\"\nQUEUE_LINE_21_NO=113\nQUEUE_LINE_21_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SKIP_FRESH GENERATION=%s\\n' \"$a\" \"$generation\"\nQUEUE_LINE_22_NO=120\nQUEUE_LINE_22_TEXT= out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"\nQUEUE_LINE_23_NO=122\nQUEUE_LINE_23_TEXT= \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\nQUEUE_LINE_24_NO=123\nQUEUE_LINE_24_TEXT= generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nQUEUE_LINE_25_NO=124\nQUEUE_LINE_25_TEXT= [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$a\" 65\nQUEUE_LINE_26_NO=125\nQUEUE_LINE_26_TEXT= printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$a\" \"$generation\"\nQUEUE_LINE_27_NO=126\nQUEUE_LINE_27_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\\n' \"$a\" \"$generation\"\nQUEUE_LINE_28_NO=133\nQUEUE_LINE_28_TEXT= if out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" latest-sealed \"$a\" 2>/dev/null)\"; then\nQUEUE_LINE_29_NO=138\nQUEUE_LINE_29_TEXT= out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"; printf '%s\\n' \"$out\"\nQUEUE_LINE_30_NO=139\nQUEUE_LINE_30_TEXT= \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\nQUEUE_LINE_31_NO=145\nQUEUE_LINE_31_TEXT= for a in \"${ALL_APPS[@]}\"; do out=\"$(\"$WORKER\" latest-sealed \"$a\")\"; printf '%s\\n' \"$out\"; done\nQUEUE_LINE_32_NO=153\nQUEUE_LINE_32_TEXT= all) [[ \"$#\" == 1 ]] || die USAGE_ALL 64; [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die DISPATCHER_MISSING 69; /usr/local/sbin/homelab-backup-dispatcher run APP_TIMER ;;\nQUEUE_SERVICE_SHOW_RC=0\nQUEUE_SERVICE_LOADSTATE=loaded\nQUEUE_SERVICE_ACTIVESTATE=inactive\nQUEUE_SERVICE_UNITFILESTATE=static\nQUEUE_SERVICE_RESULT=success\nQUEUE_SERVICE_EXECMAINSTATUS=0\nQUEUE_SERVICE_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.service\nQUEUE_SERVICE_FRAGMENT_SHA=afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a\nQUEUE_SERVICE_EXECSTART_SHA=22094bb14d8dc5f7399744e183b096583a9c9625527a2f00981f2ec5cd8e1713\nQUEUE_SERVICE_EXEC_BASENAMES=homelab-backup-v2-app-scheduled\nQUEUE_SERVICE_EXEC_ACTIONS=NONE\nQUEUE_TIMER_SHOW_RC=0\nQUEUE_TIMER_LOADSTATE=loaded\nQUEUE_TIMER_ACTIVESTATE=active\nQUEUE_TIMER_UNITFILESTATE=enabled\nQUEUE_TIMER_RESULT=success\nQUEUE_TIMER_EXECMAINSTATUS=\nQUEUE_TIMER_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.timer\nQUEUE_TIMER_FRAGMENT_SHA=af2eedce3df53029b591d31715a3f3e6688ddb997b4823f487e70c339ce88078\nQUEUE_TIMER_EXECSTART_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nQUEUE_TIMER_EXEC_BASENAMES=NONE\nQUEUE_TIMER_EXEC_ACTIONS=NONE\nWORKER_CANDIDATE_COUNT=173\nWORKER_1_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_1_PATH_HINT=//root/BACKUP_closeout_evidence_20260825T165406Z/source/homelab-app-cloud-quorum-worker\nWORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_1_EXECUTABLE=false\nWORKER_1_PRESENT=true\nWORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_1_SHEBANG=#!/usr/bin/env bash\nWORKER_1_BASH_N_RC=0\nWORKER_1_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_1_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor\nWORKER_1_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_1_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_1_INTERESTING_LINE_COUNT=57\nWORKER_1_LINE_1_NO=4\nWORKER_1_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_1_LINE_2_NO=6\nWORKER_1_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_1_LINE_3_NO=7\nWORKER_1_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_1_LINE_4_NO=8\nWORKER_1_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_1_LINE_5_NO=9\nWORKER_1_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_1_LINE_6_NO=10\nWORKER_1_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_1_LINE_7_NO=12\nWORKER_1_LINE_7_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_1_LINE_8_NO=14\nWORKER_1_LINE_8_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_1_LINE_9_NO=75\nWORKER_1_LINE_9_TEXT=logical_id(){\nWORKER_1_LINE_10_NO=82\nWORKER_1_LINE_10_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_1_LINE_11_NO=83\nWORKER_1_LINE_11_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_1_LINE_12_NO=101\nWORKER_1_LINE_12_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_1_LINE_13_NO=103\nWORKER_1_LINE_13_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_1_LINE_14_NO=104\nWORKER_1_LINE_14_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_1_LINE_15_NO=115\nWORKER_1_LINE_15_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_1_LINE_16_NO=117\nWORKER_1_LINE_16_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_1_LINE_17_NO=118\nWORKER_1_LINE_17_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_1_LINE_18_NO=119\nWORKER_1_LINE_18_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_1_LINE_19_NO=126\nWORKER_1_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_1_LINE_20_NO=141\nWORKER_1_LINE_20_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_1_LINE_21_NO=160\nWORKER_1_LINE_21_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_22_NO=191\nWORKER_1_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_23_NO=202\nWORKER_1_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_1_LINE_24_NO=218\nWORKER_1_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_25_NO=239\nWORKER_1_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_26_NO=274\nWORKER_1_LINE_26_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_1_LINE_27_NO=275\nWORKER_1_LINE_27_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_1_LINE_28_NO=293\nWORKER_1_LINE_28_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_29_NO=341\nWORKER_1_LINE_29_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_1_LINE_30_NO=346\nWORKER_1_LINE_30_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_1_LINE_31_NO=349\nWORKER_1_LINE_31_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_1_LINE_32_NO=350\nWORKER_1_LINE_32_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_1_LINE_33_NO=353\nWORKER_1_LINE_33_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_1_LINE_34_NO=354\nWORKER_1_LINE_34_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_1_LINE_35_NO=362\nWORKER_1_LINE_35_TEXT= done <\"$manifest\"\nWORKER_1_LINE_36_NO=431\nWORKER_1_LINE_36_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_1_LINE_37_NO=432\nWORKER_1_LINE_37_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_1_LINE_38_NO=466\nWORKER_1_LINE_38_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_1_LINE_39_NO=471\nWORKER_1_LINE_39_TEXT= if \"manifest.json\" not in names:\nWORKER_1_LINE_40_NO=473\nWORKER_1_LINE_40_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_1_LINE_41_NO=475\nWORKER_1_LINE_41_TEXT= for row in manifest:\nWORKER_1_LINE_42_NO=516\nWORKER_1_LINE_42_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_1_LINE_43_NO=528\nWORKER_1_LINE_43_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_1_LINE_44_NO=530\nWORKER_1_LINE_44_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_1_LINE_45_NO=536\nWORKER_1_LINE_45_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_1_LINE_46_NO=539\nWORKER_1_LINE_46_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_1_LINE_47_NO=540\nWORKER_1_LINE_47_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_1_LINE_48_NO=546\nWORKER_1_LINE_48_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_1_LINE_49_NO=553\nWORKER_1_LINE_49_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_1_LINE_50_NO=560\nWORKER_1_LINE_50_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_1_LINE_51_NO=596\nWORKER_1_LINE_51_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_1_LINE_52_NO=598\nWORKER_1_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_1_LINE_53_NO=644\nWORKER_1_LINE_53_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_1_LINE_54_NO=655\nWORKER_1_LINE_54_TEXT=ACTION=\"${1:-}\"\nWORKER_1_LINE_55_NO=656\nWORKER_1_LINE_55_TEXT=case \"$ACTION\" in\nWORKER_1_LINE_56_NO=658\nWORKER_1_LINE_56_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_1_LINE_57_NO=659\nWORKER_1_LINE_57_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_2_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_2_PATH_HINT=root/backup-registry-sha-pins-20260904T062507Z/usr/local/libexec/homelab-app-cloud-quorum-worker\nWORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_2_EXECUTABLE=true\nWORKER_2_PRESENT=true\nWORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_2_SHEBANG=#!/usr/bin/env bash\nWORKER_2_BASH_N_RC=0\nWORKER_2_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_2_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor\nWORKER_2_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_2_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_2_INTERESTING_LINE_COUNT=57\nWORKER_2_LINE_1_NO=4\nWORKER_2_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_2_LINE_2_NO=6\nWORKER_2_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_2_LINE_3_NO=7\nWORKER_2_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_2_LINE_4_NO=8\nWORKER_2_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_2_LINE_5_NO=9\nWORKER_2_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_2_LINE_6_NO=10\nWORKER_2_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_2_LINE_7_NO=12\nWORKER_2_LINE_7_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_2_LINE_8_NO=14\nWORKER_2_LINE_8_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_2_LINE_9_NO=75\nWORKER_2_LINE_9_TEXT=logical_id(){\nWORKER_2_LINE_10_NO=82\nWORKER_2_LINE_10_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_2_LINE_11_NO=83\nWORKER_2_LINE_11_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_2_LINE_12_NO=101\nWORKER_2_LINE_12_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_2_LINE_13_NO=103\nWORKER_2_LINE_13_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_2_LINE_14_NO=104\nWORKER_2_LINE_14_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_2_LINE_15_NO=115\nWORKER_2_LINE_15_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_2_LINE_16_NO=117\nWORKER_2_LINE_16_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_2_LINE_17_NO=118\nWORKER_2_LINE_17_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_2_LINE_18_NO=119\nWORKER_2_LINE_18_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_2_LINE_19_NO=126\nWORKER_2_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_2_LINE_20_NO=141\nWORKER_2_LINE_20_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_2_LINE_21_NO=160\nWORKER_2_LINE_21_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_22_NO=191\nWORKER_2_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_23_NO=202\nWORKER_2_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_2_LINE_24_NO=218\nWORKER_2_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_25_NO=239\nWORKER_2_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_26_NO=274\nWORKER_2_LINE_26_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_2_LINE_27_NO=275\nWORKER_2_LINE_27_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_2_LINE_28_NO=293\nWORKER_2_LINE_28_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_29_NO=341\nWORKER_2_LINE_29_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_2_LINE_30_NO=346\nWORKER_2_LINE_30_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_2_LINE_31_NO=349\nWORKER_2_LINE_31_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_2_LINE_32_NO=350\nWORKER_2_LINE_32_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_2_LINE_33_NO=353\nWORKER_2_LINE_33_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_2_LINE_34_NO=354\nWORKER_2_LINE_34_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_2_LINE_35_NO=362\nWORKER_2_LINE_35_TEXT= done <\"$manifest\"\nWORKER_2_LINE_36_NO=431\nWORKER_2_LINE_36_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_2_LINE_37_NO=432\nWORKER_2_LINE_37_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_2_LINE_38_NO=466\nWORKER_2_LINE_38_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_2_LINE_39_NO=471\nWORKER_2_LINE_39_TEXT= if \"manifest.json\" not in names:\nWORKER_2_LINE_40_NO=473\nWORKER_2_LINE_40_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_2_LINE_41_NO=475\nWORKER_2_LINE_41_TEXT= for row in manifest:\nWORKER_2_LINE_42_NO=516\nWORKER_2_LINE_42_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_2_LINE_43_NO=528\nWORKER_2_LINE_43_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_2_LINE_44_NO=530\nWORKER_2_LINE_44_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_2_LINE_45_NO=536\nWORKER_2_LINE_45_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_2_LINE_46_NO=539\nWORKER_2_LINE_46_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_2_LINE_47_NO=540\nWORKER_2_LINE_47_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_2_LINE_48_NO=546\nWORKER_2_LINE_48_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_2_LINE_49_NO=553\nWORKER_2_LINE_49_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_2_LINE_50_NO=560\nWORKER_2_LINE_50_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_2_LINE_51_NO=596\nWORKER_2_LINE_51_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_2_LINE_52_NO=598\nWORKER_2_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_2_LINE_53_NO=644\nWORKER_2_LINE_53_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_2_LINE_54_NO=655\nWORKER_2_LINE_54_TEXT=ACTION=\"${1:-}\"\nWORKER_2_LINE_55_NO=656\nWORKER_2_LINE_55_TEXT=case \"$ACTION\" in\nWORKER_2_LINE_56_NO=658\nWORKER_2_LINE_56_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_2_LINE_57_NO=659\nWORKER_2_LINE_57_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_3_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_3_PATH_HINT=root/homelab-ops-cr-2026-0024/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker\nWORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_3_EXECUTABLE=true\nWORKER_3_PRESENT=true\nWORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_3_SHEBANG=#!/usr/bin/env bash\nWORKER_3_BASH_N_RC=0\nWORKER_3_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_3_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_3_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work\nWORKER_3_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd\nWORKER_3_INTERESTING_LINE_COUNT=52\nWORKER_3_LINE_1_NO=4\nWORKER_3_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_3_LINE_2_NO=6\nWORKER_3_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_3_LINE_3_NO=7\nWORKER_3_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_3_LINE_4_NO=8\nWORKER_3_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_3_LINE_5_NO=9\nWORKER_3_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_3_LINE_6_NO=10\nWORKER_3_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_3_LINE_7_NO=45\nWORKER_3_LINE_7_TEXT=logical_id(){\nWORKER_3_LINE_8_NO=50\nWORKER_3_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_3_LINE_9_NO=64\nWORKER_3_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_3_LINE_10_NO=65\nWORKER_3_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_3_LINE_11_NO=83\nWORKER_3_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_3_LINE_12_NO=85\nWORKER_3_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_3_LINE_13_NO=86\nWORKER_3_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_3_LINE_14_NO=97\nWORKER_3_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_3_LINE_15_NO=99\nWORKER_3_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_3_LINE_16_NO=100\nWORKER_3_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_3_LINE_17_NO=101\nWORKER_3_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_3_LINE_18_NO=108\nWORKER_3_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_3_LINE_19_NO=123\nWORKER_3_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_3_LINE_20_NO=142\nWORKER_3_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_21_NO=150\nWORKER_3_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_3_LINE_22_NO=161\nWORKER_3_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_3_LINE_23_NO=192\nWORKER_3_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_24_NO=203\nWORKER_3_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_3_LINE_25_NO=219\nWORKER_3_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_26_NO=240\nWORKER_3_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_27_NO=256\nWORKER_3_LINE_27_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_3_LINE_28_NO=261\nWORKER_3_LINE_28_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_3_LINE_29_NO=264\nWORKER_3_LINE_29_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_3_LINE_30_NO=265\nWORKER_3_LINE_30_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_3_LINE_31_NO=268\nWORKER_3_LINE_31_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_3_LINE_32_NO=269\nWORKER_3_LINE_32_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_3_LINE_33_NO=277\nWORKER_3_LINE_33_TEXT= done <\"$manifest\"\nWORKER_3_LINE_34_NO=344\nWORKER_3_LINE_34_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_3_LINE_35_NO=349\nWORKER_3_LINE_35_TEXT= if \"manifest.json\" not in names:\nWORKER_3_LINE_36_NO=351\nWORKER_3_LINE_36_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_3_LINE_37_NO=353\nWORKER_3_LINE_37_TEXT= for row in manifest:\nWORKER_3_LINE_38_NO=394\nWORKER_3_LINE_38_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_3_LINE_39_NO=406\nWORKER_3_LINE_39_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_3_LINE_40_NO=408\nWORKER_3_LINE_40_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_3_LINE_41_NO=414\nWORKER_3_LINE_41_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_3_LINE_42_NO=417\nWORKER_3_LINE_42_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_3_LINE_43_NO=418\nWORKER_3_LINE_43_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_3_LINE_44_NO=424\nWORKER_3_LINE_44_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_3_LINE_45_NO=427\nWORKER_3_LINE_45_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_3_LINE_46_NO=434\nWORKER_3_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_3_LINE_47_NO=459\nWORKER_3_LINE_47_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_3_LINE_48_NO=461\nWORKER_3_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_3_LINE_49_NO=463\nWORKER_3_LINE_49_TEXT=ACTION=\"${1:-}\"\nWORKER_3_LINE_50_NO=464\nWORKER_3_LINE_50_TEXT=case \"$ACTION\" in\nWORKER_3_LINE_51_NO=466\nWORKER_3_LINE_51_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_3_LINE_52_NO=467\nWORKER_3_LINE_52_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_4_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_4_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker\nWORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_4_EXECUTABLE=true\nWORKER_4_PRESENT=true\nWORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_4_SHEBANG=#!/usr/bin/env bash\nWORKER_4_BASH_N_RC=0\nWORKER_4_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_4_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_4_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work\nWORKER_4_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd\nWORKER_4_INTERESTING_LINE_COUNT=52\nWORKER_4_LINE_1_NO=4\nWORKER_4_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_4_LINE_2_NO=6\nWORKER_4_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_4_LINE_3_NO=7\nWORKER_4_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_4_LINE_4_NO=8\nWORKER_4_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_4_LINE_5_NO=9\nWORKER_4_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_4_LINE_6_NO=10\nWORKER_4_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_4_LINE_7_NO=45\nWORKER_4_LINE_7_TEXT=logical_id(){\nWORKER_4_LINE_8_NO=50\nWORKER_4_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_4_LINE_9_NO=64\nWORKER_4_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_4_LINE_10_NO=65\nWORKER_4_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_4_LINE_11_NO=83\nWORKER_4_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_4_LINE_12_NO=85\nWORKER_4_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_4_LINE_13_NO=86\nWORKER_4_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_4_LINE_14_NO=97\nWORKER_4_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_4_LINE_15_NO=99\nWORKER_4_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_4_LINE_16_NO=100\nWORKER_4_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_4_LINE_17_NO=101\nWORKER_4_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_4_LINE_18_NO=108\nWORKER_4_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_4_LINE_19_NO=123\nWORKER_4_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_4_LINE_20_NO=142\nWORKER_4_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_21_NO=150\nWORKER_4_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_4_LINE_22_NO=161\nWORKER_4_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_4_LINE_23_NO=192\nWORKER_4_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_24_NO=203\nWORKER_4_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_4_LINE_25_NO=219\nWORKER_4_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_26_NO=240\nWORKER_4_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_27_NO=256\nWORKER_4_LINE_27_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_4_LINE_28_NO=261\nWORKER_4_LINE_28_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_4_LINE_29_NO=264\nWORKER_4_LINE_29_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_4_LINE_30_NO=265\nWORKER_4_LINE_30_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_4_LINE_31_NO=268\nWORKER_4_LINE_31_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_4_LINE_32_NO=269\nWORKER_4_LINE_32_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_4_LINE_33_NO=277\nWORKER_4_LINE_33_TEXT= done <\"$manifest\"\nWORKER_4_LINE_34_NO=344\nWORKER_4_LINE_34_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_4_LINE_35_NO=349\nWORKER_4_LINE_35_TEXT= if \"manifest.json\" not in names:\nWORKER_4_LINE_36_NO=351\nWORKER_4_LINE_36_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_4_LINE_37_NO=353\nWORKER_4_LINE_37_TEXT= for row in manifest:\nWORKER_4_LINE_38_NO=394\nWORKER_4_LINE_38_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_4_LINE_39_NO=406\nWORKER_4_LINE_39_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_4_LINE_40_NO=408\nWORKER_4_LINE_40_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_4_LINE_41_NO=414\nWORKER_4_LINE_41_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_4_LINE_42_NO=417\nWORKER_4_LINE_42_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_4_LINE_43_NO=418\nWORKER_4_LINE_43_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_4_LINE_44_NO=424\nWORKER_4_LINE_44_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_4_LINE_45_NO=427\nWORKER_4_LINE_45_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_4_LINE_46_NO=434\nWORKER_4_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_4_LINE_47_NO=459\nWORKER_4_LINE_47_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_4_LINE_48_NO=461\nWORKER_4_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_4_LINE_49_NO=463\nWORKER_4_LINE_49_TEXT=ACTION=\"${1:-}\"\nWORKER_4_LINE_50_NO=464\nWORKER_4_LINE_50_TEXT=case \"$ACTION\" in\nWORKER_4_LINE_51_NO=466\nWORKER_4_LINE_51_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_4_LINE_52_NO=467\nWORKER_4_LINE_52_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_5_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_5_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-scrutiny-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker\nWORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_5_EXECUTABLE=true\nWORKER_5_PRESENT=true\nWORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_5_SHEBANG=#!/usr/bin/env bash\nWORKER_5_BASH_N_RC=0\nWORKER_5_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_5_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_5_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_5_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_5_INTERESTING_LINE_COUNT=58\nWORKER_5_LINE_1_NO=4\nWORKER_5_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_5_LINE_2_NO=6\nWORKER_5_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_5_LINE_3_NO=7\nWORKER_5_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_5_LINE_4_NO=8\nWORKER_5_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_5_LINE_5_NO=9\nWORKER_5_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_5_LINE_6_NO=10\nWORKER_5_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_5_LINE_7_NO=69\nWORKER_5_LINE_7_TEXT=logical_id(){\nWORKER_5_LINE_8_NO=74\nWORKER_5_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_5_LINE_9_NO=90\nWORKER_5_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_5_LINE_10_NO=91\nWORKER_5_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_5_LINE_11_NO=109\nWORKER_5_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_5_LINE_12_NO=111\nWORKER_5_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_5_LINE_13_NO=112\nWORKER_5_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_5_LINE_14_NO=123\nWORKER_5_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_5_LINE_15_NO=125\nWORKER_5_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_5_LINE_16_NO=126\nWORKER_5_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_5_LINE_17_NO=127\nWORKER_5_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_5_LINE_18_NO=134\nWORKER_5_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_5_LINE_19_NO=149\nWORKER_5_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_5_LINE_20_NO=168\nWORKER_5_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_21_NO=176\nWORKER_5_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_5_LINE_22_NO=187\nWORKER_5_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_5_LINE_23_NO=218\nWORKER_5_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_24_NO=229\nWORKER_5_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_5_LINE_25_NO=245\nWORKER_5_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_26_NO=266\nWORKER_5_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_27_NO=301\nWORKER_5_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_5_LINE_28_NO=302\nWORKER_5_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_5_LINE_29_NO=320\nWORKER_5_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_30_NO=368\nWORKER_5_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_5_LINE_31_NO=373\nWORKER_5_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_5_LINE_32_NO=376\nWORKER_5_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_5_LINE_33_NO=377\nWORKER_5_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_5_LINE_34_NO=380\nWORKER_5_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_5_LINE_35_NO=381\nWORKER_5_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_5_LINE_36_NO=389\nWORKER_5_LINE_36_TEXT= done <\"$manifest\"\nWORKER_5_LINE_37_NO=464\nWORKER_5_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_5_LINE_38_NO=465\nWORKER_5_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_5_LINE_39_NO=499\nWORKER_5_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_5_LINE_40_NO=504\nWORKER_5_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_5_LINE_41_NO=506\nWORKER_5_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_5_LINE_42_NO=508\nWORKER_5_LINE_42_TEXT= for row in manifest:\nWORKER_5_LINE_43_NO=549\nWORKER_5_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_5_LINE_44_NO=561\nWORKER_5_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_5_LINE_45_NO=563\nWORKER_5_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_5_LINE_46_NO=569\nWORKER_5_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_5_LINE_47_NO=572\nWORKER_5_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_5_LINE_48_NO=573\nWORKER_5_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_5_LINE_49_NO=579\nWORKER_5_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_5_LINE_50_NO=586\nWORKER_5_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_5_LINE_51_NO=593\nWORKER_5_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_5_LINE_52_NO=630\nWORKER_5_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_5_LINE_53_NO=632\nWORKER_5_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_5_LINE_54_NO=678\nWORKER_5_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_5_LINE_55_NO=689\nWORKER_5_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_5_LINE_56_NO=690\nWORKER_5_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_5_LINE_57_NO=692\nWORKER_5_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_5_LINE_58_NO=693\nWORKER_5_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_6_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_6_PATH_HINT=homelab-ops-cr-2026-0029/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_6_EXECUTABLE=true\nWORKER_6_PRESENT=true\nWORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_6_SHEBANG=#!/usr/bin/env bash\nWORKER_6_BASH_N_RC=0\nWORKER_6_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_6_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_6_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_6_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_6_INTERESTING_LINE_COUNT=58\nWORKER_6_LINE_1_NO=4\nWORKER_6_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_6_LINE_2_NO=6\nWORKER_6_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_6_LINE_3_NO=7\nWORKER_6_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_6_LINE_4_NO=8\nWORKER_6_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_6_LINE_5_NO=9\nWORKER_6_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_6_LINE_6_NO=10\nWORKER_6_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_6_LINE_7_NO=69\nWORKER_6_LINE_7_TEXT=logical_id(){\nWORKER_6_LINE_8_NO=74\nWORKER_6_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_6_LINE_9_NO=90\nWORKER_6_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_6_LINE_10_NO=91\nWORKER_6_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_6_LINE_11_NO=109\nWORKER_6_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_6_LINE_12_NO=111\nWORKER_6_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_6_LINE_13_NO=112\nWORKER_6_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_6_LINE_14_NO=123\nWORKER_6_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_6_LINE_15_NO=125\nWORKER_6_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_6_LINE_16_NO=126\nWORKER_6_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_6_LINE_17_NO=127\nWORKER_6_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_6_LINE_18_NO=134\nWORKER_6_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_6_LINE_19_NO=149\nWORKER_6_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_6_LINE_20_NO=168\nWORKER_6_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_21_NO=176\nWORKER_6_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_6_LINE_22_NO=187\nWORKER_6_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_6_LINE_23_NO=218\nWORKER_6_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_24_NO=229\nWORKER_6_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_6_LINE_25_NO=245\nWORKER_6_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_26_NO=266\nWORKER_6_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_27_NO=301\nWORKER_6_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_6_LINE_28_NO=302\nWORKER_6_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_6_LINE_29_NO=320\nWORKER_6_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_30_NO=368\nWORKER_6_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_6_LINE_31_NO=373\nWORKER_6_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_6_LINE_32_NO=376\nWORKER_6_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_6_LINE_33_NO=377\nWORKER_6_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_6_LINE_34_NO=380\nWORKER_6_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_6_LINE_35_NO=381\nWORKER_6_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_6_LINE_36_NO=389\nWORKER_6_LINE_36_TEXT= done <\"$manifest\"\nWORKER_6_LINE_37_NO=464\nWORKER_6_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_6_LINE_38_NO=465\nWORKER_6_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_6_LINE_39_NO=499\nWORKER_6_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_6_LINE_40_NO=504\nWORKER_6_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_6_LINE_41_NO=506\nWORKER_6_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_6_LINE_42_NO=508\nWORKER_6_LINE_42_TEXT= for row in manifest:\nWORKER_6_LINE_43_NO=549\nWORKER_6_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_6_LINE_44_NO=561\nWORKER_6_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_6_LINE_45_NO=563\nWORKER_6_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_6_LINE_46_NO=569\nWORKER_6_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_6_LINE_47_NO=572\nWORKER_6_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_6_LINE_48_NO=573\nWORKER_6_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_6_LINE_49_NO=579\nWORKER_6_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_6_LINE_50_NO=586\nWORKER_6_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_6_LINE_51_NO=593\nWORKER_6_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_6_LINE_52_NO=630\nWORKER_6_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_6_LINE_53_NO=632\nWORKER_6_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_6_LINE_54_NO=678\nWORKER_6_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_6_LINE_55_NO=689\nWORKER_6_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_6_LINE_56_NO=690\nWORKER_6_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_6_LINE_57_NO=692\nWORKER_6_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_6_LINE_58_NO=693\nWORKER_6_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_7_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_7_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_7_EXECUTABLE=true\nWORKER_7_PRESENT=true\nWORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_7_SHEBANG=#!/usr/bin/env bash\nWORKER_7_BASH_N_RC=0\nWORKER_7_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_7_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_7_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_7_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_7_INTERESTING_LINE_COUNT=58\nWORKER_7_LINE_1_NO=4\nWORKER_7_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_7_LINE_2_NO=6\nWORKER_7_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_7_LINE_3_NO=7\nWORKER_7_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_7_LINE_4_NO=8\nWORKER_7_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_7_LINE_5_NO=9\nWORKER_7_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_7_LINE_6_NO=10\nWORKER_7_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_7_LINE_7_NO=69\nWORKER_7_LINE_7_TEXT=logical_id(){\nWORKER_7_LINE_8_NO=74\nWORKER_7_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_7_LINE_9_NO=90\nWORKER_7_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_7_LINE_10_NO=91\nWORKER_7_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_7_LINE_11_NO=109\nWORKER_7_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_7_LINE_12_NO=111\nWORKER_7_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_7_LINE_13_NO=112\nWORKER_7_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_7_LINE_14_NO=123\nWORKER_7_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_7_LINE_15_NO=125\nWORKER_7_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_7_LINE_16_NO=126\nWORKER_7_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_7_LINE_17_NO=127\nWORKER_7_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_7_LINE_18_NO=134\nWORKER_7_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_7_LINE_19_NO=149\nWORKER_7_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_7_LINE_20_NO=168\nWORKER_7_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_21_NO=176\nWORKER_7_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_7_LINE_22_NO=187\nWORKER_7_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_7_LINE_23_NO=218\nWORKER_7_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_24_NO=229\nWORKER_7_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_7_LINE_25_NO=245\nWORKER_7_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_26_NO=266\nWORKER_7_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_27_NO=301\nWORKER_7_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_7_LINE_28_NO=302\nWORKER_7_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_7_LINE_29_NO=320\nWORKER_7_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_30_NO=368\nWORKER_7_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_7_LINE_31_NO=373\nWORKER_7_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_7_LINE_32_NO=376\nWORKER_7_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_7_LINE_33_NO=377\nWORKER_7_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_7_LINE_34_NO=380\nWORKER_7_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_7_LINE_35_NO=381\nWORKER_7_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_7_LINE_36_NO=389\nWORKER_7_LINE_36_TEXT= done <\"$manifest\"\nWORKER_7_LINE_37_NO=464\nWORKER_7_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_7_LINE_38_NO=465\nWORKER_7_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_7_LINE_39_NO=499\nWORKER_7_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_7_LINE_40_NO=504\nWORKER_7_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_7_LINE_41_NO=506\nWORKER_7_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_7_LINE_42_NO=508\nWORKER_7_LINE_42_TEXT= for row in manifest:\nWORKER_7_LINE_43_NO=549\nWORKER_7_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_7_LINE_44_NO=561\nWORKER_7_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_7_LINE_45_NO=563\nWORKER_7_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_7_LINE_46_NO=569\nWORKER_7_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_7_LINE_47_NO=572\nWORKER_7_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_7_LINE_48_NO=573\nWORKER_7_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_7_LINE_49_NO=579\nWORKER_7_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_7_LINE_50_NO=586\nWORKER_7_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_7_LINE_51_NO=593\nWORKER_7_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_7_LINE_52_NO=630\nWORKER_7_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_7_LINE_53_NO=632\nWORKER_7_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_7_LINE_54_NO=678\nWORKER_7_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_7_LINE_55_NO=689\nWORKER_7_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_7_LINE_56_NO=690\nWORKER_7_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_7_LINE_57_NO=692\nWORKER_7_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_7_LINE_58_NO=693\nWORKER_7_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_8_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_8_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_8_EXECUTABLE=true\nWORKER_8_PRESENT=true\nWORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_8_SHEBANG=#!/usr/bin/env bash\nWORKER_8_BASH_N_RC=0\nWORKER_8_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_8_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_8_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_8_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_8_INTERESTING_LINE_COUNT=58\nWORKER_8_LINE_1_NO=4\nWORKER_8_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_8_LINE_2_NO=6\nWORKER_8_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_8_LINE_3_NO=7\nWORKER_8_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_8_LINE_4_NO=8\nWORKER_8_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_8_LINE_5_NO=9\nWORKER_8_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_8_LINE_6_NO=10\nWORKER_8_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_8_LINE_7_NO=69\nWORKER_8_LINE_7_TEXT=logical_id(){\nWORKER_8_LINE_8_NO=74\nWORKER_8_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_8_LINE_9_NO=90\nWORKER_8_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_8_LINE_10_NO=91\nWORKER_8_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_8_LINE_11_NO=109\nWORKER_8_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_8_LINE_12_NO=111\nWORKER_8_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_8_LINE_13_NO=112\nWORKER_8_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_8_LINE_14_NO=123\nWORKER_8_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_8_LINE_15_NO=125\nWORKER_8_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_8_LINE_16_NO=126\nWORKER_8_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_8_LINE_17_NO=127\nWORKER_8_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_8_LINE_18_NO=134\nWORKER_8_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_8_LINE_19_NO=149\nWORKER_8_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_8_LINE_20_NO=168\nWORKER_8_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_21_NO=176\nWORKER_8_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_8_LINE_22_NO=187\nWORKER_8_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_8_LINE_23_NO=218\nWORKER_8_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_24_NO=229\nWORKER_8_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_8_LINE_25_NO=245\nWORKER_8_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_26_NO=266\nWORKER_8_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_27_NO=301\nWORKER_8_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_8_LINE_28_NO=302\nWORKER_8_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_8_LINE_29_NO=320\nWORKER_8_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_30_NO=368\nWORKER_8_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_8_LINE_31_NO=373\nWORKER_8_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_8_LINE_32_NO=376\nWORKER_8_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_8_LINE_33_NO=377\nWORKER_8_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_8_LINE_34_NO=380\nWORKER_8_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_8_LINE_35_NO=381\nWORKER_8_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_8_LINE_36_NO=389\nWORKER_8_LINE_36_TEXT= done <\"$manifest\"\nWORKER_8_LINE_37_NO=464\nWORKER_8_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_8_LINE_38_NO=465\nWORKER_8_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_8_LINE_39_NO=499\nWORKER_8_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_8_LINE_40_NO=504\nWORKER_8_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_8_LINE_41_NO=506\nWORKER_8_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_8_LINE_42_NO=508\nWORKER_8_LINE_42_TEXT= for row in manifest:\nWORKER_8_LINE_43_NO=549\nWORKER_8_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_8_LINE_44_NO=561\nWORKER_8_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_8_LINE_45_NO=563\nWORKER_8_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_8_LINE_46_NO=569\nWORKER_8_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_8_LINE_47_NO=572\nWORKER_8_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_8_LINE_48_NO=573\nWORKER_8_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_8_LINE_49_NO=579\nWORKER_8_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_8_LINE_50_NO=586\nWORKER_8_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_8_LINE_51_NO=593\nWORKER_8_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_8_LINE_52_NO=630\nWORKER_8_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_8_LINE_53_NO=632\nWORKER_8_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_8_LINE_54_NO=678\nWORKER_8_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_8_LINE_55_NO=689\nWORKER_8_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_8_LINE_56_NO=690\nWORKER_8_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_8_LINE_57_NO=692\nWORKER_8_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_8_LINE_58_NO=693\nWORKER_8_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_9_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_9_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_9_EXECUTABLE=true\nWORKER_9_PRESENT=true\nWORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_9_SHEBANG=#!/usr/bin/env bash\nWORKER_9_BASH_N_RC=0\nWORKER_9_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_9_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_9_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_9_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_9_INTERESTING_LINE_COUNT=58\nWORKER_9_LINE_1_NO=4\nWORKER_9_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_9_LINE_2_NO=6\nWORKER_9_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_9_LINE_3_NO=7\nWORKER_9_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_9_LINE_4_NO=8\nWORKER_9_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_9_LINE_5_NO=9\nWORKER_9_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_9_LINE_6_NO=10\nWORKER_9_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_9_LINE_7_NO=69\nWORKER_9_LINE_7_TEXT=logical_id(){\nWORKER_9_LINE_8_NO=74\nWORKER_9_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_9_LINE_9_NO=90\nWORKER_9_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_9_LINE_10_NO=91\nWORKER_9_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_9_LINE_11_NO=109\nWORKER_9_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_9_LINE_12_NO=111\nWORKER_9_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_9_LINE_13_NO=112\nWORKER_9_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_9_LINE_14_NO=123\nWORKER_9_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_9_LINE_15_NO=125\nWORKER_9_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_9_LINE_16_NO=126\nWORKER_9_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_9_LINE_17_NO=127\nWORKER_9_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_9_LINE_18_NO=134\nWORKER_9_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_9_LINE_19_NO=149\nWORKER_9_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_9_LINE_20_NO=168\nWORKER_9_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_21_NO=176\nWORKER_9_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_9_LINE_22_NO=187\nWORKER_9_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_9_LINE_23_NO=218\nWORKER_9_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_24_NO=229\nWORKER_9_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_9_LINE_25_NO=245\nWORKER_9_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_26_NO=266\nWORKER_9_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_27_NO=301\nWORKER_9_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_9_LINE_28_NO=302\nWORKER_9_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_9_LINE_29_NO=320\nWORKER_9_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_30_NO=368\nWORKER_9_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_9_LINE_31_NO=373\nWORKER_9_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_9_LINE_32_NO=376\nWORKER_9_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_9_LINE_33_NO=377\nWORKER_9_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_9_LINE_34_NO=380\nWORKER_9_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_9_LINE_35_NO=381\nWORKER_9_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_9_LINE_36_NO=389\nWORKER_9_LINE_36_TEXT= done <\"$manifest\"\nWORKER_9_LINE_37_NO=464\nWORKER_9_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_9_LINE_38_NO=465\nWORKER_9_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_9_LINE_39_NO=499\nWORKER_9_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_9_LINE_40_NO=504\nWORKER_9_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_9_LINE_41_NO=506\nWORKER_9_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_9_LINE_42_NO=508\nWORKER_9_LINE_42_TEXT= for row in manifest:\nWORKER_9_LINE_43_NO=549\nWORKER_9_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_9_LINE_44_NO=561\nWORKER_9_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_9_LINE_45_NO=563\nWORKER_9_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_9_LINE_46_NO=569\nWORKER_9_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_9_LINE_47_NO=572\nWORKER_9_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_9_LINE_48_NO=573\nWORKER_9_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_9_LINE_49_NO=579\nWORKER_9_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_9_LINE_50_NO=586\nWORKER_9_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_9_LINE_51_NO=593\nWORKER_9_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_9_LINE_52_NO=630\nWORKER_9_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_9_LINE_53_NO=632\nWORKER_9_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_9_LINE_54_NO=678\nWORKER_9_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_9_LINE_55_NO=689\nWORKER_9_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_9_LINE_56_NO=690\nWORKER_9_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_9_LINE_57_NO=692\nWORKER_9_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_9_LINE_58_NO=693\nWORKER_9_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_10_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_10_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_10_EXECUTABLE=true\nWORKER_10_PRESENT=true\nWORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_10_SHEBANG=#!/usr/bin/env bash\nWORKER_10_BASH_N_RC=0\nWORKER_10_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_10_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_10_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_10_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_10_INTERESTING_LINE_COUNT=58\nWORKER_10_LINE_1_NO=4\nWORKER_10_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_10_LINE_2_NO=6\nWORKER_10_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_10_LINE_3_NO=7\nWORKER_10_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_10_LINE_4_NO=8\nWORKER_10_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_10_LINE_5_NO=9\nWORKER_10_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_10_LINE_6_NO=10\nWORKER_10_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_10_LINE_7_NO=69\nWORKER_10_LINE_7_TEXT=logical_id(){\nWORKER_10_LINE_8_NO=74\nWORKER_10_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_10_LINE_9_NO=90\nWORKER_10_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_10_LINE_10_NO=91\nWORKER_10_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_10_LINE_11_NO=109\nWORKER_10_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_10_LINE_12_NO=111\nWORKER_10_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_10_LINE_13_NO=112\nWORKER_10_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_10_LINE_14_NO=123\nWORKER_10_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_10_LINE_15_NO=125\nWORKER_10_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_10_LINE_16_NO=126\nWORKER_10_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_10_LINE_17_NO=127\nWORKER_10_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_10_LINE_18_NO=134\nWORKER_10_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_10_LINE_19_NO=149\nWORKER_10_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_10_LINE_20_NO=168\nWORKER_10_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_21_NO=176\nWORKER_10_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_10_LINE_22_NO=187\nWORKER_10_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_10_LINE_23_NO=218\nWORKER_10_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_24_NO=229\nWORKER_10_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_10_LINE_25_NO=245\nWORKER_10_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_26_NO=266\nWORKER_10_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_27_NO=301\nWORKER_10_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_10_LINE_28_NO=302\nWORKER_10_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_10_LINE_29_NO=320\nWORKER_10_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_30_NO=368\nWORKER_10_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_10_LINE_31_NO=373\nWORKER_10_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_10_LINE_32_NO=376\nWORKER_10_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_10_LINE_33_NO=377\nWORKER_10_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_10_LINE_34_NO=380\nWORKER_10_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_10_LINE_35_NO=381\nWORKER_10_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_10_LINE_36_NO=389\nWORKER_10_LINE_36_TEXT= done <\"$manifest\"\nWORKER_10_LINE_37_NO=464\nWORKER_10_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_10_LINE_38_NO=465\nWORKER_10_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_10_LINE_39_NO=499\nWORKER_10_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_10_LINE_40_NO=504\nWORKER_10_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_10_LINE_41_NO=506\nWORKER_10_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_10_LINE_42_NO=508\nWORKER_10_LINE_42_TEXT= for row in manifest:\nWORKER_10_LINE_43_NO=549\nWORKER_10_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_10_LINE_44_NO=561\nWORKER_10_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_10_LINE_45_NO=563\nWORKER_10_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_10_LINE_46_NO=569\nWORKER_10_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_10_LINE_47_NO=572\nWORKER_10_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_10_LINE_48_NO=573\nWORKER_10_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_10_LINE_49_NO=579\nWORKER_10_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_10_LINE_50_NO=586\nWORKER_10_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_10_LINE_51_NO=593\nWORKER_10_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_10_LINE_52_NO=630\nWORKER_10_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_10_LINE_53_NO=632\nWORKER_10_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_10_LINE_54_NO=678\nWORKER_10_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_10_LINE_55_NO=689\nWORKER_10_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_10_LINE_56_NO=690\nWORKER_10_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_10_LINE_57_NO=692\nWORKER_10_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_10_LINE_58_NO=693\nWORKER_10_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_11_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_11_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_11_EXECUTABLE=true\nWORKER_11_PRESENT=true\nWORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_11_SHEBANG=#!/usr/bin/env bash\nWORKER_11_BASH_N_RC=0\nWORKER_11_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_11_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_11_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_11_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_11_INTERESTING_LINE_COUNT=58\nWORKER_11_LINE_1_NO=4\nWORKER_11_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_11_LINE_2_NO=6\nWORKER_11_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_11_LINE_3_NO=7\nWORKER_11_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_11_LINE_4_NO=8\nWORKER_11_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_11_LINE_5_NO=9\nWORKER_11_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_11_LINE_6_NO=10\nWORKER_11_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_11_LINE_7_NO=69\nWORKER_11_LINE_7_TEXT=logical_id(){\nWORKER_11_LINE_8_NO=74\nWORKER_11_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_11_LINE_9_NO=90\nWORKER_11_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_11_LINE_10_NO=91\nWORKER_11_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_11_LINE_11_NO=109\nWORKER_11_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_11_LINE_12_NO=111\nWORKER_11_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_11_LINE_13_NO=112\nWORKER_11_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_11_LINE_14_NO=123\nWORKER_11_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_11_LINE_15_NO=125\nWORKER_11_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_11_LINE_16_NO=126\nWORKER_11_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_11_LINE_17_NO=127\nWORKER_11_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_11_LINE_18_NO=134\nWORKER_11_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_11_LINE_19_NO=149\nWORKER_11_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_11_LINE_20_NO=168\nWORKER_11_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_21_NO=176\nWORKER_11_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_11_LINE_22_NO=187\nWORKER_11_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_11_LINE_23_NO=218\nWORKER_11_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_24_NO=229\nWORKER_11_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_11_LINE_25_NO=245\nWORKER_11_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_26_NO=266\nWORKER_11_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_27_NO=301\nWORKER_11_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_11_LINE_28_NO=302\nWORKER_11_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_11_LINE_29_NO=320\nWORKER_11_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_30_NO=368\nWORKER_11_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_11_LINE_31_NO=373\nWORKER_11_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_11_LINE_32_NO=376\nWORKER_11_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_11_LINE_33_NO=377\nWORKER_11_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_11_LINE_34_NO=380\nWORKER_11_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_11_LINE_35_NO=381\nWORKER_11_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_11_LINE_36_NO=389\nWORKER_11_LINE_36_TEXT= done <\"$manifest\"\nWORKER_11_LINE_37_NO=464\nWORKER_11_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_11_LINE_38_NO=465\nWORKER_11_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_11_LINE_39_NO=499\nWORKER_11_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_11_LINE_40_NO=504\nWORKER_11_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_11_LINE_41_NO=506\nWORKER_11_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_11_LINE_42_NO=508\nWORKER_11_LINE_42_TEXT= for row in manifest:\nWORKER_11_LINE_43_NO=549\nWORKER_11_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_11_LINE_44_NO=561\nWORKER_11_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_11_LINE_45_NO=563\nWORKER_11_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_11_LINE_46_NO=569\nWORKER_11_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_11_LINE_47_NO=572\nWORKER_11_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_11_LINE_48_NO=573\nWORKER_11_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_11_LINE_49_NO=579\nWORKER_11_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_11_LINE_50_NO=586\nWORKER_11_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_11_LINE_51_NO=593\nWORKER_11_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_11_LINE_52_NO=630\nWORKER_11_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_11_LINE_53_NO=632\nWORKER_11_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_11_LINE_54_NO=678\nWORKER_11_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_11_LINE_55_NO=689\nWORKER_11_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_11_LINE_56_NO=690\nWORKER_11_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_11_LINE_57_NO=692\nWORKER_11_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_11_LINE_58_NO=693\nWORKER_11_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_12_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_12_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_12_EXECUTABLE=true\nWORKER_12_PRESENT=true\nWORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_12_SHEBANG=#!/usr/bin/env bash\nWORKER_12_BASH_N_RC=0\nWORKER_12_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_12_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_12_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_12_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_12_INTERESTING_LINE_COUNT=58\nWORKER_12_LINE_1_NO=4\nWORKER_12_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_12_LINE_2_NO=6\nWORKER_12_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_12_LINE_3_NO=7\nWORKER_12_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_12_LINE_4_NO=8\nWORKER_12_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_12_LINE_5_NO=9\nWORKER_12_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_12_LINE_6_NO=10\nWORKER_12_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_12_LINE_7_NO=69\nWORKER_12_LINE_7_TEXT=logical_id(){\nWORKER_12_LINE_8_NO=74\nWORKER_12_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_12_LINE_9_NO=90\nWORKER_12_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_12_LINE_10_NO=91\nWORKER_12_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_12_LINE_11_NO=109\nWORKER_12_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_12_LINE_12_NO=111\nWORKER_12_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_12_LINE_13_NO=112\nWORKER_12_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_12_LINE_14_NO=123\nWORKER_12_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_12_LINE_15_NO=125\nWORKER_12_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_12_LINE_16_NO=126\nWORKER_12_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_12_LINE_17_NO=127\nWORKER_12_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_12_LINE_18_NO=134\nWORKER_12_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_12_LINE_19_NO=149\nWORKER_12_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_12_LINE_20_NO=168\nWORKER_12_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_21_NO=176\nWORKER_12_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_12_LINE_22_NO=187\nWORKER_12_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_12_LINE_23_NO=218\nWORKER_12_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_24_NO=229\nWORKER_12_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_12_LINE_25_NO=245\nWORKER_12_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_26_NO=266\nWORKER_12_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_27_NO=301\nWORKER_12_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_12_LINE_28_NO=302\nWORKER_12_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_12_LINE_29_NO=320\nWORKER_12_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_30_NO=368\nWORKER_12_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_12_LINE_31_NO=373\nWORKER_12_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_12_LINE_32_NO=376\nWORKER_12_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_12_LINE_33_NO=377\nWORKER_12_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_12_LINE_34_NO=380\nWORKER_12_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_12_LINE_35_NO=381\nWORKER_12_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_12_LINE_36_NO=389\nWORKER_12_LINE_36_TEXT= done <\"$manifest\"\nWORKER_12_LINE_37_NO=464\nWORKER_12_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_12_LINE_38_NO=465\nWORKER_12_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_12_LINE_39_NO=499\nWORKER_12_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_12_LINE_40_NO=504\nWORKER_12_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_12_LINE_41_NO=506\nWORKER_12_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_12_LINE_42_NO=508\nWORKER_12_LINE_42_TEXT= for row in manifest:\nWORKER_12_LINE_43_NO=549\nWORKER_12_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_12_LINE_44_NO=561\nWORKER_12_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_12_LINE_45_NO=563\nWORKER_12_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_12_LINE_46_NO=569\nWORKER_12_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_12_LINE_47_NO=572\nWORKER_12_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_12_LINE_48_NO=573\nWORKER_12_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_12_LINE_49_NO=579\nWORKER_12_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_12_LINE_50_NO=586\nWORKER_12_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_12_LINE_51_NO=593\nWORKER_12_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_12_LINE_52_NO=630\nWORKER_12_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_12_LINE_53_NO=632\nWORKER_12_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_12_LINE_54_NO=678\nWORKER_12_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_12_LINE_55_NO=689\nWORKER_12_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_12_LINE_56_NO=690\nWORKER_12_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_12_LINE_57_NO=692\nWORKER_12_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_12_LINE_58_NO=693\nWORKER_12_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_13_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_13_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_13_EXECUTABLE=true\nWORKER_13_PRESENT=true\nWORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_13_SHEBANG=#!/usr/bin/env bash\nWORKER_13_BASH_N_RC=0\nWORKER_13_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_13_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_13_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_13_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_13_INTERESTING_LINE_COUNT=58\nWORKER_13_LINE_1_NO=4\nWORKER_13_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_13_LINE_2_NO=6\nWORKER_13_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_13_LINE_3_NO=7\nWORKER_13_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_13_LINE_4_NO=8\nWORKER_13_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_13_LINE_5_NO=9\nWORKER_13_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_13_LINE_6_NO=10\nWORKER_13_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_13_LINE_7_NO=69\nWORKER_13_LINE_7_TEXT=logical_id(){\nWORKER_13_LINE_8_NO=74\nWORKER_13_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_13_LINE_9_NO=90\nWORKER_13_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_13_LINE_10_NO=91\nWORKER_13_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_13_LINE_11_NO=109\nWORKER_13_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_13_LINE_12_NO=111\nWORKER_13_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_13_LINE_13_NO=112\nWORKER_13_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_13_LINE_14_NO=123\nWORKER_13_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_13_LINE_15_NO=125\nWORKER_13_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_13_LINE_16_NO=126\nWORKER_13_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_13_LINE_17_NO=127\nWORKER_13_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_13_LINE_18_NO=134\nWORKER_13_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_13_LINE_19_NO=149\nWORKER_13_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_13_LINE_20_NO=168\nWORKER_13_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_21_NO=176\nWORKER_13_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_13_LINE_22_NO=187\nWORKER_13_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_13_LINE_23_NO=218\nWORKER_13_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_24_NO=229\nWORKER_13_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_13_LINE_25_NO=245\nWORKER_13_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_26_NO=266\nWORKER_13_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_27_NO=301\nWORKER_13_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_13_LINE_28_NO=302\nWORKER_13_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_13_LINE_29_NO=320\nWORKER_13_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_30_NO=368\nWORKER_13_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_13_LINE_31_NO=373\nWORKER_13_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_13_LINE_32_NO=376\nWORKER_13_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_13_LINE_33_NO=377\nWORKER_13_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_13_LINE_34_NO=380\nWORKER_13_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_13_LINE_35_NO=381\nWORKER_13_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_13_LINE_36_NO=389\nWORKER_13_LINE_36_TEXT= done <\"$manifest\"\nWORKER_13_LINE_37_NO=464\nWORKER_13_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_13_LINE_38_NO=465\nWORKER_13_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_13_LINE_39_NO=499\nWORKER_13_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_13_LINE_40_NO=504\nWORKER_13_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_13_LINE_41_NO=506\nWORKER_13_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_13_LINE_42_NO=508\nWORKER_13_LINE_42_TEXT= for row in manifest:\nWORKER_13_LINE_43_NO=549\nWORKER_13_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_13_LINE_44_NO=561\nWORKER_13_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_13_LINE_45_NO=563\nWORKER_13_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_13_LINE_46_NO=569\nWORKER_13_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_13_LINE_47_NO=572\nWORKER_13_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_13_LINE_48_NO=573\nWORKER_13_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_13_LINE_49_NO=579\nWORKER_13_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_13_LINE_50_NO=586\nWORKER_13_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_13_LINE_51_NO=593\nWORKER_13_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_13_LINE_52_NO=630\nWORKER_13_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_13_LINE_53_NO=632\nWORKER_13_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_13_LINE_54_NO=678\nWORKER_13_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_13_LINE_55_NO=689\nWORKER_13_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_13_LINE_56_NO=690\nWORKER_13_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_13_LINE_57_NO=692\nWORKER_13_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_13_LINE_58_NO=693\nWORKER_13_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_14_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_14_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_14_EXECUTABLE=true\nWORKER_14_PRESENT=true\nWORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_14_SHEBANG=#!/usr/bin/env bash\nWORKER_14_BASH_N_RC=0\nWORKER_14_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_14_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_14_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_14_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_14_INTERESTING_LINE_COUNT=58\nWORKER_14_LINE_1_NO=4\nWORKER_14_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_14_LINE_2_NO=6\nWORKER_14_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_14_LINE_3_NO=7\nWORKER_14_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_14_LINE_4_NO=8\nWORKER_14_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_14_LINE_5_NO=9\nWORKER_14_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_14_LINE_6_NO=10\nWORKER_14_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_14_LINE_7_NO=69\nWORKER_14_LINE_7_TEXT=logical_id(){\nWORKER_14_LINE_8_NO=74\nWORKER_14_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_14_LINE_9_NO=90\nWORKER_14_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_14_LINE_10_NO=91\nWORKER_14_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_14_LINE_11_NO=109\nWORKER_14_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_14_LINE_12_NO=111\nWORKER_14_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_14_LINE_13_NO=112\nWORKER_14_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_14_LINE_14_NO=123\nWORKER_14_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_14_LINE_15_NO=125\nWORKER_14_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_14_LINE_16_NO=126\nWORKER_14_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_14_LINE_17_NO=127\nWORKER_14_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_14_LINE_18_NO=134\nWORKER_14_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_14_LINE_19_NO=149\nWORKER_14_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_14_LINE_20_NO=168\nWORKER_14_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_21_NO=176\nWORKER_14_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_14_LINE_22_NO=187\nWORKER_14_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_14_LINE_23_NO=218\nWORKER_14_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_24_NO=229\nWORKER_14_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_14_LINE_25_NO=245\nWORKER_14_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_26_NO=266\nWORKER_14_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_27_NO=301\nWORKER_14_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_14_LINE_28_NO=302\nWORKER_14_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_14_LINE_29_NO=320\nWORKER_14_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_30_NO=368\nWORKER_14_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_14_LINE_31_NO=373\nWORKER_14_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_14_LINE_32_NO=376\nWORKER_14_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_14_LINE_33_NO=377\nWORKER_14_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_14_LINE_34_NO=380\nWORKER_14_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_14_LINE_35_NO=381\nWORKER_14_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_14_LINE_36_NO=389\nWORKER_14_LINE_36_TEXT= done <\"$manifest\"\nWORKER_14_LINE_37_NO=464\nWORKER_14_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_14_LINE_38_NO=465\nWORKER_14_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_14_LINE_39_NO=499\nWORKER_14_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_14_LINE_40_NO=504\nWORKER_14_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_14_LINE_41_NO=506\nWORKER_14_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_14_LINE_42_NO=508\nWORKER_14_LINE_42_TEXT= for row in manifest:\nWORKER_14_LINE_43_NO=549\nWORKER_14_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_14_LINE_44_NO=561\nWORKER_14_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_14_LINE_45_NO=563\nWORKER_14_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_14_LINE_46_NO=569\nWORKER_14_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_14_LINE_47_NO=572\nWORKER_14_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_14_LINE_48_NO=573\nWORKER_14_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_14_LINE_49_NO=579\nWORKER_14_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_14_LINE_50_NO=586\nWORKER_14_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_14_LINE_51_NO=593\nWORKER_14_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_14_LINE_52_NO=630\nWORKER_14_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_14_LINE_53_NO=632\nWORKER_14_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_14_LINE_54_NO=678\nWORKER_14_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_14_LINE_55_NO=689\nWORKER_14_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_14_LINE_56_NO=690\nWORKER_14_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_14_LINE_57_NO=692\nWORKER_14_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_14_LINE_58_NO=693\nWORKER_14_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_15_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_15_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_15_EXECUTABLE=true\nWORKER_15_PRESENT=true\nWORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_15_SHEBANG=#!/usr/bin/env bash\nWORKER_15_BASH_N_RC=0\nWORKER_15_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_15_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_15_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_15_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_15_INTERESTING_LINE_COUNT=58\nWORKER_15_LINE_1_NO=4\nWORKER_15_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_15_LINE_2_NO=6\nWORKER_15_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_15_LINE_3_NO=7\nWORKER_15_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_15_LINE_4_NO=8\nWORKER_15_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_15_LINE_5_NO=9\nWORKER_15_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_15_LINE_6_NO=10\nWORKER_15_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_15_LINE_7_NO=69\nWORKER_15_LINE_7_TEXT=logical_id(){\nWORKER_15_LINE_8_NO=74\nWORKER_15_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_15_LINE_9_NO=90\nWORKER_15_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_15_LINE_10_NO=91\nWORKER_15_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_15_LINE_11_NO=109\nWORKER_15_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_15_LINE_12_NO=111\nWORKER_15_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_15_LINE_13_NO=112\nWORKER_15_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_15_LINE_14_NO=123\nWORKER_15_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_15_LINE_15_NO=125\nWORKER_15_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_15_LINE_16_NO=126\nWORKER_15_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_15_LINE_17_NO=127\nWORKER_15_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_15_LINE_18_NO=134\nWORKER_15_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_15_LINE_19_NO=149\nWORKER_15_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_15_LINE_20_NO=168\nWORKER_15_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_21_NO=176\nWORKER_15_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_15_LINE_22_NO=187\nWORKER_15_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_15_LINE_23_NO=218\nWORKER_15_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_24_NO=229\nWORKER_15_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_15_LINE_25_NO=245\nWORKER_15_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_26_NO=266\nWORKER_15_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_27_NO=301\nWORKER_15_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_15_LINE_28_NO=302\nWORKER_15_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_15_LINE_29_NO=320\nWORKER_15_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_30_NO=368\nWORKER_15_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_15_LINE_31_NO=373\nWORKER_15_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_15_LINE_32_NO=376\nWORKER_15_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_15_LINE_33_NO=377\nWORKER_15_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_15_LINE_34_NO=380\nWORKER_15_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_15_LINE_35_NO=381\nWORKER_15_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_15_LINE_36_NO=389\nWORKER_15_LINE_36_TEXT= done <\"$manifest\"\nWORKER_15_LINE_37_NO=464\nWORKER_15_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_15_LINE_38_NO=465\nWORKER_15_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_15_LINE_39_NO=499\nWORKER_15_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_15_LINE_40_NO=504\nWORKER_15_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_15_LINE_41_NO=506\nWORKER_15_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_15_LINE_42_NO=508\nWORKER_15_LINE_42_TEXT= for row in manifest:\nWORKER_15_LINE_43_NO=549\nWORKER_15_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_15_LINE_44_NO=561\nWORKER_15_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_15_LINE_45_NO=563\nWORKER_15_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_15_LINE_46_NO=569\nWORKER_15_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_15_LINE_47_NO=572\nWORKER_15_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_15_LINE_48_NO=573\nWORKER_15_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_15_LINE_49_NO=579\nWORKER_15_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_15_LINE_50_NO=586\nWORKER_15_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_15_LINE_51_NO=593\nWORKER_15_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_15_LINE_52_NO=630\nWORKER_15_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_15_LINE_53_NO=632\nWORKER_15_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_15_LINE_54_NO=678\nWORKER_15_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_15_LINE_55_NO=689\nWORKER_15_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_15_LINE_56_NO=690\nWORKER_15_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_15_LINE_57_NO=692\nWORKER_15_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_15_LINE_58_NO=693\nWORKER_15_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_16_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_16_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_16_EXECUTABLE=true\nWORKER_16_PRESENT=true\nWORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_16_SHEBANG=#!/usr/bin/env bash\nWORKER_16_BASH_N_RC=0\nWORKER_16_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_16_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_16_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_16_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_16_INTERESTING_LINE_COUNT=58\nWORKER_16_LINE_1_NO=4\nWORKER_16_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_16_LINE_2_NO=6\nWORKER_16_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_16_LINE_3_NO=7\nWORKER_16_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_16_LINE_4_NO=8\nWORKER_16_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_16_LINE_5_NO=9\nWORKER_16_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_16_LINE_6_NO=10\nWORKER_16_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_16_LINE_7_NO=69\nWORKER_16_LINE_7_TEXT=logical_id(){\nWORKER_16_LINE_8_NO=74\nWORKER_16_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_16_LINE_9_NO=90\nWORKER_16_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_16_LINE_10_NO=91\nWORKER_16_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_16_LINE_11_NO=109\nWORKER_16_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_16_LINE_12_NO=111\nWORKER_16_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_16_LINE_13_NO=112\nWORKER_16_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_16_LINE_14_NO=123\nWORKER_16_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_16_LINE_15_NO=125\nWORKER_16_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_16_LINE_16_NO=126\nWORKER_16_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_16_LINE_17_NO=127\nWORKER_16_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_16_LINE_18_NO=134\nWORKER_16_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_16_LINE_19_NO=149\nWORKER_16_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_16_LINE_20_NO=168\nWORKER_16_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_21_NO=176\nWORKER_16_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_16_LINE_22_NO=187\nWORKER_16_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_16_LINE_23_NO=218\nWORKER_16_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_24_NO=229\nWORKER_16_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_16_LINE_25_NO=245\nWORKER_16_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_26_NO=266\nWORKER_16_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_27_NO=301\nWORKER_16_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_16_LINE_28_NO=302\nWORKER_16_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_16_LINE_29_NO=320\nWORKER_16_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_30_NO=368\nWORKER_16_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_16_LINE_31_NO=373\nWORKER_16_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_16_LINE_32_NO=376\nWORKER_16_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_16_LINE_33_NO=377\nWORKER_16_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_16_LINE_34_NO=380\nWORKER_16_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_16_LINE_35_NO=381\nWORKER_16_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_16_LINE_36_NO=389\nWORKER_16_LINE_36_TEXT= done <\"$manifest\"\nWORKER_16_LINE_37_NO=464\nWORKER_16_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_16_LINE_38_NO=465\nWORKER_16_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_16_LINE_39_NO=499\nWORKER_16_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_16_LINE_40_NO=504\nWORKER_16_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_16_LINE_41_NO=506\nWORKER_16_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_16_LINE_42_NO=508\nWORKER_16_LINE_42_TEXT= for row in manifest:\nWORKER_16_LINE_43_NO=549\nWORKER_16_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_16_LINE_44_NO=561\nWORKER_16_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_16_LINE_45_NO=563\nWORKER_16_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_16_LINE_46_NO=569\nWORKER_16_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_16_LINE_47_NO=572\nWORKER_16_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_16_LINE_48_NO=573\nWORKER_16_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_16_LINE_49_NO=579\nWORKER_16_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_16_LINE_50_NO=586\nWORKER_16_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_16_LINE_51_NO=593\nWORKER_16_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_16_LINE_52_NO=630\nWORKER_16_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_16_LINE_53_NO=632\nWORKER_16_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_16_LINE_54_NO=678\nWORKER_16_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_16_LINE_55_NO=689\nWORKER_16_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_16_LINE_56_NO=690\nWORKER_16_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_16_LINE_57_NO=692\nWORKER_16_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_16_LINE_58_NO=693\nWORKER_16_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_17_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_17_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_17_EXECUTABLE=true\nWORKER_17_PRESENT=true\nWORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_17_SHEBANG=#!/usr/bin/env bash\nWORKER_17_BASH_N_RC=0\nWORKER_17_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_17_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_17_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_17_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_17_INTERESTING_LINE_COUNT=58\nWORKER_17_LINE_1_NO=4\nWORKER_17_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_17_LINE_2_NO=6\nWORKER_17_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_17_LINE_3_NO=7\nWORKER_17_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_17_LINE_4_NO=8\nWORKER_17_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_17_LINE_5_NO=9\nWORKER_17_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_17_LINE_6_NO=10\nWORKER_17_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_17_LINE_7_NO=69\nWORKER_17_LINE_7_TEXT=logical_id(){\nWORKER_17_LINE_8_NO=74\nWORKER_17_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_17_LINE_9_NO=90\nWORKER_17_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_17_LINE_10_NO=91\nWORKER_17_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_17_LINE_11_NO=109\nWORKER_17_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_17_LINE_12_NO=111\nWORKER_17_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_17_LINE_13_NO=112\nWORKER_17_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_17_LINE_14_NO=123\nWORKER_17_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_17_LINE_15_NO=125\nWORKER_17_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_17_LINE_16_NO=126\nWORKER_17_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_17_LINE_17_NO=127\nWORKER_17_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_17_LINE_18_NO=134\nWORKER_17_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_17_LINE_19_NO=149\nWORKER_17_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_17_LINE_20_NO=168\nWORKER_17_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_21_NO=176\nWORKER_17_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_17_LINE_22_NO=187\nWORKER_17_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_17_LINE_23_NO=218\nWORKER_17_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_24_NO=229\nWORKER_17_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_17_LINE_25_NO=245\nWORKER_17_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_26_NO=266\nWORKER_17_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_27_NO=301\nWORKER_17_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_17_LINE_28_NO=302\nWORKER_17_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_17_LINE_29_NO=320\nWORKER_17_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_30_NO=368\nWORKER_17_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_17_LINE_31_NO=373\nWORKER_17_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_17_LINE_32_NO=376\nWORKER_17_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_17_LINE_33_NO=377\nWORKER_17_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_17_LINE_34_NO=380\nWORKER_17_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_17_LINE_35_NO=381\nWORKER_17_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_17_LINE_36_NO=389\nWORKER_17_LINE_36_TEXT= done <\"$manifest\"\nWORKER_17_LINE_37_NO=464\nWORKER_17_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_17_LINE_38_NO=465\nWORKER_17_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_17_LINE_39_NO=499\nWORKER_17_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_17_LINE_40_NO=504\nWORKER_17_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_17_LINE_41_NO=506\nWORKER_17_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_17_LINE_42_NO=508\nWORKER_17_LINE_42_TEXT= for row in manifest:\nWORKER_17_LINE_43_NO=549\nWORKER_17_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_17_LINE_44_NO=561\nWORKER_17_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_17_LINE_45_NO=563\nWORKER_17_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_17_LINE_46_NO=569\nWORKER_17_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_17_LINE_47_NO=572\nWORKER_17_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_17_LINE_48_NO=573\nWORKER_17_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_17_LINE_49_NO=579\nWORKER_17_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_17_LINE_50_NO=586\nWORKER_17_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_17_LINE_51_NO=593\nWORKER_17_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_17_LINE_52_NO=630\nWORKER_17_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_17_LINE_53_NO=632\nWORKER_17_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_17_LINE_54_NO=678\nWORKER_17_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_17_LINE_55_NO=689\nWORKER_17_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_17_LINE_56_NO=690\nWORKER_17_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_17_LINE_57_NO=692\nWORKER_17_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_17_LINE_58_NO=693\nWORKER_17_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_18_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_18_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_18_EXECUTABLE=true\nWORKER_18_PRESENT=true\nWORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_18_SHEBANG=#!/usr/bin/env bash\nWORKER_18_BASH_N_RC=0\nWORKER_18_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_18_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_18_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_18_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_18_INTERESTING_LINE_COUNT=58\nWORKER_18_LINE_1_NO=4\nWORKER_18_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_18_LINE_2_NO=6\nWORKER_18_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_18_LINE_3_NO=7\nWORKER_18_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_18_LINE_4_NO=8\nWORKER_18_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_18_LINE_5_NO=9\nWORKER_18_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_18_LINE_6_NO=10\nWORKER_18_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_18_LINE_7_NO=69\nWORKER_18_LINE_7_TEXT=logical_id(){\nWORKER_18_LINE_8_NO=74\nWORKER_18_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_18_LINE_9_NO=90\nWORKER_18_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_18_LINE_10_NO=91\nWORKER_18_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_18_LINE_11_NO=109\nWORKER_18_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_18_LINE_12_NO=111\nWORKER_18_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_18_LINE_13_NO=112\nWORKER_18_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_18_LINE_14_NO=123\nWORKER_18_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_18_LINE_15_NO=125\nWORKER_18_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_18_LINE_16_NO=126\nWORKER_18_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_18_LINE_17_NO=127\nWORKER_18_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_18_LINE_18_NO=134\nWORKER_18_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_18_LINE_19_NO=149\nWORKER_18_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_18_LINE_20_NO=168\nWORKER_18_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_21_NO=176\nWORKER_18_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_18_LINE_22_NO=187\nWORKER_18_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_18_LINE_23_NO=218\nWORKER_18_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_24_NO=229\nWORKER_18_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_18_LINE_25_NO=245\nWORKER_18_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_26_NO=266\nWORKER_18_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_27_NO=301\nWORKER_18_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_18_LINE_28_NO=302\nWORKER_18_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_18_LINE_29_NO=320\nWORKER_18_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_30_NO=368\nWORKER_18_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_18_LINE_31_NO=373\nWORKER_18_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_18_LINE_32_NO=376\nWORKER_18_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_18_LINE_33_NO=377\nWORKER_18_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_18_LINE_34_NO=380\nWORKER_18_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_18_LINE_35_NO=381\nWORKER_18_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_18_LINE_36_NO=389\nWORKER_18_LINE_36_TEXT= done <\"$manifest\"\nWORKER_18_LINE_37_NO=464\nWORKER_18_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_18_LINE_38_NO=465\nWORKER_18_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_18_LINE_39_NO=499\nWORKER_18_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_18_LINE_40_NO=504\nWORKER_18_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_18_LINE_41_NO=506\nWORKER_18_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_18_LINE_42_NO=508\nWORKER_18_LINE_42_TEXT= for row in manifest:\nWORKER_18_LINE_43_NO=549\nWORKER_18_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_18_LINE_44_NO=561\nWORKER_18_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_18_LINE_45_NO=563\nWORKER_18_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_18_LINE_46_NO=569\nWORKER_18_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_18_LINE_47_NO=572\nWORKER_18_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_18_LINE_48_NO=573\nWORKER_18_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_18_LINE_49_NO=579\nWORKER_18_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_18_LINE_50_NO=586\nWORKER_18_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_18_LINE_51_NO=593\nWORKER_18_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_18_LINE_52_NO=630\nWORKER_18_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_18_LINE_53_NO=632\nWORKER_18_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_18_LINE_54_NO=678\nWORKER_18_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_18_LINE_55_NO=689\nWORKER_18_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_18_LINE_56_NO=690\nWORKER_18_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_18_LINE_57_NO=692\nWORKER_18_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_18_LINE_58_NO=693\nWORKER_18_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_19_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_19_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_19_EXECUTABLE=true\nWORKER_19_PRESENT=true\nWORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_19_SHEBANG=#!/usr/bin/env bash\nWORKER_19_BASH_N_RC=0\nWORKER_19_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_19_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_19_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_19_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_19_INTERESTING_LINE_COUNT=58\nWORKER_19_LINE_1_NO=4\nWORKER_19_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_19_LINE_2_NO=6\nWORKER_19_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_19_LINE_3_NO=7\nWORKER_19_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_19_LINE_4_NO=8\nWORKER_19_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_19_LINE_5_NO=9\nWORKER_19_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_19_LINE_6_NO=10\nWORKER_19_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_19_LINE_7_NO=69\nWORKER_19_LINE_7_TEXT=logical_id(){\nWORKER_19_LINE_8_NO=74\nWORKER_19_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_19_LINE_9_NO=90\nWORKER_19_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_19_LINE_10_NO=91\nWORKER_19_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_19_LINE_11_NO=109\nWORKER_19_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_19_LINE_12_NO=111\nWORKER_19_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_19_LINE_13_NO=112\nWORKER_19_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_19_LINE_14_NO=123\nWORKER_19_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_19_LINE_15_NO=125\nWORKER_19_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_19_LINE_16_NO=126\nWORKER_19_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_19_LINE_17_NO=127\nWORKER_19_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_19_LINE_18_NO=134\nWORKER_19_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_19_LINE_19_NO=149\nWORKER_19_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_19_LINE_20_NO=168\nWORKER_19_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_21_NO=176\nWORKER_19_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_19_LINE_22_NO=187\nWORKER_19_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_19_LINE_23_NO=218\nWORKER_19_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_24_NO=229\nWORKER_19_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_19_LINE_25_NO=245\nWORKER_19_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_26_NO=266\nWORKER_19_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_27_NO=301\nWORKER_19_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_19_LINE_28_NO=302\nWORKER_19_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_19_LINE_29_NO=320\nWORKER_19_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_30_NO=368\nWORKER_19_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_19_LINE_31_NO=373\nWORKER_19_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_19_LINE_32_NO=376\nWORKER_19_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_19_LINE_33_NO=377\nWORKER_19_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_19_LINE_34_NO=380\nWORKER_19_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_19_LINE_35_NO=381\nWORKER_19_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_19_LINE_36_NO=389\nWORKER_19_LINE_36_TEXT= done <\"$manifest\"\nWORKER_19_LINE_37_NO=464\nWORKER_19_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_19_LINE_38_NO=465\nWORKER_19_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_19_LINE_39_NO=499\nWORKER_19_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_19_LINE_40_NO=504\nWORKER_19_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_19_LINE_41_NO=506\nWORKER_19_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_19_LINE_42_NO=508\nWORKER_19_LINE_42_TEXT= for row in manifest:\nWORKER_19_LINE_43_NO=549\nWORKER_19_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_19_LINE_44_NO=561\nWORKER_19_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_19_LINE_45_NO=563\nWORKER_19_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_19_LINE_46_NO=569\nWORKER_19_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_19_LINE_47_NO=572\nWORKER_19_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_19_LINE_48_NO=573\nWORKER_19_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_19_LINE_49_NO=579\nWORKER_19_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_19_LINE_50_NO=586\nWORKER_19_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_19_LINE_51_NO=593\nWORKER_19_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_19_LINE_52_NO=630\nWORKER_19_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_19_LINE_53_NO=632\nWORKER_19_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_19_LINE_54_NO=678\nWORKER_19_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_19_LINE_55_NO=689\nWORKER_19_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_19_LINE_56_NO=690\nWORKER_19_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_19_LINE_57_NO=692\nWORKER_19_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_19_LINE_58_NO=693\nWORKER_19_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_20_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_20_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_20_EXECUTABLE=true\nWORKER_20_PRESENT=true\nWORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_20_SHEBANG=#!/usr/bin/env bash\nWORKER_20_BASH_N_RC=0\nWORKER_20_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_20_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_20_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_20_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_20_INTERESTING_LINE_COUNT=58\nWORKER_20_LINE_1_NO=4\nWORKER_20_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_20_LINE_2_NO=6\nWORKER_20_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_20_LINE_3_NO=7\nWORKER_20_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_20_LINE_4_NO=8\nWORKER_20_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_20_LINE_5_NO=9\nWORKER_20_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_20_LINE_6_NO=10\nWORKER_20_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_20_LINE_7_NO=69\nWORKER_20_LINE_7_TEXT=logical_id(){\nWORKER_20_LINE_8_NO=74\nWORKER_20_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_20_LINE_9_NO=90\nWORKER_20_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_20_LINE_10_NO=91\nWORKER_20_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_20_LINE_11_NO=109\nWORKER_20_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_20_LINE_12_NO=111\nWORKER_20_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_20_LINE_13_NO=112\nWORKER_20_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_20_LINE_14_NO=123\nWORKER_20_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_20_LINE_15_NO=125\nWORKER_20_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_20_LINE_16_NO=126\nWORKER_20_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_20_LINE_17_NO=127\nWORKER_20_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_20_LINE_18_NO=134\nWORKER_20_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_20_LINE_19_NO=149\nWORKER_20_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_20_LINE_20_NO=168\nWORKER_20_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_21_NO=176\nWORKER_20_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_20_LINE_22_NO=187\nWORKER_20_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_20_LINE_23_NO=218\nWORKER_20_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_24_NO=229\nWORKER_20_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_20_LINE_25_NO=245\nWORKER_20_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_26_NO=266\nWORKER_20_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_27_NO=301\nWORKER_20_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_20_LINE_28_NO=302\nWORKER_20_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_20_LINE_29_NO=320\nWORKER_20_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_30_NO=368\nWORKER_20_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_20_LINE_31_NO=373\nWORKER_20_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_20_LINE_32_NO=376\nWORKER_20_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_20_LINE_33_NO=377\nWORKER_20_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_20_LINE_34_NO=380\nWORKER_20_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_20_LINE_35_NO=381\nWORKER_20_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_20_LINE_36_NO=389\nWORKER_20_LINE_36_TEXT= done <\"$manifest\"\nWORKER_20_LINE_37_NO=464\nWORKER_20_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_20_LINE_38_NO=465\nWORKER_20_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_20_LINE_39_NO=499\nWORKER_20_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_20_LINE_40_NO=504\nWORKER_20_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_20_LINE_41_NO=506\nWORKER_20_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_20_LINE_42_NO=508\nWORKER_20_LINE_42_TEXT= for row in manifest:\nWORKER_20_LINE_43_NO=549\nWORKER_20_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_20_LINE_44_NO=561\nWORKER_20_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_20_LINE_45_NO=563\nWORKER_20_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_20_LINE_46_NO=569\nWORKER_20_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_20_LINE_47_NO=572\nWORKER_20_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_20_LINE_48_NO=573\nWORKER_20_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_20_LINE_49_NO=579\nWORKER_20_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_20_LINE_50_NO=586\nWORKER_20_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_20_LINE_51_NO=593\nWORKER_20_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_20_LINE_52_NO=630\nWORKER_20_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_20_LINE_53_NO=632\nWORKER_20_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_20_LINE_54_NO=678\nWORKER_20_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_20_LINE_55_NO=689\nWORKER_20_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_20_LINE_56_NO=690\nWORKER_20_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_20_LINE_57_NO=692\nWORKER_20_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_20_LINE_58_NO=693\nWORKER_20_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nTARGETS_PRESENT=true\nTARGETS_SHA=2f526330152d49461ff66f50adab57d1a195ea5d1b4c292945d523411b8a68a8\nTARGET_OBJECT_FOUND=true\nTARGET_KEYS=id,kind,lifecycle,logical_id,migrate,owner,source_host\nTARGET_ID=skladchik-reports-monitor\nTARGET_KIND=APP\nTARGET_LIFECYCLE=ACTIVE\nTARGET_LOGICAL_ID=app-skladchik-reports-monitor\nTARGET_MIGRATE=True\nTARGET_OWNER=pve01\nTARGET_SOURCE_HOST_BASENAME=[PRIVATE_IP]\nTARGET_SOURCE_HOST_SHA=3eb2319e39eac307b3c7bd11f67663c0a0c814fd603989f5965083f9d1c80fb0\nQUEUE_APPID_LITERAL_COUNT=0\nQUEUE_TARGETS_BASENAME_LITERAL_COUNT=0\nBACKUP_AUDIT_RC=0\nBACKUP_AUDIT_RELEVANT_COUNT=2\nBACKUP_AUDIT_RELEVANT_1=TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=22759|generation=app-skladchik-reports-monitor-pve01-20261005T003035Z\nBACKUP_AUDIT_RELEVANT_2=TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=32145|legacy_age_seconds=1627370|replica_count=2|state_id=vm170|generation=v2-vm170-20261004T214054Z-70a537ab4e91\nTARGETED_SEAL_DESIGN_READY=true\nMAIL_SENT_BY_COMMAND=false\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nDECISION=PASS_QUEUE_WORKER_RCA\nNEXT_GATE=DESIGN_ATOMIC_SOURCE_BACKUP_SEAL\nTASK_RESULT=SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA\n" +} diff --git a/runtime/history/SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1.txt b/runtime/history/SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1.txt new file mode 100644 index 00000000..d8f9c957 --- /dev/null +++ b/runtime/history/SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1.txt @@ -0,0 +1,2759 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=skladchik-durability-queue-worker-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=9786047247e48e71139ff52c618c59da24ce317cd7093ee00b699a69b178af52 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=df86e4a999ff180077eddd8c50c0647c83310a5062be61a88ccaab005bb577ea +SANITIZED_OUTPUT_SHA256=3aa1864388e614772490c9c29217a4ed56c483df64b4b67276683b5c5078e47e +OUTPUT_BEGIN +SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA=V1 +NO_MAIL_SEND=true +NO_SERVICE_RESTART=true +NO_TIMER_MUTATION=true +NO_NETWORK_MUTATION=true +NO_SOURCE_MUTATION=true +NO_BACKUP_MUTATION=true +NO_GIT_PUSH=true +MODE=read-only +HOST_PVE01=true +QUEUE_VM170_QM_RC=0 +QUEUE_VM170_STDERR_PRESENT=false +QUEUE_VM170_JSON=true +QUEUE_VM170_GUEST_EXITCODE=0 +VM170_MONITOR_SHA=19123e359f2f7b9a2d373fd7e4464dd426d31b91cabc567c252e70cad40d77c8 +VM170_TIMER_ACTIVE=active +VM170_TIMER_ENABLED=enabled +VM170_SERVICE_RESULT=success +VM170_HEALTH_STATUS=OK +VM170_HEALTH_TS=2026-10-05T06:49:35Z +VM170_HEALTH_REPORTS=22 +VM170_HEALTH_OPEN=13 +VM170_HEALTH_RESOLVED=9 +VM170_BASELINE_OK=true +SUPERVISOR_SHA=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2 +SUPERVISOR_TOP_CONST_1_NAME=BACKUP_HEALTH_AGE_MAX +SUPERVISOR_TOP_CONST_1_EXPR_SHA=056f62a1cb138698ac25abded3e6493c5519d37a485bb3900a74bad1806f70b3 +SUPERVISOR_TOP_CONST_2_NAME=RESTORE_HEALTH_AGE_MAX +SUPERVISOR_TOP_CONST_2_EXPR_SHA=ce6a9ab58cc2e0db3f07ff3ef7d288b29b6611afff056f5b818c706075d9dbd5 +SUPERVISOR_TOP_CONST_3_NAME=APP_FRESHNESS_PROBE +SUPERVISOR_TOP_CONST_3_EXPR_SHA=d6593f7b36fcf7550285e4af7a5f5766c13c2ee0e0a992b808caaed52e54693a +SUPERVISOR_TOP_CONST_4_NAME=LEGACY_BACKUP_PROBLEMS +SUPERVISOR_TOP_CONST_4_EXPR_SHA=d84baf639fd4e63d9c728a793f4dc6c457b6bee5737789745170d2150d858ddd +SUPERVISOR_TOP_CONST_COUNT=4 +SUPERVISOR_TEST_RC=0 +SUPERVISOR_TEST_STDOUT_SHA=9b4438f36f4a40de86d41fd480c6900c28603079d96011398674df7637868c5e +SUPERVISOR_TEST_OUTPUT_BEGIN +SUPERVISOR_STATUS=OK +SUPERVISOR_PROBLEMS=none +MONITOR=OK reports=22 open=13 resolved=9 +CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z +REGISTRY_SOURCE= +> +NETBIRD_ROUTE= dev wt0 table 7120 src uid 0 +SUPERVISOR_TEST_OUTPUT_END +SUPTEST_SUPERVISOR_STATUS=OK +SUPTEST_SUPERVISOR_PROBLEMS=none +SUPTEST_MONITOR=OK reports=22 open=13 resolved=9 +SUPTEST_CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z +SUPTEST_REGISTRY_SOURCE=[PRIVATE_IP] +SUPTEST_WORKER_TARGET=debian@[PRIVATE_IP] +SUPTEST_NETBIRD_ROUTE=1.1.1.1 dev wt0 table 7120 src 100.125.187.138 uid 0 +QUEUE_PRESENT=true +QUEUE_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650 +QUEUE_SHEBANG=#!/usr/bin/env bash +QUEUE_BASH_N_RC=0 +QUEUE_FUNCTIONS=backup_list,die,dispatch_one,dispatch_plan,freshness_decision,freshness_plan,lock_heavy,migrate_list,plan,verify_all +QUEUE_ACTIONS=BACKUP_RESUME,BACKUP_UNVERIFIED,SEALED_PROVEN,SKIP_FRESH +QUEUE_VAR_NAMES=DUE_SECONDS,FRESHNESS_SECONDS,HEAVY_LOCK,PROBE,QUEUE_LOCK,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,WORKER,age,decision,freshness,generation,now_epoch,out,sealed_epoch +QUEUE_COMMAND_BASENAMES=flock,homelab-app-cloud-quorum-worker,sha256sum +QUEUE_INTERESTING_LINE_COUNT=32 +QUEUE_LINE_1_NO=4 +QUEUE_LINE_1_TEXT=WORKER=/usr/local/libexec/homelab-app-cloud-quorum-worker +QUEUE_LINE_2_NO=5 +QUEUE_LINE_2_TEXT=PROBE=/usr/local/libexec/homelab-app-cloud-quorum-freshness-probe +QUEUE_LINE_3_NO=6 +QUEUE_LINE_3_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +QUEUE_LINE_4_NO=8 +QUEUE_LINE_4_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +QUEUE_LINE_5_NO=9 +QUEUE_LINE_5_TEXT=QUEUE_LOCK=/run/lock/homelab-app-cloud-quorum-queue.lock +QUEUE_LINE_6_NO=10 +QUEUE_LINE_6_TEXT=HEAVY_LOCK=/run/homelab-backup-heavy.lock +QUEUE_LINE_7_NO=21 +QUEUE_LINE_7_TEXT=[[ -x "$WORKER" && ! -L "$WORKER" ]] || die WORKER_MISSING 69 +QUEUE_LINE_8_NO=29 +QUEUE_LINE_8_TEXT= for a in "${ALL_APPS[@]}"; do "$WORKER" plan "$a"; done +QUEUE_LINE_9_NO=32 +QUEUE_LINE_9_TEXT= local app="$1" now_epoch="$2" out decision generation sealed_epoch age +QUEUE_LINE_10_NO=38 +QUEUE_LINE_10_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +QUEUE_LINE_11_NO=39 +QUEUE_LINE_11_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_RESUME GENERATION=%s\n' "$app" "$generation" +QUEUE_LINE_12_NO=42 +QUEUE_LINE_12_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +QUEUE_LINE_13_NO=50 +QUEUE_LINE_13_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS" +QUEUE_LINE_14_NO=52 +QUEUE_LINE_14_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS" +QUEUE_LINE_15_NO=54 +QUEUE_LINE_15_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS" +QUEUE_LINE_16_NO=56 +QUEUE_LINE_16_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\n' "$app" "$generation" "$age" +QUEUE_LINE_17_NO=84 +QUEUE_LINE_17_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"; printf '%s\n' "$out" +QUEUE_LINE_18_NO=85 +QUEUE_LINE_18_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null +QUEUE_LINE_19_NO=93 +QUEUE_LINE_19_TEXT= local a="$1" freshness decision now_epoch out generation +QUEUE_LINE_20_NO=112 +QUEUE_LINE_20_TEXT= generation="$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$freshness" | tail -1)" +QUEUE_LINE_21_NO=113 +QUEUE_LINE_21_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SKIP_FRESH GENERATION=%s\n' "$a" "$generation" +QUEUE_LINE_22_NO=120 +QUEUE_LINE_22_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")" +QUEUE_LINE_23_NO=122 +QUEUE_LINE_23_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null +QUEUE_LINE_24_NO=123 +QUEUE_LINE_24_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +QUEUE_LINE_25_NO=124 +QUEUE_LINE_25_TEXT= [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$a" 65 +QUEUE_LINE_26_NO=125 +QUEUE_LINE_26_TEXT= printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$a" "$generation" +QUEUE_LINE_27_NO=126 +QUEUE_LINE_27_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\n' "$a" "$generation" +QUEUE_LINE_28_NO=133 +QUEUE_LINE_28_TEXT= if out="$(exec 8>&-; exec 9>&-; "$WORKER" latest-sealed "$a" 2>/dev/null)"; then +QUEUE_LINE_29_NO=138 +QUEUE_LINE_29_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"; printf '%s\n' "$out" +QUEUE_LINE_30_NO=139 +QUEUE_LINE_30_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null +QUEUE_LINE_31_NO=145 +QUEUE_LINE_31_TEXT= for a in "${ALL_APPS[@]}"; do out="$("$WORKER" latest-sealed "$a")"; printf '%s\n' "$out"; done +QUEUE_LINE_32_NO=153 +QUEUE_LINE_32_TEXT= all) [[ "$#" == 1 ]] || die USAGE_ALL 64; [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die DISPATCHER_MISSING 69; /usr/local/sbin/homelab-backup-dispatcher run APP_TIMER ;; +QUEUE_SERVICE_SHOW_RC=0 +QUEUE_SERVICE_LOADSTATE=loaded +QUEUE_SERVICE_ACTIVESTATE=inactive +QUEUE_SERVICE_UNITFILESTATE=static +QUEUE_SERVICE_RESULT=success +QUEUE_SERVICE_EXECMAINSTATUS=0 +QUEUE_SERVICE_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.service +QUEUE_SERVICE_FRAGMENT_SHA=afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a +QUEUE_SERVICE_EXECSTART_SHA=22094bb14d8dc5f7399744e183b096583a9c9625527a2f00981f2ec5cd8e1713 +QUEUE_SERVICE_EXEC_BASENAMES=homelab-backup-v2-app-scheduled +QUEUE_SERVICE_EXEC_ACTIONS=NONE +QUEUE_TIMER_SHOW_RC=0 +QUEUE_TIMER_LOADSTATE=loaded +QUEUE_TIMER_ACTIVESTATE=active +QUEUE_TIMER_UNITFILESTATE=enabled +QUEUE_TIMER_RESULT=success +QUEUE_TIMER_EXECMAINSTATUS= +QUEUE_TIMER_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.timer +QUEUE_TIMER_FRAGMENT_SHA=af2eedce3df53029b591d31715a3f3e6688ddb997b4823f487e70c339ce88078 +QUEUE_TIMER_EXECSTART_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +QUEUE_TIMER_EXEC_BASENAMES=NONE +QUEUE_TIMER_EXEC_ACTIONS=NONE +WORKER_CANDIDATE_COUNT=173 +WORKER_1_BASENAME=homelab-app-cloud-quorum-worker +WORKER_1_PATH_HINT=//root/BACKUP_closeout_evidence_20260825T165406Z/source/homelab-app-cloud-quorum-worker +WORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 +WORKER_1_EXECUTABLE=false +WORKER_1_PRESENT=true +WORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 +WORKER_1_SHEBANG=#!/usr/bin/env bash +WORKER_1_BASH_N_RC=0 +WORKER_1_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_1_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor +WORKER_1_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_1_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_1_INTERESTING_LINE_COUNT=57 +WORKER_1_LINE_1_NO=4 +WORKER_1_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_1_LINE_2_NO=6 +WORKER_1_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_1_LINE_3_NO=7 +WORKER_1_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_1_LINE_4_NO=8 +WORKER_1_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_1_LINE_5_NO=9 +WORKER_1_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_1_LINE_6_NO=10 +WORKER_1_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_1_LINE_7_NO=12 +WORKER_1_LINE_7_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +WORKER_1_LINE_8_NO=14 +WORKER_1_LINE_8_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +WORKER_1_LINE_9_NO=75 +WORKER_1_LINE_9_TEXT=logical_id(){ +WORKER_1_LINE_10_NO=82 +WORKER_1_LINE_10_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_1_LINE_11_NO=83 +WORKER_1_LINE_11_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_1_LINE_12_NO=101 +WORKER_1_LINE_12_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_1_LINE_13_NO=103 +WORKER_1_LINE_13_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_1_LINE_14_NO=104 +WORKER_1_LINE_14_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_1_LINE_15_NO=115 +WORKER_1_LINE_15_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_1_LINE_16_NO=117 +WORKER_1_LINE_16_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_1_LINE_17_NO=118 +WORKER_1_LINE_17_TEXT= if [[ -s "$manifest" ]]; then +WORKER_1_LINE_18_NO=119 +WORKER_1_LINE_18_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_1_LINE_19_NO=126 +WORKER_1_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_1_LINE_20_NO=141 +WORKER_1_LINE_20_TEXT= logical="$(logical_id "$app")" +WORKER_1_LINE_21_NO=160 +WORKER_1_LINE_21_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_1_LINE_22_NO=191 +WORKER_1_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_1_LINE_23_NO=202 +WORKER_1_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_1_LINE_24_NO=218 +WORKER_1_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_1_LINE_25_NO=239 +WORKER_1_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_1_LINE_26_NO=274 +WORKER_1_LINE_26_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_1_LINE_27_NO=275 +WORKER_1_LINE_27_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_1_LINE_28_NO=293 +WORKER_1_LINE_28_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_1_LINE_29_NO=341 +WORKER_1_LINE_29_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_1_LINE_30_NO=346 +WORKER_1_LINE_30_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_1_LINE_31_NO=349 +WORKER_1_LINE_31_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_1_LINE_32_NO=350 +WORKER_1_LINE_32_TEXT= logical="$(logical_id "$app")" +WORKER_1_LINE_33_NO=353 +WORKER_1_LINE_33_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_1_LINE_34_NO=354 +WORKER_1_LINE_34_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_1_LINE_35_NO=362 +WORKER_1_LINE_35_TEXT= done <"$manifest" +WORKER_1_LINE_36_NO=431 +WORKER_1_LINE_36_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_1_LINE_37_NO=432 +WORKER_1_LINE_37_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_1_LINE_38_NO=466 +WORKER_1_LINE_38_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_1_LINE_39_NO=471 +WORKER_1_LINE_39_TEXT= if "manifest.json" not in names: +WORKER_1_LINE_40_NO=473 +WORKER_1_LINE_40_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_1_LINE_41_NO=475 +WORKER_1_LINE_41_TEXT= for row in manifest: +WORKER_1_LINE_42_NO=516 +WORKER_1_LINE_42_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_1_LINE_43_NO=528 +WORKER_1_LINE_43_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_1_LINE_44_NO=530 +WORKER_1_LINE_44_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_1_LINE_45_NO=536 +WORKER_1_LINE_45_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_1_LINE_46_NO=539 +WORKER_1_LINE_46_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_1_LINE_47_NO=540 +WORKER_1_LINE_47_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_1_LINE_48_NO=546 +WORKER_1_LINE_48_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_1_LINE_49_NO=553 +WORKER_1_LINE_49_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_1_LINE_50_NO=560 +WORKER_1_LINE_50_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_1_LINE_51_NO=596 +WORKER_1_LINE_51_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_1_LINE_52_NO=598 +WORKER_1_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_1_LINE_53_NO=644 +WORKER_1_LINE_53_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_1_LINE_54_NO=655 +WORKER_1_LINE_54_TEXT=ACTION="${1:-}" +WORKER_1_LINE_55_NO=656 +WORKER_1_LINE_55_TEXT=case "$ACTION" in +WORKER_1_LINE_56_NO=658 +WORKER_1_LINE_56_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_1_LINE_57_NO=659 +WORKER_1_LINE_57_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_2_BASENAME=homelab-app-cloud-quorum-worker +WORKER_2_PATH_HINT=root/backup-registry-sha-pins-20260904T062507Z/usr/local/libexec/homelab-app-cloud-quorum-worker +WORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 +WORKER_2_EXECUTABLE=true +WORKER_2_PRESENT=true +WORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 +WORKER_2_SHEBANG=#!/usr/bin/env bash +WORKER_2_BASH_N_RC=0 +WORKER_2_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_2_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor +WORKER_2_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_2_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_2_INTERESTING_LINE_COUNT=57 +WORKER_2_LINE_1_NO=4 +WORKER_2_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_2_LINE_2_NO=6 +WORKER_2_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_2_LINE_3_NO=7 +WORKER_2_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_2_LINE_4_NO=8 +WORKER_2_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_2_LINE_5_NO=9 +WORKER_2_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_2_LINE_6_NO=10 +WORKER_2_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_2_LINE_7_NO=12 +WORKER_2_LINE_7_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +WORKER_2_LINE_8_NO=14 +WORKER_2_LINE_8_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +WORKER_2_LINE_9_NO=75 +WORKER_2_LINE_9_TEXT=logical_id(){ +WORKER_2_LINE_10_NO=82 +WORKER_2_LINE_10_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_2_LINE_11_NO=83 +WORKER_2_LINE_11_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_2_LINE_12_NO=101 +WORKER_2_LINE_12_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_2_LINE_13_NO=103 +WORKER_2_LINE_13_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_2_LINE_14_NO=104 +WORKER_2_LINE_14_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_2_LINE_15_NO=115 +WORKER_2_LINE_15_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_2_LINE_16_NO=117 +WORKER_2_LINE_16_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_2_LINE_17_NO=118 +WORKER_2_LINE_17_TEXT= if [[ -s "$manifest" ]]; then +WORKER_2_LINE_18_NO=119 +WORKER_2_LINE_18_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_2_LINE_19_NO=126 +WORKER_2_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_2_LINE_20_NO=141 +WORKER_2_LINE_20_TEXT= logical="$(logical_id "$app")" +WORKER_2_LINE_21_NO=160 +WORKER_2_LINE_21_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_2_LINE_22_NO=191 +WORKER_2_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_2_LINE_23_NO=202 +WORKER_2_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_2_LINE_24_NO=218 +WORKER_2_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_2_LINE_25_NO=239 +WORKER_2_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_2_LINE_26_NO=274 +WORKER_2_LINE_26_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_2_LINE_27_NO=275 +WORKER_2_LINE_27_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_2_LINE_28_NO=293 +WORKER_2_LINE_28_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_2_LINE_29_NO=341 +WORKER_2_LINE_29_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_2_LINE_30_NO=346 +WORKER_2_LINE_30_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_2_LINE_31_NO=349 +WORKER_2_LINE_31_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_2_LINE_32_NO=350 +WORKER_2_LINE_32_TEXT= logical="$(logical_id "$app")" +WORKER_2_LINE_33_NO=353 +WORKER_2_LINE_33_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_2_LINE_34_NO=354 +WORKER_2_LINE_34_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_2_LINE_35_NO=362 +WORKER_2_LINE_35_TEXT= done <"$manifest" +WORKER_2_LINE_36_NO=431 +WORKER_2_LINE_36_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_2_LINE_37_NO=432 +WORKER_2_LINE_37_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_2_LINE_38_NO=466 +WORKER_2_LINE_38_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_2_LINE_39_NO=471 +WORKER_2_LINE_39_TEXT= if "manifest.json" not in names: +WORKER_2_LINE_40_NO=473 +WORKER_2_LINE_40_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_2_LINE_41_NO=475 +WORKER_2_LINE_41_TEXT= for row in manifest: +WORKER_2_LINE_42_NO=516 +WORKER_2_LINE_42_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_2_LINE_43_NO=528 +WORKER_2_LINE_43_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_2_LINE_44_NO=530 +WORKER_2_LINE_44_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_2_LINE_45_NO=536 +WORKER_2_LINE_45_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_2_LINE_46_NO=539 +WORKER_2_LINE_46_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_2_LINE_47_NO=540 +WORKER_2_LINE_47_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_2_LINE_48_NO=546 +WORKER_2_LINE_48_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_2_LINE_49_NO=553 +WORKER_2_LINE_49_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_2_LINE_50_NO=560 +WORKER_2_LINE_50_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_2_LINE_51_NO=596 +WORKER_2_LINE_51_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_2_LINE_52_NO=598 +WORKER_2_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_2_LINE_53_NO=644 +WORKER_2_LINE_53_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_2_LINE_54_NO=655 +WORKER_2_LINE_54_TEXT=ACTION="${1:-}" +WORKER_2_LINE_55_NO=656 +WORKER_2_LINE_55_TEXT=case "$ACTION" in +WORKER_2_LINE_56_NO=658 +WORKER_2_LINE_56_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_2_LINE_57_NO=659 +WORKER_2_LINE_57_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_3_BASENAME=homelab-app-cloud-quorum-worker +WORKER_3_PATH_HINT=root/homelab-ops-cr-2026-0024/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker +WORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 +WORKER_3_EXECUTABLE=true +WORKER_3_PRESENT=true +WORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 +WORKER_3_SHEBANG=#!/usr/bin/env bash +WORKER_3_BASH_N_RC=0 +WORKER_3_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_3_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_3_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work +WORKER_3_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd +WORKER_3_INTERESTING_LINE_COUNT=52 +WORKER_3_LINE_1_NO=4 +WORKER_3_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_3_LINE_2_NO=6 +WORKER_3_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_3_LINE_3_NO=7 +WORKER_3_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_3_LINE_4_NO=8 +WORKER_3_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_3_LINE_5_NO=9 +WORKER_3_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_3_LINE_6_NO=10 +WORKER_3_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_3_LINE_7_NO=45 +WORKER_3_LINE_7_TEXT=logical_id(){ +WORKER_3_LINE_8_NO=50 +WORKER_3_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_3_LINE_9_NO=64 +WORKER_3_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_3_LINE_10_NO=65 +WORKER_3_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_3_LINE_11_NO=83 +WORKER_3_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_3_LINE_12_NO=85 +WORKER_3_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_3_LINE_13_NO=86 +WORKER_3_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_3_LINE_14_NO=97 +WORKER_3_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_3_LINE_15_NO=99 +WORKER_3_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_3_LINE_16_NO=100 +WORKER_3_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_3_LINE_17_NO=101 +WORKER_3_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_3_LINE_18_NO=108 +WORKER_3_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_3_LINE_19_NO=123 +WORKER_3_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_3_LINE_20_NO=142 +WORKER_3_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_3_LINE_21_NO=150 +WORKER_3_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_3_LINE_22_NO=161 +WORKER_3_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_3_LINE_23_NO=192 +WORKER_3_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_3_LINE_24_NO=203 +WORKER_3_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_3_LINE_25_NO=219 +WORKER_3_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_3_LINE_26_NO=240 +WORKER_3_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_3_LINE_27_NO=256 +WORKER_3_LINE_27_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_3_LINE_28_NO=261 +WORKER_3_LINE_28_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_3_LINE_29_NO=264 +WORKER_3_LINE_29_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_3_LINE_30_NO=265 +WORKER_3_LINE_30_TEXT= logical="$(logical_id "$app")" +WORKER_3_LINE_31_NO=268 +WORKER_3_LINE_31_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_3_LINE_32_NO=269 +WORKER_3_LINE_32_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_3_LINE_33_NO=277 +WORKER_3_LINE_33_TEXT= done <"$manifest" +WORKER_3_LINE_34_NO=344 +WORKER_3_LINE_34_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_3_LINE_35_NO=349 +WORKER_3_LINE_35_TEXT= if "manifest.json" not in names: +WORKER_3_LINE_36_NO=351 +WORKER_3_LINE_36_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_3_LINE_37_NO=353 +WORKER_3_LINE_37_TEXT= for row in manifest: +WORKER_3_LINE_38_NO=394 +WORKER_3_LINE_38_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_3_LINE_39_NO=406 +WORKER_3_LINE_39_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_3_LINE_40_NO=408 +WORKER_3_LINE_40_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_3_LINE_41_NO=414 +WORKER_3_LINE_41_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_3_LINE_42_NO=417 +WORKER_3_LINE_42_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_3_LINE_43_NO=418 +WORKER_3_LINE_43_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_3_LINE_44_NO=424 +WORKER_3_LINE_44_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_3_LINE_45_NO=427 +WORKER_3_LINE_45_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_3_LINE_46_NO=434 +WORKER_3_LINE_46_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_3_LINE_47_NO=459 +WORKER_3_LINE_47_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_3_LINE_48_NO=461 +WORKER_3_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_3_LINE_49_NO=463 +WORKER_3_LINE_49_TEXT=ACTION="${1:-}" +WORKER_3_LINE_50_NO=464 +WORKER_3_LINE_50_TEXT=case "$ACTION" in +WORKER_3_LINE_51_NO=466 +WORKER_3_LINE_51_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_3_LINE_52_NO=467 +WORKER_3_LINE_52_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_4_BASENAME=homelab-app-cloud-quorum-worker +WORKER_4_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker +WORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 +WORKER_4_EXECUTABLE=true +WORKER_4_PRESENT=true +WORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 +WORKER_4_SHEBANG=#!/usr/bin/env bash +WORKER_4_BASH_N_RC=0 +WORKER_4_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_4_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_4_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work +WORKER_4_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd +WORKER_4_INTERESTING_LINE_COUNT=52 +WORKER_4_LINE_1_NO=4 +WORKER_4_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_4_LINE_2_NO=6 +WORKER_4_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_4_LINE_3_NO=7 +WORKER_4_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_4_LINE_4_NO=8 +WORKER_4_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_4_LINE_5_NO=9 +WORKER_4_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_4_LINE_6_NO=10 +WORKER_4_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_4_LINE_7_NO=45 +WORKER_4_LINE_7_TEXT=logical_id(){ +WORKER_4_LINE_8_NO=50 +WORKER_4_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_4_LINE_9_NO=64 +WORKER_4_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_4_LINE_10_NO=65 +WORKER_4_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_4_LINE_11_NO=83 +WORKER_4_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_4_LINE_12_NO=85 +WORKER_4_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_4_LINE_13_NO=86 +WORKER_4_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_4_LINE_14_NO=97 +WORKER_4_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_4_LINE_15_NO=99 +WORKER_4_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_4_LINE_16_NO=100 +WORKER_4_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_4_LINE_17_NO=101 +WORKER_4_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_4_LINE_18_NO=108 +WORKER_4_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_4_LINE_19_NO=123 +WORKER_4_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_4_LINE_20_NO=142 +WORKER_4_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_4_LINE_21_NO=150 +WORKER_4_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_4_LINE_22_NO=161 +WORKER_4_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_4_LINE_23_NO=192 +WORKER_4_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_4_LINE_24_NO=203 +WORKER_4_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_4_LINE_25_NO=219 +WORKER_4_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_4_LINE_26_NO=240 +WORKER_4_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_4_LINE_27_NO=256 +WORKER_4_LINE_27_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_4_LINE_28_NO=261 +WORKER_4_LINE_28_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_4_LINE_29_NO=264 +WORKER_4_LINE_29_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_4_LINE_30_NO=265 +WORKER_4_LINE_30_TEXT= logical="$(logical_id "$app")" +WORKER_4_LINE_31_NO=268 +WORKER_4_LINE_31_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_4_LINE_32_NO=269 +WORKER_4_LINE_32_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_4_LINE_33_NO=277 +WORKER_4_LINE_33_TEXT= done <"$manifest" +WORKER_4_LINE_34_NO=344 +WORKER_4_LINE_34_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_4_LINE_35_NO=349 +WORKER_4_LINE_35_TEXT= if "manifest.json" not in names: +WORKER_4_LINE_36_NO=351 +WORKER_4_LINE_36_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_4_LINE_37_NO=353 +WORKER_4_LINE_37_TEXT= for row in manifest: +WORKER_4_LINE_38_NO=394 +WORKER_4_LINE_38_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_4_LINE_39_NO=406 +WORKER_4_LINE_39_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_4_LINE_40_NO=408 +WORKER_4_LINE_40_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_4_LINE_41_NO=414 +WORKER_4_LINE_41_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_4_LINE_42_NO=417 +WORKER_4_LINE_42_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_4_LINE_43_NO=418 +WORKER_4_LINE_43_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_4_LINE_44_NO=424 +WORKER_4_LINE_44_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_4_LINE_45_NO=427 +WORKER_4_LINE_45_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_4_LINE_46_NO=434 +WORKER_4_LINE_46_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_4_LINE_47_NO=459 +WORKER_4_LINE_47_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_4_LINE_48_NO=461 +WORKER_4_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_4_LINE_49_NO=463 +WORKER_4_LINE_49_TEXT=ACTION="${1:-}" +WORKER_4_LINE_50_NO=464 +WORKER_4_LINE_50_TEXT=case "$ACTION" in +WORKER_4_LINE_51_NO=466 +WORKER_4_LINE_51_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_4_LINE_52_NO=467 +WORKER_4_LINE_52_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_5_BASENAME=homelab-app-cloud-quorum-worker +WORKER_5_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-scrutiny-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker +WORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_5_EXECUTABLE=true +WORKER_5_PRESENT=true +WORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_5_SHEBANG=#!/usr/bin/env bash +WORKER_5_BASH_N_RC=0 +WORKER_5_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_5_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_5_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_5_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_5_INTERESTING_LINE_COUNT=58 +WORKER_5_LINE_1_NO=4 +WORKER_5_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_5_LINE_2_NO=6 +WORKER_5_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_5_LINE_3_NO=7 +WORKER_5_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_5_LINE_4_NO=8 +WORKER_5_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_5_LINE_5_NO=9 +WORKER_5_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_5_LINE_6_NO=10 +WORKER_5_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_5_LINE_7_NO=69 +WORKER_5_LINE_7_TEXT=logical_id(){ +WORKER_5_LINE_8_NO=74 +WORKER_5_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_5_LINE_9_NO=90 +WORKER_5_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_5_LINE_10_NO=91 +WORKER_5_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_5_LINE_11_NO=109 +WORKER_5_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_5_LINE_12_NO=111 +WORKER_5_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_5_LINE_13_NO=112 +WORKER_5_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_5_LINE_14_NO=123 +WORKER_5_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_5_LINE_15_NO=125 +WORKER_5_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_5_LINE_16_NO=126 +WORKER_5_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_5_LINE_17_NO=127 +WORKER_5_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_5_LINE_18_NO=134 +WORKER_5_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_5_LINE_19_NO=149 +WORKER_5_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_5_LINE_20_NO=168 +WORKER_5_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_5_LINE_21_NO=176 +WORKER_5_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_5_LINE_22_NO=187 +WORKER_5_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_5_LINE_23_NO=218 +WORKER_5_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_5_LINE_24_NO=229 +WORKER_5_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_5_LINE_25_NO=245 +WORKER_5_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_5_LINE_26_NO=266 +WORKER_5_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_5_LINE_27_NO=301 +WORKER_5_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_5_LINE_28_NO=302 +WORKER_5_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_5_LINE_29_NO=320 +WORKER_5_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_5_LINE_30_NO=368 +WORKER_5_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_5_LINE_31_NO=373 +WORKER_5_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_5_LINE_32_NO=376 +WORKER_5_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_5_LINE_33_NO=377 +WORKER_5_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_5_LINE_34_NO=380 +WORKER_5_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_5_LINE_35_NO=381 +WORKER_5_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_5_LINE_36_NO=389 +WORKER_5_LINE_36_TEXT= done <"$manifest" +WORKER_5_LINE_37_NO=464 +WORKER_5_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_5_LINE_38_NO=465 +WORKER_5_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_5_LINE_39_NO=499 +WORKER_5_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_5_LINE_40_NO=504 +WORKER_5_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_5_LINE_41_NO=506 +WORKER_5_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_5_LINE_42_NO=508 +WORKER_5_LINE_42_TEXT= for row in manifest: +WORKER_5_LINE_43_NO=549 +WORKER_5_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_5_LINE_44_NO=561 +WORKER_5_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_5_LINE_45_NO=563 +WORKER_5_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_5_LINE_46_NO=569 +WORKER_5_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_5_LINE_47_NO=572 +WORKER_5_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_5_LINE_48_NO=573 +WORKER_5_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_5_LINE_49_NO=579 +WORKER_5_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_5_LINE_50_NO=586 +WORKER_5_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_5_LINE_51_NO=593 +WORKER_5_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_5_LINE_52_NO=630 +WORKER_5_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_5_LINE_53_NO=632 +WORKER_5_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_5_LINE_54_NO=678 +WORKER_5_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_5_LINE_55_NO=689 +WORKER_5_LINE_55_TEXT=ACTION="${1:-}" +WORKER_5_LINE_56_NO=690 +WORKER_5_LINE_56_TEXT=case "$ACTION" in +WORKER_5_LINE_57_NO=692 +WORKER_5_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_5_LINE_58_NO=693 +WORKER_5_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_6_BASENAME=homelab-app-cloud-quorum-worker +WORKER_6_PATH_HINT=homelab-ops-cr-2026-0029/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_6_EXECUTABLE=true +WORKER_6_PRESENT=true +WORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_6_SHEBANG=#!/usr/bin/env bash +WORKER_6_BASH_N_RC=0 +WORKER_6_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_6_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_6_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_6_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_6_INTERESTING_LINE_COUNT=58 +WORKER_6_LINE_1_NO=4 +WORKER_6_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_6_LINE_2_NO=6 +WORKER_6_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_6_LINE_3_NO=7 +WORKER_6_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_6_LINE_4_NO=8 +WORKER_6_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_6_LINE_5_NO=9 +WORKER_6_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_6_LINE_6_NO=10 +WORKER_6_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_6_LINE_7_NO=69 +WORKER_6_LINE_7_TEXT=logical_id(){ +WORKER_6_LINE_8_NO=74 +WORKER_6_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_6_LINE_9_NO=90 +WORKER_6_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_6_LINE_10_NO=91 +WORKER_6_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_6_LINE_11_NO=109 +WORKER_6_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_6_LINE_12_NO=111 +WORKER_6_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_6_LINE_13_NO=112 +WORKER_6_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_6_LINE_14_NO=123 +WORKER_6_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_6_LINE_15_NO=125 +WORKER_6_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_6_LINE_16_NO=126 +WORKER_6_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_6_LINE_17_NO=127 +WORKER_6_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_6_LINE_18_NO=134 +WORKER_6_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_6_LINE_19_NO=149 +WORKER_6_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_6_LINE_20_NO=168 +WORKER_6_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_6_LINE_21_NO=176 +WORKER_6_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_6_LINE_22_NO=187 +WORKER_6_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_6_LINE_23_NO=218 +WORKER_6_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_6_LINE_24_NO=229 +WORKER_6_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_6_LINE_25_NO=245 +WORKER_6_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_6_LINE_26_NO=266 +WORKER_6_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_6_LINE_27_NO=301 +WORKER_6_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_6_LINE_28_NO=302 +WORKER_6_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_6_LINE_29_NO=320 +WORKER_6_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_6_LINE_30_NO=368 +WORKER_6_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_6_LINE_31_NO=373 +WORKER_6_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_6_LINE_32_NO=376 +WORKER_6_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_6_LINE_33_NO=377 +WORKER_6_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_6_LINE_34_NO=380 +WORKER_6_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_6_LINE_35_NO=381 +WORKER_6_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_6_LINE_36_NO=389 +WORKER_6_LINE_36_TEXT= done <"$manifest" +WORKER_6_LINE_37_NO=464 +WORKER_6_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_6_LINE_38_NO=465 +WORKER_6_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_6_LINE_39_NO=499 +WORKER_6_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_6_LINE_40_NO=504 +WORKER_6_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_6_LINE_41_NO=506 +WORKER_6_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_6_LINE_42_NO=508 +WORKER_6_LINE_42_TEXT= for row in manifest: +WORKER_6_LINE_43_NO=549 +WORKER_6_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_6_LINE_44_NO=561 +WORKER_6_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_6_LINE_45_NO=563 +WORKER_6_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_6_LINE_46_NO=569 +WORKER_6_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_6_LINE_47_NO=572 +WORKER_6_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_6_LINE_48_NO=573 +WORKER_6_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_6_LINE_49_NO=579 +WORKER_6_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_6_LINE_50_NO=586 +WORKER_6_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_6_LINE_51_NO=593 +WORKER_6_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_6_LINE_52_NO=630 +WORKER_6_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_6_LINE_53_NO=632 +WORKER_6_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_6_LINE_54_NO=678 +WORKER_6_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_6_LINE_55_NO=689 +WORKER_6_LINE_55_TEXT=ACTION="${1:-}" +WORKER_6_LINE_56_NO=690 +WORKER_6_LINE_56_TEXT=case "$ACTION" in +WORKER_6_LINE_57_NO=692 +WORKER_6_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_6_LINE_58_NO=693 +WORKER_6_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_7_BASENAME=homelab-app-cloud-quorum-worker +WORKER_7_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_7_EXECUTABLE=true +WORKER_7_PRESENT=true +WORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_7_SHEBANG=#!/usr/bin/env bash +WORKER_7_BASH_N_RC=0 +WORKER_7_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_7_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_7_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_7_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_7_INTERESTING_LINE_COUNT=58 +WORKER_7_LINE_1_NO=4 +WORKER_7_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_7_LINE_2_NO=6 +WORKER_7_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_7_LINE_3_NO=7 +WORKER_7_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_7_LINE_4_NO=8 +WORKER_7_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_7_LINE_5_NO=9 +WORKER_7_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_7_LINE_6_NO=10 +WORKER_7_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_7_LINE_7_NO=69 +WORKER_7_LINE_7_TEXT=logical_id(){ +WORKER_7_LINE_8_NO=74 +WORKER_7_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_7_LINE_9_NO=90 +WORKER_7_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_7_LINE_10_NO=91 +WORKER_7_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_7_LINE_11_NO=109 +WORKER_7_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_7_LINE_12_NO=111 +WORKER_7_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_7_LINE_13_NO=112 +WORKER_7_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_7_LINE_14_NO=123 +WORKER_7_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_7_LINE_15_NO=125 +WORKER_7_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_7_LINE_16_NO=126 +WORKER_7_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_7_LINE_17_NO=127 +WORKER_7_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_7_LINE_18_NO=134 +WORKER_7_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_7_LINE_19_NO=149 +WORKER_7_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_7_LINE_20_NO=168 +WORKER_7_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_7_LINE_21_NO=176 +WORKER_7_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_7_LINE_22_NO=187 +WORKER_7_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_7_LINE_23_NO=218 +WORKER_7_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_7_LINE_24_NO=229 +WORKER_7_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_7_LINE_25_NO=245 +WORKER_7_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_7_LINE_26_NO=266 +WORKER_7_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_7_LINE_27_NO=301 +WORKER_7_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_7_LINE_28_NO=302 +WORKER_7_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_7_LINE_29_NO=320 +WORKER_7_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_7_LINE_30_NO=368 +WORKER_7_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_7_LINE_31_NO=373 +WORKER_7_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_7_LINE_32_NO=376 +WORKER_7_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_7_LINE_33_NO=377 +WORKER_7_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_7_LINE_34_NO=380 +WORKER_7_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_7_LINE_35_NO=381 +WORKER_7_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_7_LINE_36_NO=389 +WORKER_7_LINE_36_TEXT= done <"$manifest" +WORKER_7_LINE_37_NO=464 +WORKER_7_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_7_LINE_38_NO=465 +WORKER_7_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_7_LINE_39_NO=499 +WORKER_7_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_7_LINE_40_NO=504 +WORKER_7_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_7_LINE_41_NO=506 +WORKER_7_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_7_LINE_42_NO=508 +WORKER_7_LINE_42_TEXT= for row in manifest: +WORKER_7_LINE_43_NO=549 +WORKER_7_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_7_LINE_44_NO=561 +WORKER_7_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_7_LINE_45_NO=563 +WORKER_7_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_7_LINE_46_NO=569 +WORKER_7_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_7_LINE_47_NO=572 +WORKER_7_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_7_LINE_48_NO=573 +WORKER_7_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_7_LINE_49_NO=579 +WORKER_7_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_7_LINE_50_NO=586 +WORKER_7_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_7_LINE_51_NO=593 +WORKER_7_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_7_LINE_52_NO=630 +WORKER_7_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_7_LINE_53_NO=632 +WORKER_7_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_7_LINE_54_NO=678 +WORKER_7_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_7_LINE_55_NO=689 +WORKER_7_LINE_55_TEXT=ACTION="${1:-}" +WORKER_7_LINE_56_NO=690 +WORKER_7_LINE_56_TEXT=case "$ACTION" in +WORKER_7_LINE_57_NO=692 +WORKER_7_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_7_LINE_58_NO=693 +WORKER_7_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_8_BASENAME=homelab-app-cloud-quorum-worker +WORKER_8_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_8_EXECUTABLE=true +WORKER_8_PRESENT=true +WORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_8_SHEBANG=#!/usr/bin/env bash +WORKER_8_BASH_N_RC=0 +WORKER_8_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_8_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_8_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_8_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_8_INTERESTING_LINE_COUNT=58 +WORKER_8_LINE_1_NO=4 +WORKER_8_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_8_LINE_2_NO=6 +WORKER_8_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_8_LINE_3_NO=7 +WORKER_8_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_8_LINE_4_NO=8 +WORKER_8_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_8_LINE_5_NO=9 +WORKER_8_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_8_LINE_6_NO=10 +WORKER_8_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_8_LINE_7_NO=69 +WORKER_8_LINE_7_TEXT=logical_id(){ +WORKER_8_LINE_8_NO=74 +WORKER_8_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_8_LINE_9_NO=90 +WORKER_8_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_8_LINE_10_NO=91 +WORKER_8_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_8_LINE_11_NO=109 +WORKER_8_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_8_LINE_12_NO=111 +WORKER_8_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_8_LINE_13_NO=112 +WORKER_8_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_8_LINE_14_NO=123 +WORKER_8_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_8_LINE_15_NO=125 +WORKER_8_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_8_LINE_16_NO=126 +WORKER_8_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_8_LINE_17_NO=127 +WORKER_8_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_8_LINE_18_NO=134 +WORKER_8_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_8_LINE_19_NO=149 +WORKER_8_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_8_LINE_20_NO=168 +WORKER_8_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_8_LINE_21_NO=176 +WORKER_8_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_8_LINE_22_NO=187 +WORKER_8_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_8_LINE_23_NO=218 +WORKER_8_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_8_LINE_24_NO=229 +WORKER_8_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_8_LINE_25_NO=245 +WORKER_8_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_8_LINE_26_NO=266 +WORKER_8_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_8_LINE_27_NO=301 +WORKER_8_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_8_LINE_28_NO=302 +WORKER_8_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_8_LINE_29_NO=320 +WORKER_8_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_8_LINE_30_NO=368 +WORKER_8_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_8_LINE_31_NO=373 +WORKER_8_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_8_LINE_32_NO=376 +WORKER_8_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_8_LINE_33_NO=377 +WORKER_8_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_8_LINE_34_NO=380 +WORKER_8_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_8_LINE_35_NO=381 +WORKER_8_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_8_LINE_36_NO=389 +WORKER_8_LINE_36_TEXT= done <"$manifest" +WORKER_8_LINE_37_NO=464 +WORKER_8_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_8_LINE_38_NO=465 +WORKER_8_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_8_LINE_39_NO=499 +WORKER_8_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_8_LINE_40_NO=504 +WORKER_8_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_8_LINE_41_NO=506 +WORKER_8_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_8_LINE_42_NO=508 +WORKER_8_LINE_42_TEXT= for row in manifest: +WORKER_8_LINE_43_NO=549 +WORKER_8_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_8_LINE_44_NO=561 +WORKER_8_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_8_LINE_45_NO=563 +WORKER_8_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_8_LINE_46_NO=569 +WORKER_8_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_8_LINE_47_NO=572 +WORKER_8_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_8_LINE_48_NO=573 +WORKER_8_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_8_LINE_49_NO=579 +WORKER_8_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_8_LINE_50_NO=586 +WORKER_8_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_8_LINE_51_NO=593 +WORKER_8_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_8_LINE_52_NO=630 +WORKER_8_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_8_LINE_53_NO=632 +WORKER_8_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_8_LINE_54_NO=678 +WORKER_8_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_8_LINE_55_NO=689 +WORKER_8_LINE_55_TEXT=ACTION="${1:-}" +WORKER_8_LINE_56_NO=690 +WORKER_8_LINE_56_TEXT=case "$ACTION" in +WORKER_8_LINE_57_NO=692 +WORKER_8_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_8_LINE_58_NO=693 +WORKER_8_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_9_BASENAME=homelab-app-cloud-quorum-worker +WORKER_9_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_9_EXECUTABLE=true +WORKER_9_PRESENT=true +WORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_9_SHEBANG=#!/usr/bin/env bash +WORKER_9_BASH_N_RC=0 +WORKER_9_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_9_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_9_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_9_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_9_INTERESTING_LINE_COUNT=58 +WORKER_9_LINE_1_NO=4 +WORKER_9_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_9_LINE_2_NO=6 +WORKER_9_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_9_LINE_3_NO=7 +WORKER_9_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_9_LINE_4_NO=8 +WORKER_9_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_9_LINE_5_NO=9 +WORKER_9_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_9_LINE_6_NO=10 +WORKER_9_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_9_LINE_7_NO=69 +WORKER_9_LINE_7_TEXT=logical_id(){ +WORKER_9_LINE_8_NO=74 +WORKER_9_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_9_LINE_9_NO=90 +WORKER_9_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_9_LINE_10_NO=91 +WORKER_9_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_9_LINE_11_NO=109 +WORKER_9_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_9_LINE_12_NO=111 +WORKER_9_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_9_LINE_13_NO=112 +WORKER_9_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_9_LINE_14_NO=123 +WORKER_9_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_9_LINE_15_NO=125 +WORKER_9_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_9_LINE_16_NO=126 +WORKER_9_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_9_LINE_17_NO=127 +WORKER_9_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_9_LINE_18_NO=134 +WORKER_9_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_9_LINE_19_NO=149 +WORKER_9_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_9_LINE_20_NO=168 +WORKER_9_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_9_LINE_21_NO=176 +WORKER_9_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_9_LINE_22_NO=187 +WORKER_9_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_9_LINE_23_NO=218 +WORKER_9_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_9_LINE_24_NO=229 +WORKER_9_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_9_LINE_25_NO=245 +WORKER_9_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_9_LINE_26_NO=266 +WORKER_9_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_9_LINE_27_NO=301 +WORKER_9_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_9_LINE_28_NO=302 +WORKER_9_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_9_LINE_29_NO=320 +WORKER_9_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_9_LINE_30_NO=368 +WORKER_9_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_9_LINE_31_NO=373 +WORKER_9_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_9_LINE_32_NO=376 +WORKER_9_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_9_LINE_33_NO=377 +WORKER_9_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_9_LINE_34_NO=380 +WORKER_9_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_9_LINE_35_NO=381 +WORKER_9_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_9_LINE_36_NO=389 +WORKER_9_LINE_36_TEXT= done <"$manifest" +WORKER_9_LINE_37_NO=464 +WORKER_9_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_9_LINE_38_NO=465 +WORKER_9_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_9_LINE_39_NO=499 +WORKER_9_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_9_LINE_40_NO=504 +WORKER_9_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_9_LINE_41_NO=506 +WORKER_9_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_9_LINE_42_NO=508 +WORKER_9_LINE_42_TEXT= for row in manifest: +WORKER_9_LINE_43_NO=549 +WORKER_9_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_9_LINE_44_NO=561 +WORKER_9_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_9_LINE_45_NO=563 +WORKER_9_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_9_LINE_46_NO=569 +WORKER_9_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_9_LINE_47_NO=572 +WORKER_9_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_9_LINE_48_NO=573 +WORKER_9_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_9_LINE_49_NO=579 +WORKER_9_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_9_LINE_50_NO=586 +WORKER_9_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_9_LINE_51_NO=593 +WORKER_9_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_9_LINE_52_NO=630 +WORKER_9_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_9_LINE_53_NO=632 +WORKER_9_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_9_LINE_54_NO=678 +WORKER_9_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_9_LINE_55_NO=689 +WORKER_9_LINE_55_TEXT=ACTION="${1:-}" +WORKER_9_LINE_56_NO=690 +WORKER_9_LINE_56_TEXT=case "$ACTION" in +WORKER_9_LINE_57_NO=692 +WORKER_9_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_9_LINE_58_NO=693 +WORKER_9_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_10_BASENAME=homelab-app-cloud-quorum-worker +WORKER_10_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_10_EXECUTABLE=true +WORKER_10_PRESENT=true +WORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_10_SHEBANG=#!/usr/bin/env bash +WORKER_10_BASH_N_RC=0 +WORKER_10_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_10_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_10_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_10_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_10_INTERESTING_LINE_COUNT=58 +WORKER_10_LINE_1_NO=4 +WORKER_10_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_10_LINE_2_NO=6 +WORKER_10_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_10_LINE_3_NO=7 +WORKER_10_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_10_LINE_4_NO=8 +WORKER_10_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_10_LINE_5_NO=9 +WORKER_10_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_10_LINE_6_NO=10 +WORKER_10_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_10_LINE_7_NO=69 +WORKER_10_LINE_7_TEXT=logical_id(){ +WORKER_10_LINE_8_NO=74 +WORKER_10_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_10_LINE_9_NO=90 +WORKER_10_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_10_LINE_10_NO=91 +WORKER_10_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_10_LINE_11_NO=109 +WORKER_10_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_10_LINE_12_NO=111 +WORKER_10_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_10_LINE_13_NO=112 +WORKER_10_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_10_LINE_14_NO=123 +WORKER_10_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_10_LINE_15_NO=125 +WORKER_10_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_10_LINE_16_NO=126 +WORKER_10_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_10_LINE_17_NO=127 +WORKER_10_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_10_LINE_18_NO=134 +WORKER_10_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_10_LINE_19_NO=149 +WORKER_10_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_10_LINE_20_NO=168 +WORKER_10_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_10_LINE_21_NO=176 +WORKER_10_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_10_LINE_22_NO=187 +WORKER_10_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_10_LINE_23_NO=218 +WORKER_10_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_10_LINE_24_NO=229 +WORKER_10_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_10_LINE_25_NO=245 +WORKER_10_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_10_LINE_26_NO=266 +WORKER_10_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_10_LINE_27_NO=301 +WORKER_10_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_10_LINE_28_NO=302 +WORKER_10_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_10_LINE_29_NO=320 +WORKER_10_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_10_LINE_30_NO=368 +WORKER_10_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_10_LINE_31_NO=373 +WORKER_10_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_10_LINE_32_NO=376 +WORKER_10_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_10_LINE_33_NO=377 +WORKER_10_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_10_LINE_34_NO=380 +WORKER_10_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_10_LINE_35_NO=381 +WORKER_10_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_10_LINE_36_NO=389 +WORKER_10_LINE_36_TEXT= done <"$manifest" +WORKER_10_LINE_37_NO=464 +WORKER_10_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_10_LINE_38_NO=465 +WORKER_10_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_10_LINE_39_NO=499 +WORKER_10_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_10_LINE_40_NO=504 +WORKER_10_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_10_LINE_41_NO=506 +WORKER_10_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_10_LINE_42_NO=508 +WORKER_10_LINE_42_TEXT= for row in manifest: +WORKER_10_LINE_43_NO=549 +WORKER_10_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_10_LINE_44_NO=561 +WORKER_10_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_10_LINE_45_NO=563 +WORKER_10_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_10_LINE_46_NO=569 +WORKER_10_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_10_LINE_47_NO=572 +WORKER_10_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_10_LINE_48_NO=573 +WORKER_10_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_10_LINE_49_NO=579 +WORKER_10_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_10_LINE_50_NO=586 +WORKER_10_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_10_LINE_51_NO=593 +WORKER_10_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_10_LINE_52_NO=630 +WORKER_10_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_10_LINE_53_NO=632 +WORKER_10_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_10_LINE_54_NO=678 +WORKER_10_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_10_LINE_55_NO=689 +WORKER_10_LINE_55_TEXT=ACTION="${1:-}" +WORKER_10_LINE_56_NO=690 +WORKER_10_LINE_56_TEXT=case "$ACTION" in +WORKER_10_LINE_57_NO=692 +WORKER_10_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_10_LINE_58_NO=693 +WORKER_10_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_11_BASENAME=homelab-app-cloud-quorum-worker +WORKER_11_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_11_EXECUTABLE=true +WORKER_11_PRESENT=true +WORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_11_SHEBANG=#!/usr/bin/env bash +WORKER_11_BASH_N_RC=0 +WORKER_11_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_11_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_11_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_11_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_11_INTERESTING_LINE_COUNT=58 +WORKER_11_LINE_1_NO=4 +WORKER_11_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_11_LINE_2_NO=6 +WORKER_11_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_11_LINE_3_NO=7 +WORKER_11_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_11_LINE_4_NO=8 +WORKER_11_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_11_LINE_5_NO=9 +WORKER_11_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_11_LINE_6_NO=10 +WORKER_11_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_11_LINE_7_NO=69 +WORKER_11_LINE_7_TEXT=logical_id(){ +WORKER_11_LINE_8_NO=74 +WORKER_11_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_11_LINE_9_NO=90 +WORKER_11_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_11_LINE_10_NO=91 +WORKER_11_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_11_LINE_11_NO=109 +WORKER_11_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_11_LINE_12_NO=111 +WORKER_11_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_11_LINE_13_NO=112 +WORKER_11_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_11_LINE_14_NO=123 +WORKER_11_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_11_LINE_15_NO=125 +WORKER_11_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_11_LINE_16_NO=126 +WORKER_11_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_11_LINE_17_NO=127 +WORKER_11_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_11_LINE_18_NO=134 +WORKER_11_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_11_LINE_19_NO=149 +WORKER_11_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_11_LINE_20_NO=168 +WORKER_11_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_11_LINE_21_NO=176 +WORKER_11_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_11_LINE_22_NO=187 +WORKER_11_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_11_LINE_23_NO=218 +WORKER_11_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_11_LINE_24_NO=229 +WORKER_11_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_11_LINE_25_NO=245 +WORKER_11_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_11_LINE_26_NO=266 +WORKER_11_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_11_LINE_27_NO=301 +WORKER_11_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_11_LINE_28_NO=302 +WORKER_11_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_11_LINE_29_NO=320 +WORKER_11_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_11_LINE_30_NO=368 +WORKER_11_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_11_LINE_31_NO=373 +WORKER_11_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_11_LINE_32_NO=376 +WORKER_11_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_11_LINE_33_NO=377 +WORKER_11_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_11_LINE_34_NO=380 +WORKER_11_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_11_LINE_35_NO=381 +WORKER_11_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_11_LINE_36_NO=389 +WORKER_11_LINE_36_TEXT= done <"$manifest" +WORKER_11_LINE_37_NO=464 +WORKER_11_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_11_LINE_38_NO=465 +WORKER_11_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_11_LINE_39_NO=499 +WORKER_11_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_11_LINE_40_NO=504 +WORKER_11_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_11_LINE_41_NO=506 +WORKER_11_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_11_LINE_42_NO=508 +WORKER_11_LINE_42_TEXT= for row in manifest: +WORKER_11_LINE_43_NO=549 +WORKER_11_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_11_LINE_44_NO=561 +WORKER_11_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_11_LINE_45_NO=563 +WORKER_11_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_11_LINE_46_NO=569 +WORKER_11_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_11_LINE_47_NO=572 +WORKER_11_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_11_LINE_48_NO=573 +WORKER_11_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_11_LINE_49_NO=579 +WORKER_11_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_11_LINE_50_NO=586 +WORKER_11_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_11_LINE_51_NO=593 +WORKER_11_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_11_LINE_52_NO=630 +WORKER_11_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_11_LINE_53_NO=632 +WORKER_11_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_11_LINE_54_NO=678 +WORKER_11_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_11_LINE_55_NO=689 +WORKER_11_LINE_55_TEXT=ACTION="${1:-}" +WORKER_11_LINE_56_NO=690 +WORKER_11_LINE_56_TEXT=case "$ACTION" in +WORKER_11_LINE_57_NO=692 +WORKER_11_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_11_LINE_58_NO=693 +WORKER_11_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_12_BASENAME=homelab-app-cloud-quorum-worker +WORKER_12_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_12_EXECUTABLE=true +WORKER_12_PRESENT=true +WORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_12_SHEBANG=#!/usr/bin/env bash +WORKER_12_BASH_N_RC=0 +WORKER_12_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_12_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_12_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_12_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_12_INTERESTING_LINE_COUNT=58 +WORKER_12_LINE_1_NO=4 +WORKER_12_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_12_LINE_2_NO=6 +WORKER_12_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_12_LINE_3_NO=7 +WORKER_12_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_12_LINE_4_NO=8 +WORKER_12_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_12_LINE_5_NO=9 +WORKER_12_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_12_LINE_6_NO=10 +WORKER_12_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_12_LINE_7_NO=69 +WORKER_12_LINE_7_TEXT=logical_id(){ +WORKER_12_LINE_8_NO=74 +WORKER_12_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_12_LINE_9_NO=90 +WORKER_12_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_12_LINE_10_NO=91 +WORKER_12_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_12_LINE_11_NO=109 +WORKER_12_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_12_LINE_12_NO=111 +WORKER_12_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_12_LINE_13_NO=112 +WORKER_12_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_12_LINE_14_NO=123 +WORKER_12_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_12_LINE_15_NO=125 +WORKER_12_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_12_LINE_16_NO=126 +WORKER_12_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_12_LINE_17_NO=127 +WORKER_12_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_12_LINE_18_NO=134 +WORKER_12_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_12_LINE_19_NO=149 +WORKER_12_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_12_LINE_20_NO=168 +WORKER_12_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_12_LINE_21_NO=176 +WORKER_12_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_12_LINE_22_NO=187 +WORKER_12_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_12_LINE_23_NO=218 +WORKER_12_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_12_LINE_24_NO=229 +WORKER_12_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_12_LINE_25_NO=245 +WORKER_12_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_12_LINE_26_NO=266 +WORKER_12_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_12_LINE_27_NO=301 +WORKER_12_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_12_LINE_28_NO=302 +WORKER_12_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_12_LINE_29_NO=320 +WORKER_12_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_12_LINE_30_NO=368 +WORKER_12_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_12_LINE_31_NO=373 +WORKER_12_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_12_LINE_32_NO=376 +WORKER_12_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_12_LINE_33_NO=377 +WORKER_12_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_12_LINE_34_NO=380 +WORKER_12_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_12_LINE_35_NO=381 +WORKER_12_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_12_LINE_36_NO=389 +WORKER_12_LINE_36_TEXT= done <"$manifest" +WORKER_12_LINE_37_NO=464 +WORKER_12_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_12_LINE_38_NO=465 +WORKER_12_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_12_LINE_39_NO=499 +WORKER_12_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_12_LINE_40_NO=504 +WORKER_12_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_12_LINE_41_NO=506 +WORKER_12_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_12_LINE_42_NO=508 +WORKER_12_LINE_42_TEXT= for row in manifest: +WORKER_12_LINE_43_NO=549 +WORKER_12_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_12_LINE_44_NO=561 +WORKER_12_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_12_LINE_45_NO=563 +WORKER_12_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_12_LINE_46_NO=569 +WORKER_12_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_12_LINE_47_NO=572 +WORKER_12_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_12_LINE_48_NO=573 +WORKER_12_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_12_LINE_49_NO=579 +WORKER_12_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_12_LINE_50_NO=586 +WORKER_12_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_12_LINE_51_NO=593 +WORKER_12_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_12_LINE_52_NO=630 +WORKER_12_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_12_LINE_53_NO=632 +WORKER_12_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_12_LINE_54_NO=678 +WORKER_12_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_12_LINE_55_NO=689 +WORKER_12_LINE_55_TEXT=ACTION="${1:-}" +WORKER_12_LINE_56_NO=690 +WORKER_12_LINE_56_TEXT=case "$ACTION" in +WORKER_12_LINE_57_NO=692 +WORKER_12_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_12_LINE_58_NO=693 +WORKER_12_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_13_BASENAME=homelab-app-cloud-quorum-worker +WORKER_13_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_13_EXECUTABLE=true +WORKER_13_PRESENT=true +WORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_13_SHEBANG=#!/usr/bin/env bash +WORKER_13_BASH_N_RC=0 +WORKER_13_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_13_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_13_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_13_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_13_INTERESTING_LINE_COUNT=58 +WORKER_13_LINE_1_NO=4 +WORKER_13_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_13_LINE_2_NO=6 +WORKER_13_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_13_LINE_3_NO=7 +WORKER_13_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_13_LINE_4_NO=8 +WORKER_13_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_13_LINE_5_NO=9 +WORKER_13_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_13_LINE_6_NO=10 +WORKER_13_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_13_LINE_7_NO=69 +WORKER_13_LINE_7_TEXT=logical_id(){ +WORKER_13_LINE_8_NO=74 +WORKER_13_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_13_LINE_9_NO=90 +WORKER_13_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_13_LINE_10_NO=91 +WORKER_13_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_13_LINE_11_NO=109 +WORKER_13_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_13_LINE_12_NO=111 +WORKER_13_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_13_LINE_13_NO=112 +WORKER_13_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_13_LINE_14_NO=123 +WORKER_13_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_13_LINE_15_NO=125 +WORKER_13_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_13_LINE_16_NO=126 +WORKER_13_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_13_LINE_17_NO=127 +WORKER_13_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_13_LINE_18_NO=134 +WORKER_13_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_13_LINE_19_NO=149 +WORKER_13_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_13_LINE_20_NO=168 +WORKER_13_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_13_LINE_21_NO=176 +WORKER_13_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_13_LINE_22_NO=187 +WORKER_13_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_13_LINE_23_NO=218 +WORKER_13_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_13_LINE_24_NO=229 +WORKER_13_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_13_LINE_25_NO=245 +WORKER_13_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_13_LINE_26_NO=266 +WORKER_13_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_13_LINE_27_NO=301 +WORKER_13_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_13_LINE_28_NO=302 +WORKER_13_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_13_LINE_29_NO=320 +WORKER_13_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_13_LINE_30_NO=368 +WORKER_13_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_13_LINE_31_NO=373 +WORKER_13_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_13_LINE_32_NO=376 +WORKER_13_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_13_LINE_33_NO=377 +WORKER_13_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_13_LINE_34_NO=380 +WORKER_13_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_13_LINE_35_NO=381 +WORKER_13_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_13_LINE_36_NO=389 +WORKER_13_LINE_36_TEXT= done <"$manifest" +WORKER_13_LINE_37_NO=464 +WORKER_13_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_13_LINE_38_NO=465 +WORKER_13_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_13_LINE_39_NO=499 +WORKER_13_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_13_LINE_40_NO=504 +WORKER_13_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_13_LINE_41_NO=506 +WORKER_13_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_13_LINE_42_NO=508 +WORKER_13_LINE_42_TEXT= for row in manifest: +WORKER_13_LINE_43_NO=549 +WORKER_13_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_13_LINE_44_NO=561 +WORKER_13_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_13_LINE_45_NO=563 +WORKER_13_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_13_LINE_46_NO=569 +WORKER_13_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_13_LINE_47_NO=572 +WORKER_13_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_13_LINE_48_NO=573 +WORKER_13_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_13_LINE_49_NO=579 +WORKER_13_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_13_LINE_50_NO=586 +WORKER_13_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_13_LINE_51_NO=593 +WORKER_13_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_13_LINE_52_NO=630 +WORKER_13_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_13_LINE_53_NO=632 +WORKER_13_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_13_LINE_54_NO=678 +WORKER_13_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_13_LINE_55_NO=689 +WORKER_13_LINE_55_TEXT=ACTION="${1:-}" +WORKER_13_LINE_56_NO=690 +WORKER_13_LINE_56_TEXT=case "$ACTION" in +WORKER_13_LINE_57_NO=692 +WORKER_13_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_13_LINE_58_NO=693 +WORKER_13_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_14_BASENAME=homelab-app-cloud-quorum-worker +WORKER_14_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_14_EXECUTABLE=true +WORKER_14_PRESENT=true +WORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_14_SHEBANG=#!/usr/bin/env bash +WORKER_14_BASH_N_RC=0 +WORKER_14_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_14_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_14_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_14_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_14_INTERESTING_LINE_COUNT=58 +WORKER_14_LINE_1_NO=4 +WORKER_14_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_14_LINE_2_NO=6 +WORKER_14_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_14_LINE_3_NO=7 +WORKER_14_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_14_LINE_4_NO=8 +WORKER_14_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_14_LINE_5_NO=9 +WORKER_14_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_14_LINE_6_NO=10 +WORKER_14_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_14_LINE_7_NO=69 +WORKER_14_LINE_7_TEXT=logical_id(){ +WORKER_14_LINE_8_NO=74 +WORKER_14_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_14_LINE_9_NO=90 +WORKER_14_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_14_LINE_10_NO=91 +WORKER_14_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_14_LINE_11_NO=109 +WORKER_14_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_14_LINE_12_NO=111 +WORKER_14_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_14_LINE_13_NO=112 +WORKER_14_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_14_LINE_14_NO=123 +WORKER_14_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_14_LINE_15_NO=125 +WORKER_14_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_14_LINE_16_NO=126 +WORKER_14_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_14_LINE_17_NO=127 +WORKER_14_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_14_LINE_18_NO=134 +WORKER_14_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_14_LINE_19_NO=149 +WORKER_14_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_14_LINE_20_NO=168 +WORKER_14_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_14_LINE_21_NO=176 +WORKER_14_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_14_LINE_22_NO=187 +WORKER_14_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_14_LINE_23_NO=218 +WORKER_14_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_14_LINE_24_NO=229 +WORKER_14_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_14_LINE_25_NO=245 +WORKER_14_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_14_LINE_26_NO=266 +WORKER_14_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_14_LINE_27_NO=301 +WORKER_14_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_14_LINE_28_NO=302 +WORKER_14_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_14_LINE_29_NO=320 +WORKER_14_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_14_LINE_30_NO=368 +WORKER_14_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_14_LINE_31_NO=373 +WORKER_14_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_14_LINE_32_NO=376 +WORKER_14_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_14_LINE_33_NO=377 +WORKER_14_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_14_LINE_34_NO=380 +WORKER_14_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_14_LINE_35_NO=381 +WORKER_14_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_14_LINE_36_NO=389 +WORKER_14_LINE_36_TEXT= done <"$manifest" +WORKER_14_LINE_37_NO=464 +WORKER_14_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_14_LINE_38_NO=465 +WORKER_14_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_14_LINE_39_NO=499 +WORKER_14_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_14_LINE_40_NO=504 +WORKER_14_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_14_LINE_41_NO=506 +WORKER_14_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_14_LINE_42_NO=508 +WORKER_14_LINE_42_TEXT= for row in manifest: +WORKER_14_LINE_43_NO=549 +WORKER_14_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_14_LINE_44_NO=561 +WORKER_14_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_14_LINE_45_NO=563 +WORKER_14_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_14_LINE_46_NO=569 +WORKER_14_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_14_LINE_47_NO=572 +WORKER_14_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_14_LINE_48_NO=573 +WORKER_14_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_14_LINE_49_NO=579 +WORKER_14_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_14_LINE_50_NO=586 +WORKER_14_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_14_LINE_51_NO=593 +WORKER_14_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_14_LINE_52_NO=630 +WORKER_14_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_14_LINE_53_NO=632 +WORKER_14_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_14_LINE_54_NO=678 +WORKER_14_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_14_LINE_55_NO=689 +WORKER_14_LINE_55_TEXT=ACTION="${1:-}" +WORKER_14_LINE_56_NO=690 +WORKER_14_LINE_56_TEXT=case "$ACTION" in +WORKER_14_LINE_57_NO=692 +WORKER_14_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_14_LINE_58_NO=693 +WORKER_14_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_15_BASENAME=homelab-app-cloud-quorum-worker +WORKER_15_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_15_EXECUTABLE=true +WORKER_15_PRESENT=true +WORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_15_SHEBANG=#!/usr/bin/env bash +WORKER_15_BASH_N_RC=0 +WORKER_15_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_15_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_15_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_15_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_15_INTERESTING_LINE_COUNT=58 +WORKER_15_LINE_1_NO=4 +WORKER_15_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_15_LINE_2_NO=6 +WORKER_15_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_15_LINE_3_NO=7 +WORKER_15_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_15_LINE_4_NO=8 +WORKER_15_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_15_LINE_5_NO=9 +WORKER_15_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_15_LINE_6_NO=10 +WORKER_15_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_15_LINE_7_NO=69 +WORKER_15_LINE_7_TEXT=logical_id(){ +WORKER_15_LINE_8_NO=74 +WORKER_15_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_15_LINE_9_NO=90 +WORKER_15_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_15_LINE_10_NO=91 +WORKER_15_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_15_LINE_11_NO=109 +WORKER_15_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_15_LINE_12_NO=111 +WORKER_15_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_15_LINE_13_NO=112 +WORKER_15_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_15_LINE_14_NO=123 +WORKER_15_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_15_LINE_15_NO=125 +WORKER_15_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_15_LINE_16_NO=126 +WORKER_15_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_15_LINE_17_NO=127 +WORKER_15_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_15_LINE_18_NO=134 +WORKER_15_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_15_LINE_19_NO=149 +WORKER_15_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_15_LINE_20_NO=168 +WORKER_15_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_15_LINE_21_NO=176 +WORKER_15_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_15_LINE_22_NO=187 +WORKER_15_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_15_LINE_23_NO=218 +WORKER_15_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_15_LINE_24_NO=229 +WORKER_15_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_15_LINE_25_NO=245 +WORKER_15_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_15_LINE_26_NO=266 +WORKER_15_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_15_LINE_27_NO=301 +WORKER_15_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_15_LINE_28_NO=302 +WORKER_15_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_15_LINE_29_NO=320 +WORKER_15_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_15_LINE_30_NO=368 +WORKER_15_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_15_LINE_31_NO=373 +WORKER_15_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_15_LINE_32_NO=376 +WORKER_15_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_15_LINE_33_NO=377 +WORKER_15_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_15_LINE_34_NO=380 +WORKER_15_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_15_LINE_35_NO=381 +WORKER_15_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_15_LINE_36_NO=389 +WORKER_15_LINE_36_TEXT= done <"$manifest" +WORKER_15_LINE_37_NO=464 +WORKER_15_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_15_LINE_38_NO=465 +WORKER_15_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_15_LINE_39_NO=499 +WORKER_15_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_15_LINE_40_NO=504 +WORKER_15_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_15_LINE_41_NO=506 +WORKER_15_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_15_LINE_42_NO=508 +WORKER_15_LINE_42_TEXT= for row in manifest: +WORKER_15_LINE_43_NO=549 +WORKER_15_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_15_LINE_44_NO=561 +WORKER_15_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_15_LINE_45_NO=563 +WORKER_15_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_15_LINE_46_NO=569 +WORKER_15_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_15_LINE_47_NO=572 +WORKER_15_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_15_LINE_48_NO=573 +WORKER_15_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_15_LINE_49_NO=579 +WORKER_15_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_15_LINE_50_NO=586 +WORKER_15_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_15_LINE_51_NO=593 +WORKER_15_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_15_LINE_52_NO=630 +WORKER_15_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_15_LINE_53_NO=632 +WORKER_15_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_15_LINE_54_NO=678 +WORKER_15_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_15_LINE_55_NO=689 +WORKER_15_LINE_55_TEXT=ACTION="${1:-}" +WORKER_15_LINE_56_NO=690 +WORKER_15_LINE_56_TEXT=case "$ACTION" in +WORKER_15_LINE_57_NO=692 +WORKER_15_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_15_LINE_58_NO=693 +WORKER_15_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_16_BASENAME=homelab-app-cloud-quorum-worker +WORKER_16_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_16_EXECUTABLE=true +WORKER_16_PRESENT=true +WORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_16_SHEBANG=#!/usr/bin/env bash +WORKER_16_BASH_N_RC=0 +WORKER_16_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_16_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_16_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_16_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_16_INTERESTING_LINE_COUNT=58 +WORKER_16_LINE_1_NO=4 +WORKER_16_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_16_LINE_2_NO=6 +WORKER_16_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_16_LINE_3_NO=7 +WORKER_16_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_16_LINE_4_NO=8 +WORKER_16_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_16_LINE_5_NO=9 +WORKER_16_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_16_LINE_6_NO=10 +WORKER_16_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_16_LINE_7_NO=69 +WORKER_16_LINE_7_TEXT=logical_id(){ +WORKER_16_LINE_8_NO=74 +WORKER_16_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_16_LINE_9_NO=90 +WORKER_16_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_16_LINE_10_NO=91 +WORKER_16_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_16_LINE_11_NO=109 +WORKER_16_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_16_LINE_12_NO=111 +WORKER_16_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_16_LINE_13_NO=112 +WORKER_16_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_16_LINE_14_NO=123 +WORKER_16_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_16_LINE_15_NO=125 +WORKER_16_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_16_LINE_16_NO=126 +WORKER_16_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_16_LINE_17_NO=127 +WORKER_16_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_16_LINE_18_NO=134 +WORKER_16_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_16_LINE_19_NO=149 +WORKER_16_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_16_LINE_20_NO=168 +WORKER_16_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_16_LINE_21_NO=176 +WORKER_16_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_16_LINE_22_NO=187 +WORKER_16_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_16_LINE_23_NO=218 +WORKER_16_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_16_LINE_24_NO=229 +WORKER_16_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_16_LINE_25_NO=245 +WORKER_16_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_16_LINE_26_NO=266 +WORKER_16_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_16_LINE_27_NO=301 +WORKER_16_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_16_LINE_28_NO=302 +WORKER_16_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_16_LINE_29_NO=320 +WORKER_16_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_16_LINE_30_NO=368 +WORKER_16_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_16_LINE_31_NO=373 +WORKER_16_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_16_LINE_32_NO=376 +WORKER_16_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_16_LINE_33_NO=377 +WORKER_16_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_16_LINE_34_NO=380 +WORKER_16_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_16_LINE_35_NO=381 +WORKER_16_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_16_LINE_36_NO=389 +WORKER_16_LINE_36_TEXT= done <"$manifest" +WORKER_16_LINE_37_NO=464 +WORKER_16_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_16_LINE_38_NO=465 +WORKER_16_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_16_LINE_39_NO=499 +WORKER_16_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_16_LINE_40_NO=504 +WORKER_16_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_16_LINE_41_NO=506 +WORKER_16_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_16_LINE_42_NO=508 +WORKER_16_LINE_42_TEXT= for row in manifest: +WORKER_16_LINE_43_NO=549 +WORKER_16_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_16_LINE_44_NO=561 +WORKER_16_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_16_LINE_45_NO=563 +WORKER_16_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_16_LINE_46_NO=569 +WORKER_16_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_16_LINE_47_NO=572 +WORKER_16_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_16_LINE_48_NO=573 +WORKER_16_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_16_LINE_49_NO=579 +WORKER_16_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_16_LINE_50_NO=586 +WORKER_16_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_16_LINE_51_NO=593 +WORKER_16_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_16_LINE_52_NO=630 +WORKER_16_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_16_LINE_53_NO=632 +WORKER_16_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_16_LINE_54_NO=678 +WORKER_16_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_16_LINE_55_NO=689 +WORKER_16_LINE_55_TEXT=ACTION="${1:-}" +WORKER_16_LINE_56_NO=690 +WORKER_16_LINE_56_TEXT=case "$ACTION" in +WORKER_16_LINE_57_NO=692 +WORKER_16_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_16_LINE_58_NO=693 +WORKER_16_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_17_BASENAME=homelab-app-cloud-quorum-worker +WORKER_17_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_17_EXECUTABLE=true +WORKER_17_PRESENT=true +WORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_17_SHEBANG=#!/usr/bin/env bash +WORKER_17_BASH_N_RC=0 +WORKER_17_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_17_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_17_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_17_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_17_INTERESTING_LINE_COUNT=58 +WORKER_17_LINE_1_NO=4 +WORKER_17_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_17_LINE_2_NO=6 +WORKER_17_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_17_LINE_3_NO=7 +WORKER_17_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_17_LINE_4_NO=8 +WORKER_17_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_17_LINE_5_NO=9 +WORKER_17_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_17_LINE_6_NO=10 +WORKER_17_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_17_LINE_7_NO=69 +WORKER_17_LINE_7_TEXT=logical_id(){ +WORKER_17_LINE_8_NO=74 +WORKER_17_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_17_LINE_9_NO=90 +WORKER_17_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_17_LINE_10_NO=91 +WORKER_17_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_17_LINE_11_NO=109 +WORKER_17_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_17_LINE_12_NO=111 +WORKER_17_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_17_LINE_13_NO=112 +WORKER_17_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_17_LINE_14_NO=123 +WORKER_17_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_17_LINE_15_NO=125 +WORKER_17_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_17_LINE_16_NO=126 +WORKER_17_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_17_LINE_17_NO=127 +WORKER_17_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_17_LINE_18_NO=134 +WORKER_17_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_17_LINE_19_NO=149 +WORKER_17_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_17_LINE_20_NO=168 +WORKER_17_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_17_LINE_21_NO=176 +WORKER_17_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_17_LINE_22_NO=187 +WORKER_17_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_17_LINE_23_NO=218 +WORKER_17_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_17_LINE_24_NO=229 +WORKER_17_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_17_LINE_25_NO=245 +WORKER_17_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_17_LINE_26_NO=266 +WORKER_17_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_17_LINE_27_NO=301 +WORKER_17_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_17_LINE_28_NO=302 +WORKER_17_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_17_LINE_29_NO=320 +WORKER_17_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_17_LINE_30_NO=368 +WORKER_17_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_17_LINE_31_NO=373 +WORKER_17_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_17_LINE_32_NO=376 +WORKER_17_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_17_LINE_33_NO=377 +WORKER_17_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_17_LINE_34_NO=380 +WORKER_17_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_17_LINE_35_NO=381 +WORKER_17_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_17_LINE_36_NO=389 +WORKER_17_LINE_36_TEXT= done <"$manifest" +WORKER_17_LINE_37_NO=464 +WORKER_17_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_17_LINE_38_NO=465 +WORKER_17_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_17_LINE_39_NO=499 +WORKER_17_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_17_LINE_40_NO=504 +WORKER_17_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_17_LINE_41_NO=506 +WORKER_17_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_17_LINE_42_NO=508 +WORKER_17_LINE_42_TEXT= for row in manifest: +WORKER_17_LINE_43_NO=549 +WORKER_17_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_17_LINE_44_NO=561 +WORKER_17_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_17_LINE_45_NO=563 +WORKER_17_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_17_LINE_46_NO=569 +WORKER_17_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_17_LINE_47_NO=572 +WORKER_17_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_17_LINE_48_NO=573 +WORKER_17_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_17_LINE_49_NO=579 +WORKER_17_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_17_LINE_50_NO=586 +WORKER_17_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_17_LINE_51_NO=593 +WORKER_17_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_17_LINE_52_NO=630 +WORKER_17_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_17_LINE_53_NO=632 +WORKER_17_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_17_LINE_54_NO=678 +WORKER_17_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_17_LINE_55_NO=689 +WORKER_17_LINE_55_TEXT=ACTION="${1:-}" +WORKER_17_LINE_56_NO=690 +WORKER_17_LINE_56_TEXT=case "$ACTION" in +WORKER_17_LINE_57_NO=692 +WORKER_17_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_17_LINE_58_NO=693 +WORKER_17_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_18_BASENAME=homelab-app-cloud-quorum-worker +WORKER_18_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_18_EXECUTABLE=true +WORKER_18_PRESENT=true +WORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_18_SHEBANG=#!/usr/bin/env bash +WORKER_18_BASH_N_RC=0 +WORKER_18_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_18_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_18_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_18_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_18_INTERESTING_LINE_COUNT=58 +WORKER_18_LINE_1_NO=4 +WORKER_18_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_18_LINE_2_NO=6 +WORKER_18_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_18_LINE_3_NO=7 +WORKER_18_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_18_LINE_4_NO=8 +WORKER_18_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_18_LINE_5_NO=9 +WORKER_18_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_18_LINE_6_NO=10 +WORKER_18_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_18_LINE_7_NO=69 +WORKER_18_LINE_7_TEXT=logical_id(){ +WORKER_18_LINE_8_NO=74 +WORKER_18_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_18_LINE_9_NO=90 +WORKER_18_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_18_LINE_10_NO=91 +WORKER_18_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_18_LINE_11_NO=109 +WORKER_18_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_18_LINE_12_NO=111 +WORKER_18_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_18_LINE_13_NO=112 +WORKER_18_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_18_LINE_14_NO=123 +WORKER_18_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_18_LINE_15_NO=125 +WORKER_18_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_18_LINE_16_NO=126 +WORKER_18_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_18_LINE_17_NO=127 +WORKER_18_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_18_LINE_18_NO=134 +WORKER_18_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_18_LINE_19_NO=149 +WORKER_18_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_18_LINE_20_NO=168 +WORKER_18_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_18_LINE_21_NO=176 +WORKER_18_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_18_LINE_22_NO=187 +WORKER_18_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_18_LINE_23_NO=218 +WORKER_18_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_18_LINE_24_NO=229 +WORKER_18_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_18_LINE_25_NO=245 +WORKER_18_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_18_LINE_26_NO=266 +WORKER_18_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_18_LINE_27_NO=301 +WORKER_18_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_18_LINE_28_NO=302 +WORKER_18_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_18_LINE_29_NO=320 +WORKER_18_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_18_LINE_30_NO=368 +WORKER_18_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_18_LINE_31_NO=373 +WORKER_18_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_18_LINE_32_NO=376 +WORKER_18_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_18_LINE_33_NO=377 +WORKER_18_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_18_LINE_34_NO=380 +WORKER_18_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_18_LINE_35_NO=381 +WORKER_18_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_18_LINE_36_NO=389 +WORKER_18_LINE_36_TEXT= done <"$manifest" +WORKER_18_LINE_37_NO=464 +WORKER_18_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_18_LINE_38_NO=465 +WORKER_18_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_18_LINE_39_NO=499 +WORKER_18_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_18_LINE_40_NO=504 +WORKER_18_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_18_LINE_41_NO=506 +WORKER_18_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_18_LINE_42_NO=508 +WORKER_18_LINE_42_TEXT= for row in manifest: +WORKER_18_LINE_43_NO=549 +WORKER_18_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_18_LINE_44_NO=561 +WORKER_18_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_18_LINE_45_NO=563 +WORKER_18_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_18_LINE_46_NO=569 +WORKER_18_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_18_LINE_47_NO=572 +WORKER_18_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_18_LINE_48_NO=573 +WORKER_18_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_18_LINE_49_NO=579 +WORKER_18_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_18_LINE_50_NO=586 +WORKER_18_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_18_LINE_51_NO=593 +WORKER_18_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_18_LINE_52_NO=630 +WORKER_18_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_18_LINE_53_NO=632 +WORKER_18_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_18_LINE_54_NO=678 +WORKER_18_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_18_LINE_55_NO=689 +WORKER_18_LINE_55_TEXT=ACTION="${1:-}" +WORKER_18_LINE_56_NO=690 +WORKER_18_LINE_56_TEXT=case "$ACTION" in +WORKER_18_LINE_57_NO=692 +WORKER_18_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_18_LINE_58_NO=693 +WORKER_18_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_19_BASENAME=homelab-app-cloud-quorum-worker +WORKER_19_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_19_EXECUTABLE=true +WORKER_19_PRESENT=true +WORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_19_SHEBANG=#!/usr/bin/env bash +WORKER_19_BASH_N_RC=0 +WORKER_19_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_19_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_19_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_19_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_19_INTERESTING_LINE_COUNT=58 +WORKER_19_LINE_1_NO=4 +WORKER_19_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_19_LINE_2_NO=6 +WORKER_19_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_19_LINE_3_NO=7 +WORKER_19_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_19_LINE_4_NO=8 +WORKER_19_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_19_LINE_5_NO=9 +WORKER_19_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_19_LINE_6_NO=10 +WORKER_19_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_19_LINE_7_NO=69 +WORKER_19_LINE_7_TEXT=logical_id(){ +WORKER_19_LINE_8_NO=74 +WORKER_19_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_19_LINE_9_NO=90 +WORKER_19_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_19_LINE_10_NO=91 +WORKER_19_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_19_LINE_11_NO=109 +WORKER_19_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_19_LINE_12_NO=111 +WORKER_19_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_19_LINE_13_NO=112 +WORKER_19_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_19_LINE_14_NO=123 +WORKER_19_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_19_LINE_15_NO=125 +WORKER_19_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_19_LINE_16_NO=126 +WORKER_19_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_19_LINE_17_NO=127 +WORKER_19_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_19_LINE_18_NO=134 +WORKER_19_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_19_LINE_19_NO=149 +WORKER_19_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_19_LINE_20_NO=168 +WORKER_19_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_19_LINE_21_NO=176 +WORKER_19_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_19_LINE_22_NO=187 +WORKER_19_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_19_LINE_23_NO=218 +WORKER_19_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_19_LINE_24_NO=229 +WORKER_19_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_19_LINE_25_NO=245 +WORKER_19_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_19_LINE_26_NO=266 +WORKER_19_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_19_LINE_27_NO=301 +WORKER_19_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_19_LINE_28_NO=302 +WORKER_19_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_19_LINE_29_NO=320 +WORKER_19_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_19_LINE_30_NO=368 +WORKER_19_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_19_LINE_31_NO=373 +WORKER_19_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_19_LINE_32_NO=376 +WORKER_19_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_19_LINE_33_NO=377 +WORKER_19_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_19_LINE_34_NO=380 +WORKER_19_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_19_LINE_35_NO=381 +WORKER_19_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_19_LINE_36_NO=389 +WORKER_19_LINE_36_TEXT= done <"$manifest" +WORKER_19_LINE_37_NO=464 +WORKER_19_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_19_LINE_38_NO=465 +WORKER_19_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_19_LINE_39_NO=499 +WORKER_19_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_19_LINE_40_NO=504 +WORKER_19_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_19_LINE_41_NO=506 +WORKER_19_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_19_LINE_42_NO=508 +WORKER_19_LINE_42_TEXT= for row in manifest: +WORKER_19_LINE_43_NO=549 +WORKER_19_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_19_LINE_44_NO=561 +WORKER_19_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_19_LINE_45_NO=563 +WORKER_19_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_19_LINE_46_NO=569 +WORKER_19_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_19_LINE_47_NO=572 +WORKER_19_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_19_LINE_48_NO=573 +WORKER_19_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_19_LINE_49_NO=579 +WORKER_19_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_19_LINE_50_NO=586 +WORKER_19_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_19_LINE_51_NO=593 +WORKER_19_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_19_LINE_52_NO=630 +WORKER_19_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_19_LINE_53_NO=632 +WORKER_19_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_19_LINE_54_NO=678 +WORKER_19_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_19_LINE_55_NO=689 +WORKER_19_LINE_55_TEXT=ACTION="${1:-}" +WORKER_19_LINE_56_NO=690 +WORKER_19_LINE_56_TEXT=case "$ACTION" in +WORKER_19_LINE_57_NO=692 +WORKER_19_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_19_LINE_58_NO=693 +WORKER_19_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_20_BASENAME=homelab-app-cloud-quorum-worker +WORKER_20_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_20_EXECUTABLE=true +WORKER_20_PRESENT=true +WORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_20_SHEBANG=#!/usr/bin/env bash +WORKER_20_BASH_N_RC=0 +WORKER_20_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_20_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_20_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_20_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_20_INTERESTING_LINE_COUNT=58 +WORKER_20_LINE_1_NO=4 +WORKER_20_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_20_LINE_2_NO=6 +WORKER_20_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_20_LINE_3_NO=7 +WORKER_20_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_20_LINE_4_NO=8 +WORKER_20_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_20_LINE_5_NO=9 +WORKER_20_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_20_LINE_6_NO=10 +WORKER_20_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_20_LINE_7_NO=69 +WORKER_20_LINE_7_TEXT=logical_id(){ +WORKER_20_LINE_8_NO=74 +WORKER_20_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_20_LINE_9_NO=90 +WORKER_20_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_20_LINE_10_NO=91 +WORKER_20_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_20_LINE_11_NO=109 +WORKER_20_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_20_LINE_12_NO=111 +WORKER_20_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_20_LINE_13_NO=112 +WORKER_20_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_20_LINE_14_NO=123 +WORKER_20_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_20_LINE_15_NO=125 +WORKER_20_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_20_LINE_16_NO=126 +WORKER_20_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_20_LINE_17_NO=127 +WORKER_20_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_20_LINE_18_NO=134 +WORKER_20_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_20_LINE_19_NO=149 +WORKER_20_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_20_LINE_20_NO=168 +WORKER_20_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_20_LINE_21_NO=176 +WORKER_20_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_20_LINE_22_NO=187 +WORKER_20_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_20_LINE_23_NO=218 +WORKER_20_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_20_LINE_24_NO=229 +WORKER_20_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_20_LINE_25_NO=245 +WORKER_20_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_20_LINE_26_NO=266 +WORKER_20_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_20_LINE_27_NO=301 +WORKER_20_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_20_LINE_28_NO=302 +WORKER_20_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_20_LINE_29_NO=320 +WORKER_20_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_20_LINE_30_NO=368 +WORKER_20_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_20_LINE_31_NO=373 +WORKER_20_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_20_LINE_32_NO=376 +WORKER_20_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_20_LINE_33_NO=377 +WORKER_20_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_20_LINE_34_NO=380 +WORKER_20_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_20_LINE_35_NO=381 +WORKER_20_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_20_LINE_36_NO=389 +WORKER_20_LINE_36_TEXT= done <"$manifest" +WORKER_20_LINE_37_NO=464 +WORKER_20_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_20_LINE_38_NO=465 +WORKER_20_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_20_LINE_39_NO=499 +WORKER_20_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_20_LINE_40_NO=504 +WORKER_20_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_20_LINE_41_NO=506 +WORKER_20_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_20_LINE_42_NO=508 +WORKER_20_LINE_42_TEXT= for row in manifest: +WORKER_20_LINE_43_NO=549 +WORKER_20_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_20_LINE_44_NO=561 +WORKER_20_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_20_LINE_45_NO=563 +WORKER_20_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_20_LINE_46_NO=569 +WORKER_20_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_20_LINE_47_NO=572 +WORKER_20_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_20_LINE_48_NO=573 +WORKER_20_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_20_LINE_49_NO=579 +WORKER_20_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_20_LINE_50_NO=586 +WORKER_20_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_20_LINE_51_NO=593 +WORKER_20_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_20_LINE_52_NO=630 +WORKER_20_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_20_LINE_53_NO=632 +WORKER_20_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_20_LINE_54_NO=678 +WORKER_20_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_20_LINE_55_NO=689 +WORKER_20_LINE_55_TEXT=ACTION="${1:-}" +WORKER_20_LINE_56_NO=690 +WORKER_20_LINE_56_TEXT=case "$ACTION" in +WORKER_20_LINE_57_NO=692 +WORKER_20_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_20_LINE_58_NO=693 +WORKER_20_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +TARGETS_PRESENT=true +TARGETS_SHA=2f526330152d49461ff66f50adab57d1a195ea5d1b4c292945d523411b8a68a8 +TARGET_OBJECT_FOUND=true +TARGET_KEYS=id,kind,lifecycle,logical_id,migrate,owner,source_host +TARGET_ID=skladchik-reports-monitor +TARGET_KIND=APP +TARGET_LIFECYCLE=ACTIVE +TARGET_LOGICAL_ID=app-skladchik-reports-monitor +TARGET_MIGRATE=True +TARGET_OWNER=pve01 +TARGET_SOURCE_HOST_BASENAME=[PRIVATE_IP] +TARGET_SOURCE_HOST_SHA=3eb2319e39eac307b3c7bd11f67663c0a0c814fd603989f5965083f9d1c80fb0 +QUEUE_APPID_LITERAL_COUNT=0 +QUEUE_TARGETS_BASENAME_LITERAL_COUNT=0 +BACKUP_AUDIT_RC=0 +BACKUP_AUDIT_RELEVANT_COUNT=2 +BACKUP_AUDIT_RELEVANT_1=TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=22759|generation=app-skladchik-reports-monitor-pve01-20261005T003035Z +BACKUP_AUDIT_RELEVANT_2=TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=32145|legacy_age_seconds=1627370|replica_count=2|state_id=vm170|generation=v2-vm170-20261004T214054Z-70a537ab4e91 +TARGETED_SEAL_DESIGN_READY=true +MAIL_SENT_BY_COMMAND=false +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +DECISION=PASS_QUEUE_WORKER_RCA +NEXT_GATE=DESIGN_ATOMIC_SOURCE_BACKUP_SEAL +TASK_RESULT=SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 8c274cf4..88b54e4f 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-261005-SNIKKET-OPS-GAPS-RECONCILE-99577R1", + "command_id": "SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "snikket-ops-gaps-reconcile", - "started_at_utc": "2026-10-05T06:46:19Z", - "finished_at_utc": "2026-10-05T06:46:30Z", + "component": "skladchik-durability-queue-worker-rca", + "started_at_utc": "2026-10-05T06:49:57Z", + "finished_at_utc": "2026-10-05T06:50:09Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "42c5c98e1e7e36e4af7ab23f644308660eed0d45271f7207290956264ade5a2a", + "command_sha256": "9786047247e48e71139ff52c618c59da24ce317cd7093ee00b699a69b178af52", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "b273ebd9e84019c6ed048e74e5e888c0818a877184fc6e455881b787ece315f4", - "sanitized_output_sha256": "b273ebd9e84019c6ed048e74e5e888c0818a877184fc6e455881b787ece315f4", + "raw_evidence_sha256": "df86e4a999ff180077eddd8c50c0647c83310a5062be61a88ccaab005bb577ea", + "sanitized_output_sha256": "3aa1864388e614772490c9c29217a4ed56c483df64b4b67276683b5c5078e47e", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "SNIKKET_OPS_GAPS_RECONCILE_WRAPPER=V1\nMODE=read-only\nNO_DNS_MUTATION=true\nNO_VM_MUTATION=true\nNO_CONTAINER_RESTART=true\nNO_SERVICE_START=true\nNO_SOURCE_MUTATION=true\nHOST_PVE01=true\nINGRESS_HOSTKEY_SELF_BOUND=true\nVM150_INVENTORY_PRESENT=true\nVM150_RUNNING=true\nVM150_EXPECTED_NODE=true\nVM130_INVENTORY_PRESENT=true\nVM130_RUNNING=true\nVM130_EXPECTED_NODE=true\nVM150_ONBOOT=true\nVM150_AGENT_CONFIG_LINE_PRESENT=true\nVM150_QGA_PING=true\nPVE_BACKUP_JOBS_QUERY_RC=0\nPVE_BACKUP_JOB_COVERAGE_COUNT=0\nPVE_BACKUP_JOB_ENABLED_COVERAGE_COUNT=0\nPVE_BACKUP_STORAGE_QUERY_RC=0\nPVE_BACKUP_STORAGE_COUNT=1\nPVE_BACKUP_STORAGE_TYPE_COUNT=1\nPVE_VM150_BACKUP_OBJECT_COUNT=0\nPVE_VM150_BACKUP_STORAGE_WITH_OBJECT_COUNT=0\nPVE_VM150_NEWEST_BACKUP_AGE_HOURS=-1\nPVE_VM150_BACKUP_WITHIN_48H=false\nPVE_VM150_BACKUP_WITHIN_7D=false\nHOMELAB_BACKUP_V2_TIMER_ACTIVE=active\nHOMELAB_BACKUP_V2_TIMER_ENABLED=enabled\nHOMELAB_BACKUP_V2_TIMER_RESULT=success\nHOMELAB_BACKUP_V2_TIMER_EXEC_MAIN_STATUS=\nHOMELAB_BACKUP_V2_SERVICE_ACTIVE=activating\nHOMELAB_BACKUP_V2_SERVICE_ENABLED=static\nHOMELAB_BACKUP_V2_SERVICE_RESULT=success\nHOMELAB_BACKUP_V2_SERVICE_EXEC_MAIN_STATUS=0\nHOMELAB_BACKUP_AUDIT_TIMER_ACTIVE=active\nHOMELAB_BACKUP_AUDIT_TIMER_ENABLED=enabled\nHOMELAB_BACKUP_AUDIT_TIMER_RESULT=success\nHOMELAB_BACKUP_AUDIT_TIMER_EXEC_MAIN_STATUS=\nHOMELAB_BACKUP_AUDIT_SERVICE_ACTIVE=inactive\nHOMELAB_BACKUP_AUDIT_SERVICE_ENABLED=static\nHOMELAB_BACKUP_AUDIT_SERVICE_RESULT=success\nHOMELAB_BACKUP_AUDIT_SERVICE_EXEC_MAIN_STATUS=0\nBACKUP_V2_14D_VM150_REF_COUNT=40\nBACKUP_V2_14D_VM150_SUCCESS_COUNT=40\nBACKUP_V2_14D_VM150_FAIL_COUNT=0\nBACKUP_AUDIT_30D_VM150_REF_COUNT=2379\nBACKUP_AUDIT_30D_VM150_RESTORE_REF_COUNT=764\nBACKUP_AUDIT_30D_VM150_PASS_REF_COUNT=763\nBACKUP_SOURCE_FILE_COUNT=135\nBACKUP_SOURCE_VM150_EXPLICIT_REF_FILE_COUNT=17\nBACKUP_SOURCE_DYNAMIC_VM_DISCOVERY_FILE_COUNT=46\nSOURCE_GIT_REPO_SCAN_COUNT=64\nSOURCE_RECON_EXACT_SHA_MATCH_COUNT=0\nSOURCE_RECON_CORRECT_LOGIC_TRACKED_COUNT=0\nSOURCE_DNS_ALL3_TRACKED_COUNT=343\nSOURCE_DNS_ALL3_CURRENT_TARGET_TRACKED_COUNT=294\nMONITOR_SOURCE_FILE_COUNT=86\nMONITOR_CHAT_DOMAIN_REF_FILE_COUNT=0\nMONITOR_ALL3_DOMAIN_REF_FILE_COUNT=0\nMONITOR_XMPP_5222_REF_FILE_COUNT=0\nMONITOR_XMPP_5269_REF_FILE_COUNT=0\nMONITOR_TURN_3479_REF_FILE_COUNT=0\nMONITOR_TURN_STUN_REF_FILE_COUNT=0\nMONITOR_CERT_EXPIRY_REF_FILE_COUNT=2\nHOMELAB_EXTERNAL_PROBE_VPS_HEALTH_TIMER_ACTIVE=active\nHOMELAB_EXTERNAL_PROBE_VPS_HEALTH_TIMER_ENABLED=enabled\nHOMELAB_KUMA_MONITOR_POLICY_TIMER_ACTIVE=active\nHOMELAB_KUMA_MONITOR_POLICY_TIMER_ENABLED=enabled\nHOMELAB_OVERALL_HEALTH_TIMER_ACTIVE=active\nHOMELAB_OVERALL_HEALTH_TIMER_ENABLED=enabled\nHOMELAB_ALERTING_HEALTH_TIMER_ACTIVE=active\nHOMELAB_ALERTING_HEALTH_TIMER_ENABLED=enabled\nPUBLIC_DNS_OK_COUNT=3\nPUBLIC_HTTPS_OK_COUNT=3\nPUBLIC_STUN_3479_PASS=true\nPUBLIC_STUN_3479_CLASS=BINDING_SUCCESS\nVM150_QGA_EXEC_INIT_RC=0\nVM150_QGA_POLL_RC=0\nVM150_GUEST_EXITCODE=0\nVM150_GUEST_OUTPUT_MARKER=true\nSNIKKET_OPS_GAPS_GUEST=V1\nMODE=read-only\nNO_DNS_MUTATION=true\nNO_VM_MUTATION=true\nNO_CONTAINER_RESTART=true\nNO_SERVICE_START=true\nNO_SOURCE_MUTATION=true\nGUEST_HOSTNAME_SNIKKET=true\nDOCKER_OK=true\nCONTAINER_SNIKKET_PRESENT=true\nCONTAINER_SNIKKET_RUNNING=true\nCONTAINER_SNIKKET_IMAGE_ID=c392b78a2bd3447d314054caa75b3556cf4fc3579288599e2d0a15dbcd690972\nCONTAINER_SNIKKET_IMAGE_CREATED=2026-06-12T14:41:53.\nPROSODY_ABOUT_PRESENT=true\nPROSODY_ABOUT_SHA256=df326196b07e19c252252fd9213b8920957e8ff733d5b7a548ef1b004586ff38\nSNIKKET_UPDATE_CHECK_EFFECTIVE=true\nSNIKKET_RETENTION_EXPLICIT=false\nSNIKKET_UPLOAD_LIMIT_EXPLICIT=false\nCERT_DAYS_REMAINING=-1\nROOT_FS_USED_PERCENT=4\nGUEST_AUDIT_DONE=true\nUPSTREAM_COMPOSE_FETCH_RC=0\nUPSTREAM_COMPOSE_PRESENT=true\nUPSTREAM_COMPOSE_SHA256=5773abc3efc852e3d4082de5947b7e93acdf3708e416272f6edd7c3f00de69b8\nUPSTREAM_SNIKKET_IMAGE_LINE_COUNT=4\nOPS_GAPS_RECONCILE_COMPLETE=true\nDECISION=ANALYZE_EXACT_99577_EVIDENCE\nNEXT_GATE=APPLY_ONLY_PROVEN_OPERATIONAL_GAPS\nTASK_RESULT=SNIKKET_OPS_GAPS_RECONCILE\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\n" + "output": "SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA=V1\nNO_MAIL_SEND=true\nNO_SERVICE_RESTART=true\nNO_TIMER_MUTATION=true\nNO_NETWORK_MUTATION=true\nNO_SOURCE_MUTATION=true\nNO_BACKUP_MUTATION=true\nNO_GIT_PUSH=true\nMODE=read-only\nHOST_PVE01=true\nQUEUE_VM170_QM_RC=0\nQUEUE_VM170_STDERR_PRESENT=false\nQUEUE_VM170_JSON=true\nQUEUE_VM170_GUEST_EXITCODE=0\nVM170_MONITOR_SHA=19123e359f2f7b9a2d373fd7e4464dd426d31b91cabc567c252e70cad40d77c8\nVM170_TIMER_ACTIVE=active\nVM170_TIMER_ENABLED=enabled\nVM170_SERVICE_RESULT=success\nVM170_HEALTH_STATUS=OK\nVM170_HEALTH_TS=2026-10-05T06:49:35Z\nVM170_HEALTH_REPORTS=22\nVM170_HEALTH_OPEN=13\nVM170_HEALTH_RESOLVED=9\nVM170_BASELINE_OK=true\nSUPERVISOR_SHA=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2\nSUPERVISOR_TOP_CONST_1_NAME=BACKUP_HEALTH_AGE_MAX\nSUPERVISOR_TOP_CONST_1_EXPR_SHA=056f62a1cb138698ac25abded3e6493c5519d37a485bb3900a74bad1806f70b3\nSUPERVISOR_TOP_CONST_2_NAME=RESTORE_HEALTH_AGE_MAX\nSUPERVISOR_TOP_CONST_2_EXPR_SHA=ce6a9ab58cc2e0db3f07ff3ef7d288b29b6611afff056f5b818c706075d9dbd5\nSUPERVISOR_TOP_CONST_3_NAME=APP_FRESHNESS_PROBE\nSUPERVISOR_TOP_CONST_3_EXPR_SHA=d6593f7b36fcf7550285e4af7a5f5766c13c2ee0e0a992b808caaed52e54693a\nSUPERVISOR_TOP_CONST_4_NAME=LEGACY_BACKUP_PROBLEMS\nSUPERVISOR_TOP_CONST_4_EXPR_SHA=d84baf639fd4e63d9c728a793f4dc6c457b6bee5737789745170d2150d858ddd\nSUPERVISOR_TOP_CONST_COUNT=4\nSUPERVISOR_TEST_RC=0\nSUPERVISOR_TEST_STDOUT_SHA=9b4438f36f4a40de86d41fd480c6900c28603079d96011398674df7637868c5e\nSUPERVISOR_TEST_OUTPUT_BEGIN\nSUPERVISOR_STATUS=OK\nSUPERVISOR_PROBLEMS=none\nMONITOR=OK reports=22 open=13 resolved=9\nCANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z\nREGISTRY_SOURCE=\n>\nNETBIRD_ROUTE= dev wt0 table 7120 src uid 0\nSUPERVISOR_TEST_OUTPUT_END\nSUPTEST_SUPERVISOR_STATUS=OK\nSUPTEST_SUPERVISOR_PROBLEMS=none\nSUPTEST_MONITOR=OK reports=22 open=13 resolved=9\nSUPTEST_CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z\nSUPTEST_REGISTRY_SOURCE=[PRIVATE_IP]\nSUPTEST_WORKER_TARGET=debian@[PRIVATE_IP]\nSUPTEST_NETBIRD_ROUTE=1.1.1.1 dev wt0 table 7120 src 100.125.187.138 uid 0\nQUEUE_PRESENT=true\nQUEUE_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\nQUEUE_SHEBANG=#!/usr/bin/env bash\nQUEUE_BASH_N_RC=0\nQUEUE_FUNCTIONS=backup_list,die,dispatch_one,dispatch_plan,freshness_decision,freshness_plan,lock_heavy,migrate_list,plan,verify_all\nQUEUE_ACTIONS=BACKUP_RESUME,BACKUP_UNVERIFIED,SEALED_PROVEN,SKIP_FRESH\nQUEUE_VAR_NAMES=DUE_SECONDS,FRESHNESS_SECONDS,HEAVY_LOCK,PROBE,QUEUE_LOCK,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,WORKER,age,decision,freshness,generation,now_epoch,out,sealed_epoch\nQUEUE_COMMAND_BASENAMES=flock,homelab-app-cloud-quorum-worker,sha256sum\nQUEUE_INTERESTING_LINE_COUNT=32\nQUEUE_LINE_1_NO=4\nQUEUE_LINE_1_TEXT=WORKER=/usr/local/libexec/homelab-app-cloud-quorum-worker\nQUEUE_LINE_2_NO=5\nQUEUE_LINE_2_TEXT=PROBE=/usr/local/libexec/homelab-app-cloud-quorum-freshness-probe\nQUEUE_LINE_3_NO=6\nQUEUE_LINE_3_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nQUEUE_LINE_4_NO=8\nQUEUE_LINE_4_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nQUEUE_LINE_5_NO=9\nQUEUE_LINE_5_TEXT=QUEUE_LOCK=/run/lock/homelab-app-cloud-quorum-queue.lock\nQUEUE_LINE_6_NO=10\nQUEUE_LINE_6_TEXT=HEAVY_LOCK=/run/homelab-backup-heavy.lock\nQUEUE_LINE_7_NO=21\nQUEUE_LINE_7_TEXT=[[ -x \"$WORKER\" && ! -L \"$WORKER\" ]] || die WORKER_MISSING 69\nQUEUE_LINE_8_NO=29\nQUEUE_LINE_8_TEXT= for a in \"${ALL_APPS[@]}\"; do \"$WORKER\" plan \"$a\"; done\nQUEUE_LINE_9_NO=32\nQUEUE_LINE_9_TEXT= local app=\"$1\" now_epoch=\"$2\" out decision generation sealed_epoch age\nQUEUE_LINE_10_NO=38\nQUEUE_LINE_10_TEXT= generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nQUEUE_LINE_11_NO=39\nQUEUE_LINE_11_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_RESUME GENERATION=%s\\n' \"$app\" \"$generation\"\nQUEUE_LINE_12_NO=42\nQUEUE_LINE_12_TEXT= generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nQUEUE_LINE_13_NO=50\nQUEUE_LINE_13_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\"\nQUEUE_LINE_14_NO=52\nQUEUE_LINE_14_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\"\nQUEUE_LINE_15_NO=54\nQUEUE_LINE_15_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\"\nQUEUE_LINE_16_NO=56\nQUEUE_LINE_16_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\"\nQUEUE_LINE_17_NO=84\nQUEUE_LINE_17_TEXT= out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"; printf '%s\\n' \"$out\"\nQUEUE_LINE_18_NO=85\nQUEUE_LINE_18_TEXT= \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\nQUEUE_LINE_19_NO=93\nQUEUE_LINE_19_TEXT= local a=\"$1\" freshness decision now_epoch out generation\nQUEUE_LINE_20_NO=112\nQUEUE_LINE_20_TEXT= generation=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\"\nQUEUE_LINE_21_NO=113\nQUEUE_LINE_21_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SKIP_FRESH GENERATION=%s\\n' \"$a\" \"$generation\"\nQUEUE_LINE_22_NO=120\nQUEUE_LINE_22_TEXT= out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"\nQUEUE_LINE_23_NO=122\nQUEUE_LINE_23_TEXT= \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\nQUEUE_LINE_24_NO=123\nQUEUE_LINE_24_TEXT= generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nQUEUE_LINE_25_NO=124\nQUEUE_LINE_25_TEXT= [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$a\" 65\nQUEUE_LINE_26_NO=125\nQUEUE_LINE_26_TEXT= printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$a\" \"$generation\"\nQUEUE_LINE_27_NO=126\nQUEUE_LINE_27_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\\n' \"$a\" \"$generation\"\nQUEUE_LINE_28_NO=133\nQUEUE_LINE_28_TEXT= if out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" latest-sealed \"$a\" 2>/dev/null)\"; then\nQUEUE_LINE_29_NO=138\nQUEUE_LINE_29_TEXT= out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"; printf '%s\\n' \"$out\"\nQUEUE_LINE_30_NO=139\nQUEUE_LINE_30_TEXT= \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\nQUEUE_LINE_31_NO=145\nQUEUE_LINE_31_TEXT= for a in \"${ALL_APPS[@]}\"; do out=\"$(\"$WORKER\" latest-sealed \"$a\")\"; printf '%s\\n' \"$out\"; done\nQUEUE_LINE_32_NO=153\nQUEUE_LINE_32_TEXT= all) [[ \"$#\" == 1 ]] || die USAGE_ALL 64; [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die DISPATCHER_MISSING 69; /usr/local/sbin/homelab-backup-dispatcher run APP_TIMER ;;\nQUEUE_SERVICE_SHOW_RC=0\nQUEUE_SERVICE_LOADSTATE=loaded\nQUEUE_SERVICE_ACTIVESTATE=inactive\nQUEUE_SERVICE_UNITFILESTATE=static\nQUEUE_SERVICE_RESULT=success\nQUEUE_SERVICE_EXECMAINSTATUS=0\nQUEUE_SERVICE_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.service\nQUEUE_SERVICE_FRAGMENT_SHA=afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a\nQUEUE_SERVICE_EXECSTART_SHA=22094bb14d8dc5f7399744e183b096583a9c9625527a2f00981f2ec5cd8e1713\nQUEUE_SERVICE_EXEC_BASENAMES=homelab-backup-v2-app-scheduled\nQUEUE_SERVICE_EXEC_ACTIONS=NONE\nQUEUE_TIMER_SHOW_RC=0\nQUEUE_TIMER_LOADSTATE=loaded\nQUEUE_TIMER_ACTIVESTATE=active\nQUEUE_TIMER_UNITFILESTATE=enabled\nQUEUE_TIMER_RESULT=success\nQUEUE_TIMER_EXECMAINSTATUS=\nQUEUE_TIMER_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.timer\nQUEUE_TIMER_FRAGMENT_SHA=af2eedce3df53029b591d31715a3f3e6688ddb997b4823f487e70c339ce88078\nQUEUE_TIMER_EXECSTART_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nQUEUE_TIMER_EXEC_BASENAMES=NONE\nQUEUE_TIMER_EXEC_ACTIONS=NONE\nWORKER_CANDIDATE_COUNT=173\nWORKER_1_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_1_PATH_HINT=//root/BACKUP_closeout_evidence_20260825T165406Z/source/homelab-app-cloud-quorum-worker\nWORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_1_EXECUTABLE=false\nWORKER_1_PRESENT=true\nWORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_1_SHEBANG=#!/usr/bin/env bash\nWORKER_1_BASH_N_RC=0\nWORKER_1_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_1_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor\nWORKER_1_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_1_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_1_INTERESTING_LINE_COUNT=57\nWORKER_1_LINE_1_NO=4\nWORKER_1_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_1_LINE_2_NO=6\nWORKER_1_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_1_LINE_3_NO=7\nWORKER_1_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_1_LINE_4_NO=8\nWORKER_1_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_1_LINE_5_NO=9\nWORKER_1_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_1_LINE_6_NO=10\nWORKER_1_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_1_LINE_7_NO=12\nWORKER_1_LINE_7_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_1_LINE_8_NO=14\nWORKER_1_LINE_8_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_1_LINE_9_NO=75\nWORKER_1_LINE_9_TEXT=logical_id(){\nWORKER_1_LINE_10_NO=82\nWORKER_1_LINE_10_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_1_LINE_11_NO=83\nWORKER_1_LINE_11_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_1_LINE_12_NO=101\nWORKER_1_LINE_12_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_1_LINE_13_NO=103\nWORKER_1_LINE_13_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_1_LINE_14_NO=104\nWORKER_1_LINE_14_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_1_LINE_15_NO=115\nWORKER_1_LINE_15_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_1_LINE_16_NO=117\nWORKER_1_LINE_16_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_1_LINE_17_NO=118\nWORKER_1_LINE_17_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_1_LINE_18_NO=119\nWORKER_1_LINE_18_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_1_LINE_19_NO=126\nWORKER_1_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_1_LINE_20_NO=141\nWORKER_1_LINE_20_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_1_LINE_21_NO=160\nWORKER_1_LINE_21_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_22_NO=191\nWORKER_1_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_23_NO=202\nWORKER_1_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_1_LINE_24_NO=218\nWORKER_1_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_25_NO=239\nWORKER_1_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_26_NO=274\nWORKER_1_LINE_26_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_1_LINE_27_NO=275\nWORKER_1_LINE_27_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_1_LINE_28_NO=293\nWORKER_1_LINE_28_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_29_NO=341\nWORKER_1_LINE_29_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_1_LINE_30_NO=346\nWORKER_1_LINE_30_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_1_LINE_31_NO=349\nWORKER_1_LINE_31_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_1_LINE_32_NO=350\nWORKER_1_LINE_32_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_1_LINE_33_NO=353\nWORKER_1_LINE_33_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_1_LINE_34_NO=354\nWORKER_1_LINE_34_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_1_LINE_35_NO=362\nWORKER_1_LINE_35_TEXT= done <\"$manifest\"\nWORKER_1_LINE_36_NO=431\nWORKER_1_LINE_36_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_1_LINE_37_NO=432\nWORKER_1_LINE_37_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_1_LINE_38_NO=466\nWORKER_1_LINE_38_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_1_LINE_39_NO=471\nWORKER_1_LINE_39_TEXT= if \"manifest.json\" not in names:\nWORKER_1_LINE_40_NO=473\nWORKER_1_LINE_40_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_1_LINE_41_NO=475\nWORKER_1_LINE_41_TEXT= for row in manifest:\nWORKER_1_LINE_42_NO=516\nWORKER_1_LINE_42_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_1_LINE_43_NO=528\nWORKER_1_LINE_43_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_1_LINE_44_NO=530\nWORKER_1_LINE_44_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_1_LINE_45_NO=536\nWORKER_1_LINE_45_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_1_LINE_46_NO=539\nWORKER_1_LINE_46_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_1_LINE_47_NO=540\nWORKER_1_LINE_47_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_1_LINE_48_NO=546\nWORKER_1_LINE_48_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_1_LINE_49_NO=553\nWORKER_1_LINE_49_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_1_LINE_50_NO=560\nWORKER_1_LINE_50_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_1_LINE_51_NO=596\nWORKER_1_LINE_51_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_1_LINE_52_NO=598\nWORKER_1_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_1_LINE_53_NO=644\nWORKER_1_LINE_53_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_1_LINE_54_NO=655\nWORKER_1_LINE_54_TEXT=ACTION=\"${1:-}\"\nWORKER_1_LINE_55_NO=656\nWORKER_1_LINE_55_TEXT=case \"$ACTION\" in\nWORKER_1_LINE_56_NO=658\nWORKER_1_LINE_56_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_1_LINE_57_NO=659\nWORKER_1_LINE_57_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_2_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_2_PATH_HINT=root/backup-registry-sha-pins-20260904T062507Z/usr/local/libexec/homelab-app-cloud-quorum-worker\nWORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_2_EXECUTABLE=true\nWORKER_2_PRESENT=true\nWORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_2_SHEBANG=#!/usr/bin/env bash\nWORKER_2_BASH_N_RC=0\nWORKER_2_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_2_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor\nWORKER_2_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_2_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_2_INTERESTING_LINE_COUNT=57\nWORKER_2_LINE_1_NO=4\nWORKER_2_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_2_LINE_2_NO=6\nWORKER_2_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_2_LINE_3_NO=7\nWORKER_2_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_2_LINE_4_NO=8\nWORKER_2_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_2_LINE_5_NO=9\nWORKER_2_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_2_LINE_6_NO=10\nWORKER_2_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_2_LINE_7_NO=12\nWORKER_2_LINE_7_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_2_LINE_8_NO=14\nWORKER_2_LINE_8_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_2_LINE_9_NO=75\nWORKER_2_LINE_9_TEXT=logical_id(){\nWORKER_2_LINE_10_NO=82\nWORKER_2_LINE_10_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_2_LINE_11_NO=83\nWORKER_2_LINE_11_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_2_LINE_12_NO=101\nWORKER_2_LINE_12_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_2_LINE_13_NO=103\nWORKER_2_LINE_13_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_2_LINE_14_NO=104\nWORKER_2_LINE_14_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_2_LINE_15_NO=115\nWORKER_2_LINE_15_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_2_LINE_16_NO=117\nWORKER_2_LINE_16_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_2_LINE_17_NO=118\nWORKER_2_LINE_17_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_2_LINE_18_NO=119\nWORKER_2_LINE_18_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_2_LINE_19_NO=126\nWORKER_2_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_2_LINE_20_NO=141\nWORKER_2_LINE_20_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_2_LINE_21_NO=160\nWORKER_2_LINE_21_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_22_NO=191\nWORKER_2_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_23_NO=202\nWORKER_2_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_2_LINE_24_NO=218\nWORKER_2_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_25_NO=239\nWORKER_2_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_26_NO=274\nWORKER_2_LINE_26_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_2_LINE_27_NO=275\nWORKER_2_LINE_27_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_2_LINE_28_NO=293\nWORKER_2_LINE_28_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_29_NO=341\nWORKER_2_LINE_29_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_2_LINE_30_NO=346\nWORKER_2_LINE_30_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_2_LINE_31_NO=349\nWORKER_2_LINE_31_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_2_LINE_32_NO=350\nWORKER_2_LINE_32_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_2_LINE_33_NO=353\nWORKER_2_LINE_33_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_2_LINE_34_NO=354\nWORKER_2_LINE_34_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_2_LINE_35_NO=362\nWORKER_2_LINE_35_TEXT= done <\"$manifest\"\nWORKER_2_LINE_36_NO=431\nWORKER_2_LINE_36_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_2_LINE_37_NO=432\nWORKER_2_LINE_37_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_2_LINE_38_NO=466\nWORKER_2_LINE_38_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_2_LINE_39_NO=471\nWORKER_2_LINE_39_TEXT= if \"manifest.json\" not in names:\nWORKER_2_LINE_40_NO=473\nWORKER_2_LINE_40_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_2_LINE_41_NO=475\nWORKER_2_LINE_41_TEXT= for row in manifest:\nWORKER_2_LINE_42_NO=516\nWORKER_2_LINE_42_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_2_LINE_43_NO=528\nWORKER_2_LINE_43_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_2_LINE_44_NO=530\nWORKER_2_LINE_44_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_2_LINE_45_NO=536\nWORKER_2_LINE_45_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_2_LINE_46_NO=539\nWORKER_2_LINE_46_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_2_LINE_47_NO=540\nWORKER_2_LINE_47_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_2_LINE_48_NO=546\nWORKER_2_LINE_48_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_2_LINE_49_NO=553\nWORKER_2_LINE_49_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_2_LINE_50_NO=560\nWORKER_2_LINE_50_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_2_LINE_51_NO=596\nWORKER_2_LINE_51_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_2_LINE_52_NO=598\nWORKER_2_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_2_LINE_53_NO=644\nWORKER_2_LINE_53_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_2_LINE_54_NO=655\nWORKER_2_LINE_54_TEXT=ACTION=\"${1:-}\"\nWORKER_2_LINE_55_NO=656\nWORKER_2_LINE_55_TEXT=case \"$ACTION\" in\nWORKER_2_LINE_56_NO=658\nWORKER_2_LINE_56_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_2_LINE_57_NO=659\nWORKER_2_LINE_57_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_3_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_3_PATH_HINT=root/homelab-ops-cr-2026-0024/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker\nWORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_3_EXECUTABLE=true\nWORKER_3_PRESENT=true\nWORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_3_SHEBANG=#!/usr/bin/env bash\nWORKER_3_BASH_N_RC=0\nWORKER_3_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_3_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_3_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work\nWORKER_3_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd\nWORKER_3_INTERESTING_LINE_COUNT=52\nWORKER_3_LINE_1_NO=4\nWORKER_3_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_3_LINE_2_NO=6\nWORKER_3_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_3_LINE_3_NO=7\nWORKER_3_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_3_LINE_4_NO=8\nWORKER_3_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_3_LINE_5_NO=9\nWORKER_3_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_3_LINE_6_NO=10\nWORKER_3_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_3_LINE_7_NO=45\nWORKER_3_LINE_7_TEXT=logical_id(){\nWORKER_3_LINE_8_NO=50\nWORKER_3_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_3_LINE_9_NO=64\nWORKER_3_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_3_LINE_10_NO=65\nWORKER_3_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_3_LINE_11_NO=83\nWORKER_3_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_3_LINE_12_NO=85\nWORKER_3_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_3_LINE_13_NO=86\nWORKER_3_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_3_LINE_14_NO=97\nWORKER_3_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_3_LINE_15_NO=99\nWORKER_3_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_3_LINE_16_NO=100\nWORKER_3_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_3_LINE_17_NO=101\nWORKER_3_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_3_LINE_18_NO=108\nWORKER_3_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_3_LINE_19_NO=123\nWORKER_3_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_3_LINE_20_NO=142\nWORKER_3_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_21_NO=150\nWORKER_3_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_3_LINE_22_NO=161\nWORKER_3_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_3_LINE_23_NO=192\nWORKER_3_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_24_NO=203\nWORKER_3_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_3_LINE_25_NO=219\nWORKER_3_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_26_NO=240\nWORKER_3_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_27_NO=256\nWORKER_3_LINE_27_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_3_LINE_28_NO=261\nWORKER_3_LINE_28_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_3_LINE_29_NO=264\nWORKER_3_LINE_29_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_3_LINE_30_NO=265\nWORKER_3_LINE_30_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_3_LINE_31_NO=268\nWORKER_3_LINE_31_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_3_LINE_32_NO=269\nWORKER_3_LINE_32_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_3_LINE_33_NO=277\nWORKER_3_LINE_33_TEXT= done <\"$manifest\"\nWORKER_3_LINE_34_NO=344\nWORKER_3_LINE_34_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_3_LINE_35_NO=349\nWORKER_3_LINE_35_TEXT= if \"manifest.json\" not in names:\nWORKER_3_LINE_36_NO=351\nWORKER_3_LINE_36_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_3_LINE_37_NO=353\nWORKER_3_LINE_37_TEXT= for row in manifest:\nWORKER_3_LINE_38_NO=394\nWORKER_3_LINE_38_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_3_LINE_39_NO=406\nWORKER_3_LINE_39_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_3_LINE_40_NO=408\nWORKER_3_LINE_40_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_3_LINE_41_NO=414\nWORKER_3_LINE_41_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_3_LINE_42_NO=417\nWORKER_3_LINE_42_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_3_LINE_43_NO=418\nWORKER_3_LINE_43_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_3_LINE_44_NO=424\nWORKER_3_LINE_44_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_3_LINE_45_NO=427\nWORKER_3_LINE_45_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_3_LINE_46_NO=434\nWORKER_3_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_3_LINE_47_NO=459\nWORKER_3_LINE_47_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_3_LINE_48_NO=461\nWORKER_3_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_3_LINE_49_NO=463\nWORKER_3_LINE_49_TEXT=ACTION=\"${1:-}\"\nWORKER_3_LINE_50_NO=464\nWORKER_3_LINE_50_TEXT=case \"$ACTION\" in\nWORKER_3_LINE_51_NO=466\nWORKER_3_LINE_51_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_3_LINE_52_NO=467\nWORKER_3_LINE_52_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_4_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_4_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker\nWORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_4_EXECUTABLE=true\nWORKER_4_PRESENT=true\nWORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_4_SHEBANG=#!/usr/bin/env bash\nWORKER_4_BASH_N_RC=0\nWORKER_4_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_4_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_4_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work\nWORKER_4_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd\nWORKER_4_INTERESTING_LINE_COUNT=52\nWORKER_4_LINE_1_NO=4\nWORKER_4_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_4_LINE_2_NO=6\nWORKER_4_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_4_LINE_3_NO=7\nWORKER_4_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_4_LINE_4_NO=8\nWORKER_4_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_4_LINE_5_NO=9\nWORKER_4_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_4_LINE_6_NO=10\nWORKER_4_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_4_LINE_7_NO=45\nWORKER_4_LINE_7_TEXT=logical_id(){\nWORKER_4_LINE_8_NO=50\nWORKER_4_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_4_LINE_9_NO=64\nWORKER_4_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_4_LINE_10_NO=65\nWORKER_4_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_4_LINE_11_NO=83\nWORKER_4_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_4_LINE_12_NO=85\nWORKER_4_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_4_LINE_13_NO=86\nWORKER_4_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_4_LINE_14_NO=97\nWORKER_4_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_4_LINE_15_NO=99\nWORKER_4_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_4_LINE_16_NO=100\nWORKER_4_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_4_LINE_17_NO=101\nWORKER_4_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_4_LINE_18_NO=108\nWORKER_4_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_4_LINE_19_NO=123\nWORKER_4_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_4_LINE_20_NO=142\nWORKER_4_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_21_NO=150\nWORKER_4_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_4_LINE_22_NO=161\nWORKER_4_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_4_LINE_23_NO=192\nWORKER_4_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_24_NO=203\nWORKER_4_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_4_LINE_25_NO=219\nWORKER_4_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_26_NO=240\nWORKER_4_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_27_NO=256\nWORKER_4_LINE_27_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_4_LINE_28_NO=261\nWORKER_4_LINE_28_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_4_LINE_29_NO=264\nWORKER_4_LINE_29_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_4_LINE_30_NO=265\nWORKER_4_LINE_30_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_4_LINE_31_NO=268\nWORKER_4_LINE_31_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_4_LINE_32_NO=269\nWORKER_4_LINE_32_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_4_LINE_33_NO=277\nWORKER_4_LINE_33_TEXT= done <\"$manifest\"\nWORKER_4_LINE_34_NO=344\nWORKER_4_LINE_34_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_4_LINE_35_NO=349\nWORKER_4_LINE_35_TEXT= if \"manifest.json\" not in names:\nWORKER_4_LINE_36_NO=351\nWORKER_4_LINE_36_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_4_LINE_37_NO=353\nWORKER_4_LINE_37_TEXT= for row in manifest:\nWORKER_4_LINE_38_NO=394\nWORKER_4_LINE_38_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_4_LINE_39_NO=406\nWORKER_4_LINE_39_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_4_LINE_40_NO=408\nWORKER_4_LINE_40_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_4_LINE_41_NO=414\nWORKER_4_LINE_41_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_4_LINE_42_NO=417\nWORKER_4_LINE_42_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_4_LINE_43_NO=418\nWORKER_4_LINE_43_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_4_LINE_44_NO=424\nWORKER_4_LINE_44_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_4_LINE_45_NO=427\nWORKER_4_LINE_45_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_4_LINE_46_NO=434\nWORKER_4_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_4_LINE_47_NO=459\nWORKER_4_LINE_47_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_4_LINE_48_NO=461\nWORKER_4_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_4_LINE_49_NO=463\nWORKER_4_LINE_49_TEXT=ACTION=\"${1:-}\"\nWORKER_4_LINE_50_NO=464\nWORKER_4_LINE_50_TEXT=case \"$ACTION\" in\nWORKER_4_LINE_51_NO=466\nWORKER_4_LINE_51_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_4_LINE_52_NO=467\nWORKER_4_LINE_52_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_5_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_5_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-scrutiny-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker\nWORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_5_EXECUTABLE=true\nWORKER_5_PRESENT=true\nWORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_5_SHEBANG=#!/usr/bin/env bash\nWORKER_5_BASH_N_RC=0\nWORKER_5_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_5_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_5_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_5_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_5_INTERESTING_LINE_COUNT=58\nWORKER_5_LINE_1_NO=4\nWORKER_5_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_5_LINE_2_NO=6\nWORKER_5_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_5_LINE_3_NO=7\nWORKER_5_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_5_LINE_4_NO=8\nWORKER_5_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_5_LINE_5_NO=9\nWORKER_5_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_5_LINE_6_NO=10\nWORKER_5_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_5_LINE_7_NO=69\nWORKER_5_LINE_7_TEXT=logical_id(){\nWORKER_5_LINE_8_NO=74\nWORKER_5_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_5_LINE_9_NO=90\nWORKER_5_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_5_LINE_10_NO=91\nWORKER_5_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_5_LINE_11_NO=109\nWORKER_5_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_5_LINE_12_NO=111\nWORKER_5_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_5_LINE_13_NO=112\nWORKER_5_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_5_LINE_14_NO=123\nWORKER_5_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_5_LINE_15_NO=125\nWORKER_5_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_5_LINE_16_NO=126\nWORKER_5_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_5_LINE_17_NO=127\nWORKER_5_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_5_LINE_18_NO=134\nWORKER_5_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_5_LINE_19_NO=149\nWORKER_5_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_5_LINE_20_NO=168\nWORKER_5_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_21_NO=176\nWORKER_5_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_5_LINE_22_NO=187\nWORKER_5_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_5_LINE_23_NO=218\nWORKER_5_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_24_NO=229\nWORKER_5_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_5_LINE_25_NO=245\nWORKER_5_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_26_NO=266\nWORKER_5_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_27_NO=301\nWORKER_5_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_5_LINE_28_NO=302\nWORKER_5_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_5_LINE_29_NO=320\nWORKER_5_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_30_NO=368\nWORKER_5_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_5_LINE_31_NO=373\nWORKER_5_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_5_LINE_32_NO=376\nWORKER_5_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_5_LINE_33_NO=377\nWORKER_5_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_5_LINE_34_NO=380\nWORKER_5_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_5_LINE_35_NO=381\nWORKER_5_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_5_LINE_36_NO=389\nWORKER_5_LINE_36_TEXT= done <\"$manifest\"\nWORKER_5_LINE_37_NO=464\nWORKER_5_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_5_LINE_38_NO=465\nWORKER_5_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_5_LINE_39_NO=499\nWORKER_5_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_5_LINE_40_NO=504\nWORKER_5_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_5_LINE_41_NO=506\nWORKER_5_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_5_LINE_42_NO=508\nWORKER_5_LINE_42_TEXT= for row in manifest:\nWORKER_5_LINE_43_NO=549\nWORKER_5_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_5_LINE_44_NO=561\nWORKER_5_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_5_LINE_45_NO=563\nWORKER_5_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_5_LINE_46_NO=569\nWORKER_5_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_5_LINE_47_NO=572\nWORKER_5_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_5_LINE_48_NO=573\nWORKER_5_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_5_LINE_49_NO=579\nWORKER_5_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_5_LINE_50_NO=586\nWORKER_5_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_5_LINE_51_NO=593\nWORKER_5_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_5_LINE_52_NO=630\nWORKER_5_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_5_LINE_53_NO=632\nWORKER_5_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_5_LINE_54_NO=678\nWORKER_5_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_5_LINE_55_NO=689\nWORKER_5_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_5_LINE_56_NO=690\nWORKER_5_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_5_LINE_57_NO=692\nWORKER_5_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_5_LINE_58_NO=693\nWORKER_5_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_6_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_6_PATH_HINT=homelab-ops-cr-2026-0029/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_6_EXECUTABLE=true\nWORKER_6_PRESENT=true\nWORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_6_SHEBANG=#!/usr/bin/env bash\nWORKER_6_BASH_N_RC=0\nWORKER_6_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_6_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_6_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_6_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_6_INTERESTING_LINE_COUNT=58\nWORKER_6_LINE_1_NO=4\nWORKER_6_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_6_LINE_2_NO=6\nWORKER_6_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_6_LINE_3_NO=7\nWORKER_6_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_6_LINE_4_NO=8\nWORKER_6_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_6_LINE_5_NO=9\nWORKER_6_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_6_LINE_6_NO=10\nWORKER_6_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_6_LINE_7_NO=69\nWORKER_6_LINE_7_TEXT=logical_id(){\nWORKER_6_LINE_8_NO=74\nWORKER_6_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_6_LINE_9_NO=90\nWORKER_6_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_6_LINE_10_NO=91\nWORKER_6_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_6_LINE_11_NO=109\nWORKER_6_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_6_LINE_12_NO=111\nWORKER_6_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_6_LINE_13_NO=112\nWORKER_6_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_6_LINE_14_NO=123\nWORKER_6_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_6_LINE_15_NO=125\nWORKER_6_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_6_LINE_16_NO=126\nWORKER_6_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_6_LINE_17_NO=127\nWORKER_6_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_6_LINE_18_NO=134\nWORKER_6_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_6_LINE_19_NO=149\nWORKER_6_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_6_LINE_20_NO=168\nWORKER_6_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_21_NO=176\nWORKER_6_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_6_LINE_22_NO=187\nWORKER_6_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_6_LINE_23_NO=218\nWORKER_6_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_24_NO=229\nWORKER_6_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_6_LINE_25_NO=245\nWORKER_6_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_26_NO=266\nWORKER_6_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_27_NO=301\nWORKER_6_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_6_LINE_28_NO=302\nWORKER_6_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_6_LINE_29_NO=320\nWORKER_6_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_30_NO=368\nWORKER_6_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_6_LINE_31_NO=373\nWORKER_6_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_6_LINE_32_NO=376\nWORKER_6_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_6_LINE_33_NO=377\nWORKER_6_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_6_LINE_34_NO=380\nWORKER_6_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_6_LINE_35_NO=381\nWORKER_6_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_6_LINE_36_NO=389\nWORKER_6_LINE_36_TEXT= done <\"$manifest\"\nWORKER_6_LINE_37_NO=464\nWORKER_6_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_6_LINE_38_NO=465\nWORKER_6_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_6_LINE_39_NO=499\nWORKER_6_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_6_LINE_40_NO=504\nWORKER_6_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_6_LINE_41_NO=506\nWORKER_6_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_6_LINE_42_NO=508\nWORKER_6_LINE_42_TEXT= for row in manifest:\nWORKER_6_LINE_43_NO=549\nWORKER_6_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_6_LINE_44_NO=561\nWORKER_6_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_6_LINE_45_NO=563\nWORKER_6_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_6_LINE_46_NO=569\nWORKER_6_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_6_LINE_47_NO=572\nWORKER_6_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_6_LINE_48_NO=573\nWORKER_6_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_6_LINE_49_NO=579\nWORKER_6_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_6_LINE_50_NO=586\nWORKER_6_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_6_LINE_51_NO=593\nWORKER_6_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_6_LINE_52_NO=630\nWORKER_6_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_6_LINE_53_NO=632\nWORKER_6_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_6_LINE_54_NO=678\nWORKER_6_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_6_LINE_55_NO=689\nWORKER_6_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_6_LINE_56_NO=690\nWORKER_6_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_6_LINE_57_NO=692\nWORKER_6_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_6_LINE_58_NO=693\nWORKER_6_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_7_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_7_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_7_EXECUTABLE=true\nWORKER_7_PRESENT=true\nWORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_7_SHEBANG=#!/usr/bin/env bash\nWORKER_7_BASH_N_RC=0\nWORKER_7_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_7_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_7_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_7_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_7_INTERESTING_LINE_COUNT=58\nWORKER_7_LINE_1_NO=4\nWORKER_7_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_7_LINE_2_NO=6\nWORKER_7_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_7_LINE_3_NO=7\nWORKER_7_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_7_LINE_4_NO=8\nWORKER_7_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_7_LINE_5_NO=9\nWORKER_7_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_7_LINE_6_NO=10\nWORKER_7_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_7_LINE_7_NO=69\nWORKER_7_LINE_7_TEXT=logical_id(){\nWORKER_7_LINE_8_NO=74\nWORKER_7_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_7_LINE_9_NO=90\nWORKER_7_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_7_LINE_10_NO=91\nWORKER_7_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_7_LINE_11_NO=109\nWORKER_7_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_7_LINE_12_NO=111\nWORKER_7_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_7_LINE_13_NO=112\nWORKER_7_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_7_LINE_14_NO=123\nWORKER_7_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_7_LINE_15_NO=125\nWORKER_7_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_7_LINE_16_NO=126\nWORKER_7_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_7_LINE_17_NO=127\nWORKER_7_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_7_LINE_18_NO=134\nWORKER_7_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_7_LINE_19_NO=149\nWORKER_7_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_7_LINE_20_NO=168\nWORKER_7_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_21_NO=176\nWORKER_7_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_7_LINE_22_NO=187\nWORKER_7_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_7_LINE_23_NO=218\nWORKER_7_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_24_NO=229\nWORKER_7_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_7_LINE_25_NO=245\nWORKER_7_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_26_NO=266\nWORKER_7_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_27_NO=301\nWORKER_7_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_7_LINE_28_NO=302\nWORKER_7_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_7_LINE_29_NO=320\nWORKER_7_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_30_NO=368\nWORKER_7_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_7_LINE_31_NO=373\nWORKER_7_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_7_LINE_32_NO=376\nWORKER_7_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_7_LINE_33_NO=377\nWORKER_7_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_7_LINE_34_NO=380\nWORKER_7_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_7_LINE_35_NO=381\nWORKER_7_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_7_LINE_36_NO=389\nWORKER_7_LINE_36_TEXT= done <\"$manifest\"\nWORKER_7_LINE_37_NO=464\nWORKER_7_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_7_LINE_38_NO=465\nWORKER_7_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_7_LINE_39_NO=499\nWORKER_7_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_7_LINE_40_NO=504\nWORKER_7_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_7_LINE_41_NO=506\nWORKER_7_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_7_LINE_42_NO=508\nWORKER_7_LINE_42_TEXT= for row in manifest:\nWORKER_7_LINE_43_NO=549\nWORKER_7_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_7_LINE_44_NO=561\nWORKER_7_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_7_LINE_45_NO=563\nWORKER_7_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_7_LINE_46_NO=569\nWORKER_7_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_7_LINE_47_NO=572\nWORKER_7_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_7_LINE_48_NO=573\nWORKER_7_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_7_LINE_49_NO=579\nWORKER_7_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_7_LINE_50_NO=586\nWORKER_7_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_7_LINE_51_NO=593\nWORKER_7_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_7_LINE_52_NO=630\nWORKER_7_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_7_LINE_53_NO=632\nWORKER_7_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_7_LINE_54_NO=678\nWORKER_7_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_7_LINE_55_NO=689\nWORKER_7_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_7_LINE_56_NO=690\nWORKER_7_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_7_LINE_57_NO=692\nWORKER_7_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_7_LINE_58_NO=693\nWORKER_7_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_8_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_8_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_8_EXECUTABLE=true\nWORKER_8_PRESENT=true\nWORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_8_SHEBANG=#!/usr/bin/env bash\nWORKER_8_BASH_N_RC=0\nWORKER_8_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_8_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_8_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_8_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_8_INTERESTING_LINE_COUNT=58\nWORKER_8_LINE_1_NO=4\nWORKER_8_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_8_LINE_2_NO=6\nWORKER_8_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_8_LINE_3_NO=7\nWORKER_8_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_8_LINE_4_NO=8\nWORKER_8_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_8_LINE_5_NO=9\nWORKER_8_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_8_LINE_6_NO=10\nWORKER_8_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_8_LINE_7_NO=69\nWORKER_8_LINE_7_TEXT=logical_id(){\nWORKER_8_LINE_8_NO=74\nWORKER_8_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_8_LINE_9_NO=90\nWORKER_8_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_8_LINE_10_NO=91\nWORKER_8_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_8_LINE_11_NO=109\nWORKER_8_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_8_LINE_12_NO=111\nWORKER_8_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_8_LINE_13_NO=112\nWORKER_8_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_8_LINE_14_NO=123\nWORKER_8_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_8_LINE_15_NO=125\nWORKER_8_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_8_LINE_16_NO=126\nWORKER_8_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_8_LINE_17_NO=127\nWORKER_8_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_8_LINE_18_NO=134\nWORKER_8_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_8_LINE_19_NO=149\nWORKER_8_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_8_LINE_20_NO=168\nWORKER_8_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_21_NO=176\nWORKER_8_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_8_LINE_22_NO=187\nWORKER_8_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_8_LINE_23_NO=218\nWORKER_8_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_24_NO=229\nWORKER_8_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_8_LINE_25_NO=245\nWORKER_8_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_26_NO=266\nWORKER_8_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_27_NO=301\nWORKER_8_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_8_LINE_28_NO=302\nWORKER_8_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_8_LINE_29_NO=320\nWORKER_8_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_30_NO=368\nWORKER_8_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_8_LINE_31_NO=373\nWORKER_8_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_8_LINE_32_NO=376\nWORKER_8_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_8_LINE_33_NO=377\nWORKER_8_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_8_LINE_34_NO=380\nWORKER_8_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_8_LINE_35_NO=381\nWORKER_8_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_8_LINE_36_NO=389\nWORKER_8_LINE_36_TEXT= done <\"$manifest\"\nWORKER_8_LINE_37_NO=464\nWORKER_8_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_8_LINE_38_NO=465\nWORKER_8_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_8_LINE_39_NO=499\nWORKER_8_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_8_LINE_40_NO=504\nWORKER_8_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_8_LINE_41_NO=506\nWORKER_8_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_8_LINE_42_NO=508\nWORKER_8_LINE_42_TEXT= for row in manifest:\nWORKER_8_LINE_43_NO=549\nWORKER_8_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_8_LINE_44_NO=561\nWORKER_8_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_8_LINE_45_NO=563\nWORKER_8_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_8_LINE_46_NO=569\nWORKER_8_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_8_LINE_47_NO=572\nWORKER_8_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_8_LINE_48_NO=573\nWORKER_8_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_8_LINE_49_NO=579\nWORKER_8_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_8_LINE_50_NO=586\nWORKER_8_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_8_LINE_51_NO=593\nWORKER_8_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_8_LINE_52_NO=630\nWORKER_8_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_8_LINE_53_NO=632\nWORKER_8_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_8_LINE_54_NO=678\nWORKER_8_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_8_LINE_55_NO=689\nWORKER_8_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_8_LINE_56_NO=690\nWORKER_8_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_8_LINE_57_NO=692\nWORKER_8_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_8_LINE_58_NO=693\nWORKER_8_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_9_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_9_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_9_EXECUTABLE=true\nWORKER_9_PRESENT=true\nWORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_9_SHEBANG=#!/usr/bin/env bash\nWORKER_9_BASH_N_RC=0\nWORKER_9_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_9_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_9_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_9_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_9_INTERESTING_LINE_COUNT=58\nWORKER_9_LINE_1_NO=4\nWORKER_9_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_9_LINE_2_NO=6\nWORKER_9_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_9_LINE_3_NO=7\nWORKER_9_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_9_LINE_4_NO=8\nWORKER_9_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_9_LINE_5_NO=9\nWORKER_9_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_9_LINE_6_NO=10\nWORKER_9_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_9_LINE_7_NO=69\nWORKER_9_LINE_7_TEXT=logical_id(){\nWORKER_9_LINE_8_NO=74\nWORKER_9_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_9_LINE_9_NO=90\nWORKER_9_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_9_LINE_10_NO=91\nWORKER_9_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_9_LINE_11_NO=109\nWORKER_9_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_9_LINE_12_NO=111\nWORKER_9_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_9_LINE_13_NO=112\nWORKER_9_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_9_LINE_14_NO=123\nWORKER_9_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_9_LINE_15_NO=125\nWORKER_9_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_9_LINE_16_NO=126\nWORKER_9_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_9_LINE_17_NO=127\nWORKER_9_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_9_LINE_18_NO=134\nWORKER_9_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_9_LINE_19_NO=149\nWORKER_9_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_9_LINE_20_NO=168\nWORKER_9_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_21_NO=176\nWORKER_9_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_9_LINE_22_NO=187\nWORKER_9_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_9_LINE_23_NO=218\nWORKER_9_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_24_NO=229\nWORKER_9_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_9_LINE_25_NO=245\nWORKER_9_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_26_NO=266\nWORKER_9_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_27_NO=301\nWORKER_9_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_9_LINE_28_NO=302\nWORKER_9_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_9_LINE_29_NO=320\nWORKER_9_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_30_NO=368\nWORKER_9_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_9_LINE_31_NO=373\nWORKER_9_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_9_LINE_32_NO=376\nWORKER_9_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_9_LINE_33_NO=377\nWORKER_9_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_9_LINE_34_NO=380\nWORKER_9_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_9_LINE_35_NO=381\nWORKER_9_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_9_LINE_36_NO=389\nWORKER_9_LINE_36_TEXT= done <\"$manifest\"\nWORKER_9_LINE_37_NO=464\nWORKER_9_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_9_LINE_38_NO=465\nWORKER_9_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_9_LINE_39_NO=499\nWORKER_9_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_9_LINE_40_NO=504\nWORKER_9_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_9_LINE_41_NO=506\nWORKER_9_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_9_LINE_42_NO=508\nWORKER_9_LINE_42_TEXT= for row in manifest:\nWORKER_9_LINE_43_NO=549\nWORKER_9_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_9_LINE_44_NO=561\nWORKER_9_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_9_LINE_45_NO=563\nWORKER_9_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_9_LINE_46_NO=569\nWORKER_9_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_9_LINE_47_NO=572\nWORKER_9_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_9_LINE_48_NO=573\nWORKER_9_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_9_LINE_49_NO=579\nWORKER_9_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_9_LINE_50_NO=586\nWORKER_9_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_9_LINE_51_NO=593\nWORKER_9_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_9_LINE_52_NO=630\nWORKER_9_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_9_LINE_53_NO=632\nWORKER_9_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_9_LINE_54_NO=678\nWORKER_9_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_9_LINE_55_NO=689\nWORKER_9_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_9_LINE_56_NO=690\nWORKER_9_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_9_LINE_57_NO=692\nWORKER_9_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_9_LINE_58_NO=693\nWORKER_9_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_10_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_10_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_10_EXECUTABLE=true\nWORKER_10_PRESENT=true\nWORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_10_SHEBANG=#!/usr/bin/env bash\nWORKER_10_BASH_N_RC=0\nWORKER_10_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_10_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_10_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_10_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_10_INTERESTING_LINE_COUNT=58\nWORKER_10_LINE_1_NO=4\nWORKER_10_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_10_LINE_2_NO=6\nWORKER_10_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_10_LINE_3_NO=7\nWORKER_10_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_10_LINE_4_NO=8\nWORKER_10_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_10_LINE_5_NO=9\nWORKER_10_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_10_LINE_6_NO=10\nWORKER_10_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_10_LINE_7_NO=69\nWORKER_10_LINE_7_TEXT=logical_id(){\nWORKER_10_LINE_8_NO=74\nWORKER_10_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_10_LINE_9_NO=90\nWORKER_10_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_10_LINE_10_NO=91\nWORKER_10_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_10_LINE_11_NO=109\nWORKER_10_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_10_LINE_12_NO=111\nWORKER_10_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_10_LINE_13_NO=112\nWORKER_10_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_10_LINE_14_NO=123\nWORKER_10_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_10_LINE_15_NO=125\nWORKER_10_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_10_LINE_16_NO=126\nWORKER_10_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_10_LINE_17_NO=127\nWORKER_10_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_10_LINE_18_NO=134\nWORKER_10_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_10_LINE_19_NO=149\nWORKER_10_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_10_LINE_20_NO=168\nWORKER_10_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_21_NO=176\nWORKER_10_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_10_LINE_22_NO=187\nWORKER_10_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_10_LINE_23_NO=218\nWORKER_10_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_24_NO=229\nWORKER_10_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_10_LINE_25_NO=245\nWORKER_10_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_26_NO=266\nWORKER_10_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_27_NO=301\nWORKER_10_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_10_LINE_28_NO=302\nWORKER_10_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_10_LINE_29_NO=320\nWORKER_10_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_30_NO=368\nWORKER_10_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_10_LINE_31_NO=373\nWORKER_10_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_10_LINE_32_NO=376\nWORKER_10_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_10_LINE_33_NO=377\nWORKER_10_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_10_LINE_34_NO=380\nWORKER_10_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_10_LINE_35_NO=381\nWORKER_10_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_10_LINE_36_NO=389\nWORKER_10_LINE_36_TEXT= done <\"$manifest\"\nWORKER_10_LINE_37_NO=464\nWORKER_10_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_10_LINE_38_NO=465\nWORKER_10_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_10_LINE_39_NO=499\nWORKER_10_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_10_LINE_40_NO=504\nWORKER_10_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_10_LINE_41_NO=506\nWORKER_10_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_10_LINE_42_NO=508\nWORKER_10_LINE_42_TEXT= for row in manifest:\nWORKER_10_LINE_43_NO=549\nWORKER_10_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_10_LINE_44_NO=561\nWORKER_10_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_10_LINE_45_NO=563\nWORKER_10_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_10_LINE_46_NO=569\nWORKER_10_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_10_LINE_47_NO=572\nWORKER_10_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_10_LINE_48_NO=573\nWORKER_10_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_10_LINE_49_NO=579\nWORKER_10_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_10_LINE_50_NO=586\nWORKER_10_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_10_LINE_51_NO=593\nWORKER_10_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_10_LINE_52_NO=630\nWORKER_10_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_10_LINE_53_NO=632\nWORKER_10_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_10_LINE_54_NO=678\nWORKER_10_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_10_LINE_55_NO=689\nWORKER_10_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_10_LINE_56_NO=690\nWORKER_10_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_10_LINE_57_NO=692\nWORKER_10_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_10_LINE_58_NO=693\nWORKER_10_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_11_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_11_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_11_EXECUTABLE=true\nWORKER_11_PRESENT=true\nWORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_11_SHEBANG=#!/usr/bin/env bash\nWORKER_11_BASH_N_RC=0\nWORKER_11_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_11_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_11_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_11_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_11_INTERESTING_LINE_COUNT=58\nWORKER_11_LINE_1_NO=4\nWORKER_11_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_11_LINE_2_NO=6\nWORKER_11_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_11_LINE_3_NO=7\nWORKER_11_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_11_LINE_4_NO=8\nWORKER_11_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_11_LINE_5_NO=9\nWORKER_11_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_11_LINE_6_NO=10\nWORKER_11_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_11_LINE_7_NO=69\nWORKER_11_LINE_7_TEXT=logical_id(){\nWORKER_11_LINE_8_NO=74\nWORKER_11_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_11_LINE_9_NO=90\nWORKER_11_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_11_LINE_10_NO=91\nWORKER_11_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_11_LINE_11_NO=109\nWORKER_11_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_11_LINE_12_NO=111\nWORKER_11_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_11_LINE_13_NO=112\nWORKER_11_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_11_LINE_14_NO=123\nWORKER_11_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_11_LINE_15_NO=125\nWORKER_11_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_11_LINE_16_NO=126\nWORKER_11_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_11_LINE_17_NO=127\nWORKER_11_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_11_LINE_18_NO=134\nWORKER_11_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_11_LINE_19_NO=149\nWORKER_11_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_11_LINE_20_NO=168\nWORKER_11_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_21_NO=176\nWORKER_11_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_11_LINE_22_NO=187\nWORKER_11_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_11_LINE_23_NO=218\nWORKER_11_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_24_NO=229\nWORKER_11_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_11_LINE_25_NO=245\nWORKER_11_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_26_NO=266\nWORKER_11_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_27_NO=301\nWORKER_11_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_11_LINE_28_NO=302\nWORKER_11_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_11_LINE_29_NO=320\nWORKER_11_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_30_NO=368\nWORKER_11_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_11_LINE_31_NO=373\nWORKER_11_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_11_LINE_32_NO=376\nWORKER_11_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_11_LINE_33_NO=377\nWORKER_11_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_11_LINE_34_NO=380\nWORKER_11_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_11_LINE_35_NO=381\nWORKER_11_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_11_LINE_36_NO=389\nWORKER_11_LINE_36_TEXT= done <\"$manifest\"\nWORKER_11_LINE_37_NO=464\nWORKER_11_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_11_LINE_38_NO=465\nWORKER_11_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_11_LINE_39_NO=499\nWORKER_11_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_11_LINE_40_NO=504\nWORKER_11_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_11_LINE_41_NO=506\nWORKER_11_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_11_LINE_42_NO=508\nWORKER_11_LINE_42_TEXT= for row in manifest:\nWORKER_11_LINE_43_NO=549\nWORKER_11_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_11_LINE_44_NO=561\nWORKER_11_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_11_LINE_45_NO=563\nWORKER_11_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_11_LINE_46_NO=569\nWORKER_11_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_11_LINE_47_NO=572\nWORKER_11_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_11_LINE_48_NO=573\nWORKER_11_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_11_LINE_49_NO=579\nWORKER_11_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_11_LINE_50_NO=586\nWORKER_11_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_11_LINE_51_NO=593\nWORKER_11_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_11_LINE_52_NO=630\nWORKER_11_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_11_LINE_53_NO=632\nWORKER_11_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_11_LINE_54_NO=678\nWORKER_11_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_11_LINE_55_NO=689\nWORKER_11_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_11_LINE_56_NO=690\nWORKER_11_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_11_LINE_57_NO=692\nWORKER_11_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_11_LINE_58_NO=693\nWORKER_11_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_12_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_12_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_12_EXECUTABLE=true\nWORKER_12_PRESENT=true\nWORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_12_SHEBANG=#!/usr/bin/env bash\nWORKER_12_BASH_N_RC=0\nWORKER_12_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_12_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_12_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_12_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_12_INTERESTING_LINE_COUNT=58\nWORKER_12_LINE_1_NO=4\nWORKER_12_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_12_LINE_2_NO=6\nWORKER_12_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_12_LINE_3_NO=7\nWORKER_12_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_12_LINE_4_NO=8\nWORKER_12_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_12_LINE_5_NO=9\nWORKER_12_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_12_LINE_6_NO=10\nWORKER_12_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_12_LINE_7_NO=69\nWORKER_12_LINE_7_TEXT=logical_id(){\nWORKER_12_LINE_8_NO=74\nWORKER_12_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_12_LINE_9_NO=90\nWORKER_12_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_12_LINE_10_NO=91\nWORKER_12_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_12_LINE_11_NO=109\nWORKER_12_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_12_LINE_12_NO=111\nWORKER_12_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_12_LINE_13_NO=112\nWORKER_12_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_12_LINE_14_NO=123\nWORKER_12_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_12_LINE_15_NO=125\nWORKER_12_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_12_LINE_16_NO=126\nWORKER_12_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_12_LINE_17_NO=127\nWORKER_12_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_12_LINE_18_NO=134\nWORKER_12_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_12_LINE_19_NO=149\nWORKER_12_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_12_LINE_20_NO=168\nWORKER_12_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_21_NO=176\nWORKER_12_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_12_LINE_22_NO=187\nWORKER_12_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_12_LINE_23_NO=218\nWORKER_12_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_24_NO=229\nWORKER_12_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_12_LINE_25_NO=245\nWORKER_12_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_26_NO=266\nWORKER_12_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_27_NO=301\nWORKER_12_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_12_LINE_28_NO=302\nWORKER_12_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_12_LINE_29_NO=320\nWORKER_12_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_30_NO=368\nWORKER_12_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_12_LINE_31_NO=373\nWORKER_12_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_12_LINE_32_NO=376\nWORKER_12_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_12_LINE_33_NO=377\nWORKER_12_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_12_LINE_34_NO=380\nWORKER_12_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_12_LINE_35_NO=381\nWORKER_12_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_12_LINE_36_NO=389\nWORKER_12_LINE_36_TEXT= done <\"$manifest\"\nWORKER_12_LINE_37_NO=464\nWORKER_12_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_12_LINE_38_NO=465\nWORKER_12_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_12_LINE_39_NO=499\nWORKER_12_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_12_LINE_40_NO=504\nWORKER_12_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_12_LINE_41_NO=506\nWORKER_12_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_12_LINE_42_NO=508\nWORKER_12_LINE_42_TEXT= for row in manifest:\nWORKER_12_LINE_43_NO=549\nWORKER_12_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_12_LINE_44_NO=561\nWORKER_12_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_12_LINE_45_NO=563\nWORKER_12_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_12_LINE_46_NO=569\nWORKER_12_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_12_LINE_47_NO=572\nWORKER_12_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_12_LINE_48_NO=573\nWORKER_12_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_12_LINE_49_NO=579\nWORKER_12_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_12_LINE_50_NO=586\nWORKER_12_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_12_LINE_51_NO=593\nWORKER_12_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_12_LINE_52_NO=630\nWORKER_12_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_12_LINE_53_NO=632\nWORKER_12_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_12_LINE_54_NO=678\nWORKER_12_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_12_LINE_55_NO=689\nWORKER_12_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_12_LINE_56_NO=690\nWORKER_12_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_12_LINE_57_NO=692\nWORKER_12_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_12_LINE_58_NO=693\nWORKER_12_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_13_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_13_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_13_EXECUTABLE=true\nWORKER_13_PRESENT=true\nWORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_13_SHEBANG=#!/usr/bin/env bash\nWORKER_13_BASH_N_RC=0\nWORKER_13_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_13_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_13_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_13_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_13_INTERESTING_LINE_COUNT=58\nWORKER_13_LINE_1_NO=4\nWORKER_13_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_13_LINE_2_NO=6\nWORKER_13_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_13_LINE_3_NO=7\nWORKER_13_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_13_LINE_4_NO=8\nWORKER_13_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_13_LINE_5_NO=9\nWORKER_13_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_13_LINE_6_NO=10\nWORKER_13_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_13_LINE_7_NO=69\nWORKER_13_LINE_7_TEXT=logical_id(){\nWORKER_13_LINE_8_NO=74\nWORKER_13_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_13_LINE_9_NO=90\nWORKER_13_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_13_LINE_10_NO=91\nWORKER_13_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_13_LINE_11_NO=109\nWORKER_13_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_13_LINE_12_NO=111\nWORKER_13_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_13_LINE_13_NO=112\nWORKER_13_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_13_LINE_14_NO=123\nWORKER_13_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_13_LINE_15_NO=125\nWORKER_13_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_13_LINE_16_NO=126\nWORKER_13_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_13_LINE_17_NO=127\nWORKER_13_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_13_LINE_18_NO=134\nWORKER_13_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_13_LINE_19_NO=149\nWORKER_13_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_13_LINE_20_NO=168\nWORKER_13_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_21_NO=176\nWORKER_13_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_13_LINE_22_NO=187\nWORKER_13_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_13_LINE_23_NO=218\nWORKER_13_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_24_NO=229\nWORKER_13_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_13_LINE_25_NO=245\nWORKER_13_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_26_NO=266\nWORKER_13_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_27_NO=301\nWORKER_13_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_13_LINE_28_NO=302\nWORKER_13_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_13_LINE_29_NO=320\nWORKER_13_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_30_NO=368\nWORKER_13_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_13_LINE_31_NO=373\nWORKER_13_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_13_LINE_32_NO=376\nWORKER_13_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_13_LINE_33_NO=377\nWORKER_13_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_13_LINE_34_NO=380\nWORKER_13_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_13_LINE_35_NO=381\nWORKER_13_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_13_LINE_36_NO=389\nWORKER_13_LINE_36_TEXT= done <\"$manifest\"\nWORKER_13_LINE_37_NO=464\nWORKER_13_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_13_LINE_38_NO=465\nWORKER_13_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_13_LINE_39_NO=499\nWORKER_13_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_13_LINE_40_NO=504\nWORKER_13_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_13_LINE_41_NO=506\nWORKER_13_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_13_LINE_42_NO=508\nWORKER_13_LINE_42_TEXT= for row in manifest:\nWORKER_13_LINE_43_NO=549\nWORKER_13_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_13_LINE_44_NO=561\nWORKER_13_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_13_LINE_45_NO=563\nWORKER_13_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_13_LINE_46_NO=569\nWORKER_13_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_13_LINE_47_NO=572\nWORKER_13_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_13_LINE_48_NO=573\nWORKER_13_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_13_LINE_49_NO=579\nWORKER_13_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_13_LINE_50_NO=586\nWORKER_13_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_13_LINE_51_NO=593\nWORKER_13_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_13_LINE_52_NO=630\nWORKER_13_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_13_LINE_53_NO=632\nWORKER_13_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_13_LINE_54_NO=678\nWORKER_13_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_13_LINE_55_NO=689\nWORKER_13_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_13_LINE_56_NO=690\nWORKER_13_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_13_LINE_57_NO=692\nWORKER_13_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_13_LINE_58_NO=693\nWORKER_13_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_14_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_14_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_14_EXECUTABLE=true\nWORKER_14_PRESENT=true\nWORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_14_SHEBANG=#!/usr/bin/env bash\nWORKER_14_BASH_N_RC=0\nWORKER_14_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_14_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_14_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_14_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_14_INTERESTING_LINE_COUNT=58\nWORKER_14_LINE_1_NO=4\nWORKER_14_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_14_LINE_2_NO=6\nWORKER_14_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_14_LINE_3_NO=7\nWORKER_14_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_14_LINE_4_NO=8\nWORKER_14_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_14_LINE_5_NO=9\nWORKER_14_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_14_LINE_6_NO=10\nWORKER_14_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_14_LINE_7_NO=69\nWORKER_14_LINE_7_TEXT=logical_id(){\nWORKER_14_LINE_8_NO=74\nWORKER_14_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_14_LINE_9_NO=90\nWORKER_14_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_14_LINE_10_NO=91\nWORKER_14_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_14_LINE_11_NO=109\nWORKER_14_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_14_LINE_12_NO=111\nWORKER_14_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_14_LINE_13_NO=112\nWORKER_14_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_14_LINE_14_NO=123\nWORKER_14_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_14_LINE_15_NO=125\nWORKER_14_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_14_LINE_16_NO=126\nWORKER_14_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_14_LINE_17_NO=127\nWORKER_14_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_14_LINE_18_NO=134\nWORKER_14_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_14_LINE_19_NO=149\nWORKER_14_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_14_LINE_20_NO=168\nWORKER_14_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_21_NO=176\nWORKER_14_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_14_LINE_22_NO=187\nWORKER_14_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_14_LINE_23_NO=218\nWORKER_14_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_24_NO=229\nWORKER_14_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_14_LINE_25_NO=245\nWORKER_14_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_26_NO=266\nWORKER_14_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_27_NO=301\nWORKER_14_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_14_LINE_28_NO=302\nWORKER_14_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_14_LINE_29_NO=320\nWORKER_14_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_30_NO=368\nWORKER_14_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_14_LINE_31_NO=373\nWORKER_14_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_14_LINE_32_NO=376\nWORKER_14_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_14_LINE_33_NO=377\nWORKER_14_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_14_LINE_34_NO=380\nWORKER_14_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_14_LINE_35_NO=381\nWORKER_14_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_14_LINE_36_NO=389\nWORKER_14_LINE_36_TEXT= done <\"$manifest\"\nWORKER_14_LINE_37_NO=464\nWORKER_14_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_14_LINE_38_NO=465\nWORKER_14_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_14_LINE_39_NO=499\nWORKER_14_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_14_LINE_40_NO=504\nWORKER_14_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_14_LINE_41_NO=506\nWORKER_14_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_14_LINE_42_NO=508\nWORKER_14_LINE_42_TEXT= for row in manifest:\nWORKER_14_LINE_43_NO=549\nWORKER_14_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_14_LINE_44_NO=561\nWORKER_14_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_14_LINE_45_NO=563\nWORKER_14_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_14_LINE_46_NO=569\nWORKER_14_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_14_LINE_47_NO=572\nWORKER_14_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_14_LINE_48_NO=573\nWORKER_14_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_14_LINE_49_NO=579\nWORKER_14_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_14_LINE_50_NO=586\nWORKER_14_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_14_LINE_51_NO=593\nWORKER_14_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_14_LINE_52_NO=630\nWORKER_14_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_14_LINE_53_NO=632\nWORKER_14_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_14_LINE_54_NO=678\nWORKER_14_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_14_LINE_55_NO=689\nWORKER_14_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_14_LINE_56_NO=690\nWORKER_14_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_14_LINE_57_NO=692\nWORKER_14_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_14_LINE_58_NO=693\nWORKER_14_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_15_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_15_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_15_EXECUTABLE=true\nWORKER_15_PRESENT=true\nWORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_15_SHEBANG=#!/usr/bin/env bash\nWORKER_15_BASH_N_RC=0\nWORKER_15_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_15_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_15_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_15_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_15_INTERESTING_LINE_COUNT=58\nWORKER_15_LINE_1_NO=4\nWORKER_15_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_15_LINE_2_NO=6\nWORKER_15_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_15_LINE_3_NO=7\nWORKER_15_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_15_LINE_4_NO=8\nWORKER_15_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_15_LINE_5_NO=9\nWORKER_15_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_15_LINE_6_NO=10\nWORKER_15_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_15_LINE_7_NO=69\nWORKER_15_LINE_7_TEXT=logical_id(){\nWORKER_15_LINE_8_NO=74\nWORKER_15_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_15_LINE_9_NO=90\nWORKER_15_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_15_LINE_10_NO=91\nWORKER_15_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_15_LINE_11_NO=109\nWORKER_15_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_15_LINE_12_NO=111\nWORKER_15_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_15_LINE_13_NO=112\nWORKER_15_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_15_LINE_14_NO=123\nWORKER_15_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_15_LINE_15_NO=125\nWORKER_15_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_15_LINE_16_NO=126\nWORKER_15_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_15_LINE_17_NO=127\nWORKER_15_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_15_LINE_18_NO=134\nWORKER_15_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_15_LINE_19_NO=149\nWORKER_15_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_15_LINE_20_NO=168\nWORKER_15_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_21_NO=176\nWORKER_15_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_15_LINE_22_NO=187\nWORKER_15_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_15_LINE_23_NO=218\nWORKER_15_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_24_NO=229\nWORKER_15_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_15_LINE_25_NO=245\nWORKER_15_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_26_NO=266\nWORKER_15_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_27_NO=301\nWORKER_15_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_15_LINE_28_NO=302\nWORKER_15_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_15_LINE_29_NO=320\nWORKER_15_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_30_NO=368\nWORKER_15_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_15_LINE_31_NO=373\nWORKER_15_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_15_LINE_32_NO=376\nWORKER_15_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_15_LINE_33_NO=377\nWORKER_15_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_15_LINE_34_NO=380\nWORKER_15_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_15_LINE_35_NO=381\nWORKER_15_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_15_LINE_36_NO=389\nWORKER_15_LINE_36_TEXT= done <\"$manifest\"\nWORKER_15_LINE_37_NO=464\nWORKER_15_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_15_LINE_38_NO=465\nWORKER_15_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_15_LINE_39_NO=499\nWORKER_15_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_15_LINE_40_NO=504\nWORKER_15_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_15_LINE_41_NO=506\nWORKER_15_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_15_LINE_42_NO=508\nWORKER_15_LINE_42_TEXT= for row in manifest:\nWORKER_15_LINE_43_NO=549\nWORKER_15_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_15_LINE_44_NO=561\nWORKER_15_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_15_LINE_45_NO=563\nWORKER_15_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_15_LINE_46_NO=569\nWORKER_15_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_15_LINE_47_NO=572\nWORKER_15_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_15_LINE_48_NO=573\nWORKER_15_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_15_LINE_49_NO=579\nWORKER_15_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_15_LINE_50_NO=586\nWORKER_15_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_15_LINE_51_NO=593\nWORKER_15_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_15_LINE_52_NO=630\nWORKER_15_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_15_LINE_53_NO=632\nWORKER_15_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_15_LINE_54_NO=678\nWORKER_15_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_15_LINE_55_NO=689\nWORKER_15_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_15_LINE_56_NO=690\nWORKER_15_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_15_LINE_57_NO=692\nWORKER_15_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_15_LINE_58_NO=693\nWORKER_15_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_16_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_16_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_16_EXECUTABLE=true\nWORKER_16_PRESENT=true\nWORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_16_SHEBANG=#!/usr/bin/env bash\nWORKER_16_BASH_N_RC=0\nWORKER_16_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_16_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_16_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_16_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_16_INTERESTING_LINE_COUNT=58\nWORKER_16_LINE_1_NO=4\nWORKER_16_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_16_LINE_2_NO=6\nWORKER_16_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_16_LINE_3_NO=7\nWORKER_16_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_16_LINE_4_NO=8\nWORKER_16_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_16_LINE_5_NO=9\nWORKER_16_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_16_LINE_6_NO=10\nWORKER_16_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_16_LINE_7_NO=69\nWORKER_16_LINE_7_TEXT=logical_id(){\nWORKER_16_LINE_8_NO=74\nWORKER_16_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_16_LINE_9_NO=90\nWORKER_16_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_16_LINE_10_NO=91\nWORKER_16_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_16_LINE_11_NO=109\nWORKER_16_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_16_LINE_12_NO=111\nWORKER_16_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_16_LINE_13_NO=112\nWORKER_16_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_16_LINE_14_NO=123\nWORKER_16_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_16_LINE_15_NO=125\nWORKER_16_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_16_LINE_16_NO=126\nWORKER_16_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_16_LINE_17_NO=127\nWORKER_16_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_16_LINE_18_NO=134\nWORKER_16_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_16_LINE_19_NO=149\nWORKER_16_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_16_LINE_20_NO=168\nWORKER_16_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_21_NO=176\nWORKER_16_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_16_LINE_22_NO=187\nWORKER_16_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_16_LINE_23_NO=218\nWORKER_16_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_24_NO=229\nWORKER_16_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_16_LINE_25_NO=245\nWORKER_16_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_26_NO=266\nWORKER_16_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_27_NO=301\nWORKER_16_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_16_LINE_28_NO=302\nWORKER_16_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_16_LINE_29_NO=320\nWORKER_16_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_30_NO=368\nWORKER_16_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_16_LINE_31_NO=373\nWORKER_16_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_16_LINE_32_NO=376\nWORKER_16_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_16_LINE_33_NO=377\nWORKER_16_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_16_LINE_34_NO=380\nWORKER_16_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_16_LINE_35_NO=381\nWORKER_16_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_16_LINE_36_NO=389\nWORKER_16_LINE_36_TEXT= done <\"$manifest\"\nWORKER_16_LINE_37_NO=464\nWORKER_16_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_16_LINE_38_NO=465\nWORKER_16_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_16_LINE_39_NO=499\nWORKER_16_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_16_LINE_40_NO=504\nWORKER_16_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_16_LINE_41_NO=506\nWORKER_16_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_16_LINE_42_NO=508\nWORKER_16_LINE_42_TEXT= for row in manifest:\nWORKER_16_LINE_43_NO=549\nWORKER_16_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_16_LINE_44_NO=561\nWORKER_16_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_16_LINE_45_NO=563\nWORKER_16_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_16_LINE_46_NO=569\nWORKER_16_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_16_LINE_47_NO=572\nWORKER_16_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_16_LINE_48_NO=573\nWORKER_16_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_16_LINE_49_NO=579\nWORKER_16_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_16_LINE_50_NO=586\nWORKER_16_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_16_LINE_51_NO=593\nWORKER_16_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_16_LINE_52_NO=630\nWORKER_16_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_16_LINE_53_NO=632\nWORKER_16_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_16_LINE_54_NO=678\nWORKER_16_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_16_LINE_55_NO=689\nWORKER_16_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_16_LINE_56_NO=690\nWORKER_16_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_16_LINE_57_NO=692\nWORKER_16_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_16_LINE_58_NO=693\nWORKER_16_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_17_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_17_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_17_EXECUTABLE=true\nWORKER_17_PRESENT=true\nWORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_17_SHEBANG=#!/usr/bin/env bash\nWORKER_17_BASH_N_RC=0\nWORKER_17_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_17_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_17_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_17_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_17_INTERESTING_LINE_COUNT=58\nWORKER_17_LINE_1_NO=4\nWORKER_17_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_17_LINE_2_NO=6\nWORKER_17_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_17_LINE_3_NO=7\nWORKER_17_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_17_LINE_4_NO=8\nWORKER_17_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_17_LINE_5_NO=9\nWORKER_17_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_17_LINE_6_NO=10\nWORKER_17_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_17_LINE_7_NO=69\nWORKER_17_LINE_7_TEXT=logical_id(){\nWORKER_17_LINE_8_NO=74\nWORKER_17_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_17_LINE_9_NO=90\nWORKER_17_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_17_LINE_10_NO=91\nWORKER_17_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_17_LINE_11_NO=109\nWORKER_17_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_17_LINE_12_NO=111\nWORKER_17_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_17_LINE_13_NO=112\nWORKER_17_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_17_LINE_14_NO=123\nWORKER_17_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_17_LINE_15_NO=125\nWORKER_17_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_17_LINE_16_NO=126\nWORKER_17_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_17_LINE_17_NO=127\nWORKER_17_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_17_LINE_18_NO=134\nWORKER_17_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_17_LINE_19_NO=149\nWORKER_17_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_17_LINE_20_NO=168\nWORKER_17_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_21_NO=176\nWORKER_17_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_17_LINE_22_NO=187\nWORKER_17_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_17_LINE_23_NO=218\nWORKER_17_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_24_NO=229\nWORKER_17_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_17_LINE_25_NO=245\nWORKER_17_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_26_NO=266\nWORKER_17_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_27_NO=301\nWORKER_17_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_17_LINE_28_NO=302\nWORKER_17_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_17_LINE_29_NO=320\nWORKER_17_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_30_NO=368\nWORKER_17_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_17_LINE_31_NO=373\nWORKER_17_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_17_LINE_32_NO=376\nWORKER_17_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_17_LINE_33_NO=377\nWORKER_17_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_17_LINE_34_NO=380\nWORKER_17_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_17_LINE_35_NO=381\nWORKER_17_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_17_LINE_36_NO=389\nWORKER_17_LINE_36_TEXT= done <\"$manifest\"\nWORKER_17_LINE_37_NO=464\nWORKER_17_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_17_LINE_38_NO=465\nWORKER_17_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_17_LINE_39_NO=499\nWORKER_17_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_17_LINE_40_NO=504\nWORKER_17_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_17_LINE_41_NO=506\nWORKER_17_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_17_LINE_42_NO=508\nWORKER_17_LINE_42_TEXT= for row in manifest:\nWORKER_17_LINE_43_NO=549\nWORKER_17_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_17_LINE_44_NO=561\nWORKER_17_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_17_LINE_45_NO=563\nWORKER_17_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_17_LINE_46_NO=569\nWORKER_17_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_17_LINE_47_NO=572\nWORKER_17_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_17_LINE_48_NO=573\nWORKER_17_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_17_LINE_49_NO=579\nWORKER_17_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_17_LINE_50_NO=586\nWORKER_17_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_17_LINE_51_NO=593\nWORKER_17_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_17_LINE_52_NO=630\nWORKER_17_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_17_LINE_53_NO=632\nWORKER_17_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_17_LINE_54_NO=678\nWORKER_17_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_17_LINE_55_NO=689\nWORKER_17_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_17_LINE_56_NO=690\nWORKER_17_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_17_LINE_57_NO=692\nWORKER_17_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_17_LINE_58_NO=693\nWORKER_17_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_18_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_18_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_18_EXECUTABLE=true\nWORKER_18_PRESENT=true\nWORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_18_SHEBANG=#!/usr/bin/env bash\nWORKER_18_BASH_N_RC=0\nWORKER_18_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_18_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_18_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_18_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_18_INTERESTING_LINE_COUNT=58\nWORKER_18_LINE_1_NO=4\nWORKER_18_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_18_LINE_2_NO=6\nWORKER_18_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_18_LINE_3_NO=7\nWORKER_18_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_18_LINE_4_NO=8\nWORKER_18_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_18_LINE_5_NO=9\nWORKER_18_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_18_LINE_6_NO=10\nWORKER_18_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_18_LINE_7_NO=69\nWORKER_18_LINE_7_TEXT=logical_id(){\nWORKER_18_LINE_8_NO=74\nWORKER_18_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_18_LINE_9_NO=90\nWORKER_18_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_18_LINE_10_NO=91\nWORKER_18_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_18_LINE_11_NO=109\nWORKER_18_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_18_LINE_12_NO=111\nWORKER_18_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_18_LINE_13_NO=112\nWORKER_18_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_18_LINE_14_NO=123\nWORKER_18_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_18_LINE_15_NO=125\nWORKER_18_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_18_LINE_16_NO=126\nWORKER_18_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_18_LINE_17_NO=127\nWORKER_18_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_18_LINE_18_NO=134\nWORKER_18_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_18_LINE_19_NO=149\nWORKER_18_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_18_LINE_20_NO=168\nWORKER_18_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_21_NO=176\nWORKER_18_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_18_LINE_22_NO=187\nWORKER_18_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_18_LINE_23_NO=218\nWORKER_18_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_24_NO=229\nWORKER_18_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_18_LINE_25_NO=245\nWORKER_18_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_26_NO=266\nWORKER_18_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_27_NO=301\nWORKER_18_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_18_LINE_28_NO=302\nWORKER_18_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_18_LINE_29_NO=320\nWORKER_18_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_30_NO=368\nWORKER_18_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_18_LINE_31_NO=373\nWORKER_18_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_18_LINE_32_NO=376\nWORKER_18_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_18_LINE_33_NO=377\nWORKER_18_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_18_LINE_34_NO=380\nWORKER_18_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_18_LINE_35_NO=381\nWORKER_18_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_18_LINE_36_NO=389\nWORKER_18_LINE_36_TEXT= done <\"$manifest\"\nWORKER_18_LINE_37_NO=464\nWORKER_18_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_18_LINE_38_NO=465\nWORKER_18_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_18_LINE_39_NO=499\nWORKER_18_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_18_LINE_40_NO=504\nWORKER_18_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_18_LINE_41_NO=506\nWORKER_18_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_18_LINE_42_NO=508\nWORKER_18_LINE_42_TEXT= for row in manifest:\nWORKER_18_LINE_43_NO=549\nWORKER_18_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_18_LINE_44_NO=561\nWORKER_18_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_18_LINE_45_NO=563\nWORKER_18_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_18_LINE_46_NO=569\nWORKER_18_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_18_LINE_47_NO=572\nWORKER_18_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_18_LINE_48_NO=573\nWORKER_18_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_18_LINE_49_NO=579\nWORKER_18_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_18_LINE_50_NO=586\nWORKER_18_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_18_LINE_51_NO=593\nWORKER_18_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_18_LINE_52_NO=630\nWORKER_18_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_18_LINE_53_NO=632\nWORKER_18_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_18_LINE_54_NO=678\nWORKER_18_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_18_LINE_55_NO=689\nWORKER_18_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_18_LINE_56_NO=690\nWORKER_18_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_18_LINE_57_NO=692\nWORKER_18_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_18_LINE_58_NO=693\nWORKER_18_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_19_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_19_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_19_EXECUTABLE=true\nWORKER_19_PRESENT=true\nWORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_19_SHEBANG=#!/usr/bin/env bash\nWORKER_19_BASH_N_RC=0\nWORKER_19_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_19_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_19_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_19_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_19_INTERESTING_LINE_COUNT=58\nWORKER_19_LINE_1_NO=4\nWORKER_19_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_19_LINE_2_NO=6\nWORKER_19_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_19_LINE_3_NO=7\nWORKER_19_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_19_LINE_4_NO=8\nWORKER_19_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_19_LINE_5_NO=9\nWORKER_19_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_19_LINE_6_NO=10\nWORKER_19_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_19_LINE_7_NO=69\nWORKER_19_LINE_7_TEXT=logical_id(){\nWORKER_19_LINE_8_NO=74\nWORKER_19_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_19_LINE_9_NO=90\nWORKER_19_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_19_LINE_10_NO=91\nWORKER_19_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_19_LINE_11_NO=109\nWORKER_19_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_19_LINE_12_NO=111\nWORKER_19_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_19_LINE_13_NO=112\nWORKER_19_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_19_LINE_14_NO=123\nWORKER_19_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_19_LINE_15_NO=125\nWORKER_19_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_19_LINE_16_NO=126\nWORKER_19_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_19_LINE_17_NO=127\nWORKER_19_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_19_LINE_18_NO=134\nWORKER_19_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_19_LINE_19_NO=149\nWORKER_19_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_19_LINE_20_NO=168\nWORKER_19_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_21_NO=176\nWORKER_19_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_19_LINE_22_NO=187\nWORKER_19_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_19_LINE_23_NO=218\nWORKER_19_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_24_NO=229\nWORKER_19_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_19_LINE_25_NO=245\nWORKER_19_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_26_NO=266\nWORKER_19_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_27_NO=301\nWORKER_19_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_19_LINE_28_NO=302\nWORKER_19_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_19_LINE_29_NO=320\nWORKER_19_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_30_NO=368\nWORKER_19_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_19_LINE_31_NO=373\nWORKER_19_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_19_LINE_32_NO=376\nWORKER_19_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_19_LINE_33_NO=377\nWORKER_19_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_19_LINE_34_NO=380\nWORKER_19_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_19_LINE_35_NO=381\nWORKER_19_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_19_LINE_36_NO=389\nWORKER_19_LINE_36_TEXT= done <\"$manifest\"\nWORKER_19_LINE_37_NO=464\nWORKER_19_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_19_LINE_38_NO=465\nWORKER_19_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_19_LINE_39_NO=499\nWORKER_19_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_19_LINE_40_NO=504\nWORKER_19_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_19_LINE_41_NO=506\nWORKER_19_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_19_LINE_42_NO=508\nWORKER_19_LINE_42_TEXT= for row in manifest:\nWORKER_19_LINE_43_NO=549\nWORKER_19_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_19_LINE_44_NO=561\nWORKER_19_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_19_LINE_45_NO=563\nWORKER_19_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_19_LINE_46_NO=569\nWORKER_19_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_19_LINE_47_NO=572\nWORKER_19_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_19_LINE_48_NO=573\nWORKER_19_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_19_LINE_49_NO=579\nWORKER_19_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_19_LINE_50_NO=586\nWORKER_19_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_19_LINE_51_NO=593\nWORKER_19_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_19_LINE_52_NO=630\nWORKER_19_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_19_LINE_53_NO=632\nWORKER_19_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_19_LINE_54_NO=678\nWORKER_19_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_19_LINE_55_NO=689\nWORKER_19_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_19_LINE_56_NO=690\nWORKER_19_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_19_LINE_57_NO=692\nWORKER_19_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_19_LINE_58_NO=693\nWORKER_19_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_20_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_20_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_20_EXECUTABLE=true\nWORKER_20_PRESENT=true\nWORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_20_SHEBANG=#!/usr/bin/env bash\nWORKER_20_BASH_N_RC=0\nWORKER_20_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_20_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_20_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_20_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_20_INTERESTING_LINE_COUNT=58\nWORKER_20_LINE_1_NO=4\nWORKER_20_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_20_LINE_2_NO=6\nWORKER_20_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_20_LINE_3_NO=7\nWORKER_20_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_20_LINE_4_NO=8\nWORKER_20_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_20_LINE_5_NO=9\nWORKER_20_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_20_LINE_6_NO=10\nWORKER_20_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_20_LINE_7_NO=69\nWORKER_20_LINE_7_TEXT=logical_id(){\nWORKER_20_LINE_8_NO=74\nWORKER_20_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_20_LINE_9_NO=90\nWORKER_20_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_20_LINE_10_NO=91\nWORKER_20_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_20_LINE_11_NO=109\nWORKER_20_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_20_LINE_12_NO=111\nWORKER_20_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_20_LINE_13_NO=112\nWORKER_20_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_20_LINE_14_NO=123\nWORKER_20_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_20_LINE_15_NO=125\nWORKER_20_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_20_LINE_16_NO=126\nWORKER_20_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_20_LINE_17_NO=127\nWORKER_20_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_20_LINE_18_NO=134\nWORKER_20_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_20_LINE_19_NO=149\nWORKER_20_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_20_LINE_20_NO=168\nWORKER_20_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_21_NO=176\nWORKER_20_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_20_LINE_22_NO=187\nWORKER_20_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_20_LINE_23_NO=218\nWORKER_20_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_24_NO=229\nWORKER_20_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_20_LINE_25_NO=245\nWORKER_20_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_26_NO=266\nWORKER_20_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_27_NO=301\nWORKER_20_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_20_LINE_28_NO=302\nWORKER_20_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_20_LINE_29_NO=320\nWORKER_20_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_30_NO=368\nWORKER_20_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_20_LINE_31_NO=373\nWORKER_20_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_20_LINE_32_NO=376\nWORKER_20_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_20_LINE_33_NO=377\nWORKER_20_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_20_LINE_34_NO=380\nWORKER_20_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_20_LINE_35_NO=381\nWORKER_20_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_20_LINE_36_NO=389\nWORKER_20_LINE_36_TEXT= done <\"$manifest\"\nWORKER_20_LINE_37_NO=464\nWORKER_20_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_20_LINE_38_NO=465\nWORKER_20_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_20_LINE_39_NO=499\nWORKER_20_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_20_LINE_40_NO=504\nWORKER_20_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_20_LINE_41_NO=506\nWORKER_20_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_20_LINE_42_NO=508\nWORKER_20_LINE_42_TEXT= for row in manifest:\nWORKER_20_LINE_43_NO=549\nWORKER_20_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_20_LINE_44_NO=561\nWORKER_20_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_20_LINE_45_NO=563\nWORKER_20_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_20_LINE_46_NO=569\nWORKER_20_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_20_LINE_47_NO=572\nWORKER_20_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_20_LINE_48_NO=573\nWORKER_20_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_20_LINE_49_NO=579\nWORKER_20_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_20_LINE_50_NO=586\nWORKER_20_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_20_LINE_51_NO=593\nWORKER_20_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_20_LINE_52_NO=630\nWORKER_20_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_20_LINE_53_NO=632\nWORKER_20_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_20_LINE_54_NO=678\nWORKER_20_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_20_LINE_55_NO=689\nWORKER_20_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_20_LINE_56_NO=690\nWORKER_20_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_20_LINE_57_NO=692\nWORKER_20_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_20_LINE_58_NO=693\nWORKER_20_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nTARGETS_PRESENT=true\nTARGETS_SHA=2f526330152d49461ff66f50adab57d1a195ea5d1b4c292945d523411b8a68a8\nTARGET_OBJECT_FOUND=true\nTARGET_KEYS=id,kind,lifecycle,logical_id,migrate,owner,source_host\nTARGET_ID=skladchik-reports-monitor\nTARGET_KIND=APP\nTARGET_LIFECYCLE=ACTIVE\nTARGET_LOGICAL_ID=app-skladchik-reports-monitor\nTARGET_MIGRATE=True\nTARGET_OWNER=pve01\nTARGET_SOURCE_HOST_BASENAME=[PRIVATE_IP]\nTARGET_SOURCE_HOST_SHA=3eb2319e39eac307b3c7bd11f67663c0a0c814fd603989f5965083f9d1c80fb0\nQUEUE_APPID_LITERAL_COUNT=0\nQUEUE_TARGETS_BASENAME_LITERAL_COUNT=0\nBACKUP_AUDIT_RC=0\nBACKUP_AUDIT_RELEVANT_COUNT=2\nBACKUP_AUDIT_RELEVANT_1=TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=22759|generation=app-skladchik-reports-monitor-pve01-20261005T003035Z\nBACKUP_AUDIT_RELEVANT_2=TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=32145|legacy_age_seconds=1627370|replica_count=2|state_id=vm170|generation=v2-vm170-20261004T214054Z-70a537ab4e91\nTARGETED_SEAL_DESIGN_READY=true\nMAIL_SENT_BY_COMMAND=false\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nDECISION=PASS_QUEUE_WORKER_RCA\nNEXT_GATE=DESIGN_ATOMIC_SOURCE_BACKUP_SEAL\nTASK_RESULT=SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 07f78fad..d8f9c957 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-261005-SNIKKET-OPS-GAPS-RECONCILE-99577R1 +COMMAND_ID=SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=snikket-ops-gaps-reconcile +COMPONENT=skladchik-durability-queue-worker-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=42c5c98e1e7e36e4af7ab23f644308660eed0d45271f7207290956264ade5a2a +COMMAND_SHA256=9786047247e48e71139ff52c618c59da24ce317cd7093ee00b699a69b178af52 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,125 +24,2736 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=b273ebd9e84019c6ed048e74e5e888c0818a877184fc6e455881b787ece315f4 -SANITIZED_OUTPUT_SHA256=b273ebd9e84019c6ed048e74e5e888c0818a877184fc6e455881b787ece315f4 +RAW_EVIDENCE_SHA256=df86e4a999ff180077eddd8c50c0647c83310a5062be61a88ccaab005bb577ea +SANITIZED_OUTPUT_SHA256=3aa1864388e614772490c9c29217a4ed56c483df64b4b67276683b5c5078e47e OUTPUT_BEGIN -SNIKKET_OPS_GAPS_RECONCILE_WRAPPER=V1 -MODE=read-only -NO_DNS_MUTATION=true -NO_VM_MUTATION=true -NO_CONTAINER_RESTART=true -NO_SERVICE_START=true +SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA=V1 +NO_MAIL_SEND=true +NO_SERVICE_RESTART=true +NO_TIMER_MUTATION=true +NO_NETWORK_MUTATION=true NO_SOURCE_MUTATION=true +NO_BACKUP_MUTATION=true +NO_GIT_PUSH=true +MODE=read-only HOST_PVE01=true -INGRESS_HOSTKEY_SELF_BOUND=true -VM150_INVENTORY_PRESENT=true -VM150_RUNNING=true -VM150_EXPECTED_NODE=true -VM130_INVENTORY_PRESENT=true -VM130_RUNNING=true -VM130_EXPECTED_NODE=true -VM150_ONBOOT=true -VM150_AGENT_CONFIG_LINE_PRESENT=true -VM150_QGA_PING=true -PVE_BACKUP_JOBS_QUERY_RC=0 -PVE_BACKUP_JOB_COVERAGE_COUNT=0 -PVE_BACKUP_JOB_ENABLED_COVERAGE_COUNT=0 -PVE_BACKUP_STORAGE_QUERY_RC=0 -PVE_BACKUP_STORAGE_COUNT=1 -PVE_BACKUP_STORAGE_TYPE_COUNT=1 -PVE_VM150_BACKUP_OBJECT_COUNT=0 -PVE_VM150_BACKUP_STORAGE_WITH_OBJECT_COUNT=0 -PVE_VM150_NEWEST_BACKUP_AGE_HOURS=-1 -PVE_VM150_BACKUP_WITHIN_48H=false -PVE_VM150_BACKUP_WITHIN_7D=false -HOMELAB_BACKUP_V2_TIMER_ACTIVE=active -HOMELAB_BACKUP_V2_TIMER_ENABLED=enabled -HOMELAB_BACKUP_V2_TIMER_RESULT=success -HOMELAB_BACKUP_V2_TIMER_EXEC_MAIN_STATUS= -HOMELAB_BACKUP_V2_SERVICE_ACTIVE=activating -HOMELAB_BACKUP_V2_SERVICE_ENABLED=static -HOMELAB_BACKUP_V2_SERVICE_RESULT=success -HOMELAB_BACKUP_V2_SERVICE_EXEC_MAIN_STATUS=0 -HOMELAB_BACKUP_AUDIT_TIMER_ACTIVE=active -HOMELAB_BACKUP_AUDIT_TIMER_ENABLED=enabled -HOMELAB_BACKUP_AUDIT_TIMER_RESULT=success -HOMELAB_BACKUP_AUDIT_TIMER_EXEC_MAIN_STATUS= -HOMELAB_BACKUP_AUDIT_SERVICE_ACTIVE=inactive -HOMELAB_BACKUP_AUDIT_SERVICE_ENABLED=static -HOMELAB_BACKUP_AUDIT_SERVICE_RESULT=success -HOMELAB_BACKUP_AUDIT_SERVICE_EXEC_MAIN_STATUS=0 -BACKUP_V2_14D_VM150_REF_COUNT=40 -BACKUP_V2_14D_VM150_SUCCESS_COUNT=40 -BACKUP_V2_14D_VM150_FAIL_COUNT=0 -BACKUP_AUDIT_30D_VM150_REF_COUNT=2379 -BACKUP_AUDIT_30D_VM150_RESTORE_REF_COUNT=764 -BACKUP_AUDIT_30D_VM150_PASS_REF_COUNT=763 -BACKUP_SOURCE_FILE_COUNT=135 -BACKUP_SOURCE_VM150_EXPLICIT_REF_FILE_COUNT=17 -BACKUP_SOURCE_DYNAMIC_VM_DISCOVERY_FILE_COUNT=46 -SOURCE_GIT_REPO_SCAN_COUNT=64 -SOURCE_RECON_EXACT_SHA_MATCH_COUNT=0 -SOURCE_RECON_CORRECT_LOGIC_TRACKED_COUNT=0 -SOURCE_DNS_ALL3_TRACKED_COUNT=343 -SOURCE_DNS_ALL3_CURRENT_TARGET_TRACKED_COUNT=294 -MONITOR_SOURCE_FILE_COUNT=86 -MONITOR_CHAT_DOMAIN_REF_FILE_COUNT=0 -MONITOR_ALL3_DOMAIN_REF_FILE_COUNT=0 -MONITOR_XMPP_5222_REF_FILE_COUNT=0 -MONITOR_XMPP_5269_REF_FILE_COUNT=0 -MONITOR_TURN_3479_REF_FILE_COUNT=0 -MONITOR_TURN_STUN_REF_FILE_COUNT=0 -MONITOR_CERT_EXPIRY_REF_FILE_COUNT=2 -HOMELAB_EXTERNAL_PROBE_VPS_HEALTH_TIMER_ACTIVE=active -HOMELAB_EXTERNAL_PROBE_VPS_HEALTH_TIMER_ENABLED=enabled -HOMELAB_KUMA_MONITOR_POLICY_TIMER_ACTIVE=active -HOMELAB_KUMA_MONITOR_POLICY_TIMER_ENABLED=enabled -HOMELAB_OVERALL_HEALTH_TIMER_ACTIVE=active -HOMELAB_OVERALL_HEALTH_TIMER_ENABLED=enabled -HOMELAB_ALERTING_HEALTH_TIMER_ACTIVE=active -HOMELAB_ALERTING_HEALTH_TIMER_ENABLED=enabled -PUBLIC_DNS_OK_COUNT=3 -PUBLIC_HTTPS_OK_COUNT=3 -PUBLIC_STUN_3479_PASS=true -PUBLIC_STUN_3479_CLASS=BINDING_SUCCESS -VM150_QGA_EXEC_INIT_RC=0 -VM150_QGA_POLL_RC=0 -VM150_GUEST_EXITCODE=0 -VM150_GUEST_OUTPUT_MARKER=true -SNIKKET_OPS_GAPS_GUEST=V1 -MODE=read-only -NO_DNS_MUTATION=true -NO_VM_MUTATION=true -NO_CONTAINER_RESTART=true -NO_SERVICE_START=true -NO_SOURCE_MUTATION=true -GUEST_HOSTNAME_SNIKKET=true -DOCKER_OK=true -CONTAINER_SNIKKET_PRESENT=true -CONTAINER_SNIKKET_RUNNING=true -CONTAINER_SNIKKET_IMAGE_ID=c392b78a2bd3447d314054caa75b3556cf4fc3579288599e2d0a15dbcd690972 -CONTAINER_SNIKKET_IMAGE_CREATED=2026-06-12T14:41:53. -PROSODY_ABOUT_PRESENT=true -PROSODY_ABOUT_SHA256=df326196b07e19c252252fd9213b8920957e8ff733d5b7a548ef1b004586ff38 -SNIKKET_UPDATE_CHECK_EFFECTIVE=true -SNIKKET_RETENTION_EXPLICIT=false -SNIKKET_UPLOAD_LIMIT_EXPLICIT=false -CERT_DAYS_REMAINING=-1 -ROOT_FS_USED_PERCENT=4 -GUEST_AUDIT_DONE=true -UPSTREAM_COMPOSE_FETCH_RC=0 -UPSTREAM_COMPOSE_PRESENT=true -UPSTREAM_COMPOSE_SHA256=5773abc3efc852e3d4082de5947b7e93acdf3708e416272f6edd7c3f00de69b8 -UPSTREAM_SNIKKET_IMAGE_LINE_COUNT=4 -OPS_GAPS_RECONCILE_COMPLETE=true -DECISION=ANALYZE_EXACT_99577_EVIDENCE -NEXT_GATE=APPLY_ONLY_PROVEN_OPERATIONAL_GAPS -TASK_RESULT=SNIKKET_OPS_GAPS_RECONCILE +QUEUE_VM170_QM_RC=0 +QUEUE_VM170_STDERR_PRESENT=false +QUEUE_VM170_JSON=true +QUEUE_VM170_GUEST_EXITCODE=0 +VM170_MONITOR_SHA=19123e359f2f7b9a2d373fd7e4464dd426d31b91cabc567c252e70cad40d77c8 +VM170_TIMER_ACTIVE=active +VM170_TIMER_ENABLED=enabled +VM170_SERVICE_RESULT=success +VM170_HEALTH_STATUS=OK +VM170_HEALTH_TS=2026-10-05T06:49:35Z +VM170_HEALTH_REPORTS=22 +VM170_HEALTH_OPEN=13 +VM170_HEALTH_RESOLVED=9 +VM170_BASELINE_OK=true +SUPERVISOR_SHA=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2 +SUPERVISOR_TOP_CONST_1_NAME=BACKUP_HEALTH_AGE_MAX +SUPERVISOR_TOP_CONST_1_EXPR_SHA=056f62a1cb138698ac25abded3e6493c5519d37a485bb3900a74bad1806f70b3 +SUPERVISOR_TOP_CONST_2_NAME=RESTORE_HEALTH_AGE_MAX +SUPERVISOR_TOP_CONST_2_EXPR_SHA=ce6a9ab58cc2e0db3f07ff3ef7d288b29b6611afff056f5b818c706075d9dbd5 +SUPERVISOR_TOP_CONST_3_NAME=APP_FRESHNESS_PROBE +SUPERVISOR_TOP_CONST_3_EXPR_SHA=d6593f7b36fcf7550285e4af7a5f5766c13c2ee0e0a992b808caaed52e54693a +SUPERVISOR_TOP_CONST_4_NAME=LEGACY_BACKUP_PROBLEMS +SUPERVISOR_TOP_CONST_4_EXPR_SHA=d84baf639fd4e63d9c728a793f4dc6c457b6bee5737789745170d2150d858ddd +SUPERVISOR_TOP_CONST_COUNT=4 +SUPERVISOR_TEST_RC=0 +SUPERVISOR_TEST_STDOUT_SHA=9b4438f36f4a40de86d41fd480c6900c28603079d96011398674df7637868c5e +SUPERVISOR_TEST_OUTPUT_BEGIN +SUPERVISOR_STATUS=OK +SUPERVISOR_PROBLEMS=none +MONITOR=OK reports=22 open=13 resolved=9 +CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z +REGISTRY_SOURCE= +> +NETBIRD_ROUTE= dev wt0 table 7120 src uid 0 +SUPERVISOR_TEST_OUTPUT_END +SUPTEST_SUPERVISOR_STATUS=OK +SUPTEST_SUPERVISOR_PROBLEMS=none +SUPTEST_MONITOR=OK reports=22 open=13 resolved=9 +SUPTEST_CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z +SUPTEST_REGISTRY_SOURCE=[PRIVATE_IP] +SUPTEST_WORKER_TARGET=debian@[PRIVATE_IP] +SUPTEST_NETBIRD_ROUTE=1.1.1.1 dev wt0 table 7120 src 100.125.187.138 uid 0 +QUEUE_PRESENT=true +QUEUE_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650 +QUEUE_SHEBANG=#!/usr/bin/env bash +QUEUE_BASH_N_RC=0 +QUEUE_FUNCTIONS=backup_list,die,dispatch_one,dispatch_plan,freshness_decision,freshness_plan,lock_heavy,migrate_list,plan,verify_all +QUEUE_ACTIONS=BACKUP_RESUME,BACKUP_UNVERIFIED,SEALED_PROVEN,SKIP_FRESH +QUEUE_VAR_NAMES=DUE_SECONDS,FRESHNESS_SECONDS,HEAVY_LOCK,PROBE,QUEUE_LOCK,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,WORKER,age,decision,freshness,generation,now_epoch,out,sealed_epoch +QUEUE_COMMAND_BASENAMES=flock,homelab-app-cloud-quorum-worker,sha256sum +QUEUE_INTERESTING_LINE_COUNT=32 +QUEUE_LINE_1_NO=4 +QUEUE_LINE_1_TEXT=WORKER=/usr/local/libexec/homelab-app-cloud-quorum-worker +QUEUE_LINE_2_NO=5 +QUEUE_LINE_2_TEXT=PROBE=/usr/local/libexec/homelab-app-cloud-quorum-freshness-probe +QUEUE_LINE_3_NO=6 +QUEUE_LINE_3_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +QUEUE_LINE_4_NO=8 +QUEUE_LINE_4_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +QUEUE_LINE_5_NO=9 +QUEUE_LINE_5_TEXT=QUEUE_LOCK=/run/lock/homelab-app-cloud-quorum-queue.lock +QUEUE_LINE_6_NO=10 +QUEUE_LINE_6_TEXT=HEAVY_LOCK=/run/homelab-backup-heavy.lock +QUEUE_LINE_7_NO=21 +QUEUE_LINE_7_TEXT=[[ -x "$WORKER" && ! -L "$WORKER" ]] || die WORKER_MISSING 69 +QUEUE_LINE_8_NO=29 +QUEUE_LINE_8_TEXT= for a in "${ALL_APPS[@]}"; do "$WORKER" plan "$a"; done +QUEUE_LINE_9_NO=32 +QUEUE_LINE_9_TEXT= local app="$1" now_epoch="$2" out decision generation sealed_epoch age +QUEUE_LINE_10_NO=38 +QUEUE_LINE_10_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +QUEUE_LINE_11_NO=39 +QUEUE_LINE_11_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_RESUME GENERATION=%s\n' "$app" "$generation" +QUEUE_LINE_12_NO=42 +QUEUE_LINE_12_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +QUEUE_LINE_13_NO=50 +QUEUE_LINE_13_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS" +QUEUE_LINE_14_NO=52 +QUEUE_LINE_14_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS" +QUEUE_LINE_15_NO=54 +QUEUE_LINE_15_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS" +QUEUE_LINE_16_NO=56 +QUEUE_LINE_16_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\n' "$app" "$generation" "$age" +QUEUE_LINE_17_NO=84 +QUEUE_LINE_17_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"; printf '%s\n' "$out" +QUEUE_LINE_18_NO=85 +QUEUE_LINE_18_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null +QUEUE_LINE_19_NO=93 +QUEUE_LINE_19_TEXT= local a="$1" freshness decision now_epoch out generation +QUEUE_LINE_20_NO=112 +QUEUE_LINE_20_TEXT= generation="$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$freshness" | tail -1)" +QUEUE_LINE_21_NO=113 +QUEUE_LINE_21_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SKIP_FRESH GENERATION=%s\n' "$a" "$generation" +QUEUE_LINE_22_NO=120 +QUEUE_LINE_22_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")" +QUEUE_LINE_23_NO=122 +QUEUE_LINE_23_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null +QUEUE_LINE_24_NO=123 +QUEUE_LINE_24_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" +QUEUE_LINE_25_NO=124 +QUEUE_LINE_25_TEXT= [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$a" 65 +QUEUE_LINE_26_NO=125 +QUEUE_LINE_26_TEXT= printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$a" "$generation" +QUEUE_LINE_27_NO=126 +QUEUE_LINE_27_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\n' "$a" "$generation" +QUEUE_LINE_28_NO=133 +QUEUE_LINE_28_TEXT= if out="$(exec 8>&-; exec 9>&-; "$WORKER" latest-sealed "$a" 2>/dev/null)"; then +QUEUE_LINE_29_NO=138 +QUEUE_LINE_29_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"; printf '%s\n' "$out" +QUEUE_LINE_30_NO=139 +QUEUE_LINE_30_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null +QUEUE_LINE_31_NO=145 +QUEUE_LINE_31_TEXT= for a in "${ALL_APPS[@]}"; do out="$("$WORKER" latest-sealed "$a")"; printf '%s\n' "$out"; done +QUEUE_LINE_32_NO=153 +QUEUE_LINE_32_TEXT= all) [[ "$#" == 1 ]] || die USAGE_ALL 64; [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die DISPATCHER_MISSING 69; /usr/local/sbin/homelab-backup-dispatcher run APP_TIMER ;; +QUEUE_SERVICE_SHOW_RC=0 +QUEUE_SERVICE_LOADSTATE=loaded +QUEUE_SERVICE_ACTIVESTATE=inactive +QUEUE_SERVICE_UNITFILESTATE=static +QUEUE_SERVICE_RESULT=success +QUEUE_SERVICE_EXECMAINSTATUS=0 +QUEUE_SERVICE_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.service +QUEUE_SERVICE_FRAGMENT_SHA=afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a +QUEUE_SERVICE_EXECSTART_SHA=22094bb14d8dc5f7399744e183b096583a9c9625527a2f00981f2ec5cd8e1713 +QUEUE_SERVICE_EXEC_BASENAMES=homelab-backup-v2-app-scheduled +QUEUE_SERVICE_EXEC_ACTIONS=NONE +QUEUE_TIMER_SHOW_RC=0 +QUEUE_TIMER_LOADSTATE=loaded +QUEUE_TIMER_ACTIVESTATE=active +QUEUE_TIMER_UNITFILESTATE=enabled +QUEUE_TIMER_RESULT=success +QUEUE_TIMER_EXECMAINSTATUS= +QUEUE_TIMER_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.timer +QUEUE_TIMER_FRAGMENT_SHA=af2eedce3df53029b591d31715a3f3e6688ddb997b4823f487e70c339ce88078 +QUEUE_TIMER_EXECSTART_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +QUEUE_TIMER_EXEC_BASENAMES=NONE +QUEUE_TIMER_EXEC_ACTIONS=NONE +WORKER_CANDIDATE_COUNT=173 +WORKER_1_BASENAME=homelab-app-cloud-quorum-worker +WORKER_1_PATH_HINT=//root/BACKUP_closeout_evidence_20260825T165406Z/source/homelab-app-cloud-quorum-worker +WORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 +WORKER_1_EXECUTABLE=false +WORKER_1_PRESENT=true +WORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 +WORKER_1_SHEBANG=#!/usr/bin/env bash +WORKER_1_BASH_N_RC=0 +WORKER_1_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_1_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor +WORKER_1_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_1_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_1_INTERESTING_LINE_COUNT=57 +WORKER_1_LINE_1_NO=4 +WORKER_1_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_1_LINE_2_NO=6 +WORKER_1_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_1_LINE_3_NO=7 +WORKER_1_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_1_LINE_4_NO=8 +WORKER_1_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_1_LINE_5_NO=9 +WORKER_1_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_1_LINE_6_NO=10 +WORKER_1_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_1_LINE_7_NO=12 +WORKER_1_LINE_7_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +WORKER_1_LINE_8_NO=14 +WORKER_1_LINE_8_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +WORKER_1_LINE_9_NO=75 +WORKER_1_LINE_9_TEXT=logical_id(){ +WORKER_1_LINE_10_NO=82 +WORKER_1_LINE_10_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_1_LINE_11_NO=83 +WORKER_1_LINE_11_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_1_LINE_12_NO=101 +WORKER_1_LINE_12_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_1_LINE_13_NO=103 +WORKER_1_LINE_13_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_1_LINE_14_NO=104 +WORKER_1_LINE_14_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_1_LINE_15_NO=115 +WORKER_1_LINE_15_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_1_LINE_16_NO=117 +WORKER_1_LINE_16_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_1_LINE_17_NO=118 +WORKER_1_LINE_17_TEXT= if [[ -s "$manifest" ]]; then +WORKER_1_LINE_18_NO=119 +WORKER_1_LINE_18_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_1_LINE_19_NO=126 +WORKER_1_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_1_LINE_20_NO=141 +WORKER_1_LINE_20_TEXT= logical="$(logical_id "$app")" +WORKER_1_LINE_21_NO=160 +WORKER_1_LINE_21_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_1_LINE_22_NO=191 +WORKER_1_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_1_LINE_23_NO=202 +WORKER_1_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_1_LINE_24_NO=218 +WORKER_1_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_1_LINE_25_NO=239 +WORKER_1_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_1_LINE_26_NO=274 +WORKER_1_LINE_26_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_1_LINE_27_NO=275 +WORKER_1_LINE_27_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_1_LINE_28_NO=293 +WORKER_1_LINE_28_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_1_LINE_29_NO=341 +WORKER_1_LINE_29_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_1_LINE_30_NO=346 +WORKER_1_LINE_30_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_1_LINE_31_NO=349 +WORKER_1_LINE_31_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_1_LINE_32_NO=350 +WORKER_1_LINE_32_TEXT= logical="$(logical_id "$app")" +WORKER_1_LINE_33_NO=353 +WORKER_1_LINE_33_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_1_LINE_34_NO=354 +WORKER_1_LINE_34_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_1_LINE_35_NO=362 +WORKER_1_LINE_35_TEXT= done <"$manifest" +WORKER_1_LINE_36_NO=431 +WORKER_1_LINE_36_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_1_LINE_37_NO=432 +WORKER_1_LINE_37_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_1_LINE_38_NO=466 +WORKER_1_LINE_38_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_1_LINE_39_NO=471 +WORKER_1_LINE_39_TEXT= if "manifest.json" not in names: +WORKER_1_LINE_40_NO=473 +WORKER_1_LINE_40_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_1_LINE_41_NO=475 +WORKER_1_LINE_41_TEXT= for row in manifest: +WORKER_1_LINE_42_NO=516 +WORKER_1_LINE_42_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_1_LINE_43_NO=528 +WORKER_1_LINE_43_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_1_LINE_44_NO=530 +WORKER_1_LINE_44_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_1_LINE_45_NO=536 +WORKER_1_LINE_45_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_1_LINE_46_NO=539 +WORKER_1_LINE_46_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_1_LINE_47_NO=540 +WORKER_1_LINE_47_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_1_LINE_48_NO=546 +WORKER_1_LINE_48_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_1_LINE_49_NO=553 +WORKER_1_LINE_49_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_1_LINE_50_NO=560 +WORKER_1_LINE_50_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_1_LINE_51_NO=596 +WORKER_1_LINE_51_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_1_LINE_52_NO=598 +WORKER_1_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_1_LINE_53_NO=644 +WORKER_1_LINE_53_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_1_LINE_54_NO=655 +WORKER_1_LINE_54_TEXT=ACTION="${1:-}" +WORKER_1_LINE_55_NO=656 +WORKER_1_LINE_55_TEXT=case "$ACTION" in +WORKER_1_LINE_56_NO=658 +WORKER_1_LINE_56_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_1_LINE_57_NO=659 +WORKER_1_LINE_57_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_2_BASENAME=homelab-app-cloud-quorum-worker +WORKER_2_PATH_HINT=root/backup-registry-sha-pins-20260904T062507Z/usr/local/libexec/homelab-app-cloud-quorum-worker +WORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 +WORKER_2_EXECUTABLE=true +WORKER_2_PRESENT=true +WORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 +WORKER_2_SHEBANG=#!/usr/bin/env bash +WORKER_2_BASH_N_RC=0 +WORKER_2_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_2_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor +WORKER_2_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_2_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_2_INTERESTING_LINE_COUNT=57 +WORKER_2_LINE_1_NO=4 +WORKER_2_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_2_LINE_2_NO=6 +WORKER_2_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_2_LINE_3_NO=7 +WORKER_2_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_2_LINE_4_NO=8 +WORKER_2_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_2_LINE_5_NO=9 +WORKER_2_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_2_LINE_6_NO=10 +WORKER_2_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_2_LINE_7_NO=12 +WORKER_2_LINE_7_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +WORKER_2_LINE_8_NO=14 +WORKER_2_LINE_8_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" +WORKER_2_LINE_9_NO=75 +WORKER_2_LINE_9_TEXT=logical_id(){ +WORKER_2_LINE_10_NO=82 +WORKER_2_LINE_10_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_2_LINE_11_NO=83 +WORKER_2_LINE_11_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_2_LINE_12_NO=101 +WORKER_2_LINE_12_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_2_LINE_13_NO=103 +WORKER_2_LINE_13_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_2_LINE_14_NO=104 +WORKER_2_LINE_14_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_2_LINE_15_NO=115 +WORKER_2_LINE_15_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_2_LINE_16_NO=117 +WORKER_2_LINE_16_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_2_LINE_17_NO=118 +WORKER_2_LINE_17_TEXT= if [[ -s "$manifest" ]]; then +WORKER_2_LINE_18_NO=119 +WORKER_2_LINE_18_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_2_LINE_19_NO=126 +WORKER_2_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_2_LINE_20_NO=141 +WORKER_2_LINE_20_TEXT= logical="$(logical_id "$app")" +WORKER_2_LINE_21_NO=160 +WORKER_2_LINE_21_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_2_LINE_22_NO=191 +WORKER_2_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_2_LINE_23_NO=202 +WORKER_2_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_2_LINE_24_NO=218 +WORKER_2_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_2_LINE_25_NO=239 +WORKER_2_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_2_LINE_26_NO=274 +WORKER_2_LINE_26_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_2_LINE_27_NO=275 +WORKER_2_LINE_27_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_2_LINE_28_NO=293 +WORKER_2_LINE_28_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_2_LINE_29_NO=341 +WORKER_2_LINE_29_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_2_LINE_30_NO=346 +WORKER_2_LINE_30_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_2_LINE_31_NO=349 +WORKER_2_LINE_31_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_2_LINE_32_NO=350 +WORKER_2_LINE_32_TEXT= logical="$(logical_id "$app")" +WORKER_2_LINE_33_NO=353 +WORKER_2_LINE_33_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_2_LINE_34_NO=354 +WORKER_2_LINE_34_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_2_LINE_35_NO=362 +WORKER_2_LINE_35_TEXT= done <"$manifest" +WORKER_2_LINE_36_NO=431 +WORKER_2_LINE_36_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_2_LINE_37_NO=432 +WORKER_2_LINE_37_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_2_LINE_38_NO=466 +WORKER_2_LINE_38_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_2_LINE_39_NO=471 +WORKER_2_LINE_39_TEXT= if "manifest.json" not in names: +WORKER_2_LINE_40_NO=473 +WORKER_2_LINE_40_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_2_LINE_41_NO=475 +WORKER_2_LINE_41_TEXT= for row in manifest: +WORKER_2_LINE_42_NO=516 +WORKER_2_LINE_42_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_2_LINE_43_NO=528 +WORKER_2_LINE_43_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_2_LINE_44_NO=530 +WORKER_2_LINE_44_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_2_LINE_45_NO=536 +WORKER_2_LINE_45_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_2_LINE_46_NO=539 +WORKER_2_LINE_46_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_2_LINE_47_NO=540 +WORKER_2_LINE_47_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_2_LINE_48_NO=546 +WORKER_2_LINE_48_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_2_LINE_49_NO=553 +WORKER_2_LINE_49_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_2_LINE_50_NO=560 +WORKER_2_LINE_50_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_2_LINE_51_NO=596 +WORKER_2_LINE_51_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_2_LINE_52_NO=598 +WORKER_2_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_2_LINE_53_NO=644 +WORKER_2_LINE_53_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_2_LINE_54_NO=655 +WORKER_2_LINE_54_TEXT=ACTION="${1:-}" +WORKER_2_LINE_55_NO=656 +WORKER_2_LINE_55_TEXT=case "$ACTION" in +WORKER_2_LINE_56_NO=658 +WORKER_2_LINE_56_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_2_LINE_57_NO=659 +WORKER_2_LINE_57_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_3_BASENAME=homelab-app-cloud-quorum-worker +WORKER_3_PATH_HINT=root/homelab-ops-cr-2026-0024/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker +WORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 +WORKER_3_EXECUTABLE=true +WORKER_3_PRESENT=true +WORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 +WORKER_3_SHEBANG=#!/usr/bin/env bash +WORKER_3_BASH_N_RC=0 +WORKER_3_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_3_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_3_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work +WORKER_3_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd +WORKER_3_INTERESTING_LINE_COUNT=52 +WORKER_3_LINE_1_NO=4 +WORKER_3_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_3_LINE_2_NO=6 +WORKER_3_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_3_LINE_3_NO=7 +WORKER_3_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_3_LINE_4_NO=8 +WORKER_3_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_3_LINE_5_NO=9 +WORKER_3_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_3_LINE_6_NO=10 +WORKER_3_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_3_LINE_7_NO=45 +WORKER_3_LINE_7_TEXT=logical_id(){ +WORKER_3_LINE_8_NO=50 +WORKER_3_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_3_LINE_9_NO=64 +WORKER_3_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_3_LINE_10_NO=65 +WORKER_3_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_3_LINE_11_NO=83 +WORKER_3_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_3_LINE_12_NO=85 +WORKER_3_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_3_LINE_13_NO=86 +WORKER_3_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_3_LINE_14_NO=97 +WORKER_3_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_3_LINE_15_NO=99 +WORKER_3_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_3_LINE_16_NO=100 +WORKER_3_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_3_LINE_17_NO=101 +WORKER_3_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_3_LINE_18_NO=108 +WORKER_3_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_3_LINE_19_NO=123 +WORKER_3_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_3_LINE_20_NO=142 +WORKER_3_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_3_LINE_21_NO=150 +WORKER_3_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_3_LINE_22_NO=161 +WORKER_3_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_3_LINE_23_NO=192 +WORKER_3_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_3_LINE_24_NO=203 +WORKER_3_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_3_LINE_25_NO=219 +WORKER_3_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_3_LINE_26_NO=240 +WORKER_3_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_3_LINE_27_NO=256 +WORKER_3_LINE_27_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_3_LINE_28_NO=261 +WORKER_3_LINE_28_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_3_LINE_29_NO=264 +WORKER_3_LINE_29_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_3_LINE_30_NO=265 +WORKER_3_LINE_30_TEXT= logical="$(logical_id "$app")" +WORKER_3_LINE_31_NO=268 +WORKER_3_LINE_31_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_3_LINE_32_NO=269 +WORKER_3_LINE_32_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_3_LINE_33_NO=277 +WORKER_3_LINE_33_TEXT= done <"$manifest" +WORKER_3_LINE_34_NO=344 +WORKER_3_LINE_34_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_3_LINE_35_NO=349 +WORKER_3_LINE_35_TEXT= if "manifest.json" not in names: +WORKER_3_LINE_36_NO=351 +WORKER_3_LINE_36_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_3_LINE_37_NO=353 +WORKER_3_LINE_37_TEXT= for row in manifest: +WORKER_3_LINE_38_NO=394 +WORKER_3_LINE_38_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_3_LINE_39_NO=406 +WORKER_3_LINE_39_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_3_LINE_40_NO=408 +WORKER_3_LINE_40_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_3_LINE_41_NO=414 +WORKER_3_LINE_41_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_3_LINE_42_NO=417 +WORKER_3_LINE_42_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_3_LINE_43_NO=418 +WORKER_3_LINE_43_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_3_LINE_44_NO=424 +WORKER_3_LINE_44_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_3_LINE_45_NO=427 +WORKER_3_LINE_45_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_3_LINE_46_NO=434 +WORKER_3_LINE_46_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_3_LINE_47_NO=459 +WORKER_3_LINE_47_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_3_LINE_48_NO=461 +WORKER_3_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_3_LINE_49_NO=463 +WORKER_3_LINE_49_TEXT=ACTION="${1:-}" +WORKER_3_LINE_50_NO=464 +WORKER_3_LINE_50_TEXT=case "$ACTION" in +WORKER_3_LINE_51_NO=466 +WORKER_3_LINE_51_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_3_LINE_52_NO=467 +WORKER_3_LINE_52_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_4_BASENAME=homelab-app-cloud-quorum-worker +WORKER_4_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker +WORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 +WORKER_4_EXECUTABLE=true +WORKER_4_PRESENT=true +WORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 +WORKER_4_SHEBANG=#!/usr/bin/env bash +WORKER_4_BASH_N_RC=0 +WORKER_4_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_4_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_4_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work +WORKER_4_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd +WORKER_4_INTERESTING_LINE_COUNT=52 +WORKER_4_LINE_1_NO=4 +WORKER_4_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_4_LINE_2_NO=6 +WORKER_4_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_4_LINE_3_NO=7 +WORKER_4_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_4_LINE_4_NO=8 +WORKER_4_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_4_LINE_5_NO=9 +WORKER_4_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_4_LINE_6_NO=10 +WORKER_4_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_4_LINE_7_NO=45 +WORKER_4_LINE_7_TEXT=logical_id(){ +WORKER_4_LINE_8_NO=50 +WORKER_4_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_4_LINE_9_NO=64 +WORKER_4_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_4_LINE_10_NO=65 +WORKER_4_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_4_LINE_11_NO=83 +WORKER_4_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_4_LINE_12_NO=85 +WORKER_4_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_4_LINE_13_NO=86 +WORKER_4_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_4_LINE_14_NO=97 +WORKER_4_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_4_LINE_15_NO=99 +WORKER_4_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_4_LINE_16_NO=100 +WORKER_4_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_4_LINE_17_NO=101 +WORKER_4_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_4_LINE_18_NO=108 +WORKER_4_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_4_LINE_19_NO=123 +WORKER_4_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_4_LINE_20_NO=142 +WORKER_4_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_4_LINE_21_NO=150 +WORKER_4_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_4_LINE_22_NO=161 +WORKER_4_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_4_LINE_23_NO=192 +WORKER_4_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_4_LINE_24_NO=203 +WORKER_4_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_4_LINE_25_NO=219 +WORKER_4_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_4_LINE_26_NO=240 +WORKER_4_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_4_LINE_27_NO=256 +WORKER_4_LINE_27_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_4_LINE_28_NO=261 +WORKER_4_LINE_28_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_4_LINE_29_NO=264 +WORKER_4_LINE_29_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_4_LINE_30_NO=265 +WORKER_4_LINE_30_TEXT= logical="$(logical_id "$app")" +WORKER_4_LINE_31_NO=268 +WORKER_4_LINE_31_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_4_LINE_32_NO=269 +WORKER_4_LINE_32_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_4_LINE_33_NO=277 +WORKER_4_LINE_33_TEXT= done <"$manifest" +WORKER_4_LINE_34_NO=344 +WORKER_4_LINE_34_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_4_LINE_35_NO=349 +WORKER_4_LINE_35_TEXT= if "manifest.json" not in names: +WORKER_4_LINE_36_NO=351 +WORKER_4_LINE_36_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_4_LINE_37_NO=353 +WORKER_4_LINE_37_TEXT= for row in manifest: +WORKER_4_LINE_38_NO=394 +WORKER_4_LINE_38_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_4_LINE_39_NO=406 +WORKER_4_LINE_39_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_4_LINE_40_NO=408 +WORKER_4_LINE_40_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_4_LINE_41_NO=414 +WORKER_4_LINE_41_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_4_LINE_42_NO=417 +WORKER_4_LINE_42_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_4_LINE_43_NO=418 +WORKER_4_LINE_43_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_4_LINE_44_NO=424 +WORKER_4_LINE_44_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_4_LINE_45_NO=427 +WORKER_4_LINE_45_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_4_LINE_46_NO=434 +WORKER_4_LINE_46_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_4_LINE_47_NO=459 +WORKER_4_LINE_47_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_4_LINE_48_NO=461 +WORKER_4_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_4_LINE_49_NO=463 +WORKER_4_LINE_49_TEXT=ACTION="${1:-}" +WORKER_4_LINE_50_NO=464 +WORKER_4_LINE_50_TEXT=case "$ACTION" in +WORKER_4_LINE_51_NO=466 +WORKER_4_LINE_51_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_4_LINE_52_NO=467 +WORKER_4_LINE_52_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_5_BASENAME=homelab-app-cloud-quorum-worker +WORKER_5_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-scrutiny-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker +WORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_5_EXECUTABLE=true +WORKER_5_PRESENT=true +WORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_5_SHEBANG=#!/usr/bin/env bash +WORKER_5_BASH_N_RC=0 +WORKER_5_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_5_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_5_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_5_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_5_INTERESTING_LINE_COUNT=58 +WORKER_5_LINE_1_NO=4 +WORKER_5_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_5_LINE_2_NO=6 +WORKER_5_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_5_LINE_3_NO=7 +WORKER_5_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_5_LINE_4_NO=8 +WORKER_5_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_5_LINE_5_NO=9 +WORKER_5_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_5_LINE_6_NO=10 +WORKER_5_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_5_LINE_7_NO=69 +WORKER_5_LINE_7_TEXT=logical_id(){ +WORKER_5_LINE_8_NO=74 +WORKER_5_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_5_LINE_9_NO=90 +WORKER_5_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_5_LINE_10_NO=91 +WORKER_5_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_5_LINE_11_NO=109 +WORKER_5_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_5_LINE_12_NO=111 +WORKER_5_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_5_LINE_13_NO=112 +WORKER_5_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_5_LINE_14_NO=123 +WORKER_5_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_5_LINE_15_NO=125 +WORKER_5_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_5_LINE_16_NO=126 +WORKER_5_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_5_LINE_17_NO=127 +WORKER_5_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_5_LINE_18_NO=134 +WORKER_5_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_5_LINE_19_NO=149 +WORKER_5_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_5_LINE_20_NO=168 +WORKER_5_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_5_LINE_21_NO=176 +WORKER_5_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_5_LINE_22_NO=187 +WORKER_5_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_5_LINE_23_NO=218 +WORKER_5_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_5_LINE_24_NO=229 +WORKER_5_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_5_LINE_25_NO=245 +WORKER_5_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_5_LINE_26_NO=266 +WORKER_5_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_5_LINE_27_NO=301 +WORKER_5_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_5_LINE_28_NO=302 +WORKER_5_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_5_LINE_29_NO=320 +WORKER_5_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_5_LINE_30_NO=368 +WORKER_5_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_5_LINE_31_NO=373 +WORKER_5_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_5_LINE_32_NO=376 +WORKER_5_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_5_LINE_33_NO=377 +WORKER_5_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_5_LINE_34_NO=380 +WORKER_5_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_5_LINE_35_NO=381 +WORKER_5_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_5_LINE_36_NO=389 +WORKER_5_LINE_36_TEXT= done <"$manifest" +WORKER_5_LINE_37_NO=464 +WORKER_5_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_5_LINE_38_NO=465 +WORKER_5_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_5_LINE_39_NO=499 +WORKER_5_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_5_LINE_40_NO=504 +WORKER_5_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_5_LINE_41_NO=506 +WORKER_5_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_5_LINE_42_NO=508 +WORKER_5_LINE_42_TEXT= for row in manifest: +WORKER_5_LINE_43_NO=549 +WORKER_5_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_5_LINE_44_NO=561 +WORKER_5_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_5_LINE_45_NO=563 +WORKER_5_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_5_LINE_46_NO=569 +WORKER_5_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_5_LINE_47_NO=572 +WORKER_5_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_5_LINE_48_NO=573 +WORKER_5_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_5_LINE_49_NO=579 +WORKER_5_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_5_LINE_50_NO=586 +WORKER_5_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_5_LINE_51_NO=593 +WORKER_5_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_5_LINE_52_NO=630 +WORKER_5_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_5_LINE_53_NO=632 +WORKER_5_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_5_LINE_54_NO=678 +WORKER_5_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_5_LINE_55_NO=689 +WORKER_5_LINE_55_TEXT=ACTION="${1:-}" +WORKER_5_LINE_56_NO=690 +WORKER_5_LINE_56_TEXT=case "$ACTION" in +WORKER_5_LINE_57_NO=692 +WORKER_5_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_5_LINE_58_NO=693 +WORKER_5_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_6_BASENAME=homelab-app-cloud-quorum-worker +WORKER_6_PATH_HINT=homelab-ops-cr-2026-0029/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_6_EXECUTABLE=true +WORKER_6_PRESENT=true +WORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_6_SHEBANG=#!/usr/bin/env bash +WORKER_6_BASH_N_RC=0 +WORKER_6_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_6_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_6_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_6_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_6_INTERESTING_LINE_COUNT=58 +WORKER_6_LINE_1_NO=4 +WORKER_6_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_6_LINE_2_NO=6 +WORKER_6_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_6_LINE_3_NO=7 +WORKER_6_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_6_LINE_4_NO=8 +WORKER_6_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_6_LINE_5_NO=9 +WORKER_6_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_6_LINE_6_NO=10 +WORKER_6_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_6_LINE_7_NO=69 +WORKER_6_LINE_7_TEXT=logical_id(){ +WORKER_6_LINE_8_NO=74 +WORKER_6_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_6_LINE_9_NO=90 +WORKER_6_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_6_LINE_10_NO=91 +WORKER_6_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_6_LINE_11_NO=109 +WORKER_6_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_6_LINE_12_NO=111 +WORKER_6_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_6_LINE_13_NO=112 +WORKER_6_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_6_LINE_14_NO=123 +WORKER_6_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_6_LINE_15_NO=125 +WORKER_6_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_6_LINE_16_NO=126 +WORKER_6_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_6_LINE_17_NO=127 +WORKER_6_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_6_LINE_18_NO=134 +WORKER_6_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_6_LINE_19_NO=149 +WORKER_6_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_6_LINE_20_NO=168 +WORKER_6_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_6_LINE_21_NO=176 +WORKER_6_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_6_LINE_22_NO=187 +WORKER_6_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_6_LINE_23_NO=218 +WORKER_6_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_6_LINE_24_NO=229 +WORKER_6_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_6_LINE_25_NO=245 +WORKER_6_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_6_LINE_26_NO=266 +WORKER_6_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_6_LINE_27_NO=301 +WORKER_6_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_6_LINE_28_NO=302 +WORKER_6_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_6_LINE_29_NO=320 +WORKER_6_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_6_LINE_30_NO=368 +WORKER_6_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_6_LINE_31_NO=373 +WORKER_6_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_6_LINE_32_NO=376 +WORKER_6_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_6_LINE_33_NO=377 +WORKER_6_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_6_LINE_34_NO=380 +WORKER_6_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_6_LINE_35_NO=381 +WORKER_6_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_6_LINE_36_NO=389 +WORKER_6_LINE_36_TEXT= done <"$manifest" +WORKER_6_LINE_37_NO=464 +WORKER_6_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_6_LINE_38_NO=465 +WORKER_6_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_6_LINE_39_NO=499 +WORKER_6_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_6_LINE_40_NO=504 +WORKER_6_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_6_LINE_41_NO=506 +WORKER_6_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_6_LINE_42_NO=508 +WORKER_6_LINE_42_TEXT= for row in manifest: +WORKER_6_LINE_43_NO=549 +WORKER_6_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_6_LINE_44_NO=561 +WORKER_6_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_6_LINE_45_NO=563 +WORKER_6_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_6_LINE_46_NO=569 +WORKER_6_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_6_LINE_47_NO=572 +WORKER_6_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_6_LINE_48_NO=573 +WORKER_6_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_6_LINE_49_NO=579 +WORKER_6_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_6_LINE_50_NO=586 +WORKER_6_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_6_LINE_51_NO=593 +WORKER_6_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_6_LINE_52_NO=630 +WORKER_6_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_6_LINE_53_NO=632 +WORKER_6_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_6_LINE_54_NO=678 +WORKER_6_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_6_LINE_55_NO=689 +WORKER_6_LINE_55_TEXT=ACTION="${1:-}" +WORKER_6_LINE_56_NO=690 +WORKER_6_LINE_56_TEXT=case "$ACTION" in +WORKER_6_LINE_57_NO=692 +WORKER_6_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_6_LINE_58_NO=693 +WORKER_6_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_7_BASENAME=homelab-app-cloud-quorum-worker +WORKER_7_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_7_EXECUTABLE=true +WORKER_7_PRESENT=true +WORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_7_SHEBANG=#!/usr/bin/env bash +WORKER_7_BASH_N_RC=0 +WORKER_7_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_7_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_7_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_7_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_7_INTERESTING_LINE_COUNT=58 +WORKER_7_LINE_1_NO=4 +WORKER_7_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_7_LINE_2_NO=6 +WORKER_7_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_7_LINE_3_NO=7 +WORKER_7_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_7_LINE_4_NO=8 +WORKER_7_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_7_LINE_5_NO=9 +WORKER_7_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_7_LINE_6_NO=10 +WORKER_7_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_7_LINE_7_NO=69 +WORKER_7_LINE_7_TEXT=logical_id(){ +WORKER_7_LINE_8_NO=74 +WORKER_7_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_7_LINE_9_NO=90 +WORKER_7_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_7_LINE_10_NO=91 +WORKER_7_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_7_LINE_11_NO=109 +WORKER_7_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_7_LINE_12_NO=111 +WORKER_7_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_7_LINE_13_NO=112 +WORKER_7_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_7_LINE_14_NO=123 +WORKER_7_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_7_LINE_15_NO=125 +WORKER_7_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_7_LINE_16_NO=126 +WORKER_7_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_7_LINE_17_NO=127 +WORKER_7_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_7_LINE_18_NO=134 +WORKER_7_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_7_LINE_19_NO=149 +WORKER_7_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_7_LINE_20_NO=168 +WORKER_7_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_7_LINE_21_NO=176 +WORKER_7_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_7_LINE_22_NO=187 +WORKER_7_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_7_LINE_23_NO=218 +WORKER_7_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_7_LINE_24_NO=229 +WORKER_7_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_7_LINE_25_NO=245 +WORKER_7_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_7_LINE_26_NO=266 +WORKER_7_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_7_LINE_27_NO=301 +WORKER_7_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_7_LINE_28_NO=302 +WORKER_7_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_7_LINE_29_NO=320 +WORKER_7_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_7_LINE_30_NO=368 +WORKER_7_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_7_LINE_31_NO=373 +WORKER_7_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_7_LINE_32_NO=376 +WORKER_7_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_7_LINE_33_NO=377 +WORKER_7_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_7_LINE_34_NO=380 +WORKER_7_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_7_LINE_35_NO=381 +WORKER_7_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_7_LINE_36_NO=389 +WORKER_7_LINE_36_TEXT= done <"$manifest" +WORKER_7_LINE_37_NO=464 +WORKER_7_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_7_LINE_38_NO=465 +WORKER_7_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_7_LINE_39_NO=499 +WORKER_7_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_7_LINE_40_NO=504 +WORKER_7_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_7_LINE_41_NO=506 +WORKER_7_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_7_LINE_42_NO=508 +WORKER_7_LINE_42_TEXT= for row in manifest: +WORKER_7_LINE_43_NO=549 +WORKER_7_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_7_LINE_44_NO=561 +WORKER_7_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_7_LINE_45_NO=563 +WORKER_7_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_7_LINE_46_NO=569 +WORKER_7_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_7_LINE_47_NO=572 +WORKER_7_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_7_LINE_48_NO=573 +WORKER_7_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_7_LINE_49_NO=579 +WORKER_7_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_7_LINE_50_NO=586 +WORKER_7_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_7_LINE_51_NO=593 +WORKER_7_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_7_LINE_52_NO=630 +WORKER_7_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_7_LINE_53_NO=632 +WORKER_7_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_7_LINE_54_NO=678 +WORKER_7_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_7_LINE_55_NO=689 +WORKER_7_LINE_55_TEXT=ACTION="${1:-}" +WORKER_7_LINE_56_NO=690 +WORKER_7_LINE_56_TEXT=case "$ACTION" in +WORKER_7_LINE_57_NO=692 +WORKER_7_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_7_LINE_58_NO=693 +WORKER_7_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_8_BASENAME=homelab-app-cloud-quorum-worker +WORKER_8_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_8_EXECUTABLE=true +WORKER_8_PRESENT=true +WORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_8_SHEBANG=#!/usr/bin/env bash +WORKER_8_BASH_N_RC=0 +WORKER_8_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_8_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_8_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_8_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_8_INTERESTING_LINE_COUNT=58 +WORKER_8_LINE_1_NO=4 +WORKER_8_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_8_LINE_2_NO=6 +WORKER_8_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_8_LINE_3_NO=7 +WORKER_8_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_8_LINE_4_NO=8 +WORKER_8_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_8_LINE_5_NO=9 +WORKER_8_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_8_LINE_6_NO=10 +WORKER_8_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_8_LINE_7_NO=69 +WORKER_8_LINE_7_TEXT=logical_id(){ +WORKER_8_LINE_8_NO=74 +WORKER_8_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_8_LINE_9_NO=90 +WORKER_8_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_8_LINE_10_NO=91 +WORKER_8_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_8_LINE_11_NO=109 +WORKER_8_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_8_LINE_12_NO=111 +WORKER_8_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_8_LINE_13_NO=112 +WORKER_8_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_8_LINE_14_NO=123 +WORKER_8_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_8_LINE_15_NO=125 +WORKER_8_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_8_LINE_16_NO=126 +WORKER_8_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_8_LINE_17_NO=127 +WORKER_8_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_8_LINE_18_NO=134 +WORKER_8_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_8_LINE_19_NO=149 +WORKER_8_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_8_LINE_20_NO=168 +WORKER_8_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_8_LINE_21_NO=176 +WORKER_8_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_8_LINE_22_NO=187 +WORKER_8_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_8_LINE_23_NO=218 +WORKER_8_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_8_LINE_24_NO=229 +WORKER_8_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_8_LINE_25_NO=245 +WORKER_8_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_8_LINE_26_NO=266 +WORKER_8_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_8_LINE_27_NO=301 +WORKER_8_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_8_LINE_28_NO=302 +WORKER_8_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_8_LINE_29_NO=320 +WORKER_8_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_8_LINE_30_NO=368 +WORKER_8_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_8_LINE_31_NO=373 +WORKER_8_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_8_LINE_32_NO=376 +WORKER_8_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_8_LINE_33_NO=377 +WORKER_8_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_8_LINE_34_NO=380 +WORKER_8_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_8_LINE_35_NO=381 +WORKER_8_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_8_LINE_36_NO=389 +WORKER_8_LINE_36_TEXT= done <"$manifest" +WORKER_8_LINE_37_NO=464 +WORKER_8_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_8_LINE_38_NO=465 +WORKER_8_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_8_LINE_39_NO=499 +WORKER_8_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_8_LINE_40_NO=504 +WORKER_8_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_8_LINE_41_NO=506 +WORKER_8_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_8_LINE_42_NO=508 +WORKER_8_LINE_42_TEXT= for row in manifest: +WORKER_8_LINE_43_NO=549 +WORKER_8_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_8_LINE_44_NO=561 +WORKER_8_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_8_LINE_45_NO=563 +WORKER_8_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_8_LINE_46_NO=569 +WORKER_8_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_8_LINE_47_NO=572 +WORKER_8_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_8_LINE_48_NO=573 +WORKER_8_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_8_LINE_49_NO=579 +WORKER_8_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_8_LINE_50_NO=586 +WORKER_8_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_8_LINE_51_NO=593 +WORKER_8_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_8_LINE_52_NO=630 +WORKER_8_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_8_LINE_53_NO=632 +WORKER_8_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_8_LINE_54_NO=678 +WORKER_8_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_8_LINE_55_NO=689 +WORKER_8_LINE_55_TEXT=ACTION="${1:-}" +WORKER_8_LINE_56_NO=690 +WORKER_8_LINE_56_TEXT=case "$ACTION" in +WORKER_8_LINE_57_NO=692 +WORKER_8_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_8_LINE_58_NO=693 +WORKER_8_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_9_BASENAME=homelab-app-cloud-quorum-worker +WORKER_9_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_9_EXECUTABLE=true +WORKER_9_PRESENT=true +WORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_9_SHEBANG=#!/usr/bin/env bash +WORKER_9_BASH_N_RC=0 +WORKER_9_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_9_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_9_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_9_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_9_INTERESTING_LINE_COUNT=58 +WORKER_9_LINE_1_NO=4 +WORKER_9_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_9_LINE_2_NO=6 +WORKER_9_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_9_LINE_3_NO=7 +WORKER_9_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_9_LINE_4_NO=8 +WORKER_9_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_9_LINE_5_NO=9 +WORKER_9_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_9_LINE_6_NO=10 +WORKER_9_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_9_LINE_7_NO=69 +WORKER_9_LINE_7_TEXT=logical_id(){ +WORKER_9_LINE_8_NO=74 +WORKER_9_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_9_LINE_9_NO=90 +WORKER_9_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_9_LINE_10_NO=91 +WORKER_9_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_9_LINE_11_NO=109 +WORKER_9_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_9_LINE_12_NO=111 +WORKER_9_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_9_LINE_13_NO=112 +WORKER_9_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_9_LINE_14_NO=123 +WORKER_9_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_9_LINE_15_NO=125 +WORKER_9_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_9_LINE_16_NO=126 +WORKER_9_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_9_LINE_17_NO=127 +WORKER_9_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_9_LINE_18_NO=134 +WORKER_9_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_9_LINE_19_NO=149 +WORKER_9_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_9_LINE_20_NO=168 +WORKER_9_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_9_LINE_21_NO=176 +WORKER_9_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_9_LINE_22_NO=187 +WORKER_9_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_9_LINE_23_NO=218 +WORKER_9_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_9_LINE_24_NO=229 +WORKER_9_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_9_LINE_25_NO=245 +WORKER_9_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_9_LINE_26_NO=266 +WORKER_9_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_9_LINE_27_NO=301 +WORKER_9_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_9_LINE_28_NO=302 +WORKER_9_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_9_LINE_29_NO=320 +WORKER_9_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_9_LINE_30_NO=368 +WORKER_9_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_9_LINE_31_NO=373 +WORKER_9_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_9_LINE_32_NO=376 +WORKER_9_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_9_LINE_33_NO=377 +WORKER_9_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_9_LINE_34_NO=380 +WORKER_9_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_9_LINE_35_NO=381 +WORKER_9_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_9_LINE_36_NO=389 +WORKER_9_LINE_36_TEXT= done <"$manifest" +WORKER_9_LINE_37_NO=464 +WORKER_9_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_9_LINE_38_NO=465 +WORKER_9_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_9_LINE_39_NO=499 +WORKER_9_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_9_LINE_40_NO=504 +WORKER_9_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_9_LINE_41_NO=506 +WORKER_9_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_9_LINE_42_NO=508 +WORKER_9_LINE_42_TEXT= for row in manifest: +WORKER_9_LINE_43_NO=549 +WORKER_9_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_9_LINE_44_NO=561 +WORKER_9_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_9_LINE_45_NO=563 +WORKER_9_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_9_LINE_46_NO=569 +WORKER_9_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_9_LINE_47_NO=572 +WORKER_9_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_9_LINE_48_NO=573 +WORKER_9_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_9_LINE_49_NO=579 +WORKER_9_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_9_LINE_50_NO=586 +WORKER_9_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_9_LINE_51_NO=593 +WORKER_9_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_9_LINE_52_NO=630 +WORKER_9_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_9_LINE_53_NO=632 +WORKER_9_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_9_LINE_54_NO=678 +WORKER_9_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_9_LINE_55_NO=689 +WORKER_9_LINE_55_TEXT=ACTION="${1:-}" +WORKER_9_LINE_56_NO=690 +WORKER_9_LINE_56_TEXT=case "$ACTION" in +WORKER_9_LINE_57_NO=692 +WORKER_9_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_9_LINE_58_NO=693 +WORKER_9_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_10_BASENAME=homelab-app-cloud-quorum-worker +WORKER_10_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_10_EXECUTABLE=true +WORKER_10_PRESENT=true +WORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_10_SHEBANG=#!/usr/bin/env bash +WORKER_10_BASH_N_RC=0 +WORKER_10_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_10_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_10_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_10_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_10_INTERESTING_LINE_COUNT=58 +WORKER_10_LINE_1_NO=4 +WORKER_10_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_10_LINE_2_NO=6 +WORKER_10_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_10_LINE_3_NO=7 +WORKER_10_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_10_LINE_4_NO=8 +WORKER_10_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_10_LINE_5_NO=9 +WORKER_10_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_10_LINE_6_NO=10 +WORKER_10_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_10_LINE_7_NO=69 +WORKER_10_LINE_7_TEXT=logical_id(){ +WORKER_10_LINE_8_NO=74 +WORKER_10_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_10_LINE_9_NO=90 +WORKER_10_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_10_LINE_10_NO=91 +WORKER_10_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_10_LINE_11_NO=109 +WORKER_10_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_10_LINE_12_NO=111 +WORKER_10_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_10_LINE_13_NO=112 +WORKER_10_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_10_LINE_14_NO=123 +WORKER_10_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_10_LINE_15_NO=125 +WORKER_10_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_10_LINE_16_NO=126 +WORKER_10_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_10_LINE_17_NO=127 +WORKER_10_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_10_LINE_18_NO=134 +WORKER_10_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_10_LINE_19_NO=149 +WORKER_10_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_10_LINE_20_NO=168 +WORKER_10_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_10_LINE_21_NO=176 +WORKER_10_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_10_LINE_22_NO=187 +WORKER_10_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_10_LINE_23_NO=218 +WORKER_10_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_10_LINE_24_NO=229 +WORKER_10_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_10_LINE_25_NO=245 +WORKER_10_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_10_LINE_26_NO=266 +WORKER_10_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_10_LINE_27_NO=301 +WORKER_10_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_10_LINE_28_NO=302 +WORKER_10_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_10_LINE_29_NO=320 +WORKER_10_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_10_LINE_30_NO=368 +WORKER_10_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_10_LINE_31_NO=373 +WORKER_10_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_10_LINE_32_NO=376 +WORKER_10_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_10_LINE_33_NO=377 +WORKER_10_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_10_LINE_34_NO=380 +WORKER_10_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_10_LINE_35_NO=381 +WORKER_10_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_10_LINE_36_NO=389 +WORKER_10_LINE_36_TEXT= done <"$manifest" +WORKER_10_LINE_37_NO=464 +WORKER_10_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_10_LINE_38_NO=465 +WORKER_10_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_10_LINE_39_NO=499 +WORKER_10_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_10_LINE_40_NO=504 +WORKER_10_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_10_LINE_41_NO=506 +WORKER_10_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_10_LINE_42_NO=508 +WORKER_10_LINE_42_TEXT= for row in manifest: +WORKER_10_LINE_43_NO=549 +WORKER_10_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_10_LINE_44_NO=561 +WORKER_10_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_10_LINE_45_NO=563 +WORKER_10_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_10_LINE_46_NO=569 +WORKER_10_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_10_LINE_47_NO=572 +WORKER_10_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_10_LINE_48_NO=573 +WORKER_10_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_10_LINE_49_NO=579 +WORKER_10_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_10_LINE_50_NO=586 +WORKER_10_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_10_LINE_51_NO=593 +WORKER_10_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_10_LINE_52_NO=630 +WORKER_10_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_10_LINE_53_NO=632 +WORKER_10_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_10_LINE_54_NO=678 +WORKER_10_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_10_LINE_55_NO=689 +WORKER_10_LINE_55_TEXT=ACTION="${1:-}" +WORKER_10_LINE_56_NO=690 +WORKER_10_LINE_56_TEXT=case "$ACTION" in +WORKER_10_LINE_57_NO=692 +WORKER_10_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_10_LINE_58_NO=693 +WORKER_10_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_11_BASENAME=homelab-app-cloud-quorum-worker +WORKER_11_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_11_EXECUTABLE=true +WORKER_11_PRESENT=true +WORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_11_SHEBANG=#!/usr/bin/env bash +WORKER_11_BASH_N_RC=0 +WORKER_11_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_11_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_11_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_11_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_11_INTERESTING_LINE_COUNT=58 +WORKER_11_LINE_1_NO=4 +WORKER_11_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_11_LINE_2_NO=6 +WORKER_11_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_11_LINE_3_NO=7 +WORKER_11_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_11_LINE_4_NO=8 +WORKER_11_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_11_LINE_5_NO=9 +WORKER_11_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_11_LINE_6_NO=10 +WORKER_11_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_11_LINE_7_NO=69 +WORKER_11_LINE_7_TEXT=logical_id(){ +WORKER_11_LINE_8_NO=74 +WORKER_11_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_11_LINE_9_NO=90 +WORKER_11_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_11_LINE_10_NO=91 +WORKER_11_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_11_LINE_11_NO=109 +WORKER_11_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_11_LINE_12_NO=111 +WORKER_11_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_11_LINE_13_NO=112 +WORKER_11_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_11_LINE_14_NO=123 +WORKER_11_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_11_LINE_15_NO=125 +WORKER_11_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_11_LINE_16_NO=126 +WORKER_11_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_11_LINE_17_NO=127 +WORKER_11_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_11_LINE_18_NO=134 +WORKER_11_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_11_LINE_19_NO=149 +WORKER_11_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_11_LINE_20_NO=168 +WORKER_11_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_11_LINE_21_NO=176 +WORKER_11_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_11_LINE_22_NO=187 +WORKER_11_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_11_LINE_23_NO=218 +WORKER_11_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_11_LINE_24_NO=229 +WORKER_11_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_11_LINE_25_NO=245 +WORKER_11_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_11_LINE_26_NO=266 +WORKER_11_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_11_LINE_27_NO=301 +WORKER_11_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_11_LINE_28_NO=302 +WORKER_11_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_11_LINE_29_NO=320 +WORKER_11_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_11_LINE_30_NO=368 +WORKER_11_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_11_LINE_31_NO=373 +WORKER_11_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_11_LINE_32_NO=376 +WORKER_11_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_11_LINE_33_NO=377 +WORKER_11_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_11_LINE_34_NO=380 +WORKER_11_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_11_LINE_35_NO=381 +WORKER_11_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_11_LINE_36_NO=389 +WORKER_11_LINE_36_TEXT= done <"$manifest" +WORKER_11_LINE_37_NO=464 +WORKER_11_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_11_LINE_38_NO=465 +WORKER_11_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_11_LINE_39_NO=499 +WORKER_11_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_11_LINE_40_NO=504 +WORKER_11_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_11_LINE_41_NO=506 +WORKER_11_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_11_LINE_42_NO=508 +WORKER_11_LINE_42_TEXT= for row in manifest: +WORKER_11_LINE_43_NO=549 +WORKER_11_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_11_LINE_44_NO=561 +WORKER_11_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_11_LINE_45_NO=563 +WORKER_11_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_11_LINE_46_NO=569 +WORKER_11_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_11_LINE_47_NO=572 +WORKER_11_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_11_LINE_48_NO=573 +WORKER_11_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_11_LINE_49_NO=579 +WORKER_11_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_11_LINE_50_NO=586 +WORKER_11_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_11_LINE_51_NO=593 +WORKER_11_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_11_LINE_52_NO=630 +WORKER_11_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_11_LINE_53_NO=632 +WORKER_11_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_11_LINE_54_NO=678 +WORKER_11_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_11_LINE_55_NO=689 +WORKER_11_LINE_55_TEXT=ACTION="${1:-}" +WORKER_11_LINE_56_NO=690 +WORKER_11_LINE_56_TEXT=case "$ACTION" in +WORKER_11_LINE_57_NO=692 +WORKER_11_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_11_LINE_58_NO=693 +WORKER_11_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_12_BASENAME=homelab-app-cloud-quorum-worker +WORKER_12_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_12_EXECUTABLE=true +WORKER_12_PRESENT=true +WORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_12_SHEBANG=#!/usr/bin/env bash +WORKER_12_BASH_N_RC=0 +WORKER_12_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_12_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_12_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_12_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_12_INTERESTING_LINE_COUNT=58 +WORKER_12_LINE_1_NO=4 +WORKER_12_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_12_LINE_2_NO=6 +WORKER_12_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_12_LINE_3_NO=7 +WORKER_12_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_12_LINE_4_NO=8 +WORKER_12_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_12_LINE_5_NO=9 +WORKER_12_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_12_LINE_6_NO=10 +WORKER_12_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_12_LINE_7_NO=69 +WORKER_12_LINE_7_TEXT=logical_id(){ +WORKER_12_LINE_8_NO=74 +WORKER_12_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_12_LINE_9_NO=90 +WORKER_12_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_12_LINE_10_NO=91 +WORKER_12_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_12_LINE_11_NO=109 +WORKER_12_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_12_LINE_12_NO=111 +WORKER_12_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_12_LINE_13_NO=112 +WORKER_12_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_12_LINE_14_NO=123 +WORKER_12_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_12_LINE_15_NO=125 +WORKER_12_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_12_LINE_16_NO=126 +WORKER_12_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_12_LINE_17_NO=127 +WORKER_12_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_12_LINE_18_NO=134 +WORKER_12_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_12_LINE_19_NO=149 +WORKER_12_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_12_LINE_20_NO=168 +WORKER_12_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_12_LINE_21_NO=176 +WORKER_12_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_12_LINE_22_NO=187 +WORKER_12_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_12_LINE_23_NO=218 +WORKER_12_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_12_LINE_24_NO=229 +WORKER_12_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_12_LINE_25_NO=245 +WORKER_12_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_12_LINE_26_NO=266 +WORKER_12_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_12_LINE_27_NO=301 +WORKER_12_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_12_LINE_28_NO=302 +WORKER_12_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_12_LINE_29_NO=320 +WORKER_12_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_12_LINE_30_NO=368 +WORKER_12_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_12_LINE_31_NO=373 +WORKER_12_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_12_LINE_32_NO=376 +WORKER_12_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_12_LINE_33_NO=377 +WORKER_12_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_12_LINE_34_NO=380 +WORKER_12_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_12_LINE_35_NO=381 +WORKER_12_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_12_LINE_36_NO=389 +WORKER_12_LINE_36_TEXT= done <"$manifest" +WORKER_12_LINE_37_NO=464 +WORKER_12_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_12_LINE_38_NO=465 +WORKER_12_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_12_LINE_39_NO=499 +WORKER_12_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_12_LINE_40_NO=504 +WORKER_12_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_12_LINE_41_NO=506 +WORKER_12_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_12_LINE_42_NO=508 +WORKER_12_LINE_42_TEXT= for row in manifest: +WORKER_12_LINE_43_NO=549 +WORKER_12_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_12_LINE_44_NO=561 +WORKER_12_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_12_LINE_45_NO=563 +WORKER_12_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_12_LINE_46_NO=569 +WORKER_12_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_12_LINE_47_NO=572 +WORKER_12_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_12_LINE_48_NO=573 +WORKER_12_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_12_LINE_49_NO=579 +WORKER_12_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_12_LINE_50_NO=586 +WORKER_12_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_12_LINE_51_NO=593 +WORKER_12_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_12_LINE_52_NO=630 +WORKER_12_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_12_LINE_53_NO=632 +WORKER_12_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_12_LINE_54_NO=678 +WORKER_12_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_12_LINE_55_NO=689 +WORKER_12_LINE_55_TEXT=ACTION="${1:-}" +WORKER_12_LINE_56_NO=690 +WORKER_12_LINE_56_TEXT=case "$ACTION" in +WORKER_12_LINE_57_NO=692 +WORKER_12_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_12_LINE_58_NO=693 +WORKER_12_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_13_BASENAME=homelab-app-cloud-quorum-worker +WORKER_13_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_13_EXECUTABLE=true +WORKER_13_PRESENT=true +WORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_13_SHEBANG=#!/usr/bin/env bash +WORKER_13_BASH_N_RC=0 +WORKER_13_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_13_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_13_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_13_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_13_INTERESTING_LINE_COUNT=58 +WORKER_13_LINE_1_NO=4 +WORKER_13_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_13_LINE_2_NO=6 +WORKER_13_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_13_LINE_3_NO=7 +WORKER_13_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_13_LINE_4_NO=8 +WORKER_13_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_13_LINE_5_NO=9 +WORKER_13_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_13_LINE_6_NO=10 +WORKER_13_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_13_LINE_7_NO=69 +WORKER_13_LINE_7_TEXT=logical_id(){ +WORKER_13_LINE_8_NO=74 +WORKER_13_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_13_LINE_9_NO=90 +WORKER_13_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_13_LINE_10_NO=91 +WORKER_13_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_13_LINE_11_NO=109 +WORKER_13_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_13_LINE_12_NO=111 +WORKER_13_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_13_LINE_13_NO=112 +WORKER_13_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_13_LINE_14_NO=123 +WORKER_13_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_13_LINE_15_NO=125 +WORKER_13_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_13_LINE_16_NO=126 +WORKER_13_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_13_LINE_17_NO=127 +WORKER_13_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_13_LINE_18_NO=134 +WORKER_13_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_13_LINE_19_NO=149 +WORKER_13_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_13_LINE_20_NO=168 +WORKER_13_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_13_LINE_21_NO=176 +WORKER_13_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_13_LINE_22_NO=187 +WORKER_13_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_13_LINE_23_NO=218 +WORKER_13_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_13_LINE_24_NO=229 +WORKER_13_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_13_LINE_25_NO=245 +WORKER_13_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_13_LINE_26_NO=266 +WORKER_13_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_13_LINE_27_NO=301 +WORKER_13_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_13_LINE_28_NO=302 +WORKER_13_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_13_LINE_29_NO=320 +WORKER_13_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_13_LINE_30_NO=368 +WORKER_13_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_13_LINE_31_NO=373 +WORKER_13_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_13_LINE_32_NO=376 +WORKER_13_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_13_LINE_33_NO=377 +WORKER_13_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_13_LINE_34_NO=380 +WORKER_13_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_13_LINE_35_NO=381 +WORKER_13_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_13_LINE_36_NO=389 +WORKER_13_LINE_36_TEXT= done <"$manifest" +WORKER_13_LINE_37_NO=464 +WORKER_13_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_13_LINE_38_NO=465 +WORKER_13_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_13_LINE_39_NO=499 +WORKER_13_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_13_LINE_40_NO=504 +WORKER_13_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_13_LINE_41_NO=506 +WORKER_13_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_13_LINE_42_NO=508 +WORKER_13_LINE_42_TEXT= for row in manifest: +WORKER_13_LINE_43_NO=549 +WORKER_13_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_13_LINE_44_NO=561 +WORKER_13_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_13_LINE_45_NO=563 +WORKER_13_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_13_LINE_46_NO=569 +WORKER_13_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_13_LINE_47_NO=572 +WORKER_13_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_13_LINE_48_NO=573 +WORKER_13_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_13_LINE_49_NO=579 +WORKER_13_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_13_LINE_50_NO=586 +WORKER_13_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_13_LINE_51_NO=593 +WORKER_13_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_13_LINE_52_NO=630 +WORKER_13_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_13_LINE_53_NO=632 +WORKER_13_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_13_LINE_54_NO=678 +WORKER_13_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_13_LINE_55_NO=689 +WORKER_13_LINE_55_TEXT=ACTION="${1:-}" +WORKER_13_LINE_56_NO=690 +WORKER_13_LINE_56_TEXT=case "$ACTION" in +WORKER_13_LINE_57_NO=692 +WORKER_13_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_13_LINE_58_NO=693 +WORKER_13_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_14_BASENAME=homelab-app-cloud-quorum-worker +WORKER_14_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_14_EXECUTABLE=true +WORKER_14_PRESENT=true +WORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_14_SHEBANG=#!/usr/bin/env bash +WORKER_14_BASH_N_RC=0 +WORKER_14_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_14_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_14_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_14_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_14_INTERESTING_LINE_COUNT=58 +WORKER_14_LINE_1_NO=4 +WORKER_14_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_14_LINE_2_NO=6 +WORKER_14_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_14_LINE_3_NO=7 +WORKER_14_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_14_LINE_4_NO=8 +WORKER_14_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_14_LINE_5_NO=9 +WORKER_14_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_14_LINE_6_NO=10 +WORKER_14_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_14_LINE_7_NO=69 +WORKER_14_LINE_7_TEXT=logical_id(){ +WORKER_14_LINE_8_NO=74 +WORKER_14_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_14_LINE_9_NO=90 +WORKER_14_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_14_LINE_10_NO=91 +WORKER_14_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_14_LINE_11_NO=109 +WORKER_14_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_14_LINE_12_NO=111 +WORKER_14_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_14_LINE_13_NO=112 +WORKER_14_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_14_LINE_14_NO=123 +WORKER_14_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_14_LINE_15_NO=125 +WORKER_14_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_14_LINE_16_NO=126 +WORKER_14_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_14_LINE_17_NO=127 +WORKER_14_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_14_LINE_18_NO=134 +WORKER_14_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_14_LINE_19_NO=149 +WORKER_14_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_14_LINE_20_NO=168 +WORKER_14_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_14_LINE_21_NO=176 +WORKER_14_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_14_LINE_22_NO=187 +WORKER_14_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_14_LINE_23_NO=218 +WORKER_14_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_14_LINE_24_NO=229 +WORKER_14_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_14_LINE_25_NO=245 +WORKER_14_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_14_LINE_26_NO=266 +WORKER_14_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_14_LINE_27_NO=301 +WORKER_14_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_14_LINE_28_NO=302 +WORKER_14_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_14_LINE_29_NO=320 +WORKER_14_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_14_LINE_30_NO=368 +WORKER_14_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_14_LINE_31_NO=373 +WORKER_14_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_14_LINE_32_NO=376 +WORKER_14_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_14_LINE_33_NO=377 +WORKER_14_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_14_LINE_34_NO=380 +WORKER_14_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_14_LINE_35_NO=381 +WORKER_14_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_14_LINE_36_NO=389 +WORKER_14_LINE_36_TEXT= done <"$manifest" +WORKER_14_LINE_37_NO=464 +WORKER_14_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_14_LINE_38_NO=465 +WORKER_14_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_14_LINE_39_NO=499 +WORKER_14_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_14_LINE_40_NO=504 +WORKER_14_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_14_LINE_41_NO=506 +WORKER_14_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_14_LINE_42_NO=508 +WORKER_14_LINE_42_TEXT= for row in manifest: +WORKER_14_LINE_43_NO=549 +WORKER_14_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_14_LINE_44_NO=561 +WORKER_14_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_14_LINE_45_NO=563 +WORKER_14_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_14_LINE_46_NO=569 +WORKER_14_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_14_LINE_47_NO=572 +WORKER_14_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_14_LINE_48_NO=573 +WORKER_14_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_14_LINE_49_NO=579 +WORKER_14_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_14_LINE_50_NO=586 +WORKER_14_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_14_LINE_51_NO=593 +WORKER_14_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_14_LINE_52_NO=630 +WORKER_14_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_14_LINE_53_NO=632 +WORKER_14_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_14_LINE_54_NO=678 +WORKER_14_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_14_LINE_55_NO=689 +WORKER_14_LINE_55_TEXT=ACTION="${1:-}" +WORKER_14_LINE_56_NO=690 +WORKER_14_LINE_56_TEXT=case "$ACTION" in +WORKER_14_LINE_57_NO=692 +WORKER_14_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_14_LINE_58_NO=693 +WORKER_14_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_15_BASENAME=homelab-app-cloud-quorum-worker +WORKER_15_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_15_EXECUTABLE=true +WORKER_15_PRESENT=true +WORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_15_SHEBANG=#!/usr/bin/env bash +WORKER_15_BASH_N_RC=0 +WORKER_15_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_15_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_15_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_15_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_15_INTERESTING_LINE_COUNT=58 +WORKER_15_LINE_1_NO=4 +WORKER_15_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_15_LINE_2_NO=6 +WORKER_15_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_15_LINE_3_NO=7 +WORKER_15_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_15_LINE_4_NO=8 +WORKER_15_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_15_LINE_5_NO=9 +WORKER_15_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_15_LINE_6_NO=10 +WORKER_15_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_15_LINE_7_NO=69 +WORKER_15_LINE_7_TEXT=logical_id(){ +WORKER_15_LINE_8_NO=74 +WORKER_15_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_15_LINE_9_NO=90 +WORKER_15_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_15_LINE_10_NO=91 +WORKER_15_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_15_LINE_11_NO=109 +WORKER_15_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_15_LINE_12_NO=111 +WORKER_15_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_15_LINE_13_NO=112 +WORKER_15_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_15_LINE_14_NO=123 +WORKER_15_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_15_LINE_15_NO=125 +WORKER_15_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_15_LINE_16_NO=126 +WORKER_15_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_15_LINE_17_NO=127 +WORKER_15_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_15_LINE_18_NO=134 +WORKER_15_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_15_LINE_19_NO=149 +WORKER_15_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_15_LINE_20_NO=168 +WORKER_15_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_15_LINE_21_NO=176 +WORKER_15_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_15_LINE_22_NO=187 +WORKER_15_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_15_LINE_23_NO=218 +WORKER_15_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_15_LINE_24_NO=229 +WORKER_15_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_15_LINE_25_NO=245 +WORKER_15_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_15_LINE_26_NO=266 +WORKER_15_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_15_LINE_27_NO=301 +WORKER_15_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_15_LINE_28_NO=302 +WORKER_15_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_15_LINE_29_NO=320 +WORKER_15_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_15_LINE_30_NO=368 +WORKER_15_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_15_LINE_31_NO=373 +WORKER_15_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_15_LINE_32_NO=376 +WORKER_15_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_15_LINE_33_NO=377 +WORKER_15_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_15_LINE_34_NO=380 +WORKER_15_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_15_LINE_35_NO=381 +WORKER_15_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_15_LINE_36_NO=389 +WORKER_15_LINE_36_TEXT= done <"$manifest" +WORKER_15_LINE_37_NO=464 +WORKER_15_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_15_LINE_38_NO=465 +WORKER_15_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_15_LINE_39_NO=499 +WORKER_15_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_15_LINE_40_NO=504 +WORKER_15_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_15_LINE_41_NO=506 +WORKER_15_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_15_LINE_42_NO=508 +WORKER_15_LINE_42_TEXT= for row in manifest: +WORKER_15_LINE_43_NO=549 +WORKER_15_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_15_LINE_44_NO=561 +WORKER_15_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_15_LINE_45_NO=563 +WORKER_15_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_15_LINE_46_NO=569 +WORKER_15_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_15_LINE_47_NO=572 +WORKER_15_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_15_LINE_48_NO=573 +WORKER_15_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_15_LINE_49_NO=579 +WORKER_15_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_15_LINE_50_NO=586 +WORKER_15_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_15_LINE_51_NO=593 +WORKER_15_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_15_LINE_52_NO=630 +WORKER_15_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_15_LINE_53_NO=632 +WORKER_15_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_15_LINE_54_NO=678 +WORKER_15_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_15_LINE_55_NO=689 +WORKER_15_LINE_55_TEXT=ACTION="${1:-}" +WORKER_15_LINE_56_NO=690 +WORKER_15_LINE_56_TEXT=case "$ACTION" in +WORKER_15_LINE_57_NO=692 +WORKER_15_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_15_LINE_58_NO=693 +WORKER_15_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_16_BASENAME=homelab-app-cloud-quorum-worker +WORKER_16_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_16_EXECUTABLE=true +WORKER_16_PRESENT=true +WORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_16_SHEBANG=#!/usr/bin/env bash +WORKER_16_BASH_N_RC=0 +WORKER_16_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_16_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_16_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_16_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_16_INTERESTING_LINE_COUNT=58 +WORKER_16_LINE_1_NO=4 +WORKER_16_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_16_LINE_2_NO=6 +WORKER_16_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_16_LINE_3_NO=7 +WORKER_16_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_16_LINE_4_NO=8 +WORKER_16_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_16_LINE_5_NO=9 +WORKER_16_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_16_LINE_6_NO=10 +WORKER_16_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_16_LINE_7_NO=69 +WORKER_16_LINE_7_TEXT=logical_id(){ +WORKER_16_LINE_8_NO=74 +WORKER_16_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_16_LINE_9_NO=90 +WORKER_16_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_16_LINE_10_NO=91 +WORKER_16_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_16_LINE_11_NO=109 +WORKER_16_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_16_LINE_12_NO=111 +WORKER_16_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_16_LINE_13_NO=112 +WORKER_16_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_16_LINE_14_NO=123 +WORKER_16_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_16_LINE_15_NO=125 +WORKER_16_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_16_LINE_16_NO=126 +WORKER_16_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_16_LINE_17_NO=127 +WORKER_16_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_16_LINE_18_NO=134 +WORKER_16_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_16_LINE_19_NO=149 +WORKER_16_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_16_LINE_20_NO=168 +WORKER_16_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_16_LINE_21_NO=176 +WORKER_16_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_16_LINE_22_NO=187 +WORKER_16_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_16_LINE_23_NO=218 +WORKER_16_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_16_LINE_24_NO=229 +WORKER_16_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_16_LINE_25_NO=245 +WORKER_16_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_16_LINE_26_NO=266 +WORKER_16_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_16_LINE_27_NO=301 +WORKER_16_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_16_LINE_28_NO=302 +WORKER_16_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_16_LINE_29_NO=320 +WORKER_16_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_16_LINE_30_NO=368 +WORKER_16_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_16_LINE_31_NO=373 +WORKER_16_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_16_LINE_32_NO=376 +WORKER_16_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_16_LINE_33_NO=377 +WORKER_16_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_16_LINE_34_NO=380 +WORKER_16_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_16_LINE_35_NO=381 +WORKER_16_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_16_LINE_36_NO=389 +WORKER_16_LINE_36_TEXT= done <"$manifest" +WORKER_16_LINE_37_NO=464 +WORKER_16_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_16_LINE_38_NO=465 +WORKER_16_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_16_LINE_39_NO=499 +WORKER_16_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_16_LINE_40_NO=504 +WORKER_16_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_16_LINE_41_NO=506 +WORKER_16_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_16_LINE_42_NO=508 +WORKER_16_LINE_42_TEXT= for row in manifest: +WORKER_16_LINE_43_NO=549 +WORKER_16_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_16_LINE_44_NO=561 +WORKER_16_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_16_LINE_45_NO=563 +WORKER_16_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_16_LINE_46_NO=569 +WORKER_16_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_16_LINE_47_NO=572 +WORKER_16_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_16_LINE_48_NO=573 +WORKER_16_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_16_LINE_49_NO=579 +WORKER_16_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_16_LINE_50_NO=586 +WORKER_16_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_16_LINE_51_NO=593 +WORKER_16_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_16_LINE_52_NO=630 +WORKER_16_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_16_LINE_53_NO=632 +WORKER_16_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_16_LINE_54_NO=678 +WORKER_16_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_16_LINE_55_NO=689 +WORKER_16_LINE_55_TEXT=ACTION="${1:-}" +WORKER_16_LINE_56_NO=690 +WORKER_16_LINE_56_TEXT=case "$ACTION" in +WORKER_16_LINE_57_NO=692 +WORKER_16_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_16_LINE_58_NO=693 +WORKER_16_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_17_BASENAME=homelab-app-cloud-quorum-worker +WORKER_17_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_17_EXECUTABLE=true +WORKER_17_PRESENT=true +WORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_17_SHEBANG=#!/usr/bin/env bash +WORKER_17_BASH_N_RC=0 +WORKER_17_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_17_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_17_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_17_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_17_INTERESTING_LINE_COUNT=58 +WORKER_17_LINE_1_NO=4 +WORKER_17_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_17_LINE_2_NO=6 +WORKER_17_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_17_LINE_3_NO=7 +WORKER_17_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_17_LINE_4_NO=8 +WORKER_17_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_17_LINE_5_NO=9 +WORKER_17_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_17_LINE_6_NO=10 +WORKER_17_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_17_LINE_7_NO=69 +WORKER_17_LINE_7_TEXT=logical_id(){ +WORKER_17_LINE_8_NO=74 +WORKER_17_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_17_LINE_9_NO=90 +WORKER_17_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_17_LINE_10_NO=91 +WORKER_17_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_17_LINE_11_NO=109 +WORKER_17_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_17_LINE_12_NO=111 +WORKER_17_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_17_LINE_13_NO=112 +WORKER_17_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_17_LINE_14_NO=123 +WORKER_17_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_17_LINE_15_NO=125 +WORKER_17_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_17_LINE_16_NO=126 +WORKER_17_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_17_LINE_17_NO=127 +WORKER_17_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_17_LINE_18_NO=134 +WORKER_17_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_17_LINE_19_NO=149 +WORKER_17_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_17_LINE_20_NO=168 +WORKER_17_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_17_LINE_21_NO=176 +WORKER_17_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_17_LINE_22_NO=187 +WORKER_17_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_17_LINE_23_NO=218 +WORKER_17_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_17_LINE_24_NO=229 +WORKER_17_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_17_LINE_25_NO=245 +WORKER_17_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_17_LINE_26_NO=266 +WORKER_17_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_17_LINE_27_NO=301 +WORKER_17_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_17_LINE_28_NO=302 +WORKER_17_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_17_LINE_29_NO=320 +WORKER_17_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_17_LINE_30_NO=368 +WORKER_17_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_17_LINE_31_NO=373 +WORKER_17_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_17_LINE_32_NO=376 +WORKER_17_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_17_LINE_33_NO=377 +WORKER_17_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_17_LINE_34_NO=380 +WORKER_17_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_17_LINE_35_NO=381 +WORKER_17_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_17_LINE_36_NO=389 +WORKER_17_LINE_36_TEXT= done <"$manifest" +WORKER_17_LINE_37_NO=464 +WORKER_17_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_17_LINE_38_NO=465 +WORKER_17_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_17_LINE_39_NO=499 +WORKER_17_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_17_LINE_40_NO=504 +WORKER_17_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_17_LINE_41_NO=506 +WORKER_17_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_17_LINE_42_NO=508 +WORKER_17_LINE_42_TEXT= for row in manifest: +WORKER_17_LINE_43_NO=549 +WORKER_17_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_17_LINE_44_NO=561 +WORKER_17_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_17_LINE_45_NO=563 +WORKER_17_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_17_LINE_46_NO=569 +WORKER_17_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_17_LINE_47_NO=572 +WORKER_17_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_17_LINE_48_NO=573 +WORKER_17_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_17_LINE_49_NO=579 +WORKER_17_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_17_LINE_50_NO=586 +WORKER_17_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_17_LINE_51_NO=593 +WORKER_17_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_17_LINE_52_NO=630 +WORKER_17_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_17_LINE_53_NO=632 +WORKER_17_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_17_LINE_54_NO=678 +WORKER_17_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_17_LINE_55_NO=689 +WORKER_17_LINE_55_TEXT=ACTION="${1:-}" +WORKER_17_LINE_56_NO=690 +WORKER_17_LINE_56_TEXT=case "$ACTION" in +WORKER_17_LINE_57_NO=692 +WORKER_17_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_17_LINE_58_NO=693 +WORKER_17_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_18_BASENAME=homelab-app-cloud-quorum-worker +WORKER_18_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_18_EXECUTABLE=true +WORKER_18_PRESENT=true +WORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_18_SHEBANG=#!/usr/bin/env bash +WORKER_18_BASH_N_RC=0 +WORKER_18_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_18_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_18_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_18_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_18_INTERESTING_LINE_COUNT=58 +WORKER_18_LINE_1_NO=4 +WORKER_18_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_18_LINE_2_NO=6 +WORKER_18_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_18_LINE_3_NO=7 +WORKER_18_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_18_LINE_4_NO=8 +WORKER_18_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_18_LINE_5_NO=9 +WORKER_18_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_18_LINE_6_NO=10 +WORKER_18_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_18_LINE_7_NO=69 +WORKER_18_LINE_7_TEXT=logical_id(){ +WORKER_18_LINE_8_NO=74 +WORKER_18_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_18_LINE_9_NO=90 +WORKER_18_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_18_LINE_10_NO=91 +WORKER_18_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_18_LINE_11_NO=109 +WORKER_18_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_18_LINE_12_NO=111 +WORKER_18_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_18_LINE_13_NO=112 +WORKER_18_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_18_LINE_14_NO=123 +WORKER_18_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_18_LINE_15_NO=125 +WORKER_18_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_18_LINE_16_NO=126 +WORKER_18_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_18_LINE_17_NO=127 +WORKER_18_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_18_LINE_18_NO=134 +WORKER_18_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_18_LINE_19_NO=149 +WORKER_18_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_18_LINE_20_NO=168 +WORKER_18_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_18_LINE_21_NO=176 +WORKER_18_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_18_LINE_22_NO=187 +WORKER_18_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_18_LINE_23_NO=218 +WORKER_18_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_18_LINE_24_NO=229 +WORKER_18_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_18_LINE_25_NO=245 +WORKER_18_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_18_LINE_26_NO=266 +WORKER_18_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_18_LINE_27_NO=301 +WORKER_18_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_18_LINE_28_NO=302 +WORKER_18_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_18_LINE_29_NO=320 +WORKER_18_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_18_LINE_30_NO=368 +WORKER_18_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_18_LINE_31_NO=373 +WORKER_18_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_18_LINE_32_NO=376 +WORKER_18_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_18_LINE_33_NO=377 +WORKER_18_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_18_LINE_34_NO=380 +WORKER_18_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_18_LINE_35_NO=381 +WORKER_18_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_18_LINE_36_NO=389 +WORKER_18_LINE_36_TEXT= done <"$manifest" +WORKER_18_LINE_37_NO=464 +WORKER_18_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_18_LINE_38_NO=465 +WORKER_18_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_18_LINE_39_NO=499 +WORKER_18_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_18_LINE_40_NO=504 +WORKER_18_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_18_LINE_41_NO=506 +WORKER_18_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_18_LINE_42_NO=508 +WORKER_18_LINE_42_TEXT= for row in manifest: +WORKER_18_LINE_43_NO=549 +WORKER_18_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_18_LINE_44_NO=561 +WORKER_18_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_18_LINE_45_NO=563 +WORKER_18_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_18_LINE_46_NO=569 +WORKER_18_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_18_LINE_47_NO=572 +WORKER_18_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_18_LINE_48_NO=573 +WORKER_18_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_18_LINE_49_NO=579 +WORKER_18_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_18_LINE_50_NO=586 +WORKER_18_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_18_LINE_51_NO=593 +WORKER_18_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_18_LINE_52_NO=630 +WORKER_18_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_18_LINE_53_NO=632 +WORKER_18_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_18_LINE_54_NO=678 +WORKER_18_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_18_LINE_55_NO=689 +WORKER_18_LINE_55_TEXT=ACTION="${1:-}" +WORKER_18_LINE_56_NO=690 +WORKER_18_LINE_56_TEXT=case "$ACTION" in +WORKER_18_LINE_57_NO=692 +WORKER_18_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_18_LINE_58_NO=693 +WORKER_18_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_19_BASENAME=homelab-app-cloud-quorum-worker +WORKER_19_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_19_EXECUTABLE=true +WORKER_19_PRESENT=true +WORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_19_SHEBANG=#!/usr/bin/env bash +WORKER_19_BASH_N_RC=0 +WORKER_19_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_19_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_19_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_19_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_19_INTERESTING_LINE_COUNT=58 +WORKER_19_LINE_1_NO=4 +WORKER_19_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_19_LINE_2_NO=6 +WORKER_19_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_19_LINE_3_NO=7 +WORKER_19_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_19_LINE_4_NO=8 +WORKER_19_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_19_LINE_5_NO=9 +WORKER_19_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_19_LINE_6_NO=10 +WORKER_19_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_19_LINE_7_NO=69 +WORKER_19_LINE_7_TEXT=logical_id(){ +WORKER_19_LINE_8_NO=74 +WORKER_19_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_19_LINE_9_NO=90 +WORKER_19_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_19_LINE_10_NO=91 +WORKER_19_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_19_LINE_11_NO=109 +WORKER_19_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_19_LINE_12_NO=111 +WORKER_19_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_19_LINE_13_NO=112 +WORKER_19_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_19_LINE_14_NO=123 +WORKER_19_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_19_LINE_15_NO=125 +WORKER_19_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_19_LINE_16_NO=126 +WORKER_19_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_19_LINE_17_NO=127 +WORKER_19_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_19_LINE_18_NO=134 +WORKER_19_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_19_LINE_19_NO=149 +WORKER_19_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_19_LINE_20_NO=168 +WORKER_19_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_19_LINE_21_NO=176 +WORKER_19_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_19_LINE_22_NO=187 +WORKER_19_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_19_LINE_23_NO=218 +WORKER_19_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_19_LINE_24_NO=229 +WORKER_19_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_19_LINE_25_NO=245 +WORKER_19_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_19_LINE_26_NO=266 +WORKER_19_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_19_LINE_27_NO=301 +WORKER_19_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_19_LINE_28_NO=302 +WORKER_19_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_19_LINE_29_NO=320 +WORKER_19_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_19_LINE_30_NO=368 +WORKER_19_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_19_LINE_31_NO=373 +WORKER_19_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_19_LINE_32_NO=376 +WORKER_19_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_19_LINE_33_NO=377 +WORKER_19_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_19_LINE_34_NO=380 +WORKER_19_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_19_LINE_35_NO=381 +WORKER_19_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_19_LINE_36_NO=389 +WORKER_19_LINE_36_TEXT= done <"$manifest" +WORKER_19_LINE_37_NO=464 +WORKER_19_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_19_LINE_38_NO=465 +WORKER_19_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_19_LINE_39_NO=499 +WORKER_19_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_19_LINE_40_NO=504 +WORKER_19_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_19_LINE_41_NO=506 +WORKER_19_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_19_LINE_42_NO=508 +WORKER_19_LINE_42_TEXT= for row in manifest: +WORKER_19_LINE_43_NO=549 +WORKER_19_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_19_LINE_44_NO=561 +WORKER_19_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_19_LINE_45_NO=563 +WORKER_19_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_19_LINE_46_NO=569 +WORKER_19_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_19_LINE_47_NO=572 +WORKER_19_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_19_LINE_48_NO=573 +WORKER_19_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_19_LINE_49_NO=579 +WORKER_19_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_19_LINE_50_NO=586 +WORKER_19_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_19_LINE_51_NO=593 +WORKER_19_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_19_LINE_52_NO=630 +WORKER_19_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_19_LINE_53_NO=632 +WORKER_19_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_19_LINE_54_NO=678 +WORKER_19_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_19_LINE_55_NO=689 +WORKER_19_LINE_55_TEXT=ACTION="${1:-}" +WORKER_19_LINE_56_NO=690 +WORKER_19_LINE_56_TEXT=case "$ACTION" in +WORKER_19_LINE_57_NO=692 +WORKER_19_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_19_LINE_58_NO=693 +WORKER_19_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +WORKER_20_BASENAME=homelab-app-cloud-quorum-worker +WORKER_20_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker +WORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_20_EXECUTABLE=true +WORKER_20_PRESENT=true +WORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 +WORKER_20_SHEBANG=#!/usr/bin/env bash +WORKER_20_BASH_N_RC=0 +WORKER_20_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding +WORKER_20_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor +WORKER_20_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work +WORKER_20_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd +WORKER_20_INTERESTING_LINE_COUNT=58 +WORKER_20_LINE_1_NO=4 +WORKER_20_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload +WORKER_20_LINE_2_NO=6 +WORKER_20_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper +WORKER_20_LINE_3_NO=7 +WORKER_20_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper +WORKER_20_LINE_4_NO=8 +WORKER_20_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum +WORKER_20_LINE_5_NO=9 +WORKER_20_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation +WORKER_20_LINE_6_NO=10 +WORKER_20_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum +WORKER_20_LINE_7_NO=69 +WORKER_20_LINE_7_TEXT=logical_id(){ +WORKER_20_LINE_8_NO=74 +WORKER_20_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; +WORKER_20_LINE_9_NO=90 +WORKER_20_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } +WORKER_20_LINE_10_NO=91 +WORKER_20_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } +WORKER_20_LINE_11_NO=109 +WORKER_20_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null +WORKER_20_LINE_12_NO=111 +WORKER_20_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } +WORKER_20_LINE_13_NO=112 +WORKER_20_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } +WORKER_20_LINE_14_NO=123 +WORKER_20_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" +WORKER_20_LINE_15_NO=125 +WORKER_20_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 +WORKER_20_LINE_16_NO=126 +WORKER_20_LINE_16_TEXT= if [[ -s "$manifest" ]]; then +WORKER_20_LINE_17_NO=127 +WORKER_20_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 +WORKER_20_LINE_18_NO=134 +WORKER_20_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" +WORKER_20_LINE_19_NO=149 +WORKER_20_LINE_19_TEXT= logical="$(logical_id "$app")" +WORKER_20_LINE_20_NO=168 +WORKER_20_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_20_LINE_21_NO=176 +WORKER_20_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" +WORKER_20_LINE_22_NO=187 +WORKER_20_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" +WORKER_20_LINE_23_NO=218 +WORKER_20_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_20_LINE_24_NO=229 +WORKER_20_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" +WORKER_20_LINE_25_NO=245 +WORKER_20_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_20_LINE_26_NO=266 +WORKER_20_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_20_LINE_27_NO=301 +WORKER_20_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_20_LINE_28_NO=302 +WORKER_20_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_20_LINE_29_NO=320 +WORKER_20_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" +WORKER_20_LINE_30_NO=368 +WORKER_20_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 +WORKER_20_LINE_31_NO=373 +WORKER_20_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") +WORKER_20_LINE_32_NO=376 +WORKER_20_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual +WORKER_20_LINE_33_NO=377 +WORKER_20_LINE_33_TEXT= logical="$(logical_id "$app")" +WORKER_20_LINE_34_NO=380 +WORKER_20_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" +WORKER_20_LINE_35_NO=381 +WORKER_20_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 +WORKER_20_LINE_36_NO=389 +WORKER_20_LINE_36_TEXT= done <"$manifest" +WORKER_20_LINE_37_NO=464 +WORKER_20_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) +WORKER_20_LINE_38_NO=465 +WORKER_20_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) +WORKER_20_LINE_39_NO=499 +WORKER_20_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} +WORKER_20_LINE_40_NO=504 +WORKER_20_LINE_40_TEXT= if "manifest.json" not in names: +WORKER_20_LINE_41_NO=506 +WORKER_20_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) +WORKER_20_LINE_42_NO=508 +WORKER_20_LINE_42_TEXT= for row in manifest: +WORKER_20_LINE_43_NO=549 +WORKER_20_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" +WORKER_20_LINE_44_NO=561 +WORKER_20_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" +WORKER_20_LINE_45_NO=563 +WORKER_20_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null +WORKER_20_LINE_46_NO=569 +WORKER_20_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" +WORKER_20_LINE_47_NO=572 +WORKER_20_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" +WORKER_20_LINE_48_NO=573 +WORKER_20_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_20_LINE_49_NO=579 +WORKER_20_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi +WORKER_20_LINE_50_NO=586 +WORKER_20_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_20_LINE_51_NO=593 +WORKER_20_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" +WORKER_20_LINE_52_NO=630 +WORKER_20_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" +WORKER_20_LINE_53_NO=632 +WORKER_20_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" +WORKER_20_LINE_54_NO=678 +WORKER_20_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" +WORKER_20_LINE_55_NO=689 +WORKER_20_LINE_55_TEXT=ACTION="${1:-}" +WORKER_20_LINE_56_NO=690 +WORKER_20_LINE_56_TEXT=case "$ACTION" in +WORKER_20_LINE_57_NO=692 +WORKER_20_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; +WORKER_20_LINE_58_NO=693 +WORKER_20_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; +TARGETS_PRESENT=true +TARGETS_SHA=2f526330152d49461ff66f50adab57d1a195ea5d1b4c292945d523411b8a68a8 +TARGET_OBJECT_FOUND=true +TARGET_KEYS=id,kind,lifecycle,logical_id,migrate,owner,source_host +TARGET_ID=skladchik-reports-monitor +TARGET_KIND=APP +TARGET_LIFECYCLE=ACTIVE +TARGET_LOGICAL_ID=app-skladchik-reports-monitor +TARGET_MIGRATE=True +TARGET_OWNER=pve01 +TARGET_SOURCE_HOST_BASENAME=[PRIVATE_IP] +TARGET_SOURCE_HOST_SHA=3eb2319e39eac307b3c7bd11f67663c0a0c814fd603989f5965083f9d1c80fb0 +QUEUE_APPID_LITERAL_COUNT=0 +QUEUE_TARGETS_BASENAME_LITERAL_COUNT=0 +BACKUP_AUDIT_RC=0 +BACKUP_AUDIT_RELEVANT_COUNT=2 +BACKUP_AUDIT_RELEVANT_1=TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=22759|generation=app-skladchik-reports-monitor-pve01-20261005T003035Z +BACKUP_AUDIT_RELEVANT_2=TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=32145|legacy_age_seconds=1627370|replica_count=2|state_id=vm170|generation=v2-vm170-20261004T214054Z-70a537ab4e91 +TARGETED_SEAL_DESIGN_READY=true +MAIL_SENT_BY_COMMAND=false CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null MUTATION_OUTCOME=NO_MUTATION +DECISION=PASS_QUEUE_WORKER_RCA +NEXT_GATE=DESIGN_ATOMIC_SOURCE_BACKUP_SEAL +TASK_RESULT=SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA OUTPUT_END CHAT_OUTPUT_END