diff --git a/runtime/history/SUPPORT-261005-SNIKKET-OPS-OWNERSHIP-RCA-99578R1.json b/runtime/history/SUPPORT-261005-SNIKKET-OPS-OWNERSHIP-RCA-99578R1.json new file mode 100644 index 00000000..c1f49bf3 --- /dev/null +++ b/runtime/history/SUPPORT-261005-SNIKKET-OPS-OWNERSHIP-RCA-99578R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-261005-SNIKKET-OPS-OWNERSHIP-RCA-99578R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "snikket-ops-ownership-rca", + "started_at_utc": "2026-10-05T06:54:25Z", + "finished_at_utc": "2026-10-05T06:54:40Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "7c2094112fc9cb5e843bd3559354125d99d3543ddd0f2055aca907ad84c4b4f5", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "5ae23df042db0d52f6523a7361f49070039ea122f2db6e864fd4a5d979c98317", + "sanitized_output_sha256": "5ae23df042db0d52f6523a7361f49070039ea122f2db6e864fd4a5d979c98317", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "SNIKKET_OPS_OWNERSHIP_RCA_WRAPPER=V1\nMODE=read-only\nNO_DNS_MUTATION=true\nNO_VM_MUTATION=true\nNO_CONTAINER_RESTART=true\nNO_SERVICE_START=true\nNO_SOURCE_MUTATION=true\nHOST_PVE01=true\nINGRESS_HOSTKEY_SELF_BOUND=true\nVM150_INVENTORY_PRESENT=true\nVM150_RUNNING=true\nVM150_EXPECTED_NODE=true\nVM171_INVENTORY_PRESENT=true\nVM171_RUNNING=true\nVM171_EXPECTED_NODE=true\nVM150_QGA_PING=true\nVM171_QGA_PING=true\nPVE_BACKUP_JOBS_QUERY_RC=0\nPVE_BACKUP_JOB_COVERAGE_COUNT=0\nPVE_BACKUP_JOB_ENABLED_COVERAGE_COUNT=0\nPVE_BACKUP_STORAGE_QUERY_RC=0\nPVE_BACKUP_STORAGE_COUNT=1\nPVE_VM150_BACKUP_OBJECT_COUNT=0\nPVE_VM150_BACKUP_STORAGE_WITH_OBJECT_COUNT=0\nPVE_VM150_NEWEST_BACKUP_AGE_HOURS=-1\nBACKUP_ENGINE_VM150_REF_14D_COUNT=40\nBACKUP_ENGINE_VM150_SUCCESS_14D_COUNT=40\nBACKUP_ENGINE_VM150_LATEST_SUCCESS_AGE_HOURS=16\nBACKUP_AUDIT_VM150_REF_30D_COUNT=2379\nBACKUP_AUDIT_VM150_RESTORE_PASS_30D_COUNT=0\nBACKUP_AUDIT_VM150_LATEST_RESTORE_PASS_AGE_HOURS=-1\nBACKUP_EVIDENCE_CLASS_VZDUMP=true\nBACKUP_EVIDENCE_CLASS_RESTIC=false\nBACKUP_EVIDENCE_CLASS_BORG=false\nBACKUP_EVIDENCE_CLASS_RCLONE=false\nBACKUP_EVIDENCE_CLASS_RSYNC_TAR=false\nBACKUP_EVIDENCE_CLASS_DOCKER_DATA=false\nBACKUP_EVIDENCE_CLASS_REMOTE=true\nBACKUP_SOURCE_FILE_COUNT=482\nBACKUP_SOURCE_VM150_REF_COUNT=45\nBACKUP_SOURCE_CAP_VZDUMP=true\nBACKUP_SOURCE_CAP_RESTIC=false\nBACKUP_SOURCE_CAP_BORG=false\nBACKUP_SOURCE_CAP_RCLONE=true\nBACKUP_SOURCE_CAP_RSYNC_TAR=true\nBACKUP_SOURCE_CAP_DYNAMIC_VM=true\nPVE01_GIT_REPO_COUNT=64\nPVE01_SNIKKET_SOURCE_CANDIDATE_REPO_COUNT=49\nPVE01_SOURCE_RECON_EXACT_SHA_COUNT=0\nPVE01_SOURCE_COTURN_EXACT_SHA_COUNT=49\nPVE01_SOURCE_OVERRIDE_EXACT_SHA_COUNT=49\nPVE01_SOURCE_RECON_CORRECT_LOGIC_COUNT=0\nPVE01_SOURCE_DNS_ALL3_COUNT=343\nPVE01_SOURCE_DNS_CURRENT_TARGET_COUNT=294\nPVE01_MONITOR_SOURCE_FILE_COUNT=86\nPVE01_MONITOR_CHAT_REF=false\nPVE01_MONITOR_XMPP_5222_REF=false\nPVE01_MONITOR_XMPP_5269_REF=false\nPVE01_MONITOR_TURN_3479_REF=false\nPVE01_MONITOR_STUN_TURN_REF=false\nVM150_QGA_EXEC_INIT_RC=0\nVM150_QGA_POLL_RC=0\nVM_GUEST_EXITCODE=0\nVM_GUEST_OUTPUT_MARKER=true\nSNIKKET_OPS_OWNERSHIP_GUEST150=V1\nMODE=read-only\nNO_MUTATION=true\nGUEST150_HOSTNAME_OK=true\nGUEST150_LIVE_RECON_PRESENT=true\nGUEST150_LIVE_RECON_SHA256=bb692a3a044d41e6e04518aff5bdfa646f533660a70ea793f87d040c881a6e98\nGUEST150_LIVE_COTURN_PRESENT=true\nGUEST150_LIVE_COTURN_SHA256=2de0e47c09b3b4fc7a6c32c5ef061efaaa7c7f4db3d7a25fdf4da8fbb2e6f483\nGUEST150_LIVE_OVERRIDE_PRESENT=true\nGUEST150_LIVE_OVERRIDE_SHA256=54a42711db5c2eb99b3886184c7ce3e01a30aba844ff2022bc74a45414a2241c\nGUEST150_GIT_REPO_COUNT=0\nGUEST150_SNIKKET_TRACKED_FILE_COUNT=0\nGUEST150_SOURCE_CANDIDATE_REPO_COUNT=0\nGUEST150_SOURCE_RECON_EXACT_SHA_COUNT=0\nGUEST150_SOURCE_COTURN_EXACT_SHA_COUNT=0\nGUEST150_SOURCE_OVERRIDE_EXACT_SHA_COUNT=0\nGUEST150_SOURCE_RECON_CORRECT_LOGIC_COUNT=0\nGUEST150_RECON_SERVICE_UNIT_PRESENT=true\nGUEST150_RECON_TIMER_UNIT_PRESENT=true\nGUEST150_EDGE_UNIT_PRESENT=true\nGUEST150_TCP3479_COUNT=16\nGUEST150_UDP3479_COUNT=16\nGUEST150_RECON_SERVICE_RESULT=success\nGUEST150_RECON_SERVICE_EXEC_MAIN_STATUS=0\nGUEST150_AUDIT_DONE=true\nVM171_QGA_EXEC_INIT_RC=0\nVM171_QGA_POLL_RC=0\nVM_GUEST_EXITCODE=0\nVM_GUEST_OUTPUT_MARKER=true\nSNIKKET_OPS_OWNERSHIP_GUEST171=V1\nMODE=read-only\nNO_MUTATION=true\nGUEST171_HOSTNAME_MONITORING=true\nGUEST171_MONITOR_TEXT_FILE_COUNT=373\nGUEST171_MONITOR_CHAT_REF=false\nGUEST171_MONITOR_GROUPS_REF=false\nGUEST171_MONITOR_SHARE_REF=false\nGUEST171_MONITOR_PORT_443_REF=true\nGUEST171_MONITOR_PORT_5000_REF=false\nGUEST171_MONITOR_PORT_5222_REF=true\nGUEST171_MONITOR_PORT_5269_REF=false\nGUEST171_MONITOR_PORT_3479_REF=true\nGUEST171_MONITOR_PORT_5349_REF=false\nGUEST171_MONITOR_PORT_5350_REF=false\nGUEST171_MONITOR_STUN_TURN_REF=false\nGUEST171_MONITOR_CERT_EXPIRY_REF=true\nGUEST171_KUMA_DB_COUNT=2\nGUEST171_KUMA_MONITOR_COUNT=192\nGUEST171_KUMA_CHAT_MATCH_COUNT=0\nGUEST171_KUMA_GROUPS_MATCH_COUNT=0\nGUEST171_KUMA_SHARE_MATCH_COUNT=0\nGUEST171_KUMA_5222_MATCH_COUNT=0\nGUEST171_KUMA_5269_MATCH_COUNT=0\nGUEST171_KUMA_3479_MATCH_COUNT=0\nGUEST171_KUMA_STUN_TURN_MATCH_COUNT=0\nGUEST171_AUDIT_DONE=true\nPUBLIC_DNS_OK_COUNT=3\nPUBLIC_HTTPS_OK_COUNT=3\nPUBLIC_STUN_3479_PASS=true\nPUBLIC_STUN_3479_CLASS=BINDING_SUCCESS\nOPS_OWNERSHIP_RCA_COMPLETE=true\nDECISION=ANALYZE_EXACT_99578_OWNERSHIP\nNEXT_GATE=TARGETED_OPERATIONAL_HARDENING\nTASK_RESULT=SNIKKET_OPS_OWNERSHIP_RCA\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\n" +} diff --git a/runtime/history/SUPPORT-261005-SNIKKET-OPS-OWNERSHIP-RCA-99578R1.txt b/runtime/history/SUPPORT-261005-SNIKKET-OPS-OWNERSHIP-RCA-99578R1.txt new file mode 100644 index 00000000..e146413c --- /dev/null +++ b/runtime/history/SUPPORT-261005-SNIKKET-OPS-OWNERSHIP-RCA-99578R1.txt @@ -0,0 +1,164 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-261005-SNIKKET-OPS-OWNERSHIP-RCA-99578R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=snikket-ops-ownership-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=7c2094112fc9cb5e843bd3559354125d99d3543ddd0f2055aca907ad84c4b4f5 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=5ae23df042db0d52f6523a7361f49070039ea122f2db6e864fd4a5d979c98317 +SANITIZED_OUTPUT_SHA256=5ae23df042db0d52f6523a7361f49070039ea122f2db6e864fd4a5d979c98317 +OUTPUT_BEGIN +SNIKKET_OPS_OWNERSHIP_RCA_WRAPPER=V1 +MODE=read-only +NO_DNS_MUTATION=true +NO_VM_MUTATION=true +NO_CONTAINER_RESTART=true +NO_SERVICE_START=true +NO_SOURCE_MUTATION=true +HOST_PVE01=true +INGRESS_HOSTKEY_SELF_BOUND=true +VM150_INVENTORY_PRESENT=true +VM150_RUNNING=true +VM150_EXPECTED_NODE=true +VM171_INVENTORY_PRESENT=true +VM171_RUNNING=true +VM171_EXPECTED_NODE=true +VM150_QGA_PING=true +VM171_QGA_PING=true +PVE_BACKUP_JOBS_QUERY_RC=0 +PVE_BACKUP_JOB_COVERAGE_COUNT=0 +PVE_BACKUP_JOB_ENABLED_COVERAGE_COUNT=0 +PVE_BACKUP_STORAGE_QUERY_RC=0 +PVE_BACKUP_STORAGE_COUNT=1 +PVE_VM150_BACKUP_OBJECT_COUNT=0 +PVE_VM150_BACKUP_STORAGE_WITH_OBJECT_COUNT=0 +PVE_VM150_NEWEST_BACKUP_AGE_HOURS=-1 +BACKUP_ENGINE_VM150_REF_14D_COUNT=40 +BACKUP_ENGINE_VM150_SUCCESS_14D_COUNT=40 +BACKUP_ENGINE_VM150_LATEST_SUCCESS_AGE_HOURS=16 +BACKUP_AUDIT_VM150_REF_30D_COUNT=2379 +BACKUP_AUDIT_VM150_RESTORE_PASS_30D_COUNT=0 +BACKUP_AUDIT_VM150_LATEST_RESTORE_PASS_AGE_HOURS=-1 +BACKUP_EVIDENCE_CLASS_VZDUMP=true +BACKUP_EVIDENCE_CLASS_RESTIC=false +BACKUP_EVIDENCE_CLASS_BORG=false +BACKUP_EVIDENCE_CLASS_RCLONE=false +BACKUP_EVIDENCE_CLASS_RSYNC_TAR=false +BACKUP_EVIDENCE_CLASS_DOCKER_DATA=false +BACKUP_EVIDENCE_CLASS_REMOTE=true +BACKUP_SOURCE_FILE_COUNT=482 +BACKUP_SOURCE_VM150_REF_COUNT=45 +BACKUP_SOURCE_CAP_VZDUMP=true +BACKUP_SOURCE_CAP_RESTIC=false +BACKUP_SOURCE_CAP_BORG=false +BACKUP_SOURCE_CAP_RCLONE=true +BACKUP_SOURCE_CAP_RSYNC_TAR=true +BACKUP_SOURCE_CAP_DYNAMIC_VM=true +PVE01_GIT_REPO_COUNT=64 +PVE01_SNIKKET_SOURCE_CANDIDATE_REPO_COUNT=49 +PVE01_SOURCE_RECON_EXACT_SHA_COUNT=0 +PVE01_SOURCE_COTURN_EXACT_SHA_COUNT=49 +PVE01_SOURCE_OVERRIDE_EXACT_SHA_COUNT=49 +PVE01_SOURCE_RECON_CORRECT_LOGIC_COUNT=0 +PVE01_SOURCE_DNS_ALL3_COUNT=343 +PVE01_SOURCE_DNS_CURRENT_TARGET_COUNT=294 +PVE01_MONITOR_SOURCE_FILE_COUNT=86 +PVE01_MONITOR_CHAT_REF=false +PVE01_MONITOR_XMPP_5222_REF=false +PVE01_MONITOR_XMPP_5269_REF=false +PVE01_MONITOR_TURN_3479_REF=false +PVE01_MONITOR_STUN_TURN_REF=false +VM150_QGA_EXEC_INIT_RC=0 +VM150_QGA_POLL_RC=0 +VM_GUEST_EXITCODE=0 +VM_GUEST_OUTPUT_MARKER=true +SNIKKET_OPS_OWNERSHIP_GUEST150=V1 +MODE=read-only +NO_MUTATION=true +GUEST150_HOSTNAME_OK=true +GUEST150_LIVE_RECON_PRESENT=true +GUEST150_LIVE_RECON_SHA256=bb692a3a044d41e6e04518aff5bdfa646f533660a70ea793f87d040c881a6e98 +GUEST150_LIVE_COTURN_PRESENT=true +GUEST150_LIVE_COTURN_SHA256=2de0e47c09b3b4fc7a6c32c5ef061efaaa7c7f4db3d7a25fdf4da8fbb2e6f483 +GUEST150_LIVE_OVERRIDE_PRESENT=true +GUEST150_LIVE_OVERRIDE_SHA256=54a42711db5c2eb99b3886184c7ce3e01a30aba844ff2022bc74a45414a2241c +GUEST150_GIT_REPO_COUNT=0 +GUEST150_SNIKKET_TRACKED_FILE_COUNT=0 +GUEST150_SOURCE_CANDIDATE_REPO_COUNT=0 +GUEST150_SOURCE_RECON_EXACT_SHA_COUNT=0 +GUEST150_SOURCE_COTURN_EXACT_SHA_COUNT=0 +GUEST150_SOURCE_OVERRIDE_EXACT_SHA_COUNT=0 +GUEST150_SOURCE_RECON_CORRECT_LOGIC_COUNT=0 +GUEST150_RECON_SERVICE_UNIT_PRESENT=true +GUEST150_RECON_TIMER_UNIT_PRESENT=true +GUEST150_EDGE_UNIT_PRESENT=true +GUEST150_TCP3479_COUNT=16 +GUEST150_UDP3479_COUNT=16 +GUEST150_RECON_SERVICE_RESULT=success +GUEST150_RECON_SERVICE_EXEC_MAIN_STATUS=0 +GUEST150_AUDIT_DONE=true +VM171_QGA_EXEC_INIT_RC=0 +VM171_QGA_POLL_RC=0 +VM_GUEST_EXITCODE=0 +VM_GUEST_OUTPUT_MARKER=true +SNIKKET_OPS_OWNERSHIP_GUEST171=V1 +MODE=read-only +NO_MUTATION=true +GUEST171_HOSTNAME_MONITORING=true +GUEST171_MONITOR_TEXT_FILE_COUNT=373 +GUEST171_MONITOR_CHAT_REF=false +GUEST171_MONITOR_GROUPS_REF=false +GUEST171_MONITOR_SHARE_REF=false +GUEST171_MONITOR_PORT_443_REF=true +GUEST171_MONITOR_PORT_5000_REF=false +GUEST171_MONITOR_PORT_5222_REF=true +GUEST171_MONITOR_PORT_5269_REF=false +GUEST171_MONITOR_PORT_3479_REF=true +GUEST171_MONITOR_PORT_5349_REF=false +GUEST171_MONITOR_PORT_5350_REF=false +GUEST171_MONITOR_STUN_TURN_REF=false +GUEST171_MONITOR_CERT_EXPIRY_REF=true +GUEST171_KUMA_DB_COUNT=2 +GUEST171_KUMA_MONITOR_COUNT=192 +GUEST171_KUMA_CHAT_MATCH_COUNT=0 +GUEST171_KUMA_GROUPS_MATCH_COUNT=0 +GUEST171_KUMA_SHARE_MATCH_COUNT=0 +GUEST171_KUMA_5222_MATCH_COUNT=0 +GUEST171_KUMA_5269_MATCH_COUNT=0 +GUEST171_KUMA_3479_MATCH_COUNT=0 +GUEST171_KUMA_STUN_TURN_MATCH_COUNT=0 +GUEST171_AUDIT_DONE=true +PUBLIC_DNS_OK_COUNT=3 +PUBLIC_HTTPS_OK_COUNT=3 +PUBLIC_STUN_3479_PASS=true +PUBLIC_STUN_3479_CLASS=BINDING_SUCCESS +OPS_OWNERSHIP_RCA_COMPLETE=true +DECISION=ANALYZE_EXACT_99578_OWNERSHIP +NEXT_GATE=TARGETED_OPERATIONAL_HARDENING +TASK_RESULT=SNIKKET_OPS_OWNERSHIP_RCA +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 88b54e4f..c1f49bf3 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1", + "command_id": "SUPPORT-261005-SNIKKET-OPS-OWNERSHIP-RCA-99578R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "skladchik-durability-queue-worker-rca", - "started_at_utc": "2026-10-05T06:49:57Z", - "finished_at_utc": "2026-10-05T06:50:09Z", + "component": "snikket-ops-ownership-rca", + "started_at_utc": "2026-10-05T06:54:25Z", + "finished_at_utc": "2026-10-05T06:54:40Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "9786047247e48e71139ff52c618c59da24ce317cd7093ee00b699a69b178af52", + "command_sha256": "7c2094112fc9cb5e843bd3559354125d99d3543ddd0f2055aca907ad84c4b4f5", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "df86e4a999ff180077eddd8c50c0647c83310a5062be61a88ccaab005bb577ea", - "sanitized_output_sha256": "3aa1864388e614772490c9c29217a4ed56c483df64b4b67276683b5c5078e47e", + "raw_evidence_sha256": "5ae23df042db0d52f6523a7361f49070039ea122f2db6e864fd4a5d979c98317", + "sanitized_output_sha256": "5ae23df042db0d52f6523a7361f49070039ea122f2db6e864fd4a5d979c98317", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA=V1\nNO_MAIL_SEND=true\nNO_SERVICE_RESTART=true\nNO_TIMER_MUTATION=true\nNO_NETWORK_MUTATION=true\nNO_SOURCE_MUTATION=true\nNO_BACKUP_MUTATION=true\nNO_GIT_PUSH=true\nMODE=read-only\nHOST_PVE01=true\nQUEUE_VM170_QM_RC=0\nQUEUE_VM170_STDERR_PRESENT=false\nQUEUE_VM170_JSON=true\nQUEUE_VM170_GUEST_EXITCODE=0\nVM170_MONITOR_SHA=19123e359f2f7b9a2d373fd7e4464dd426d31b91cabc567c252e70cad40d77c8\nVM170_TIMER_ACTIVE=active\nVM170_TIMER_ENABLED=enabled\nVM170_SERVICE_RESULT=success\nVM170_HEALTH_STATUS=OK\nVM170_HEALTH_TS=2026-10-05T06:49:35Z\nVM170_HEALTH_REPORTS=22\nVM170_HEALTH_OPEN=13\nVM170_HEALTH_RESOLVED=9\nVM170_BASELINE_OK=true\nSUPERVISOR_SHA=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2\nSUPERVISOR_TOP_CONST_1_NAME=BACKUP_HEALTH_AGE_MAX\nSUPERVISOR_TOP_CONST_1_EXPR_SHA=056f62a1cb138698ac25abded3e6493c5519d37a485bb3900a74bad1806f70b3\nSUPERVISOR_TOP_CONST_2_NAME=RESTORE_HEALTH_AGE_MAX\nSUPERVISOR_TOP_CONST_2_EXPR_SHA=ce6a9ab58cc2e0db3f07ff3ef7d288b29b6611afff056f5b818c706075d9dbd5\nSUPERVISOR_TOP_CONST_3_NAME=APP_FRESHNESS_PROBE\nSUPERVISOR_TOP_CONST_3_EXPR_SHA=d6593f7b36fcf7550285e4af7a5f5766c13c2ee0e0a992b808caaed52e54693a\nSUPERVISOR_TOP_CONST_4_NAME=LEGACY_BACKUP_PROBLEMS\nSUPERVISOR_TOP_CONST_4_EXPR_SHA=d84baf639fd4e63d9c728a793f4dc6c457b6bee5737789745170d2150d858ddd\nSUPERVISOR_TOP_CONST_COUNT=4\nSUPERVISOR_TEST_RC=0\nSUPERVISOR_TEST_STDOUT_SHA=9b4438f36f4a40de86d41fd480c6900c28603079d96011398674df7637868c5e\nSUPERVISOR_TEST_OUTPUT_BEGIN\nSUPERVISOR_STATUS=OK\nSUPERVISOR_PROBLEMS=none\nMONITOR=OK reports=22 open=13 resolved=9\nCANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z\nREGISTRY_SOURCE=\n>\nNETBIRD_ROUTE= dev wt0 table 7120 src uid 0\nSUPERVISOR_TEST_OUTPUT_END\nSUPTEST_SUPERVISOR_STATUS=OK\nSUPTEST_SUPERVISOR_PROBLEMS=none\nSUPTEST_MONITOR=OK reports=22 open=13 resolved=9\nSUPTEST_CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z\nSUPTEST_REGISTRY_SOURCE=[PRIVATE_IP]\nSUPTEST_WORKER_TARGET=debian@[PRIVATE_IP]\nSUPTEST_NETBIRD_ROUTE=1.1.1.1 dev wt0 table 7120 src 100.125.187.138 uid 0\nQUEUE_PRESENT=true\nQUEUE_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\nQUEUE_SHEBANG=#!/usr/bin/env bash\nQUEUE_BASH_N_RC=0\nQUEUE_FUNCTIONS=backup_list,die,dispatch_one,dispatch_plan,freshness_decision,freshness_plan,lock_heavy,migrate_list,plan,verify_all\nQUEUE_ACTIONS=BACKUP_RESUME,BACKUP_UNVERIFIED,SEALED_PROVEN,SKIP_FRESH\nQUEUE_VAR_NAMES=DUE_SECONDS,FRESHNESS_SECONDS,HEAVY_LOCK,PROBE,QUEUE_LOCK,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,WORKER,age,decision,freshness,generation,now_epoch,out,sealed_epoch\nQUEUE_COMMAND_BASENAMES=flock,homelab-app-cloud-quorum-worker,sha256sum\nQUEUE_INTERESTING_LINE_COUNT=32\nQUEUE_LINE_1_NO=4\nQUEUE_LINE_1_TEXT=WORKER=/usr/local/libexec/homelab-app-cloud-quorum-worker\nQUEUE_LINE_2_NO=5\nQUEUE_LINE_2_TEXT=PROBE=/usr/local/libexec/homelab-app-cloud-quorum-freshness-probe\nQUEUE_LINE_3_NO=6\nQUEUE_LINE_3_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nQUEUE_LINE_4_NO=8\nQUEUE_LINE_4_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nQUEUE_LINE_5_NO=9\nQUEUE_LINE_5_TEXT=QUEUE_LOCK=/run/lock/homelab-app-cloud-quorum-queue.lock\nQUEUE_LINE_6_NO=10\nQUEUE_LINE_6_TEXT=HEAVY_LOCK=/run/homelab-backup-heavy.lock\nQUEUE_LINE_7_NO=21\nQUEUE_LINE_7_TEXT=[[ -x \"$WORKER\" && ! -L \"$WORKER\" ]] || die WORKER_MISSING 69\nQUEUE_LINE_8_NO=29\nQUEUE_LINE_8_TEXT= for a in \"${ALL_APPS[@]}\"; do \"$WORKER\" plan \"$a\"; done\nQUEUE_LINE_9_NO=32\nQUEUE_LINE_9_TEXT= local app=\"$1\" now_epoch=\"$2\" out decision generation sealed_epoch age\nQUEUE_LINE_10_NO=38\nQUEUE_LINE_10_TEXT= generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nQUEUE_LINE_11_NO=39\nQUEUE_LINE_11_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_RESUME GENERATION=%s\\n' \"$app\" \"$generation\"\nQUEUE_LINE_12_NO=42\nQUEUE_LINE_12_TEXT= generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nQUEUE_LINE_13_NO=50\nQUEUE_LINE_13_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\"\nQUEUE_LINE_14_NO=52\nQUEUE_LINE_14_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\"\nQUEUE_LINE_15_NO=54\nQUEUE_LINE_15_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\" \"$DUE_SECONDS\" \"$FRESHNESS_SECONDS\"\nQUEUE_LINE_16_NO=56\nQUEUE_LINE_16_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\\n' \"$app\" \"$generation\" \"$age\"\nQUEUE_LINE_17_NO=84\nQUEUE_LINE_17_TEXT= out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"; printf '%s\\n' \"$out\"\nQUEUE_LINE_18_NO=85\nQUEUE_LINE_18_TEXT= \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\nQUEUE_LINE_19_NO=93\nQUEUE_LINE_19_TEXT= local a=\"$1\" freshness decision now_epoch out generation\nQUEUE_LINE_20_NO=112\nQUEUE_LINE_20_TEXT= generation=\"$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,\"\",$i);print $i}}' <<<\"$freshness\" | tail -1)\"\nQUEUE_LINE_21_NO=113\nQUEUE_LINE_21_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SKIP_FRESH GENERATION=%s\\n' \"$a\" \"$generation\"\nQUEUE_LINE_22_NO=120\nQUEUE_LINE_22_TEXT= out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"\nQUEUE_LINE_23_NO=122\nQUEUE_LINE_23_TEXT= \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\nQUEUE_LINE_24_NO=123\nQUEUE_LINE_24_TEXT= generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nQUEUE_LINE_25_NO=124\nQUEUE_LINE_25_TEXT= [[ -n \"$generation\" ]] || die \"GENERATION_NOT_RETURNED_$a\" 65\nQUEUE_LINE_26_NO=125\nQUEUE_LINE_26_TEXT= printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\\n' \"$a\" \"$generation\"\nQUEUE_LINE_27_NO=126\nQUEUE_LINE_27_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\\n' \"$a\" \"$generation\"\nQUEUE_LINE_28_NO=133\nQUEUE_LINE_28_TEXT= if out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" latest-sealed \"$a\" 2>/dev/null)\"; then\nQUEUE_LINE_29_NO=138\nQUEUE_LINE_29_TEXT= out=\"$(exec 8>&-; exec 9>&-; \"$WORKER\" backup \"$a\")\"; printf '%s\\n' \"$out\"\nQUEUE_LINE_30_NO=139\nQUEUE_LINE_30_TEXT= \"$WORKER\" latest-sealed \"$a\" 8>&- 9>&- >/dev/null\nQUEUE_LINE_31_NO=145\nQUEUE_LINE_31_TEXT= for a in \"${ALL_APPS[@]}\"; do out=\"$(\"$WORKER\" latest-sealed \"$a\")\"; printf '%s\\n' \"$out\"; done\nQUEUE_LINE_32_NO=153\nQUEUE_LINE_32_TEXT= all) [[ \"$#\" == 1 ]] || die USAGE_ALL 64; [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die DISPATCHER_MISSING 69; /usr/local/sbin/homelab-backup-dispatcher run APP_TIMER ;;\nQUEUE_SERVICE_SHOW_RC=0\nQUEUE_SERVICE_LOADSTATE=loaded\nQUEUE_SERVICE_ACTIVESTATE=inactive\nQUEUE_SERVICE_UNITFILESTATE=static\nQUEUE_SERVICE_RESULT=success\nQUEUE_SERVICE_EXECMAINSTATUS=0\nQUEUE_SERVICE_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.service\nQUEUE_SERVICE_FRAGMENT_SHA=afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a\nQUEUE_SERVICE_EXECSTART_SHA=22094bb14d8dc5f7399744e183b096583a9c9625527a2f00981f2ec5cd8e1713\nQUEUE_SERVICE_EXEC_BASENAMES=homelab-backup-v2-app-scheduled\nQUEUE_SERVICE_EXEC_ACTIONS=NONE\nQUEUE_TIMER_SHOW_RC=0\nQUEUE_TIMER_LOADSTATE=loaded\nQUEUE_TIMER_ACTIVESTATE=active\nQUEUE_TIMER_UNITFILESTATE=enabled\nQUEUE_TIMER_RESULT=success\nQUEUE_TIMER_EXECMAINSTATUS=\nQUEUE_TIMER_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.timer\nQUEUE_TIMER_FRAGMENT_SHA=af2eedce3df53029b591d31715a3f3e6688ddb997b4823f487e70c339ce88078\nQUEUE_TIMER_EXECSTART_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nQUEUE_TIMER_EXEC_BASENAMES=NONE\nQUEUE_TIMER_EXEC_ACTIONS=NONE\nWORKER_CANDIDATE_COUNT=173\nWORKER_1_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_1_PATH_HINT=//root/BACKUP_closeout_evidence_20260825T165406Z/source/homelab-app-cloud-quorum-worker\nWORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_1_EXECUTABLE=false\nWORKER_1_PRESENT=true\nWORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_1_SHEBANG=#!/usr/bin/env bash\nWORKER_1_BASH_N_RC=0\nWORKER_1_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_1_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor\nWORKER_1_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_1_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_1_INTERESTING_LINE_COUNT=57\nWORKER_1_LINE_1_NO=4\nWORKER_1_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_1_LINE_2_NO=6\nWORKER_1_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_1_LINE_3_NO=7\nWORKER_1_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_1_LINE_4_NO=8\nWORKER_1_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_1_LINE_5_NO=9\nWORKER_1_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_1_LINE_6_NO=10\nWORKER_1_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_1_LINE_7_NO=12\nWORKER_1_LINE_7_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_1_LINE_8_NO=14\nWORKER_1_LINE_8_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_1_LINE_9_NO=75\nWORKER_1_LINE_9_TEXT=logical_id(){\nWORKER_1_LINE_10_NO=82\nWORKER_1_LINE_10_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_1_LINE_11_NO=83\nWORKER_1_LINE_11_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_1_LINE_12_NO=101\nWORKER_1_LINE_12_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_1_LINE_13_NO=103\nWORKER_1_LINE_13_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_1_LINE_14_NO=104\nWORKER_1_LINE_14_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_1_LINE_15_NO=115\nWORKER_1_LINE_15_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_1_LINE_16_NO=117\nWORKER_1_LINE_16_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_1_LINE_17_NO=118\nWORKER_1_LINE_17_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_1_LINE_18_NO=119\nWORKER_1_LINE_18_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_1_LINE_19_NO=126\nWORKER_1_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_1_LINE_20_NO=141\nWORKER_1_LINE_20_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_1_LINE_21_NO=160\nWORKER_1_LINE_21_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_22_NO=191\nWORKER_1_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_23_NO=202\nWORKER_1_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_1_LINE_24_NO=218\nWORKER_1_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_25_NO=239\nWORKER_1_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_26_NO=274\nWORKER_1_LINE_26_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_1_LINE_27_NO=275\nWORKER_1_LINE_27_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_1_LINE_28_NO=293\nWORKER_1_LINE_28_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_1_LINE_29_NO=341\nWORKER_1_LINE_29_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_1_LINE_30_NO=346\nWORKER_1_LINE_30_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_1_LINE_31_NO=349\nWORKER_1_LINE_31_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_1_LINE_32_NO=350\nWORKER_1_LINE_32_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_1_LINE_33_NO=353\nWORKER_1_LINE_33_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_1_LINE_34_NO=354\nWORKER_1_LINE_34_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_1_LINE_35_NO=362\nWORKER_1_LINE_35_TEXT= done <\"$manifest\"\nWORKER_1_LINE_36_NO=431\nWORKER_1_LINE_36_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_1_LINE_37_NO=432\nWORKER_1_LINE_37_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_1_LINE_38_NO=466\nWORKER_1_LINE_38_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_1_LINE_39_NO=471\nWORKER_1_LINE_39_TEXT= if \"manifest.json\" not in names:\nWORKER_1_LINE_40_NO=473\nWORKER_1_LINE_40_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_1_LINE_41_NO=475\nWORKER_1_LINE_41_TEXT= for row in manifest:\nWORKER_1_LINE_42_NO=516\nWORKER_1_LINE_42_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_1_LINE_43_NO=528\nWORKER_1_LINE_43_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_1_LINE_44_NO=530\nWORKER_1_LINE_44_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_1_LINE_45_NO=536\nWORKER_1_LINE_45_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_1_LINE_46_NO=539\nWORKER_1_LINE_46_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_1_LINE_47_NO=540\nWORKER_1_LINE_47_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_1_LINE_48_NO=546\nWORKER_1_LINE_48_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_1_LINE_49_NO=553\nWORKER_1_LINE_49_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_1_LINE_50_NO=560\nWORKER_1_LINE_50_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_1_LINE_51_NO=596\nWORKER_1_LINE_51_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_1_LINE_52_NO=598\nWORKER_1_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_1_LINE_53_NO=644\nWORKER_1_LINE_53_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_1_LINE_54_NO=655\nWORKER_1_LINE_54_TEXT=ACTION=\"${1:-}\"\nWORKER_1_LINE_55_NO=656\nWORKER_1_LINE_55_TEXT=case \"$ACTION\" in\nWORKER_1_LINE_56_NO=658\nWORKER_1_LINE_56_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_1_LINE_57_NO=659\nWORKER_1_LINE_57_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_2_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_2_PATH_HINT=root/backup-registry-sha-pins-20260904T062507Z/usr/local/libexec/homelab-app-cloud-quorum-worker\nWORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_2_EXECUTABLE=true\nWORKER_2_PRESENT=true\nWORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913\nWORKER_2_SHEBANG=#!/usr/bin/env bash\nWORKER_2_BASH_N_RC=0\nWORKER_2_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_2_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor\nWORKER_2_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_2_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_2_INTERESTING_LINE_COUNT=57\nWORKER_2_LINE_1_NO=4\nWORKER_2_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_2_LINE_2_NO=6\nWORKER_2_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_2_LINE_3_NO=7\nWORKER_2_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_2_LINE_4_NO=8\nWORKER_2_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_2_LINE_5_NO=9\nWORKER_2_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_2_LINE_6_NO=10\nWORKER_2_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_2_LINE_7_NO=12\nWORKER_2_LINE_7_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_2_LINE_8_NO=14\nWORKER_2_LINE_8_TEXT=REGISTRY_HELPER=\"${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}\"\nWORKER_2_LINE_9_NO=75\nWORKER_2_LINE_9_TEXT=logical_id(){\nWORKER_2_LINE_10_NO=82\nWORKER_2_LINE_10_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_2_LINE_11_NO=83\nWORKER_2_LINE_11_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_2_LINE_12_NO=101\nWORKER_2_LINE_12_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_2_LINE_13_NO=103\nWORKER_2_LINE_13_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_2_LINE_14_NO=104\nWORKER_2_LINE_14_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_2_LINE_15_NO=115\nWORKER_2_LINE_15_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_2_LINE_16_NO=117\nWORKER_2_LINE_16_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_2_LINE_17_NO=118\nWORKER_2_LINE_17_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_2_LINE_18_NO=119\nWORKER_2_LINE_18_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_2_LINE_19_NO=126\nWORKER_2_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_2_LINE_20_NO=141\nWORKER_2_LINE_20_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_2_LINE_21_NO=160\nWORKER_2_LINE_21_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_22_NO=191\nWORKER_2_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_23_NO=202\nWORKER_2_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_2_LINE_24_NO=218\nWORKER_2_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_25_NO=239\nWORKER_2_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_26_NO=274\nWORKER_2_LINE_26_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_2_LINE_27_NO=275\nWORKER_2_LINE_27_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_2_LINE_28_NO=293\nWORKER_2_LINE_28_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_2_LINE_29_NO=341\nWORKER_2_LINE_29_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_2_LINE_30_NO=346\nWORKER_2_LINE_30_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_2_LINE_31_NO=349\nWORKER_2_LINE_31_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_2_LINE_32_NO=350\nWORKER_2_LINE_32_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_2_LINE_33_NO=353\nWORKER_2_LINE_33_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_2_LINE_34_NO=354\nWORKER_2_LINE_34_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_2_LINE_35_NO=362\nWORKER_2_LINE_35_TEXT= done <\"$manifest\"\nWORKER_2_LINE_36_NO=431\nWORKER_2_LINE_36_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_2_LINE_37_NO=432\nWORKER_2_LINE_37_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_2_LINE_38_NO=466\nWORKER_2_LINE_38_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_2_LINE_39_NO=471\nWORKER_2_LINE_39_TEXT= if \"manifest.json\" not in names:\nWORKER_2_LINE_40_NO=473\nWORKER_2_LINE_40_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_2_LINE_41_NO=475\nWORKER_2_LINE_41_TEXT= for row in manifest:\nWORKER_2_LINE_42_NO=516\nWORKER_2_LINE_42_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_2_LINE_43_NO=528\nWORKER_2_LINE_43_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_2_LINE_44_NO=530\nWORKER_2_LINE_44_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_2_LINE_45_NO=536\nWORKER_2_LINE_45_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_2_LINE_46_NO=539\nWORKER_2_LINE_46_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_2_LINE_47_NO=540\nWORKER_2_LINE_47_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_2_LINE_48_NO=546\nWORKER_2_LINE_48_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_2_LINE_49_NO=553\nWORKER_2_LINE_49_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_2_LINE_50_NO=560\nWORKER_2_LINE_50_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_2_LINE_51_NO=596\nWORKER_2_LINE_51_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_2_LINE_52_NO=598\nWORKER_2_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_2_LINE_53_NO=644\nWORKER_2_LINE_53_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_2_LINE_54_NO=655\nWORKER_2_LINE_54_TEXT=ACTION=\"${1:-}\"\nWORKER_2_LINE_55_NO=656\nWORKER_2_LINE_55_TEXT=case \"$ACTION\" in\nWORKER_2_LINE_56_NO=658\nWORKER_2_LINE_56_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_2_LINE_57_NO=659\nWORKER_2_LINE_57_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_3_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_3_PATH_HINT=root/homelab-ops-cr-2026-0024/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker\nWORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_3_EXECUTABLE=true\nWORKER_3_PRESENT=true\nWORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_3_SHEBANG=#!/usr/bin/env bash\nWORKER_3_BASH_N_RC=0\nWORKER_3_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_3_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_3_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work\nWORKER_3_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd\nWORKER_3_INTERESTING_LINE_COUNT=52\nWORKER_3_LINE_1_NO=4\nWORKER_3_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_3_LINE_2_NO=6\nWORKER_3_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_3_LINE_3_NO=7\nWORKER_3_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_3_LINE_4_NO=8\nWORKER_3_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_3_LINE_5_NO=9\nWORKER_3_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_3_LINE_6_NO=10\nWORKER_3_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_3_LINE_7_NO=45\nWORKER_3_LINE_7_TEXT=logical_id(){\nWORKER_3_LINE_8_NO=50\nWORKER_3_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_3_LINE_9_NO=64\nWORKER_3_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_3_LINE_10_NO=65\nWORKER_3_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_3_LINE_11_NO=83\nWORKER_3_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_3_LINE_12_NO=85\nWORKER_3_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_3_LINE_13_NO=86\nWORKER_3_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_3_LINE_14_NO=97\nWORKER_3_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_3_LINE_15_NO=99\nWORKER_3_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_3_LINE_16_NO=100\nWORKER_3_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_3_LINE_17_NO=101\nWORKER_3_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_3_LINE_18_NO=108\nWORKER_3_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_3_LINE_19_NO=123\nWORKER_3_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_3_LINE_20_NO=142\nWORKER_3_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_21_NO=150\nWORKER_3_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_3_LINE_22_NO=161\nWORKER_3_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_3_LINE_23_NO=192\nWORKER_3_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_24_NO=203\nWORKER_3_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_3_LINE_25_NO=219\nWORKER_3_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_26_NO=240\nWORKER_3_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_3_LINE_27_NO=256\nWORKER_3_LINE_27_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_3_LINE_28_NO=261\nWORKER_3_LINE_28_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_3_LINE_29_NO=264\nWORKER_3_LINE_29_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_3_LINE_30_NO=265\nWORKER_3_LINE_30_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_3_LINE_31_NO=268\nWORKER_3_LINE_31_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_3_LINE_32_NO=269\nWORKER_3_LINE_32_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_3_LINE_33_NO=277\nWORKER_3_LINE_33_TEXT= done <\"$manifest\"\nWORKER_3_LINE_34_NO=344\nWORKER_3_LINE_34_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_3_LINE_35_NO=349\nWORKER_3_LINE_35_TEXT= if \"manifest.json\" not in names:\nWORKER_3_LINE_36_NO=351\nWORKER_3_LINE_36_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_3_LINE_37_NO=353\nWORKER_3_LINE_37_TEXT= for row in manifest:\nWORKER_3_LINE_38_NO=394\nWORKER_3_LINE_38_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_3_LINE_39_NO=406\nWORKER_3_LINE_39_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_3_LINE_40_NO=408\nWORKER_3_LINE_40_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_3_LINE_41_NO=414\nWORKER_3_LINE_41_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_3_LINE_42_NO=417\nWORKER_3_LINE_42_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_3_LINE_43_NO=418\nWORKER_3_LINE_43_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_3_LINE_44_NO=424\nWORKER_3_LINE_44_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_3_LINE_45_NO=427\nWORKER_3_LINE_45_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_3_LINE_46_NO=434\nWORKER_3_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_3_LINE_47_NO=459\nWORKER_3_LINE_47_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_3_LINE_48_NO=461\nWORKER_3_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_3_LINE_49_NO=463\nWORKER_3_LINE_49_TEXT=ACTION=\"${1:-}\"\nWORKER_3_LINE_50_NO=464\nWORKER_3_LINE_50_TEXT=case \"$ACTION\" in\nWORKER_3_LINE_51_NO=466\nWORKER_3_LINE_51_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_3_LINE_52_NO=467\nWORKER_3_LINE_52_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_4_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_4_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker\nWORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_4_EXECUTABLE=true\nWORKER_4_PRESENT=true\nWORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309\nWORKER_4_SHEBANG=#!/usr/bin/env bash\nWORKER_4_BASH_N_RC=0\nWORKER_4_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_4_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_4_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work\nWORKER_4_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd\nWORKER_4_INTERESTING_LINE_COUNT=52\nWORKER_4_LINE_1_NO=4\nWORKER_4_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_4_LINE_2_NO=6\nWORKER_4_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_4_LINE_3_NO=7\nWORKER_4_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_4_LINE_4_NO=8\nWORKER_4_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_4_LINE_5_NO=9\nWORKER_4_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_4_LINE_6_NO=10\nWORKER_4_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_4_LINE_7_NO=45\nWORKER_4_LINE_7_TEXT=logical_id(){\nWORKER_4_LINE_8_NO=50\nWORKER_4_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_4_LINE_9_NO=64\nWORKER_4_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_4_LINE_10_NO=65\nWORKER_4_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_4_LINE_11_NO=83\nWORKER_4_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_4_LINE_12_NO=85\nWORKER_4_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_4_LINE_13_NO=86\nWORKER_4_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_4_LINE_14_NO=97\nWORKER_4_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_4_LINE_15_NO=99\nWORKER_4_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_4_LINE_16_NO=100\nWORKER_4_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_4_LINE_17_NO=101\nWORKER_4_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_4_LINE_18_NO=108\nWORKER_4_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_4_LINE_19_NO=123\nWORKER_4_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_4_LINE_20_NO=142\nWORKER_4_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_21_NO=150\nWORKER_4_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_4_LINE_22_NO=161\nWORKER_4_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_4_LINE_23_NO=192\nWORKER_4_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_24_NO=203\nWORKER_4_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_4_LINE_25_NO=219\nWORKER_4_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_26_NO=240\nWORKER_4_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_4_LINE_27_NO=256\nWORKER_4_LINE_27_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_4_LINE_28_NO=261\nWORKER_4_LINE_28_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_4_LINE_29_NO=264\nWORKER_4_LINE_29_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_4_LINE_30_NO=265\nWORKER_4_LINE_30_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_4_LINE_31_NO=268\nWORKER_4_LINE_31_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_4_LINE_32_NO=269\nWORKER_4_LINE_32_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_4_LINE_33_NO=277\nWORKER_4_LINE_33_TEXT= done <\"$manifest\"\nWORKER_4_LINE_34_NO=344\nWORKER_4_LINE_34_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_4_LINE_35_NO=349\nWORKER_4_LINE_35_TEXT= if \"manifest.json\" not in names:\nWORKER_4_LINE_36_NO=351\nWORKER_4_LINE_36_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_4_LINE_37_NO=353\nWORKER_4_LINE_37_TEXT= for row in manifest:\nWORKER_4_LINE_38_NO=394\nWORKER_4_LINE_38_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_4_LINE_39_NO=406\nWORKER_4_LINE_39_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_4_LINE_40_NO=408\nWORKER_4_LINE_40_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_4_LINE_41_NO=414\nWORKER_4_LINE_41_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_4_LINE_42_NO=417\nWORKER_4_LINE_42_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_4_LINE_43_NO=418\nWORKER_4_LINE_43_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_4_LINE_44_NO=424\nWORKER_4_LINE_44_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_4_LINE_45_NO=427\nWORKER_4_LINE_45_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_4_LINE_46_NO=434\nWORKER_4_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_4_LINE_47_NO=459\nWORKER_4_LINE_47_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_4_LINE_48_NO=461\nWORKER_4_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_4_LINE_49_NO=463\nWORKER_4_LINE_49_TEXT=ACTION=\"${1:-}\"\nWORKER_4_LINE_50_NO=464\nWORKER_4_LINE_50_TEXT=case \"$ACTION\" in\nWORKER_4_LINE_51_NO=466\nWORKER_4_LINE_51_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_4_LINE_52_NO=467\nWORKER_4_LINE_52_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_5_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_5_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-scrutiny-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker\nWORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_5_EXECUTABLE=true\nWORKER_5_PRESENT=true\nWORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_5_SHEBANG=#!/usr/bin/env bash\nWORKER_5_BASH_N_RC=0\nWORKER_5_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_5_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_5_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_5_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_5_INTERESTING_LINE_COUNT=58\nWORKER_5_LINE_1_NO=4\nWORKER_5_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_5_LINE_2_NO=6\nWORKER_5_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_5_LINE_3_NO=7\nWORKER_5_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_5_LINE_4_NO=8\nWORKER_5_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_5_LINE_5_NO=9\nWORKER_5_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_5_LINE_6_NO=10\nWORKER_5_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_5_LINE_7_NO=69\nWORKER_5_LINE_7_TEXT=logical_id(){\nWORKER_5_LINE_8_NO=74\nWORKER_5_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_5_LINE_9_NO=90\nWORKER_5_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_5_LINE_10_NO=91\nWORKER_5_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_5_LINE_11_NO=109\nWORKER_5_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_5_LINE_12_NO=111\nWORKER_5_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_5_LINE_13_NO=112\nWORKER_5_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_5_LINE_14_NO=123\nWORKER_5_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_5_LINE_15_NO=125\nWORKER_5_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_5_LINE_16_NO=126\nWORKER_5_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_5_LINE_17_NO=127\nWORKER_5_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_5_LINE_18_NO=134\nWORKER_5_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_5_LINE_19_NO=149\nWORKER_5_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_5_LINE_20_NO=168\nWORKER_5_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_21_NO=176\nWORKER_5_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_5_LINE_22_NO=187\nWORKER_5_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_5_LINE_23_NO=218\nWORKER_5_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_24_NO=229\nWORKER_5_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_5_LINE_25_NO=245\nWORKER_5_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_26_NO=266\nWORKER_5_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_27_NO=301\nWORKER_5_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_5_LINE_28_NO=302\nWORKER_5_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_5_LINE_29_NO=320\nWORKER_5_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_5_LINE_30_NO=368\nWORKER_5_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_5_LINE_31_NO=373\nWORKER_5_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_5_LINE_32_NO=376\nWORKER_5_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_5_LINE_33_NO=377\nWORKER_5_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_5_LINE_34_NO=380\nWORKER_5_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_5_LINE_35_NO=381\nWORKER_5_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_5_LINE_36_NO=389\nWORKER_5_LINE_36_TEXT= done <\"$manifest\"\nWORKER_5_LINE_37_NO=464\nWORKER_5_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_5_LINE_38_NO=465\nWORKER_5_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_5_LINE_39_NO=499\nWORKER_5_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_5_LINE_40_NO=504\nWORKER_5_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_5_LINE_41_NO=506\nWORKER_5_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_5_LINE_42_NO=508\nWORKER_5_LINE_42_TEXT= for row in manifest:\nWORKER_5_LINE_43_NO=549\nWORKER_5_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_5_LINE_44_NO=561\nWORKER_5_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_5_LINE_45_NO=563\nWORKER_5_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_5_LINE_46_NO=569\nWORKER_5_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_5_LINE_47_NO=572\nWORKER_5_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_5_LINE_48_NO=573\nWORKER_5_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_5_LINE_49_NO=579\nWORKER_5_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_5_LINE_50_NO=586\nWORKER_5_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_5_LINE_51_NO=593\nWORKER_5_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_5_LINE_52_NO=630\nWORKER_5_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_5_LINE_53_NO=632\nWORKER_5_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_5_LINE_54_NO=678\nWORKER_5_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_5_LINE_55_NO=689\nWORKER_5_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_5_LINE_56_NO=690\nWORKER_5_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_5_LINE_57_NO=692\nWORKER_5_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_5_LINE_58_NO=693\nWORKER_5_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_6_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_6_PATH_HINT=homelab-ops-cr-2026-0029/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_6_EXECUTABLE=true\nWORKER_6_PRESENT=true\nWORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_6_SHEBANG=#!/usr/bin/env bash\nWORKER_6_BASH_N_RC=0\nWORKER_6_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_6_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_6_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_6_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_6_INTERESTING_LINE_COUNT=58\nWORKER_6_LINE_1_NO=4\nWORKER_6_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_6_LINE_2_NO=6\nWORKER_6_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_6_LINE_3_NO=7\nWORKER_6_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_6_LINE_4_NO=8\nWORKER_6_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_6_LINE_5_NO=9\nWORKER_6_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_6_LINE_6_NO=10\nWORKER_6_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_6_LINE_7_NO=69\nWORKER_6_LINE_7_TEXT=logical_id(){\nWORKER_6_LINE_8_NO=74\nWORKER_6_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_6_LINE_9_NO=90\nWORKER_6_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_6_LINE_10_NO=91\nWORKER_6_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_6_LINE_11_NO=109\nWORKER_6_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_6_LINE_12_NO=111\nWORKER_6_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_6_LINE_13_NO=112\nWORKER_6_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_6_LINE_14_NO=123\nWORKER_6_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_6_LINE_15_NO=125\nWORKER_6_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_6_LINE_16_NO=126\nWORKER_6_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_6_LINE_17_NO=127\nWORKER_6_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_6_LINE_18_NO=134\nWORKER_6_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_6_LINE_19_NO=149\nWORKER_6_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_6_LINE_20_NO=168\nWORKER_6_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_21_NO=176\nWORKER_6_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_6_LINE_22_NO=187\nWORKER_6_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_6_LINE_23_NO=218\nWORKER_6_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_24_NO=229\nWORKER_6_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_6_LINE_25_NO=245\nWORKER_6_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_26_NO=266\nWORKER_6_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_27_NO=301\nWORKER_6_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_6_LINE_28_NO=302\nWORKER_6_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_6_LINE_29_NO=320\nWORKER_6_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_6_LINE_30_NO=368\nWORKER_6_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_6_LINE_31_NO=373\nWORKER_6_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_6_LINE_32_NO=376\nWORKER_6_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_6_LINE_33_NO=377\nWORKER_6_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_6_LINE_34_NO=380\nWORKER_6_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_6_LINE_35_NO=381\nWORKER_6_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_6_LINE_36_NO=389\nWORKER_6_LINE_36_TEXT= done <\"$manifest\"\nWORKER_6_LINE_37_NO=464\nWORKER_6_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_6_LINE_38_NO=465\nWORKER_6_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_6_LINE_39_NO=499\nWORKER_6_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_6_LINE_40_NO=504\nWORKER_6_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_6_LINE_41_NO=506\nWORKER_6_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_6_LINE_42_NO=508\nWORKER_6_LINE_42_TEXT= for row in manifest:\nWORKER_6_LINE_43_NO=549\nWORKER_6_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_6_LINE_44_NO=561\nWORKER_6_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_6_LINE_45_NO=563\nWORKER_6_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_6_LINE_46_NO=569\nWORKER_6_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_6_LINE_47_NO=572\nWORKER_6_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_6_LINE_48_NO=573\nWORKER_6_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_6_LINE_49_NO=579\nWORKER_6_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_6_LINE_50_NO=586\nWORKER_6_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_6_LINE_51_NO=593\nWORKER_6_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_6_LINE_52_NO=630\nWORKER_6_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_6_LINE_53_NO=632\nWORKER_6_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_6_LINE_54_NO=678\nWORKER_6_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_6_LINE_55_NO=689\nWORKER_6_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_6_LINE_56_NO=690\nWORKER_6_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_6_LINE_57_NO=692\nWORKER_6_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_6_LINE_58_NO=693\nWORKER_6_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_7_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_7_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_7_EXECUTABLE=true\nWORKER_7_PRESENT=true\nWORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_7_SHEBANG=#!/usr/bin/env bash\nWORKER_7_BASH_N_RC=0\nWORKER_7_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_7_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_7_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_7_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_7_INTERESTING_LINE_COUNT=58\nWORKER_7_LINE_1_NO=4\nWORKER_7_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_7_LINE_2_NO=6\nWORKER_7_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_7_LINE_3_NO=7\nWORKER_7_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_7_LINE_4_NO=8\nWORKER_7_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_7_LINE_5_NO=9\nWORKER_7_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_7_LINE_6_NO=10\nWORKER_7_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_7_LINE_7_NO=69\nWORKER_7_LINE_7_TEXT=logical_id(){\nWORKER_7_LINE_8_NO=74\nWORKER_7_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_7_LINE_9_NO=90\nWORKER_7_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_7_LINE_10_NO=91\nWORKER_7_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_7_LINE_11_NO=109\nWORKER_7_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_7_LINE_12_NO=111\nWORKER_7_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_7_LINE_13_NO=112\nWORKER_7_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_7_LINE_14_NO=123\nWORKER_7_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_7_LINE_15_NO=125\nWORKER_7_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_7_LINE_16_NO=126\nWORKER_7_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_7_LINE_17_NO=127\nWORKER_7_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_7_LINE_18_NO=134\nWORKER_7_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_7_LINE_19_NO=149\nWORKER_7_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_7_LINE_20_NO=168\nWORKER_7_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_21_NO=176\nWORKER_7_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_7_LINE_22_NO=187\nWORKER_7_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_7_LINE_23_NO=218\nWORKER_7_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_24_NO=229\nWORKER_7_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_7_LINE_25_NO=245\nWORKER_7_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_26_NO=266\nWORKER_7_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_27_NO=301\nWORKER_7_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_7_LINE_28_NO=302\nWORKER_7_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_7_LINE_29_NO=320\nWORKER_7_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_7_LINE_30_NO=368\nWORKER_7_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_7_LINE_31_NO=373\nWORKER_7_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_7_LINE_32_NO=376\nWORKER_7_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_7_LINE_33_NO=377\nWORKER_7_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_7_LINE_34_NO=380\nWORKER_7_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_7_LINE_35_NO=381\nWORKER_7_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_7_LINE_36_NO=389\nWORKER_7_LINE_36_TEXT= done <\"$manifest\"\nWORKER_7_LINE_37_NO=464\nWORKER_7_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_7_LINE_38_NO=465\nWORKER_7_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_7_LINE_39_NO=499\nWORKER_7_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_7_LINE_40_NO=504\nWORKER_7_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_7_LINE_41_NO=506\nWORKER_7_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_7_LINE_42_NO=508\nWORKER_7_LINE_42_TEXT= for row in manifest:\nWORKER_7_LINE_43_NO=549\nWORKER_7_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_7_LINE_44_NO=561\nWORKER_7_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_7_LINE_45_NO=563\nWORKER_7_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_7_LINE_46_NO=569\nWORKER_7_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_7_LINE_47_NO=572\nWORKER_7_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_7_LINE_48_NO=573\nWORKER_7_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_7_LINE_49_NO=579\nWORKER_7_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_7_LINE_50_NO=586\nWORKER_7_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_7_LINE_51_NO=593\nWORKER_7_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_7_LINE_52_NO=630\nWORKER_7_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_7_LINE_53_NO=632\nWORKER_7_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_7_LINE_54_NO=678\nWORKER_7_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_7_LINE_55_NO=689\nWORKER_7_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_7_LINE_56_NO=690\nWORKER_7_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_7_LINE_57_NO=692\nWORKER_7_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_7_LINE_58_NO=693\nWORKER_7_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_8_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_8_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_8_EXECUTABLE=true\nWORKER_8_PRESENT=true\nWORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_8_SHEBANG=#!/usr/bin/env bash\nWORKER_8_BASH_N_RC=0\nWORKER_8_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_8_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_8_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_8_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_8_INTERESTING_LINE_COUNT=58\nWORKER_8_LINE_1_NO=4\nWORKER_8_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_8_LINE_2_NO=6\nWORKER_8_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_8_LINE_3_NO=7\nWORKER_8_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_8_LINE_4_NO=8\nWORKER_8_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_8_LINE_5_NO=9\nWORKER_8_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_8_LINE_6_NO=10\nWORKER_8_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_8_LINE_7_NO=69\nWORKER_8_LINE_7_TEXT=logical_id(){\nWORKER_8_LINE_8_NO=74\nWORKER_8_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_8_LINE_9_NO=90\nWORKER_8_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_8_LINE_10_NO=91\nWORKER_8_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_8_LINE_11_NO=109\nWORKER_8_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_8_LINE_12_NO=111\nWORKER_8_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_8_LINE_13_NO=112\nWORKER_8_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_8_LINE_14_NO=123\nWORKER_8_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_8_LINE_15_NO=125\nWORKER_8_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_8_LINE_16_NO=126\nWORKER_8_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_8_LINE_17_NO=127\nWORKER_8_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_8_LINE_18_NO=134\nWORKER_8_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_8_LINE_19_NO=149\nWORKER_8_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_8_LINE_20_NO=168\nWORKER_8_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_21_NO=176\nWORKER_8_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_8_LINE_22_NO=187\nWORKER_8_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_8_LINE_23_NO=218\nWORKER_8_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_24_NO=229\nWORKER_8_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_8_LINE_25_NO=245\nWORKER_8_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_26_NO=266\nWORKER_8_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_27_NO=301\nWORKER_8_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_8_LINE_28_NO=302\nWORKER_8_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_8_LINE_29_NO=320\nWORKER_8_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_8_LINE_30_NO=368\nWORKER_8_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_8_LINE_31_NO=373\nWORKER_8_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_8_LINE_32_NO=376\nWORKER_8_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_8_LINE_33_NO=377\nWORKER_8_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_8_LINE_34_NO=380\nWORKER_8_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_8_LINE_35_NO=381\nWORKER_8_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_8_LINE_36_NO=389\nWORKER_8_LINE_36_TEXT= done <\"$manifest\"\nWORKER_8_LINE_37_NO=464\nWORKER_8_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_8_LINE_38_NO=465\nWORKER_8_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_8_LINE_39_NO=499\nWORKER_8_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_8_LINE_40_NO=504\nWORKER_8_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_8_LINE_41_NO=506\nWORKER_8_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_8_LINE_42_NO=508\nWORKER_8_LINE_42_TEXT= for row in manifest:\nWORKER_8_LINE_43_NO=549\nWORKER_8_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_8_LINE_44_NO=561\nWORKER_8_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_8_LINE_45_NO=563\nWORKER_8_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_8_LINE_46_NO=569\nWORKER_8_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_8_LINE_47_NO=572\nWORKER_8_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_8_LINE_48_NO=573\nWORKER_8_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_8_LINE_49_NO=579\nWORKER_8_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_8_LINE_50_NO=586\nWORKER_8_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_8_LINE_51_NO=593\nWORKER_8_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_8_LINE_52_NO=630\nWORKER_8_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_8_LINE_53_NO=632\nWORKER_8_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_8_LINE_54_NO=678\nWORKER_8_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_8_LINE_55_NO=689\nWORKER_8_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_8_LINE_56_NO=690\nWORKER_8_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_8_LINE_57_NO=692\nWORKER_8_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_8_LINE_58_NO=693\nWORKER_8_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_9_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_9_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_9_EXECUTABLE=true\nWORKER_9_PRESENT=true\nWORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_9_SHEBANG=#!/usr/bin/env bash\nWORKER_9_BASH_N_RC=0\nWORKER_9_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_9_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_9_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_9_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_9_INTERESTING_LINE_COUNT=58\nWORKER_9_LINE_1_NO=4\nWORKER_9_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_9_LINE_2_NO=6\nWORKER_9_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_9_LINE_3_NO=7\nWORKER_9_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_9_LINE_4_NO=8\nWORKER_9_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_9_LINE_5_NO=9\nWORKER_9_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_9_LINE_6_NO=10\nWORKER_9_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_9_LINE_7_NO=69\nWORKER_9_LINE_7_TEXT=logical_id(){\nWORKER_9_LINE_8_NO=74\nWORKER_9_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_9_LINE_9_NO=90\nWORKER_9_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_9_LINE_10_NO=91\nWORKER_9_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_9_LINE_11_NO=109\nWORKER_9_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_9_LINE_12_NO=111\nWORKER_9_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_9_LINE_13_NO=112\nWORKER_9_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_9_LINE_14_NO=123\nWORKER_9_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_9_LINE_15_NO=125\nWORKER_9_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_9_LINE_16_NO=126\nWORKER_9_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_9_LINE_17_NO=127\nWORKER_9_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_9_LINE_18_NO=134\nWORKER_9_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_9_LINE_19_NO=149\nWORKER_9_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_9_LINE_20_NO=168\nWORKER_9_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_21_NO=176\nWORKER_9_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_9_LINE_22_NO=187\nWORKER_9_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_9_LINE_23_NO=218\nWORKER_9_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_24_NO=229\nWORKER_9_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_9_LINE_25_NO=245\nWORKER_9_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_26_NO=266\nWORKER_9_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_27_NO=301\nWORKER_9_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_9_LINE_28_NO=302\nWORKER_9_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_9_LINE_29_NO=320\nWORKER_9_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_9_LINE_30_NO=368\nWORKER_9_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_9_LINE_31_NO=373\nWORKER_9_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_9_LINE_32_NO=376\nWORKER_9_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_9_LINE_33_NO=377\nWORKER_9_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_9_LINE_34_NO=380\nWORKER_9_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_9_LINE_35_NO=381\nWORKER_9_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_9_LINE_36_NO=389\nWORKER_9_LINE_36_TEXT= done <\"$manifest\"\nWORKER_9_LINE_37_NO=464\nWORKER_9_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_9_LINE_38_NO=465\nWORKER_9_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_9_LINE_39_NO=499\nWORKER_9_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_9_LINE_40_NO=504\nWORKER_9_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_9_LINE_41_NO=506\nWORKER_9_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_9_LINE_42_NO=508\nWORKER_9_LINE_42_TEXT= for row in manifest:\nWORKER_9_LINE_43_NO=549\nWORKER_9_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_9_LINE_44_NO=561\nWORKER_9_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_9_LINE_45_NO=563\nWORKER_9_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_9_LINE_46_NO=569\nWORKER_9_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_9_LINE_47_NO=572\nWORKER_9_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_9_LINE_48_NO=573\nWORKER_9_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_9_LINE_49_NO=579\nWORKER_9_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_9_LINE_50_NO=586\nWORKER_9_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_9_LINE_51_NO=593\nWORKER_9_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_9_LINE_52_NO=630\nWORKER_9_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_9_LINE_53_NO=632\nWORKER_9_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_9_LINE_54_NO=678\nWORKER_9_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_9_LINE_55_NO=689\nWORKER_9_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_9_LINE_56_NO=690\nWORKER_9_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_9_LINE_57_NO=692\nWORKER_9_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_9_LINE_58_NO=693\nWORKER_9_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_10_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_10_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_10_EXECUTABLE=true\nWORKER_10_PRESENT=true\nWORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_10_SHEBANG=#!/usr/bin/env bash\nWORKER_10_BASH_N_RC=0\nWORKER_10_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_10_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_10_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_10_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_10_INTERESTING_LINE_COUNT=58\nWORKER_10_LINE_1_NO=4\nWORKER_10_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_10_LINE_2_NO=6\nWORKER_10_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_10_LINE_3_NO=7\nWORKER_10_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_10_LINE_4_NO=8\nWORKER_10_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_10_LINE_5_NO=9\nWORKER_10_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_10_LINE_6_NO=10\nWORKER_10_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_10_LINE_7_NO=69\nWORKER_10_LINE_7_TEXT=logical_id(){\nWORKER_10_LINE_8_NO=74\nWORKER_10_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_10_LINE_9_NO=90\nWORKER_10_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_10_LINE_10_NO=91\nWORKER_10_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_10_LINE_11_NO=109\nWORKER_10_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_10_LINE_12_NO=111\nWORKER_10_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_10_LINE_13_NO=112\nWORKER_10_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_10_LINE_14_NO=123\nWORKER_10_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_10_LINE_15_NO=125\nWORKER_10_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_10_LINE_16_NO=126\nWORKER_10_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_10_LINE_17_NO=127\nWORKER_10_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_10_LINE_18_NO=134\nWORKER_10_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_10_LINE_19_NO=149\nWORKER_10_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_10_LINE_20_NO=168\nWORKER_10_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_21_NO=176\nWORKER_10_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_10_LINE_22_NO=187\nWORKER_10_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_10_LINE_23_NO=218\nWORKER_10_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_24_NO=229\nWORKER_10_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_10_LINE_25_NO=245\nWORKER_10_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_26_NO=266\nWORKER_10_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_27_NO=301\nWORKER_10_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_10_LINE_28_NO=302\nWORKER_10_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_10_LINE_29_NO=320\nWORKER_10_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_10_LINE_30_NO=368\nWORKER_10_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_10_LINE_31_NO=373\nWORKER_10_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_10_LINE_32_NO=376\nWORKER_10_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_10_LINE_33_NO=377\nWORKER_10_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_10_LINE_34_NO=380\nWORKER_10_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_10_LINE_35_NO=381\nWORKER_10_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_10_LINE_36_NO=389\nWORKER_10_LINE_36_TEXT= done <\"$manifest\"\nWORKER_10_LINE_37_NO=464\nWORKER_10_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_10_LINE_38_NO=465\nWORKER_10_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_10_LINE_39_NO=499\nWORKER_10_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_10_LINE_40_NO=504\nWORKER_10_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_10_LINE_41_NO=506\nWORKER_10_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_10_LINE_42_NO=508\nWORKER_10_LINE_42_TEXT= for row in manifest:\nWORKER_10_LINE_43_NO=549\nWORKER_10_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_10_LINE_44_NO=561\nWORKER_10_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_10_LINE_45_NO=563\nWORKER_10_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_10_LINE_46_NO=569\nWORKER_10_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_10_LINE_47_NO=572\nWORKER_10_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_10_LINE_48_NO=573\nWORKER_10_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_10_LINE_49_NO=579\nWORKER_10_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_10_LINE_50_NO=586\nWORKER_10_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_10_LINE_51_NO=593\nWORKER_10_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_10_LINE_52_NO=630\nWORKER_10_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_10_LINE_53_NO=632\nWORKER_10_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_10_LINE_54_NO=678\nWORKER_10_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_10_LINE_55_NO=689\nWORKER_10_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_10_LINE_56_NO=690\nWORKER_10_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_10_LINE_57_NO=692\nWORKER_10_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_10_LINE_58_NO=693\nWORKER_10_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_11_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_11_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_11_EXECUTABLE=true\nWORKER_11_PRESENT=true\nWORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_11_SHEBANG=#!/usr/bin/env bash\nWORKER_11_BASH_N_RC=0\nWORKER_11_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_11_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_11_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_11_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_11_INTERESTING_LINE_COUNT=58\nWORKER_11_LINE_1_NO=4\nWORKER_11_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_11_LINE_2_NO=6\nWORKER_11_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_11_LINE_3_NO=7\nWORKER_11_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_11_LINE_4_NO=8\nWORKER_11_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_11_LINE_5_NO=9\nWORKER_11_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_11_LINE_6_NO=10\nWORKER_11_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_11_LINE_7_NO=69\nWORKER_11_LINE_7_TEXT=logical_id(){\nWORKER_11_LINE_8_NO=74\nWORKER_11_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_11_LINE_9_NO=90\nWORKER_11_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_11_LINE_10_NO=91\nWORKER_11_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_11_LINE_11_NO=109\nWORKER_11_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_11_LINE_12_NO=111\nWORKER_11_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_11_LINE_13_NO=112\nWORKER_11_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_11_LINE_14_NO=123\nWORKER_11_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_11_LINE_15_NO=125\nWORKER_11_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_11_LINE_16_NO=126\nWORKER_11_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_11_LINE_17_NO=127\nWORKER_11_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_11_LINE_18_NO=134\nWORKER_11_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_11_LINE_19_NO=149\nWORKER_11_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_11_LINE_20_NO=168\nWORKER_11_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_21_NO=176\nWORKER_11_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_11_LINE_22_NO=187\nWORKER_11_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_11_LINE_23_NO=218\nWORKER_11_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_24_NO=229\nWORKER_11_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_11_LINE_25_NO=245\nWORKER_11_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_26_NO=266\nWORKER_11_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_27_NO=301\nWORKER_11_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_11_LINE_28_NO=302\nWORKER_11_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_11_LINE_29_NO=320\nWORKER_11_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_11_LINE_30_NO=368\nWORKER_11_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_11_LINE_31_NO=373\nWORKER_11_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_11_LINE_32_NO=376\nWORKER_11_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_11_LINE_33_NO=377\nWORKER_11_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_11_LINE_34_NO=380\nWORKER_11_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_11_LINE_35_NO=381\nWORKER_11_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_11_LINE_36_NO=389\nWORKER_11_LINE_36_TEXT= done <\"$manifest\"\nWORKER_11_LINE_37_NO=464\nWORKER_11_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_11_LINE_38_NO=465\nWORKER_11_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_11_LINE_39_NO=499\nWORKER_11_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_11_LINE_40_NO=504\nWORKER_11_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_11_LINE_41_NO=506\nWORKER_11_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_11_LINE_42_NO=508\nWORKER_11_LINE_42_TEXT= for row in manifest:\nWORKER_11_LINE_43_NO=549\nWORKER_11_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_11_LINE_44_NO=561\nWORKER_11_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_11_LINE_45_NO=563\nWORKER_11_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_11_LINE_46_NO=569\nWORKER_11_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_11_LINE_47_NO=572\nWORKER_11_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_11_LINE_48_NO=573\nWORKER_11_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_11_LINE_49_NO=579\nWORKER_11_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_11_LINE_50_NO=586\nWORKER_11_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_11_LINE_51_NO=593\nWORKER_11_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_11_LINE_52_NO=630\nWORKER_11_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_11_LINE_53_NO=632\nWORKER_11_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_11_LINE_54_NO=678\nWORKER_11_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_11_LINE_55_NO=689\nWORKER_11_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_11_LINE_56_NO=690\nWORKER_11_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_11_LINE_57_NO=692\nWORKER_11_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_11_LINE_58_NO=693\nWORKER_11_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_12_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_12_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_12_EXECUTABLE=true\nWORKER_12_PRESENT=true\nWORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_12_SHEBANG=#!/usr/bin/env bash\nWORKER_12_BASH_N_RC=0\nWORKER_12_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_12_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_12_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_12_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_12_INTERESTING_LINE_COUNT=58\nWORKER_12_LINE_1_NO=4\nWORKER_12_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_12_LINE_2_NO=6\nWORKER_12_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_12_LINE_3_NO=7\nWORKER_12_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_12_LINE_4_NO=8\nWORKER_12_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_12_LINE_5_NO=9\nWORKER_12_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_12_LINE_6_NO=10\nWORKER_12_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_12_LINE_7_NO=69\nWORKER_12_LINE_7_TEXT=logical_id(){\nWORKER_12_LINE_8_NO=74\nWORKER_12_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_12_LINE_9_NO=90\nWORKER_12_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_12_LINE_10_NO=91\nWORKER_12_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_12_LINE_11_NO=109\nWORKER_12_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_12_LINE_12_NO=111\nWORKER_12_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_12_LINE_13_NO=112\nWORKER_12_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_12_LINE_14_NO=123\nWORKER_12_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_12_LINE_15_NO=125\nWORKER_12_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_12_LINE_16_NO=126\nWORKER_12_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_12_LINE_17_NO=127\nWORKER_12_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_12_LINE_18_NO=134\nWORKER_12_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_12_LINE_19_NO=149\nWORKER_12_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_12_LINE_20_NO=168\nWORKER_12_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_21_NO=176\nWORKER_12_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_12_LINE_22_NO=187\nWORKER_12_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_12_LINE_23_NO=218\nWORKER_12_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_24_NO=229\nWORKER_12_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_12_LINE_25_NO=245\nWORKER_12_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_26_NO=266\nWORKER_12_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_27_NO=301\nWORKER_12_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_12_LINE_28_NO=302\nWORKER_12_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_12_LINE_29_NO=320\nWORKER_12_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_12_LINE_30_NO=368\nWORKER_12_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_12_LINE_31_NO=373\nWORKER_12_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_12_LINE_32_NO=376\nWORKER_12_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_12_LINE_33_NO=377\nWORKER_12_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_12_LINE_34_NO=380\nWORKER_12_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_12_LINE_35_NO=381\nWORKER_12_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_12_LINE_36_NO=389\nWORKER_12_LINE_36_TEXT= done <\"$manifest\"\nWORKER_12_LINE_37_NO=464\nWORKER_12_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_12_LINE_38_NO=465\nWORKER_12_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_12_LINE_39_NO=499\nWORKER_12_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_12_LINE_40_NO=504\nWORKER_12_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_12_LINE_41_NO=506\nWORKER_12_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_12_LINE_42_NO=508\nWORKER_12_LINE_42_TEXT= for row in manifest:\nWORKER_12_LINE_43_NO=549\nWORKER_12_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_12_LINE_44_NO=561\nWORKER_12_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_12_LINE_45_NO=563\nWORKER_12_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_12_LINE_46_NO=569\nWORKER_12_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_12_LINE_47_NO=572\nWORKER_12_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_12_LINE_48_NO=573\nWORKER_12_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_12_LINE_49_NO=579\nWORKER_12_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_12_LINE_50_NO=586\nWORKER_12_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_12_LINE_51_NO=593\nWORKER_12_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_12_LINE_52_NO=630\nWORKER_12_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_12_LINE_53_NO=632\nWORKER_12_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_12_LINE_54_NO=678\nWORKER_12_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_12_LINE_55_NO=689\nWORKER_12_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_12_LINE_56_NO=690\nWORKER_12_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_12_LINE_57_NO=692\nWORKER_12_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_12_LINE_58_NO=693\nWORKER_12_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_13_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_13_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_13_EXECUTABLE=true\nWORKER_13_PRESENT=true\nWORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_13_SHEBANG=#!/usr/bin/env bash\nWORKER_13_BASH_N_RC=0\nWORKER_13_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_13_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_13_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_13_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_13_INTERESTING_LINE_COUNT=58\nWORKER_13_LINE_1_NO=4\nWORKER_13_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_13_LINE_2_NO=6\nWORKER_13_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_13_LINE_3_NO=7\nWORKER_13_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_13_LINE_4_NO=8\nWORKER_13_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_13_LINE_5_NO=9\nWORKER_13_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_13_LINE_6_NO=10\nWORKER_13_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_13_LINE_7_NO=69\nWORKER_13_LINE_7_TEXT=logical_id(){\nWORKER_13_LINE_8_NO=74\nWORKER_13_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_13_LINE_9_NO=90\nWORKER_13_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_13_LINE_10_NO=91\nWORKER_13_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_13_LINE_11_NO=109\nWORKER_13_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_13_LINE_12_NO=111\nWORKER_13_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_13_LINE_13_NO=112\nWORKER_13_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_13_LINE_14_NO=123\nWORKER_13_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_13_LINE_15_NO=125\nWORKER_13_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_13_LINE_16_NO=126\nWORKER_13_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_13_LINE_17_NO=127\nWORKER_13_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_13_LINE_18_NO=134\nWORKER_13_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_13_LINE_19_NO=149\nWORKER_13_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_13_LINE_20_NO=168\nWORKER_13_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_21_NO=176\nWORKER_13_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_13_LINE_22_NO=187\nWORKER_13_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_13_LINE_23_NO=218\nWORKER_13_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_24_NO=229\nWORKER_13_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_13_LINE_25_NO=245\nWORKER_13_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_26_NO=266\nWORKER_13_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_27_NO=301\nWORKER_13_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_13_LINE_28_NO=302\nWORKER_13_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_13_LINE_29_NO=320\nWORKER_13_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_13_LINE_30_NO=368\nWORKER_13_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_13_LINE_31_NO=373\nWORKER_13_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_13_LINE_32_NO=376\nWORKER_13_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_13_LINE_33_NO=377\nWORKER_13_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_13_LINE_34_NO=380\nWORKER_13_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_13_LINE_35_NO=381\nWORKER_13_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_13_LINE_36_NO=389\nWORKER_13_LINE_36_TEXT= done <\"$manifest\"\nWORKER_13_LINE_37_NO=464\nWORKER_13_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_13_LINE_38_NO=465\nWORKER_13_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_13_LINE_39_NO=499\nWORKER_13_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_13_LINE_40_NO=504\nWORKER_13_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_13_LINE_41_NO=506\nWORKER_13_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_13_LINE_42_NO=508\nWORKER_13_LINE_42_TEXT= for row in manifest:\nWORKER_13_LINE_43_NO=549\nWORKER_13_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_13_LINE_44_NO=561\nWORKER_13_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_13_LINE_45_NO=563\nWORKER_13_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_13_LINE_46_NO=569\nWORKER_13_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_13_LINE_47_NO=572\nWORKER_13_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_13_LINE_48_NO=573\nWORKER_13_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_13_LINE_49_NO=579\nWORKER_13_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_13_LINE_50_NO=586\nWORKER_13_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_13_LINE_51_NO=593\nWORKER_13_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_13_LINE_52_NO=630\nWORKER_13_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_13_LINE_53_NO=632\nWORKER_13_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_13_LINE_54_NO=678\nWORKER_13_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_13_LINE_55_NO=689\nWORKER_13_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_13_LINE_56_NO=690\nWORKER_13_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_13_LINE_57_NO=692\nWORKER_13_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_13_LINE_58_NO=693\nWORKER_13_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_14_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_14_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_14_EXECUTABLE=true\nWORKER_14_PRESENT=true\nWORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_14_SHEBANG=#!/usr/bin/env bash\nWORKER_14_BASH_N_RC=0\nWORKER_14_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_14_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_14_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_14_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_14_INTERESTING_LINE_COUNT=58\nWORKER_14_LINE_1_NO=4\nWORKER_14_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_14_LINE_2_NO=6\nWORKER_14_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_14_LINE_3_NO=7\nWORKER_14_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_14_LINE_4_NO=8\nWORKER_14_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_14_LINE_5_NO=9\nWORKER_14_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_14_LINE_6_NO=10\nWORKER_14_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_14_LINE_7_NO=69\nWORKER_14_LINE_7_TEXT=logical_id(){\nWORKER_14_LINE_8_NO=74\nWORKER_14_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_14_LINE_9_NO=90\nWORKER_14_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_14_LINE_10_NO=91\nWORKER_14_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_14_LINE_11_NO=109\nWORKER_14_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_14_LINE_12_NO=111\nWORKER_14_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_14_LINE_13_NO=112\nWORKER_14_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_14_LINE_14_NO=123\nWORKER_14_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_14_LINE_15_NO=125\nWORKER_14_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_14_LINE_16_NO=126\nWORKER_14_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_14_LINE_17_NO=127\nWORKER_14_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_14_LINE_18_NO=134\nWORKER_14_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_14_LINE_19_NO=149\nWORKER_14_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_14_LINE_20_NO=168\nWORKER_14_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_21_NO=176\nWORKER_14_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_14_LINE_22_NO=187\nWORKER_14_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_14_LINE_23_NO=218\nWORKER_14_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_24_NO=229\nWORKER_14_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_14_LINE_25_NO=245\nWORKER_14_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_26_NO=266\nWORKER_14_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_27_NO=301\nWORKER_14_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_14_LINE_28_NO=302\nWORKER_14_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_14_LINE_29_NO=320\nWORKER_14_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_14_LINE_30_NO=368\nWORKER_14_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_14_LINE_31_NO=373\nWORKER_14_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_14_LINE_32_NO=376\nWORKER_14_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_14_LINE_33_NO=377\nWORKER_14_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_14_LINE_34_NO=380\nWORKER_14_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_14_LINE_35_NO=381\nWORKER_14_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_14_LINE_36_NO=389\nWORKER_14_LINE_36_TEXT= done <\"$manifest\"\nWORKER_14_LINE_37_NO=464\nWORKER_14_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_14_LINE_38_NO=465\nWORKER_14_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_14_LINE_39_NO=499\nWORKER_14_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_14_LINE_40_NO=504\nWORKER_14_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_14_LINE_41_NO=506\nWORKER_14_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_14_LINE_42_NO=508\nWORKER_14_LINE_42_TEXT= for row in manifest:\nWORKER_14_LINE_43_NO=549\nWORKER_14_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_14_LINE_44_NO=561\nWORKER_14_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_14_LINE_45_NO=563\nWORKER_14_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_14_LINE_46_NO=569\nWORKER_14_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_14_LINE_47_NO=572\nWORKER_14_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_14_LINE_48_NO=573\nWORKER_14_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_14_LINE_49_NO=579\nWORKER_14_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_14_LINE_50_NO=586\nWORKER_14_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_14_LINE_51_NO=593\nWORKER_14_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_14_LINE_52_NO=630\nWORKER_14_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_14_LINE_53_NO=632\nWORKER_14_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_14_LINE_54_NO=678\nWORKER_14_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_14_LINE_55_NO=689\nWORKER_14_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_14_LINE_56_NO=690\nWORKER_14_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_14_LINE_57_NO=692\nWORKER_14_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_14_LINE_58_NO=693\nWORKER_14_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_15_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_15_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_15_EXECUTABLE=true\nWORKER_15_PRESENT=true\nWORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_15_SHEBANG=#!/usr/bin/env bash\nWORKER_15_BASH_N_RC=0\nWORKER_15_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_15_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_15_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_15_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_15_INTERESTING_LINE_COUNT=58\nWORKER_15_LINE_1_NO=4\nWORKER_15_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_15_LINE_2_NO=6\nWORKER_15_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_15_LINE_3_NO=7\nWORKER_15_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_15_LINE_4_NO=8\nWORKER_15_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_15_LINE_5_NO=9\nWORKER_15_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_15_LINE_6_NO=10\nWORKER_15_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_15_LINE_7_NO=69\nWORKER_15_LINE_7_TEXT=logical_id(){\nWORKER_15_LINE_8_NO=74\nWORKER_15_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_15_LINE_9_NO=90\nWORKER_15_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_15_LINE_10_NO=91\nWORKER_15_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_15_LINE_11_NO=109\nWORKER_15_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_15_LINE_12_NO=111\nWORKER_15_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_15_LINE_13_NO=112\nWORKER_15_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_15_LINE_14_NO=123\nWORKER_15_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_15_LINE_15_NO=125\nWORKER_15_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_15_LINE_16_NO=126\nWORKER_15_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_15_LINE_17_NO=127\nWORKER_15_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_15_LINE_18_NO=134\nWORKER_15_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_15_LINE_19_NO=149\nWORKER_15_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_15_LINE_20_NO=168\nWORKER_15_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_21_NO=176\nWORKER_15_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_15_LINE_22_NO=187\nWORKER_15_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_15_LINE_23_NO=218\nWORKER_15_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_24_NO=229\nWORKER_15_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_15_LINE_25_NO=245\nWORKER_15_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_26_NO=266\nWORKER_15_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_27_NO=301\nWORKER_15_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_15_LINE_28_NO=302\nWORKER_15_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_15_LINE_29_NO=320\nWORKER_15_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_15_LINE_30_NO=368\nWORKER_15_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_15_LINE_31_NO=373\nWORKER_15_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_15_LINE_32_NO=376\nWORKER_15_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_15_LINE_33_NO=377\nWORKER_15_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_15_LINE_34_NO=380\nWORKER_15_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_15_LINE_35_NO=381\nWORKER_15_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_15_LINE_36_NO=389\nWORKER_15_LINE_36_TEXT= done <\"$manifest\"\nWORKER_15_LINE_37_NO=464\nWORKER_15_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_15_LINE_38_NO=465\nWORKER_15_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_15_LINE_39_NO=499\nWORKER_15_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_15_LINE_40_NO=504\nWORKER_15_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_15_LINE_41_NO=506\nWORKER_15_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_15_LINE_42_NO=508\nWORKER_15_LINE_42_TEXT= for row in manifest:\nWORKER_15_LINE_43_NO=549\nWORKER_15_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_15_LINE_44_NO=561\nWORKER_15_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_15_LINE_45_NO=563\nWORKER_15_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_15_LINE_46_NO=569\nWORKER_15_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_15_LINE_47_NO=572\nWORKER_15_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_15_LINE_48_NO=573\nWORKER_15_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_15_LINE_49_NO=579\nWORKER_15_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_15_LINE_50_NO=586\nWORKER_15_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_15_LINE_51_NO=593\nWORKER_15_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_15_LINE_52_NO=630\nWORKER_15_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_15_LINE_53_NO=632\nWORKER_15_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_15_LINE_54_NO=678\nWORKER_15_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_15_LINE_55_NO=689\nWORKER_15_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_15_LINE_56_NO=690\nWORKER_15_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_15_LINE_57_NO=692\nWORKER_15_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_15_LINE_58_NO=693\nWORKER_15_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_16_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_16_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_16_EXECUTABLE=true\nWORKER_16_PRESENT=true\nWORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_16_SHEBANG=#!/usr/bin/env bash\nWORKER_16_BASH_N_RC=0\nWORKER_16_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_16_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_16_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_16_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_16_INTERESTING_LINE_COUNT=58\nWORKER_16_LINE_1_NO=4\nWORKER_16_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_16_LINE_2_NO=6\nWORKER_16_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_16_LINE_3_NO=7\nWORKER_16_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_16_LINE_4_NO=8\nWORKER_16_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_16_LINE_5_NO=9\nWORKER_16_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_16_LINE_6_NO=10\nWORKER_16_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_16_LINE_7_NO=69\nWORKER_16_LINE_7_TEXT=logical_id(){\nWORKER_16_LINE_8_NO=74\nWORKER_16_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_16_LINE_9_NO=90\nWORKER_16_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_16_LINE_10_NO=91\nWORKER_16_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_16_LINE_11_NO=109\nWORKER_16_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_16_LINE_12_NO=111\nWORKER_16_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_16_LINE_13_NO=112\nWORKER_16_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_16_LINE_14_NO=123\nWORKER_16_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_16_LINE_15_NO=125\nWORKER_16_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_16_LINE_16_NO=126\nWORKER_16_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_16_LINE_17_NO=127\nWORKER_16_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_16_LINE_18_NO=134\nWORKER_16_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_16_LINE_19_NO=149\nWORKER_16_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_16_LINE_20_NO=168\nWORKER_16_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_21_NO=176\nWORKER_16_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_16_LINE_22_NO=187\nWORKER_16_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_16_LINE_23_NO=218\nWORKER_16_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_24_NO=229\nWORKER_16_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_16_LINE_25_NO=245\nWORKER_16_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_26_NO=266\nWORKER_16_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_27_NO=301\nWORKER_16_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_16_LINE_28_NO=302\nWORKER_16_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_16_LINE_29_NO=320\nWORKER_16_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_16_LINE_30_NO=368\nWORKER_16_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_16_LINE_31_NO=373\nWORKER_16_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_16_LINE_32_NO=376\nWORKER_16_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_16_LINE_33_NO=377\nWORKER_16_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_16_LINE_34_NO=380\nWORKER_16_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_16_LINE_35_NO=381\nWORKER_16_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_16_LINE_36_NO=389\nWORKER_16_LINE_36_TEXT= done <\"$manifest\"\nWORKER_16_LINE_37_NO=464\nWORKER_16_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_16_LINE_38_NO=465\nWORKER_16_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_16_LINE_39_NO=499\nWORKER_16_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_16_LINE_40_NO=504\nWORKER_16_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_16_LINE_41_NO=506\nWORKER_16_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_16_LINE_42_NO=508\nWORKER_16_LINE_42_TEXT= for row in manifest:\nWORKER_16_LINE_43_NO=549\nWORKER_16_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_16_LINE_44_NO=561\nWORKER_16_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_16_LINE_45_NO=563\nWORKER_16_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_16_LINE_46_NO=569\nWORKER_16_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_16_LINE_47_NO=572\nWORKER_16_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_16_LINE_48_NO=573\nWORKER_16_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_16_LINE_49_NO=579\nWORKER_16_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_16_LINE_50_NO=586\nWORKER_16_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_16_LINE_51_NO=593\nWORKER_16_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_16_LINE_52_NO=630\nWORKER_16_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_16_LINE_53_NO=632\nWORKER_16_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_16_LINE_54_NO=678\nWORKER_16_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_16_LINE_55_NO=689\nWORKER_16_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_16_LINE_56_NO=690\nWORKER_16_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_16_LINE_57_NO=692\nWORKER_16_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_16_LINE_58_NO=693\nWORKER_16_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_17_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_17_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_17_EXECUTABLE=true\nWORKER_17_PRESENT=true\nWORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_17_SHEBANG=#!/usr/bin/env bash\nWORKER_17_BASH_N_RC=0\nWORKER_17_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_17_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_17_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_17_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_17_INTERESTING_LINE_COUNT=58\nWORKER_17_LINE_1_NO=4\nWORKER_17_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_17_LINE_2_NO=6\nWORKER_17_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_17_LINE_3_NO=7\nWORKER_17_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_17_LINE_4_NO=8\nWORKER_17_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_17_LINE_5_NO=9\nWORKER_17_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_17_LINE_6_NO=10\nWORKER_17_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_17_LINE_7_NO=69\nWORKER_17_LINE_7_TEXT=logical_id(){\nWORKER_17_LINE_8_NO=74\nWORKER_17_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_17_LINE_9_NO=90\nWORKER_17_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_17_LINE_10_NO=91\nWORKER_17_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_17_LINE_11_NO=109\nWORKER_17_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_17_LINE_12_NO=111\nWORKER_17_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_17_LINE_13_NO=112\nWORKER_17_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_17_LINE_14_NO=123\nWORKER_17_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_17_LINE_15_NO=125\nWORKER_17_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_17_LINE_16_NO=126\nWORKER_17_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_17_LINE_17_NO=127\nWORKER_17_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_17_LINE_18_NO=134\nWORKER_17_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_17_LINE_19_NO=149\nWORKER_17_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_17_LINE_20_NO=168\nWORKER_17_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_21_NO=176\nWORKER_17_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_17_LINE_22_NO=187\nWORKER_17_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_17_LINE_23_NO=218\nWORKER_17_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_24_NO=229\nWORKER_17_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_17_LINE_25_NO=245\nWORKER_17_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_26_NO=266\nWORKER_17_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_27_NO=301\nWORKER_17_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_17_LINE_28_NO=302\nWORKER_17_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_17_LINE_29_NO=320\nWORKER_17_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_17_LINE_30_NO=368\nWORKER_17_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_17_LINE_31_NO=373\nWORKER_17_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_17_LINE_32_NO=376\nWORKER_17_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_17_LINE_33_NO=377\nWORKER_17_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_17_LINE_34_NO=380\nWORKER_17_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_17_LINE_35_NO=381\nWORKER_17_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_17_LINE_36_NO=389\nWORKER_17_LINE_36_TEXT= done <\"$manifest\"\nWORKER_17_LINE_37_NO=464\nWORKER_17_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_17_LINE_38_NO=465\nWORKER_17_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_17_LINE_39_NO=499\nWORKER_17_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_17_LINE_40_NO=504\nWORKER_17_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_17_LINE_41_NO=506\nWORKER_17_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_17_LINE_42_NO=508\nWORKER_17_LINE_42_TEXT= for row in manifest:\nWORKER_17_LINE_43_NO=549\nWORKER_17_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_17_LINE_44_NO=561\nWORKER_17_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_17_LINE_45_NO=563\nWORKER_17_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_17_LINE_46_NO=569\nWORKER_17_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_17_LINE_47_NO=572\nWORKER_17_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_17_LINE_48_NO=573\nWORKER_17_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_17_LINE_49_NO=579\nWORKER_17_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_17_LINE_50_NO=586\nWORKER_17_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_17_LINE_51_NO=593\nWORKER_17_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_17_LINE_52_NO=630\nWORKER_17_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_17_LINE_53_NO=632\nWORKER_17_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_17_LINE_54_NO=678\nWORKER_17_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_17_LINE_55_NO=689\nWORKER_17_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_17_LINE_56_NO=690\nWORKER_17_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_17_LINE_57_NO=692\nWORKER_17_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_17_LINE_58_NO=693\nWORKER_17_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_18_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_18_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_18_EXECUTABLE=true\nWORKER_18_PRESENT=true\nWORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_18_SHEBANG=#!/usr/bin/env bash\nWORKER_18_BASH_N_RC=0\nWORKER_18_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_18_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_18_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_18_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_18_INTERESTING_LINE_COUNT=58\nWORKER_18_LINE_1_NO=4\nWORKER_18_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_18_LINE_2_NO=6\nWORKER_18_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_18_LINE_3_NO=7\nWORKER_18_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_18_LINE_4_NO=8\nWORKER_18_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_18_LINE_5_NO=9\nWORKER_18_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_18_LINE_6_NO=10\nWORKER_18_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_18_LINE_7_NO=69\nWORKER_18_LINE_7_TEXT=logical_id(){\nWORKER_18_LINE_8_NO=74\nWORKER_18_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_18_LINE_9_NO=90\nWORKER_18_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_18_LINE_10_NO=91\nWORKER_18_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_18_LINE_11_NO=109\nWORKER_18_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_18_LINE_12_NO=111\nWORKER_18_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_18_LINE_13_NO=112\nWORKER_18_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_18_LINE_14_NO=123\nWORKER_18_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_18_LINE_15_NO=125\nWORKER_18_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_18_LINE_16_NO=126\nWORKER_18_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_18_LINE_17_NO=127\nWORKER_18_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_18_LINE_18_NO=134\nWORKER_18_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_18_LINE_19_NO=149\nWORKER_18_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_18_LINE_20_NO=168\nWORKER_18_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_21_NO=176\nWORKER_18_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_18_LINE_22_NO=187\nWORKER_18_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_18_LINE_23_NO=218\nWORKER_18_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_24_NO=229\nWORKER_18_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_18_LINE_25_NO=245\nWORKER_18_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_26_NO=266\nWORKER_18_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_27_NO=301\nWORKER_18_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_18_LINE_28_NO=302\nWORKER_18_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_18_LINE_29_NO=320\nWORKER_18_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_18_LINE_30_NO=368\nWORKER_18_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_18_LINE_31_NO=373\nWORKER_18_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_18_LINE_32_NO=376\nWORKER_18_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_18_LINE_33_NO=377\nWORKER_18_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_18_LINE_34_NO=380\nWORKER_18_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_18_LINE_35_NO=381\nWORKER_18_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_18_LINE_36_NO=389\nWORKER_18_LINE_36_TEXT= done <\"$manifest\"\nWORKER_18_LINE_37_NO=464\nWORKER_18_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_18_LINE_38_NO=465\nWORKER_18_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_18_LINE_39_NO=499\nWORKER_18_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_18_LINE_40_NO=504\nWORKER_18_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_18_LINE_41_NO=506\nWORKER_18_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_18_LINE_42_NO=508\nWORKER_18_LINE_42_TEXT= for row in manifest:\nWORKER_18_LINE_43_NO=549\nWORKER_18_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_18_LINE_44_NO=561\nWORKER_18_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_18_LINE_45_NO=563\nWORKER_18_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_18_LINE_46_NO=569\nWORKER_18_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_18_LINE_47_NO=572\nWORKER_18_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_18_LINE_48_NO=573\nWORKER_18_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_18_LINE_49_NO=579\nWORKER_18_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_18_LINE_50_NO=586\nWORKER_18_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_18_LINE_51_NO=593\nWORKER_18_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_18_LINE_52_NO=630\nWORKER_18_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_18_LINE_53_NO=632\nWORKER_18_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_18_LINE_54_NO=678\nWORKER_18_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_18_LINE_55_NO=689\nWORKER_18_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_18_LINE_56_NO=690\nWORKER_18_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_18_LINE_57_NO=692\nWORKER_18_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_18_LINE_58_NO=693\nWORKER_18_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_19_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_19_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_19_EXECUTABLE=true\nWORKER_19_PRESENT=true\nWORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_19_SHEBANG=#!/usr/bin/env bash\nWORKER_19_BASH_N_RC=0\nWORKER_19_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_19_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_19_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_19_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_19_INTERESTING_LINE_COUNT=58\nWORKER_19_LINE_1_NO=4\nWORKER_19_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_19_LINE_2_NO=6\nWORKER_19_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_19_LINE_3_NO=7\nWORKER_19_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_19_LINE_4_NO=8\nWORKER_19_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_19_LINE_5_NO=9\nWORKER_19_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_19_LINE_6_NO=10\nWORKER_19_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_19_LINE_7_NO=69\nWORKER_19_LINE_7_TEXT=logical_id(){\nWORKER_19_LINE_8_NO=74\nWORKER_19_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_19_LINE_9_NO=90\nWORKER_19_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_19_LINE_10_NO=91\nWORKER_19_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_19_LINE_11_NO=109\nWORKER_19_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_19_LINE_12_NO=111\nWORKER_19_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_19_LINE_13_NO=112\nWORKER_19_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_19_LINE_14_NO=123\nWORKER_19_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_19_LINE_15_NO=125\nWORKER_19_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_19_LINE_16_NO=126\nWORKER_19_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_19_LINE_17_NO=127\nWORKER_19_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_19_LINE_18_NO=134\nWORKER_19_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_19_LINE_19_NO=149\nWORKER_19_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_19_LINE_20_NO=168\nWORKER_19_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_21_NO=176\nWORKER_19_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_19_LINE_22_NO=187\nWORKER_19_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_19_LINE_23_NO=218\nWORKER_19_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_24_NO=229\nWORKER_19_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_19_LINE_25_NO=245\nWORKER_19_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_26_NO=266\nWORKER_19_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_27_NO=301\nWORKER_19_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_19_LINE_28_NO=302\nWORKER_19_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_19_LINE_29_NO=320\nWORKER_19_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_19_LINE_30_NO=368\nWORKER_19_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_19_LINE_31_NO=373\nWORKER_19_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_19_LINE_32_NO=376\nWORKER_19_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_19_LINE_33_NO=377\nWORKER_19_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_19_LINE_34_NO=380\nWORKER_19_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_19_LINE_35_NO=381\nWORKER_19_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_19_LINE_36_NO=389\nWORKER_19_LINE_36_TEXT= done <\"$manifest\"\nWORKER_19_LINE_37_NO=464\nWORKER_19_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_19_LINE_38_NO=465\nWORKER_19_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_19_LINE_39_NO=499\nWORKER_19_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_19_LINE_40_NO=504\nWORKER_19_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_19_LINE_41_NO=506\nWORKER_19_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_19_LINE_42_NO=508\nWORKER_19_LINE_42_TEXT= for row in manifest:\nWORKER_19_LINE_43_NO=549\nWORKER_19_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_19_LINE_44_NO=561\nWORKER_19_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_19_LINE_45_NO=563\nWORKER_19_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_19_LINE_46_NO=569\nWORKER_19_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_19_LINE_47_NO=572\nWORKER_19_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_19_LINE_48_NO=573\nWORKER_19_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_19_LINE_49_NO=579\nWORKER_19_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_19_LINE_50_NO=586\nWORKER_19_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_19_LINE_51_NO=593\nWORKER_19_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_19_LINE_52_NO=630\nWORKER_19_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_19_LINE_53_NO=632\nWORKER_19_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_19_LINE_54_NO=678\nWORKER_19_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_19_LINE_55_NO=689\nWORKER_19_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_19_LINE_56_NO=690\nWORKER_19_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_19_LINE_57_NO=692\nWORKER_19_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_19_LINE_58_NO=693\nWORKER_19_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nWORKER_20_BASENAME=homelab-app-cloud-quorum-worker\nWORKER_20_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker\nWORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_20_EXECUTABLE=true\nWORKER_20_PRESENT=true\nWORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72\nWORKER_20_SHEBANG=#!/usr/bin/env bash\nWORKER_20_BASH_N_RC=0\nWORKER_20_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding\nWORKER_20_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor\nWORKER_20_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work\nWORKER_20_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd\nWORKER_20_INTERESTING_LINE_COUNT=58\nWORKER_20_LINE_1_NO=4\nWORKER_20_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload\nWORKER_20_LINE_2_NO=6\nWORKER_20_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper\nWORKER_20_LINE_3_NO=7\nWORKER_20_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper\nWORKER_20_LINE_4_NO=8\nWORKER_20_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum\nWORKER_20_LINE_5_NO=9\nWORKER_20_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation\nWORKER_20_LINE_6_NO=10\nWORKER_20_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum\nWORKER_20_LINE_7_NO=69\nWORKER_20_LINE_7_TEXT=logical_id(){\nWORKER_20_LINE_8_NO=74\nWORKER_20_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;;\nWORKER_20_LINE_9_NO=90\nWORKER_20_LINE_9_TEXT=state_dir(){ echo \"$STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_20_LINE_10_NO=91\nWORKER_20_LINE_10_TEXT=restore_dir(){ echo \"$RESTORE_STATE_ROOT/$(logical_id \"$1\")\"; }\nWORKER_20_LINE_11_NO=109\nWORKER_20_LINE_11_TEXT= jq -e --arg l \"$(logical_id \"$app\")\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .transport_state==\"SEALED\" and .restore_source==\"Cloud A\" and .payload_sha256_verified==true and .result==\"OK\"' \"$f\" >/dev/null\nWORKER_20_LINE_12_NO=111\nWORKER_20_LINE_12_TEXT=remote_core(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$CORE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$CORE_HELPER\"; }\nWORKER_20_LINE_13_NO=112\nWORKER_20_LINE_13_TEXT=remote_edge(){ local action=\"$1\" gen=\"$2\"; ssh \"${SSH_BASE[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$action\" \"$gen\" <\"$EDGE_HELPER\"; }\nWORKER_20_LINE_14_NO=123\nWORKER_20_LINE_14_TEXT= local dir=\"$1\" manifest=\"$2\" label=\"$3\"\nWORKER_20_LINE_15_NO=125\nWORKER_20_LINE_15_TEXT= [[ -f \"$manifest\" && ! -L \"$manifest\" ]] || die \"MANIFEST_FILE_INVALID_${label}\" 70\nWORKER_20_LINE_16_NO=126\nWORKER_20_LINE_16_TEXT= if [[ -s \"$manifest\" ]]; then\nWORKER_20_LINE_17_NO=127\nWORKER_20_LINE_17_TEXT= (cd \"$dir\"; sha256sum -c \"$manifest\" >/dev/null) || die \"MANIFEST_CHECK_FAILED_${label}\" 70\nWORKER_20_LINE_18_NO=134\nWORKER_20_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg source \"$(source_host \"$app\")\" --arg started \"$started\" --arg completed \"$completed\" --arg qstart \"$qstart\" --arg qend \"$qend\" --argjson components \"$components_json\" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >\"$file\"\nWORKER_20_LINE_19_NO=149\nWORKER_20_LINE_19_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_20_LINE_20_NO=168\nWORKER_20_LINE_20_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-memos \"$gen\" memos \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_21_NO=176\nWORKER_20_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 \".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'\"\nWORKER_20_LINE_22_NO=187\nWORKER_20_LINE_22_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-moderator-assistant \"$gen\" skladchik-moderator-assistant \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_20_LINE_23_NO=218\nWORKER_20_LINE_23_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-filebrowser \"$gen\" filebrowser \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_24_NO=229\nWORKER_20_LINE_24_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-skladchik-reports-monitor \"$gen\" skladchik-reports-monitor \"$started\" \"$completed\" \"\" \"\" \"$comps\"\nWORKER_20_LINE_25_NO=245\nWORKER_20_LINE_25_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-immich \"$gen\" immich \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_26_NO=266\nWORKER_20_LINE_26_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-hbarhub \"$gen\" hbarhub \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_27_NO=301\nWORKER_20_LINE_27_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_20_LINE_28_NO=302\nWORKER_20_LINE_28_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_20_LINE_29_NO=320\nWORKER_20_LINE_29_TEXT= write_binding \"$CAPTURE/app-generation.json\" app-scrutiny \"$gen\" scrutiny \"$started\" \"$completed\" \"$qstart\" \"$qend\" \"$comps\"\nWORKER_20_LINE_30_NO=368\nWORKER_20_LINE_30_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.schema==\"homelab-app-generation-v1\" and .logical_id==$l and .generation==$g and (.components|type==\"object\" and length>0)' \"$ex/app-generation.json\" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70\nWORKER_20_LINE_31_NO=373\nWORKER_20_LINE_31_TEXT= done < <(jq -r '.' \"$ex/app-generation.json\")\nWORKER_20_LINE_32_NO=376\nWORKER_20_LINE_32_TEXT= local app=\"$1\" gen=\"$2\" state=\"$3\" out=\"$4\" logical base manifest expected_msha got_msha idx name size sha part actual\nWORKER_20_LINE_33_NO=377\nWORKER_20_LINE_33_TEXT= logical=\"$(logical_id \"$app\")\"\nWORKER_20_LINE_34_NO=380\nWORKER_20_LINE_34_TEXT= manifest=\"$(dirname \"$out\")/PARTS.tsv\"; rclone cat \"${base%/}/PARTS.tsv\" >\"$manifest\"\nWORKER_20_LINE_35_NO=381\nWORKER_20_LINE_35_TEXT= got_msha=\"$(sha256sum \"$manifest\"|awk '{print $1}')\"; [[ \"$got_msha\" == \"$expected_msha\" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70\nWORKER_20_LINE_36_NO=389\nWORKER_20_LINE_36_TEXT= done <\"$manifest\"\nWORKER_20_LINE_37_NO=464\nWORKER_20_LINE_37_TEXT= if \"manifest.json\" not in names: raise SystemExit(70)\nWORKER_20_LINE_38_NO=465\nWORKER_20_LINE_38_TEXT= rows=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_20_LINE_39_NO=499\nWORKER_20_LINE_39_TEXT=expected_tags={\"hbarhub-web:latest\",\"hbarhub-worker:latest\",\"hbarhub-scheduler:latest\"}\nWORKER_20_LINE_40_NO=504\nWORKER_20_LINE_40_TEXT= if \"manifest.json\" not in names:\nWORKER_20_LINE_41_NO=506\nWORKER_20_LINE_41_TEXT= manifest=json.load(tf.extractfile(\"manifest.json\"))\nWORKER_20_LINE_42_NO=508\nWORKER_20_LINE_42_TEXT= for row in manifest:\nWORKER_20_LINE_43_NO=549\nWORKER_20_LINE_43_TEXT= logical=\"$(logical_id \"$app\")\"; state=\"$STATE_ROOT/$logical/$gen.json\"\nWORKER_20_LINE_44_NO=561\nWORKER_20_LINE_44_TEXT= jq -n --arg logical_id \"$logical\" --arg generation \"$gen\" --arg app \"$app\" --arg sha \"$expected_sha\" --arg at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" --arg state \"$state\" --argjson checks \"$checks\" '{schema:\"homelab-app-restore-validation-v1\",logical_id:$logical_id,generation:$generation,app:$app,transport_state:\"SEALED\",transport_state_file:$state,restore_source:\"Cloud A\",payload_sha256:$sha,payload_sha256_verified:true,result:\"OK\",validated_at_utc:$at,checks:$checks}' >\"$tmp\"\nWORKER_20_LINE_45_NO=563\nWORKER_20_LINE_45_TEXT= jq -e --arg l \"$logical\" --arg g \"$gen\" '.logical_id==$l and .generation==$g and .result==\"OK\" and .transport_state==\"SEALED\" and .payload_sha256_verified==true' \"$out\" >/dev/null\nWORKER_20_LINE_46_NO=569\nWORKER_20_LINE_46_TEXT= logical=\"$(logical_id \"$app\")\"; mkdir -p \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT/$logical\"; chmod 0700 \"$STATE_ROOT\" \"$STATE_ROOT/$logical\" \"$RESTORE_STATE_ROOT\" \"$RESTORE_STATE_ROOT/$logical\" \"$STAGE_ROOT\" \"$STAGE_ROOT/$logical\"\nWORKER_20_LINE_47_NO=572\nWORKER_20_LINE_47_TEXT= state=\"${pending[0]}\"; gen=\"$(jq -er '.generation' \"$state\")\"; payload=\"$(jq -er '.payload_path' \"$state\")\"; printf 'RESUME=YES\\nAPP=%s\\nGENERATION=%s\\nSTAGE=TRANSPORT_RESUME\\n' \"$app\" \"$gen\"\nWORKER_20_LINE_48_NO=573\nWORKER_20_LINE_48_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_20_LINE_49_NO=579\nWORKER_20_LINE_49_TEXT= if [[ -n \"$latest\" ]]; then gen=\"$(jq -er '.generation' \"$latest\")\"; if ! restore_ok \"$app\" \"$gen\"; then printf 'RESUME_RESTORE_VALIDATION=YES\\nAPP=%s\\nGENERATION=%s\\n' \"$app\" \"$gen\"; restore_validate \"$app\" \"$gen\"; printf 'BACKUP_STATUS=SEALED\\nAPP=%s\\nGENERATION=%s\\nCLOUD_QUORUM=2\\n' \"$app\" \"$gen\"; return; fi; fi\nWORKER_20_LINE_50_NO=586\nWORKER_20_LINE_50_TEXT= printf 'STAGE=TRANSPORT\\n'; \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$payload\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_20_LINE_51_NO=593\nWORKER_20_LINE_51_TEXT= logical=\"$(logical_id \"$app\")\"; free=\"$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')\"\nWORKER_20_LINE_52_NO=630\nWORKER_20_LINE_52_TEXT= f=\"$(latest_sealed_state \"$app\")\"; [[ -n \"$f\" ]] || die \"SEALED_STATE_MISSING_$app\" 65; gen=\"$(jq -er '.generation' \"$f\")\"\nWORKER_20_LINE_53_NO=632\nWORKER_20_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\\nAPP=%s\\nLOGICAL_ID=%s\\nGENERATION=%s\\nSTATE_FILE=%s\\nRESTORE_EVIDENCE=%s\\n' \"$app\" \"$(logical_id \"$app\")\" \"$gen\" \"$f\" \"$(restore_dir \"$app\")/$gen.json\"\nWORKER_20_LINE_54_NO=678\nWORKER_20_LINE_54_TEXT= \"$TRANSPORT\" backup-delete \"$logical\" \"$gen\" \"$PAYLOAD\" \"$REMOTE_A\" \"$REMOTE_B\" \"$STATE_ROOT\"\nWORKER_20_LINE_55_NO=689\nWORKER_20_LINE_55_TEXT=ACTION=\"${1:-}\"\nWORKER_20_LINE_56_NO=690\nWORKER_20_LINE_56_TEXT=case \"$ACTION\" in\nWORKER_20_LINE_57_NO=692\nWORKER_20_LINE_57_TEXT= backup) [[ \"$#\" == 2 ]] || die USAGE_BACKUP 64; backup_one \"$2\" ;;\nWORKER_20_LINE_58_NO=693\nWORKER_20_LINE_58_TEXT= restore-validate) [[ \"$#\" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id \"$2\" >/dev/null; restore_validate \"$2\" \"$3\" ;;\nTARGETS_PRESENT=true\nTARGETS_SHA=2f526330152d49461ff66f50adab57d1a195ea5d1b4c292945d523411b8a68a8\nTARGET_OBJECT_FOUND=true\nTARGET_KEYS=id,kind,lifecycle,logical_id,migrate,owner,source_host\nTARGET_ID=skladchik-reports-monitor\nTARGET_KIND=APP\nTARGET_LIFECYCLE=ACTIVE\nTARGET_LOGICAL_ID=app-skladchik-reports-monitor\nTARGET_MIGRATE=True\nTARGET_OWNER=pve01\nTARGET_SOURCE_HOST_BASENAME=[PRIVATE_IP]\nTARGET_SOURCE_HOST_SHA=3eb2319e39eac307b3c7bd11f67663c0a0c814fd603989f5965083f9d1c80fb0\nQUEUE_APPID_LITERAL_COUNT=0\nQUEUE_TARGETS_BASENAME_LITERAL_COUNT=0\nBACKUP_AUDIT_RC=0\nBACKUP_AUDIT_RELEVANT_COUNT=2\nBACKUP_AUDIT_RELEVANT_1=TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=22759|generation=app-skladchik-reports-monitor-pve01-20261005T003035Z\nBACKUP_AUDIT_RELEVANT_2=TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=32145|legacy_age_seconds=1627370|replica_count=2|state_id=vm170|generation=v2-vm170-20261004T214054Z-70a537ab4e91\nTARGETED_SEAL_DESIGN_READY=true\nMAIL_SENT_BY_COMMAND=false\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\nDECISION=PASS_QUEUE_WORKER_RCA\nNEXT_GATE=DESIGN_ATOMIC_SOURCE_BACKUP_SEAL\nTASK_RESULT=SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA\n" + "output": "SNIKKET_OPS_OWNERSHIP_RCA_WRAPPER=V1\nMODE=read-only\nNO_DNS_MUTATION=true\nNO_VM_MUTATION=true\nNO_CONTAINER_RESTART=true\nNO_SERVICE_START=true\nNO_SOURCE_MUTATION=true\nHOST_PVE01=true\nINGRESS_HOSTKEY_SELF_BOUND=true\nVM150_INVENTORY_PRESENT=true\nVM150_RUNNING=true\nVM150_EXPECTED_NODE=true\nVM171_INVENTORY_PRESENT=true\nVM171_RUNNING=true\nVM171_EXPECTED_NODE=true\nVM150_QGA_PING=true\nVM171_QGA_PING=true\nPVE_BACKUP_JOBS_QUERY_RC=0\nPVE_BACKUP_JOB_COVERAGE_COUNT=0\nPVE_BACKUP_JOB_ENABLED_COVERAGE_COUNT=0\nPVE_BACKUP_STORAGE_QUERY_RC=0\nPVE_BACKUP_STORAGE_COUNT=1\nPVE_VM150_BACKUP_OBJECT_COUNT=0\nPVE_VM150_BACKUP_STORAGE_WITH_OBJECT_COUNT=0\nPVE_VM150_NEWEST_BACKUP_AGE_HOURS=-1\nBACKUP_ENGINE_VM150_REF_14D_COUNT=40\nBACKUP_ENGINE_VM150_SUCCESS_14D_COUNT=40\nBACKUP_ENGINE_VM150_LATEST_SUCCESS_AGE_HOURS=16\nBACKUP_AUDIT_VM150_REF_30D_COUNT=2379\nBACKUP_AUDIT_VM150_RESTORE_PASS_30D_COUNT=0\nBACKUP_AUDIT_VM150_LATEST_RESTORE_PASS_AGE_HOURS=-1\nBACKUP_EVIDENCE_CLASS_VZDUMP=true\nBACKUP_EVIDENCE_CLASS_RESTIC=false\nBACKUP_EVIDENCE_CLASS_BORG=false\nBACKUP_EVIDENCE_CLASS_RCLONE=false\nBACKUP_EVIDENCE_CLASS_RSYNC_TAR=false\nBACKUP_EVIDENCE_CLASS_DOCKER_DATA=false\nBACKUP_EVIDENCE_CLASS_REMOTE=true\nBACKUP_SOURCE_FILE_COUNT=482\nBACKUP_SOURCE_VM150_REF_COUNT=45\nBACKUP_SOURCE_CAP_VZDUMP=true\nBACKUP_SOURCE_CAP_RESTIC=false\nBACKUP_SOURCE_CAP_BORG=false\nBACKUP_SOURCE_CAP_RCLONE=true\nBACKUP_SOURCE_CAP_RSYNC_TAR=true\nBACKUP_SOURCE_CAP_DYNAMIC_VM=true\nPVE01_GIT_REPO_COUNT=64\nPVE01_SNIKKET_SOURCE_CANDIDATE_REPO_COUNT=49\nPVE01_SOURCE_RECON_EXACT_SHA_COUNT=0\nPVE01_SOURCE_COTURN_EXACT_SHA_COUNT=49\nPVE01_SOURCE_OVERRIDE_EXACT_SHA_COUNT=49\nPVE01_SOURCE_RECON_CORRECT_LOGIC_COUNT=0\nPVE01_SOURCE_DNS_ALL3_COUNT=343\nPVE01_SOURCE_DNS_CURRENT_TARGET_COUNT=294\nPVE01_MONITOR_SOURCE_FILE_COUNT=86\nPVE01_MONITOR_CHAT_REF=false\nPVE01_MONITOR_XMPP_5222_REF=false\nPVE01_MONITOR_XMPP_5269_REF=false\nPVE01_MONITOR_TURN_3479_REF=false\nPVE01_MONITOR_STUN_TURN_REF=false\nVM150_QGA_EXEC_INIT_RC=0\nVM150_QGA_POLL_RC=0\nVM_GUEST_EXITCODE=0\nVM_GUEST_OUTPUT_MARKER=true\nSNIKKET_OPS_OWNERSHIP_GUEST150=V1\nMODE=read-only\nNO_MUTATION=true\nGUEST150_HOSTNAME_OK=true\nGUEST150_LIVE_RECON_PRESENT=true\nGUEST150_LIVE_RECON_SHA256=bb692a3a044d41e6e04518aff5bdfa646f533660a70ea793f87d040c881a6e98\nGUEST150_LIVE_COTURN_PRESENT=true\nGUEST150_LIVE_COTURN_SHA256=2de0e47c09b3b4fc7a6c32c5ef061efaaa7c7f4db3d7a25fdf4da8fbb2e6f483\nGUEST150_LIVE_OVERRIDE_PRESENT=true\nGUEST150_LIVE_OVERRIDE_SHA256=54a42711db5c2eb99b3886184c7ce3e01a30aba844ff2022bc74a45414a2241c\nGUEST150_GIT_REPO_COUNT=0\nGUEST150_SNIKKET_TRACKED_FILE_COUNT=0\nGUEST150_SOURCE_CANDIDATE_REPO_COUNT=0\nGUEST150_SOURCE_RECON_EXACT_SHA_COUNT=0\nGUEST150_SOURCE_COTURN_EXACT_SHA_COUNT=0\nGUEST150_SOURCE_OVERRIDE_EXACT_SHA_COUNT=0\nGUEST150_SOURCE_RECON_CORRECT_LOGIC_COUNT=0\nGUEST150_RECON_SERVICE_UNIT_PRESENT=true\nGUEST150_RECON_TIMER_UNIT_PRESENT=true\nGUEST150_EDGE_UNIT_PRESENT=true\nGUEST150_TCP3479_COUNT=16\nGUEST150_UDP3479_COUNT=16\nGUEST150_RECON_SERVICE_RESULT=success\nGUEST150_RECON_SERVICE_EXEC_MAIN_STATUS=0\nGUEST150_AUDIT_DONE=true\nVM171_QGA_EXEC_INIT_RC=0\nVM171_QGA_POLL_RC=0\nVM_GUEST_EXITCODE=0\nVM_GUEST_OUTPUT_MARKER=true\nSNIKKET_OPS_OWNERSHIP_GUEST171=V1\nMODE=read-only\nNO_MUTATION=true\nGUEST171_HOSTNAME_MONITORING=true\nGUEST171_MONITOR_TEXT_FILE_COUNT=373\nGUEST171_MONITOR_CHAT_REF=false\nGUEST171_MONITOR_GROUPS_REF=false\nGUEST171_MONITOR_SHARE_REF=false\nGUEST171_MONITOR_PORT_443_REF=true\nGUEST171_MONITOR_PORT_5000_REF=false\nGUEST171_MONITOR_PORT_5222_REF=true\nGUEST171_MONITOR_PORT_5269_REF=false\nGUEST171_MONITOR_PORT_3479_REF=true\nGUEST171_MONITOR_PORT_5349_REF=false\nGUEST171_MONITOR_PORT_5350_REF=false\nGUEST171_MONITOR_STUN_TURN_REF=false\nGUEST171_MONITOR_CERT_EXPIRY_REF=true\nGUEST171_KUMA_DB_COUNT=2\nGUEST171_KUMA_MONITOR_COUNT=192\nGUEST171_KUMA_CHAT_MATCH_COUNT=0\nGUEST171_KUMA_GROUPS_MATCH_COUNT=0\nGUEST171_KUMA_SHARE_MATCH_COUNT=0\nGUEST171_KUMA_5222_MATCH_COUNT=0\nGUEST171_KUMA_5269_MATCH_COUNT=0\nGUEST171_KUMA_3479_MATCH_COUNT=0\nGUEST171_KUMA_STUN_TURN_MATCH_COUNT=0\nGUEST171_AUDIT_DONE=true\nPUBLIC_DNS_OK_COUNT=3\nPUBLIC_HTTPS_OK_COUNT=3\nPUBLIC_STUN_3479_PASS=true\nPUBLIC_STUN_3479_CLASS=BINDING_SUCCESS\nOPS_OWNERSHIP_RCA_COMPLETE=true\nDECISION=ANALYZE_EXACT_99578_OWNERSHIP\nNEXT_GATE=TARGETED_OPERATIONAL_HARDENING\nTASK_RESULT=SNIKKET_OPS_OWNERSHIP_RCA\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index d8f9c957..e146413c 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-261005-SKLADCHIK-DURABILITY-QUEUE-WORKER-RCA-99579R1 +COMMAND_ID=SUPPORT-261005-SNIKKET-OPS-OWNERSHIP-RCA-99578R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=skladchik-durability-queue-worker-rca +COMPONENT=snikket-ops-ownership-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=9786047247e48e71139ff52c618c59da24ce317cd7093ee00b699a69b178af52 +COMMAND_SHA256=7c2094112fc9cb5e843bd3559354125d99d3543ddd0f2055aca907ad84c4b4f5 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,2736 +24,141 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=df86e4a999ff180077eddd8c50c0647c83310a5062be61a88ccaab005bb577ea -SANITIZED_OUTPUT_SHA256=3aa1864388e614772490c9c29217a4ed56c483df64b4b67276683b5c5078e47e +RAW_EVIDENCE_SHA256=5ae23df042db0d52f6523a7361f49070039ea122f2db6e864fd4a5d979c98317 +SANITIZED_OUTPUT_SHA256=5ae23df042db0d52f6523a7361f49070039ea122f2db6e864fd4a5d979c98317 OUTPUT_BEGIN -SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA=V1 -NO_MAIL_SEND=true -NO_SERVICE_RESTART=true -NO_TIMER_MUTATION=true -NO_NETWORK_MUTATION=true -NO_SOURCE_MUTATION=true -NO_BACKUP_MUTATION=true -NO_GIT_PUSH=true +SNIKKET_OPS_OWNERSHIP_RCA_WRAPPER=V1 MODE=read-only +NO_DNS_MUTATION=true +NO_VM_MUTATION=true +NO_CONTAINER_RESTART=true +NO_SERVICE_START=true +NO_SOURCE_MUTATION=true HOST_PVE01=true -QUEUE_VM170_QM_RC=0 -QUEUE_VM170_STDERR_PRESENT=false -QUEUE_VM170_JSON=true -QUEUE_VM170_GUEST_EXITCODE=0 -VM170_MONITOR_SHA=19123e359f2f7b9a2d373fd7e4464dd426d31b91cabc567c252e70cad40d77c8 -VM170_TIMER_ACTIVE=active -VM170_TIMER_ENABLED=enabled -VM170_SERVICE_RESULT=success -VM170_HEALTH_STATUS=OK -VM170_HEALTH_TS=2026-10-05T06:49:35Z -VM170_HEALTH_REPORTS=22 -VM170_HEALTH_OPEN=13 -VM170_HEALTH_RESOLVED=9 -VM170_BASELINE_OK=true -SUPERVISOR_SHA=3a9bf9f3f9c28dbf670300f0851a5b7f86eb41fa41cdf44d0935337ae8ef02a2 -SUPERVISOR_TOP_CONST_1_NAME=BACKUP_HEALTH_AGE_MAX -SUPERVISOR_TOP_CONST_1_EXPR_SHA=056f62a1cb138698ac25abded3e6493c5519d37a485bb3900a74bad1806f70b3 -SUPERVISOR_TOP_CONST_2_NAME=RESTORE_HEALTH_AGE_MAX -SUPERVISOR_TOP_CONST_2_EXPR_SHA=ce6a9ab58cc2e0db3f07ff3ef7d288b29b6611afff056f5b818c706075d9dbd5 -SUPERVISOR_TOP_CONST_3_NAME=APP_FRESHNESS_PROBE -SUPERVISOR_TOP_CONST_3_EXPR_SHA=d6593f7b36fcf7550285e4af7a5f5766c13c2ee0e0a992b808caaed52e54693a -SUPERVISOR_TOP_CONST_4_NAME=LEGACY_BACKUP_PROBLEMS -SUPERVISOR_TOP_CONST_4_EXPR_SHA=d84baf639fd4e63d9c728a793f4dc6c457b6bee5737789745170d2150d858ddd -SUPERVISOR_TOP_CONST_COUNT=4 -SUPERVISOR_TEST_RC=0 -SUPERVISOR_TEST_STDOUT_SHA=9b4438f36f4a40de86d41fd480c6900c28603079d96011398674df7637868c5e -SUPERVISOR_TEST_OUTPUT_BEGIN -SUPERVISOR_STATUS=OK -SUPERVISOR_PROBLEMS=none -MONITOR=OK reports=22 open=13 resolved=9 -CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z -REGISTRY_SOURCE= -> -NETBIRD_ROUTE= dev wt0 table 7120 src uid 0 -SUPERVISOR_TEST_OUTPUT_END -SUPTEST_SUPERVISOR_STATUS=OK -SUPTEST_SUPERVISOR_PROBLEMS=none -SUPTEST_MONITOR=OK reports=22 open=13 resolved=9 -SUPTEST_CANONICAL_BACKUP=SEALED_PROVEN generation=app-skladchik-reports-monitor-pve01-20261005T003035Z -SUPTEST_REGISTRY_SOURCE=[PRIVATE_IP] -SUPTEST_WORKER_TARGET=debian@[PRIVATE_IP] -SUPTEST_NETBIRD_ROUTE=1.1.1.1 dev wt0 table 7120 src 100.125.187.138 uid 0 -QUEUE_PRESENT=true -QUEUE_SHA=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650 -QUEUE_SHEBANG=#!/usr/bin/env bash -QUEUE_BASH_N_RC=0 -QUEUE_FUNCTIONS=backup_list,die,dispatch_one,dispatch_plan,freshness_decision,freshness_plan,lock_heavy,migrate_list,plan,verify_all -QUEUE_ACTIONS=BACKUP_RESUME,BACKUP_UNVERIFIED,SEALED_PROVEN,SKIP_FRESH -QUEUE_VAR_NAMES=DUE_SECONDS,FRESHNESS_SECONDS,HEAVY_LOCK,PROBE,QUEUE_LOCK,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,WORKER,age,decision,freshness,generation,now_epoch,out,sealed_epoch -QUEUE_COMMAND_BASENAMES=flock,homelab-app-cloud-quorum-worker,sha256sum -QUEUE_INTERESTING_LINE_COUNT=32 -QUEUE_LINE_1_NO=4 -QUEUE_LINE_1_TEXT=WORKER=/usr/local/libexec/homelab-app-cloud-quorum-worker -QUEUE_LINE_2_NO=5 -QUEUE_LINE_2_TEXT=PROBE=/usr/local/libexec/homelab-app-cloud-quorum-freshness-probe -QUEUE_LINE_3_NO=6 -QUEUE_LINE_3_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" -QUEUE_LINE_4_NO=8 -QUEUE_LINE_4_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" -QUEUE_LINE_5_NO=9 -QUEUE_LINE_5_TEXT=QUEUE_LOCK=/run/lock/homelab-app-cloud-quorum-queue.lock -QUEUE_LINE_6_NO=10 -QUEUE_LINE_6_TEXT=HEAVY_LOCK=/run/homelab-backup-heavy.lock -QUEUE_LINE_7_NO=21 -QUEUE_LINE_7_TEXT=[[ -x "$WORKER" && ! -L "$WORKER" ]] || die WORKER_MISSING 69 -QUEUE_LINE_8_NO=29 -QUEUE_LINE_8_TEXT= for a in "${ALL_APPS[@]}"; do "$WORKER" plan "$a"; done -QUEUE_LINE_9_NO=32 -QUEUE_LINE_9_TEXT= local app="$1" now_epoch="$2" out decision generation sealed_epoch age -QUEUE_LINE_10_NO=38 -QUEUE_LINE_10_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" -QUEUE_LINE_11_NO=39 -QUEUE_LINE_11_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_RESUME GENERATION=%s\n' "$app" "$generation" -QUEUE_LINE_12_NO=42 -QUEUE_LINE_12_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" -QUEUE_LINE_13_NO=50 -QUEUE_LINE_13_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=SKIP_FRESH GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS" -QUEUE_LINE_14_NO=52 -QUEUE_LINE_14_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_DUE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS" -QUEUE_LINE_15_NO=54 -QUEUE_LINE_15_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_STALE GENERATION=%s AGE_SECONDS=%s DUE_SECONDS=%s FRESHNESS_SECONDS=%s\n' "$app" "$generation" "$age" "$DUE_SECONDS" "$FRESHNESS_SECONDS" -QUEUE_LINE_16_NO=56 -QUEUE_LINE_16_TEXT= printf 'QUEUE_FRESHNESS APP=%s DECISION=BACKUP_UNVERIFIED REASON=SEALED_TIMESTAMP_IN_FUTURE GENERATION=%s AGE_SECONDS=%s\n' "$app" "$generation" "$age" -QUEUE_LINE_17_NO=84 -QUEUE_LINE_17_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"; printf '%s\n' "$out" -QUEUE_LINE_18_NO=85 -QUEUE_LINE_18_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null -QUEUE_LINE_19_NO=93 -QUEUE_LINE_19_TEXT= local a="$1" freshness decision now_epoch out generation -QUEUE_LINE_20_NO=112 -QUEUE_LINE_20_TEXT= generation="$(awk '/^QUEUE_FRESHNESS /{for(i=1;i<=NF;i++)if($i~/^GENERATION=/){sub(/^GENERATION=/,"",$i);print $i}}' <<<"$freshness" | tail -1)" -QUEUE_LINE_21_NO=113 -QUEUE_LINE_21_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SKIP_FRESH GENERATION=%s\n' "$a" "$generation" -QUEUE_LINE_22_NO=120 -QUEUE_LINE_22_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")" -QUEUE_LINE_23_NO=122 -QUEUE_LINE_23_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null -QUEUE_LINE_24_NO=123 -QUEUE_LINE_24_TEXT= generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" -QUEUE_LINE_25_NO=124 -QUEUE_LINE_25_TEXT= [[ -n "$generation" ]] || die "GENERATION_NOT_RETURNED_$a" 65 -QUEUE_LINE_26_NO=125 -QUEUE_LINE_26_TEXT= printf 'QUEUE_SEALED APP=%s GENERATION=%s DISPATCH=ONE_TARGET\n' "$a" "$generation" -QUEUE_LINE_27_NO=126 -QUEUE_LINE_27_TEXT= printf 'QUEUE_DISPATCH_RESULT KIND=APP APP=%s RESULT=SEALED GENERATION=%s\n' "$a" "$generation" -QUEUE_LINE_28_NO=133 -QUEUE_LINE_28_TEXT= if out="$(exec 8>&-; exec 9>&-; "$WORKER" latest-sealed "$a" 2>/dev/null)"; then -QUEUE_LINE_29_NO=138 -QUEUE_LINE_29_TEXT= out="$(exec 8>&-; exec 9>&-; "$WORKER" backup "$a")"; printf '%s\n' "$out" -QUEUE_LINE_30_NO=139 -QUEUE_LINE_30_TEXT= "$WORKER" latest-sealed "$a" 8>&- 9>&- >/dev/null -QUEUE_LINE_31_NO=145 -QUEUE_LINE_31_TEXT= for a in "${ALL_APPS[@]}"; do out="$("$WORKER" latest-sealed "$a")"; printf '%s\n' "$out"; done -QUEUE_LINE_32_NO=153 -QUEUE_LINE_32_TEXT= all) [[ "$#" == 1 ]] || die USAGE_ALL 64; [[ -x /usr/local/sbin/homelab-backup-dispatcher && ! -L /usr/local/sbin/homelab-backup-dispatcher ]] || die DISPATCHER_MISSING 69; /usr/local/sbin/homelab-backup-dispatcher run APP_TIMER ;; -QUEUE_SERVICE_SHOW_RC=0 -QUEUE_SERVICE_LOADSTATE=loaded -QUEUE_SERVICE_ACTIVESTATE=inactive -QUEUE_SERVICE_UNITFILESTATE=static -QUEUE_SERVICE_RESULT=success -QUEUE_SERVICE_EXECMAINSTATUS=0 -QUEUE_SERVICE_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.service -QUEUE_SERVICE_FRAGMENT_SHA=afaa997bd85dafdb29d2cd2b53527996171c21fd5e80ae559f8f2bee0dcbbf1a -QUEUE_SERVICE_EXECSTART_SHA=22094bb14d8dc5f7399744e183b096583a9c9625527a2f00981f2ec5cd8e1713 -QUEUE_SERVICE_EXEC_BASENAMES=homelab-backup-v2-app-scheduled -QUEUE_SERVICE_EXEC_ACTIONS=NONE -QUEUE_TIMER_SHOW_RC=0 -QUEUE_TIMER_LOADSTATE=loaded -QUEUE_TIMER_ACTIVESTATE=active -QUEUE_TIMER_UNITFILESTATE=enabled -QUEUE_TIMER_RESULT=success -QUEUE_TIMER_EXECMAINSTATUS= -QUEUE_TIMER_FRAGMENT_BASENAME=homelab-app-cloud-quorum-queue.timer -QUEUE_TIMER_FRAGMENT_SHA=af2eedce3df53029b591d31715a3f3e6688ddb997b4823f487e70c339ce88078 -QUEUE_TIMER_EXECSTART_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -QUEUE_TIMER_EXEC_BASENAMES=NONE -QUEUE_TIMER_EXEC_ACTIONS=NONE -WORKER_CANDIDATE_COUNT=173 -WORKER_1_BASENAME=homelab-app-cloud-quorum-worker -WORKER_1_PATH_HINT=//root/BACKUP_closeout_evidence_20260825T165406Z/source/homelab-app-cloud-quorum-worker -WORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 -WORKER_1_EXECUTABLE=false -WORKER_1_PRESENT=true -WORKER_1_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 -WORKER_1_SHEBANG=#!/usr/bin/env bash -WORKER_1_BASH_N_RC=0 -WORKER_1_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_1_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor -WORKER_1_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_1_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_1_INTERESTING_LINE_COUNT=57 -WORKER_1_LINE_1_NO=4 -WORKER_1_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_1_LINE_2_NO=6 -WORKER_1_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_1_LINE_3_NO=7 -WORKER_1_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_1_LINE_4_NO=8 -WORKER_1_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_1_LINE_5_NO=9 -WORKER_1_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_1_LINE_6_NO=10 -WORKER_1_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_1_LINE_7_NO=12 -WORKER_1_LINE_7_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" -WORKER_1_LINE_8_NO=14 -WORKER_1_LINE_8_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" -WORKER_1_LINE_9_NO=75 -WORKER_1_LINE_9_TEXT=logical_id(){ -WORKER_1_LINE_10_NO=82 -WORKER_1_LINE_10_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_1_LINE_11_NO=83 -WORKER_1_LINE_11_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_1_LINE_12_NO=101 -WORKER_1_LINE_12_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_1_LINE_13_NO=103 -WORKER_1_LINE_13_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_1_LINE_14_NO=104 -WORKER_1_LINE_14_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_1_LINE_15_NO=115 -WORKER_1_LINE_15_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_1_LINE_16_NO=117 -WORKER_1_LINE_16_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_1_LINE_17_NO=118 -WORKER_1_LINE_17_TEXT= if [[ -s "$manifest" ]]; then -WORKER_1_LINE_18_NO=119 -WORKER_1_LINE_18_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_1_LINE_19_NO=126 -WORKER_1_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_1_LINE_20_NO=141 -WORKER_1_LINE_20_TEXT= logical="$(logical_id "$app")" -WORKER_1_LINE_21_NO=160 -WORKER_1_LINE_21_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_1_LINE_22_NO=191 -WORKER_1_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_1_LINE_23_NO=202 -WORKER_1_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_1_LINE_24_NO=218 -WORKER_1_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_1_LINE_25_NO=239 -WORKER_1_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_1_LINE_26_NO=274 -WORKER_1_LINE_26_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_1_LINE_27_NO=275 -WORKER_1_LINE_27_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_1_LINE_28_NO=293 -WORKER_1_LINE_28_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_1_LINE_29_NO=341 -WORKER_1_LINE_29_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_1_LINE_30_NO=346 -WORKER_1_LINE_30_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_1_LINE_31_NO=349 -WORKER_1_LINE_31_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_1_LINE_32_NO=350 -WORKER_1_LINE_32_TEXT= logical="$(logical_id "$app")" -WORKER_1_LINE_33_NO=353 -WORKER_1_LINE_33_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_1_LINE_34_NO=354 -WORKER_1_LINE_34_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_1_LINE_35_NO=362 -WORKER_1_LINE_35_TEXT= done <"$manifest" -WORKER_1_LINE_36_NO=431 -WORKER_1_LINE_36_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_1_LINE_37_NO=432 -WORKER_1_LINE_37_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_1_LINE_38_NO=466 -WORKER_1_LINE_38_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_1_LINE_39_NO=471 -WORKER_1_LINE_39_TEXT= if "manifest.json" not in names: -WORKER_1_LINE_40_NO=473 -WORKER_1_LINE_40_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_1_LINE_41_NO=475 -WORKER_1_LINE_41_TEXT= for row in manifest: -WORKER_1_LINE_42_NO=516 -WORKER_1_LINE_42_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_1_LINE_43_NO=528 -WORKER_1_LINE_43_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_1_LINE_44_NO=530 -WORKER_1_LINE_44_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_1_LINE_45_NO=536 -WORKER_1_LINE_45_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_1_LINE_46_NO=539 -WORKER_1_LINE_46_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_1_LINE_47_NO=540 -WORKER_1_LINE_47_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_1_LINE_48_NO=546 -WORKER_1_LINE_48_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_1_LINE_49_NO=553 -WORKER_1_LINE_49_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_1_LINE_50_NO=560 -WORKER_1_LINE_50_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_1_LINE_51_NO=596 -WORKER_1_LINE_51_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_1_LINE_52_NO=598 -WORKER_1_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_1_LINE_53_NO=644 -WORKER_1_LINE_53_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_1_LINE_54_NO=655 -WORKER_1_LINE_54_TEXT=ACTION="${1:-}" -WORKER_1_LINE_55_NO=656 -WORKER_1_LINE_55_TEXT=case "$ACTION" in -WORKER_1_LINE_56_NO=658 -WORKER_1_LINE_56_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_1_LINE_57_NO=659 -WORKER_1_LINE_57_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_2_BASENAME=homelab-app-cloud-quorum-worker -WORKER_2_PATH_HINT=root/backup-registry-sha-pins-20260904T062507Z/usr/local/libexec/homelab-app-cloud-quorum-worker -WORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 -WORKER_2_EXECUTABLE=true -WORKER_2_PRESENT=true -WORKER_2_SHA=f82f0fa81925f4518355126d75b1e306aee2cd7b85270d09279e1e617cda2913 -WORKER_2_SHEBANG=#!/usr/bin/env bash -WORKER_2_BASH_N_RC=0 -WORKER_2_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_2_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-reports-monitor -WORKER_2_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REGISTRY_HELPER,REGISTRY_HELPER_SHA_EXPECTED,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_2_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_2_INTERESTING_LINE_COUNT=57 -WORKER_2_LINE_1_NO=4 -WORKER_2_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_2_LINE_2_NO=6 -WORKER_2_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_2_LINE_3_NO=7 -WORKER_2_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_2_LINE_4_NO=8 -WORKER_2_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_2_LINE_5_NO=9 -WORKER_2_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_2_LINE_6_NO=10 -WORKER_2_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_2_LINE_7_NO=12 -WORKER_2_LINE_7_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" -WORKER_2_LINE_8_NO=14 -WORKER_2_LINE_8_TEXT=REGISTRY_HELPER="${HOMELAB_BACKUP_TARGET_REGISTRY_HELPER:-/usr/local/libexec/homelab-backup-target-registry}" -WORKER_2_LINE_9_NO=75 -WORKER_2_LINE_9_TEXT=logical_id(){ -WORKER_2_LINE_10_NO=82 -WORKER_2_LINE_10_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_2_LINE_11_NO=83 -WORKER_2_LINE_11_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_2_LINE_12_NO=101 -WORKER_2_LINE_12_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_2_LINE_13_NO=103 -WORKER_2_LINE_13_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_2_LINE_14_NO=104 -WORKER_2_LINE_14_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_2_LINE_15_NO=115 -WORKER_2_LINE_15_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_2_LINE_16_NO=117 -WORKER_2_LINE_16_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_2_LINE_17_NO=118 -WORKER_2_LINE_17_TEXT= if [[ -s "$manifest" ]]; then -WORKER_2_LINE_18_NO=119 -WORKER_2_LINE_18_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_2_LINE_19_NO=126 -WORKER_2_LINE_19_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_2_LINE_20_NO=141 -WORKER_2_LINE_20_TEXT= logical="$(logical_id "$app")" -WORKER_2_LINE_21_NO=160 -WORKER_2_LINE_21_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_2_LINE_22_NO=191 -WORKER_2_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_2_LINE_23_NO=202 -WORKER_2_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_2_LINE_24_NO=218 -WORKER_2_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_2_LINE_25_NO=239 -WORKER_2_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_2_LINE_26_NO=274 -WORKER_2_LINE_26_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_2_LINE_27_NO=275 -WORKER_2_LINE_27_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_2_LINE_28_NO=293 -WORKER_2_LINE_28_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_2_LINE_29_NO=341 -WORKER_2_LINE_29_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_2_LINE_30_NO=346 -WORKER_2_LINE_30_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_2_LINE_31_NO=349 -WORKER_2_LINE_31_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_2_LINE_32_NO=350 -WORKER_2_LINE_32_TEXT= logical="$(logical_id "$app")" -WORKER_2_LINE_33_NO=353 -WORKER_2_LINE_33_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_2_LINE_34_NO=354 -WORKER_2_LINE_34_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_2_LINE_35_NO=362 -WORKER_2_LINE_35_TEXT= done <"$manifest" -WORKER_2_LINE_36_NO=431 -WORKER_2_LINE_36_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_2_LINE_37_NO=432 -WORKER_2_LINE_37_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_2_LINE_38_NO=466 -WORKER_2_LINE_38_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_2_LINE_39_NO=471 -WORKER_2_LINE_39_TEXT= if "manifest.json" not in names: -WORKER_2_LINE_40_NO=473 -WORKER_2_LINE_40_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_2_LINE_41_NO=475 -WORKER_2_LINE_41_TEXT= for row in manifest: -WORKER_2_LINE_42_NO=516 -WORKER_2_LINE_42_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_2_LINE_43_NO=528 -WORKER_2_LINE_43_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_2_LINE_44_NO=530 -WORKER_2_LINE_44_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_2_LINE_45_NO=536 -WORKER_2_LINE_45_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_2_LINE_46_NO=539 -WORKER_2_LINE_46_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_2_LINE_47_NO=540 -WORKER_2_LINE_47_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_2_LINE_48_NO=546 -WORKER_2_LINE_48_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_2_LINE_49_NO=553 -WORKER_2_LINE_49_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_2_LINE_50_NO=560 -WORKER_2_LINE_50_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_2_LINE_51_NO=596 -WORKER_2_LINE_51_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_2_LINE_52_NO=598 -WORKER_2_LINE_52_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_2_LINE_53_NO=644 -WORKER_2_LINE_53_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_2_LINE_54_NO=655 -WORKER_2_LINE_54_TEXT=ACTION="${1:-}" -WORKER_2_LINE_55_NO=656 -WORKER_2_LINE_55_TEXT=case "$ACTION" in -WORKER_2_LINE_56_NO=658 -WORKER_2_LINE_56_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_2_LINE_57_NO=659 -WORKER_2_LINE_57_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_3_BASENAME=homelab-app-cloud-quorum-worker -WORKER_3_PATH_HINT=root/homelab-ops-cr-2026-0024/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker -WORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 -WORKER_3_EXECUTABLE=true -WORKER_3_PRESENT=true -WORKER_3_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 -WORKER_3_SHEBANG=#!/usr/bin/env bash -WORKER_3_BASH_N_RC=0 -WORKER_3_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_3_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_3_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work -WORKER_3_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd -WORKER_3_INTERESTING_LINE_COUNT=52 -WORKER_3_LINE_1_NO=4 -WORKER_3_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_3_LINE_2_NO=6 -WORKER_3_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_3_LINE_3_NO=7 -WORKER_3_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_3_LINE_4_NO=8 -WORKER_3_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_3_LINE_5_NO=9 -WORKER_3_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_3_LINE_6_NO=10 -WORKER_3_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_3_LINE_7_NO=45 -WORKER_3_LINE_7_TEXT=logical_id(){ -WORKER_3_LINE_8_NO=50 -WORKER_3_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_3_LINE_9_NO=64 -WORKER_3_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_3_LINE_10_NO=65 -WORKER_3_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_3_LINE_11_NO=83 -WORKER_3_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_3_LINE_12_NO=85 -WORKER_3_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_3_LINE_13_NO=86 -WORKER_3_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_3_LINE_14_NO=97 -WORKER_3_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_3_LINE_15_NO=99 -WORKER_3_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_3_LINE_16_NO=100 -WORKER_3_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_3_LINE_17_NO=101 -WORKER_3_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_3_LINE_18_NO=108 -WORKER_3_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_3_LINE_19_NO=123 -WORKER_3_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_3_LINE_20_NO=142 -WORKER_3_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_3_LINE_21_NO=150 -WORKER_3_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_3_LINE_22_NO=161 -WORKER_3_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_3_LINE_23_NO=192 -WORKER_3_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_3_LINE_24_NO=203 -WORKER_3_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_3_LINE_25_NO=219 -WORKER_3_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_3_LINE_26_NO=240 -WORKER_3_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_3_LINE_27_NO=256 -WORKER_3_LINE_27_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_3_LINE_28_NO=261 -WORKER_3_LINE_28_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_3_LINE_29_NO=264 -WORKER_3_LINE_29_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_3_LINE_30_NO=265 -WORKER_3_LINE_30_TEXT= logical="$(logical_id "$app")" -WORKER_3_LINE_31_NO=268 -WORKER_3_LINE_31_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_3_LINE_32_NO=269 -WORKER_3_LINE_32_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_3_LINE_33_NO=277 -WORKER_3_LINE_33_TEXT= done <"$manifest" -WORKER_3_LINE_34_NO=344 -WORKER_3_LINE_34_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_3_LINE_35_NO=349 -WORKER_3_LINE_35_TEXT= if "manifest.json" not in names: -WORKER_3_LINE_36_NO=351 -WORKER_3_LINE_36_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_3_LINE_37_NO=353 -WORKER_3_LINE_37_TEXT= for row in manifest: -WORKER_3_LINE_38_NO=394 -WORKER_3_LINE_38_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_3_LINE_39_NO=406 -WORKER_3_LINE_39_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_3_LINE_40_NO=408 -WORKER_3_LINE_40_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_3_LINE_41_NO=414 -WORKER_3_LINE_41_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_3_LINE_42_NO=417 -WORKER_3_LINE_42_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_3_LINE_43_NO=418 -WORKER_3_LINE_43_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_3_LINE_44_NO=424 -WORKER_3_LINE_44_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_3_LINE_45_NO=427 -WORKER_3_LINE_45_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_3_LINE_46_NO=434 -WORKER_3_LINE_46_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_3_LINE_47_NO=459 -WORKER_3_LINE_47_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_3_LINE_48_NO=461 -WORKER_3_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_3_LINE_49_NO=463 -WORKER_3_LINE_49_TEXT=ACTION="${1:-}" -WORKER_3_LINE_50_NO=464 -WORKER_3_LINE_50_TEXT=case "$ACTION" in -WORKER_3_LINE_51_NO=466 -WORKER_3_LINE_51_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_3_LINE_52_NO=467 -WORKER_3_LINE_52_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_4_BASENAME=homelab-app-cloud-quorum-worker -WORKER_4_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-hbarhub-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker -WORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 -WORKER_4_EXECUTABLE=true -WORKER_4_PRESENT=true -WORKER_4_SHA=7c33994c605c7f68e3c7d71cce95b73dbd908c1b13143d2abe9f63f0ecf06309 -WORKER_4_SHEBANG=#!/usr/bin/env bash -WORKER_4_BASH_N_RC=0 -WORKER_4_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,cleanup,cleanup_filebrowser_capture_orphans,die,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_4_ACTIONS=filebrowser,hbarhub,immich,memos,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_4_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,allowed,base,cfg,checks,cnt,completed,comps,d,db,dbsha,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_msha,iid,image_hex,image_tar,imagesha,latest,logical,manifest,meta,meta_tar,metasha,names,out,part,payload,qstart,remote_dump,required,source_bytes,stacksha,started,state,tag_to_cfg,tmp,work -WORKER_4_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,tar,zstd -WORKER_4_INTERESTING_LINE_COUNT=52 -WORKER_4_LINE_1_NO=4 -WORKER_4_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_4_LINE_2_NO=6 -WORKER_4_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_4_LINE_3_NO=7 -WORKER_4_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_4_LINE_4_NO=8 -WORKER_4_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_4_LINE_5_NO=9 -WORKER_4_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_4_LINE_6_NO=10 -WORKER_4_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_4_LINE_7_NO=45 -WORKER_4_LINE_7_TEXT=logical_id(){ -WORKER_4_LINE_8_NO=50 -WORKER_4_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_4_LINE_9_NO=64 -WORKER_4_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_4_LINE_10_NO=65 -WORKER_4_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_4_LINE_11_NO=83 -WORKER_4_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_4_LINE_12_NO=85 -WORKER_4_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_4_LINE_13_NO=86 -WORKER_4_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_4_LINE_14_NO=97 -WORKER_4_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_4_LINE_15_NO=99 -WORKER_4_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_4_LINE_16_NO=100 -WORKER_4_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_4_LINE_17_NO=101 -WORKER_4_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_4_LINE_18_NO=108 -WORKER_4_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_4_LINE_19_NO=123 -WORKER_4_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_4_LINE_20_NO=142 -WORKER_4_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_4_LINE_21_NO=150 -WORKER_4_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_4_LINE_22_NO=161 -WORKER_4_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_4_LINE_23_NO=192 -WORKER_4_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_4_LINE_24_NO=203 -WORKER_4_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_4_LINE_25_NO=219 -WORKER_4_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_4_LINE_26_NO=240 -WORKER_4_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_4_LINE_27_NO=256 -WORKER_4_LINE_27_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_4_LINE_28_NO=261 -WORKER_4_LINE_28_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_4_LINE_29_NO=264 -WORKER_4_LINE_29_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_4_LINE_30_NO=265 -WORKER_4_LINE_30_TEXT= logical="$(logical_id "$app")" -WORKER_4_LINE_31_NO=268 -WORKER_4_LINE_31_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_4_LINE_32_NO=269 -WORKER_4_LINE_32_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_4_LINE_33_NO=277 -WORKER_4_LINE_33_TEXT= done <"$manifest" -WORKER_4_LINE_34_NO=344 -WORKER_4_LINE_34_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_4_LINE_35_NO=349 -WORKER_4_LINE_35_TEXT= if "manifest.json" not in names: -WORKER_4_LINE_36_NO=351 -WORKER_4_LINE_36_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_4_LINE_37_NO=353 -WORKER_4_LINE_37_TEXT= for row in manifest: -WORKER_4_LINE_38_NO=394 -WORKER_4_LINE_38_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_4_LINE_39_NO=406 -WORKER_4_LINE_39_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_4_LINE_40_NO=408 -WORKER_4_LINE_40_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_4_LINE_41_NO=414 -WORKER_4_LINE_41_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_4_LINE_42_NO=417 -WORKER_4_LINE_42_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_4_LINE_43_NO=418 -WORKER_4_LINE_43_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_4_LINE_44_NO=424 -WORKER_4_LINE_44_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_4_LINE_45_NO=427 -WORKER_4_LINE_45_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_4_LINE_46_NO=434 -WORKER_4_LINE_46_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_4_LINE_47_NO=459 -WORKER_4_LINE_47_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_4_LINE_48_NO=461 -WORKER_4_LINE_48_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_4_LINE_49_NO=463 -WORKER_4_LINE_49_TEXT=ACTION="${1:-}" -WORKER_4_LINE_50_NO=464 -WORKER_4_LINE_50_TEXT=case "$ACTION" in -WORKER_4_LINE_51_NO=466 -WORKER_4_LINE_51_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_4_LINE_52_NO=467 -WORKER_4_LINE_52_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_5_BASENAME=homelab-app-cloud-quorum-worker -WORKER_5_PATH_HINT=root/homelab-ops-cr-2026-0025/tasks/backup-scrutiny-cloud-quorum-v1/assets/homelab-app-cloud-quorum-worker -WORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_5_EXECUTABLE=true -WORKER_5_PRESENT=true -WORKER_5_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_5_SHEBANG=#!/usr/bin/env bash -WORKER_5_BASH_N_RC=0 -WORKER_5_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_5_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_5_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_5_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_5_INTERESTING_LINE_COUNT=58 -WORKER_5_LINE_1_NO=4 -WORKER_5_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_5_LINE_2_NO=6 -WORKER_5_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_5_LINE_3_NO=7 -WORKER_5_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_5_LINE_4_NO=8 -WORKER_5_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_5_LINE_5_NO=9 -WORKER_5_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_5_LINE_6_NO=10 -WORKER_5_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_5_LINE_7_NO=69 -WORKER_5_LINE_7_TEXT=logical_id(){ -WORKER_5_LINE_8_NO=74 -WORKER_5_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_5_LINE_9_NO=90 -WORKER_5_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_5_LINE_10_NO=91 -WORKER_5_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_5_LINE_11_NO=109 -WORKER_5_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_5_LINE_12_NO=111 -WORKER_5_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_5_LINE_13_NO=112 -WORKER_5_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_5_LINE_14_NO=123 -WORKER_5_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_5_LINE_15_NO=125 -WORKER_5_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_5_LINE_16_NO=126 -WORKER_5_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_5_LINE_17_NO=127 -WORKER_5_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_5_LINE_18_NO=134 -WORKER_5_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_5_LINE_19_NO=149 -WORKER_5_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_5_LINE_20_NO=168 -WORKER_5_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_5_LINE_21_NO=176 -WORKER_5_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_5_LINE_22_NO=187 -WORKER_5_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_5_LINE_23_NO=218 -WORKER_5_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_5_LINE_24_NO=229 -WORKER_5_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_5_LINE_25_NO=245 -WORKER_5_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_5_LINE_26_NO=266 -WORKER_5_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_5_LINE_27_NO=301 -WORKER_5_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_5_LINE_28_NO=302 -WORKER_5_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_5_LINE_29_NO=320 -WORKER_5_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_5_LINE_30_NO=368 -WORKER_5_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_5_LINE_31_NO=373 -WORKER_5_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_5_LINE_32_NO=376 -WORKER_5_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_5_LINE_33_NO=377 -WORKER_5_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_5_LINE_34_NO=380 -WORKER_5_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_5_LINE_35_NO=381 -WORKER_5_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_5_LINE_36_NO=389 -WORKER_5_LINE_36_TEXT= done <"$manifest" -WORKER_5_LINE_37_NO=464 -WORKER_5_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_5_LINE_38_NO=465 -WORKER_5_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_5_LINE_39_NO=499 -WORKER_5_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_5_LINE_40_NO=504 -WORKER_5_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_5_LINE_41_NO=506 -WORKER_5_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_5_LINE_42_NO=508 -WORKER_5_LINE_42_TEXT= for row in manifest: -WORKER_5_LINE_43_NO=549 -WORKER_5_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_5_LINE_44_NO=561 -WORKER_5_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_5_LINE_45_NO=563 -WORKER_5_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_5_LINE_46_NO=569 -WORKER_5_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_5_LINE_47_NO=572 -WORKER_5_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_5_LINE_48_NO=573 -WORKER_5_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_5_LINE_49_NO=579 -WORKER_5_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_5_LINE_50_NO=586 -WORKER_5_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_5_LINE_51_NO=593 -WORKER_5_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_5_LINE_52_NO=630 -WORKER_5_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_5_LINE_53_NO=632 -WORKER_5_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_5_LINE_54_NO=678 -WORKER_5_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_5_LINE_55_NO=689 -WORKER_5_LINE_55_TEXT=ACTION="${1:-}" -WORKER_5_LINE_56_NO=690 -WORKER_5_LINE_56_TEXT=case "$ACTION" in -WORKER_5_LINE_57_NO=692 -WORKER_5_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_5_LINE_58_NO=693 -WORKER_5_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_6_BASENAME=homelab-app-cloud-quorum-worker -WORKER_6_PATH_HINT=homelab-ops-cr-2026-0029/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_6_EXECUTABLE=true -WORKER_6_PRESENT=true -WORKER_6_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_6_SHEBANG=#!/usr/bin/env bash -WORKER_6_BASH_N_RC=0 -WORKER_6_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_6_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_6_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_6_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_6_INTERESTING_LINE_COUNT=58 -WORKER_6_LINE_1_NO=4 -WORKER_6_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_6_LINE_2_NO=6 -WORKER_6_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_6_LINE_3_NO=7 -WORKER_6_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_6_LINE_4_NO=8 -WORKER_6_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_6_LINE_5_NO=9 -WORKER_6_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_6_LINE_6_NO=10 -WORKER_6_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_6_LINE_7_NO=69 -WORKER_6_LINE_7_TEXT=logical_id(){ -WORKER_6_LINE_8_NO=74 -WORKER_6_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_6_LINE_9_NO=90 -WORKER_6_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_6_LINE_10_NO=91 -WORKER_6_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_6_LINE_11_NO=109 -WORKER_6_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_6_LINE_12_NO=111 -WORKER_6_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_6_LINE_13_NO=112 -WORKER_6_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_6_LINE_14_NO=123 -WORKER_6_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_6_LINE_15_NO=125 -WORKER_6_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_6_LINE_16_NO=126 -WORKER_6_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_6_LINE_17_NO=127 -WORKER_6_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_6_LINE_18_NO=134 -WORKER_6_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_6_LINE_19_NO=149 -WORKER_6_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_6_LINE_20_NO=168 -WORKER_6_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_6_LINE_21_NO=176 -WORKER_6_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_6_LINE_22_NO=187 -WORKER_6_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_6_LINE_23_NO=218 -WORKER_6_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_6_LINE_24_NO=229 -WORKER_6_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_6_LINE_25_NO=245 -WORKER_6_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_6_LINE_26_NO=266 -WORKER_6_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_6_LINE_27_NO=301 -WORKER_6_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_6_LINE_28_NO=302 -WORKER_6_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_6_LINE_29_NO=320 -WORKER_6_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_6_LINE_30_NO=368 -WORKER_6_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_6_LINE_31_NO=373 -WORKER_6_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_6_LINE_32_NO=376 -WORKER_6_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_6_LINE_33_NO=377 -WORKER_6_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_6_LINE_34_NO=380 -WORKER_6_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_6_LINE_35_NO=381 -WORKER_6_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_6_LINE_36_NO=389 -WORKER_6_LINE_36_TEXT= done <"$manifest" -WORKER_6_LINE_37_NO=464 -WORKER_6_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_6_LINE_38_NO=465 -WORKER_6_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_6_LINE_39_NO=499 -WORKER_6_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_6_LINE_40_NO=504 -WORKER_6_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_6_LINE_41_NO=506 -WORKER_6_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_6_LINE_42_NO=508 -WORKER_6_LINE_42_TEXT= for row in manifest: -WORKER_6_LINE_43_NO=549 -WORKER_6_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_6_LINE_44_NO=561 -WORKER_6_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_6_LINE_45_NO=563 -WORKER_6_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_6_LINE_46_NO=569 -WORKER_6_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_6_LINE_47_NO=572 -WORKER_6_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_6_LINE_48_NO=573 -WORKER_6_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_6_LINE_49_NO=579 -WORKER_6_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_6_LINE_50_NO=586 -WORKER_6_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_6_LINE_51_NO=593 -WORKER_6_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_6_LINE_52_NO=630 -WORKER_6_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_6_LINE_53_NO=632 -WORKER_6_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_6_LINE_54_NO=678 -WORKER_6_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_6_LINE_55_NO=689 -WORKER_6_LINE_55_TEXT=ACTION="${1:-}" -WORKER_6_LINE_56_NO=690 -WORKER_6_LINE_56_TEXT=case "$ACTION" in -WORKER_6_LINE_57_NO=692 -WORKER_6_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_6_LINE_58_NO=693 -WORKER_6_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_7_BASENAME=homelab-app-cloud-quorum-worker -WORKER_7_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_7_EXECUTABLE=true -WORKER_7_PRESENT=true -WORKER_7_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_7_SHEBANG=#!/usr/bin/env bash -WORKER_7_BASH_N_RC=0 -WORKER_7_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_7_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_7_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_7_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_7_INTERESTING_LINE_COUNT=58 -WORKER_7_LINE_1_NO=4 -WORKER_7_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_7_LINE_2_NO=6 -WORKER_7_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_7_LINE_3_NO=7 -WORKER_7_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_7_LINE_4_NO=8 -WORKER_7_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_7_LINE_5_NO=9 -WORKER_7_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_7_LINE_6_NO=10 -WORKER_7_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_7_LINE_7_NO=69 -WORKER_7_LINE_7_TEXT=logical_id(){ -WORKER_7_LINE_8_NO=74 -WORKER_7_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_7_LINE_9_NO=90 -WORKER_7_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_7_LINE_10_NO=91 -WORKER_7_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_7_LINE_11_NO=109 -WORKER_7_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_7_LINE_12_NO=111 -WORKER_7_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_7_LINE_13_NO=112 -WORKER_7_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_7_LINE_14_NO=123 -WORKER_7_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_7_LINE_15_NO=125 -WORKER_7_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_7_LINE_16_NO=126 -WORKER_7_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_7_LINE_17_NO=127 -WORKER_7_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_7_LINE_18_NO=134 -WORKER_7_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_7_LINE_19_NO=149 -WORKER_7_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_7_LINE_20_NO=168 -WORKER_7_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_7_LINE_21_NO=176 -WORKER_7_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_7_LINE_22_NO=187 -WORKER_7_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_7_LINE_23_NO=218 -WORKER_7_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_7_LINE_24_NO=229 -WORKER_7_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_7_LINE_25_NO=245 -WORKER_7_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_7_LINE_26_NO=266 -WORKER_7_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_7_LINE_27_NO=301 -WORKER_7_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_7_LINE_28_NO=302 -WORKER_7_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_7_LINE_29_NO=320 -WORKER_7_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_7_LINE_30_NO=368 -WORKER_7_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_7_LINE_31_NO=373 -WORKER_7_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_7_LINE_32_NO=376 -WORKER_7_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_7_LINE_33_NO=377 -WORKER_7_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_7_LINE_34_NO=380 -WORKER_7_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_7_LINE_35_NO=381 -WORKER_7_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_7_LINE_36_NO=389 -WORKER_7_LINE_36_TEXT= done <"$manifest" -WORKER_7_LINE_37_NO=464 -WORKER_7_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_7_LINE_38_NO=465 -WORKER_7_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_7_LINE_39_NO=499 -WORKER_7_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_7_LINE_40_NO=504 -WORKER_7_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_7_LINE_41_NO=506 -WORKER_7_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_7_LINE_42_NO=508 -WORKER_7_LINE_42_TEXT= for row in manifest: -WORKER_7_LINE_43_NO=549 -WORKER_7_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_7_LINE_44_NO=561 -WORKER_7_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_7_LINE_45_NO=563 -WORKER_7_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_7_LINE_46_NO=569 -WORKER_7_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_7_LINE_47_NO=572 -WORKER_7_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_7_LINE_48_NO=573 -WORKER_7_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_7_LINE_49_NO=579 -WORKER_7_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_7_LINE_50_NO=586 -WORKER_7_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_7_LINE_51_NO=593 -WORKER_7_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_7_LINE_52_NO=630 -WORKER_7_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_7_LINE_53_NO=632 -WORKER_7_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_7_LINE_54_NO=678 -WORKER_7_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_7_LINE_55_NO=689 -WORKER_7_LINE_55_TEXT=ACTION="${1:-}" -WORKER_7_LINE_56_NO=690 -WORKER_7_LINE_56_TEXT=case "$ACTION" in -WORKER_7_LINE_57_NO=692 -WORKER_7_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_7_LINE_58_NO=693 -WORKER_7_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_8_BASENAME=homelab-app-cloud-quorum-worker -WORKER_8_PATH_HINT=homelab-ops-cr-2026-0030/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_8_EXECUTABLE=true -WORKER_8_PRESENT=true -WORKER_8_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_8_SHEBANG=#!/usr/bin/env bash -WORKER_8_BASH_N_RC=0 -WORKER_8_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_8_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_8_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_8_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_8_INTERESTING_LINE_COUNT=58 -WORKER_8_LINE_1_NO=4 -WORKER_8_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_8_LINE_2_NO=6 -WORKER_8_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_8_LINE_3_NO=7 -WORKER_8_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_8_LINE_4_NO=8 -WORKER_8_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_8_LINE_5_NO=9 -WORKER_8_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_8_LINE_6_NO=10 -WORKER_8_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_8_LINE_7_NO=69 -WORKER_8_LINE_7_TEXT=logical_id(){ -WORKER_8_LINE_8_NO=74 -WORKER_8_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_8_LINE_9_NO=90 -WORKER_8_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_8_LINE_10_NO=91 -WORKER_8_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_8_LINE_11_NO=109 -WORKER_8_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_8_LINE_12_NO=111 -WORKER_8_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_8_LINE_13_NO=112 -WORKER_8_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_8_LINE_14_NO=123 -WORKER_8_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_8_LINE_15_NO=125 -WORKER_8_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_8_LINE_16_NO=126 -WORKER_8_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_8_LINE_17_NO=127 -WORKER_8_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_8_LINE_18_NO=134 -WORKER_8_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_8_LINE_19_NO=149 -WORKER_8_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_8_LINE_20_NO=168 -WORKER_8_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_8_LINE_21_NO=176 -WORKER_8_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_8_LINE_22_NO=187 -WORKER_8_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_8_LINE_23_NO=218 -WORKER_8_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_8_LINE_24_NO=229 -WORKER_8_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_8_LINE_25_NO=245 -WORKER_8_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_8_LINE_26_NO=266 -WORKER_8_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_8_LINE_27_NO=301 -WORKER_8_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_8_LINE_28_NO=302 -WORKER_8_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_8_LINE_29_NO=320 -WORKER_8_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_8_LINE_30_NO=368 -WORKER_8_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_8_LINE_31_NO=373 -WORKER_8_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_8_LINE_32_NO=376 -WORKER_8_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_8_LINE_33_NO=377 -WORKER_8_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_8_LINE_34_NO=380 -WORKER_8_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_8_LINE_35_NO=381 -WORKER_8_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_8_LINE_36_NO=389 -WORKER_8_LINE_36_TEXT= done <"$manifest" -WORKER_8_LINE_37_NO=464 -WORKER_8_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_8_LINE_38_NO=465 -WORKER_8_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_8_LINE_39_NO=499 -WORKER_8_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_8_LINE_40_NO=504 -WORKER_8_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_8_LINE_41_NO=506 -WORKER_8_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_8_LINE_42_NO=508 -WORKER_8_LINE_42_TEXT= for row in manifest: -WORKER_8_LINE_43_NO=549 -WORKER_8_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_8_LINE_44_NO=561 -WORKER_8_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_8_LINE_45_NO=563 -WORKER_8_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_8_LINE_46_NO=569 -WORKER_8_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_8_LINE_47_NO=572 -WORKER_8_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_8_LINE_48_NO=573 -WORKER_8_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_8_LINE_49_NO=579 -WORKER_8_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_8_LINE_50_NO=586 -WORKER_8_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_8_LINE_51_NO=593 -WORKER_8_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_8_LINE_52_NO=630 -WORKER_8_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_8_LINE_53_NO=632 -WORKER_8_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_8_LINE_54_NO=678 -WORKER_8_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_8_LINE_55_NO=689 -WORKER_8_LINE_55_TEXT=ACTION="${1:-}" -WORKER_8_LINE_56_NO=690 -WORKER_8_LINE_56_TEXT=case "$ACTION" in -WORKER_8_LINE_57_NO=692 -WORKER_8_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_8_LINE_58_NO=693 -WORKER_8_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_9_BASENAME=homelab-app-cloud-quorum-worker -WORKER_9_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_9_EXECUTABLE=true -WORKER_9_PRESENT=true -WORKER_9_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_9_SHEBANG=#!/usr/bin/env bash -WORKER_9_BASH_N_RC=0 -WORKER_9_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_9_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_9_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_9_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_9_INTERESTING_LINE_COUNT=58 -WORKER_9_LINE_1_NO=4 -WORKER_9_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_9_LINE_2_NO=6 -WORKER_9_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_9_LINE_3_NO=7 -WORKER_9_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_9_LINE_4_NO=8 -WORKER_9_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_9_LINE_5_NO=9 -WORKER_9_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_9_LINE_6_NO=10 -WORKER_9_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_9_LINE_7_NO=69 -WORKER_9_LINE_7_TEXT=logical_id(){ -WORKER_9_LINE_8_NO=74 -WORKER_9_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_9_LINE_9_NO=90 -WORKER_9_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_9_LINE_10_NO=91 -WORKER_9_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_9_LINE_11_NO=109 -WORKER_9_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_9_LINE_12_NO=111 -WORKER_9_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_9_LINE_13_NO=112 -WORKER_9_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_9_LINE_14_NO=123 -WORKER_9_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_9_LINE_15_NO=125 -WORKER_9_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_9_LINE_16_NO=126 -WORKER_9_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_9_LINE_17_NO=127 -WORKER_9_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_9_LINE_18_NO=134 -WORKER_9_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_9_LINE_19_NO=149 -WORKER_9_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_9_LINE_20_NO=168 -WORKER_9_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_9_LINE_21_NO=176 -WORKER_9_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_9_LINE_22_NO=187 -WORKER_9_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_9_LINE_23_NO=218 -WORKER_9_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_9_LINE_24_NO=229 -WORKER_9_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_9_LINE_25_NO=245 -WORKER_9_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_9_LINE_26_NO=266 -WORKER_9_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_9_LINE_27_NO=301 -WORKER_9_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_9_LINE_28_NO=302 -WORKER_9_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_9_LINE_29_NO=320 -WORKER_9_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_9_LINE_30_NO=368 -WORKER_9_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_9_LINE_31_NO=373 -WORKER_9_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_9_LINE_32_NO=376 -WORKER_9_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_9_LINE_33_NO=377 -WORKER_9_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_9_LINE_34_NO=380 -WORKER_9_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_9_LINE_35_NO=381 -WORKER_9_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_9_LINE_36_NO=389 -WORKER_9_LINE_36_TEXT= done <"$manifest" -WORKER_9_LINE_37_NO=464 -WORKER_9_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_9_LINE_38_NO=465 -WORKER_9_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_9_LINE_39_NO=499 -WORKER_9_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_9_LINE_40_NO=504 -WORKER_9_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_9_LINE_41_NO=506 -WORKER_9_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_9_LINE_42_NO=508 -WORKER_9_LINE_42_TEXT= for row in manifest: -WORKER_9_LINE_43_NO=549 -WORKER_9_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_9_LINE_44_NO=561 -WORKER_9_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_9_LINE_45_NO=563 -WORKER_9_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_9_LINE_46_NO=569 -WORKER_9_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_9_LINE_47_NO=572 -WORKER_9_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_9_LINE_48_NO=573 -WORKER_9_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_9_LINE_49_NO=579 -WORKER_9_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_9_LINE_50_NO=586 -WORKER_9_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_9_LINE_51_NO=593 -WORKER_9_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_9_LINE_52_NO=630 -WORKER_9_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_9_LINE_53_NO=632 -WORKER_9_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_9_LINE_54_NO=678 -WORKER_9_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_9_LINE_55_NO=689 -WORKER_9_LINE_55_TEXT=ACTION="${1:-}" -WORKER_9_LINE_56_NO=690 -WORKER_9_LINE_56_TEXT=case "$ACTION" in -WORKER_9_LINE_57_NO=692 -WORKER_9_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_9_LINE_58_NO=693 -WORKER_9_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_10_BASENAME=homelab-app-cloud-quorum-worker -WORKER_10_PATH_HINT=homelab-ops-cr-2026-0031/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_10_EXECUTABLE=true -WORKER_10_PRESENT=true -WORKER_10_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_10_SHEBANG=#!/usr/bin/env bash -WORKER_10_BASH_N_RC=0 -WORKER_10_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_10_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_10_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_10_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_10_INTERESTING_LINE_COUNT=58 -WORKER_10_LINE_1_NO=4 -WORKER_10_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_10_LINE_2_NO=6 -WORKER_10_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_10_LINE_3_NO=7 -WORKER_10_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_10_LINE_4_NO=8 -WORKER_10_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_10_LINE_5_NO=9 -WORKER_10_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_10_LINE_6_NO=10 -WORKER_10_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_10_LINE_7_NO=69 -WORKER_10_LINE_7_TEXT=logical_id(){ -WORKER_10_LINE_8_NO=74 -WORKER_10_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_10_LINE_9_NO=90 -WORKER_10_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_10_LINE_10_NO=91 -WORKER_10_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_10_LINE_11_NO=109 -WORKER_10_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_10_LINE_12_NO=111 -WORKER_10_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_10_LINE_13_NO=112 -WORKER_10_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_10_LINE_14_NO=123 -WORKER_10_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_10_LINE_15_NO=125 -WORKER_10_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_10_LINE_16_NO=126 -WORKER_10_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_10_LINE_17_NO=127 -WORKER_10_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_10_LINE_18_NO=134 -WORKER_10_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_10_LINE_19_NO=149 -WORKER_10_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_10_LINE_20_NO=168 -WORKER_10_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_10_LINE_21_NO=176 -WORKER_10_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_10_LINE_22_NO=187 -WORKER_10_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_10_LINE_23_NO=218 -WORKER_10_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_10_LINE_24_NO=229 -WORKER_10_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_10_LINE_25_NO=245 -WORKER_10_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_10_LINE_26_NO=266 -WORKER_10_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_10_LINE_27_NO=301 -WORKER_10_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_10_LINE_28_NO=302 -WORKER_10_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_10_LINE_29_NO=320 -WORKER_10_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_10_LINE_30_NO=368 -WORKER_10_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_10_LINE_31_NO=373 -WORKER_10_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_10_LINE_32_NO=376 -WORKER_10_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_10_LINE_33_NO=377 -WORKER_10_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_10_LINE_34_NO=380 -WORKER_10_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_10_LINE_35_NO=381 -WORKER_10_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_10_LINE_36_NO=389 -WORKER_10_LINE_36_TEXT= done <"$manifest" -WORKER_10_LINE_37_NO=464 -WORKER_10_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_10_LINE_38_NO=465 -WORKER_10_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_10_LINE_39_NO=499 -WORKER_10_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_10_LINE_40_NO=504 -WORKER_10_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_10_LINE_41_NO=506 -WORKER_10_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_10_LINE_42_NO=508 -WORKER_10_LINE_42_TEXT= for row in manifest: -WORKER_10_LINE_43_NO=549 -WORKER_10_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_10_LINE_44_NO=561 -WORKER_10_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_10_LINE_45_NO=563 -WORKER_10_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_10_LINE_46_NO=569 -WORKER_10_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_10_LINE_47_NO=572 -WORKER_10_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_10_LINE_48_NO=573 -WORKER_10_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_10_LINE_49_NO=579 -WORKER_10_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_10_LINE_50_NO=586 -WORKER_10_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_10_LINE_51_NO=593 -WORKER_10_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_10_LINE_52_NO=630 -WORKER_10_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_10_LINE_53_NO=632 -WORKER_10_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_10_LINE_54_NO=678 -WORKER_10_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_10_LINE_55_NO=689 -WORKER_10_LINE_55_TEXT=ACTION="${1:-}" -WORKER_10_LINE_56_NO=690 -WORKER_10_LINE_56_TEXT=case "$ACTION" in -WORKER_10_LINE_57_NO=692 -WORKER_10_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_10_LINE_58_NO=693 -WORKER_10_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_11_BASENAME=homelab-app-cloud-quorum-worker -WORKER_11_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_11_EXECUTABLE=true -WORKER_11_PRESENT=true -WORKER_11_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_11_SHEBANG=#!/usr/bin/env bash -WORKER_11_BASH_N_RC=0 -WORKER_11_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_11_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_11_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_11_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_11_INTERESTING_LINE_COUNT=58 -WORKER_11_LINE_1_NO=4 -WORKER_11_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_11_LINE_2_NO=6 -WORKER_11_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_11_LINE_3_NO=7 -WORKER_11_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_11_LINE_4_NO=8 -WORKER_11_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_11_LINE_5_NO=9 -WORKER_11_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_11_LINE_6_NO=10 -WORKER_11_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_11_LINE_7_NO=69 -WORKER_11_LINE_7_TEXT=logical_id(){ -WORKER_11_LINE_8_NO=74 -WORKER_11_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_11_LINE_9_NO=90 -WORKER_11_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_11_LINE_10_NO=91 -WORKER_11_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_11_LINE_11_NO=109 -WORKER_11_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_11_LINE_12_NO=111 -WORKER_11_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_11_LINE_13_NO=112 -WORKER_11_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_11_LINE_14_NO=123 -WORKER_11_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_11_LINE_15_NO=125 -WORKER_11_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_11_LINE_16_NO=126 -WORKER_11_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_11_LINE_17_NO=127 -WORKER_11_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_11_LINE_18_NO=134 -WORKER_11_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_11_LINE_19_NO=149 -WORKER_11_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_11_LINE_20_NO=168 -WORKER_11_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_11_LINE_21_NO=176 -WORKER_11_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_11_LINE_22_NO=187 -WORKER_11_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_11_LINE_23_NO=218 -WORKER_11_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_11_LINE_24_NO=229 -WORKER_11_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_11_LINE_25_NO=245 -WORKER_11_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_11_LINE_26_NO=266 -WORKER_11_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_11_LINE_27_NO=301 -WORKER_11_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_11_LINE_28_NO=302 -WORKER_11_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_11_LINE_29_NO=320 -WORKER_11_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_11_LINE_30_NO=368 -WORKER_11_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_11_LINE_31_NO=373 -WORKER_11_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_11_LINE_32_NO=376 -WORKER_11_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_11_LINE_33_NO=377 -WORKER_11_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_11_LINE_34_NO=380 -WORKER_11_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_11_LINE_35_NO=381 -WORKER_11_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_11_LINE_36_NO=389 -WORKER_11_LINE_36_TEXT= done <"$manifest" -WORKER_11_LINE_37_NO=464 -WORKER_11_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_11_LINE_38_NO=465 -WORKER_11_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_11_LINE_39_NO=499 -WORKER_11_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_11_LINE_40_NO=504 -WORKER_11_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_11_LINE_41_NO=506 -WORKER_11_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_11_LINE_42_NO=508 -WORKER_11_LINE_42_TEXT= for row in manifest: -WORKER_11_LINE_43_NO=549 -WORKER_11_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_11_LINE_44_NO=561 -WORKER_11_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_11_LINE_45_NO=563 -WORKER_11_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_11_LINE_46_NO=569 -WORKER_11_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_11_LINE_47_NO=572 -WORKER_11_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_11_LINE_48_NO=573 -WORKER_11_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_11_LINE_49_NO=579 -WORKER_11_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_11_LINE_50_NO=586 -WORKER_11_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_11_LINE_51_NO=593 -WORKER_11_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_11_LINE_52_NO=630 -WORKER_11_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_11_LINE_53_NO=632 -WORKER_11_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_11_LINE_54_NO=678 -WORKER_11_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_11_LINE_55_NO=689 -WORKER_11_LINE_55_TEXT=ACTION="${1:-}" -WORKER_11_LINE_56_NO=690 -WORKER_11_LINE_56_TEXT=case "$ACTION" in -WORKER_11_LINE_57_NO=692 -WORKER_11_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_11_LINE_58_NO=693 -WORKER_11_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_12_BASENAME=homelab-app-cloud-quorum-worker -WORKER_12_PATH_HINT=homelab-ops-cr-2026-0032/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_12_EXECUTABLE=true -WORKER_12_PRESENT=true -WORKER_12_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_12_SHEBANG=#!/usr/bin/env bash -WORKER_12_BASH_N_RC=0 -WORKER_12_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_12_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_12_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_12_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_12_INTERESTING_LINE_COUNT=58 -WORKER_12_LINE_1_NO=4 -WORKER_12_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_12_LINE_2_NO=6 -WORKER_12_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_12_LINE_3_NO=7 -WORKER_12_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_12_LINE_4_NO=8 -WORKER_12_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_12_LINE_5_NO=9 -WORKER_12_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_12_LINE_6_NO=10 -WORKER_12_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_12_LINE_7_NO=69 -WORKER_12_LINE_7_TEXT=logical_id(){ -WORKER_12_LINE_8_NO=74 -WORKER_12_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_12_LINE_9_NO=90 -WORKER_12_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_12_LINE_10_NO=91 -WORKER_12_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_12_LINE_11_NO=109 -WORKER_12_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_12_LINE_12_NO=111 -WORKER_12_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_12_LINE_13_NO=112 -WORKER_12_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_12_LINE_14_NO=123 -WORKER_12_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_12_LINE_15_NO=125 -WORKER_12_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_12_LINE_16_NO=126 -WORKER_12_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_12_LINE_17_NO=127 -WORKER_12_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_12_LINE_18_NO=134 -WORKER_12_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_12_LINE_19_NO=149 -WORKER_12_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_12_LINE_20_NO=168 -WORKER_12_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_12_LINE_21_NO=176 -WORKER_12_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_12_LINE_22_NO=187 -WORKER_12_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_12_LINE_23_NO=218 -WORKER_12_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_12_LINE_24_NO=229 -WORKER_12_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_12_LINE_25_NO=245 -WORKER_12_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_12_LINE_26_NO=266 -WORKER_12_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_12_LINE_27_NO=301 -WORKER_12_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_12_LINE_28_NO=302 -WORKER_12_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_12_LINE_29_NO=320 -WORKER_12_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_12_LINE_30_NO=368 -WORKER_12_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_12_LINE_31_NO=373 -WORKER_12_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_12_LINE_32_NO=376 -WORKER_12_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_12_LINE_33_NO=377 -WORKER_12_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_12_LINE_34_NO=380 -WORKER_12_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_12_LINE_35_NO=381 -WORKER_12_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_12_LINE_36_NO=389 -WORKER_12_LINE_36_TEXT= done <"$manifest" -WORKER_12_LINE_37_NO=464 -WORKER_12_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_12_LINE_38_NO=465 -WORKER_12_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_12_LINE_39_NO=499 -WORKER_12_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_12_LINE_40_NO=504 -WORKER_12_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_12_LINE_41_NO=506 -WORKER_12_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_12_LINE_42_NO=508 -WORKER_12_LINE_42_TEXT= for row in manifest: -WORKER_12_LINE_43_NO=549 -WORKER_12_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_12_LINE_44_NO=561 -WORKER_12_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_12_LINE_45_NO=563 -WORKER_12_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_12_LINE_46_NO=569 -WORKER_12_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_12_LINE_47_NO=572 -WORKER_12_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_12_LINE_48_NO=573 -WORKER_12_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_12_LINE_49_NO=579 -WORKER_12_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_12_LINE_50_NO=586 -WORKER_12_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_12_LINE_51_NO=593 -WORKER_12_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_12_LINE_52_NO=630 -WORKER_12_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_12_LINE_53_NO=632 -WORKER_12_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_12_LINE_54_NO=678 -WORKER_12_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_12_LINE_55_NO=689 -WORKER_12_LINE_55_TEXT=ACTION="${1:-}" -WORKER_12_LINE_56_NO=690 -WORKER_12_LINE_56_TEXT=case "$ACTION" in -WORKER_12_LINE_57_NO=692 -WORKER_12_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_12_LINE_58_NO=693 -WORKER_12_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_13_BASENAME=homelab-app-cloud-quorum-worker -WORKER_13_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_13_EXECUTABLE=true -WORKER_13_PRESENT=true -WORKER_13_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_13_SHEBANG=#!/usr/bin/env bash -WORKER_13_BASH_N_RC=0 -WORKER_13_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_13_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_13_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_13_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_13_INTERESTING_LINE_COUNT=58 -WORKER_13_LINE_1_NO=4 -WORKER_13_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_13_LINE_2_NO=6 -WORKER_13_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_13_LINE_3_NO=7 -WORKER_13_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_13_LINE_4_NO=8 -WORKER_13_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_13_LINE_5_NO=9 -WORKER_13_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_13_LINE_6_NO=10 -WORKER_13_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_13_LINE_7_NO=69 -WORKER_13_LINE_7_TEXT=logical_id(){ -WORKER_13_LINE_8_NO=74 -WORKER_13_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_13_LINE_9_NO=90 -WORKER_13_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_13_LINE_10_NO=91 -WORKER_13_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_13_LINE_11_NO=109 -WORKER_13_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_13_LINE_12_NO=111 -WORKER_13_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_13_LINE_13_NO=112 -WORKER_13_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_13_LINE_14_NO=123 -WORKER_13_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_13_LINE_15_NO=125 -WORKER_13_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_13_LINE_16_NO=126 -WORKER_13_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_13_LINE_17_NO=127 -WORKER_13_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_13_LINE_18_NO=134 -WORKER_13_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_13_LINE_19_NO=149 -WORKER_13_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_13_LINE_20_NO=168 -WORKER_13_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_13_LINE_21_NO=176 -WORKER_13_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_13_LINE_22_NO=187 -WORKER_13_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_13_LINE_23_NO=218 -WORKER_13_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_13_LINE_24_NO=229 -WORKER_13_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_13_LINE_25_NO=245 -WORKER_13_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_13_LINE_26_NO=266 -WORKER_13_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_13_LINE_27_NO=301 -WORKER_13_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_13_LINE_28_NO=302 -WORKER_13_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_13_LINE_29_NO=320 -WORKER_13_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_13_LINE_30_NO=368 -WORKER_13_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_13_LINE_31_NO=373 -WORKER_13_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_13_LINE_32_NO=376 -WORKER_13_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_13_LINE_33_NO=377 -WORKER_13_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_13_LINE_34_NO=380 -WORKER_13_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_13_LINE_35_NO=381 -WORKER_13_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_13_LINE_36_NO=389 -WORKER_13_LINE_36_TEXT= done <"$manifest" -WORKER_13_LINE_37_NO=464 -WORKER_13_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_13_LINE_38_NO=465 -WORKER_13_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_13_LINE_39_NO=499 -WORKER_13_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_13_LINE_40_NO=504 -WORKER_13_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_13_LINE_41_NO=506 -WORKER_13_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_13_LINE_42_NO=508 -WORKER_13_LINE_42_TEXT= for row in manifest: -WORKER_13_LINE_43_NO=549 -WORKER_13_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_13_LINE_44_NO=561 -WORKER_13_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_13_LINE_45_NO=563 -WORKER_13_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_13_LINE_46_NO=569 -WORKER_13_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_13_LINE_47_NO=572 -WORKER_13_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_13_LINE_48_NO=573 -WORKER_13_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_13_LINE_49_NO=579 -WORKER_13_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_13_LINE_50_NO=586 -WORKER_13_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_13_LINE_51_NO=593 -WORKER_13_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_13_LINE_52_NO=630 -WORKER_13_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_13_LINE_53_NO=632 -WORKER_13_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_13_LINE_54_NO=678 -WORKER_13_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_13_LINE_55_NO=689 -WORKER_13_LINE_55_TEXT=ACTION="${1:-}" -WORKER_13_LINE_56_NO=690 -WORKER_13_LINE_56_TEXT=case "$ACTION" in -WORKER_13_LINE_57_NO=692 -WORKER_13_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_13_LINE_58_NO=693 -WORKER_13_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_14_BASENAME=homelab-app-cloud-quorum-worker -WORKER_14_PATH_HINT=homelab-ops-cr-2026-0033/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_14_EXECUTABLE=true -WORKER_14_PRESENT=true -WORKER_14_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_14_SHEBANG=#!/usr/bin/env bash -WORKER_14_BASH_N_RC=0 -WORKER_14_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_14_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_14_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_14_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_14_INTERESTING_LINE_COUNT=58 -WORKER_14_LINE_1_NO=4 -WORKER_14_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_14_LINE_2_NO=6 -WORKER_14_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_14_LINE_3_NO=7 -WORKER_14_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_14_LINE_4_NO=8 -WORKER_14_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_14_LINE_5_NO=9 -WORKER_14_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_14_LINE_6_NO=10 -WORKER_14_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_14_LINE_7_NO=69 -WORKER_14_LINE_7_TEXT=logical_id(){ -WORKER_14_LINE_8_NO=74 -WORKER_14_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_14_LINE_9_NO=90 -WORKER_14_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_14_LINE_10_NO=91 -WORKER_14_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_14_LINE_11_NO=109 -WORKER_14_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_14_LINE_12_NO=111 -WORKER_14_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_14_LINE_13_NO=112 -WORKER_14_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_14_LINE_14_NO=123 -WORKER_14_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_14_LINE_15_NO=125 -WORKER_14_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_14_LINE_16_NO=126 -WORKER_14_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_14_LINE_17_NO=127 -WORKER_14_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_14_LINE_18_NO=134 -WORKER_14_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_14_LINE_19_NO=149 -WORKER_14_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_14_LINE_20_NO=168 -WORKER_14_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_14_LINE_21_NO=176 -WORKER_14_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_14_LINE_22_NO=187 -WORKER_14_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_14_LINE_23_NO=218 -WORKER_14_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_14_LINE_24_NO=229 -WORKER_14_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_14_LINE_25_NO=245 -WORKER_14_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_14_LINE_26_NO=266 -WORKER_14_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_14_LINE_27_NO=301 -WORKER_14_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_14_LINE_28_NO=302 -WORKER_14_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_14_LINE_29_NO=320 -WORKER_14_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_14_LINE_30_NO=368 -WORKER_14_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_14_LINE_31_NO=373 -WORKER_14_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_14_LINE_32_NO=376 -WORKER_14_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_14_LINE_33_NO=377 -WORKER_14_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_14_LINE_34_NO=380 -WORKER_14_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_14_LINE_35_NO=381 -WORKER_14_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_14_LINE_36_NO=389 -WORKER_14_LINE_36_TEXT= done <"$manifest" -WORKER_14_LINE_37_NO=464 -WORKER_14_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_14_LINE_38_NO=465 -WORKER_14_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_14_LINE_39_NO=499 -WORKER_14_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_14_LINE_40_NO=504 -WORKER_14_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_14_LINE_41_NO=506 -WORKER_14_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_14_LINE_42_NO=508 -WORKER_14_LINE_42_TEXT= for row in manifest: -WORKER_14_LINE_43_NO=549 -WORKER_14_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_14_LINE_44_NO=561 -WORKER_14_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_14_LINE_45_NO=563 -WORKER_14_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_14_LINE_46_NO=569 -WORKER_14_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_14_LINE_47_NO=572 -WORKER_14_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_14_LINE_48_NO=573 -WORKER_14_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_14_LINE_49_NO=579 -WORKER_14_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_14_LINE_50_NO=586 -WORKER_14_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_14_LINE_51_NO=593 -WORKER_14_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_14_LINE_52_NO=630 -WORKER_14_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_14_LINE_53_NO=632 -WORKER_14_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_14_LINE_54_NO=678 -WORKER_14_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_14_LINE_55_NO=689 -WORKER_14_LINE_55_TEXT=ACTION="${1:-}" -WORKER_14_LINE_56_NO=690 -WORKER_14_LINE_56_TEXT=case "$ACTION" in -WORKER_14_LINE_57_NO=692 -WORKER_14_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_14_LINE_58_NO=693 -WORKER_14_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_15_BASENAME=homelab-app-cloud-quorum-worker -WORKER_15_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_15_EXECUTABLE=true -WORKER_15_PRESENT=true -WORKER_15_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_15_SHEBANG=#!/usr/bin/env bash -WORKER_15_BASH_N_RC=0 -WORKER_15_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_15_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_15_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_15_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_15_INTERESTING_LINE_COUNT=58 -WORKER_15_LINE_1_NO=4 -WORKER_15_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_15_LINE_2_NO=6 -WORKER_15_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_15_LINE_3_NO=7 -WORKER_15_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_15_LINE_4_NO=8 -WORKER_15_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_15_LINE_5_NO=9 -WORKER_15_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_15_LINE_6_NO=10 -WORKER_15_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_15_LINE_7_NO=69 -WORKER_15_LINE_7_TEXT=logical_id(){ -WORKER_15_LINE_8_NO=74 -WORKER_15_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_15_LINE_9_NO=90 -WORKER_15_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_15_LINE_10_NO=91 -WORKER_15_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_15_LINE_11_NO=109 -WORKER_15_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_15_LINE_12_NO=111 -WORKER_15_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_15_LINE_13_NO=112 -WORKER_15_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_15_LINE_14_NO=123 -WORKER_15_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_15_LINE_15_NO=125 -WORKER_15_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_15_LINE_16_NO=126 -WORKER_15_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_15_LINE_17_NO=127 -WORKER_15_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_15_LINE_18_NO=134 -WORKER_15_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_15_LINE_19_NO=149 -WORKER_15_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_15_LINE_20_NO=168 -WORKER_15_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_15_LINE_21_NO=176 -WORKER_15_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_15_LINE_22_NO=187 -WORKER_15_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_15_LINE_23_NO=218 -WORKER_15_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_15_LINE_24_NO=229 -WORKER_15_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_15_LINE_25_NO=245 -WORKER_15_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_15_LINE_26_NO=266 -WORKER_15_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_15_LINE_27_NO=301 -WORKER_15_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_15_LINE_28_NO=302 -WORKER_15_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_15_LINE_29_NO=320 -WORKER_15_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_15_LINE_30_NO=368 -WORKER_15_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_15_LINE_31_NO=373 -WORKER_15_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_15_LINE_32_NO=376 -WORKER_15_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_15_LINE_33_NO=377 -WORKER_15_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_15_LINE_34_NO=380 -WORKER_15_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_15_LINE_35_NO=381 -WORKER_15_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_15_LINE_36_NO=389 -WORKER_15_LINE_36_TEXT= done <"$manifest" -WORKER_15_LINE_37_NO=464 -WORKER_15_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_15_LINE_38_NO=465 -WORKER_15_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_15_LINE_39_NO=499 -WORKER_15_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_15_LINE_40_NO=504 -WORKER_15_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_15_LINE_41_NO=506 -WORKER_15_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_15_LINE_42_NO=508 -WORKER_15_LINE_42_TEXT= for row in manifest: -WORKER_15_LINE_43_NO=549 -WORKER_15_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_15_LINE_44_NO=561 -WORKER_15_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_15_LINE_45_NO=563 -WORKER_15_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_15_LINE_46_NO=569 -WORKER_15_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_15_LINE_47_NO=572 -WORKER_15_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_15_LINE_48_NO=573 -WORKER_15_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_15_LINE_49_NO=579 -WORKER_15_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_15_LINE_50_NO=586 -WORKER_15_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_15_LINE_51_NO=593 -WORKER_15_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_15_LINE_52_NO=630 -WORKER_15_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_15_LINE_53_NO=632 -WORKER_15_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_15_LINE_54_NO=678 -WORKER_15_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_15_LINE_55_NO=689 -WORKER_15_LINE_55_TEXT=ACTION="${1:-}" -WORKER_15_LINE_56_NO=690 -WORKER_15_LINE_56_TEXT=case "$ACTION" in -WORKER_15_LINE_57_NO=692 -WORKER_15_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_15_LINE_58_NO=693 -WORKER_15_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_16_BASENAME=homelab-app-cloud-quorum-worker -WORKER_16_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_16_EXECUTABLE=true -WORKER_16_PRESENT=true -WORKER_16_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_16_SHEBANG=#!/usr/bin/env bash -WORKER_16_BASH_N_RC=0 -WORKER_16_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_16_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_16_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_16_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_16_INTERESTING_LINE_COUNT=58 -WORKER_16_LINE_1_NO=4 -WORKER_16_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_16_LINE_2_NO=6 -WORKER_16_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_16_LINE_3_NO=7 -WORKER_16_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_16_LINE_4_NO=8 -WORKER_16_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_16_LINE_5_NO=9 -WORKER_16_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_16_LINE_6_NO=10 -WORKER_16_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_16_LINE_7_NO=69 -WORKER_16_LINE_7_TEXT=logical_id(){ -WORKER_16_LINE_8_NO=74 -WORKER_16_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_16_LINE_9_NO=90 -WORKER_16_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_16_LINE_10_NO=91 -WORKER_16_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_16_LINE_11_NO=109 -WORKER_16_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_16_LINE_12_NO=111 -WORKER_16_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_16_LINE_13_NO=112 -WORKER_16_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_16_LINE_14_NO=123 -WORKER_16_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_16_LINE_15_NO=125 -WORKER_16_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_16_LINE_16_NO=126 -WORKER_16_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_16_LINE_17_NO=127 -WORKER_16_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_16_LINE_18_NO=134 -WORKER_16_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_16_LINE_19_NO=149 -WORKER_16_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_16_LINE_20_NO=168 -WORKER_16_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_16_LINE_21_NO=176 -WORKER_16_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_16_LINE_22_NO=187 -WORKER_16_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_16_LINE_23_NO=218 -WORKER_16_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_16_LINE_24_NO=229 -WORKER_16_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_16_LINE_25_NO=245 -WORKER_16_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_16_LINE_26_NO=266 -WORKER_16_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_16_LINE_27_NO=301 -WORKER_16_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_16_LINE_28_NO=302 -WORKER_16_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_16_LINE_29_NO=320 -WORKER_16_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_16_LINE_30_NO=368 -WORKER_16_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_16_LINE_31_NO=373 -WORKER_16_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_16_LINE_32_NO=376 -WORKER_16_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_16_LINE_33_NO=377 -WORKER_16_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_16_LINE_34_NO=380 -WORKER_16_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_16_LINE_35_NO=381 -WORKER_16_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_16_LINE_36_NO=389 -WORKER_16_LINE_36_TEXT= done <"$manifest" -WORKER_16_LINE_37_NO=464 -WORKER_16_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_16_LINE_38_NO=465 -WORKER_16_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_16_LINE_39_NO=499 -WORKER_16_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_16_LINE_40_NO=504 -WORKER_16_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_16_LINE_41_NO=506 -WORKER_16_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_16_LINE_42_NO=508 -WORKER_16_LINE_42_TEXT= for row in manifest: -WORKER_16_LINE_43_NO=549 -WORKER_16_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_16_LINE_44_NO=561 -WORKER_16_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_16_LINE_45_NO=563 -WORKER_16_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_16_LINE_46_NO=569 -WORKER_16_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_16_LINE_47_NO=572 -WORKER_16_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_16_LINE_48_NO=573 -WORKER_16_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_16_LINE_49_NO=579 -WORKER_16_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_16_LINE_50_NO=586 -WORKER_16_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_16_LINE_51_NO=593 -WORKER_16_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_16_LINE_52_NO=630 -WORKER_16_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_16_LINE_53_NO=632 -WORKER_16_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_16_LINE_54_NO=678 -WORKER_16_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_16_LINE_55_NO=689 -WORKER_16_LINE_55_TEXT=ACTION="${1:-}" -WORKER_16_LINE_56_NO=690 -WORKER_16_LINE_56_TEXT=case "$ACTION" in -WORKER_16_LINE_57_NO=692 -WORKER_16_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_16_LINE_58_NO=693 -WORKER_16_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_17_BASENAME=homelab-app-cloud-quorum-worker -WORKER_17_PATH_HINT=homelab-ops-cr-2026-0034/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_17_EXECUTABLE=true -WORKER_17_PRESENT=true -WORKER_17_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_17_SHEBANG=#!/usr/bin/env bash -WORKER_17_BASH_N_RC=0 -WORKER_17_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_17_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_17_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_17_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_17_INTERESTING_LINE_COUNT=58 -WORKER_17_LINE_1_NO=4 -WORKER_17_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_17_LINE_2_NO=6 -WORKER_17_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_17_LINE_3_NO=7 -WORKER_17_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_17_LINE_4_NO=8 -WORKER_17_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_17_LINE_5_NO=9 -WORKER_17_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_17_LINE_6_NO=10 -WORKER_17_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_17_LINE_7_NO=69 -WORKER_17_LINE_7_TEXT=logical_id(){ -WORKER_17_LINE_8_NO=74 -WORKER_17_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_17_LINE_9_NO=90 -WORKER_17_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_17_LINE_10_NO=91 -WORKER_17_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_17_LINE_11_NO=109 -WORKER_17_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_17_LINE_12_NO=111 -WORKER_17_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_17_LINE_13_NO=112 -WORKER_17_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_17_LINE_14_NO=123 -WORKER_17_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_17_LINE_15_NO=125 -WORKER_17_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_17_LINE_16_NO=126 -WORKER_17_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_17_LINE_17_NO=127 -WORKER_17_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_17_LINE_18_NO=134 -WORKER_17_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_17_LINE_19_NO=149 -WORKER_17_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_17_LINE_20_NO=168 -WORKER_17_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_17_LINE_21_NO=176 -WORKER_17_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_17_LINE_22_NO=187 -WORKER_17_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_17_LINE_23_NO=218 -WORKER_17_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_17_LINE_24_NO=229 -WORKER_17_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_17_LINE_25_NO=245 -WORKER_17_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_17_LINE_26_NO=266 -WORKER_17_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_17_LINE_27_NO=301 -WORKER_17_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_17_LINE_28_NO=302 -WORKER_17_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_17_LINE_29_NO=320 -WORKER_17_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_17_LINE_30_NO=368 -WORKER_17_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_17_LINE_31_NO=373 -WORKER_17_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_17_LINE_32_NO=376 -WORKER_17_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_17_LINE_33_NO=377 -WORKER_17_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_17_LINE_34_NO=380 -WORKER_17_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_17_LINE_35_NO=381 -WORKER_17_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_17_LINE_36_NO=389 -WORKER_17_LINE_36_TEXT= done <"$manifest" -WORKER_17_LINE_37_NO=464 -WORKER_17_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_17_LINE_38_NO=465 -WORKER_17_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_17_LINE_39_NO=499 -WORKER_17_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_17_LINE_40_NO=504 -WORKER_17_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_17_LINE_41_NO=506 -WORKER_17_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_17_LINE_42_NO=508 -WORKER_17_LINE_42_TEXT= for row in manifest: -WORKER_17_LINE_43_NO=549 -WORKER_17_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_17_LINE_44_NO=561 -WORKER_17_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_17_LINE_45_NO=563 -WORKER_17_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_17_LINE_46_NO=569 -WORKER_17_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_17_LINE_47_NO=572 -WORKER_17_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_17_LINE_48_NO=573 -WORKER_17_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_17_LINE_49_NO=579 -WORKER_17_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_17_LINE_50_NO=586 -WORKER_17_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_17_LINE_51_NO=593 -WORKER_17_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_17_LINE_52_NO=630 -WORKER_17_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_17_LINE_53_NO=632 -WORKER_17_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_17_LINE_54_NO=678 -WORKER_17_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_17_LINE_55_NO=689 -WORKER_17_LINE_55_TEXT=ACTION="${1:-}" -WORKER_17_LINE_56_NO=690 -WORKER_17_LINE_56_TEXT=case "$ACTION" in -WORKER_17_LINE_57_NO=692 -WORKER_17_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_17_LINE_58_NO=693 -WORKER_17_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_18_BASENAME=homelab-app-cloud-quorum-worker -WORKER_18_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-cloud-quorum-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_18_EXECUTABLE=true -WORKER_18_PRESENT=true -WORKER_18_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_18_SHEBANG=#!/usr/bin/env bash -WORKER_18_BASH_N_RC=0 -WORKER_18_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_18_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_18_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_18_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_18_INTERESTING_LINE_COUNT=58 -WORKER_18_LINE_1_NO=4 -WORKER_18_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_18_LINE_2_NO=6 -WORKER_18_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_18_LINE_3_NO=7 -WORKER_18_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_18_LINE_4_NO=8 -WORKER_18_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_18_LINE_5_NO=9 -WORKER_18_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_18_LINE_6_NO=10 -WORKER_18_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_18_LINE_7_NO=69 -WORKER_18_LINE_7_TEXT=logical_id(){ -WORKER_18_LINE_8_NO=74 -WORKER_18_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_18_LINE_9_NO=90 -WORKER_18_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_18_LINE_10_NO=91 -WORKER_18_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_18_LINE_11_NO=109 -WORKER_18_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_18_LINE_12_NO=111 -WORKER_18_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_18_LINE_13_NO=112 -WORKER_18_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_18_LINE_14_NO=123 -WORKER_18_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_18_LINE_15_NO=125 -WORKER_18_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_18_LINE_16_NO=126 -WORKER_18_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_18_LINE_17_NO=127 -WORKER_18_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_18_LINE_18_NO=134 -WORKER_18_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_18_LINE_19_NO=149 -WORKER_18_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_18_LINE_20_NO=168 -WORKER_18_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_18_LINE_21_NO=176 -WORKER_18_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_18_LINE_22_NO=187 -WORKER_18_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_18_LINE_23_NO=218 -WORKER_18_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_18_LINE_24_NO=229 -WORKER_18_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_18_LINE_25_NO=245 -WORKER_18_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_18_LINE_26_NO=266 -WORKER_18_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_18_LINE_27_NO=301 -WORKER_18_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_18_LINE_28_NO=302 -WORKER_18_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_18_LINE_29_NO=320 -WORKER_18_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_18_LINE_30_NO=368 -WORKER_18_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_18_LINE_31_NO=373 -WORKER_18_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_18_LINE_32_NO=376 -WORKER_18_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_18_LINE_33_NO=377 -WORKER_18_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_18_LINE_34_NO=380 -WORKER_18_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_18_LINE_35_NO=381 -WORKER_18_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_18_LINE_36_NO=389 -WORKER_18_LINE_36_TEXT= done <"$manifest" -WORKER_18_LINE_37_NO=464 -WORKER_18_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_18_LINE_38_NO=465 -WORKER_18_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_18_LINE_39_NO=499 -WORKER_18_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_18_LINE_40_NO=504 -WORKER_18_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_18_LINE_41_NO=506 -WORKER_18_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_18_LINE_42_NO=508 -WORKER_18_LINE_42_TEXT= for row in manifest: -WORKER_18_LINE_43_NO=549 -WORKER_18_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_18_LINE_44_NO=561 -WORKER_18_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_18_LINE_45_NO=563 -WORKER_18_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_18_LINE_46_NO=569 -WORKER_18_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_18_LINE_47_NO=572 -WORKER_18_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_18_LINE_48_NO=573 -WORKER_18_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_18_LINE_49_NO=579 -WORKER_18_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_18_LINE_50_NO=586 -WORKER_18_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_18_LINE_51_NO=593 -WORKER_18_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_18_LINE_52_NO=630 -WORKER_18_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_18_LINE_53_NO=632 -WORKER_18_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_18_LINE_54_NO=678 -WORKER_18_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_18_LINE_55_NO=689 -WORKER_18_LINE_55_TEXT=ACTION="${1:-}" -WORKER_18_LINE_56_NO=690 -WORKER_18_LINE_56_TEXT=case "$ACTION" in -WORKER_18_LINE_57_NO=692 -WORKER_18_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_18_LINE_58_NO=693 -WORKER_18_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_19_BASENAME=homelab-app-cloud-quorum-worker -WORKER_19_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-bootstrap-configs-source-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_19_EXECUTABLE=true -WORKER_19_PRESENT=true -WORKER_19_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_19_SHEBANG=#!/usr/bin/env bash -WORKER_19_BASH_N_RC=0 -WORKER_19_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_19_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_19_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_19_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_19_INTERESTING_LINE_COUNT=58 -WORKER_19_LINE_1_NO=4 -WORKER_19_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_19_LINE_2_NO=6 -WORKER_19_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_19_LINE_3_NO=7 -WORKER_19_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_19_LINE_4_NO=8 -WORKER_19_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_19_LINE_5_NO=9 -WORKER_19_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_19_LINE_6_NO=10 -WORKER_19_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_19_LINE_7_NO=69 -WORKER_19_LINE_7_TEXT=logical_id(){ -WORKER_19_LINE_8_NO=74 -WORKER_19_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_19_LINE_9_NO=90 -WORKER_19_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_19_LINE_10_NO=91 -WORKER_19_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_19_LINE_11_NO=109 -WORKER_19_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_19_LINE_12_NO=111 -WORKER_19_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_19_LINE_13_NO=112 -WORKER_19_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_19_LINE_14_NO=123 -WORKER_19_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_19_LINE_15_NO=125 -WORKER_19_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_19_LINE_16_NO=126 -WORKER_19_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_19_LINE_17_NO=127 -WORKER_19_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_19_LINE_18_NO=134 -WORKER_19_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_19_LINE_19_NO=149 -WORKER_19_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_19_LINE_20_NO=168 -WORKER_19_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_19_LINE_21_NO=176 -WORKER_19_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_19_LINE_22_NO=187 -WORKER_19_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_19_LINE_23_NO=218 -WORKER_19_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_19_LINE_24_NO=229 -WORKER_19_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_19_LINE_25_NO=245 -WORKER_19_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_19_LINE_26_NO=266 -WORKER_19_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_19_LINE_27_NO=301 -WORKER_19_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_19_LINE_28_NO=302 -WORKER_19_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_19_LINE_29_NO=320 -WORKER_19_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_19_LINE_30_NO=368 -WORKER_19_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_19_LINE_31_NO=373 -WORKER_19_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_19_LINE_32_NO=376 -WORKER_19_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_19_LINE_33_NO=377 -WORKER_19_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_19_LINE_34_NO=380 -WORKER_19_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_19_LINE_35_NO=381 -WORKER_19_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_19_LINE_36_NO=389 -WORKER_19_LINE_36_TEXT= done <"$manifest" -WORKER_19_LINE_37_NO=464 -WORKER_19_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_19_LINE_38_NO=465 -WORKER_19_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_19_LINE_39_NO=499 -WORKER_19_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_19_LINE_40_NO=504 -WORKER_19_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_19_LINE_41_NO=506 -WORKER_19_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_19_LINE_42_NO=508 -WORKER_19_LINE_42_TEXT= for row in manifest: -WORKER_19_LINE_43_NO=549 -WORKER_19_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_19_LINE_44_NO=561 -WORKER_19_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_19_LINE_45_NO=563 -WORKER_19_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_19_LINE_46_NO=569 -WORKER_19_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_19_LINE_47_NO=572 -WORKER_19_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_19_LINE_48_NO=573 -WORKER_19_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_19_LINE_49_NO=579 -WORKER_19_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_19_LINE_50_NO=586 -WORKER_19_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_19_LINE_51_NO=593 -WORKER_19_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_19_LINE_52_NO=630 -WORKER_19_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_19_LINE_53_NO=632 -WORKER_19_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_19_LINE_54_NO=678 -WORKER_19_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_19_LINE_55_NO=689 -WORKER_19_LINE_55_TEXT=ACTION="${1:-}" -WORKER_19_LINE_56_NO=690 -WORKER_19_LINE_56_TEXT=case "$ACTION" in -WORKER_19_LINE_57_NO=692 -WORKER_19_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_19_LINE_58_NO=693 -WORKER_19_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -WORKER_20_BASENAME=homelab-app-cloud-quorum-worker -WORKER_20_PATH_HINT=homelab-ops-cr-2026-0035/tasks/backup-control-final-generation-v1/assets/runtime/homelab-app-cloud-quorum-worker -WORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_20_EXECUTABLE=true -WORKER_20_PRESENT=true -WORKER_20_SHA=57b64f20848105e5ec23aec4473f9acb715d03121dc20ca63d6cb78cba089f72 -WORKER_20_SHEBANG=#!/usr/bin/env bash -WORKER_20_BASH_N_RC=0 -WORKER_20_FUNCTIONS=archive_capture,backup_one,capture_filebrowser,capture_hbarhub,capture_immich,capture_memos,capture_moderator,capture_new,capture_reports,capture_scrutiny,cleanup,cleanup_filebrowser_capture_orphans,die,finalize_scrutiny_capture,latest_incomplete,latest_sealed,latest_sealed_state,logical_id,manifest_tree,new_capture_paths,on_err,plan,reconstruct_cloud_a,recover_scrutiny_orphan,remote_core,remote_edge,remote_validate_archive,remote_validate_filebrowser,restore_dir,restore_ok,restore_validate,resume_if_needed,rsync_core,scrutiny_health,scrutiny_restore_cleanup,scrutiny_resume,semantic_validate,source_host,state_dir,valid_generation,verify_binding_components,verify_manifest_tree,write_binding -WORKER_20_ACTIONS=filebrowser,hbarhub,immich,memos,scrutiny,skladchik-moderator-assistant,skladchik-reports-monitor -WORKER_20_VAR_NAMES=ACTION,ACTIVE_STAGE,CAPTURE,CORE,CORE_HELPER,EDGE,EDGE_HELPER,GEN_BASE,PAYLOAD,QUIESCED,QUIESCED_GEN,REMOTE_A,REMOTE_B,RESTORE_STATE_ROOT,RESTORE_WORK_ROOT,RSYNC_RSH,SCP_BASE,SCRATCH_CONTAINER,SCRUTINY_IMAGE_DIGEST_EXPECTED,SCRUTINY_IMAGE_ID_EXPECTED,SCRUTINY_IMAGE_REF_EXPECTED,SCRUTINY_UNIT,SCRUTINY_UNIT_SHA_EXPECTED,SSH_BASE,STAGE_ROOT,STATE_ROOT,TRANSPORT,TRANSPORT_SHA_EXPECTED,actual,actual_config,actual_influx,allowed,base,cfg,cfgsha,checks,cnt,completed,comps,config_manifest_sha,d,db,dbsha,devices,digest,expect,expected_ids,expected_sha,expected_tags,extra,f,gen,got_devices,got_msha,h,iid,image_hex,image_sha,image_tar,imagesha,influx_tables,latest,logical,manifest,meta,meta_tar,metasha,name,names,obj,orphan_count,out,part,payload,qstart,ref,remote_dump,report,required,rows,runtime,scdb,series,source_bytes,stacksha,started,state,tag_to_cfg,tmp,tsm_files,unitsha,wal_files,work -WORKER_20_COMMAND_BASENAMES=flock,jq,python3,rclone,sha256sum,systemctl,tar,zstd -WORKER_20_INTERESTING_LINE_COUNT=58 -WORKER_20_LINE_1_NO=4 -WORKER_20_LINE_1_TEXT=TRANSPORT=/usr/local/libexec/homelab-cloud-quorum-payload -WORKER_20_LINE_2_NO=6 -WORKER_20_LINE_2_TEXT=CORE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-core-helper -WORKER_20_LINE_3_NO=7 -WORKER_20_LINE_3_TEXT=EDGE_HELPER=/usr/local/libexec/homelab-app-cloud-quorum-edge-helper -WORKER_20_LINE_4_NO=8 -WORKER_20_LINE_4_TEXT=STATE_ROOT=/var/lib/homelab-backup/cloud-quorum -WORKER_20_LINE_5_NO=9 -WORKER_20_LINE_5_TEXT=RESTORE_STATE_ROOT=/var/lib/homelab-backup/app-restore-validation -WORKER_20_LINE_6_NO=10 -WORKER_20_LINE_6_TEXT=STAGE_ROOT=/mnt/staging/homelab-app-cloud-quorum -WORKER_20_LINE_7_NO=69 -WORKER_20_LINE_7_TEXT=logical_id(){ -WORKER_20_LINE_8_NO=74 -WORKER_20_LINE_8_TEXT= skladchik-reports-monitor) echo app-skladchik-reports-monitor ;; -WORKER_20_LINE_9_NO=90 -WORKER_20_LINE_9_TEXT=state_dir(){ echo "$STATE_ROOT/$(logical_id "$1")"; } -WORKER_20_LINE_10_NO=91 -WORKER_20_LINE_10_TEXT=restore_dir(){ echo "$RESTORE_STATE_ROOT/$(logical_id "$1")"; } -WORKER_20_LINE_11_NO=109 -WORKER_20_LINE_11_TEXT= jq -e --arg l "$(logical_id "$app")" --arg g "$gen" '.logical_id==$l and .generation==$g and .transport_state=="SEALED" and .restore_source=="Cloud A" and .payload_sha256_verified==true and .result=="OK"' "$f" >/dev/null -WORKER_20_LINE_12_NO=111 -WORKER_20_LINE_12_TEXT=remote_core(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$CORE" sudo -n /bin/bash -s -- "$action" "$gen" <"$CORE_HELPER"; } -WORKER_20_LINE_13_NO=112 -WORKER_20_LINE_13_TEXT=remote_edge(){ local action="$1" gen="$2"; ssh "${SSH_BASE[@]}" "$EDGE" sudo -n /bin/bash -s -- "$action" "$gen" <"$EDGE_HELPER"; } -WORKER_20_LINE_14_NO=123 -WORKER_20_LINE_14_TEXT= local dir="$1" manifest="$2" label="$3" -WORKER_20_LINE_15_NO=125 -WORKER_20_LINE_15_TEXT= [[ -f "$manifest" && ! -L "$manifest" ]] || die "MANIFEST_FILE_INVALID_${label}" 70 -WORKER_20_LINE_16_NO=126 -WORKER_20_LINE_16_TEXT= if [[ -s "$manifest" ]]; then -WORKER_20_LINE_17_NO=127 -WORKER_20_LINE_17_TEXT= (cd "$dir"; sha256sum -c "$manifest" >/dev/null) || die "MANIFEST_CHECK_FAILED_${label}" 70 -WORKER_20_LINE_18_NO=134 -WORKER_20_LINE_18_TEXT= jq -n --arg schema homelab-app-generation-v1 --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg source "$(source_host "$app")" --arg started "$started" --arg completed "$completed" --arg qstart "$qstart" --arg qend "$qend" --argjson components "$components_json" '{schema:$schema,logical_id:$logical_id,generation:$generation,app:$app,source_host:$source,capture_started_at_utc:$started,capture_completed_at_utc:$completed,quiesce_started_at_utc:($qstart|select(length>0)//null),quiesce_completed_at_utc:($qend|select(length>0)//null),components:$components}' >"$file" -WORKER_20_LINE_19_NO=149 -WORKER_20_LINE_19_TEXT= logical="$(logical_id "$app")" -WORKER_20_LINE_20_NO=168 -WORKER_20_LINE_20_TEXT= write_binding "$CAPTURE/app-generation.json" app-memos "$gen" memos "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_20_LINE_21_NO=176 -WORKER_20_LINE_21_TEXT= sqlite3 /var/lib/skladchik-moderator-assistant/state.sqlite3 ".backup '$CAPTURE/skladchik-moderator-assistant/var-lib/state.sqlite3'" -WORKER_20_LINE_22_NO=187 -WORKER_20_LINE_22_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-moderator-assistant "$gen" skladchik-moderator-assistant "$started" "$completed" "" "" "$comps" -WORKER_20_LINE_23_NO=218 -WORKER_20_LINE_23_TEXT= write_binding "$CAPTURE/app-generation.json" app-filebrowser "$gen" filebrowser "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_20_LINE_24_NO=229 -WORKER_20_LINE_24_TEXT= write_binding "$CAPTURE/app-generation.json" app-skladchik-reports-monitor "$gen" skladchik-reports-monitor "$started" "$completed" "" "" "$comps" -WORKER_20_LINE_25_NO=245 -WORKER_20_LINE_25_TEXT= write_binding "$CAPTURE/app-generation.json" app-immich "$gen" immich "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_20_LINE_26_NO=266 -WORKER_20_LINE_26_TEXT= write_binding "$CAPTURE/app-generation.json" app-hbarhub "$gen" hbarhub "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_20_LINE_27_NO=301 -WORKER_20_LINE_27_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_20_LINE_28_NO=302 -WORKER_20_LINE_28_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_20_LINE_29_NO=320 -WORKER_20_LINE_29_TEXT= write_binding "$CAPTURE/app-generation.json" app-scrutiny "$gen" scrutiny "$started" "$completed" "$qstart" "$qend" "$comps" -WORKER_20_LINE_30_NO=368 -WORKER_20_LINE_30_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.schema=="homelab-app-generation-v1" and .logical_id==$l and .generation==$g and (.components|type=="object" and length>0)' "$ex/app-generation.json" >/dev/null || die RESTORE_BINDING_IDENTITY_INVALID 70 -WORKER_20_LINE_31_NO=373 -WORKER_20_LINE_31_TEXT= done < <(jq -r '.' "$ex/app-generation.json") -WORKER_20_LINE_32_NO=376 -WORKER_20_LINE_32_TEXT= local app="$1" gen="$2" state="$3" out="$4" logical base manifest expected_msha got_msha idx name size sha part actual -WORKER_20_LINE_33_NO=377 -WORKER_20_LINE_33_TEXT= logical="$(logical_id "$app")" -WORKER_20_LINE_34_NO=380 -WORKER_20_LINE_34_TEXT= manifest="$(dirname "$out")/PARTS.tsv"; rclone cat "${base%/}/PARTS.tsv" >"$manifest" -WORKER_20_LINE_35_NO=381 -WORKER_20_LINE_35_TEXT= got_msha="$(sha256sum "$manifest"|awk '{print $1}')"; [[ "$got_msha" == "$expected_msha" ]] || die RESTORE_A_MANIFEST_SHA_MISMATCH 70 -WORKER_20_LINE_36_NO=389 -WORKER_20_LINE_36_TEXT= done <"$manifest" -WORKER_20_LINE_37_NO=464 -WORKER_20_LINE_37_TEXT= if "manifest.json" not in names: raise SystemExit(70) -WORKER_20_LINE_38_NO=465 -WORKER_20_LINE_38_TEXT= rows=json.load(tf.extractfile("manifest.json")) -WORKER_20_LINE_39_NO=499 -WORKER_20_LINE_39_TEXT=expected_tags={"hbarhub-web:latest","hbarhub-worker:latest","hbarhub-scheduler:latest"} -WORKER_20_LINE_40_NO=504 -WORKER_20_LINE_40_TEXT= if "manifest.json" not in names: -WORKER_20_LINE_41_NO=506 -WORKER_20_LINE_41_TEXT= manifest=json.load(tf.extractfile("manifest.json")) -WORKER_20_LINE_42_NO=508 -WORKER_20_LINE_42_TEXT= for row in manifest: -WORKER_20_LINE_43_NO=549 -WORKER_20_LINE_43_TEXT= logical="$(logical_id "$app")"; state="$STATE_ROOT/$logical/$gen.json" -WORKER_20_LINE_44_NO=561 -WORKER_20_LINE_44_TEXT= jq -n --arg logical_id "$logical" --arg generation "$gen" --arg app "$app" --arg sha "$expected_sha" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg state "$state" --argjson checks "$checks" '{schema:"homelab-app-restore-validation-v1",logical_id:$logical_id,generation:$generation,app:$app,transport_state:"SEALED",transport_state_file:$state,restore_source:"Cloud A",payload_sha256:$sha,payload_sha256_verified:true,result:"OK",validated_at_utc:$at,checks:$checks}' >"$tmp" -WORKER_20_LINE_45_NO=563 -WORKER_20_LINE_45_TEXT= jq -e --arg l "$logical" --arg g "$gen" '.logical_id==$l and .generation==$g and .result=="OK" and .transport_state=="SEALED" and .payload_sha256_verified==true' "$out" >/dev/null -WORKER_20_LINE_46_NO=569 -WORKER_20_LINE_46_TEXT= logical="$(logical_id "$app")"; mkdir -p "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT/$logical"; chmod 0700 "$STATE_ROOT" "$STATE_ROOT/$logical" "$RESTORE_STATE_ROOT" "$RESTORE_STATE_ROOT/$logical" "$STAGE_ROOT" "$STAGE_ROOT/$logical" -WORKER_20_LINE_47_NO=572 -WORKER_20_LINE_47_TEXT= state="${pending[0]}"; gen="$(jq -er '.generation' "$state")"; payload="$(jq -er '.payload_path' "$state")"; printf 'RESUME=YES\nAPP=%s\nGENERATION=%s\nSTAGE=TRANSPORT_RESUME\n' "$app" "$gen" -WORKER_20_LINE_48_NO=573 -WORKER_20_LINE_48_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_20_LINE_49_NO=579 -WORKER_20_LINE_49_TEXT= if [[ -n "$latest" ]]; then gen="$(jq -er '.generation' "$latest")"; if ! restore_ok "$app" "$gen"; then printf 'RESUME_RESTORE_VALIDATION=YES\nAPP=%s\nGENERATION=%s\n' "$app" "$gen"; restore_validate "$app" "$gen"; printf 'BACKUP_STATUS=SEALED\nAPP=%s\nGENERATION=%s\nCLOUD_QUORUM=2\n' "$app" "$gen"; return; fi; fi -WORKER_20_LINE_50_NO=586 -WORKER_20_LINE_50_TEXT= printf 'STAGE=TRANSPORT\n'; "$TRANSPORT" backup-delete "$logical" "$gen" "$payload" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_20_LINE_51_NO=593 -WORKER_20_LINE_51_TEXT= logical="$(logical_id "$app")"; free="$(df -B1 --output=avail /mnt/staging | tail -n1 | tr -d ' ')" -WORKER_20_LINE_52_NO=630 -WORKER_20_LINE_52_TEXT= f="$(latest_sealed_state "$app")"; [[ -n "$f" ]] || die "SEALED_STATE_MISSING_$app" 65; gen="$(jq -er '.generation' "$f")" -WORKER_20_LINE_53_NO=632 -WORKER_20_LINE_53_TEXT= printf 'LATEST_SEALED_STATUS=OK\nAPP=%s\nLOGICAL_ID=%s\nGENERATION=%s\nSTATE_FILE=%s\nRESTORE_EVIDENCE=%s\n' "$app" "$(logical_id "$app")" "$gen" "$f" "$(restore_dir "$app")/$gen.json" -WORKER_20_LINE_54_NO=678 -WORKER_20_LINE_54_TEXT= "$TRANSPORT" backup-delete "$logical" "$gen" "$PAYLOAD" "$REMOTE_A" "$REMOTE_B" "$STATE_ROOT" -WORKER_20_LINE_55_NO=689 -WORKER_20_LINE_55_TEXT=ACTION="${1:-}" -WORKER_20_LINE_56_NO=690 -WORKER_20_LINE_56_TEXT=case "$ACTION" in -WORKER_20_LINE_57_NO=692 -WORKER_20_LINE_57_TEXT= backup) [[ "$#" == 2 ]] || die USAGE_BACKUP 64; backup_one "$2" ;; -WORKER_20_LINE_58_NO=693 -WORKER_20_LINE_58_TEXT= restore-validate) [[ "$#" == 3 ]] || die USAGE_RESTORE_VALIDATE 64; logical_id "$2" >/dev/null; restore_validate "$2" "$3" ;; -TARGETS_PRESENT=true -TARGETS_SHA=2f526330152d49461ff66f50adab57d1a195ea5d1b4c292945d523411b8a68a8 -TARGET_OBJECT_FOUND=true -TARGET_KEYS=id,kind,lifecycle,logical_id,migrate,owner,source_host -TARGET_ID=skladchik-reports-monitor -TARGET_KIND=APP -TARGET_LIFECYCLE=ACTIVE -TARGET_LOGICAL_ID=app-skladchik-reports-monitor -TARGET_MIGRATE=True -TARGET_OWNER=pve01 -TARGET_SOURCE_HOST_BASENAME=[PRIVATE_IP] -TARGET_SOURCE_HOST_SHA=3eb2319e39eac307b3c7bd11f67663c0a0c814fd603989f5965083f9d1c80fb0 -QUEUE_APPID_LITERAL_COUNT=0 -QUEUE_TARGETS_BASENAME_LITERAL_COUNT=0 -BACKUP_AUDIT_RC=0 -BACKUP_AUDIT_RELEVANT_COUNT=2 -BACKUP_AUDIT_RELEVANT_1=TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=22759|generation=app-skladchik-reports-monitor-pve01-20261005T003035Z -BACKUP_AUDIT_RELEVANT_2=TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=32145|legacy_age_seconds=1627370|replica_count=2|state_id=vm170|generation=v2-vm170-20261004T214054Z-70a537ab4e91 -TARGETED_SEAL_DESIGN_READY=true -MAIL_SENT_BY_COMMAND=false +INGRESS_HOSTKEY_SELF_BOUND=true +VM150_INVENTORY_PRESENT=true +VM150_RUNNING=true +VM150_EXPECTED_NODE=true +VM171_INVENTORY_PRESENT=true +VM171_RUNNING=true +VM171_EXPECTED_NODE=true +VM150_QGA_PING=true +VM171_QGA_PING=true +PVE_BACKUP_JOBS_QUERY_RC=0 +PVE_BACKUP_JOB_COVERAGE_COUNT=0 +PVE_BACKUP_JOB_ENABLED_COVERAGE_COUNT=0 +PVE_BACKUP_STORAGE_QUERY_RC=0 +PVE_BACKUP_STORAGE_COUNT=1 +PVE_VM150_BACKUP_OBJECT_COUNT=0 +PVE_VM150_BACKUP_STORAGE_WITH_OBJECT_COUNT=0 +PVE_VM150_NEWEST_BACKUP_AGE_HOURS=-1 +BACKUP_ENGINE_VM150_REF_14D_COUNT=40 +BACKUP_ENGINE_VM150_SUCCESS_14D_COUNT=40 +BACKUP_ENGINE_VM150_LATEST_SUCCESS_AGE_HOURS=16 +BACKUP_AUDIT_VM150_REF_30D_COUNT=2379 +BACKUP_AUDIT_VM150_RESTORE_PASS_30D_COUNT=0 +BACKUP_AUDIT_VM150_LATEST_RESTORE_PASS_AGE_HOURS=-1 +BACKUP_EVIDENCE_CLASS_VZDUMP=true +BACKUP_EVIDENCE_CLASS_RESTIC=false +BACKUP_EVIDENCE_CLASS_BORG=false +BACKUP_EVIDENCE_CLASS_RCLONE=false +BACKUP_EVIDENCE_CLASS_RSYNC_TAR=false +BACKUP_EVIDENCE_CLASS_DOCKER_DATA=false +BACKUP_EVIDENCE_CLASS_REMOTE=true +BACKUP_SOURCE_FILE_COUNT=482 +BACKUP_SOURCE_VM150_REF_COUNT=45 +BACKUP_SOURCE_CAP_VZDUMP=true +BACKUP_SOURCE_CAP_RESTIC=false +BACKUP_SOURCE_CAP_BORG=false +BACKUP_SOURCE_CAP_RCLONE=true +BACKUP_SOURCE_CAP_RSYNC_TAR=true +BACKUP_SOURCE_CAP_DYNAMIC_VM=true +PVE01_GIT_REPO_COUNT=64 +PVE01_SNIKKET_SOURCE_CANDIDATE_REPO_COUNT=49 +PVE01_SOURCE_RECON_EXACT_SHA_COUNT=0 +PVE01_SOURCE_COTURN_EXACT_SHA_COUNT=49 +PVE01_SOURCE_OVERRIDE_EXACT_SHA_COUNT=49 +PVE01_SOURCE_RECON_CORRECT_LOGIC_COUNT=0 +PVE01_SOURCE_DNS_ALL3_COUNT=343 +PVE01_SOURCE_DNS_CURRENT_TARGET_COUNT=294 +PVE01_MONITOR_SOURCE_FILE_COUNT=86 +PVE01_MONITOR_CHAT_REF=false +PVE01_MONITOR_XMPP_5222_REF=false +PVE01_MONITOR_XMPP_5269_REF=false +PVE01_MONITOR_TURN_3479_REF=false +PVE01_MONITOR_STUN_TURN_REF=false +VM150_QGA_EXEC_INIT_RC=0 +VM150_QGA_POLL_RC=0 +VM_GUEST_EXITCODE=0 +VM_GUEST_OUTPUT_MARKER=true +SNIKKET_OPS_OWNERSHIP_GUEST150=V1 +MODE=read-only +NO_MUTATION=true +GUEST150_HOSTNAME_OK=true +GUEST150_LIVE_RECON_PRESENT=true +GUEST150_LIVE_RECON_SHA256=bb692a3a044d41e6e04518aff5bdfa646f533660a70ea793f87d040c881a6e98 +GUEST150_LIVE_COTURN_PRESENT=true +GUEST150_LIVE_COTURN_SHA256=2de0e47c09b3b4fc7a6c32c5ef061efaaa7c7f4db3d7a25fdf4da8fbb2e6f483 +GUEST150_LIVE_OVERRIDE_PRESENT=true +GUEST150_LIVE_OVERRIDE_SHA256=54a42711db5c2eb99b3886184c7ce3e01a30aba844ff2022bc74a45414a2241c +GUEST150_GIT_REPO_COUNT=0 +GUEST150_SNIKKET_TRACKED_FILE_COUNT=0 +GUEST150_SOURCE_CANDIDATE_REPO_COUNT=0 +GUEST150_SOURCE_RECON_EXACT_SHA_COUNT=0 +GUEST150_SOURCE_COTURN_EXACT_SHA_COUNT=0 +GUEST150_SOURCE_OVERRIDE_EXACT_SHA_COUNT=0 +GUEST150_SOURCE_RECON_CORRECT_LOGIC_COUNT=0 +GUEST150_RECON_SERVICE_UNIT_PRESENT=true +GUEST150_RECON_TIMER_UNIT_PRESENT=true +GUEST150_EDGE_UNIT_PRESENT=true +GUEST150_TCP3479_COUNT=16 +GUEST150_UDP3479_COUNT=16 +GUEST150_RECON_SERVICE_RESULT=success +GUEST150_RECON_SERVICE_EXEC_MAIN_STATUS=0 +GUEST150_AUDIT_DONE=true +VM171_QGA_EXEC_INIT_RC=0 +VM171_QGA_POLL_RC=0 +VM_GUEST_EXITCODE=0 +VM_GUEST_OUTPUT_MARKER=true +SNIKKET_OPS_OWNERSHIP_GUEST171=V1 +MODE=read-only +NO_MUTATION=true +GUEST171_HOSTNAME_MONITORING=true +GUEST171_MONITOR_TEXT_FILE_COUNT=373 +GUEST171_MONITOR_CHAT_REF=false +GUEST171_MONITOR_GROUPS_REF=false +GUEST171_MONITOR_SHARE_REF=false +GUEST171_MONITOR_PORT_443_REF=true +GUEST171_MONITOR_PORT_5000_REF=false +GUEST171_MONITOR_PORT_5222_REF=true +GUEST171_MONITOR_PORT_5269_REF=false +GUEST171_MONITOR_PORT_3479_REF=true +GUEST171_MONITOR_PORT_5349_REF=false +GUEST171_MONITOR_PORT_5350_REF=false +GUEST171_MONITOR_STUN_TURN_REF=false +GUEST171_MONITOR_CERT_EXPIRY_REF=true +GUEST171_KUMA_DB_COUNT=2 +GUEST171_KUMA_MONITOR_COUNT=192 +GUEST171_KUMA_CHAT_MATCH_COUNT=0 +GUEST171_KUMA_GROUPS_MATCH_COUNT=0 +GUEST171_KUMA_SHARE_MATCH_COUNT=0 +GUEST171_KUMA_5222_MATCH_COUNT=0 +GUEST171_KUMA_5269_MATCH_COUNT=0 +GUEST171_KUMA_3479_MATCH_COUNT=0 +GUEST171_KUMA_STUN_TURN_MATCH_COUNT=0 +GUEST171_AUDIT_DONE=true +PUBLIC_DNS_OK_COUNT=3 +PUBLIC_HTTPS_OK_COUNT=3 +PUBLIC_STUN_3479_PASS=true +PUBLIC_STUN_3479_CLASS=BINDING_SUCCESS +OPS_OWNERSHIP_RCA_COMPLETE=true +DECISION=ANALYZE_EXACT_99578_OWNERSHIP +NEXT_GATE=TARGETED_OPERATIONAL_HARDENING +TASK_RESULT=SNIKKET_OPS_OWNERSHIP_RCA CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null MUTATION_OUTCOME=NO_MUTATION -DECISION=PASS_QUEUE_WORKER_RCA -NEXT_GATE=DESIGN_ATOMIC_SOURCE_BACKUP_SEAL -TASK_RESULT=SKLADCHIK_DURABILITY_QUEUE_WORKER_RCA OUTPUT_END CHAT_OUTPUT_END