diff --git a/runtime/history/SIGNALBOT-260908-INTAKE-RCA-035.json b/runtime/history/SIGNALBOT-260908-INTAKE-RCA-035.json new file mode 100644 index 00000000..065036db --- /dev/null +++ b/runtime/history/SIGNALBOT-260908-INTAKE-RCA-035.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SIGNALBOT-260908-INTAKE-RCA-035", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "signalbot-cr0116-intake-rca", + "started_at_utc": "2026-09-08T05:23:21Z", + "finished_at_utc": "2026-09-08T05:23:55Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "01c43885d0820853cc620910f562146315ad34c12ff24ffceefe7ef0181b8778", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "9bc13c343cdda506b9e7bbb4a38191d9f1ee1e4d31beb0916fa1cda673e1e097", + "sanitized_output_sha256": "9bc13c343cdda506b9e7bbb4a38191d9f1ee1e4d31beb0916fa1cda673e1e097", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "ERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nAUTHORITY_CHECK_SCOPE=READABILITY_ONLY_INCIDENTS_NOT_CLOSED\nSIGNALBOT035_VM170={\"name\":\"core-apps\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"qemu\"}\nSIGNALBOT035_VM170_SNAPSHOT={\"report\":{\"collector\":{\"cmd\":[\"python\",\"-m\",\"demonov_forward_map.bitfinex_trade_collector\"],\"entrypoint\":null,\"health\":null,\"id_short\":\"a546a70c3e8c\",\"image\":\"8020-demonov-shadow:e32760c69a4311728db9066ee8bf2bf66b1e5a70\",\"name\":\"demonov-shadow-8020-collector-1\",\"pid\":210213,\"restart_count\":13,\"running\":true,\"started_at\":\"2026-09-08T05:01:08.70385245Z\",\"status\":\"running\"},\"collector_status\":{\"age_sec\":8.9,\"exists\":true,\"mtime_epoch\":1788845024.0247085,\"path\":\"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/collector_status.json\",\"values\":{\"fail_closed_reason\":\"source reconnect detected\",\"gap_detected\":false,\"post_finality_revision\":false,\"prospective_fail_closed_reason\":\"\",\"prospective_gap_detected\":false,\"prospective_reconnect_detected\":false,\"reconnect_detected\":true,\"updated_at\":\"2026-09-08T05:23:44.028140+00:00\"}},\"db_a\":{\"d1_revisions_count\":0,\"event_kind_counts\":[[\"SEQUENCE\",179729],[\"RECONNECT\",10],[\"SESSION_START\",1]],\"exists\":true,\"finalized_d1_count\":0,\"latest_sequence_events\":[{\"event_kind\":\"SEQUENCE\",\"event_seq\":179740,\"received_at\":\"2026-09-08T05:23:20.918750+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2510},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179739,\"received_at\":\"2026-09-08T05:23:20.891856+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2509},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179738,\"received_at\":\"2026-09-08T05:23:20.552139+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2508},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179737,\"received_at\":\"2026-09-08T05:23:20.534792+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2507},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179736,\"received_at\":\"2026-09-08T05:23:20.329909+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2506},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179735,\"received_at\":\"2026-09-08T05:23:20.177708+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2505},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179734,\"received_at\":\"2026-09-08T05:23:20.170743+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2504},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179733,\"received_at\":\"2026-09-08T05:23:20.163522+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2503},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179732,\"received_at\":\"2026-09-08T05:23:20.004001+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2502},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179731,\"received_at\":\"2026-09-08T05:23:19.850034+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2501},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179730,\"received_at\":\"2026-09-08T05:23:19.841410+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2500},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179729,\"received_at\":\"2026-09-08T05:23:19.827700+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2499},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179728,\"received_at\":\"2026-09-08T05:23:19.671403+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2498},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179727,\"received_at\":\"2026-09-08T05:23:19.518174+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2497},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179726,\"received_at\":\"2026-09-08T05:23:19.357283+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2496},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179725,\"received_at\":\"2026-09-08T05:23:19.200903+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2495},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179724,\"received_at\":\"2026-09-08T05:23:19.193716+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2494},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179723,\"received_at\":\"2026-09-08T05:23:19.185835+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2493},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179722,\"received_at\":\"2026-09-08T05:23:19.177983+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2492},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179721,\"received_at\":\"2026-09-08T05:23:19.164437+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2491},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179720,\"received_at\":\"2026-09-08T05:23:19.006208+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2490},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179719,\"received_at\":\"2026-09-08T05:23:18.848474+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2489},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179718,\"received_at\":\"2026-09-08T05:23:18.697530+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2488},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179717,\"received_at\":\"2026-09-08T05:23:18.541070+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2487},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179716,\"received_at\":\"2026-09-08T05:23:18.532005+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2486},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179715,\"received_at\":\"2026-09-08T05:23:18.522984+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2485},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179714,\"received_at\":\"2026-09-08T05:23:18.515020+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2484},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179713,\"received_at\":\"2026-09-08T05:23:18.504619+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2483},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179712,\"received_at\":\"2026-09-08T05:23:18.349318+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2482},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179711,\"received_at\":\"2026-09-08T05:23:18.189830+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2481}],\"latest_sessions\":[{\"events\":2510,\"first_received_at\":\"2026-09-08T05:01:09.298101+00:00\",\"last_received_at\":\"2026-09-08T05:23:20.918750+00:00\",\"max_source_sequence\":2510,\"min_source_sequence\":1,\"session_hash\":\"253ffe9a26a5\"},{\"events\":2475,\"first_received_at\":\"2026-09-08T04:54:28.087186+00:00\",\"last_received_at\":\"2026-09-08T05:01:08.350487+00:00\",\"max_source_sequence\":2475,\"min_source_sequence\":1,\"session_hash\":\"2b3998047f77\"},{\"events\":988,\"first_received_at\":\"2026-09-08T04:43:46.442548+00:00\",\"last_received_at\":\"2026-09-08T04:54:27.005150+00:00\",\"max_source_sequence\":988,\"min_source_sequence\":1,\"session_hash\":\"b501ae120c1f\"},{\"events\":589,\"first_received_at\":\"2026-09-08T04:42:44.434580+00:00\",\"last_received_at\":\"2026-09-08T04:43:45.521859+00:00\",\"max_source_sequence\":589,\"min_source_sequence\":1,\"session_hash\":\"10709eb54a7b\"},{\"events\":3620,\"first_received_at\":\"2026-09-08T03:52:02.239825+00:00\",\"last_received_at\":\"2026-09-08T04:42:43.449781+00:00\",\"max_source_sequence\":3620,\"min_source_sequence\":1,\"session_hash\":\"210de3583c8e\"},{\"events\":72124,\"first_received_at\":\"2026-09-07T15:04:59.188298+00:00\",\"last_received_at\":\"2026-09-08T03:52:00.952767+00:00\",\"max_source_sequence\":72124,\"min_source_sequence\":1,\"session_hash\":\"69bb22240441\"},{\"events\":1688,\"first_received_at\":\"2026-09-07T14:46:51.906734+00:00\",\"last_received_at\":\"2026-09-07T15:04:47.491753+00:00\",\"max_source_sequence\":1688,\"min_source_sequence\":1,\"session_hash\":\"d1d81c259d86\"},{\"events\":10295,\"first_received_at\":\"2026-09-07T13:30:08.973280+00:00\",\"last_received_at\":\"2026-09-07T14:46:39.866418+00:00\",\"max_source_sequence\":10295,\"min_source_sequence\":1,\"session_hash\":\"a2db9de2102e\"},{\"events\":14430,\"first_received_at\":\"2026-09-07T09:50:41.075840+00:00\",\"last_received_at\":\"2026-09-07T13:30:03.165612+00:00\",\"max_source_sequence\":14430,\"min_source_sequence\":1,\"session_hash\":\"8da25a454977\"},{\"events\":71006,\"first_received_at\":\"2026-09-06T18:48:31.906387+00:00\",\"last_received_at\":\"2026-09-07T09:50:13.818640+00:00\",\"max_source_sequence\":71006,\"min_source_sequence\":1,\"session_hash\":\"184e0e63b1c9\"}],\"latest_source_receipts\":[{\"receipt_id\":\"02e5367b2692cde4f9bff73096271e6cc0598bf27413d651026f6318c8338d72\",\"received_at\":\"2026-09-08T03:52:00.952767+00:00\",\"source_sequence\":72124},{\"receipt_id\":\"568fff4620581353cea781b3cef61de359e02f12475ac54c7460e85c4baa8625\",\"received_at\":\"2026-09-08T03:52:00.943741+00:00\",\"source_sequence\":72123},{\"receipt_id\":\"3631962ca9ee6b49c4870d42ec814ad6c1c3f0ff65f78a445abecc54bcbe8567\",\"received_at\":\"2026-09-08T03:52:00.931321+00:00\",\"source_sequence\":72122},{\"receipt_id\":\"e9e23d4774b5bdc4801a3334511f50fbb5b25e117b01f8e913a98be3d98badc0\",\"received_at\":\"2026-09-08T03:52:00.790541+00:00\",\"source_sequence\":72121},{\"receipt_id\":\"59ad6dd601a0d1605e6ea3962d9bfb348cdbc67cc8666b473eb8d58b1b8c077c\",\"received_at\":\"2026-09-08T03:52:00.656138+00:00\",\"source_sequence\":72120},{\"receipt_id\":\"634a7fdd47cddc7c664450d7921d4b83093d20a6f95f72f31f6dc004b219ff0f\",\"received_at\":\"2026-09-08T03:52:00.642417+00:00\",\"source_sequence\":72119},{\"receipt_id\":\"6943e44a184672570beaf3ac076e8051162086f16c4f836d78b040c6fdb811ce\",\"received_at\":\"2026-09-08T03:52:00.504076+00:00\",\"source_sequence\":72118},{\"receipt_id\":\"063b4e0abd1065a59d6b9fe62b9e4583f4caacbbb1e83414875386170b8ba7fe\",\"received_at\":\"2026-09-08T03:52:00.494856+00:00\",\"source_sequence\":72117},{\"receipt_id\":\"0f018a85552ad881ab665baa7ddc7a0e956b6d0d66ec43ecfeee44b60e766f4c\",\"received_at\":\"2026-09-08T03:52:00.481147+00:00\",\"source_sequence\":72116},{\"receipt_id\":\"6b23823a92031d3850483879578a8528a45b5f6002b26381659904d813f5b8d2\",\"received_at\":\"2026-09-08T03:52:00.333870+00:00\",\"source_sequence\":72115},{\"receipt_id\":\"1985ce2996de079095414ce9a48bfa3fcfffe7ddcbcdcff1857e03aa0c770d7d\",\"received_at\":\"2026-09-08T03:52:00.194864+00:00\",\"source_sequence\":72114},{\"receipt_id\":\"a4ed84172daff21758b2402a6cc03ab51ddc9474ba23d7a4036f7c6c4a3af152\",\"received_at\":\"2026-09-08T03:52:00.180891+00:00\",\"source_sequence\":72113},{\"receipt_id\":\"8646a6eea56850890817a5ff2056f9639f830a876e460c3898cfebbbd881a670\",\"received_at\":\"2026-09-08T03:52:00.041606+00:00\",\"source_sequence\":72112},{\"receipt_id\":\"70ebbdab1c237840ea3cb6b543fab899e49db3b3a0ab65455e5dcb85b7844923\",\"received_at\":\"2026-09-08T03:52:00.023714+00:00\",\"source_sequence\":72111},{\"receipt_id\":\"7cb2c79c0940deed850deb26500cb3c04331baa4eee8e302a1ca9a3e1cd3023c\",\"received_at\":\"2026-09-08T03:51:59.882828+00:00\",\"source_sequence\":72110}],\"path\":\"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/bitfinex_source.db\",\"quick_check\":\"ok\",\"sequence_events\":{\"count\":179740,\"max_event_seq\":179740,\"max_received_at\":\"2026-09-08T05:23:20.918750+00:00\",\"max_source_sequence\":72124},\"source_receipts_columns\":[\"receipt_id\",\"venue_id\",\"source_kind\",\"received_at\",\"source_sequence\",\"raw_payload_hash\",\"collector_session_id\",\"receipt_json\"]},\"db_b\":{\"d1_revisions_count\":0,\"event_kind_counts\":[[\"SEQUENCE\",179781],[\"RECONNECT\",10],[\"SESSION_START\",1]],\"exists\":true,\"finalized_d1_count\":0,\"latest_sequence_events\":[{\"event_kind\":\"SEQUENCE\",\"event_seq\":179792,\"received_at\":\"2026-09-08T05:23:44.017776+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2562},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179791,\"received_at\":\"2026-09-08T05:23:43.862198+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2561},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179790,\"received_at\":\"2026-09-08T05:23:43.702914+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2560},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179789,\"received_at\":\"2026-09-08T05:23:43.541151+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2559},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179788,\"received_at\":\"2026-09-08T05:23:43.382833+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2558},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179787,\"received_at\":\"2026-09-08T05:23:43.374827+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2557},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179786,\"received_at\":\"2026-09-08T05:23:43.365183+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2556},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179785,\"received_at\":\"2026-09-08T05:23:43.356202+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2555},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179784,\"received_at\":\"2026-09-08T05:23:43.337857+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2554},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179783,\"received_at\":\"2026-09-08T05:23:43.179159+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2553},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179782,\"received_at\":\"2026-09-08T05:23:43.025806+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2552},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179781,\"received_at\":\"2026-09-08T05:23:42.863932+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2551},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179780,\"received_at\":\"2026-09-08T05:23:42.715680+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2550},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179779,\"received_at\":\"2026-09-08T05:23:42.709841+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2549},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179778,\"received_at\":\"2026-09-08T05:23:42.702802+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2548},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179777,\"received_at\":\"2026-09-08T05:23:42.695983+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2547},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179776,\"received_at\":\"2026-09-08T05:23:42.682500+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2546},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179775,\"received_at\":\"2026-09-08T05:23:33.495587+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2545},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179774,\"received_at\":\"2026-09-08T05:23:33.324435+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2544},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179773,\"received_at\":\"2026-09-08T05:23:33.170904+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2543},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179772,\"received_at\":\"2026-09-08T05:23:33.161405+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2542},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179771,\"received_at\":\"2026-09-08T05:23:33.148039+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2541},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179770,\"received_at\":\"2026-09-08T05:23:33.102924+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2540},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179769,\"received_at\":\"2026-09-08T05:23:32.054424+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2539},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179768,\"received_at\":\"2026-09-08T05:23:32.019933+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2538},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179767,\"received_at\":\"2026-09-08T05:23:30.952907+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2537},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179766,\"received_at\":\"2026-09-08T05:23:30.911940+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2536},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179765,\"received_at\":\"2026-09-08T05:23:28.052780+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2535},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179764,\"received_at\":\"2026-09-08T05:23:25.504960+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2534},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179763,\"received_at\":\"2026-09-08T05:23:25.491855+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2533}],\"latest_sessions\":[{\"events\":2562,\"first_received_at\":\"2026-09-08T05:01:09.298101+00:00\",\"last_received_at\":\"2026-09-08T05:23:44.017776+00:00\",\"max_source_sequence\":2562,\"min_source_sequence\":1,\"session_hash\":\"253ffe9a26a5\"},{\"events\":2475,\"first_received_at\":\"2026-09-08T04:54:28.087186+00:00\",\"last_received_at\":\"2026-09-08T05:01:08.350487+00:00\",\"max_source_sequence\":2475,\"min_source_sequence\":1,\"session_hash\":\"2b3998047f77\"},{\"events\":988,\"first_received_at\":\"2026-09-08T04:43:46.442548+00:00\",\"last_received_at\":\"2026-09-08T04:54:27.005150+00:00\",\"max_source_sequence\":988,\"min_source_sequence\":1,\"session_hash\":\"b501ae120c1f\"},{\"events\":589,\"first_received_at\":\"2026-09-08T04:42:44.434580+00:00\",\"last_received_at\":\"2026-09-08T04:43:45.521859+00:00\",\"max_source_sequence\":589,\"min_source_sequence\":1,\"session_hash\":\"10709eb54a7b\"},{\"events\":3620,\"first_received_at\":\"2026-09-08T03:52:02.239825+00:00\",\"last_received_at\":\"2026-09-08T04:42:43.449781+00:00\",\"max_source_sequence\":3620,\"min_source_sequence\":1,\"session_hash\":\"210de3583c8e\"},{\"events\":72124,\"first_received_at\":\"2026-09-07T15:04:59.188298+00:00\",\"last_received_at\":\"2026-09-08T03:52:00.952767+00:00\",\"max_source_sequence\":72124,\"min_source_sequence\":1,\"session_hash\":\"69bb22240441\"},{\"events\":1688,\"first_received_at\":\"2026-09-07T14:46:51.906734+00:00\",\"last_received_at\":\"2026-09-07T15:04:47.491753+00:00\",\"max_source_sequence\":1688,\"min_source_sequence\":1,\"session_hash\":\"d1d81c259d86\"},{\"events\":10295,\"first_received_at\":\"2026-09-07T13:30:08.973280+00:00\",\"last_received_at\":\"2026-09-07T14:46:39.866418+00:00\",\"max_source_sequence\":10295,\"min_source_sequence\":1,\"session_hash\":\"a2db9de2102e\"},{\"events\":14430,\"first_received_at\":\"2026-09-07T09:50:41.075840+00:00\",\"last_received_at\":\"2026-09-07T13:30:03.165612+00:00\",\"max_source_sequence\":14430,\"min_source_sequence\":1,\"session_hash\":\"8da25a454977\"},{\"events\":71006,\"first_received_at\":\"2026-09-06T18:48:31.906387+00:00\",\"last_received_at\":\"2026-09-07T09:50:13.818640+00:00\",\"max_source_sequence\":71006,\"min_source_sequence\":1,\"session_hash\":\"184e0e63b1c9\"}],\"latest_source_receipts\":[{\"receipt_id\":\"02e5367b2692cde4f9bff73096271e6cc0598bf27413d651026f6318c8338d72\",\"received_at\":\"2026-09-08T03:52:00.952767+00:00\",\"source_sequence\":72124},{\"receipt_id\":\"568fff4620581353cea781b3cef61de359e02f12475ac54c7460e85c4baa8625\",\"received_at\":\"2026-09-08T03:52:00.943741+00:00\",\"source_sequence\":72123},{\"receipt_id\":\"3631962ca9ee6b49c4870d42ec814ad6c1c3f0ff65f78a445abecc54bcbe8567\",\"received_at\":\"2026-09-08T03:52:00.931321+00:00\",\"source_sequence\":72122},{\"receipt_id\":\"e9e23d4774b5bdc4801a3334511f50fbb5b25e117b01f8e913a98be3d98badc0\",\"received_at\":\"2026-09-08T03:52:00.790541+00:00\",\"source_sequence\":72121},{\"receipt_id\":\"59ad6dd601a0d1605e6ea3962d9bfb348cdbc67cc8666b473eb8d58b1b8c077c\",\"received_at\":\"2026-09-08T03:52:00.656138+00:00\",\"source_sequence\":72120},{\"receipt_id\":\"634a7fdd47cddc7c664450d7921d4b83093d20a6f95f72f31f6dc004b219ff0f\",\"received_at\":\"2026-09-08T03:52:00.642417+00:00\",\"source_sequence\":72119},{\"receipt_id\":\"6943e44a184672570beaf3ac076e8051162086f16c4f836d78b040c6fdb811ce\",\"received_at\":\"2026-09-08T03:52:00.504076+00:00\",\"source_sequence\":72118},{\"receipt_id\":\"063b4e0abd1065a59d6b9fe62b9e4583f4caacbbb1e83414875386170b8ba7fe\",\"received_at\":\"2026-09-08T03:52:00.494856+00:00\",\"source_sequence\":72117},{\"receipt_id\":\"0f018a85552ad881ab665baa7ddc7a0e956b6d0d66ec43ecfeee44b60e766f4c\",\"received_at\":\"2026-09-08T03:52:00.481147+00:00\",\"source_sequence\":72116},{\"receipt_id\":\"6b23823a92031d3850483879578a8528a45b5f6002b26381659904d813f5b8d2\",\"received_at\":\"2026-09-08T03:52:00.333870+00:00\",\"source_sequence\":72115},{\"receipt_id\":\"1985ce2996de079095414ce9a48bfa3fcfffe7ddcbcdcff1857e03aa0c770d7d\",\"received_at\":\"2026-09-08T03:52:00.194864+00:00\",\"source_sequence\":72114},{\"receipt_id\":\"a4ed84172daff21758b2402a6cc03ab51ddc9474ba23d7a4036f7c6c4a3af152\",\"received_at\":\"2026-09-08T03:52:00.180891+00:00\",\"source_sequence\":72113},{\"receipt_id\":\"8646a6eea56850890817a5ff2056f9639f830a876e460c3898cfebbbd881a670\",\"received_at\":\"2026-09-08T03:52:00.041606+00:00\",\"source_sequence\":72112},{\"receipt_id\":\"70ebbdab1c237840ea3cb6b543fab899e49db3b3a0ab65455e5dcb85b7844923\",\"received_at\":\"2026-09-08T03:52:00.023714+00:00\",\"source_sequence\":72111},{\"receipt_id\":\"7cb2c79c0940deed850deb26500cb3c04331baa4eee8e302a1ca9a3e1cd3023c\",\"received_at\":\"2026-09-08T03:51:59.882828+00:00\",\"source_sequence\":72110}],\"path\":\"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/bitfinex_source.db\",\"quick_check\":\"ok\",\"sequence_events\":{\"count\":179792,\"max_event_seq\":179792,\"max_received_at\":\"2026-09-08T05:23:44.017776+00:00\",\"max_source_sequence\":72124},\"source_receipts_columns\":[\"receipt_id\",\"venue_id\",\"source_kind\",\"received_at\",\"source_sequence\",\"raw_payload_hash\",\"collector_session_id\",\"receipt_json\"]},\"decision\":\"HOLD_COLLECTOR_INTAKE_STALLED_WITH_PROCESS_AND_TCP_ALIVE\",\"endpoint_probe\":{\"api-pub.bitfinex.com\":{\"dns_rc\":0,\"resolved\":true},\"api.bitfinex.com\":{\"dns_rc\":0,\"resolved\":true},\"platform_status\":{\"body\":\"[1]\",\"rc\":0,\"stderr\":\"\"}},\"epoch_exists\":true,\"holds\":[\"COLLECTOR_INTAKE_STILL_STALLED\"],\"hostname\":\"core-apps\",\"last_event_age_sec\":10.6,\"logs\":{\"keyword_counts\":{\"CLOSED\":5,\"CONNECTED\":0,\"DISCONNECT\":0,\"ERROR\":5,\"EXCEPTION\":5,\"FINAL\":0,\"GAP\":0,\"HEARTBEAT\":0,\"RECEIVED\":5,\"RECONNECT\":0,\"SOURCE\":0,\"TRACEBACK\":5,\"WEBSOCKET\":15},\"last_lines\":[\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"Traceback (most recent call last):\",\"File \\\"\\\", line 203, in _run_module_as_main\",\"File \\\"\\\", line 88, in _run_code\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1246, in \",\"raise SystemExit(main())\",\"~~~~^^\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1175, in main\",\"for raw in ws:\",\"^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"yield self.recv()\",\"~~~~~~~~~^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"raise self.protocol.close_exc from self.recv_exc\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"Traceback (most recent call last):\",\"File \\\"\\\", line 203, in _run_module_as_main\",\"File \\\"\\\", line 88, in _run_code\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1246, in \",\"raise SystemExit(main())\",\"~~~~^^\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1175, in main\",\"for raw in ws:\",\"^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"yield self.recv()\",\"~~~~~~~~~^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"raise self.protocol.close_exc from self.recv_exc\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"Traceback (most recent call last):\",\"File \\\"\\\", line 203, in _run_module_as_main\",\"File \\\"\\\", line 88, in _run_code\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1246, in \",\"raise SystemExit(main())\",\"~~~~^^\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1175, in main\",\"for raw in ws:\",\"^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"yield self.recv()\",\"~~~~~~~~~^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"raise self.protocol.close_exc from self.recv_exc\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\"],\"last_relevant\":[\"Traceback (most recent call last):\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"Traceback (most recent call last):\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"Traceback (most recent call last):\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"Traceback (most recent call last):\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"Traceback (most recent call last):\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\"],\"line_count\":81,\"rc\":0},\"mutation_scope\":\"NONE_READ_ONLY\",\"sequence_window\":{\"a\":72124,\"advanced_from_034\":false,\"b\":72124,\"baseline_034\":72124,\"grew\":false,\"vs_034\":72124},\"socket\":{\"established_total\":1,\"pid\":210213,\"remote_port_counts\":{\"443\":1},\"states\":{\"01\":1,\"0A\":1}},\"stack_exists\":true,\"stats\":{\"line\":\"demonov-shadow-8020-collector-1 43.29% 26.51MiB / 9.719GiB 315kB / 126kB\",\"rc\":0,\"stderr\":\"\"},\"top\":{\"lines\":[\"PID PPID COMMAND\",\"210213 210191 python\"],\"rc\":0,\"stderr\":\"\"}},\"state\":\"COMPLETE\"}\nSIGNALBOT035_HOLDS=[\"COLLECTOR_INTAKE_STILL_STALLED\"]\nSIGNALBOT035_DECISION=HOLD_COLLECTOR_INTAKE_STALLED_WITH_PROCESS_AND_TCP_ALIVE\nSIGNALBOT035_MUTATION_SCOPE=NO_RUNTIME_NO_GIT_NO_EXCHANGE_MUTATION\n" +} diff --git a/runtime/history/SIGNALBOT-260908-INTAKE-RCA-035.txt b/runtime/history/SIGNALBOT-260908-INTAKE-RCA-035.txt new file mode 100644 index 00000000..6eff3b04 --- /dev/null +++ b/runtime/history/SIGNALBOT-260908-INTAKE-RCA-035.txt @@ -0,0 +1,40 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SIGNALBOT-260908-INTAKE-RCA-035 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=signalbot-cr0116-intake-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=01c43885d0820853cc620910f562146315ad34c12ff24ffceefe7ef0181b8778 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=9bc13c343cdda506b9e7bbb4a38191d9f1ee1e4d31beb0916fa1cda673e1e097 +SANITIZED_OUTPUT_SHA256=9bc13c343cdda506b9e7bbb4a38191d9f1ee1e4d31beb0916fa1cda673e1e097 +OUTPUT_BEGIN +ERROR_REGISTER_CHECK=OK +REFERENCE_CHECK=OK +AUTHORITY_CHECK_SCOPE=READABILITY_ONLY_INCIDENTS_NOT_CLOSED +SIGNALBOT035_VM170={"name":"core-apps","node":"pve01","status":"running","type":"qemu"} +SIGNALBOT035_VM170_SNAPSHOT={"report":{"collector":{"cmd":["python","-m","demonov_forward_map.bitfinex_trade_collector"],"entrypoint":null,"health":null,"id_short":"a546a70c3e8c","image":"8020-demonov-shadow:e32760c69a4311728db9066ee8bf2bf66b1e5a70","name":"demonov-shadow-8020-collector-1","pid":210213,"restart_count":13,"running":true,"started_at":"2026-09-08T05:01:08.70385245Z","status":"running"},"collector_status":{"age_sec":8.9,"exists":true,"mtime_epoch":1788845024.0247085,"path":"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/collector_status.json","values":{"fail_closed_reason":"source reconnect detected","gap_detected":false,"post_finality_revision":false,"prospective_fail_closed_reason":"","prospective_gap_detected":false,"prospective_reconnect_detected":false,"reconnect_detected":true,"updated_at":"2026-09-08T05:23:44.028140+00:00"}},"db_a":{"d1_revisions_count":0,"event_kind_counts":[["SEQUENCE",179729],["RECONNECT",10],["SESSION_START",1]],"exists":true,"finalized_d1_count":0,"latest_sequence_events":[{"event_kind":"SEQUENCE","event_seq":179740,"received_at":"2026-09-08T05:23:20.918750+00:00","session_hash":"253ffe9a26a5","source_sequence":2510},{"event_kind":"SEQUENCE","event_seq":179739,"received_at":"2026-09-08T05:23:20.891856+00:00","session_hash":"253ffe9a26a5","source_sequence":2509},{"event_kind":"SEQUENCE","event_seq":179738,"received_at":"2026-09-08T05:23:20.552139+00:00","session_hash":"253ffe9a26a5","source_sequence":2508},{"event_kind":"SEQUENCE","event_seq":179737,"received_at":"2026-09-08T05:23:20.534792+00:00","session_hash":"253ffe9a26a5","source_sequence":2507},{"event_kind":"SEQUENCE","event_seq":179736,"received_at":"2026-09-08T05:23:20.329909+00:00","session_hash":"253ffe9a26a5","source_sequence":2506},{"event_kind":"SEQUENCE","event_seq":179735,"received_at":"2026-09-08T05:23:20.177708+00:00","session_hash":"253ffe9a26a5","source_sequence":2505},{"event_kind":"SEQUENCE","event_seq":179734,"received_at":"2026-09-08T05:23:20.170743+00:00","session_hash":"253ffe9a26a5","source_sequence":2504},{"event_kind":"SEQUENCE","event_seq":179733,"received_at":"2026-09-08T05:23:20.163522+00:00","session_hash":"253ffe9a26a5","source_sequence":2503},{"event_kind":"SEQUENCE","event_seq":179732,"received_at":"2026-09-08T05:23:20.004001+00:00","session_hash":"253ffe9a26a5","source_sequence":2502},{"event_kind":"SEQUENCE","event_seq":179731,"received_at":"2026-09-08T05:23:19.850034+00:00","session_hash":"253ffe9a26a5","source_sequence":2501},{"event_kind":"SEQUENCE","event_seq":179730,"received_at":"2026-09-08T05:23:19.841410+00:00","session_hash":"253ffe9a26a5","source_sequence":2500},{"event_kind":"SEQUENCE","event_seq":179729,"received_at":"2026-09-08T05:23:19.827700+00:00","session_hash":"253ffe9a26a5","source_sequence":2499},{"event_kind":"SEQUENCE","event_seq":179728,"received_at":"2026-09-08T05:23:19.671403+00:00","session_hash":"253ffe9a26a5","source_sequence":2498},{"event_kind":"SEQUENCE","event_seq":179727,"received_at":"2026-09-08T05:23:19.518174+00:00","session_hash":"253ffe9a26a5","source_sequence":2497},{"event_kind":"SEQUENCE","event_seq":179726,"received_at":"2026-09-08T05:23:19.357283+00:00","session_hash":"253ffe9a26a5","source_sequence":2496},{"event_kind":"SEQUENCE","event_seq":179725,"received_at":"2026-09-08T05:23:19.200903+00:00","session_hash":"253ffe9a26a5","source_sequence":2495},{"event_kind":"SEQUENCE","event_seq":179724,"received_at":"2026-09-08T05:23:19.193716+00:00","session_hash":"253ffe9a26a5","source_sequence":2494},{"event_kind":"SEQUENCE","event_seq":179723,"received_at":"2026-09-08T05:23:19.185835+00:00","session_hash":"253ffe9a26a5","source_sequence":2493},{"event_kind":"SEQUENCE","event_seq":179722,"received_at":"2026-09-08T05:23:19.177983+00:00","session_hash":"253ffe9a26a5","source_sequence":2492},{"event_kind":"SEQUENCE","event_seq":179721,"received_at":"2026-09-08T05:23:19.164437+00:00","session_hash":"253ffe9a26a5","source_sequence":2491},{"event_kind":"SEQUENCE","event_seq":179720,"received_at":"2026-09-08T05:23:19.006208+00:00","session_hash":"253ffe9a26a5","source_sequence":2490},{"event_kind":"SEQUENCE","event_seq":179719,"received_at":"2026-09-08T05:23:18.848474+00:00","session_hash":"253ffe9a26a5","source_sequence":2489},{"event_kind":"SEQUENCE","event_seq":179718,"received_at":"2026-09-08T05:23:18.697530+00:00","session_hash":"253ffe9a26a5","source_sequence":2488},{"event_kind":"SEQUENCE","event_seq":179717,"received_at":"2026-09-08T05:23:18.541070+00:00","session_hash":"253ffe9a26a5","source_sequence":2487},{"event_kind":"SEQUENCE","event_seq":179716,"received_at":"2026-09-08T05:23:18.532005+00:00","session_hash":"253ffe9a26a5","source_sequence":2486},{"event_kind":"SEQUENCE","event_seq":179715,"received_at":"2026-09-08T05:23:18.522984+00:00","session_hash":"253ffe9a26a5","source_sequence":2485},{"event_kind":"SEQUENCE","event_seq":179714,"received_at":"2026-09-08T05:23:18.515020+00:00","session_hash":"253ffe9a26a5","source_sequence":2484},{"event_kind":"SEQUENCE","event_seq":179713,"received_at":"2026-09-08T05:23:18.504619+00:00","session_hash":"253ffe9a26a5","source_sequence":2483},{"event_kind":"SEQUENCE","event_seq":179712,"received_at":"2026-09-08T05:23:18.349318+00:00","session_hash":"253ffe9a26a5","source_sequence":2482},{"event_kind":"SEQUENCE","event_seq":179711,"received_at":"2026-09-08T05:23:18.189830+00:00","session_hash":"253ffe9a26a5","source_sequence":2481}],"latest_sessions":[{"events":2510,"first_received_at":"2026-09-08T05:01:09.298101+00:00","last_received_at":"2026-09-08T05:23:20.918750+00:00","max_source_sequence":2510,"min_source_sequence":1,"session_hash":"253ffe9a26a5"},{"events":2475,"first_received_at":"2026-09-08T04:54:28.087186+00:00","last_received_at":"2026-09-08T05:01:08.350487+00:00","max_source_sequence":2475,"min_source_sequence":1,"session_hash":"2b3998047f77"},{"events":988,"first_received_at":"2026-09-08T04:43:46.442548+00:00","last_received_at":"2026-09-08T04:54:27.005150+00:00","max_source_sequence":988,"min_source_sequence":1,"session_hash":"b501ae120c1f"},{"events":589,"first_received_at":"2026-09-08T04:42:44.434580+00:00","last_received_at":"2026-09-08T04:43:45.521859+00:00","max_source_sequence":589,"min_source_sequence":1,"session_hash":"10709eb54a7b"},{"events":3620,"first_received_at":"2026-09-08T03:52:02.239825+00:00","last_received_at":"2026-09-08T04:42:43.449781+00:00","max_source_sequence":3620,"min_source_sequence":1,"session_hash":"210de3583c8e"},{"events":72124,"first_received_at":"2026-09-07T15:04:59.188298+00:00","last_received_at":"2026-09-08T03:52:00.952767+00:00","max_source_sequence":72124,"min_source_sequence":1,"session_hash":"69bb22240441"},{"events":1688,"first_received_at":"2026-09-07T14:46:51.906734+00:00","last_received_at":"2026-09-07T15:04:47.491753+00:00","max_source_sequence":1688,"min_source_sequence":1,"session_hash":"d1d81c259d86"},{"events":10295,"first_received_at":"2026-09-07T13:30:08.973280+00:00","last_received_at":"2026-09-07T14:46:39.866418+00:00","max_source_sequence":10295,"min_source_sequence":1,"session_hash":"a2db9de2102e"},{"events":14430,"first_received_at":"2026-09-07T09:50:41.075840+00:00","last_received_at":"2026-09-07T13:30:03.165612+00:00","max_source_sequence":14430,"min_source_sequence":1,"session_hash":"8da25a454977"},{"events":71006,"first_received_at":"2026-09-06T18:48:31.906387+00:00","last_received_at":"2026-09-07T09:50:13.818640+00:00","max_source_sequence":71006,"min_source_sequence":1,"session_hash":"184e0e63b1c9"}],"latest_source_receipts":[{"receipt_id":"02e5367b2692cde4f9bff73096271e6cc0598bf27413d651026f6318c8338d72","received_at":"2026-09-08T03:52:00.952767+00:00","source_sequence":72124},{"receipt_id":"568fff4620581353cea781b3cef61de359e02f12475ac54c7460e85c4baa8625","received_at":"2026-09-08T03:52:00.943741+00:00","source_sequence":72123},{"receipt_id":"3631962ca9ee6b49c4870d42ec814ad6c1c3f0ff65f78a445abecc54bcbe8567","received_at":"2026-09-08T03:52:00.931321+00:00","source_sequence":72122},{"receipt_id":"e9e23d4774b5bdc4801a3334511f50fbb5b25e117b01f8e913a98be3d98badc0","received_at":"2026-09-08T03:52:00.790541+00:00","source_sequence":72121},{"receipt_id":"59ad6dd601a0d1605e6ea3962d9bfb348cdbc67cc8666b473eb8d58b1b8c077c","received_at":"2026-09-08T03:52:00.656138+00:00","source_sequence":72120},{"receipt_id":"634a7fdd47cddc7c664450d7921d4b83093d20a6f95f72f31f6dc004b219ff0f","received_at":"2026-09-08T03:52:00.642417+00:00","source_sequence":72119},{"receipt_id":"6943e44a184672570beaf3ac076e8051162086f16c4f836d78b040c6fdb811ce","received_at":"2026-09-08T03:52:00.504076+00:00","source_sequence":72118},{"receipt_id":"063b4e0abd1065a59d6b9fe62b9e4583f4caacbbb1e83414875386170b8ba7fe","received_at":"2026-09-08T03:52:00.494856+00:00","source_sequence":72117},{"receipt_id":"0f018a85552ad881ab665baa7ddc7a0e956b6d0d66ec43ecfeee44b60e766f4c","received_at":"2026-09-08T03:52:00.481147+00:00","source_sequence":72116},{"receipt_id":"6b23823a92031d3850483879578a8528a45b5f6002b26381659904d813f5b8d2","received_at":"2026-09-08T03:52:00.333870+00:00","source_sequence":72115},{"receipt_id":"1985ce2996de079095414ce9a48bfa3fcfffe7ddcbcdcff1857e03aa0c770d7d","received_at":"2026-09-08T03:52:00.194864+00:00","source_sequence":72114},{"receipt_id":"a4ed84172daff21758b2402a6cc03ab51ddc9474ba23d7a4036f7c6c4a3af152","received_at":"2026-09-08T03:52:00.180891+00:00","source_sequence":72113},{"receipt_id":"8646a6eea56850890817a5ff2056f9639f830a876e460c3898cfebbbd881a670","received_at":"2026-09-08T03:52:00.041606+00:00","source_sequence":72112},{"receipt_id":"70ebbdab1c237840ea3cb6b543fab899e49db3b3a0ab65455e5dcb85b7844923","received_at":"2026-09-08T03:52:00.023714+00:00","source_sequence":72111},{"receipt_id":"7cb2c79c0940deed850deb26500cb3c04331baa4eee8e302a1ca9a3e1cd3023c","received_at":"2026-09-08T03:51:59.882828+00:00","source_sequence":72110}],"path":"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/bitfinex_source.db","quick_check":"ok","sequence_events":{"count":179740,"max_event_seq":179740,"max_received_at":"2026-09-08T05:23:20.918750+00:00","max_source_sequence":72124},"source_receipts_columns":["receipt_id","venue_id","source_kind","received_at","source_sequence","raw_payload_hash","collector_session_id","receipt_json"]},"db_b":{"d1_revisions_count":0,"event_kind_counts":[["SEQUENCE",179781],["RECONNECT",10],["SESSION_START",1]],"exists":true,"finalized_d1_count":0,"latest_sequence_events":[{"event_kind":"SEQUENCE","event_seq":179792,"received_at":"2026-09-08T05:23:44.017776+00:00","session_hash":"253ffe9a26a5","source_sequence":2562},{"event_kind":"SEQUENCE","event_seq":179791,"received_at":"2026-09-08T05:23:43.862198+00:00","session_hash":"253ffe9a26a5","source_sequence":2561},{"event_kind":"SEQUENCE","event_seq":179790,"received_at":"2026-09-08T05:23:43.702914+00:00","session_hash":"253ffe9a26a5","source_sequence":2560},{"event_kind":"SEQUENCE","event_seq":179789,"received_at":"2026-09-08T05:23:43.541151+00:00","session_hash":"253ffe9a26a5","source_sequence":2559},{"event_kind":"SEQUENCE","event_seq":179788,"received_at":"2026-09-08T05:23:43.382833+00:00","session_hash":"253ffe9a26a5","source_sequence":2558},{"event_kind":"SEQUENCE","event_seq":179787,"received_at":"2026-09-08T05:23:43.374827+00:00","session_hash":"253ffe9a26a5","source_sequence":2557},{"event_kind":"SEQUENCE","event_seq":179786,"received_at":"2026-09-08T05:23:43.365183+00:00","session_hash":"253ffe9a26a5","source_sequence":2556},{"event_kind":"SEQUENCE","event_seq":179785,"received_at":"2026-09-08T05:23:43.356202+00:00","session_hash":"253ffe9a26a5","source_sequence":2555},{"event_kind":"SEQUENCE","event_seq":179784,"received_at":"2026-09-08T05:23:43.337857+00:00","session_hash":"253ffe9a26a5","source_sequence":2554},{"event_kind":"SEQUENCE","event_seq":179783,"received_at":"2026-09-08T05:23:43.179159+00:00","session_hash":"253ffe9a26a5","source_sequence":2553},{"event_kind":"SEQUENCE","event_seq":179782,"received_at":"2026-09-08T05:23:43.025806+00:00","session_hash":"253ffe9a26a5","source_sequence":2552},{"event_kind":"SEQUENCE","event_seq":179781,"received_at":"2026-09-08T05:23:42.863932+00:00","session_hash":"253ffe9a26a5","source_sequence":2551},{"event_kind":"SEQUENCE","event_seq":179780,"received_at":"2026-09-08T05:23:42.715680+00:00","session_hash":"253ffe9a26a5","source_sequence":2550},{"event_kind":"SEQUENCE","event_seq":179779,"received_at":"2026-09-08T05:23:42.709841+00:00","session_hash":"253ffe9a26a5","source_sequence":2549},{"event_kind":"SEQUENCE","event_seq":179778,"received_at":"2026-09-08T05:23:42.702802+00:00","session_hash":"253ffe9a26a5","source_sequence":2548},{"event_kind":"SEQUENCE","event_seq":179777,"received_at":"2026-09-08T05:23:42.695983+00:00","session_hash":"253ffe9a26a5","source_sequence":2547},{"event_kind":"SEQUENCE","event_seq":179776,"received_at":"2026-09-08T05:23:42.682500+00:00","session_hash":"253ffe9a26a5","source_sequence":2546},{"event_kind":"SEQUENCE","event_seq":179775,"received_at":"2026-09-08T05:23:33.495587+00:00","session_hash":"253ffe9a26a5","source_sequence":2545},{"event_kind":"SEQUENCE","event_seq":179774,"received_at":"2026-09-08T05:23:33.324435+00:00","session_hash":"253ffe9a26a5","source_sequence":2544},{"event_kind":"SEQUENCE","event_seq":179773,"received_at":"2026-09-08T05:23:33.170904+00:00","session_hash":"253ffe9a26a5","source_sequence":2543},{"event_kind":"SEQUENCE","event_seq":179772,"received_at":"2026-09-08T05:23:33.161405+00:00","session_hash":"253ffe9a26a5","source_sequence":2542},{"event_kind":"SEQUENCE","event_seq":179771,"received_at":"2026-09-08T05:23:33.148039+00:00","session_hash":"253ffe9a26a5","source_sequence":2541},{"event_kind":"SEQUENCE","event_seq":179770,"received_at":"2026-09-08T05:23:33.102924+00:00","session_hash":"253ffe9a26a5","source_sequence":2540},{"event_kind":"SEQUENCE","event_seq":179769,"received_at":"2026-09-08T05:23:32.054424+00:00","session_hash":"253ffe9a26a5","source_sequence":2539},{"event_kind":"SEQUENCE","event_seq":179768,"received_at":"2026-09-08T05:23:32.019933+00:00","session_hash":"253ffe9a26a5","source_sequence":2538},{"event_kind":"SEQUENCE","event_seq":179767,"received_at":"2026-09-08T05:23:30.952907+00:00","session_hash":"253ffe9a26a5","source_sequence":2537},{"event_kind":"SEQUENCE","event_seq":179766,"received_at":"2026-09-08T05:23:30.911940+00:00","session_hash":"253ffe9a26a5","source_sequence":2536},{"event_kind":"SEQUENCE","event_seq":179765,"received_at":"2026-09-08T05:23:28.052780+00:00","session_hash":"253ffe9a26a5","source_sequence":2535},{"event_kind":"SEQUENCE","event_seq":179764,"received_at":"2026-09-08T05:23:25.504960+00:00","session_hash":"253ffe9a26a5","source_sequence":2534},{"event_kind":"SEQUENCE","event_seq":179763,"received_at":"2026-09-08T05:23:25.491855+00:00","session_hash":"253ffe9a26a5","source_sequence":2533}],"latest_sessions":[{"events":2562,"first_received_at":"2026-09-08T05:01:09.298101+00:00","last_received_at":"2026-09-08T05:23:44.017776+00:00","max_source_sequence":2562,"min_source_sequence":1,"session_hash":"253ffe9a26a5"},{"events":2475,"first_received_at":"2026-09-08T04:54:28.087186+00:00","last_received_at":"2026-09-08T05:01:08.350487+00:00","max_source_sequence":2475,"min_source_sequence":1,"session_hash":"2b3998047f77"},{"events":988,"first_received_at":"2026-09-08T04:43:46.442548+00:00","last_received_at":"2026-09-08T04:54:27.005150+00:00","max_source_sequence":988,"min_source_sequence":1,"session_hash":"b501ae120c1f"},{"events":589,"first_received_at":"2026-09-08T04:42:44.434580+00:00","last_received_at":"2026-09-08T04:43:45.521859+00:00","max_source_sequence":589,"min_source_sequence":1,"session_hash":"10709eb54a7b"},{"events":3620,"first_received_at":"2026-09-08T03:52:02.239825+00:00","last_received_at":"2026-09-08T04:42:43.449781+00:00","max_source_sequence":3620,"min_source_sequence":1,"session_hash":"210de3583c8e"},{"events":72124,"first_received_at":"2026-09-07T15:04:59.188298+00:00","last_received_at":"2026-09-08T03:52:00.952767+00:00","max_source_sequence":72124,"min_source_sequence":1,"session_hash":"69bb22240441"},{"events":1688,"first_received_at":"2026-09-07T14:46:51.906734+00:00","last_received_at":"2026-09-07T15:04:47.491753+00:00","max_source_sequence":1688,"min_source_sequence":1,"session_hash":"d1d81c259d86"},{"events":10295,"first_received_at":"2026-09-07T13:30:08.973280+00:00","last_received_at":"2026-09-07T14:46:39.866418+00:00","max_source_sequence":10295,"min_source_sequence":1,"session_hash":"a2db9de2102e"},{"events":14430,"first_received_at":"2026-09-07T09:50:41.075840+00:00","last_received_at":"2026-09-07T13:30:03.165612+00:00","max_source_sequence":14430,"min_source_sequence":1,"session_hash":"8da25a454977"},{"events":71006,"first_received_at":"2026-09-06T18:48:31.906387+00:00","last_received_at":"2026-09-07T09:50:13.818640+00:00","max_source_sequence":71006,"min_source_sequence":1,"session_hash":"184e0e63b1c9"}],"latest_source_receipts":[{"receipt_id":"02e5367b2692cde4f9bff73096271e6cc0598bf27413d651026f6318c8338d72","received_at":"2026-09-08T03:52:00.952767+00:00","source_sequence":72124},{"receipt_id":"568fff4620581353cea781b3cef61de359e02f12475ac54c7460e85c4baa8625","received_at":"2026-09-08T03:52:00.943741+00:00","source_sequence":72123},{"receipt_id":"3631962ca9ee6b49c4870d42ec814ad6c1c3f0ff65f78a445abecc54bcbe8567","received_at":"2026-09-08T03:52:00.931321+00:00","source_sequence":72122},{"receipt_id":"e9e23d4774b5bdc4801a3334511f50fbb5b25e117b01f8e913a98be3d98badc0","received_at":"2026-09-08T03:52:00.790541+00:00","source_sequence":72121},{"receipt_id":"59ad6dd601a0d1605e6ea3962d9bfb348cdbc67cc8666b473eb8d58b1b8c077c","received_at":"2026-09-08T03:52:00.656138+00:00","source_sequence":72120},{"receipt_id":"634a7fdd47cddc7c664450d7921d4b83093d20a6f95f72f31f6dc004b219ff0f","received_at":"2026-09-08T03:52:00.642417+00:00","source_sequence":72119},{"receipt_id":"6943e44a184672570beaf3ac076e8051162086f16c4f836d78b040c6fdb811ce","received_at":"2026-09-08T03:52:00.504076+00:00","source_sequence":72118},{"receipt_id":"063b4e0abd1065a59d6b9fe62b9e4583f4caacbbb1e83414875386170b8ba7fe","received_at":"2026-09-08T03:52:00.494856+00:00","source_sequence":72117},{"receipt_id":"0f018a85552ad881ab665baa7ddc7a0e956b6d0d66ec43ecfeee44b60e766f4c","received_at":"2026-09-08T03:52:00.481147+00:00","source_sequence":72116},{"receipt_id":"6b23823a92031d3850483879578a8528a45b5f6002b26381659904d813f5b8d2","received_at":"2026-09-08T03:52:00.333870+00:00","source_sequence":72115},{"receipt_id":"1985ce2996de079095414ce9a48bfa3fcfffe7ddcbcdcff1857e03aa0c770d7d","received_at":"2026-09-08T03:52:00.194864+00:00","source_sequence":72114},{"receipt_id":"a4ed84172daff21758b2402a6cc03ab51ddc9474ba23d7a4036f7c6c4a3af152","received_at":"2026-09-08T03:52:00.180891+00:00","source_sequence":72113},{"receipt_id":"8646a6eea56850890817a5ff2056f9639f830a876e460c3898cfebbbd881a670","received_at":"2026-09-08T03:52:00.041606+00:00","source_sequence":72112},{"receipt_id":"70ebbdab1c237840ea3cb6b543fab899e49db3b3a0ab65455e5dcb85b7844923","received_at":"2026-09-08T03:52:00.023714+00:00","source_sequence":72111},{"receipt_id":"7cb2c79c0940deed850deb26500cb3c04331baa4eee8e302a1ca9a3e1cd3023c","received_at":"2026-09-08T03:51:59.882828+00:00","source_sequence":72110}],"path":"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/bitfinex_source.db","quick_check":"ok","sequence_events":{"count":179792,"max_event_seq":179792,"max_received_at":"2026-09-08T05:23:44.017776+00:00","max_source_sequence":72124},"source_receipts_columns":["receipt_id","venue_id","source_kind","received_at","source_sequence","raw_payload_hash","collector_session_id","receipt_json"]},"decision":"HOLD_COLLECTOR_INTAKE_STALLED_WITH_PROCESS_AND_TCP_ALIVE","endpoint_probe":{"api-pub.bitfinex.com":{"dns_rc":0,"resolved":true},"api.bitfinex.com":{"dns_rc":0,"resolved":true},"platform_status":{"body":"[1]","rc":0,"stderr":""}},"epoch_exists":true,"holds":["COLLECTOR_INTAKE_STILL_STALLED"],"hostname":"core-apps","last_event_age_sec":10.6,"logs":{"keyword_counts":{"CLOSED":5,"CONNECTED":0,"DISCONNECT":0,"ERROR":5,"EXCEPTION":5,"FINAL":0,"GAP":0,"HEARTBEAT":0,"RECEIVED":5,"RECONNECT":0,"SOURCE":0,"TRACEBACK":5,"WEBSOCKET":15},"last_lines":["websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received",":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour","Traceback (most recent call last):","File \"\", line 203, in _run_module_as_main","File \"\", line 88, in _run_code","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1246, in ","raise SystemExit(main())","~~~~^^","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1175, in main","for raw in ws:","^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","yield self.recv()","~~~~~~~~~^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","raise self.protocol.close_exc from self.recv_exc","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received",":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour","Traceback (most recent call last):","File \"\", line 203, in _run_module_as_main","File \"\", line 88, in _run_code","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1246, in ","raise SystemExit(main())","~~~~^^","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1175, in main","for raw in ws:","^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","yield self.recv()","~~~~~~~~~^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","raise self.protocol.close_exc from self.recv_exc","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received",":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour","Traceback (most recent call last):","File \"\", line 203, in _run_module_as_main","File \"\", line 88, in _run_code","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1246, in ","raise SystemExit(main())","~~~~^^","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1175, in main","for raw in ws:","^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","yield self.recv()","~~~~~~~~~^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","raise self.protocol.close_exc from self.recv_exc","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received",":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour"],"last_relevant":["Traceback (most recent call last):","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received","Traceback (most recent call last):","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received","Traceback (most recent call last):","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received","Traceback (most recent call last):","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received","Traceback (most recent call last):","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received"],"line_count":81,"rc":0},"mutation_scope":"NONE_READ_ONLY","sequence_window":{"a":72124,"advanced_from_034":false,"b":72124,"baseline_034":72124,"grew":false,"vs_034":72124},"socket":{"established_total":1,"pid":210213,"remote_port_counts":{"443":1},"states":{"01":1,"0A":1}},"stack_exists":true,"stats":{"line":"demonov-shadow-8020-collector-1 43.29% 26.51MiB / 9.719GiB 315kB / 126kB","rc":0,"stderr":""},"top":{"lines":["PID PPID COMMAND","210213 210191 python"],"rc":0,"stderr":""}},"state":"COMPLETE"} +SIGNALBOT035_HOLDS=["COLLECTOR_INTAKE_STILL_STALLED"] +SIGNALBOT035_DECISION=HOLD_COLLECTOR_INTAKE_STALLED_WITH_PROCESS_AND_TCP_ALIVE +SIGNALBOT035_MUTATION_SCOPE=NO_RUNTIME_NO_GIT_NO_EXCHANGE_MUTATION + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 61f8b591..065036db 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,27 +1,27 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "NEWFI-260908-A-STAGINGDNS-APPLY4", - "status": "FAIL", - "rc": 1, + "command_id": "SIGNALBOT-260908-INTAKE-RCA-035", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "newfi-staging-cloudflare-ingress-candidate-proof-dns-apply", - "started_at_utc": "2026-09-08T05:22:49Z", - "finished_at_utc": "2026-09-08T05:22:59Z", + "mode": "read-only", + "component": "signalbot-cr0116-intake-rca", + "started_at_utc": "2026-09-08T05:23:21Z", + "finished_at_utc": "2026-09-08T05:23:55Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "6e253c0d6d882ebec1a77ebcb735719f51d55260a2498f0dd79bdf7b1a0b4861", + "command_sha256": "01c43885d0820853cc620910f562146315ad34c12ff24ffceefe7ef0181b8778", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, + "change_declared": false, "result_contract_valid": true, - "result_contract_status": "FAIL", + "result_contract_status": null, "result_contract_error": null, - "command_rc": 1, + "command_rc": 0, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "5a6777ef0dfd996de4e6b912aac38ff38d3a2f6895c69a30bc91d15884f738f5", - "sanitized_output_sha256": "5a6777ef0dfd996de4e6b912aac38ff38d3a2f6895c69a30bc91d15884f738f5", + "raw_evidence_sha256": "9bc13c343cdda506b9e7bbb4a38191d9f1ee1e4d31beb0916fa1cda673e1e097", + "sanitized_output_sha256": "9bc13c343cdda506b9e7bbb4a38191d9f1ee1e4d31beb0916fa1cda673e1e097", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "STAGINGDNS_APPLY4_BEGIN=true\nCOMMAND_ID=NEWFI-260908-A-STAGINGDNS-APPLY4\nCOMPONENT=newfi-staging-cloudflare-ingress-candidate-proof-dns-apply\nMODE=change\nERROR_REGISTER_CHECK=OK\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nAUTHORITY_CHECK_SCOPE=READABILITY_ONLY_INCIDENTS_NOT_CLOSED\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nDISCOVERY1_HISTORY_SHA256=edf7a289f77b6580795698fdc7d82355903d24c8df42d76bf84ec8a36117b873\nPRIOR_DNS_APPLY3_HISTORY_SHA256=916cc5e9c705b07c36b93decd55470bf12b24cb1d6d158b1bbde1b9750d6fc89\nPORTAL_PRODUCT_STATE_GATE=PASS\nPUBLIC_DNS_PRESTATE={\"1.1.1.1/A\":{\"err\":\"\",\"out\":\";; Got answer:\\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 61269\\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\\n\\n;; OPT PSEUDOSECTION:\\n; EDNS: version: 0, flags:; udp: 1232\\n;; AUTHORITY SECTION:\\ngram1.ru.\\t\\t1800\\tIN\\tSOA\\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800\",\"rc\":0},\"1.1.1.1/AAAA\":{\"err\":\"\",\"out\":\";; Got answer:\\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 42274\\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\\n\\n;; OPT PSEUDOSECTION:\\n; EDNS: version: 0, flags:; udp: 1232\\n;; AUTHORITY SECTION:\\ngram1.ru.\\t\\t1188\\tIN\\tSOA\\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800\",\"rc\":0},\"1.1.1.1/CNAME\":{\"err\":\"\",\"out\":\";; Got answer:\\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 1534\\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\\n\\n;; OPT PSEUDOSECTION:\\n; EDNS: version: 0, flags:; udp: 1232\\n;; AUTHORITY SECTION:\\ngram1.ru.\\t\\t1189\\tIN\\tSOA\\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800\",\"rc\":0},\"8.8.8.8/A\":{\"err\":\"\",\"out\":\";; Got answer:\\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 65407\\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\\n\\n;; OPT PSEUDOSECTION:\\n; EDNS: version: 0, flags:; udp: 1232\\n;; AUTHORITY SECTION:\\ngram1.ru.\\t\\t1800\\tIN\\tSOA\\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800\",\"rc\":0},\"8.8.8.8/AAAA\":{\"err\":\"\",\"out\":\";; Got answer:\\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 40214\\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\\n\\n;; OPT PSEUDOSECTION:\\n; EDNS: version: 0, flags:; udp: 1232\\n;; AUTHORITY SECTION:\\ngram1.ru.\\t\\t1396\\tIN\\tSOA\\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800\",\"rc\":0},\"8.8.8.8/CNAME\":{\"err\":\"\",\"out\":\";; Got answer:\\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 30626\\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\\n\\n;; OPT PSEUDOSECTION:\\n; EDNS: version: 0, flags:; udp: 512\\n;; AUTHORITY SECTION:\\ngram1.ru.\\t\\t1800\\tIN\\tSOA\\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800\",\"rc\":0}}\nCLOUDFLARE_TOKEN_DISCOVERY={\"candidate_count\":0,\"scanned_files\":1486}\nEDGE_VM_IDENTITY={\"name\":\"edge-vm\",\"node\":\"pve03\",\"vmid\":130}\nEDGE_CLOUDFLARE_TOKEN_SCAN={\"candidate_count\":3,\"rc\":0,\"source_paths\":[\"/root/cert38-narrow-migration-20260904T074428Z/credentials-38.before\",\"/opt/npmplus/secure/taftauto_cf_certbot.ini\",\"/opt/npmplus/secure/forum_cf_token.env\"]}\nEDGE_CLOUDFLARE_ZONE_ACCESS_VALID_COUNT=1\nCLOUDFLARE_TOKEN_SELECTED={\"source_paths\":[\"/root/cert38-narrow-migration-20260904T074428Z/credentials-38.before\"],\"valid_token_count\":1,\"zone_id_sha256\":\"5a88f12ebe16612b4e42f11ee57b4b95ca4a13c4abb8f0009f09babfbec927e7\"}\nEDGE_VM_IDENTITY={\"name\":\"edge-vm\",\"node\":\"pve03\",\"vmid\":130}\nDNS_APPLY_FAILURE=ApplyError:EDGE_ROUTE_CONFIG_NOT_FOUND\nFINAL_DECISION=FAIL_NEWFI_STAGING_DNS_NO_MUTATION\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"NEWFI-260908-A-STAGINGDNS-APPLY4\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"FAIL\",\"version\":1}\n" + "output": "ERROR_REGISTER_CHECK=OK\nREFERENCE_CHECK=OK\nAUTHORITY_CHECK_SCOPE=READABILITY_ONLY_INCIDENTS_NOT_CLOSED\nSIGNALBOT035_VM170={\"name\":\"core-apps\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"qemu\"}\nSIGNALBOT035_VM170_SNAPSHOT={\"report\":{\"collector\":{\"cmd\":[\"python\",\"-m\",\"demonov_forward_map.bitfinex_trade_collector\"],\"entrypoint\":null,\"health\":null,\"id_short\":\"a546a70c3e8c\",\"image\":\"8020-demonov-shadow:e32760c69a4311728db9066ee8bf2bf66b1e5a70\",\"name\":\"demonov-shadow-8020-collector-1\",\"pid\":210213,\"restart_count\":13,\"running\":true,\"started_at\":\"2026-09-08T05:01:08.70385245Z\",\"status\":\"running\"},\"collector_status\":{\"age_sec\":8.9,\"exists\":true,\"mtime_epoch\":1788845024.0247085,\"path\":\"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/collector_status.json\",\"values\":{\"fail_closed_reason\":\"source reconnect detected\",\"gap_detected\":false,\"post_finality_revision\":false,\"prospective_fail_closed_reason\":\"\",\"prospective_gap_detected\":false,\"prospective_reconnect_detected\":false,\"reconnect_detected\":true,\"updated_at\":\"2026-09-08T05:23:44.028140+00:00\"}},\"db_a\":{\"d1_revisions_count\":0,\"event_kind_counts\":[[\"SEQUENCE\",179729],[\"RECONNECT\",10],[\"SESSION_START\",1]],\"exists\":true,\"finalized_d1_count\":0,\"latest_sequence_events\":[{\"event_kind\":\"SEQUENCE\",\"event_seq\":179740,\"received_at\":\"2026-09-08T05:23:20.918750+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2510},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179739,\"received_at\":\"2026-09-08T05:23:20.891856+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2509},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179738,\"received_at\":\"2026-09-08T05:23:20.552139+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2508},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179737,\"received_at\":\"2026-09-08T05:23:20.534792+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2507},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179736,\"received_at\":\"2026-09-08T05:23:20.329909+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2506},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179735,\"received_at\":\"2026-09-08T05:23:20.177708+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2505},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179734,\"received_at\":\"2026-09-08T05:23:20.170743+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2504},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179733,\"received_at\":\"2026-09-08T05:23:20.163522+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2503},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179732,\"received_at\":\"2026-09-08T05:23:20.004001+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2502},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179731,\"received_at\":\"2026-09-08T05:23:19.850034+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2501},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179730,\"received_at\":\"2026-09-08T05:23:19.841410+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2500},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179729,\"received_at\":\"2026-09-08T05:23:19.827700+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2499},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179728,\"received_at\":\"2026-09-08T05:23:19.671403+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2498},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179727,\"received_at\":\"2026-09-08T05:23:19.518174+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2497},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179726,\"received_at\":\"2026-09-08T05:23:19.357283+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2496},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179725,\"received_at\":\"2026-09-08T05:23:19.200903+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2495},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179724,\"received_at\":\"2026-09-08T05:23:19.193716+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2494},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179723,\"received_at\":\"2026-09-08T05:23:19.185835+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2493},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179722,\"received_at\":\"2026-09-08T05:23:19.177983+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2492},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179721,\"received_at\":\"2026-09-08T05:23:19.164437+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2491},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179720,\"received_at\":\"2026-09-08T05:23:19.006208+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2490},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179719,\"received_at\":\"2026-09-08T05:23:18.848474+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2489},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179718,\"received_at\":\"2026-09-08T05:23:18.697530+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2488},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179717,\"received_at\":\"2026-09-08T05:23:18.541070+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2487},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179716,\"received_at\":\"2026-09-08T05:23:18.532005+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2486},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179715,\"received_at\":\"2026-09-08T05:23:18.522984+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2485},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179714,\"received_at\":\"2026-09-08T05:23:18.515020+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2484},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179713,\"received_at\":\"2026-09-08T05:23:18.504619+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2483},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179712,\"received_at\":\"2026-09-08T05:23:18.349318+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2482},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179711,\"received_at\":\"2026-09-08T05:23:18.189830+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2481}],\"latest_sessions\":[{\"events\":2510,\"first_received_at\":\"2026-09-08T05:01:09.298101+00:00\",\"last_received_at\":\"2026-09-08T05:23:20.918750+00:00\",\"max_source_sequence\":2510,\"min_source_sequence\":1,\"session_hash\":\"253ffe9a26a5\"},{\"events\":2475,\"first_received_at\":\"2026-09-08T04:54:28.087186+00:00\",\"last_received_at\":\"2026-09-08T05:01:08.350487+00:00\",\"max_source_sequence\":2475,\"min_source_sequence\":1,\"session_hash\":\"2b3998047f77\"},{\"events\":988,\"first_received_at\":\"2026-09-08T04:43:46.442548+00:00\",\"last_received_at\":\"2026-09-08T04:54:27.005150+00:00\",\"max_source_sequence\":988,\"min_source_sequence\":1,\"session_hash\":\"b501ae120c1f\"},{\"events\":589,\"first_received_at\":\"2026-09-08T04:42:44.434580+00:00\",\"last_received_at\":\"2026-09-08T04:43:45.521859+00:00\",\"max_source_sequence\":589,\"min_source_sequence\":1,\"session_hash\":\"10709eb54a7b\"},{\"events\":3620,\"first_received_at\":\"2026-09-08T03:52:02.239825+00:00\",\"last_received_at\":\"2026-09-08T04:42:43.449781+00:00\",\"max_source_sequence\":3620,\"min_source_sequence\":1,\"session_hash\":\"210de3583c8e\"},{\"events\":72124,\"first_received_at\":\"2026-09-07T15:04:59.188298+00:00\",\"last_received_at\":\"2026-09-08T03:52:00.952767+00:00\",\"max_source_sequence\":72124,\"min_source_sequence\":1,\"session_hash\":\"69bb22240441\"},{\"events\":1688,\"first_received_at\":\"2026-09-07T14:46:51.906734+00:00\",\"last_received_at\":\"2026-09-07T15:04:47.491753+00:00\",\"max_source_sequence\":1688,\"min_source_sequence\":1,\"session_hash\":\"d1d81c259d86\"},{\"events\":10295,\"first_received_at\":\"2026-09-07T13:30:08.973280+00:00\",\"last_received_at\":\"2026-09-07T14:46:39.866418+00:00\",\"max_source_sequence\":10295,\"min_source_sequence\":1,\"session_hash\":\"a2db9de2102e\"},{\"events\":14430,\"first_received_at\":\"2026-09-07T09:50:41.075840+00:00\",\"last_received_at\":\"2026-09-07T13:30:03.165612+00:00\",\"max_source_sequence\":14430,\"min_source_sequence\":1,\"session_hash\":\"8da25a454977\"},{\"events\":71006,\"first_received_at\":\"2026-09-06T18:48:31.906387+00:00\",\"last_received_at\":\"2026-09-07T09:50:13.818640+00:00\",\"max_source_sequence\":71006,\"min_source_sequence\":1,\"session_hash\":\"184e0e63b1c9\"}],\"latest_source_receipts\":[{\"receipt_id\":\"02e5367b2692cde4f9bff73096271e6cc0598bf27413d651026f6318c8338d72\",\"received_at\":\"2026-09-08T03:52:00.952767+00:00\",\"source_sequence\":72124},{\"receipt_id\":\"568fff4620581353cea781b3cef61de359e02f12475ac54c7460e85c4baa8625\",\"received_at\":\"2026-09-08T03:52:00.943741+00:00\",\"source_sequence\":72123},{\"receipt_id\":\"3631962ca9ee6b49c4870d42ec814ad6c1c3f0ff65f78a445abecc54bcbe8567\",\"received_at\":\"2026-09-08T03:52:00.931321+00:00\",\"source_sequence\":72122},{\"receipt_id\":\"e9e23d4774b5bdc4801a3334511f50fbb5b25e117b01f8e913a98be3d98badc0\",\"received_at\":\"2026-09-08T03:52:00.790541+00:00\",\"source_sequence\":72121},{\"receipt_id\":\"59ad6dd601a0d1605e6ea3962d9bfb348cdbc67cc8666b473eb8d58b1b8c077c\",\"received_at\":\"2026-09-08T03:52:00.656138+00:00\",\"source_sequence\":72120},{\"receipt_id\":\"634a7fdd47cddc7c664450d7921d4b83093d20a6f95f72f31f6dc004b219ff0f\",\"received_at\":\"2026-09-08T03:52:00.642417+00:00\",\"source_sequence\":72119},{\"receipt_id\":\"6943e44a184672570beaf3ac076e8051162086f16c4f836d78b040c6fdb811ce\",\"received_at\":\"2026-09-08T03:52:00.504076+00:00\",\"source_sequence\":72118},{\"receipt_id\":\"063b4e0abd1065a59d6b9fe62b9e4583f4caacbbb1e83414875386170b8ba7fe\",\"received_at\":\"2026-09-08T03:52:00.494856+00:00\",\"source_sequence\":72117},{\"receipt_id\":\"0f018a85552ad881ab665baa7ddc7a0e956b6d0d66ec43ecfeee44b60e766f4c\",\"received_at\":\"2026-09-08T03:52:00.481147+00:00\",\"source_sequence\":72116},{\"receipt_id\":\"6b23823a92031d3850483879578a8528a45b5f6002b26381659904d813f5b8d2\",\"received_at\":\"2026-09-08T03:52:00.333870+00:00\",\"source_sequence\":72115},{\"receipt_id\":\"1985ce2996de079095414ce9a48bfa3fcfffe7ddcbcdcff1857e03aa0c770d7d\",\"received_at\":\"2026-09-08T03:52:00.194864+00:00\",\"source_sequence\":72114},{\"receipt_id\":\"a4ed84172daff21758b2402a6cc03ab51ddc9474ba23d7a4036f7c6c4a3af152\",\"received_at\":\"2026-09-08T03:52:00.180891+00:00\",\"source_sequence\":72113},{\"receipt_id\":\"8646a6eea56850890817a5ff2056f9639f830a876e460c3898cfebbbd881a670\",\"received_at\":\"2026-09-08T03:52:00.041606+00:00\",\"source_sequence\":72112},{\"receipt_id\":\"70ebbdab1c237840ea3cb6b543fab899e49db3b3a0ab65455e5dcb85b7844923\",\"received_at\":\"2026-09-08T03:52:00.023714+00:00\",\"source_sequence\":72111},{\"receipt_id\":\"7cb2c79c0940deed850deb26500cb3c04331baa4eee8e302a1ca9a3e1cd3023c\",\"received_at\":\"2026-09-08T03:51:59.882828+00:00\",\"source_sequence\":72110}],\"path\":\"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/bitfinex_source.db\",\"quick_check\":\"ok\",\"sequence_events\":{\"count\":179740,\"max_event_seq\":179740,\"max_received_at\":\"2026-09-08T05:23:20.918750+00:00\",\"max_source_sequence\":72124},\"source_receipts_columns\":[\"receipt_id\",\"venue_id\",\"source_kind\",\"received_at\",\"source_sequence\",\"raw_payload_hash\",\"collector_session_id\",\"receipt_json\"]},\"db_b\":{\"d1_revisions_count\":0,\"event_kind_counts\":[[\"SEQUENCE\",179781],[\"RECONNECT\",10],[\"SESSION_START\",1]],\"exists\":true,\"finalized_d1_count\":0,\"latest_sequence_events\":[{\"event_kind\":\"SEQUENCE\",\"event_seq\":179792,\"received_at\":\"2026-09-08T05:23:44.017776+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2562},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179791,\"received_at\":\"2026-09-08T05:23:43.862198+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2561},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179790,\"received_at\":\"2026-09-08T05:23:43.702914+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2560},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179789,\"received_at\":\"2026-09-08T05:23:43.541151+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2559},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179788,\"received_at\":\"2026-09-08T05:23:43.382833+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2558},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179787,\"received_at\":\"2026-09-08T05:23:43.374827+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2557},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179786,\"received_at\":\"2026-09-08T05:23:43.365183+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2556},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179785,\"received_at\":\"2026-09-08T05:23:43.356202+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2555},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179784,\"received_at\":\"2026-09-08T05:23:43.337857+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2554},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179783,\"received_at\":\"2026-09-08T05:23:43.179159+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2553},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179782,\"received_at\":\"2026-09-08T05:23:43.025806+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2552},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179781,\"received_at\":\"2026-09-08T05:23:42.863932+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2551},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179780,\"received_at\":\"2026-09-08T05:23:42.715680+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2550},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179779,\"received_at\":\"2026-09-08T05:23:42.709841+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2549},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179778,\"received_at\":\"2026-09-08T05:23:42.702802+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2548},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179777,\"received_at\":\"2026-09-08T05:23:42.695983+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2547},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179776,\"received_at\":\"2026-09-08T05:23:42.682500+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2546},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179775,\"received_at\":\"2026-09-08T05:23:33.495587+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2545},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179774,\"received_at\":\"2026-09-08T05:23:33.324435+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2544},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179773,\"received_at\":\"2026-09-08T05:23:33.170904+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2543},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179772,\"received_at\":\"2026-09-08T05:23:33.161405+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2542},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179771,\"received_at\":\"2026-09-08T05:23:33.148039+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2541},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179770,\"received_at\":\"2026-09-08T05:23:33.102924+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2540},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179769,\"received_at\":\"2026-09-08T05:23:32.054424+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2539},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179768,\"received_at\":\"2026-09-08T05:23:32.019933+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2538},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179767,\"received_at\":\"2026-09-08T05:23:30.952907+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2537},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179766,\"received_at\":\"2026-09-08T05:23:30.911940+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2536},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179765,\"received_at\":\"2026-09-08T05:23:28.052780+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2535},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179764,\"received_at\":\"2026-09-08T05:23:25.504960+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2534},{\"event_kind\":\"SEQUENCE\",\"event_seq\":179763,\"received_at\":\"2026-09-08T05:23:25.491855+00:00\",\"session_hash\":\"253ffe9a26a5\",\"source_sequence\":2533}],\"latest_sessions\":[{\"events\":2562,\"first_received_at\":\"2026-09-08T05:01:09.298101+00:00\",\"last_received_at\":\"2026-09-08T05:23:44.017776+00:00\",\"max_source_sequence\":2562,\"min_source_sequence\":1,\"session_hash\":\"253ffe9a26a5\"},{\"events\":2475,\"first_received_at\":\"2026-09-08T04:54:28.087186+00:00\",\"last_received_at\":\"2026-09-08T05:01:08.350487+00:00\",\"max_source_sequence\":2475,\"min_source_sequence\":1,\"session_hash\":\"2b3998047f77\"},{\"events\":988,\"first_received_at\":\"2026-09-08T04:43:46.442548+00:00\",\"last_received_at\":\"2026-09-08T04:54:27.005150+00:00\",\"max_source_sequence\":988,\"min_source_sequence\":1,\"session_hash\":\"b501ae120c1f\"},{\"events\":589,\"first_received_at\":\"2026-09-08T04:42:44.434580+00:00\",\"last_received_at\":\"2026-09-08T04:43:45.521859+00:00\",\"max_source_sequence\":589,\"min_source_sequence\":1,\"session_hash\":\"10709eb54a7b\"},{\"events\":3620,\"first_received_at\":\"2026-09-08T03:52:02.239825+00:00\",\"last_received_at\":\"2026-09-08T04:42:43.449781+00:00\",\"max_source_sequence\":3620,\"min_source_sequence\":1,\"session_hash\":\"210de3583c8e\"},{\"events\":72124,\"first_received_at\":\"2026-09-07T15:04:59.188298+00:00\",\"last_received_at\":\"2026-09-08T03:52:00.952767+00:00\",\"max_source_sequence\":72124,\"min_source_sequence\":1,\"session_hash\":\"69bb22240441\"},{\"events\":1688,\"first_received_at\":\"2026-09-07T14:46:51.906734+00:00\",\"last_received_at\":\"2026-09-07T15:04:47.491753+00:00\",\"max_source_sequence\":1688,\"min_source_sequence\":1,\"session_hash\":\"d1d81c259d86\"},{\"events\":10295,\"first_received_at\":\"2026-09-07T13:30:08.973280+00:00\",\"last_received_at\":\"2026-09-07T14:46:39.866418+00:00\",\"max_source_sequence\":10295,\"min_source_sequence\":1,\"session_hash\":\"a2db9de2102e\"},{\"events\":14430,\"first_received_at\":\"2026-09-07T09:50:41.075840+00:00\",\"last_received_at\":\"2026-09-07T13:30:03.165612+00:00\",\"max_source_sequence\":14430,\"min_source_sequence\":1,\"session_hash\":\"8da25a454977\"},{\"events\":71006,\"first_received_at\":\"2026-09-06T18:48:31.906387+00:00\",\"last_received_at\":\"2026-09-07T09:50:13.818640+00:00\",\"max_source_sequence\":71006,\"min_source_sequence\":1,\"session_hash\":\"184e0e63b1c9\"}],\"latest_source_receipts\":[{\"receipt_id\":\"02e5367b2692cde4f9bff73096271e6cc0598bf27413d651026f6318c8338d72\",\"received_at\":\"2026-09-08T03:52:00.952767+00:00\",\"source_sequence\":72124},{\"receipt_id\":\"568fff4620581353cea781b3cef61de359e02f12475ac54c7460e85c4baa8625\",\"received_at\":\"2026-09-08T03:52:00.943741+00:00\",\"source_sequence\":72123},{\"receipt_id\":\"3631962ca9ee6b49c4870d42ec814ad6c1c3f0ff65f78a445abecc54bcbe8567\",\"received_at\":\"2026-09-08T03:52:00.931321+00:00\",\"source_sequence\":72122},{\"receipt_id\":\"e9e23d4774b5bdc4801a3334511f50fbb5b25e117b01f8e913a98be3d98badc0\",\"received_at\":\"2026-09-08T03:52:00.790541+00:00\",\"source_sequence\":72121},{\"receipt_id\":\"59ad6dd601a0d1605e6ea3962d9bfb348cdbc67cc8666b473eb8d58b1b8c077c\",\"received_at\":\"2026-09-08T03:52:00.656138+00:00\",\"source_sequence\":72120},{\"receipt_id\":\"634a7fdd47cddc7c664450d7921d4b83093d20a6f95f72f31f6dc004b219ff0f\",\"received_at\":\"2026-09-08T03:52:00.642417+00:00\",\"source_sequence\":72119},{\"receipt_id\":\"6943e44a184672570beaf3ac076e8051162086f16c4f836d78b040c6fdb811ce\",\"received_at\":\"2026-09-08T03:52:00.504076+00:00\",\"source_sequence\":72118},{\"receipt_id\":\"063b4e0abd1065a59d6b9fe62b9e4583f4caacbbb1e83414875386170b8ba7fe\",\"received_at\":\"2026-09-08T03:52:00.494856+00:00\",\"source_sequence\":72117},{\"receipt_id\":\"0f018a85552ad881ab665baa7ddc7a0e956b6d0d66ec43ecfeee44b60e766f4c\",\"received_at\":\"2026-09-08T03:52:00.481147+00:00\",\"source_sequence\":72116},{\"receipt_id\":\"6b23823a92031d3850483879578a8528a45b5f6002b26381659904d813f5b8d2\",\"received_at\":\"2026-09-08T03:52:00.333870+00:00\",\"source_sequence\":72115},{\"receipt_id\":\"1985ce2996de079095414ce9a48bfa3fcfffe7ddcbcdcff1857e03aa0c770d7d\",\"received_at\":\"2026-09-08T03:52:00.194864+00:00\",\"source_sequence\":72114},{\"receipt_id\":\"a4ed84172daff21758b2402a6cc03ab51ddc9474ba23d7a4036f7c6c4a3af152\",\"received_at\":\"2026-09-08T03:52:00.180891+00:00\",\"source_sequence\":72113},{\"receipt_id\":\"8646a6eea56850890817a5ff2056f9639f830a876e460c3898cfebbbd881a670\",\"received_at\":\"2026-09-08T03:52:00.041606+00:00\",\"source_sequence\":72112},{\"receipt_id\":\"70ebbdab1c237840ea3cb6b543fab899e49db3b3a0ab65455e5dcb85b7844923\",\"received_at\":\"2026-09-08T03:52:00.023714+00:00\",\"source_sequence\":72111},{\"receipt_id\":\"7cb2c79c0940deed850deb26500cb3c04331baa4eee8e302a1ca9a3e1cd3023c\",\"received_at\":\"2026-09-08T03:51:59.882828+00:00\",\"source_sequence\":72110}],\"path\":\"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/bitfinex_source.db\",\"quick_check\":\"ok\",\"sequence_events\":{\"count\":179792,\"max_event_seq\":179792,\"max_received_at\":\"2026-09-08T05:23:44.017776+00:00\",\"max_source_sequence\":72124},\"source_receipts_columns\":[\"receipt_id\",\"venue_id\",\"source_kind\",\"received_at\",\"source_sequence\",\"raw_payload_hash\",\"collector_session_id\",\"receipt_json\"]},\"decision\":\"HOLD_COLLECTOR_INTAKE_STALLED_WITH_PROCESS_AND_TCP_ALIVE\",\"endpoint_probe\":{\"api-pub.bitfinex.com\":{\"dns_rc\":0,\"resolved\":true},\"api.bitfinex.com\":{\"dns_rc\":0,\"resolved\":true},\"platform_status\":{\"body\":\"[1]\",\"rc\":0,\"stderr\":\"\"}},\"epoch_exists\":true,\"holds\":[\"COLLECTOR_INTAKE_STILL_STALLED\"],\"hostname\":\"core-apps\",\"last_event_age_sec\":10.6,\"logs\":{\"keyword_counts\":{\"CLOSED\":5,\"CONNECTED\":0,\"DISCONNECT\":0,\"ERROR\":5,\"EXCEPTION\":5,\"FINAL\":0,\"GAP\":0,\"HEARTBEAT\":0,\"RECEIVED\":5,\"RECONNECT\":0,\"SOURCE\":0,\"TRACEBACK\":5,\"WEBSOCKET\":15},\"last_lines\":[\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"Traceback (most recent call last):\",\"File \\\"\\\", line 203, in _run_module_as_main\",\"File \\\"\\\", line 88, in _run_code\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1246, in \",\"raise SystemExit(main())\",\"~~~~^^\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1175, in main\",\"for raw in ws:\",\"^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"yield self.recv()\",\"~~~~~~~~~^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"raise self.protocol.close_exc from self.recv_exc\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"Traceback (most recent call last):\",\"File \\\"\\\", line 203, in _run_module_as_main\",\"File \\\"\\\", line 88, in _run_code\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1246, in \",\"raise SystemExit(main())\",\"~~~~^^\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1175, in main\",\"for raw in ws:\",\"^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"yield self.recv()\",\"~~~~~~~~~^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"raise self.protocol.close_exc from self.recv_exc\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"Traceback (most recent call last):\",\"File \\\"\\\", line 203, in _run_module_as_main\",\"File \\\"\\\", line 88, in _run_code\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1246, in \",\"raise SystemExit(main())\",\"~~~~^^\",\"File \\\"/app/demonov_forward_map/bitfinex_trade_collector.py\\\", line 1175, in main\",\"for raw in ws:\",\"^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"yield self.recv()\",\"~~~~~~~~~^^\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"raise self.protocol.close_exc from self.recv_exc\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\"],\"last_relevant\":[\"Traceback (most recent call last):\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"Traceback (most recent call last):\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"Traceback (most recent call last):\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"Traceback (most recent call last):\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"Traceback (most recent call last):\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 242, in __iter__\",\"File \\\"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\\\", line 336, in recv\",\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\"],\"line_count\":81,\"rc\":0},\"mutation_scope\":\"NONE_READ_ONLY\",\"sequence_window\":{\"a\":72124,\"advanced_from_034\":false,\"b\":72124,\"baseline_034\":72124,\"grew\":false,\"vs_034\":72124},\"socket\":{\"established_total\":1,\"pid\":210213,\"remote_port_counts\":{\"443\":1},\"states\":{\"01\":1,\"0A\":1}},\"stack_exists\":true,\"stats\":{\"line\":\"demonov-shadow-8020-collector-1 43.29% 26.51MiB / 9.719GiB 315kB / 126kB\",\"rc\":0,\"stderr\":\"\"},\"top\":{\"lines\":[\"PID PPID COMMAND\",\"210213 210191 python\"],\"rc\":0,\"stderr\":\"\"}},\"state\":\"COMPLETE\"}\nSIGNALBOT035_HOLDS=[\"COLLECTOR_INTAKE_STILL_STALLED\"]\nSIGNALBOT035_DECISION=HOLD_COLLECTOR_INTAKE_STALLED_WITH_PROCESS_AND_TCP_ALIVE\nSIGNALBOT035_MUTATION_SCOPE=NO_RUNTIME_NO_GIT_NO_EXCHANGE_MUTATION\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 36dfbc3b..6eff3b04 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=NEWFI-260908-A-STAGINGDNS-APPLY4 -STATUS=FAIL -RC=1 +COMMAND_ID=SIGNALBOT-260908-INTAKE-RCA-035 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=newfi-staging-cloudflare-ingress-candidate-proof-dns-apply +MODE=read-only +COMPONENT=signalbot-cr0116-intake-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=6e253c0d6d882ebec1a77ebcb735719f51d55260a2498f0dd79bdf7b1a0b4861 +COMMAND_SHA256=01c43885d0820853cc620910f562146315ad34c12ff24ffceefe7ef0181b8778 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true +CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=1 +COMMAND_RC=0 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,32 +24,17 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=5a6777ef0dfd996de4e6b912aac38ff38d3a2f6895c69a30bc91d15884f738f5 -SANITIZED_OUTPUT_SHA256=5a6777ef0dfd996de4e6b912aac38ff38d3a2f6895c69a30bc91d15884f738f5 +RAW_EVIDENCE_SHA256=9bc13c343cdda506b9e7bbb4a38191d9f1ee1e4d31beb0916fa1cda673e1e097 +SANITIZED_OUTPUT_SHA256=9bc13c343cdda506b9e7bbb4a38191d9f1ee1e4d31beb0916fa1cda673e1e097 OUTPUT_BEGIN -STAGINGDNS_APPLY4_BEGIN=true -COMMAND_ID=NEWFI-260908-A-STAGINGDNS-APPLY4 -COMPONENT=newfi-staging-cloudflare-ingress-candidate-proof-dns-apply -MODE=change ERROR_REGISTER_CHECK=OK -ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 REFERENCE_CHECK=OK -REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 AUTHORITY_CHECK_SCOPE=READABILITY_ONLY_INCIDENTS_NOT_CLOSED -RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -DISCOVERY1_HISTORY_SHA256=edf7a289f77b6580795698fdc7d82355903d24c8df42d76bf84ec8a36117b873 -PRIOR_DNS_APPLY3_HISTORY_SHA256=916cc5e9c705b07c36b93decd55470bf12b24cb1d6d158b1bbde1b9750d6fc89 -PORTAL_PRODUCT_STATE_GATE=PASS -PUBLIC_DNS_PRESTATE={"1.1.1.1/A":{"err":"","out":";; Got answer:\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 61269\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\n\n;; OPT PSEUDOSECTION:\n; EDNS: version: 0, flags:; udp: 1232\n;; AUTHORITY SECTION:\ngram1.ru.\t\t1800\tIN\tSOA\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800","rc":0},"1.1.1.1/AAAA":{"err":"","out":";; Got answer:\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 42274\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\n\n;; OPT PSEUDOSECTION:\n; EDNS: version: 0, flags:; udp: 1232\n;; AUTHORITY SECTION:\ngram1.ru.\t\t1188\tIN\tSOA\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800","rc":0},"1.1.1.1/CNAME":{"err":"","out":";; Got answer:\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 1534\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\n\n;; OPT PSEUDOSECTION:\n; EDNS: version: 0, flags:; udp: 1232\n;; AUTHORITY SECTION:\ngram1.ru.\t\t1189\tIN\tSOA\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800","rc":0},"8.8.8.8/A":{"err":"","out":";; Got answer:\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 65407\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\n\n;; OPT PSEUDOSECTION:\n; EDNS: version: 0, flags:; udp: 1232\n;; AUTHORITY SECTION:\ngram1.ru.\t\t1800\tIN\tSOA\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800","rc":0},"8.8.8.8/AAAA":{"err":"","out":";; Got answer:\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 40214\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\n\n;; OPT PSEUDOSECTION:\n; EDNS: version: 0, flags:; udp: 1232\n;; AUTHORITY SECTION:\ngram1.ru.\t\t1396\tIN\tSOA\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800","rc":0},"8.8.8.8/CNAME":{"err":"","out":";; Got answer:\n;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 30626\n;; flags: qr rd ra; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1\n\n;; OPT PSEUDOSECTION:\n; EDNS: version: 0, flags:; udp: 512\n;; AUTHORITY SECTION:\ngram1.ru.\t\t1800\tIN\tSOA\tsara.ns.cloudflare.com. dns.cloudflare.com. 2413993701 10000 2400 604800 1800","rc":0}} -CLOUDFLARE_TOKEN_DISCOVERY={"candidate_count":0,"scanned_files":1486} -EDGE_VM_IDENTITY={"name":"edge-vm","node":"pve03","vmid":130} -EDGE_CLOUDFLARE_TOKEN_SCAN={"candidate_count":3,"rc":0,"source_paths":["/root/cert38-narrow-migration-20260904T074428Z/credentials-38.before","/opt/npmplus/secure/taftauto_cf_certbot.ini","/opt/npmplus/secure/forum_cf_token.env"]} -EDGE_CLOUDFLARE_ZONE_ACCESS_VALID_COUNT=1 -CLOUDFLARE_TOKEN_SELECTED={"source_paths":["/root/cert38-narrow-migration-20260904T074428Z/credentials-38.before"],"valid_token_count":1,"zone_id_sha256":"5a88f12ebe16612b4e42f11ee57b4b95ca4a13c4abb8f0009f09babfbec927e7"} -EDGE_VM_IDENTITY={"name":"edge-vm","node":"pve03","vmid":130} -DNS_APPLY_FAILURE=ApplyError:EDGE_ROUTE_CONFIG_NOT_FOUND -FINAL_DECISION=FAIL_NEWFI_STAGING_DNS_NO_MUTATION -HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"NEWFI-260908-A-STAGINGDNS-APPLY4","rollback_restored":null,"rollback_started":false,"status":"FAIL","version":1} +SIGNALBOT035_VM170={"name":"core-apps","node":"pve01","status":"running","type":"qemu"} +SIGNALBOT035_VM170_SNAPSHOT={"report":{"collector":{"cmd":["python","-m","demonov_forward_map.bitfinex_trade_collector"],"entrypoint":null,"health":null,"id_short":"a546a70c3e8c","image":"8020-demonov-shadow:e32760c69a4311728db9066ee8bf2bf66b1e5a70","name":"demonov-shadow-8020-collector-1","pid":210213,"restart_count":13,"running":true,"started_at":"2026-09-08T05:01:08.70385245Z","status":"running"},"collector_status":{"age_sec":8.9,"exists":true,"mtime_epoch":1788845024.0247085,"path":"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/collector_status.json","values":{"fail_closed_reason":"source reconnect detected","gap_detected":false,"post_finality_revision":false,"prospective_fail_closed_reason":"","prospective_gap_detected":false,"prospective_reconnect_detected":false,"reconnect_detected":true,"updated_at":"2026-09-08T05:23:44.028140+00:00"}},"db_a":{"d1_revisions_count":0,"event_kind_counts":[["SEQUENCE",179729],["RECONNECT",10],["SESSION_START",1]],"exists":true,"finalized_d1_count":0,"latest_sequence_events":[{"event_kind":"SEQUENCE","event_seq":179740,"received_at":"2026-09-08T05:23:20.918750+00:00","session_hash":"253ffe9a26a5","source_sequence":2510},{"event_kind":"SEQUENCE","event_seq":179739,"received_at":"2026-09-08T05:23:20.891856+00:00","session_hash":"253ffe9a26a5","source_sequence":2509},{"event_kind":"SEQUENCE","event_seq":179738,"received_at":"2026-09-08T05:23:20.552139+00:00","session_hash":"253ffe9a26a5","source_sequence":2508},{"event_kind":"SEQUENCE","event_seq":179737,"received_at":"2026-09-08T05:23:20.534792+00:00","session_hash":"253ffe9a26a5","source_sequence":2507},{"event_kind":"SEQUENCE","event_seq":179736,"received_at":"2026-09-08T05:23:20.329909+00:00","session_hash":"253ffe9a26a5","source_sequence":2506},{"event_kind":"SEQUENCE","event_seq":179735,"received_at":"2026-09-08T05:23:20.177708+00:00","session_hash":"253ffe9a26a5","source_sequence":2505},{"event_kind":"SEQUENCE","event_seq":179734,"received_at":"2026-09-08T05:23:20.170743+00:00","session_hash":"253ffe9a26a5","source_sequence":2504},{"event_kind":"SEQUENCE","event_seq":179733,"received_at":"2026-09-08T05:23:20.163522+00:00","session_hash":"253ffe9a26a5","source_sequence":2503},{"event_kind":"SEQUENCE","event_seq":179732,"received_at":"2026-09-08T05:23:20.004001+00:00","session_hash":"253ffe9a26a5","source_sequence":2502},{"event_kind":"SEQUENCE","event_seq":179731,"received_at":"2026-09-08T05:23:19.850034+00:00","session_hash":"253ffe9a26a5","source_sequence":2501},{"event_kind":"SEQUENCE","event_seq":179730,"received_at":"2026-09-08T05:23:19.841410+00:00","session_hash":"253ffe9a26a5","source_sequence":2500},{"event_kind":"SEQUENCE","event_seq":179729,"received_at":"2026-09-08T05:23:19.827700+00:00","session_hash":"253ffe9a26a5","source_sequence":2499},{"event_kind":"SEQUENCE","event_seq":179728,"received_at":"2026-09-08T05:23:19.671403+00:00","session_hash":"253ffe9a26a5","source_sequence":2498},{"event_kind":"SEQUENCE","event_seq":179727,"received_at":"2026-09-08T05:23:19.518174+00:00","session_hash":"253ffe9a26a5","source_sequence":2497},{"event_kind":"SEQUENCE","event_seq":179726,"received_at":"2026-09-08T05:23:19.357283+00:00","session_hash":"253ffe9a26a5","source_sequence":2496},{"event_kind":"SEQUENCE","event_seq":179725,"received_at":"2026-09-08T05:23:19.200903+00:00","session_hash":"253ffe9a26a5","source_sequence":2495},{"event_kind":"SEQUENCE","event_seq":179724,"received_at":"2026-09-08T05:23:19.193716+00:00","session_hash":"253ffe9a26a5","source_sequence":2494},{"event_kind":"SEQUENCE","event_seq":179723,"received_at":"2026-09-08T05:23:19.185835+00:00","session_hash":"253ffe9a26a5","source_sequence":2493},{"event_kind":"SEQUENCE","event_seq":179722,"received_at":"2026-09-08T05:23:19.177983+00:00","session_hash":"253ffe9a26a5","source_sequence":2492},{"event_kind":"SEQUENCE","event_seq":179721,"received_at":"2026-09-08T05:23:19.164437+00:00","session_hash":"253ffe9a26a5","source_sequence":2491},{"event_kind":"SEQUENCE","event_seq":179720,"received_at":"2026-09-08T05:23:19.006208+00:00","session_hash":"253ffe9a26a5","source_sequence":2490},{"event_kind":"SEQUENCE","event_seq":179719,"received_at":"2026-09-08T05:23:18.848474+00:00","session_hash":"253ffe9a26a5","source_sequence":2489},{"event_kind":"SEQUENCE","event_seq":179718,"received_at":"2026-09-08T05:23:18.697530+00:00","session_hash":"253ffe9a26a5","source_sequence":2488},{"event_kind":"SEQUENCE","event_seq":179717,"received_at":"2026-09-08T05:23:18.541070+00:00","session_hash":"253ffe9a26a5","source_sequence":2487},{"event_kind":"SEQUENCE","event_seq":179716,"received_at":"2026-09-08T05:23:18.532005+00:00","session_hash":"253ffe9a26a5","source_sequence":2486},{"event_kind":"SEQUENCE","event_seq":179715,"received_at":"2026-09-08T05:23:18.522984+00:00","session_hash":"253ffe9a26a5","source_sequence":2485},{"event_kind":"SEQUENCE","event_seq":179714,"received_at":"2026-09-08T05:23:18.515020+00:00","session_hash":"253ffe9a26a5","source_sequence":2484},{"event_kind":"SEQUENCE","event_seq":179713,"received_at":"2026-09-08T05:23:18.504619+00:00","session_hash":"253ffe9a26a5","source_sequence":2483},{"event_kind":"SEQUENCE","event_seq":179712,"received_at":"2026-09-08T05:23:18.349318+00:00","session_hash":"253ffe9a26a5","source_sequence":2482},{"event_kind":"SEQUENCE","event_seq":179711,"received_at":"2026-09-08T05:23:18.189830+00:00","session_hash":"253ffe9a26a5","source_sequence":2481}],"latest_sessions":[{"events":2510,"first_received_at":"2026-09-08T05:01:09.298101+00:00","last_received_at":"2026-09-08T05:23:20.918750+00:00","max_source_sequence":2510,"min_source_sequence":1,"session_hash":"253ffe9a26a5"},{"events":2475,"first_received_at":"2026-09-08T04:54:28.087186+00:00","last_received_at":"2026-09-08T05:01:08.350487+00:00","max_source_sequence":2475,"min_source_sequence":1,"session_hash":"2b3998047f77"},{"events":988,"first_received_at":"2026-09-08T04:43:46.442548+00:00","last_received_at":"2026-09-08T04:54:27.005150+00:00","max_source_sequence":988,"min_source_sequence":1,"session_hash":"b501ae120c1f"},{"events":589,"first_received_at":"2026-09-08T04:42:44.434580+00:00","last_received_at":"2026-09-08T04:43:45.521859+00:00","max_source_sequence":589,"min_source_sequence":1,"session_hash":"10709eb54a7b"},{"events":3620,"first_received_at":"2026-09-08T03:52:02.239825+00:00","last_received_at":"2026-09-08T04:42:43.449781+00:00","max_source_sequence":3620,"min_source_sequence":1,"session_hash":"210de3583c8e"},{"events":72124,"first_received_at":"2026-09-07T15:04:59.188298+00:00","last_received_at":"2026-09-08T03:52:00.952767+00:00","max_source_sequence":72124,"min_source_sequence":1,"session_hash":"69bb22240441"},{"events":1688,"first_received_at":"2026-09-07T14:46:51.906734+00:00","last_received_at":"2026-09-07T15:04:47.491753+00:00","max_source_sequence":1688,"min_source_sequence":1,"session_hash":"d1d81c259d86"},{"events":10295,"first_received_at":"2026-09-07T13:30:08.973280+00:00","last_received_at":"2026-09-07T14:46:39.866418+00:00","max_source_sequence":10295,"min_source_sequence":1,"session_hash":"a2db9de2102e"},{"events":14430,"first_received_at":"2026-09-07T09:50:41.075840+00:00","last_received_at":"2026-09-07T13:30:03.165612+00:00","max_source_sequence":14430,"min_source_sequence":1,"session_hash":"8da25a454977"},{"events":71006,"first_received_at":"2026-09-06T18:48:31.906387+00:00","last_received_at":"2026-09-07T09:50:13.818640+00:00","max_source_sequence":71006,"min_source_sequence":1,"session_hash":"184e0e63b1c9"}],"latest_source_receipts":[{"receipt_id":"02e5367b2692cde4f9bff73096271e6cc0598bf27413d651026f6318c8338d72","received_at":"2026-09-08T03:52:00.952767+00:00","source_sequence":72124},{"receipt_id":"568fff4620581353cea781b3cef61de359e02f12475ac54c7460e85c4baa8625","received_at":"2026-09-08T03:52:00.943741+00:00","source_sequence":72123},{"receipt_id":"3631962ca9ee6b49c4870d42ec814ad6c1c3f0ff65f78a445abecc54bcbe8567","received_at":"2026-09-08T03:52:00.931321+00:00","source_sequence":72122},{"receipt_id":"e9e23d4774b5bdc4801a3334511f50fbb5b25e117b01f8e913a98be3d98badc0","received_at":"2026-09-08T03:52:00.790541+00:00","source_sequence":72121},{"receipt_id":"59ad6dd601a0d1605e6ea3962d9bfb348cdbc67cc8666b473eb8d58b1b8c077c","received_at":"2026-09-08T03:52:00.656138+00:00","source_sequence":72120},{"receipt_id":"634a7fdd47cddc7c664450d7921d4b83093d20a6f95f72f31f6dc004b219ff0f","received_at":"2026-09-08T03:52:00.642417+00:00","source_sequence":72119},{"receipt_id":"6943e44a184672570beaf3ac076e8051162086f16c4f836d78b040c6fdb811ce","received_at":"2026-09-08T03:52:00.504076+00:00","source_sequence":72118},{"receipt_id":"063b4e0abd1065a59d6b9fe62b9e4583f4caacbbb1e83414875386170b8ba7fe","received_at":"2026-09-08T03:52:00.494856+00:00","source_sequence":72117},{"receipt_id":"0f018a85552ad881ab665baa7ddc7a0e956b6d0d66ec43ecfeee44b60e766f4c","received_at":"2026-09-08T03:52:00.481147+00:00","source_sequence":72116},{"receipt_id":"6b23823a92031d3850483879578a8528a45b5f6002b26381659904d813f5b8d2","received_at":"2026-09-08T03:52:00.333870+00:00","source_sequence":72115},{"receipt_id":"1985ce2996de079095414ce9a48bfa3fcfffe7ddcbcdcff1857e03aa0c770d7d","received_at":"2026-09-08T03:52:00.194864+00:00","source_sequence":72114},{"receipt_id":"a4ed84172daff21758b2402a6cc03ab51ddc9474ba23d7a4036f7c6c4a3af152","received_at":"2026-09-08T03:52:00.180891+00:00","source_sequence":72113},{"receipt_id":"8646a6eea56850890817a5ff2056f9639f830a876e460c3898cfebbbd881a670","received_at":"2026-09-08T03:52:00.041606+00:00","source_sequence":72112},{"receipt_id":"70ebbdab1c237840ea3cb6b543fab899e49db3b3a0ab65455e5dcb85b7844923","received_at":"2026-09-08T03:52:00.023714+00:00","source_sequence":72111},{"receipt_id":"7cb2c79c0940deed850deb26500cb3c04331baa4eee8e302a1ca9a3e1cd3023c","received_at":"2026-09-08T03:51:59.882828+00:00","source_sequence":72110}],"path":"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/bitfinex_source.db","quick_check":"ok","sequence_events":{"count":179740,"max_event_seq":179740,"max_received_at":"2026-09-08T05:23:20.918750+00:00","max_source_sequence":72124},"source_receipts_columns":["receipt_id","venue_id","source_kind","received_at","source_sequence","raw_payload_hash","collector_session_id","receipt_json"]},"db_b":{"d1_revisions_count":0,"event_kind_counts":[["SEQUENCE",179781],["RECONNECT",10],["SESSION_START",1]],"exists":true,"finalized_d1_count":0,"latest_sequence_events":[{"event_kind":"SEQUENCE","event_seq":179792,"received_at":"2026-09-08T05:23:44.017776+00:00","session_hash":"253ffe9a26a5","source_sequence":2562},{"event_kind":"SEQUENCE","event_seq":179791,"received_at":"2026-09-08T05:23:43.862198+00:00","session_hash":"253ffe9a26a5","source_sequence":2561},{"event_kind":"SEQUENCE","event_seq":179790,"received_at":"2026-09-08T05:23:43.702914+00:00","session_hash":"253ffe9a26a5","source_sequence":2560},{"event_kind":"SEQUENCE","event_seq":179789,"received_at":"2026-09-08T05:23:43.541151+00:00","session_hash":"253ffe9a26a5","source_sequence":2559},{"event_kind":"SEQUENCE","event_seq":179788,"received_at":"2026-09-08T05:23:43.382833+00:00","session_hash":"253ffe9a26a5","source_sequence":2558},{"event_kind":"SEQUENCE","event_seq":179787,"received_at":"2026-09-08T05:23:43.374827+00:00","session_hash":"253ffe9a26a5","source_sequence":2557},{"event_kind":"SEQUENCE","event_seq":179786,"received_at":"2026-09-08T05:23:43.365183+00:00","session_hash":"253ffe9a26a5","source_sequence":2556},{"event_kind":"SEQUENCE","event_seq":179785,"received_at":"2026-09-08T05:23:43.356202+00:00","session_hash":"253ffe9a26a5","source_sequence":2555},{"event_kind":"SEQUENCE","event_seq":179784,"received_at":"2026-09-08T05:23:43.337857+00:00","session_hash":"253ffe9a26a5","source_sequence":2554},{"event_kind":"SEQUENCE","event_seq":179783,"received_at":"2026-09-08T05:23:43.179159+00:00","session_hash":"253ffe9a26a5","source_sequence":2553},{"event_kind":"SEQUENCE","event_seq":179782,"received_at":"2026-09-08T05:23:43.025806+00:00","session_hash":"253ffe9a26a5","source_sequence":2552},{"event_kind":"SEQUENCE","event_seq":179781,"received_at":"2026-09-08T05:23:42.863932+00:00","session_hash":"253ffe9a26a5","source_sequence":2551},{"event_kind":"SEQUENCE","event_seq":179780,"received_at":"2026-09-08T05:23:42.715680+00:00","session_hash":"253ffe9a26a5","source_sequence":2550},{"event_kind":"SEQUENCE","event_seq":179779,"received_at":"2026-09-08T05:23:42.709841+00:00","session_hash":"253ffe9a26a5","source_sequence":2549},{"event_kind":"SEQUENCE","event_seq":179778,"received_at":"2026-09-08T05:23:42.702802+00:00","session_hash":"253ffe9a26a5","source_sequence":2548},{"event_kind":"SEQUENCE","event_seq":179777,"received_at":"2026-09-08T05:23:42.695983+00:00","session_hash":"253ffe9a26a5","source_sequence":2547},{"event_kind":"SEQUENCE","event_seq":179776,"received_at":"2026-09-08T05:23:42.682500+00:00","session_hash":"253ffe9a26a5","source_sequence":2546},{"event_kind":"SEQUENCE","event_seq":179775,"received_at":"2026-09-08T05:23:33.495587+00:00","session_hash":"253ffe9a26a5","source_sequence":2545},{"event_kind":"SEQUENCE","event_seq":179774,"received_at":"2026-09-08T05:23:33.324435+00:00","session_hash":"253ffe9a26a5","source_sequence":2544},{"event_kind":"SEQUENCE","event_seq":179773,"received_at":"2026-09-08T05:23:33.170904+00:00","session_hash":"253ffe9a26a5","source_sequence":2543},{"event_kind":"SEQUENCE","event_seq":179772,"received_at":"2026-09-08T05:23:33.161405+00:00","session_hash":"253ffe9a26a5","source_sequence":2542},{"event_kind":"SEQUENCE","event_seq":179771,"received_at":"2026-09-08T05:23:33.148039+00:00","session_hash":"253ffe9a26a5","source_sequence":2541},{"event_kind":"SEQUENCE","event_seq":179770,"received_at":"2026-09-08T05:23:33.102924+00:00","session_hash":"253ffe9a26a5","source_sequence":2540},{"event_kind":"SEQUENCE","event_seq":179769,"received_at":"2026-09-08T05:23:32.054424+00:00","session_hash":"253ffe9a26a5","source_sequence":2539},{"event_kind":"SEQUENCE","event_seq":179768,"received_at":"2026-09-08T05:23:32.019933+00:00","session_hash":"253ffe9a26a5","source_sequence":2538},{"event_kind":"SEQUENCE","event_seq":179767,"received_at":"2026-09-08T05:23:30.952907+00:00","session_hash":"253ffe9a26a5","source_sequence":2537},{"event_kind":"SEQUENCE","event_seq":179766,"received_at":"2026-09-08T05:23:30.911940+00:00","session_hash":"253ffe9a26a5","source_sequence":2536},{"event_kind":"SEQUENCE","event_seq":179765,"received_at":"2026-09-08T05:23:28.052780+00:00","session_hash":"253ffe9a26a5","source_sequence":2535},{"event_kind":"SEQUENCE","event_seq":179764,"received_at":"2026-09-08T05:23:25.504960+00:00","session_hash":"253ffe9a26a5","source_sequence":2534},{"event_kind":"SEQUENCE","event_seq":179763,"received_at":"2026-09-08T05:23:25.491855+00:00","session_hash":"253ffe9a26a5","source_sequence":2533}],"latest_sessions":[{"events":2562,"first_received_at":"2026-09-08T05:01:09.298101+00:00","last_received_at":"2026-09-08T05:23:44.017776+00:00","max_source_sequence":2562,"min_source_sequence":1,"session_hash":"253ffe9a26a5"},{"events":2475,"first_received_at":"2026-09-08T04:54:28.087186+00:00","last_received_at":"2026-09-08T05:01:08.350487+00:00","max_source_sequence":2475,"min_source_sequence":1,"session_hash":"2b3998047f77"},{"events":988,"first_received_at":"2026-09-08T04:43:46.442548+00:00","last_received_at":"2026-09-08T04:54:27.005150+00:00","max_source_sequence":988,"min_source_sequence":1,"session_hash":"b501ae120c1f"},{"events":589,"first_received_at":"2026-09-08T04:42:44.434580+00:00","last_received_at":"2026-09-08T04:43:45.521859+00:00","max_source_sequence":589,"min_source_sequence":1,"session_hash":"10709eb54a7b"},{"events":3620,"first_received_at":"2026-09-08T03:52:02.239825+00:00","last_received_at":"2026-09-08T04:42:43.449781+00:00","max_source_sequence":3620,"min_source_sequence":1,"session_hash":"210de3583c8e"},{"events":72124,"first_received_at":"2026-09-07T15:04:59.188298+00:00","last_received_at":"2026-09-08T03:52:00.952767+00:00","max_source_sequence":72124,"min_source_sequence":1,"session_hash":"69bb22240441"},{"events":1688,"first_received_at":"2026-09-07T14:46:51.906734+00:00","last_received_at":"2026-09-07T15:04:47.491753+00:00","max_source_sequence":1688,"min_source_sequence":1,"session_hash":"d1d81c259d86"},{"events":10295,"first_received_at":"2026-09-07T13:30:08.973280+00:00","last_received_at":"2026-09-07T14:46:39.866418+00:00","max_source_sequence":10295,"min_source_sequence":1,"session_hash":"a2db9de2102e"},{"events":14430,"first_received_at":"2026-09-07T09:50:41.075840+00:00","last_received_at":"2026-09-07T13:30:03.165612+00:00","max_source_sequence":14430,"min_source_sequence":1,"session_hash":"8da25a454977"},{"events":71006,"first_received_at":"2026-09-06T18:48:31.906387+00:00","last_received_at":"2026-09-07T09:50:13.818640+00:00","max_source_sequence":71006,"min_source_sequence":1,"session_hash":"184e0e63b1c9"}],"latest_source_receipts":[{"receipt_id":"02e5367b2692cde4f9bff73096271e6cc0598bf27413d651026f6318c8338d72","received_at":"2026-09-08T03:52:00.952767+00:00","source_sequence":72124},{"receipt_id":"568fff4620581353cea781b3cef61de359e02f12475ac54c7460e85c4baa8625","received_at":"2026-09-08T03:52:00.943741+00:00","source_sequence":72123},{"receipt_id":"3631962ca9ee6b49c4870d42ec814ad6c1c3f0ff65f78a445abecc54bcbe8567","received_at":"2026-09-08T03:52:00.931321+00:00","source_sequence":72122},{"receipt_id":"e9e23d4774b5bdc4801a3334511f50fbb5b25e117b01f8e913a98be3d98badc0","received_at":"2026-09-08T03:52:00.790541+00:00","source_sequence":72121},{"receipt_id":"59ad6dd601a0d1605e6ea3962d9bfb348cdbc67cc8666b473eb8d58b1b8c077c","received_at":"2026-09-08T03:52:00.656138+00:00","source_sequence":72120},{"receipt_id":"634a7fdd47cddc7c664450d7921d4b83093d20a6f95f72f31f6dc004b219ff0f","received_at":"2026-09-08T03:52:00.642417+00:00","source_sequence":72119},{"receipt_id":"6943e44a184672570beaf3ac076e8051162086f16c4f836d78b040c6fdb811ce","received_at":"2026-09-08T03:52:00.504076+00:00","source_sequence":72118},{"receipt_id":"063b4e0abd1065a59d6b9fe62b9e4583f4caacbbb1e83414875386170b8ba7fe","received_at":"2026-09-08T03:52:00.494856+00:00","source_sequence":72117},{"receipt_id":"0f018a85552ad881ab665baa7ddc7a0e956b6d0d66ec43ecfeee44b60e766f4c","received_at":"2026-09-08T03:52:00.481147+00:00","source_sequence":72116},{"receipt_id":"6b23823a92031d3850483879578a8528a45b5f6002b26381659904d813f5b8d2","received_at":"2026-09-08T03:52:00.333870+00:00","source_sequence":72115},{"receipt_id":"1985ce2996de079095414ce9a48bfa3fcfffe7ddcbcdcff1857e03aa0c770d7d","received_at":"2026-09-08T03:52:00.194864+00:00","source_sequence":72114},{"receipt_id":"a4ed84172daff21758b2402a6cc03ab51ddc9474ba23d7a4036f7c6c4a3af152","received_at":"2026-09-08T03:52:00.180891+00:00","source_sequence":72113},{"receipt_id":"8646a6eea56850890817a5ff2056f9639f830a876e460c3898cfebbbd881a670","received_at":"2026-09-08T03:52:00.041606+00:00","source_sequence":72112},{"receipt_id":"70ebbdab1c237840ea3cb6b543fab899e49db3b3a0ab65455e5dcb85b7844923","received_at":"2026-09-08T03:52:00.023714+00:00","source_sequence":72111},{"receipt_id":"7cb2c79c0940deed850deb26500cb3c04331baa4eee8e302a1ca9a3e1cd3023c","received_at":"2026-09-08T03:51:59.882828+00:00","source_sequence":72110}],"path":"/opt/stacks/8020-demonov-shadow/data/epochs/ec477ea41ecbd5cfdef5afc259595aab20674a7d933f41a389329ff05098b338/bitfinex_source.db","quick_check":"ok","sequence_events":{"count":179792,"max_event_seq":179792,"max_received_at":"2026-09-08T05:23:44.017776+00:00","max_source_sequence":72124},"source_receipts_columns":["receipt_id","venue_id","source_kind","received_at","source_sequence","raw_payload_hash","collector_session_id","receipt_json"]},"decision":"HOLD_COLLECTOR_INTAKE_STALLED_WITH_PROCESS_AND_TCP_ALIVE","endpoint_probe":{"api-pub.bitfinex.com":{"dns_rc":0,"resolved":true},"api.bitfinex.com":{"dns_rc":0,"resolved":true},"platform_status":{"body":"[1]","rc":0,"stderr":""}},"epoch_exists":true,"holds":["COLLECTOR_INTAKE_STILL_STALLED"],"hostname":"core-apps","last_event_age_sec":10.6,"logs":{"keyword_counts":{"CLOSED":5,"CONNECTED":0,"DISCONNECT":0,"ERROR":5,"EXCEPTION":5,"FINAL":0,"GAP":0,"HEARTBEAT":0,"RECEIVED":5,"RECONNECT":0,"SOURCE":0,"TRACEBACK":5,"WEBSOCKET":15},"last_lines":["websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received",":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour","Traceback (most recent call last):","File \"\", line 203, in _run_module_as_main","File \"\", line 88, in _run_code","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1246, in ","raise SystemExit(main())","~~~~^^","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1175, in main","for raw in ws:","^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","yield self.recv()","~~~~~~~~~^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","raise self.protocol.close_exc from self.recv_exc","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received",":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour","Traceback (most recent call last):","File \"\", line 203, in _run_module_as_main","File \"\", line 88, in _run_code","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1246, in ","raise SystemExit(main())","~~~~^^","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1175, in main","for raw in ws:","^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","yield self.recv()","~~~~~~~~~^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","raise self.protocol.close_exc from self.recv_exc","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received",":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour","Traceback (most recent call last):","File \"\", line 203, in _run_module_as_main","File \"\", line 88, in _run_code","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1246, in ","raise SystemExit(main())","~~~~^^","File \"/app/demonov_forward_map/bitfinex_trade_collector.py\", line 1175, in main","for raw in ws:","^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","yield self.recv()","~~~~~~~~~^^","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","raise self.protocol.close_exc from self.recv_exc","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received",":130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour"],"last_relevant":["Traceback (most recent call last):","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received","Traceback (most recent call last):","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received","Traceback (most recent call last):","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received","Traceback (most recent call last):","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received","Traceback (most recent call last):","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 242, in __iter__","File \"/usr/local/lib/python3.13/site-packages/websockets/sync/connection.py\", line 336, in recv","websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received"],"line_count":81,"rc":0},"mutation_scope":"NONE_READ_ONLY","sequence_window":{"a":72124,"advanced_from_034":false,"b":72124,"baseline_034":72124,"grew":false,"vs_034":72124},"socket":{"established_total":1,"pid":210213,"remote_port_counts":{"443":1},"states":{"01":1,"0A":1}},"stack_exists":true,"stats":{"line":"demonov-shadow-8020-collector-1 43.29% 26.51MiB / 9.719GiB 315kB / 126kB","rc":0,"stderr":""},"top":{"lines":["PID PPID COMMAND","210213 210191 python"],"rc":0,"stderr":""}},"state":"COMPLETE"} +SIGNALBOT035_HOLDS=["COLLECTOR_INTAKE_STILL_STALLED"] +SIGNALBOT035_DECISION=HOLD_COLLECTOR_INTAKE_STALLED_WITH_PROCESS_AND_TCP_ALIVE +SIGNALBOT035_MUTATION_SCOPE=NO_RUNTIME_NO_GIT_NO_EXCHANGE_MUTATION OUTPUT_END CHAT_OUTPUT_END