homelab-runtime-publisher
39 lines
34 KiB
JSON
39 lines
34 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"channel": "homelab-runtime",
|
|
"command_id": "SIGNALBOT-260917-COLLECTOR-RESTART-CLEAN-D1-RCA-046R1",
|
|
"status": "OK",
|
|
"rc": 0,
|
|
"host": "pve01",
|
|
"mode": "read-only",
|
|
"component": "signalbot-cr0116-collector-restart-clean-d1-rca",
|
|
"started_at_utc": "2026-09-17T12:06:47Z",
|
|
"finished_at_utc": "2026-09-17T12:06:54Z",
|
|
"reference_register_checked": true,
|
|
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
|
|
"error_register_checked": true,
|
|
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
|
|
"command_sha256": "a61f3aabb628574832ca64e90b2ed41dbddaec808c9bbb83d1717d68c88b56b4",
|
|
"duplicate_failed_command_blocked": false,
|
|
"block_reason": null,
|
|
"execution_started": true,
|
|
"change_declared": false,
|
|
"result_contract_valid": true,
|
|
"result_contract_status": null,
|
|
"result_contract_error": null,
|
|
"command_rc": 0,
|
|
"changes_made": false,
|
|
"rollback_started": false,
|
|
"rollback_restored": null,
|
|
"mutation_outcome": "NO_MUTATION",
|
|
"sanitized": true,
|
|
"secrets_included": false,
|
|
"private_addresses_included": false,
|
|
"raw_evidence_retained_locally": true,
|
|
"raw_evidence_sha256": "182733d8a7d74c82d8aba6245fc2a7c2c31ca5d9b331aa0f28a8df2dff144946",
|
|
"sanitized_output_sha256": "182733d8a7d74c82d8aba6245fc2a7c2c31ca5d9b331aa0f28a8df2dff144946",
|
|
"output_truncated_in_json": false,
|
|
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
|
"output": "SIGNALBOT046_BEGIN=true\nSIGNALBOT046_MODE=READ_ONLY_COLLECTOR_RESTART_CLEAN_D1_RCA\nSIGNALBOT046_RETUNE_80_20=NO\nSIGNALBOT046_RETUNE_DEMONOV=NO\nSIGNALBOT046_REAL_EXCHANGE_EXECUTION=NO\nSIGNALBOT046_COLLECTOR_RESTART=NO\nREFERENCE_CHECK=OK\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_CHECK=OK\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nAUTHORITY_CHECK_SCOPE=READABILITY_ONLY_INCIDENTS_NOT_CLOSED\nSIGNALBOT046_RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSIGNALBOT046_VM170={\"name\":\"core-apps\",\"node\":\"pve01\",\"ok\":true,\"status\":\"running\",\"type\":\"qemu\"}\nSIGNALBOT046_GIT_CONTEXT=[{\"branch\":\"cr-2026-0116-reliability-finality\",\"dirty\":false,\"exists\":true,\"head\":\"ac7094a76ddc06f376d71a76432af7c2dd10ec7d\",\"ls_remote_rc\":0,\"path\":\"/root/8020-demonov-shadow-bot-cr0116\",\"remote_feature\":\"76e1f58baa544a0e208bba317f0150e5b6d74dbd\"},{\"branch\":\"cr-2026-0105-demonov-shadow-orchestrator\",\"dirty\":false,\"exists\":true,\"head\":\"559948bcb6baead49a47d9953b0cc0090e332fd7\",\"ls_remote_rc\":0,\"path\":\"/root/8020-demonov-shadow-bot\",\"remote_feature\":\"76e1f58baa544a0e208bba317f0150e5b6d74dbd\"}]\nSIGNALBOT046_QM={\"rc\":0,\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":31352}\nSIGNALBOT046_QGA_META={\"err_bytes\":0,\"err_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"exitcode\":0,\"exited\":1,\"out_bytes\":29930}\nSIGNALBOT046_GUEST={\"collector\":{\"env_selected\":{\"CODE_IDENTITY\":\"76e1f58baa544a0e208bba317f0150e5b6d74dbd\",\"COLLECTION_START_AT\":\"2026-09-10T00:00:00+00:00\",\"EXPERIMENT_ID\":\"55588af053b5a7b47bdc86a9231b7fc3a4cdd00b3b53e05e201e69438f480e0f\",\"RUNTIME_MANIFEST_ID\":\"a6a4f903e7e6aa309d8c7d181fbe613fae95f11077dff0fc2ceb7c331fb75389\"},\"id\":\"ddb2d923ad72\",\"image\":\"8020-demonov-shadow:76e1f58baa544a0e208bba317f0150e5b6d74dbd\",\"image_id\":\"sha256:341f71a906c91d3fa9c1d50bd89df2a849dfb0b168bb4551efb444bf61c4ab99\",\"ok\":true,\"restart_count\":561,\"restart_policy\":{\"MaximumRetryCount\":0,\"Name\":\"unless-stopped\"},\"state\":{\"Error\":\"\",\"ExitCode\":0,\"FinishedAt\":\"2026-09-17T12:04:24.012149999Z\",\"OOMKilled\":false,\"Running\":true,\"StartedAt\":\"2026-09-17T12:04:24.211673357Z\"}},\"collector_logs\":{\"168h\":{\"exception_interval_seconds\":{\"count\":887,\"max\":39863.883,\"median\":137.572,\"min\":0.0},\"last_relevant_lines\":[\"2026-09-17T07:47:29.437659348Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:13:48.036120400Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:13:48.036126335Z Traceback (most recent call last):\",\"2026-09-17T08:13:48.036147402Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:13:53.545485537Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:13:53.545505408Z Traceback (most recent call last):\",\"2026-09-17T08:13:53.545756154Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:13:54.166345254Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:17:19.424299177Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:17:19.424304259Z Traceback (most recent call last):\",\"2026-09-17T08:17:19.424320599Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:17:26.429651129Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:17:26.429673577Z Traceback (most recent call last):\",\"2026-09-17T08:17:26.429902510Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:17:27.059693454Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:32:51.439050039Z Traceback (most recent call last):\",\"2026-09-17T08:32:51.439359756Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:32:52.080092656Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:41:58.869207759Z Traceback (most recent call last):\",\"2026-09-17T08:41:58.869443994Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:41:59.469211933Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:55:46.842716641Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:55:46.842747529Z Traceback (most recent call last):\",\"2026-09-17T08:55:46.842755360Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:55:51.144847039Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:55:51.144863534Z Traceback (most recent call last):\",\"2026-09-17T08:55:51.145100293Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:55:51.765255786Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:57:57.045656443Z Traceback (most recent call last):\",\"2026-09-17T08:57:57.045919078Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:57:57.707884690Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T09:29:08.535237993Z Traceback (most recent call last):\",\"2026-09-17T09:29:08.535442751Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T09:29:09.158465479Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T09:39:51.762188516Z Traceback (most recent call last):\",\"2026-09-17T09:39:51.762530387Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T09:39:52.416931374Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T09:46:37.772960015Z TimeoutError: timed out while closing connection\",\"2026-09-17T09:46:37.772965730Z Traceback (most recent call last):\",\"2026-09-17T09:46:37.772982080Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T09:46:41.018961324Z TimeoutError: timed out while closing connection\",\"2026-09-17T09:46:41.018975904Z Traceback (most recent call last):\",\"2026-09-17T09:46:41.019075016Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T09:46:41.620979164Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T11:03:55.542306092Z Traceback (most recent call last):\",\"2026-09-17T11:03:55.542530119Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T11:03:56.176388077Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T11:17:33.035119538Z Traceback (most recent call last):\",\"2026-09-17T11:17:33.035241599Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T11:17:33.698657870Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T12:00:02.285303924Z Traceback (most recent call last):\",\"2026-09-17T12:00:02.285766938Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T12:00:02.890426666Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T12:04:18.112777831Z TimeoutError: timed out while closing connection\",\"2026-09-17T12:04:18.112788046Z Traceback (most recent call last):\",\"2026-09-17T12:04:18.112808335Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T12:04:23.863334496Z TimeoutError: timed out while closing connection\",\"2026-09-17T12:04:23.863365466Z Traceback (most recent call last):\",\"2026-09-17T12:04:23.863484787Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T12:04:24.483887284Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\"],\"line_count\":11350,\"rc\":0,\"signature_counts\":{\"ConnectionResetError: [Errno 104] Connection reset by peer\":12,\"OSError: [Errno 9] Bad file descriptor\":4,\"TimeoutError: _ssl.c:1015: The handshake operation timed out\":14,\"TimeoutError: timed out while closing connection\":212,\"websockets.exceptions.ConnectionClosedError: no close frame received or sent\":26,\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\":620},\"stderr_sha256\":\"feeff3f8ec6b68f089689d8ba85a839a145e35fd6dc043e74382695a2d1396af\",\"terminal_exception_count\":888,\"traceback_count\":676},\"24h\":{\"exception_interval_seconds\":{\"count\":202,\"max\":8837.966,\"median\":7.39,\"min\":0.0},\"last_relevant_lines\":[\"2026-09-17T07:47:29.437659348Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:13:48.036120400Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:13:48.036126335Z Traceback (most recent call last):\",\"2026-09-17T08:13:48.036147402Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:13:53.545485537Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:13:53.545505408Z Traceback (most recent call last):\",\"2026-09-17T08:13:53.545756154Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:13:54.166345254Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:17:19.424299177Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:17:19.424304259Z Traceback (most recent call last):\",\"2026-09-17T08:17:19.424320599Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:17:26.429651129Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:17:26.429673577Z Traceback (most recent call last):\",\"2026-09-17T08:17:26.429902510Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:17:27.059693454Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:32:51.439050039Z Traceback (most recent call last):\",\"2026-09-17T08:32:51.439359756Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:32:52.080092656Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:41:58.869207759Z Traceback (most recent call last):\",\"2026-09-17T08:41:58.869443994Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:41:59.469211933Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:55:46.842716641Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:55:46.842747529Z Traceback (most recent call last):\",\"2026-09-17T08:55:46.842755360Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:55:51.144847039Z TimeoutError: timed out while closing connection\",\"2026-09-17T08:55:51.144863534Z Traceback (most recent call last):\",\"2026-09-17T08:55:51.145100293Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:55:51.765255786Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T08:57:57.045656443Z Traceback (most recent call last):\",\"2026-09-17T08:57:57.045919078Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T08:57:57.707884690Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T09:29:08.535237993Z Traceback (most recent call last):\",\"2026-09-17T09:29:08.535442751Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T09:29:09.158465479Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T09:39:51.762188516Z Traceback (most recent call last):\",\"2026-09-17T09:39:51.762530387Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T09:39:52.416931374Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T09:46:37.772960015Z TimeoutError: timed out while closing connection\",\"2026-09-17T09:46:37.772965730Z Traceback (most recent call last):\",\"2026-09-17T09:46:37.772982080Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T09:46:41.018961324Z TimeoutError: timed out while closing connection\",\"2026-09-17T09:46:41.018975904Z Traceback (most recent call last):\",\"2026-09-17T09:46:41.019075016Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T09:46:41.620979164Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T11:03:55.542306092Z Traceback (most recent call last):\",\"2026-09-17T11:03:55.542530119Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T11:03:56.176388077Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T11:17:33.035119538Z Traceback (most recent call last):\",\"2026-09-17T11:17:33.035241599Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T11:17:33.698657870Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T12:00:02.285303924Z Traceback (most recent call last):\",\"2026-09-17T12:00:02.285766938Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T12:00:02.890426666Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\",\"2026-09-17T12:04:18.112777831Z TimeoutError: timed out while closing connection\",\"2026-09-17T12:04:18.112788046Z Traceback (most recent call last):\",\"2026-09-17T12:04:18.112808335Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T12:04:23.863334496Z TimeoutError: timed out while closing connection\",\"2026-09-17T12:04:23.863365466Z Traceback (most recent call last):\",\"2026-09-17T12:04:23.863484787Z websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\",\"2026-09-17T12:04:24.483887284Z <frozen runpy>:130: RuntimeWarning: 'demonov_forward_map.bitfinex_trade_collector' found in sys.modules after import of package 'demonov_forward_map', but prior to execution of 'demonov_forward_map.bitfinex_trade_collector'; this may result in unpredictable behaviour\"],\"line_count\":2246,\"rc\":0,\"signature_counts\":{\"ConnectionResetError: [Errno 104] Connection reset by peer\":2,\"OSError: [Errno 9] Bad file descriptor\":1,\"TimeoutError: timed out while closing connection\":68,\"websockets.exceptions.ConnectionClosedError: no close frame received or sent\":4,\"websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout; no close frame received\":128},\"stderr_sha256\":\"ee44934ba42545d9c3f3389b515de26e2cbe6f71bfa8e02ba95e4eb4f8be26b9\",\"terminal_exception_count\":203,\"traceback_count\":135}},\"compose_env_contract\":{\"ok\":true,\"selected\":{\"collector\":{\"CODE_IDENTITY\":\"76e1f58baa544a0e208bba317f0150e5b6d74dbd\",\"COLLECTION_START_AT\":\"2026-09-10T00:00:00+00:00\",\"EXPERIMENT_ID\":\"55588af053b5a7b47bdc86a9231b7fc3a4cdd00b3b53e05e201e69438f480e0f\",\"RUNTIME_MANIFEST_ID\":\"a6a4f903e7e6aa309d8c7d181fbe613fae95f11077dff0fc2ceb7c331fb75389\"},\"relay\":{\"CODE_IDENTITY\":\"76e1f58baa544a0e208bba317f0150e5b6d74dbd\",\"EXPERIMENT_ID\":\"55588af053b5a7b47bdc86a9231b7fc3a4cdd00b3b53e05e201e69438f480e0f\",\"RUNTIME_MANIFEST_ID\":\"a6a4f903e7e6aa309d8c7d181fbe613fae95f11077dff0fc2ceb7c331fb75389\"},\"shadow\":{\"CODE_IDENTITY\":\"76e1f58baa544a0e208bba317f0150e5b6d74dbd\",\"EXPERIMENT_ID\":\"55588af053b5a7b47bdc86a9231b7fc3a4cdd00b3b53e05e201e69438f480e0f\",\"RUNTIME_MANIFEST_ID\":\"a6a4f903e7e6aa309d8c7d181fbe613fae95f11077dff0fc2ceb7c331fb75389\"},\"worker\":{\"CODE_IDENTITY\":\"76e1f58baa544a0e208bba317f0150e5b6d74dbd\",\"EXPERIMENT_ID\":\"55588af053b5a7b47bdc86a9231b7fc3a4cdd00b3b53e05e201e69438f480e0f\",\"RUNTIME_MANIFEST_ID\":\"a6a4f903e7e6aa309d8c7d181fbe613fae95f11077dff0fc2ceb7c331fb75389\"}}},\"decision\":\"RCA_EVIDENCE_COLLECTED\",\"docker_events\":{\"die_times_tail\":[],\"event_counts\":{},\"note\":\"DOCKER_EVENT_HISTORY_IS_DAEMON_BUFFER_NOT_AUTHORITATIVE_FOR_FULL_7D\",\"ok\":true},\"expected_epoch\":\"55588af053b5a7b47bdc86a9231b7fc3a4cdd00b3b53e05e201e69438f480e0f\",\"live_db\":{\"counts\":{\"bars\":999,\"notification_state\":0,\"observations\":0,\"processed_logical_d1\":0,\"role_state\":2,\"signal_outbox\":0},\"exists\":true,\"observation_latest\":[],\"outbox_latest\":[],\"path\":\"live_prospective.db\",\"processed_latest\":[],\"quick_check\":\"ok\"},\"mutation_scope\":\"NONE_READ_ONLY\",\"rca_holds\":[\"COLLECTOR_RESTART_COUNT_NONZERO\",\"COLLECTOR_TRACEBACKS_LAST_24H\",\"NO_FINALIZED_D1\",\"NO_PROCESSED_LOGICAL_D1\",\"RECONNECT_EVENTS_SINCE_COLLECTION_START\"],\"relay_db\":{\"count\":0,\"exists\":true,\"latest\":[],\"path\":\"notification_relay.db\",\"quick_check\":\"ok\",\"status_counts\":[]},\"running_source\":{\"bytes\":42087,\"connect_calls\":[{\"close_timeout\":5,\"lineno\":1153,\"open_timeout\":10,\"ping_interval\":20,\"ping_timeout\":60}],\"ok\":true,\"sha256\":\"193dd0c7f9890fdb4026c24dbc5add825f1641d92097e881806705ca452812d5\",\"websockets_version\":\"15.0.1\"},\"schema\":\"signalbot-collector-restart-clean-d1-rca-v1\",\"source_db\":{\"counts\":{\"d1_revisions\":0,\"finalized_d1\":0,\"logical_d1\":0,\"sequence_events\":1258654,\"source_receipts\":1258654,\"trade_receipts\":605714,\"trade_revisions\":605714},\"distinct_sessions\":541,\"event_kind_totals\":[{\"count\":540,\"event_kind\":\"RECONNECT\"},{\"count\":1140040,\"event_kind\":\"SEQUENCE\"}],\"event_time_bounds\":{\"first_at\":\"2026-09-10T00:00:02.188633+00:00\",\"last_at\":\"2026-09-17T12:06:52.123856+00:00\",\"max_event_seq\":1258654},\"events_by_day\":[{\"RECONNECT\":4,\"SEQUENCE\":295888,\"day\":\"2026-09-10\"},{\"RECONNECT\":61,\"SEQUENCE\":159157,\"day\":\"2026-09-11\"},{\"RECONNECT\":57,\"SEQUENCE\":118201,\"day\":\"2026-09-12\"},{\"RECONNECT\":26,\"SEQUENCE\":95757,\"day\":\"2026-09-13\"},{\"RECONNECT\":144,\"SEQUENCE\":157972,\"day\":\"2026-09-14\"},{\"RECONNECT\":115,\"SEQUENCE\":137854,\"day\":\"2026-09-15\"},{\"RECONNECT\":103,\"SEQUENCE\":121537,\"day\":\"2026-09-16\"},{\"RECONNECT\":30,\"SEQUENCE\":53674,\"day\":\"2026-09-17\"}],\"exists\":true,\"finality_state_counts\":[],\"finalized_latest\":[],\"latest_discontinuities\":[{\"event_kind\":\"RECONNECT\",\"event_seq\":1258416,\"received_at\":\"2026-09-17T12:04:24.930627+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1257965,\"received_at\":\"2026-09-17T12:00:03.243290+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1255571,\"received_at\":\"2026-09-17T11:17:34.118706+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1254732,\"received_at\":\"2026-09-17T11:03:56.498939+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1249672,\"received_at\":\"2026-09-17T09:46:41.930402+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1249142,\"received_at\":\"2026-09-17T09:39:52.825977+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1248209,\"received_at\":\"2026-09-17T09:29:09.474554+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1245733,\"received_at\":\"2026-09-17T08:57:58.020295+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1245537,\"received_at\":\"2026-09-17T08:55:52.141573+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1244630,\"received_at\":\"2026-09-17T08:41:59.761500+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1243913,\"received_at\":\"2026-09-17T08:32:52.396179+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1242867,\"received_at\":\"2026-09-17T08:17:27.357535+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1242574,\"received_at\":\"2026-09-17T08:13:54.479336+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1240645,\"received_at\":\"2026-09-17T07:47:29.786175+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1240009,\"received_at\":\"2026-09-17T07:39:02.739596+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1239783,\"received_at\":\"2026-09-17T07:35:55.722019+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1239643,\"received_at\":\"2026-09-17T07:34:26.864866+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1238438,\"received_at\":\"2026-09-17T07:21:40.190699+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1237690,\"received_at\":\"2026-09-17T07:11:55.541803+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1237546,\"received_at\":\"2026-09-17T07:10:31.186291+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1233669,\"received_at\":\"2026-09-17T06:17:18.042490+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1228580,\"received_at\":\"2026-09-17T05:03:12.650068+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1218754,\"received_at\":\"2026-09-17T02:35:53.586905+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1217771,\"received_at\":\"2026-09-17T02:22:56.747351+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1215788,\"received_at\":\"2026-09-17T02:04:15.512319+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1213086,\"received_at\":\"2026-09-17T01:35:11.201054+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1212649,\"received_at\":\"2026-09-17T01:31:11.442572+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1212381,\"received_at\":\"2026-09-17T01:28:48.225894+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1212125,\"received_at\":\"2026-09-17T01:26:31.875962+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1206235,\"received_at\":\"2026-09-17T00:13:17.260092+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1197388,\"received_at\":\"2026-09-16T22:04:56.520321+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1194456,\"received_at\":\"2026-09-16T21:12:40.035746+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1192942,\"received_at\":\"2026-09-16T20:44:20.614279+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1192406,\"received_at\":\"2026-09-16T20:36:40.321230+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1191486,\"received_at\":\"2026-09-16T20:24:40.899370+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1191248,\"received_at\":\"2026-09-16T20:21:50.762985+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1190157,\"received_at\":\"2026-09-16T20:08:32.485209+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1188255,\"received_at\":\"2026-09-16T19:47:17.294083+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1187862,\"received_at\":\"2026-09-16T19:43:17.526019+00:00\",\"source_sequence\":1},{\"event_kind\":\"RECONNECT\",\"event_seq\":1187249,\"received_at\":\"2026-09-16T19:37:29.012473+00:00\",\"source_sequence\":1}],\"logical_latest\":[],\"meta_selected\":[{\"k\":\"collection_start_at\",\"v\":\"2026-09-10T00:00:00+00:00\"},{\"k\":\"active_session_id\",\"v\":\"fddcb98edfca44219dc17de32e8c6664\"},{\"k\":\"reconnect_detected\",\"v\":\"1\"},{\"k\":\"fail_closed_reason\",\"v\":\"source reconnect detected\"}],\"path\":\"bitfinex_source.db\",\"quick_check\":\"ok\",\"receipt_time_bounds\":{\"count\":1140580,\"first_at\":\"2026-09-10T00:00:02.188633+00:00\",\"last_at\":\"2026-09-17T12:06:52.123856+00:00\"}},\"status\":{\"collector\":{\"exists\":true,\"sha256\":\"f9f6f58bc7aa307698435bca1195598a074ef537091ee91eb24c5a246ac85d1b\",\"values\":{\"fail_closed_reason\":\"source reconnect detected\",\"gap_detected\":false,\"post_finality_revision\":false,\"prospective_fail_closed_reason\":\"\",\"prospective_gap_detected\":false,\"prospective_reconnect_detected\":false,\"reconnect_detected\":true,\"updated_at\":\"2026-09-17T12:06:52.129845+00:00\"}},\"live\":{\"exists\":true,\"sha256\":\"6ff334079324f152c7df8010b46f57475576539db9f61e60190cf10dc91e468b\",\"values\":{\"code_identity\":\"76e1f58baa544a0e208bba317f0150e5b6d74dbd\",\"execution_authority\":false,\"experiment_id\":\"55588af053b5a7b47bdc86a9231b7fc3a4cdd00b3b53e05e201e69438f480e0f\",\"finalized_d1_available_count\":0,\"finalized_d1_backlog_count\":0,\"finalized_d1_processed_this_cycle\":0,\"heartbeat_at\":\"2026-09-17T12:05:42.643395+00:00\",\"last_primary_closed_d1\":null,\"production_launch_allowed\":false,\"prospective_research_collection_allowed\":true,\"risk_authority\":false,\"service\":\"8020-demonov-live-worker\",\"source_fail_closed_reason\":\"\",\"source_gap_detected\":false,\"source_reconnect_detected\":false,\"status\":\"AWAITING_PRIMARY_FINALITY\"}},\"relay\":{\"exists\":true,\"sha256\":\"6d3cdc72f29037237391cf3f7c25bfbf758d1dde4babb1b526b213ec1bcbc0ee\",\"values\":{\"execution_authority\":false,\"heartbeat_at\":\"2026-09-17T12:06:02.547454+00:00\",\"production_launch_allowed\":false,\"risk_authority\":false,\"service\":\"8020-demonov-notification-relay\",\"status\":\"NOTIFICATION_RELAY_RUNNING\"}}}}\nSIGNALBOT046_RCA_HOLDS=[\"COLLECTOR_RESTART_COUNT_NONZERO\",\"COLLECTOR_TRACEBACKS_LAST_24H\",\"NO_FINALIZED_D1\",\"NO_PROCESSED_LOGICAL_D1\",\"RECONNECT_EVENTS_SINCE_COLLECTION_START\"]\nSIGNALBOT046_DECISION=RCA_EVIDENCE_COLLECTED\nSIGNALBOT046_MUTATION_SCOPE=NONE_READ_ONLY\nSIGNALBOT046_NEXT_GATE=ANALYZE_IMMUTABLE_046_NO_AUTOMATIC_REPAIR\nSIGNALBOT046_END=true\n"
|
|
}
|