homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-09-21 09:10:59 +00:00
parent b217d66f55
commit 04505ebfd3
4 changed files with 89 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1.json
+38
View File
@@ -0,0 +1,38 @@
{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "read-only",
"component": "signalbot-final-hold-rca",
"started_at_utc": "2026-09-21T09:10:56Z",
"finished_at_utc": "2026-09-21T09:10:58Z",
"reference_register_checked": true,
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
"error_register_checked": true,
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
"command_sha256": "ce652581521352689a86613bb7462e51a3505b95ab3aae5b1a650ed6465bfb90",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
"change_declared": false,
"result_contract_valid": true,
"result_contract_status": null,
"result_contract_error": null,
"command_rc": 0,
"changes_made": false,
"rollback_started": false,
"rollback_restored": null,
"mutation_outcome": "NO_MUTATION",
"sanitized": true,
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "bc8169c6ed3b2790278fcf64192893e9662de210b9d5fe4818963a69bb0e8dd7",
"sanitized_output_sha256": "bc8169c6ed3b2790278fcf64192893e9662de210b9d5fe4818963a69bb0e8dd7",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "SIGNALBOT052FINALRCA1_RESULT={\"outer_rc\":0,\"guest_exitcode\":0,\"rca\":{\"containers\":[{\"name\":\"demonov-shadow-8020-collector-1\",\"id\":\"8a65d323beda0410f180baf1e11144aaf53bf00f3329d758623bb711c1959f1f\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false},{\"name\":\"demonov-shadow-8020-worker-1\",\"id\":\"8677c7c7388e1f023ddc214a703de207085d21ceebca7565f673dd80b93c414d\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false},{\"name\":\"demonov-shadow-8020-relay-1\",\"id\":\"8c7689a8e46e72f94747f002af43d3a3b68c75af81f92d9287f0828c60738e6a\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false},{\"name\":\"demonov-shadow-8020-shadow-1\",\"id\":\"80ab5a8cad2b0eb99c4d0b86d66de1a673e4e40ff1fb7ac52b444d5a657bddb3\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false}],\"candidate_actual_id\":\"sha256:525cf1e50b232943e82bcd76c8cc2ea5f297bf7319db49bf6fd3470d60bfe555\",\"quick_check\":\"ok\",\"reconciliation_counts\":{\"RECONCILED\":20,\"UNRESOLVED\":1},\"unresolved\":[{\"window_id\":\"5ceb35ba092d9fff7a7520784ddf1b087441d7855b2d24e4673d8c32c282caaf\",\"reconnect_event_seq\":513728,\"reconnect_received_at\":\"2026-09-21T08:48:58.777306+00:00\",\"previous_trade_identity\":\"bb18a73cdc0b5831145ca37c469476c25e641e6c40b000489753104562b5fc74\",\"previous_trade_mts\":\"2026-09-21T08:48:56.940000+00:00\",\"next_trade_identity\":\"a543b607170f909e7f008d809f8f778972b2ae6a58aa1ad3cdd66af92943c9ec\",\"next_trade_mts\":\"2026-09-21T08:48:59.795000+00:00\",\"status\":\"UNRESOLVED\",\"evidence_sha256\":\"0933e981b764b89edd58396a737136747668a903a239eac64d538d5f4f2d5391\",\"recovered_row_count\":0,\"failure_reason\":\"REST_FETCH_OR_VALIDATION_FAILED\",\"resolved_at\":\"2026-09-21T08:48:59.886028+00:00\"}],\"accepted\":[\"bb9cbf18d81e605641e7aa9b792d35d1f55c3cae51fbe1abfe1617b72f0eaa14\",\"BTC|BITFINEX|SPOT\",\"D1\",\"2026-09-18T00:00:00+00:00\",\"2026-09-19T00:00:00+00:00\",\"FINALIZED\",\"94755f8715fe97b4fde2d846638af84ea4a829625bb0ca6ece98a78803416b2e\",\"33cf3eb2b33466a5bb2c2779602fb3119cb00e628759f941456df554e31cb774\"],\"revision_count\":1,\"sequence_events\":521614,\"collector_status.json\":{\"mtime\":1789981854.9718356,\"age_sec\":2.559959888458252,\"filtered_paths\":[[\"post_finality_revision\",false],[\"prospective_fail_closed_reason\",\"REST_FETCH_OR_VALIDATION_FAILED\"],[\"prospective_gap_detected\",false],[\"prospective_reconnect_detected\",true],[\"reconciliation_last_reason\",\"REST_FETCH_OR_VALIDATION_FAILED\"],[\"reconciliation_pending_count\",0],[\"reconciliation_unresolved_count\",1],[\"updated_at\",\"2026-09-21T09:10:54.977508+00:00\"]]},\"live_status.json\":{\"mtime\":1789981842.9316573,\"age_sec\":14.600192785263062,\"filtered_paths\":[[\"execution_authority\",false],[\"finalized_d1_available_count\",0],[\"finalized_d1_backlog_count\",0],[\"finalized_d1_processed_this_cycle\",0],[\"heartbeat_at\",\"2026-09-21T09:10:42.936294+00:00\"],[\"last_primary_closed_d1\",null],[\"production_launch_allowed\",false],[\"prospective_research_collection_allowed\",true],[\"prospective_row_count\",4],[\"risk_authority\",false],[\"source_post_finality_revision\",false],[\"status\",\"FAIL_CLOSED\"]]},\"live_matches\":[[\"observations\",2],[\"processed_logical_d1\",1]],\"expected\":{\"active_tag\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"active_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"candidate_id\":\"sha256:525cf1e50b232943e82bcd76c8cc2ea5f297bf7319db49bf6fd3470d60bfe555\",\"accepted\":[\"bb9cbf18d81e605641e7aa9b792d35d1f55c3cae51fbe1abfe1617b72f0eaa14\",\"BTC|BITFINEX|SPOT\",\"D1\",\"2026-09-18T00:00:00+00:00\",\"2026-09-19T00:00:00+00:00\",\"FINALIZED\",\"94755f8715fe97b4fde2d846638af84ea4a829625bb0ca6ece98a78803416b2e\",\"33cf3eb2b33466a5bb2c2779602fb3119cb00e628759f941456df554e31cb774\"]}}}\nTASK_RESULT=PASS_SIGNALBOT_052FINALRCA1_BLOCKERS_CAPTURED\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n"
}
runtime/history/SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1.txt
+35
View File
@@ -0,0 +1,35 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=signalbot-final-hold-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=ce652581521352689a86613bb7462e51a3505b95ab3aae5b1a650ed6465bfb90
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=bc8169c6ed3b2790278fcf64192893e9662de210b9d5fe4818963a69bb0e8dd7
SANITIZED_OUTPUT_SHA256=bc8169c6ed3b2790278fcf64192893e9662de210b9d5fe4818963a69bb0e8dd7
OUTPUT_BEGIN
SIGNALBOT052FINALRCA1_RESULT={"outer_rc":0,"guest_exitcode":0,"rca":{"containers":[{"name":"demonov-shadow-8020-collector-1","id":"8a65d323beda0410f180baf1e11144aaf53bf00f3329d758623bb711c1959f1f","image_id":"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60","config_image":"8020-demonov-shadow:signalbot-050r6-749073ae0f89","restart":0,"running":true,"paused":false,"restarting":false},{"name":"demonov-shadow-8020-worker-1","id":"8677c7c7388e1f023ddc214a703de207085d21ceebca7565f673dd80b93c414d","image_id":"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60","config_image":"8020-demonov-shadow:signalbot-050r6-749073ae0f89","restart":0,"running":true,"paused":false,"restarting":false},{"name":"demonov-shadow-8020-relay-1","id":"8c7689a8e46e72f94747f002af43d3a3b68c75af81f92d9287f0828c60738e6a","image_id":"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60","config_image":"8020-demonov-shadow:signalbot-050r6-749073ae0f89","restart":0,"running":true,"paused":false,"restarting":false},{"name":"demonov-shadow-8020-shadow-1","id":"80ab5a8cad2b0eb99c4d0b86d66de1a673e4e40ff1fb7ac52b444d5a657bddb3","image_id":"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60","config_image":"8020-demonov-shadow:signalbot-050r6-749073ae0f89","restart":0,"running":true,"paused":false,"restarting":false}],"candidate_actual_id":"sha256:525cf1e50b232943e82bcd76c8cc2ea5f297bf7319db49bf6fd3470d60bfe555","quick_check":"ok","reconciliation_counts":{"RECONCILED":20,"UNRESOLVED":1},"unresolved":[{"window_id":"5ceb35ba092d9fff7a7520784ddf1b087441d7855b2d24e4673d8c32c282caaf","reconnect_event_seq":513728,"reconnect_received_at":"2026-09-21T08:48:58.777306+00:00","previous_trade_identity":"bb18a73cdc0b5831145ca37c469476c25e641e6c40b000489753104562b5fc74","previous_trade_mts":"2026-09-21T08:48:56.940000+00:00","next_trade_identity":"a543b607170f909e7f008d809f8f778972b2ae6a58aa1ad3cdd66af92943c9ec","next_trade_mts":"2026-09-21T08:48:59.795000+00:00","status":"UNRESOLVED","evidence_sha256":"0933e981b764b89edd58396a737136747668a903a239eac64d538d5f4f2d5391","recovered_row_count":0,"failure_reason":"REST_FETCH_OR_VALIDATION_FAILED","resolved_at":"2026-09-21T08:48:59.886028+00:00"}],"accepted":["bb9cbf18d81e605641e7aa9b792d35d1f55c3cae51fbe1abfe1617b72f0eaa14","BTC|BITFINEX|SPOT","D1","2026-09-18T00:00:00+00:00","2026-09-19T00:00:00+00:00","FINALIZED","94755f8715fe97b4fde2d846638af84ea4a829625bb0ca6ece98a78803416b2e","33cf3eb2b33466a5bb2c2779602fb3119cb00e628759f941456df554e31cb774"],"revision_count":1,"sequence_events":521614,"collector_status.json":{"mtime":1789981854.9718356,"age_sec":2.559959888458252,"filtered_paths":[["post_finality_revision",false],["prospective_fail_closed_reason","REST_FETCH_OR_VALIDATION_FAILED"],["prospective_gap_detected",false],["prospective_reconnect_detected",true],["reconciliation_last_reason","REST_FETCH_OR_VALIDATION_FAILED"],["reconciliation_pending_count",0],["reconciliation_unresolved_count",1],["updated_at","2026-09-21T09:10:54.977508+00:00"]]},"live_status.json":{"mtime":1789981842.9316573,"age_sec":14.600192785263062,"filtered_paths":[["execution_authority",false],["finalized_d1_available_count",0],["finalized_d1_backlog_count",0],["finalized_d1_processed_this_cycle",0],["heartbeat_at","2026-09-21T09:10:42.936294+00:00"],["last_primary_closed_d1",null],["production_launch_allowed",false],["prospective_research_collection_allowed",true],["prospective_row_count",4],["risk_authority",false],["source_post_finality_revision",false],["status","FAIL_CLOSED"]]},"live_matches":[["observations",2],["processed_logical_d1",1]],"expected":{"active_tag":"8020-demonov-shadow:signalbot-050r6-749073ae0f89","active_id":"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60","candidate_id":"sha256:525cf1e50b232943e82bcd76c8cc2ea5f297bf7319db49bf6fd3470d60bfe555","accepted":["bb9cbf18d81e605641e7aa9b792d35d1f55c3cae51fbe1abfe1617b72f0eaa14","BTC|BITFINEX|SPOT","D1","2026-09-18T00:00:00+00:00","2026-09-19T00:00:00+00:00","FINALIZED","94755f8715fe97b4fde2d846638af84ea4a829625bb0ca6ece98a78803416b2e","33cf3eb2b33466a5bb2c2779602fb3119cb00e628759f941456df554e31cb774"]}}}
TASK_RESULT=PASS_SIGNALBOT_052FINALRCA1_BLOCKERS_CAPTURED
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+8 -8
View File
@@ -1,19 +1,19 @@
{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1",
"command_id": "SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "read-only",
"component": "homelab-backup-busy-rca",
"started_at_utc": "2026-09-21T09:07:14Z",
"finished_at_utc": "2026-09-21T09:07:14Z",
"component": "signalbot-final-hold-rca",
"started_at_utc": "2026-09-21T09:10:56Z",
"finished_at_utc": "2026-09-21T09:10:58Z",
"reference_register_checked": true,
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
"error_register_checked": true,
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
"command_sha256": "1d13fe3e140bfc2bc4a6c4003134cad2f1041197c7c8ddf2403689b1cffa1391",
"command_sha256": "ce652581521352689a86613bb7462e51a3505b95ab3aae5b1a650ed6465bfb90",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
@@ -30,9 +30,9 @@
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "96264514854b5aa0f18be36121dab5745543b61e700d11c7aec4d597410b6a94",
"sanitized_output_sha256": "96264514854b5aa0f18be36121dab5745543b61e700d11c7aec4d597410b6a94",
"raw_evidence_sha256": "bc8169c6ed3b2790278fcf64192893e9662de210b9d5fe4818963a69bb0e8dd7",
"sanitized_output_sha256": "bc8169c6ed3b2790278fcf64192893e9662de210b9d5fe4818963a69bb0e8dd7",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "WORKERS2_BACKUP_BUSY_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1\nMODE=read-only\nCOMPONENT=homelab-backup-busy-rca\nMUTATION_BOUNDARY=READ_ONLY_NO_SERVICE_STOP_NO_TIMER_STOP_NO_LOCK_BREAK_NO_CONFIG_CHANGE_NO_BACKUP_NO_GC_NO_GIT_WRITE\nSOURCE_BASELINE={\"clean\":true,\"head\":\"4e59d5afaa670899c56f87a84bc887175f299614\",\"remote_branch_sha\":null}\nBACKUP_SERVICE={\"props\":{\"ActiveEnterTimestamp\":\"\",\"ActiveState\":\"activating\",\"ExecMainCode\":\"0\",\"ExecMainExitTimestamp\":\"\",\"ExecMainPID\":\"3117776\",\"ExecMainStartTimestamp\":\"Mon 2026-09-21 10:42:58 MSK\",\"ExecMainStatus\":\"0\",\"FragmentPath\":\"/etc/systemd/system/homelab-backup-v2.service\",\"InactiveEnterTimestamp\":\"Mon 2026-09-21 10:11:59 MSK\",\"MainPID\":\"3117776\",\"RemainAfterExit\":\"no\",\"Result\":\"success\",\"SubState\":\"start\",\"Type\":\"oneshot\"},\"rc\":0}\nBACKUP_TIMER={\"props\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-backup-v2.timer\",\"LastTriggerUSec\":\"Mon 2026-09-21 10:42:58 MSK\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"running\",\"Triggers\":\"homelab-backup-v2.service\",\"Unit\":\"homelab-backup-v2.service\"},\"rc\":0}\nSERVICE_UNIT_RELEVANT=[\"Type=oneshot\",\"ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\"]\nTIMER_UNIT_RELEVANT=[\"OnUnitActiveSec=30min\",\"RandomizedDelaySec=10min\"]\nHEAVY_LOCK={\"exists\":true,\"flock_rc\":1,\"free\":false,\"fuser_pids\":[3117776],\"lslocks\":[{\"command\":\"homelab-backup-\",\"mode\":\"WRITE\",\"path\":\"/run/homelab-backup-heavy.lock\",\"pid\":3117776,\"type\":\"FLOCK\"}]}\nHOLDER_PROCESSES=[{\"cmd_hash\":\"cd76277a7385c9f3\",\"pid\":3117776,\"ps\":\"3117776 1 5055 SNs homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\",\"state\":\"S\"}]\nBACKUP_RELATED_PROCESSES=[\"3117776 1 5055 SNs homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\"]\nSERVICE_JOURNAL_1076_WINDOW=[]\nSERVICE_JOURNAL_RECENT_TAIL=[\"2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2339216]: SCHEDULE_TARGET=xf-dsmods\",\"2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2339216]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 1min 18.429s CPU time, 713.4M memory peak.\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\",\"2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2759410]: SELFTEST_REPLACEMENT486=PASS\",\"2026-09-21T09:41:22+03:00 pve01 homelab-backup-v2[2759410]: SCHEDULE_TARGET=xf-kingofwolk\",\"2026-09-21T09:41:22+03:00 pve01 homelab-backup-v2[2759410]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T09:41:22+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\",\"2026-09-21T09:41:22+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-21T09:41:22+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 11.004s CPU time, 143.6M memory peak.\",\"2026-09-21T10:04:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\",\"2026-09-21T10:04:32+03:00 pve01 homelab-backup-v2[2947207]: SELFTEST_REPLACEMENT486=PASS\",\"2026-09-21T10:11:59+03:00 pve01 homelab-backup-v2[2947207]: SCHEDULE_TARGET=xf-newfi\",\"2026-09-21T10:11:59+03:00 pve01 homelab-backup-v2[2947207]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T10:11:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\",\"2026-09-21T10:11:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-21T10:11:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 1min 35.661s CPU time, 162.2M memory peak.\",\"2026-09-21T10:42:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\",\"2026-09-21T10:42:59+03:00 pve01 homelab-backup-v2[3117776]: SELFTEST_REPLACEMENT486=PASS\"]\nRCA_CLASSIFICATION={\"fix\":\"IDENTIFY_OWNER;STOP_TIMER_FIRST_THEN_WAIT_FOR_OWNER_TO_EXIT;DO_NOT_BREAK_LOCK\",\"root_cause\":\"HEAVY_LOCK_HELD_BY_LIVE_PROCESS\"}\nNO_MUTATION=true\nNO_SERVICE_STOP=true\nNO_TIMER_STOP=true\nNO_LOCK_BREAK=true\nNO_BACKUP_STARTED=true\nNO_RETENTION_STARTED=true\nTASK_RESULT=PASS_HOMELAB_BACKUP_BUSY_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_BUSY_RCA_END=1\n"
"output": "SIGNALBOT052FINALRCA1_RESULT={\"outer_rc\":0,\"guest_exitcode\":0,\"rca\":{\"containers\":[{\"name\":\"demonov-shadow-8020-collector-1\",\"id\":\"8a65d323beda0410f180baf1e11144aaf53bf00f3329d758623bb711c1959f1f\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false},{\"name\":\"demonov-shadow-8020-worker-1\",\"id\":\"8677c7c7388e1f023ddc214a703de207085d21ceebca7565f673dd80b93c414d\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false},{\"name\":\"demonov-shadow-8020-relay-1\",\"id\":\"8c7689a8e46e72f94747f002af43d3a3b68c75af81f92d9287f0828c60738e6a\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false},{\"name\":\"demonov-shadow-8020-shadow-1\",\"id\":\"80ab5a8cad2b0eb99c4d0b86d66de1a673e4e40ff1fb7ac52b444d5a657bddb3\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false}],\"candidate_actual_id\":\"sha256:525cf1e50b232943e82bcd76c8cc2ea5f297bf7319db49bf6fd3470d60bfe555\",\"quick_check\":\"ok\",\"reconciliation_counts\":{\"RECONCILED\":20,\"UNRESOLVED\":1},\"unresolved\":[{\"window_id\":\"5ceb35ba092d9fff7a7520784ddf1b087441d7855b2d24e4673d8c32c282caaf\",\"reconnect_event_seq\":513728,\"reconnect_received_at\":\"2026-09-21T08:48:58.777306+00:00\",\"previous_trade_identity\":\"bb18a73cdc0b5831145ca37c469476c25e641e6c40b000489753104562b5fc74\",\"previous_trade_mts\":\"2026-09-21T08:48:56.940000+00:00\",\"next_trade_identity\":\"a543b607170f909e7f008d809f8f778972b2ae6a58aa1ad3cdd66af92943c9ec\",\"next_trade_mts\":\"2026-09-21T08:48:59.795000+00:00\",\"status\":\"UNRESOLVED\",\"evidence_sha256\":\"0933e981b764b89edd58396a737136747668a903a239eac64d538d5f4f2d5391\",\"recovered_row_count\":0,\"failure_reason\":\"REST_FETCH_OR_VALIDATION_FAILED\",\"resolved_at\":\"2026-09-21T08:48:59.886028+00:00\"}],\"accepted\":[\"bb9cbf18d81e605641e7aa9b792d35d1f55c3cae51fbe1abfe1617b72f0eaa14\",\"BTC|BITFINEX|SPOT\",\"D1\",\"2026-09-18T00:00:00+00:00\",\"2026-09-19T00:00:00+00:00\",\"FINALIZED\",\"94755f8715fe97b4fde2d846638af84ea4a829625bb0ca6ece98a78803416b2e\",\"33cf3eb2b33466a5bb2c2779602fb3119cb00e628759f941456df554e31cb774\"],\"revision_count\":1,\"sequence_events\":521614,\"collector_status.json\":{\"mtime\":1789981854.9718356,\"age_sec\":2.559959888458252,\"filtered_paths\":[[\"post_finality_revision\",false],[\"prospective_fail_closed_reason\",\"REST_FETCH_OR_VALIDATION_FAILED\"],[\"prospective_gap_detected\",false],[\"prospective_reconnect_detected\",true],[\"reconciliation_last_reason\",\"REST_FETCH_OR_VALIDATION_FAILED\"],[\"reconciliation_pending_count\",0],[\"reconciliation_unresolved_count\",1],[\"updated_at\",\"2026-09-21T09:10:54.977508+00:00\"]]},\"live_status.json\":{\"mtime\":1789981842.9316573,\"age_sec\":14.600192785263062,\"filtered_paths\":[[\"execution_authority\",false],[\"finalized_d1_available_count\",0],[\"finalized_d1_backlog_count\",0],[\"finalized_d1_processed_this_cycle\",0],[\"heartbeat_at\",\"2026-09-21T09:10:42.936294+00:00\"],[\"last_primary_closed_d1\",null],[\"production_launch_allowed\",false],[\"prospective_research_collection_allowed\",true],[\"prospective_row_count\",4],[\"risk_authority\",false],[\"source_post_finality_revision\",false],[\"status\",\"FAIL_CLOSED\"]]},\"live_matches\":[[\"observations\",2],[\"processed_logical_d1\",1]],\"expected\":{\"active_tag\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"active_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"candidate_id\":\"sha256:525cf1e50b232943e82bcd76c8cc2ea5f297bf7319db49bf6fd3470d60bfe555\",\"accepted\":[\"bb9cbf18d81e605641e7aa9b792d35d1f55c3cae51fbe1abfe1617b72f0eaa14\",\"BTC|BITFINEX|SPOT\",\"D1\",\"2026-09-18T00:00:00+00:00\",\"2026-09-19T00:00:00+00:00\",\"FINALIZED\",\"94755f8715fe97b4fde2d846638af84ea4a829625bb0ca6ece98a78803416b2e\",\"33cf3eb2b33466a5bb2c2779602fb3119cb00e628759f941456df554e31cb774\"]}}}\nTASK_RESULT=PASS_SIGNALBOT_052FINALRCA1_BLOCKERS_CAPTURED\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n"
}
runtime/latest.txt
+8 -30
View File
@@ -1,15 +1,15 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1
COMMAND_ID=SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=homelab-backup-busy-rca
COMPONENT=signalbot-final-hold-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=1d13fe3e140bfc2bc4a6c4003134cad2f1041197c7c8ddf2403689b1cffa1391
COMMAND_SHA256=ce652581521352689a86613bb7462e51a3505b95ab3aae5b1a650ed6465bfb90
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
@@ -24,34 +24,12 @@ MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=96264514854b5aa0f18be36121dab5745543b61e700d11c7aec4d597410b6a94
SANITIZED_OUTPUT_SHA256=96264514854b5aa0f18be36121dab5745543b61e700d11c7aec4d597410b6a94
RAW_EVIDENCE_SHA256=bc8169c6ed3b2790278fcf64192893e9662de210b9d5fe4818963a69bb0e8dd7
SANITIZED_OUTPUT_SHA256=bc8169c6ed3b2790278fcf64192893e9662de210b9d5fe4818963a69bb0e8dd7
OUTPUT_BEGIN
WORKERS2_BACKUP_BUSY_RCA_BEGIN=1
COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1
MODE=read-only
COMPONENT=homelab-backup-busy-rca
MUTATION_BOUNDARY=READ_ONLY_NO_SERVICE_STOP_NO_TIMER_STOP_NO_LOCK_BREAK_NO_CONFIG_CHANGE_NO_BACKUP_NO_GC_NO_GIT_WRITE
SOURCE_BASELINE={"clean":true,"head":"4e59d5afaa670899c56f87a84bc887175f299614","remote_branch_sha":null}
BACKUP_SERVICE={"props":{"ActiveEnterTimestamp":"","ActiveState":"activating","ExecMainCode":"0","ExecMainExitTimestamp":"","ExecMainPID":"3117776","ExecMainStartTimestamp":"Mon 2026-09-21 10:42:58 MSK","ExecMainStatus":"0","FragmentPath":"/etc/systemd/system/homelab-backup-v2.service","InactiveEnterTimestamp":"Mon 2026-09-21 10:11:59 MSK","MainPID":"3117776","RemainAfterExit":"no","Result":"success","SubState":"start","Type":"oneshot"},"rc":0}
BACKUP_TIMER={"props":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-backup-v2.timer","LastTriggerUSec":"Mon 2026-09-21 10:42:58 MSK","NextElapseUSecRealtime":"","Result":"success","SubState":"running","Triggers":"homelab-backup-v2.service","Unit":"homelab-backup-v2.service"},"rc":0}
SERVICE_UNIT_RELEVANT=["Type=oneshot","ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled"]
TIMER_UNIT_RELEVANT=["OnUnitActiveSec=30min","RandomizedDelaySec=10min"]
HEAVY_LOCK={"exists":true,"flock_rc":1,"free":false,"fuser_pids":[3117776],"lslocks":[{"command":"homelab-backup-","mode":"WRITE","path":"/run/homelab-backup-heavy.lock","pid":3117776,"type":"FLOCK"}]}
HOLDER_PROCESSES=[{"cmd_hash":"cd76277a7385c9f3","pid":3117776,"ps":"3117776 1 5055 SNs homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled","state":"S"}]
BACKUP_RELATED_PROCESSES=["3117776 1 5055 SNs homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled"]
SERVICE_JOURNAL_1076_WINDOW=[]
SERVICE_JOURNAL_RECENT_TAIL=["2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2339216]: SCHEDULE_TARGET=xf-dsmods","2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2339216]: SCHEDULE_RESULT=SUCCESS","2026-09-21T09:31:02+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.","2026-09-21T09:31:02+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-21T09:31:02+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 1min 18.429s CPU time, 713.4M memory peak.","2026-09-21T09:31:02+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...","2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2759410]: SELFTEST_REPLACEMENT486=PASS","2026-09-21T09:41:22+03:00 pve01 homelab-backup-v2[2759410]: SCHEDULE_TARGET=xf-kingofwolk","2026-09-21T09:41:22+03:00 pve01 homelab-backup-v2[2759410]: SCHEDULE_RESULT=SUCCESS","2026-09-21T09:41:22+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.","2026-09-21T09:41:22+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-21T09:41:22+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 11.004s CPU time, 143.6M memory peak.","2026-09-21T10:04:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...","2026-09-21T10:04:32+03:00 pve01 homelab-backup-v2[2947207]: SELFTEST_REPLACEMENT486=PASS","2026-09-21T10:11:59+03:00 pve01 homelab-backup-v2[2947207]: SCHEDULE_TARGET=xf-newfi","2026-09-21T10:11:59+03:00 pve01 homelab-backup-v2[2947207]: SCHEDULE_RESULT=SUCCESS","2026-09-21T10:11:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.","2026-09-21T10:11:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-21T10:11:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 1min 35.661s CPU time, 162.2M memory peak.","2026-09-21T10:42:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...","2026-09-21T10:42:59+03:00 pve01 homelab-backup-v2[3117776]: SELFTEST_REPLACEMENT486=PASS"]
RCA_CLASSIFICATION={"fix":"IDENTIFY_OWNER;STOP_TIMER_FIRST_THEN_WAIT_FOR_OWNER_TO_EXIT;DO_NOT_BREAK_LOCK","root_cause":"HEAVY_LOCK_HELD_BY_LIVE_PROCESS"}
NO_MUTATION=true
NO_SERVICE_STOP=true
NO_TIMER_STOP=true
NO_LOCK_BREAK=true
NO_BACKUP_STARTED=true
NO_RETENTION_STARTED=true
TASK_RESULT=PASS_HOMELAB_BACKUP_BUSY_RCA_READONLY
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
WORKERS2_BACKUP_BUSY_RCA_END=1
SIGNALBOT052FINALRCA1_RESULT={"outer_rc":0,"guest_exitcode":0,"rca":{"containers":[{"name":"demonov-shadow-8020-collector-1","id":"8a65d323beda0410f180baf1e11144aaf53bf00f3329d758623bb711c1959f1f","image_id":"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60","config_image":"8020-demonov-shadow:signalbot-050r6-749073ae0f89","restart":0,"running":true,"paused":false,"restarting":false},{"name":"demonov-shadow-8020-worker-1","id":"8677c7c7388e1f023ddc214a703de207085d21ceebca7565f673dd80b93c414d","image_id":"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60","config_image":"8020-demonov-shadow:signalbot-050r6-749073ae0f89","restart":0,"running":true,"paused":false,"restarting":false},{"name":"demonov-shadow-8020-relay-1","id":"8c7689a8e46e72f94747f002af43d3a3b68c75af81f92d9287f0828c60738e6a","image_id":"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60","config_image":"8020-demonov-shadow:signalbot-050r6-749073ae0f89","restart":0,"running":true,"paused":false,"restarting":false},{"name":"demonov-shadow-8020-shadow-1","id":"80ab5a8cad2b0eb99c4d0b86d66de1a673e4e40ff1fb7ac52b444d5a657bddb3","image_id":"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60","config_image":"8020-demonov-shadow:signalbot-050r6-749073ae0f89","restart":0,"running":true,"paused":false,"restarting":false}],"candidate_actual_id":"sha256:525cf1e50b232943e82bcd76c8cc2ea5f297bf7319db49bf6fd3470d60bfe555","quick_check":"ok","reconciliation_counts":{"RECONCILED":20,"UNRESOLVED":1},"unresolved":[{"window_id":"5ceb35ba092d9fff7a7520784ddf1b087441d7855b2d24e4673d8c32c282caaf","reconnect_event_seq":513728,"reconnect_received_at":"2026-09-21T08:48:58.777306+00:00","previous_trade_identity":"bb18a73cdc0b5831145ca37c469476c25e641e6c40b000489753104562b5fc74","previous_trade_mts":"2026-09-21T08:48:56.940000+00:00","next_trade_identity":"a543b607170f909e7f008d809f8f778972b2ae6a58aa1ad3cdd66af92943c9ec","next_trade_mts":"2026-09-21T08:48:59.795000+00:00","status":"UNRESOLVED","evidence_sha256":"0933e981b764b89edd58396a737136747668a903a239eac64d538d5f4f2d5391","recovered_row_count":0,"failure_reason":"REST_FETCH_OR_VALIDATION_FAILED","resolved_at":"2026-09-21T08:48:59.886028+00:00"}],"accepted":["bb9cbf18d81e605641e7aa9b792d35d1f55c3cae51fbe1abfe1617b72f0eaa14","BTC|BITFINEX|SPOT","D1","2026-09-18T00:00:00+00:00","2026-09-19T00:00:00+00:00","FINALIZED","94755f8715fe97b4fde2d846638af84ea4a829625bb0ca6ece98a78803416b2e","33cf3eb2b33466a5bb2c2779602fb3119cb00e628759f941456df554e31cb774"],"revision_count":1,"sequence_events":521614,"collector_status.json":{"mtime":1789981854.9718356,"age_sec":2.559959888458252,"filtered_paths":[["post_finality_revision",false],["prospective_fail_closed_reason","REST_FETCH_OR_VALIDATION_FAILED"],["prospective_gap_detected",false],["prospective_reconnect_detected",true],["reconciliation_last_reason","REST_FETCH_OR_VALIDATION_FAILED"],["reconciliation_pending_count",0],["reconciliation_unresolved_count",1],["updated_at","2026-09-21T09:10:54.977508+00:00"]]},"live_status.json":{"mtime":1789981842.9316573,"age_sec":14.600192785263062,"filtered_paths":[["execution_authority",false],["finalized_d1_available_count",0],["finalized_d1_backlog_count",0],["finalized_d1_processed_this_cycle",0],["heartbeat_at","2026-09-21T09:10:42.936294+00:00"],["last_primary_closed_d1",null],["production_launch_allowed",false],["prospective_research_collection_allowed",true],["prospective_row_count",4],["risk_authority",false],["source_post_finality_revision",false],["status","FAIL_CLOSED"]]},"live_matches":[["observations",2],["processed_logical_d1",1]],"expected":{"active_tag":"8020-demonov-shadow:signalbot-050r6-749073ae0f89","active_id":"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60","candidate_id":"sha256:525cf1e50b232943e82bcd76c8cc2ea5f297bf7319db49bf6fd3470d60bfe555","accepted":["bb9cbf18d81e605641e7aa9b792d35d1f55c3cae51fbe1abfe1617b72f0eaa14","BTC|BITFINEX|SPOT","D1","2026-09-18T00:00:00+00:00","2026-09-19T00:00:00+00:00","FINALIZED","94755f8715fe97b4fde2d846638af84ea4a829625bb0ca6ece98a78803416b2e","33cf3eb2b33466a5bb2c2779602fb3119cb00e628759f941456df554e31cb774"]}}}
TASK_RESULT=PASS_SIGNALBOT_052FINALRCA1_BLOCKERS_CAPTURED
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END