homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-09-21 09:10:59 +00:00
parent b217d66f55
commit 04505ebfd3
4 changed files with 89 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/latest.json
+8 -8
View File
@@ -1,19 +1,19 @@
{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1",
"command_id": "SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "read-only",
"component": "homelab-backup-busy-rca",
"started_at_utc": "2026-09-21T09:07:14Z",
"finished_at_utc": "2026-09-21T09:07:14Z",
"component": "signalbot-final-hold-rca",
"started_at_utc": "2026-09-21T09:10:56Z",
"finished_at_utc": "2026-09-21T09:10:58Z",
"reference_register_checked": true,
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
"error_register_checked": true,
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
"command_sha256": "1d13fe3e140bfc2bc4a6c4003134cad2f1041197c7c8ddf2403689b1cffa1391",
"command_sha256": "ce652581521352689a86613bb7462e51a3505b95ab3aae5b1a650ed6465bfb90",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
@@ -30,9 +30,9 @@
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "96264514854b5aa0f18be36121dab5745543b61e700d11c7aec4d597410b6a94",
"sanitized_output_sha256": "96264514854b5aa0f18be36121dab5745543b61e700d11c7aec4d597410b6a94",
"raw_evidence_sha256": "bc8169c6ed3b2790278fcf64192893e9662de210b9d5fe4818963a69bb0e8dd7",
"sanitized_output_sha256": "bc8169c6ed3b2790278fcf64192893e9662de210b9d5fe4818963a69bb0e8dd7",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "WORKERS2_BACKUP_BUSY_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1\nMODE=read-only\nCOMPONENT=homelab-backup-busy-rca\nMUTATION_BOUNDARY=READ_ONLY_NO_SERVICE_STOP_NO_TIMER_STOP_NO_LOCK_BREAK_NO_CONFIG_CHANGE_NO_BACKUP_NO_GC_NO_GIT_WRITE\nSOURCE_BASELINE={\"clean\":true,\"head\":\"4e59d5afaa670899c56f87a84bc887175f299614\",\"remote_branch_sha\":null}\nBACKUP_SERVICE={\"props\":{\"ActiveEnterTimestamp\":\"\",\"ActiveState\":\"activating\",\"ExecMainCode\":\"0\",\"ExecMainExitTimestamp\":\"\",\"ExecMainPID\":\"3117776\",\"ExecMainStartTimestamp\":\"Mon 2026-09-21 10:42:58 MSK\",\"ExecMainStatus\":\"0\",\"FragmentPath\":\"/etc/systemd/system/homelab-backup-v2.service\",\"InactiveEnterTimestamp\":\"Mon 2026-09-21 10:11:59 MSK\",\"MainPID\":\"3117776\",\"RemainAfterExit\":\"no\",\"Result\":\"success\",\"SubState\":\"start\",\"Type\":\"oneshot\"},\"rc\":0}\nBACKUP_TIMER={\"props\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-backup-v2.timer\",\"LastTriggerUSec\":\"Mon 2026-09-21 10:42:58 MSK\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"running\",\"Triggers\":\"homelab-backup-v2.service\",\"Unit\":\"homelab-backup-v2.service\"},\"rc\":0}\nSERVICE_UNIT_RELEVANT=[\"Type=oneshot\",\"ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\"]\nTIMER_UNIT_RELEVANT=[\"OnUnitActiveSec=30min\",\"RandomizedDelaySec=10min\"]\nHEAVY_LOCK={\"exists\":true,\"flock_rc\":1,\"free\":false,\"fuser_pids\":[3117776],\"lslocks\":[{\"command\":\"homelab-backup-\",\"mode\":\"WRITE\",\"path\":\"/run/homelab-backup-heavy.lock\",\"pid\":3117776,\"type\":\"FLOCK\"}]}\nHOLDER_PROCESSES=[{\"cmd_hash\":\"cd76277a7385c9f3\",\"pid\":3117776,\"ps\":\"3117776 1 5055 SNs homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\",\"state\":\"S\"}]\nBACKUP_RELATED_PROCESSES=[\"3117776 1 5055 SNs homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\"]\nSERVICE_JOURNAL_1076_WINDOW=[]\nSERVICE_JOURNAL_RECENT_TAIL=[\"2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2339216]: SCHEDULE_TARGET=xf-dsmods\",\"2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2339216]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 1min 18.429s CPU time, 713.4M memory peak.\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\",\"2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2759410]: SELFTEST_REPLACEMENT486=PASS\",\"2026-09-21T09:41:22+03:00 pve01 homelab-backup-v2[2759410]: SCHEDULE_TARGET=xf-kingofwolk\",\"2026-09-21T09:41:22+03:00 pve01 homelab-backup-v2[2759410]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T09:41:22+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\",\"2026-09-21T09:41:22+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-21T09:41:22+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 11.004s CPU time, 143.6M memory peak.\",\"2026-09-21T10:04:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\",\"2026-09-21T10:04:32+03:00 pve01 homelab-backup-v2[2947207]: SELFTEST_REPLACEMENT486=PASS\",\"2026-09-21T10:11:59+03:00 pve01 homelab-backup-v2[2947207]: SCHEDULE_TARGET=xf-newfi\",\"2026-09-21T10:11:59+03:00 pve01 homelab-backup-v2[2947207]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T10:11:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\",\"2026-09-21T10:11:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-21T10:11:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 1min 35.661s CPU time, 162.2M memory peak.\",\"2026-09-21T10:42:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\",\"2026-09-21T10:42:59+03:00 pve01 homelab-backup-v2[3117776]: SELFTEST_REPLACEMENT486=PASS\"]\nRCA_CLASSIFICATION={\"fix\":\"IDENTIFY_OWNER;STOP_TIMER_FIRST_THEN_WAIT_FOR_OWNER_TO_EXIT;DO_NOT_BREAK_LOCK\",\"root_cause\":\"HEAVY_LOCK_HELD_BY_LIVE_PROCESS\"}\nNO_MUTATION=true\nNO_SERVICE_STOP=true\nNO_TIMER_STOP=true\nNO_LOCK_BREAK=true\nNO_BACKUP_STARTED=true\nNO_RETENTION_STARTED=true\nTASK_RESULT=PASS_HOMELAB_BACKUP_BUSY_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_BUSY_RCA_END=1\n"
"output": "SIGNALBOT052FINALRCA1_RESULT={\"outer_rc\":0,\"guest_exitcode\":0,\"rca\":{\"containers\":[{\"name\":\"demonov-shadow-8020-collector-1\",\"id\":\"8a65d323beda0410f180baf1e11144aaf53bf00f3329d758623bb711c1959f1f\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false},{\"name\":\"demonov-shadow-8020-worker-1\",\"id\":\"8677c7c7388e1f023ddc214a703de207085d21ceebca7565f673dd80b93c414d\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false},{\"name\":\"demonov-shadow-8020-relay-1\",\"id\":\"8c7689a8e46e72f94747f002af43d3a3b68c75af81f92d9287f0828c60738e6a\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false},{\"name\":\"demonov-shadow-8020-shadow-1\",\"id\":\"80ab5a8cad2b0eb99c4d0b86d66de1a673e4e40ff1fb7ac52b444d5a657bddb3\",\"image_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"config_image\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"restart\":0,\"running\":true,\"paused\":false,\"restarting\":false}],\"candidate_actual_id\":\"sha256:525cf1e50b232943e82bcd76c8cc2ea5f297bf7319db49bf6fd3470d60bfe555\",\"quick_check\":\"ok\",\"reconciliation_counts\":{\"RECONCILED\":20,\"UNRESOLVED\":1},\"unresolved\":[{\"window_id\":\"5ceb35ba092d9fff7a7520784ddf1b087441d7855b2d24e4673d8c32c282caaf\",\"reconnect_event_seq\":513728,\"reconnect_received_at\":\"2026-09-21T08:48:58.777306+00:00\",\"previous_trade_identity\":\"bb18a73cdc0b5831145ca37c469476c25e641e6c40b000489753104562b5fc74\",\"previous_trade_mts\":\"2026-09-21T08:48:56.940000+00:00\",\"next_trade_identity\":\"a543b607170f909e7f008d809f8f778972b2ae6a58aa1ad3cdd66af92943c9ec\",\"next_trade_mts\":\"2026-09-21T08:48:59.795000+00:00\",\"status\":\"UNRESOLVED\",\"evidence_sha256\":\"0933e981b764b89edd58396a737136747668a903a239eac64d538d5f4f2d5391\",\"recovered_row_count\":0,\"failure_reason\":\"REST_FETCH_OR_VALIDATION_FAILED\",\"resolved_at\":\"2026-09-21T08:48:59.886028+00:00\"}],\"accepted\":[\"bb9cbf18d81e605641e7aa9b792d35d1f55c3cae51fbe1abfe1617b72f0eaa14\",\"BTC|BITFINEX|SPOT\",\"D1\",\"2026-09-18T00:00:00+00:00\",\"2026-09-19T00:00:00+00:00\",\"FINALIZED\",\"94755f8715fe97b4fde2d846638af84ea4a829625bb0ca6ece98a78803416b2e\",\"33cf3eb2b33466a5bb2c2779602fb3119cb00e628759f941456df554e31cb774\"],\"revision_count\":1,\"sequence_events\":521614,\"collector_status.json\":{\"mtime\":1789981854.9718356,\"age_sec\":2.559959888458252,\"filtered_paths\":[[\"post_finality_revision\",false],[\"prospective_fail_closed_reason\",\"REST_FETCH_OR_VALIDATION_FAILED\"],[\"prospective_gap_detected\",false],[\"prospective_reconnect_detected\",true],[\"reconciliation_last_reason\",\"REST_FETCH_OR_VALIDATION_FAILED\"],[\"reconciliation_pending_count\",0],[\"reconciliation_unresolved_count\",1],[\"updated_at\",\"2026-09-21T09:10:54.977508+00:00\"]]},\"live_status.json\":{\"mtime\":1789981842.9316573,\"age_sec\":14.600192785263062,\"filtered_paths\":[[\"execution_authority\",false],[\"finalized_d1_available_count\",0],[\"finalized_d1_backlog_count\",0],[\"finalized_d1_processed_this_cycle\",0],[\"heartbeat_at\",\"2026-09-21T09:10:42.936294+00:00\"],[\"last_primary_closed_d1\",null],[\"production_launch_allowed\",false],[\"prospective_research_collection_allowed\",true],[\"prospective_row_count\",4],[\"risk_authority\",false],[\"source_post_finality_revision\",false],[\"status\",\"FAIL_CLOSED\"]]},\"live_matches\":[[\"observations\",2],[\"processed_logical_d1\",1]],\"expected\":{\"active_tag\":\"8020-demonov-shadow:signalbot-050r6-749073ae0f89\",\"active_id\":\"sha256:bc7fe43a1627532cf2ffbea2489352bb97c9a9e445e097cd5837b80a9e90cc60\",\"candidate_id\":\"sha256:525cf1e50b232943e82bcd76c8cc2ea5f297bf7319db49bf6fd3470d60bfe555\",\"accepted\":[\"bb9cbf18d81e605641e7aa9b792d35d1f55c3cae51fbe1abfe1617b72f0eaa14\",\"BTC|BITFINEX|SPOT\",\"D1\",\"2026-09-18T00:00:00+00:00\",\"2026-09-19T00:00:00+00:00\",\"FINALIZED\",\"94755f8715fe97b4fde2d846638af84ea4a829625bb0ca6ece98a78803416b2e\",\"33cf3eb2b33466a5bb2c2779602fb3119cb00e628759f941456df554e31cb774\"]}}}\nTASK_RESULT=PASS_SIGNALBOT_052FINALRCA1_BLOCKERS_CAPTURED\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SIGNALBOT-260921-FINAL-HOLD-RCA-052FINALRCA1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n"
}