runtime: publish SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1

2026-09-21 09:07:15 +00:00
parent ce99d2ccf9
commit b217d66f55
4 changed files with 143 additions and 41 deletions
@@ -0,0 +1,38 @@
+{
+  "schema_version": 1,
+  "channel": "homelab-runtime",
+  "command_id": "SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1",
+  "status": "OK",
+  "rc": 0,
+  "host": "pve01",
+  "mode": "read-only",
+  "component": "homelab-backup-busy-rca",
+  "started_at_utc": "2026-09-21T09:07:14Z",
+  "finished_at_utc": "2026-09-21T09:07:14Z",
+  "reference_register_checked": true,
+  "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
+  "error_register_checked": true,
+  "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
+  "command_sha256": "1d13fe3e140bfc2bc4a6c4003134cad2f1041197c7c8ddf2403689b1cffa1391",
+  "duplicate_failed_command_blocked": false,
+  "block_reason": null,
+  "execution_started": true,
+  "change_declared": false,
+  "result_contract_valid": true,
+  "result_contract_status": null,
+  "result_contract_error": null,
+  "command_rc": 0,
+  "changes_made": false,
+  "rollback_started": false,
+  "rollback_restored": null,
+  "mutation_outcome": "NO_MUTATION",
+  "sanitized": true,
+  "secrets_included": false,
+  "private_addresses_included": false,
+  "raw_evidence_retained_locally": true,
+  "raw_evidence_sha256": "96264514854b5aa0f18be36121dab5745543b61e700d11c7aec4d597410b6a94",
+  "sanitized_output_sha256": "96264514854b5aa0f18be36121dab5745543b61e700d11c7aec4d597410b6a94",
+  "output_truncated_in_json": false,
+  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
+  "output": "WORKERS2_BACKUP_BUSY_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1\nMODE=read-only\nCOMPONENT=homelab-backup-busy-rca\nMUTATION_BOUNDARY=READ_ONLY_NO_SERVICE_STOP_NO_TIMER_STOP_NO_LOCK_BREAK_NO_CONFIG_CHANGE_NO_BACKUP_NO_GC_NO_GIT_WRITE\nSOURCE_BASELINE={\"clean\":true,\"head\":\"4e59d5afaa670899c56f87a84bc887175f299614\",\"remote_branch_sha\":null}\nBACKUP_SERVICE={\"props\":{\"ActiveEnterTimestamp\":\"\",\"ActiveState\":\"activating\",\"ExecMainCode\":\"0\",\"ExecMainExitTimestamp\":\"\",\"ExecMainPID\":\"3117776\",\"ExecMainStartTimestamp\":\"Mon 2026-09-21 10:42:58 MSK\",\"ExecMainStatus\":\"0\",\"FragmentPath\":\"/etc/systemd/system/homelab-backup-v2.service\",\"InactiveEnterTimestamp\":\"Mon 2026-09-21 10:11:59 MSK\",\"MainPID\":\"3117776\",\"RemainAfterExit\":\"no\",\"Result\":\"success\",\"SubState\":\"start\",\"Type\":\"oneshot\"},\"rc\":0}\nBACKUP_TIMER={\"props\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-backup-v2.timer\",\"LastTriggerUSec\":\"Mon 2026-09-21 10:42:58 MSK\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"running\",\"Triggers\":\"homelab-backup-v2.service\",\"Unit\":\"homelab-backup-v2.service\"},\"rc\":0}\nSERVICE_UNIT_RELEVANT=[\"Type=oneshot\",\"ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\"]\nTIMER_UNIT_RELEVANT=[\"OnUnitActiveSec=30min\",\"RandomizedDelaySec=10min\"]\nHEAVY_LOCK={\"exists\":true,\"flock_rc\":1,\"free\":false,\"fuser_pids\":[3117776],\"lslocks\":[{\"command\":\"homelab-backup-\",\"mode\":\"WRITE\",\"path\":\"/run/homelab-backup-heavy.lock\",\"pid\":3117776,\"type\":\"FLOCK\"}]}\nHOLDER_PROCESSES=[{\"cmd_hash\":\"cd76277a7385c9f3\",\"pid\":3117776,\"ps\":\"3117776       1    5055 SNs  homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\",\"state\":\"S\"}]\nBACKUP_RELATED_PROCESSES=[\"3117776       1    5055 SNs  homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\"]\nSERVICE_JOURNAL_1076_WINDOW=[]\nSERVICE_JOURNAL_RECENT_TAIL=[\"2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2339216]: SCHEDULE_TARGET=xf-dsmods\",\"2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2339216]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 1min 18.429s CPU time, 713.4M memory peak.\",\"2026-09-21T09:31:02+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\",\"2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2759410]: SELFTEST_REPLACEMENT486=PASS\",\"2026-09-21T09:41:22+03:00 pve01 homelab-backup-v2[2759410]: SCHEDULE_TARGET=xf-kingofwolk\",\"2026-09-21T09:41:22+03:00 pve01 homelab-backup-v2[2759410]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T09:41:22+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\",\"2026-09-21T09:41:22+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-21T09:41:22+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 11.004s CPU time, 143.6M memory peak.\",\"2026-09-21T10:04:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\",\"2026-09-21T10:04:32+03:00 pve01 homelab-backup-v2[2947207]: SELFTEST_REPLACEMENT486=PASS\",\"2026-09-21T10:11:59+03:00 pve01 homelab-backup-v2[2947207]: SCHEDULE_TARGET=xf-newfi\",\"2026-09-21T10:11:59+03:00 pve01 homelab-backup-v2[2947207]: SCHEDULE_RESULT=SUCCESS\",\"2026-09-21T10:11:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\",\"2026-09-21T10:11:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\",\"2026-09-21T10:11:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 1min 35.661s CPU time, 162.2M memory peak.\",\"2026-09-21T10:42:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\",\"2026-09-21T10:42:59+03:00 pve01 homelab-backup-v2[3117776]: SELFTEST_REPLACEMENT486=PASS\"]\nRCA_CLASSIFICATION={\"fix\":\"IDENTIFY_OWNER;STOP_TIMER_FIRST_THEN_WAIT_FOR_OWNER_TO_EXIT;DO_NOT_BREAK_LOCK\",\"root_cause\":\"HEAVY_LOCK_HELD_BY_LIVE_PROCESS\"}\nNO_MUTATION=true\nNO_SERVICE_STOP=true\nNO_TIMER_STOP=true\nNO_LOCK_BREAK=true\nNO_BACKUP_STARTED=true\nNO_RETENTION_STARTED=true\nTASK_RESULT=PASS_HOMELAB_BACKUP_BUSY_RCA_READONLY\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_BACKUP_BUSY_RCA_END=1\n"
+}
@@ -0,0 +1,57 @@
+CHAT_OUTPUT_BEGIN
+COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1
+STATUS=OK
+RC=0
+HOST=pve01
+MODE=read-only
+COMPONENT=homelab-backup-busy-rca
+REFERENCE_REGISTER_CHECK=OK
+REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
+ERROR_REGISTER_CHECK=OK
+ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
+COMMAND_SHA256=1d13fe3e140bfc2bc4a6c4003134cad2f1041197c7c8ddf2403689b1cffa1391
+DUPLICATE_FAILED_COMMAND_BLOCKED=false
+EXECUTION_STARTED=true
+CHANGE_DECLARED=false
+RESULT_CONTRACT_VALID=true
+RESULT_CONTRACT_STATUS=NOT_APPLICABLE
+RESULT_CONTRACT_ERROR=NONE
+COMMAND_RC=0
+CHANGES_MADE=false
+ROLLBACK_STARTED=false
+ROLLBACK_RESTORED=null
+MUTATION_OUTCOME=NO_MUTATION
+SANITIZED=yes
+SECRETS_INCLUDED=no
+PRIVATE_ADDRESSES_INCLUDED=no
+RAW_EVIDENCE_SHA256=96264514854b5aa0f18be36121dab5745543b61e700d11c7aec4d597410b6a94
+SANITIZED_OUTPUT_SHA256=96264514854b5aa0f18be36121dab5745543b61e700d11c7aec4d597410b6a94
+OUTPUT_BEGIN
+WORKERS2_BACKUP_BUSY_RCA_BEGIN=1
+COMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1
+MODE=read-only
+COMPONENT=homelab-backup-busy-rca
+MUTATION_BOUNDARY=READ_ONLY_NO_SERVICE_STOP_NO_TIMER_STOP_NO_LOCK_BREAK_NO_CONFIG_CHANGE_NO_BACKUP_NO_GC_NO_GIT_WRITE
+SOURCE_BASELINE={"clean":true,"head":"4e59d5afaa670899c56f87a84bc887175f299614","remote_branch_sha":null}
+BACKUP_SERVICE={"props":{"ActiveEnterTimestamp":"","ActiveState":"activating","ExecMainCode":"0","ExecMainExitTimestamp":"","ExecMainPID":"3117776","ExecMainStartTimestamp":"Mon 2026-09-21 10:42:58 MSK","ExecMainStatus":"0","FragmentPath":"/etc/systemd/system/homelab-backup-v2.service","InactiveEnterTimestamp":"Mon 2026-09-21 10:11:59 MSK","MainPID":"3117776","RemainAfterExit":"no","Result":"success","SubState":"start","Type":"oneshot"},"rc":0}
+BACKUP_TIMER={"props":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-backup-v2.timer","LastTriggerUSec":"Mon 2026-09-21 10:42:58 MSK","NextElapseUSecRealtime":"","Result":"success","SubState":"running","Triggers":"homelab-backup-v2.service","Unit":"homelab-backup-v2.service"},"rc":0}
+SERVICE_UNIT_RELEVANT=["Type=oneshot","ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled"]
+TIMER_UNIT_RELEVANT=["OnUnitActiveSec=30min","RandomizedDelaySec=10min"]
+HEAVY_LOCK={"exists":true,"flock_rc":1,"free":false,"fuser_pids":[3117776],"lslocks":[{"command":"homelab-backup-","mode":"WRITE","path":"/run/homelab-backup-heavy.lock","pid":3117776,"type":"FLOCK"}]}
+HOLDER_PROCESSES=[{"cmd_hash":"cd76277a7385c9f3","pid":3117776,"ps":"3117776       1    5055 SNs  homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled","state":"S"}]
+BACKUP_RELATED_PROCESSES=["3117776       1    5055 SNs  homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled"]
+SERVICE_JOURNAL_1076_WINDOW=[]
+SERVICE_JOURNAL_RECENT_TAIL=["2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2339216]: SCHEDULE_TARGET=xf-dsmods","2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2339216]: SCHEDULE_RESULT=SUCCESS","2026-09-21T09:31:02+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.","2026-09-21T09:31:02+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-21T09:31:02+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 1min 18.429s CPU time, 713.4M memory peak.","2026-09-21T09:31:02+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...","2026-09-21T09:31:02+03:00 pve01 homelab-backup-v2[2759410]: SELFTEST_REPLACEMENT486=PASS","2026-09-21T09:41:22+03:00 pve01 homelab-backup-v2[2759410]: SCHEDULE_TARGET=xf-kingofwolk","2026-09-21T09:41:22+03:00 pve01 homelab-backup-v2[2759410]: SCHEDULE_RESULT=SUCCESS","2026-09-21T09:41:22+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.","2026-09-21T09:41:22+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-21T09:41:22+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 11.004s CPU time, 143.6M memory peak.","2026-09-21T10:04:32+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...","2026-09-21T10:04:32+03:00 pve01 homelab-backup-v2[2947207]: SELFTEST_REPLACEMENT486=PASS","2026-09-21T10:11:59+03:00 pve01 homelab-backup-v2[2947207]: SCHEDULE_TARGET=xf-newfi","2026-09-21T10:11:59+03:00 pve01 homelab-backup-v2[2947207]: SCHEDULE_RESULT=SUCCESS","2026-09-21T10:11:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.","2026-09-21T10:11:59+03:00 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.","2026-09-21T10:11:59+03:00 pve01 systemd[1]: homelab-backup-v2.service: Consumed 1min 35.661s CPU time, 162.2M memory peak.","2026-09-21T10:42:58+03:00 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...","2026-09-21T10:42:59+03:00 pve01 homelab-backup-v2[3117776]: SELFTEST_REPLACEMENT486=PASS"]
+RCA_CLASSIFICATION={"fix":"IDENTIFY_OWNER;STOP_TIMER_FIRST_THEN_WAIT_FOR_OWNER_TO_EXIT;DO_NOT_BREAK_LOCK","root_cause":"HEAVY_LOCK_HELD_BY_LIVE_PROCESS"}
+NO_MUTATION=true
+NO_SERVICE_STOP=true
+NO_TIMER_STOP=true
+NO_LOCK_BREAK=true
+NO_BACKUP_STARTED=true
+NO_RETENTION_STARTED=true
+TASK_RESULT=PASS_HOMELAB_BACKUP_BUSY_RCA_READONLY
+HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260921-HOMELAB-BACKUP-BUSY-RCA-READONLY-1077R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
+WORKERS2_BACKUP_BUSY_RCA_END=1
+
+OUTPUT_END
+CHAT_OUTPUT_END