From 7525f9042b5cb9e4b5d75f6f77e28680783c005b Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Mon, 5 Oct 2026 17:55:04 +0000 Subject: [PATCH] runtime: publish SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1 --- ...0-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1.json | 38 ++++ ...60-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1.txt | 96 ++++++++++ runtime/latest.json | 32 ++-- runtime/latest.txt | 173 ++++++++---------- 4 files changed, 222 insertions(+), 117 deletions(-) create mode 100644 runtime/history/SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1.json create mode 100644 runtime/history/SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1.txt diff --git a/runtime/history/SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1.json b/runtime/history/SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1.json new file mode 100644 index 00000000..904f1efd --- /dev/null +++ b/runtime/history/SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1", + "status": "FAIL", + "rc": 68, + "host": "pve01", + "mode": "change", + "component": "vm160-backup-quiesce-watchdog-fix", + "started_at_utc": "2026-10-05T17:53:19Z", + "finished_at_utc": "2026-10-05T17:55:03Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "d8edc0bcd4d08329eae8a011660c73b1a355cfb2a456146ef6590ed72bc36b2d", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": true, + "result_contract_valid": true, + "result_contract_status": "FAIL", + "result_contract_error": null, + "command_rc": 68, + "changes_made": true, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "MUTATED", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d", + "sanitized_output_sha256": "3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "VM160_BACKUP_QUIESCE_WATCHDOG_FIX=V1\nMODE=change\nTARGET_VM=160\nTARGET_NODE=pve02\nWATCHDOG_RCA_PROVEN=true\nWATCHDOG_RCA=CONTROLLER_PVESH_TIMEOUT_45S_INTERRUPTED_BACKUP\nBACKUP_LAUNCH_MODEL=DETACHED_SYSTEMD_TRANSIENT_PVESH\nNO_VM_SHUTDOWN=true\nNO_DNS_MUTATION=true\nNO_PROXY_MUTATION=true\nNO_BEGET_MUTATION=true\nNO_DB_CONTENT_MUTATION=true\nPUBLIC_PRE_ALL_SIX_OK=true\nCODEVIPE_RU_PUBLIC_PRE_OK=true\nCODEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true\nHKMODS_RU_PUBLIC_PRE_OK=true\nHKMODS_RU_PUBLIC_PRE_BEGET_HINT=true\nDSMODS_RU_PUBLIC_PRE_OK=true\nDSMODS_RU_PUBLIC_PRE_BEGET_HINT=true\nGAMEVIPE_RU_PUBLIC_PRE_OK=true\nGAMEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true\nKINGOFWOLK_RU_PUBLIC_PRE_OK=true\nKINGOFWOLK_RU_PUBLIC_PRE_BEGET_HINT=true\nZAKRUTIM_RU_PUBLIC_PRE_OK=true\nZAKRUTIM_RU_PUBLIC_PRE_BEGET_HINT=true\nVM160_PRE_FOUND=true\nVM160_PRE_NODE=pve02\nVM160_PRE_STATE=running\nPRE_GUEST_QGA_RC=0\nPRE_GUEST_EXITCODE=0\nCRON_ACTIVE_PRESENT=true\nCRON_99621_DISABLED_PRESENT=false\nSTATE_99621_DIR_PRESENT=false\nRUNNER_PROCESS_COUNT=0\nMATCH_TIMER_COUNT=1\nMATCH_TIMER_ACTIVE=1\nMATCH_TIMER_ENABLED=1\nDB_ROOT_FILE_COUNT=6\nVM160_PRECHANGE_GUEST_OK=true\nALL_SIX_SOURCE_BOARDS_INACTIVE=true\nQUIESCE_UNTOUCHED_PRE=true\nBACKUP_STORAGE_CANDIDATE_COUNT=1\nBACKUP_STORAGE_NAME=local\nBACKUP_STORAGE_AVAILABLE_BYTES=87478521856\nBACKUP_STORAGE_TOTAL_BYTES=105089261568\nBACKUP_REQUIRED_MIN_FREE_BYTES=34359738368\nBACKUP_SPACE_PREFLIGHT_OK=true\nVM160_BACKUP_TASK_ALREADY_RUNNING=false\nVM160_BACKUP_COUNT_PRE=0\nTRANSIENT_UNIT_PREEXISTS=false\nBACKUP_START_EPOCH=1791222811\nDETACHED_LAUNCH_RC=0\nDETACHED_LAUNCH_OUTPUT_SHA=dff3498d460ccce6681bd424902dec8994b870f02d9ffb229900bb3c122499c3\nBACKUP_TASK_DISCOVERED=false\nBACKUP_TASK_UPID_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nDECISION=HOLD_BACKUP_TASK_NOT_DISCOVERED\nNEXT_GATE=VM160_BACKUP_TASK_RECONCILE\nCOMMANDS_REMAINING_AFTER_SUCCESS=2\nTASK_RESULT=VM160_BACKUP_QUIESCE_WATCHDOG_FIX\nCHANGES_MADE=true\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=DETACHED_LAUNCH_SUBMITTED_TASK_NOT_DISCOVERED\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1.txt b/runtime/history/SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1.txt new file mode 100644 index 00000000..bdac9087 --- /dev/null +++ b/runtime/history/SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1.txt @@ -0,0 +1,96 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1 +STATUS=FAIL +RC=68 +HOST=pve01 +MODE=change +COMPONENT=vm160-backup-quiesce-watchdog-fix +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=d8edc0bcd4d08329eae8a011660c73b1a355cfb2a456146ef6590ed72bc36b2d +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=true +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=68 +CHANGES_MADE=true +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=MUTATED +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d +SANITIZED_OUTPUT_SHA256=3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d +OUTPUT_BEGIN +VM160_BACKUP_QUIESCE_WATCHDOG_FIX=V1 +MODE=change +TARGET_VM=160 +TARGET_NODE=pve02 +WATCHDOG_RCA_PROVEN=true +WATCHDOG_RCA=CONTROLLER_PVESH_TIMEOUT_45S_INTERRUPTED_BACKUP +BACKUP_LAUNCH_MODEL=DETACHED_SYSTEMD_TRANSIENT_PVESH +NO_VM_SHUTDOWN=true +NO_DNS_MUTATION=true +NO_PROXY_MUTATION=true +NO_BEGET_MUTATION=true +NO_DB_CONTENT_MUTATION=true +PUBLIC_PRE_ALL_SIX_OK=true +CODEVIPE_RU_PUBLIC_PRE_OK=true +CODEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true +HKMODS_RU_PUBLIC_PRE_OK=true +HKMODS_RU_PUBLIC_PRE_BEGET_HINT=true +DSMODS_RU_PUBLIC_PRE_OK=true +DSMODS_RU_PUBLIC_PRE_BEGET_HINT=true +GAMEVIPE_RU_PUBLIC_PRE_OK=true +GAMEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true +KINGOFWOLK_RU_PUBLIC_PRE_OK=true +KINGOFWOLK_RU_PUBLIC_PRE_BEGET_HINT=true +ZAKRUTIM_RU_PUBLIC_PRE_OK=true +ZAKRUTIM_RU_PUBLIC_PRE_BEGET_HINT=true +VM160_PRE_FOUND=true +VM160_PRE_NODE=pve02 +VM160_PRE_STATE=running +PRE_GUEST_QGA_RC=0 +PRE_GUEST_EXITCODE=0 +CRON_ACTIVE_PRESENT=true +CRON_99621_DISABLED_PRESENT=false +STATE_99621_DIR_PRESENT=false +RUNNER_PROCESS_COUNT=0 +MATCH_TIMER_COUNT=1 +MATCH_TIMER_ACTIVE=1 +MATCH_TIMER_ENABLED=1 +DB_ROOT_FILE_COUNT=6 +VM160_PRECHANGE_GUEST_OK=true +ALL_SIX_SOURCE_BOARDS_INACTIVE=true +QUIESCE_UNTOUCHED_PRE=true +BACKUP_STORAGE_CANDIDATE_COUNT=1 +BACKUP_STORAGE_NAME=local +BACKUP_STORAGE_AVAILABLE_BYTES=87478521856 +BACKUP_STORAGE_TOTAL_BYTES=105089261568 +BACKUP_REQUIRED_MIN_FREE_BYTES=34359738368 +BACKUP_SPACE_PREFLIGHT_OK=true +VM160_BACKUP_TASK_ALREADY_RUNNING=false +VM160_BACKUP_COUNT_PRE=0 +TRANSIENT_UNIT_PREEXISTS=false +BACKUP_START_EPOCH=1791222811 +DETACHED_LAUNCH_RC=0 +DETACHED_LAUNCH_OUTPUT_SHA=dff3498d460ccce6681bd424902dec8994b870f02d9ffb229900bb3c122499c3 +BACKUP_TASK_DISCOVERED=false +BACKUP_TASK_UPID_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +DECISION=HOLD_BACKUP_TASK_NOT_DISCOVERED +NEXT_GATE=VM160_BACKUP_TASK_RECONCILE +COMMANDS_REMAINING_AFTER_SUCCESS=2 +TASK_RESULT=VM160_BACKUP_QUIESCE_WATCHDOG_FIX +CHANGES_MADE=true +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=DETACHED_LAUNCH_SUBMITTED_TASK_NOT_DISCOVERED +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 518153cd..904f1efd 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-261005-VM160-VZDUMP-PIPE-PRODUCER-RCA-99620R1", - "status": "OK", - "rc": 0, + "command_id": "SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1", + "status": "FAIL", + "rc": 68, "host": "pve01", - "mode": "read-only", - "component": "vm160-vzdump-pipe-producer-rca", - "started_at_utc": "2026-10-05T17:37:54Z", - "finished_at_utc": "2026-10-05T17:37:59Z", + "mode": "change", + "component": "vm160-backup-quiesce-watchdog-fix", + "started_at_utc": "2026-10-05T17:53:19Z", + "finished_at_utc": "2026-10-05T17:55:03Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "72066f03903738f8c012e4b65a4a9af21e10c52b7e5b57d9e204688234754e24", + "command_sha256": "d8edc0bcd4d08329eae8a011660c73b1a355cfb2a456146ef6590ed72bc36b2d", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": false, + "change_declared": true, "result_contract_valid": true, - "result_contract_status": null, + "result_contract_status": "FAIL", "result_contract_error": null, - "command_rc": 0, - "changes_made": false, + "command_rc": 68, + "changes_made": true, "rollback_started": false, "rollback_restored": null, - "mutation_outcome": "NO_MUTATION", + "mutation_outcome": "MUTATED", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "24493c6b989dfc38fcbc58b91461e936132070e60d58af938cecdb4feda4ea6f", - "sanitized_output_sha256": "24493c6b989dfc38fcbc58b91461e936132070e60d58af938cecdb4feda4ea6f", + "raw_evidence_sha256": "3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d", + "sanitized_output_sha256": "3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "VM160_VZDUMP_PIPE_PRODUCER_RCA=V1\nMODE=read-only\nNO_VM_MUTATION=true\nNO_BACKUP_CREATE=true\nNO_BACKUP_DELETE=true\nNO_CONFIG_WRITE=true\nNO_SERVICE_RESTART=true\nNO_GUEST_MUTATION=true\nNO_DNS_MUTATION=true\nNO_PROXY_MUTATION=true\nTARGET_TASK_MATCH_COUNT=1\nTARGET_TASK_UPID_SHA=be6ac2a9c3def71ef248f7fa41bd4e273e39c3b4317424e31af2b6cb5b8e5b3b\nTARGET_TASK_START=1791217449\nTARGET_TASK_END=1791217497\nTARGET_TASK_STATUS=broken pipe\nTARGET_TASK_EXITSTATUS=\nTASK_LOG_RC=0\nTASK_LOG_LINE_COUNT=28\nTASK_LOG_SHA=6bfdb2dbb2eafb6f3701b2f84f41728ade02e640eb64813f0292ab77b810e529\nTASK_LOG_01=INFO: starting new backup job: vzdump 160 --notes-template 'WORKERS2 VM160 retirement rollback backup 99615' --mode snapshot --storage local --compress zstd --node pve02\nTASK_LOG_02=INFO: Starting Backup of VM 160 (qemu)\nTASK_LOG_03=INFO: Backup started at 2026-10-05 19:24:09\nTASK_LOG_04=INFO: status = running\nTASK_LOG_05=INFO: VM Name: forum-prod\nTASK_LOG_06=INFO: include disk 'scsi0' 'local-lvm:vm-160-disk-0' 200G\nTASK_LOG_07=INFO: backup mode: snapshot\nTASK_LOG_08=INFO: ionice priority: 7\nTASK_LOG_09=INFO: creating vzdump archive '/var/lib/vz/dump/vzdump-qemu-160-2026_10_05-19_24_09.vma.zst'\nTASK_LOG_10=INFO: issuing guest-agent 'fs-freeze' command\nTASK_LOG_11=INFO: starting backup via QMP command\nTASK_LOG_12=INFO: issuing guest-agent 'fs-thaw' command\nTASK_LOG_13=INFO: started backup task '41ea49b6-7bd9-4883-a24f-722e1d18fca0'\nTASK_LOG_14=INFO: resuming VM again\nTASK_LOG_15=INFO: 0% (1.3 GiB of 200.0 GiB) in 3s, read: 455.5 MiB/s, write: 189.7 MiB/s\nTASK_LOG_16=INFO: 2% (4.1 GiB of 200.0 GiB) in 6s, read: 952.8 MiB/s, write: 158.5 MiB/s\nTASK_LOG_17=INFO: 4% (8.9 GiB of 200.0 GiB) in 9s, read: 1.6 GiB/s, write: 126.9 MiB/s\nTASK_LOG_18=INFO: 6% (12.2 GiB of 200.0 GiB) in 12s, read: 1.1 GiB/s, write: 136.8 MiB/s\nTASK_LOG_19=INFO: 7% (14.3 GiB of 200.0 GiB) in 15s, read: 686.3 MiB/s, write: 142.4 MiB/s\nTASK_LOG_20=INFO: 9% (18.4 GiB of 200.0 GiB) in 18s, read: 1.4 GiB/s, write: 72.2 MiB/s\nTASK_LOG_21=INFO: 10% (20.3 GiB of 200.0 GiB) in 22s, read: 474.9 MiB/s, write: 163.3 MiB/s\nTASK_LOG_22=INFO: 11% (22.6 GiB of 200.0 GiB) in 25s, read: 795.1 MiB/s, write: 165.3 MiB/s\nTASK_LOG_23=INFO: 12% (24.4 GiB of 200.0 GiB) in 28s, read: 599.3 MiB/s, write: 192.6 MiB/s\nTASK_LOG_24=INFO: 13% (26.1 GiB of 200.0 GiB) in 35s, read: 257.2 MiB/s, write: 111.2 MiB/s\nTASK_LOG_25=INFO: 14% (28.3 GiB of 200.0 GiB) in 38s, read: 727.7 MiB/s, write: 161.0 MiB/s\nTASK_LOG_26=INFO: 16% (32.1 GiB of 200.0 GiB) in 41s, read: 1.3 GiB/s, write: 118.1 MiB/s\nTASK_LOG_27=INFO: 18% (36.9 GiB of 200.0 GiB) in 44s, read: 1.6 GiB/s, write: 56.1 MiB/s\nTASK_LOG_28=TASK ERROR: broken pipe\nLOG_HAS_ZSTD_ERROR=false\nLOG_HAS_NO_SPACE=false\nLOG_HAS_IO_ERROR=false\nLOG_HAS_VMA_QUEUE_WRITE=false\nLOG_HAS_ABORTING=false\nLOG_HAS_GUEST_THAW=true\nLOG_HAS_PROGRESS=true\nPROGRESS_LINE_COUNT=13\nLAST_PROGRESS_LINE=INFO: 18% (36.9 GiB of 200.0 GiB) in 44s, read: 1.6 GiB/s, write: 56.1 MiB/s\nNODE_JOURNAL_RC=0\nNODE_JOURNAL_LINE_COUNT=10002\nNODE_JOURNAL_SHA=b78892f57eea13b97d0d00ae4a3abc977fe08fbd927622c0b4848733dfddfd6e\nNODE_JOURNAL_RELEVANT_COUNT=63\nNODE_JOURNAL_WINDOW_COUNT=6\nNODE_JOURNAL_01=Oct 05 19:24:09 pve02 pvesh[4125258]: starting task UPID:pve02:003EF24D:13A18FC0:6AC3CF29:vzdump:160:root@pam:\nNODE_JOURNAL_02=Oct 05 19:24:09 pve02 pvesh[4125261]: INFO: starting new backup job: vzdump 160 --notes-template 'WORKERS2 VM160 retirement rollback backup 99615' --mode snapshot --storage local --compress zstd --node pve02\nNODE_JOURNAL_03=Oct 05 19:24:09 pve02 pvesh[4125261]: INFO: Starting Backup of VM 160 (qemu)\nNODE_JOURNAL_04=Oct 05 19:24:56 pve02 pvesh[4125261]: ERROR: Backup of VM 160 failed - interrupted by signal\nNODE_JOURNAL_05=Oct 05 19:24:56 pve02 pvesh[4125261]: ERROR: Backup job failed - interrupted by signal\nNODE_JOURNAL_06=Oct 05 19:24:57 pve02 pvesh[4125258]: end task UPID:pve02:003EF24D:13A18FC0:6AC3CF29:vzdump:160:root@pam: broken pipe\nJOURNAL_OOM_SIGNAL=false\nJOURNAL_IO_ERROR_SIGNAL=false\nJOURNAL_EXT4_ERROR_SIGNAL=false\nJOURNAL_ZSTD_SIGNAL=true\nJOURNAL_QEMU_SIGNAL=true\nLEFTOVER_FIND_RC=0\nLEFTOVER_160_FILE_COUNT=0\nPKG_PVE_MANAGER_VER_SHA=4bc0532243c6a7480dabf2cb6704e8645fef799b9561a78bc98dae955bec9d19\nPKG_QEMU_SERVER_VER_SHA=86b252f483e9d9c1ecf7a79de6136570ddae05a37b103287f5e5d290b6792269\nPKG_PVE_QEMU_KVM_VER_SHA=6f9cdce1fb8c1849f0ed78ae68aadf9744230ece98a33a5bcabcfd4e914dbb38\nPKG_ZSTD_VER_SHA=42f9875a37197a0a9eaa49c6fc729b93cf63e2d3c2dc11e7d28dd8798ca7e5da\nHOST_MEMINFO_SHA=890013975f41093d8759e94305a292094c46b0061a63aff051fd908f14499b6e\nHOST_MEMTOTAL_KB=32596696\nHOST_MEMAVAILABLE_KB=8851192\nHOST_SWAPTOTAL_KB=10485756\nHOST_SWAPFREE_KB=7745016\nPIPE_ROOT_CLASS=PIPE_CLOSED_WITHOUT_EXPLICIT_CONSUMER_ERROR\nRCA_PROVEN=true\nDECISION=PASS_VM160_VZDUMP_PIPE_PRODUCER_RCA\nNEXT_GATE=VM160_BACKUP_PIPE_SAFE_FIX\nCOMMANDS_REMAINING_AFTER_SUCCESS=2\nTASK_RESULT=VM160_VZDUMP_PIPE_PRODUCER_RCA\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\n" + "output": "VM160_BACKUP_QUIESCE_WATCHDOG_FIX=V1\nMODE=change\nTARGET_VM=160\nTARGET_NODE=pve02\nWATCHDOG_RCA_PROVEN=true\nWATCHDOG_RCA=CONTROLLER_PVESH_TIMEOUT_45S_INTERRUPTED_BACKUP\nBACKUP_LAUNCH_MODEL=DETACHED_SYSTEMD_TRANSIENT_PVESH\nNO_VM_SHUTDOWN=true\nNO_DNS_MUTATION=true\nNO_PROXY_MUTATION=true\nNO_BEGET_MUTATION=true\nNO_DB_CONTENT_MUTATION=true\nPUBLIC_PRE_ALL_SIX_OK=true\nCODEVIPE_RU_PUBLIC_PRE_OK=true\nCODEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true\nHKMODS_RU_PUBLIC_PRE_OK=true\nHKMODS_RU_PUBLIC_PRE_BEGET_HINT=true\nDSMODS_RU_PUBLIC_PRE_OK=true\nDSMODS_RU_PUBLIC_PRE_BEGET_HINT=true\nGAMEVIPE_RU_PUBLIC_PRE_OK=true\nGAMEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true\nKINGOFWOLK_RU_PUBLIC_PRE_OK=true\nKINGOFWOLK_RU_PUBLIC_PRE_BEGET_HINT=true\nZAKRUTIM_RU_PUBLIC_PRE_OK=true\nZAKRUTIM_RU_PUBLIC_PRE_BEGET_HINT=true\nVM160_PRE_FOUND=true\nVM160_PRE_NODE=pve02\nVM160_PRE_STATE=running\nPRE_GUEST_QGA_RC=0\nPRE_GUEST_EXITCODE=0\nCRON_ACTIVE_PRESENT=true\nCRON_99621_DISABLED_PRESENT=false\nSTATE_99621_DIR_PRESENT=false\nRUNNER_PROCESS_COUNT=0\nMATCH_TIMER_COUNT=1\nMATCH_TIMER_ACTIVE=1\nMATCH_TIMER_ENABLED=1\nDB_ROOT_FILE_COUNT=6\nVM160_PRECHANGE_GUEST_OK=true\nALL_SIX_SOURCE_BOARDS_INACTIVE=true\nQUIESCE_UNTOUCHED_PRE=true\nBACKUP_STORAGE_CANDIDATE_COUNT=1\nBACKUP_STORAGE_NAME=local\nBACKUP_STORAGE_AVAILABLE_BYTES=87478521856\nBACKUP_STORAGE_TOTAL_BYTES=105089261568\nBACKUP_REQUIRED_MIN_FREE_BYTES=34359738368\nBACKUP_SPACE_PREFLIGHT_OK=true\nVM160_BACKUP_TASK_ALREADY_RUNNING=false\nVM160_BACKUP_COUNT_PRE=0\nTRANSIENT_UNIT_PREEXISTS=false\nBACKUP_START_EPOCH=1791222811\nDETACHED_LAUNCH_RC=0\nDETACHED_LAUNCH_OUTPUT_SHA=dff3498d460ccce6681bd424902dec8994b870f02d9ffb229900bb3c122499c3\nBACKUP_TASK_DISCOVERED=false\nBACKUP_TASK_UPID_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nDECISION=HOLD_BACKUP_TASK_NOT_DISCOVERED\nNEXT_GATE=VM160_BACKUP_TASK_RECONCILE\nCOMMANDS_REMAINING_AFTER_SUCCESS=2\nTASK_RESULT=VM160_BACKUP_QUIESCE_WATCHDOG_FIX\nCHANGES_MADE=true\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=DETACHED_LAUNCH_SUBMITTED_TASK_NOT_DISCOVERED\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 48efb6af..bdac9087 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,125 +1,96 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-261005-VM160-VZDUMP-PIPE-PRODUCER-RCA-99620R1 -STATUS=OK -RC=0 +COMMAND_ID=SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1 +STATUS=FAIL +RC=68 HOST=pve01 -MODE=read-only -COMPONENT=vm160-vzdump-pipe-producer-rca +MODE=change +COMPONENT=vm160-backup-quiesce-watchdog-fix REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=72066f03903738f8c012e4b65a4a9af21e10c52b7e5b57d9e204688234754e24 +COMMAND_SHA256=d8edc0bcd4d08329eae8a011660c73b1a355cfb2a456146ef6590ed72bc36b2d DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=false +CHANGE_DECLARED=true RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_STATUS=FAIL RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=0 -CHANGES_MADE=false +COMMAND_RC=68 +CHANGES_MADE=true ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=NO_MUTATION +MUTATION_OUTCOME=MUTATED SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=24493c6b989dfc38fcbc58b91461e936132070e60d58af938cecdb4feda4ea6f -SANITIZED_OUTPUT_SHA256=24493c6b989dfc38fcbc58b91461e936132070e60d58af938cecdb4feda4ea6f +RAW_EVIDENCE_SHA256=3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d +SANITIZED_OUTPUT_SHA256=3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d OUTPUT_BEGIN -VM160_VZDUMP_PIPE_PRODUCER_RCA=V1 -MODE=read-only -NO_VM_MUTATION=true -NO_BACKUP_CREATE=true -NO_BACKUP_DELETE=true -NO_CONFIG_WRITE=true -NO_SERVICE_RESTART=true -NO_GUEST_MUTATION=true +VM160_BACKUP_QUIESCE_WATCHDOG_FIX=V1 +MODE=change +TARGET_VM=160 +TARGET_NODE=pve02 +WATCHDOG_RCA_PROVEN=true +WATCHDOG_RCA=CONTROLLER_PVESH_TIMEOUT_45S_INTERRUPTED_BACKUP +BACKUP_LAUNCH_MODEL=DETACHED_SYSTEMD_TRANSIENT_PVESH +NO_VM_SHUTDOWN=true NO_DNS_MUTATION=true NO_PROXY_MUTATION=true -TARGET_TASK_MATCH_COUNT=1 -TARGET_TASK_UPID_SHA=be6ac2a9c3def71ef248f7fa41bd4e273e39c3b4317424e31af2b6cb5b8e5b3b -TARGET_TASK_START=1791217449 -TARGET_TASK_END=1791217497 -TARGET_TASK_STATUS=broken pipe -TARGET_TASK_EXITSTATUS= -TASK_LOG_RC=0 -TASK_LOG_LINE_COUNT=28 -TASK_LOG_SHA=6bfdb2dbb2eafb6f3701b2f84f41728ade02e640eb64813f0292ab77b810e529 -TASK_LOG_01=INFO: starting new backup job: vzdump 160 --notes-template 'WORKERS2 VM160 retirement rollback backup 99615' --mode snapshot --storage local --compress zstd --node pve02 -TASK_LOG_02=INFO: Starting Backup of VM 160 (qemu) -TASK_LOG_03=INFO: Backup started at 2026-10-05 19:24:09 -TASK_LOG_04=INFO: status = running -TASK_LOG_05=INFO: VM Name: forum-prod -TASK_LOG_06=INFO: include disk 'scsi0' 'local-lvm:vm-160-disk-0' 200G -TASK_LOG_07=INFO: backup mode: snapshot -TASK_LOG_08=INFO: ionice priority: 7 -TASK_LOG_09=INFO: creating vzdump archive '/var/lib/vz/dump/vzdump-qemu-160-2026_10_05-19_24_09.vma.zst' -TASK_LOG_10=INFO: issuing guest-agent 'fs-freeze' command -TASK_LOG_11=INFO: starting backup via QMP command -TASK_LOG_12=INFO: issuing guest-agent 'fs-thaw' command -TASK_LOG_13=INFO: started backup task '41ea49b6-7bd9-4883-a24f-722e1d18fca0' -TASK_LOG_14=INFO: resuming VM again -TASK_LOG_15=INFO: 0% (1.3 GiB of 200.0 GiB) in 3s, read: 455.5 MiB/s, write: 189.7 MiB/s -TASK_LOG_16=INFO: 2% (4.1 GiB of 200.0 GiB) in 6s, read: 952.8 MiB/s, write: 158.5 MiB/s -TASK_LOG_17=INFO: 4% (8.9 GiB of 200.0 GiB) in 9s, read: 1.6 GiB/s, write: 126.9 MiB/s -TASK_LOG_18=INFO: 6% (12.2 GiB of 200.0 GiB) in 12s, read: 1.1 GiB/s, write: 136.8 MiB/s -TASK_LOG_19=INFO: 7% (14.3 GiB of 200.0 GiB) in 15s, read: 686.3 MiB/s, write: 142.4 MiB/s -TASK_LOG_20=INFO: 9% (18.4 GiB of 200.0 GiB) in 18s, read: 1.4 GiB/s, write: 72.2 MiB/s -TASK_LOG_21=INFO: 10% (20.3 GiB of 200.0 GiB) in 22s, read: 474.9 MiB/s, write: 163.3 MiB/s -TASK_LOG_22=INFO: 11% (22.6 GiB of 200.0 GiB) in 25s, read: 795.1 MiB/s, write: 165.3 MiB/s -TASK_LOG_23=INFO: 12% (24.4 GiB of 200.0 GiB) in 28s, read: 599.3 MiB/s, write: 192.6 MiB/s -TASK_LOG_24=INFO: 13% (26.1 GiB of 200.0 GiB) in 35s, read: 257.2 MiB/s, write: 111.2 MiB/s -TASK_LOG_25=INFO: 14% (28.3 GiB of 200.0 GiB) in 38s, read: 727.7 MiB/s, write: 161.0 MiB/s -TASK_LOG_26=INFO: 16% (32.1 GiB of 200.0 GiB) in 41s, read: 1.3 GiB/s, write: 118.1 MiB/s -TASK_LOG_27=INFO: 18% (36.9 GiB of 200.0 GiB) in 44s, read: 1.6 GiB/s, write: 56.1 MiB/s -TASK_LOG_28=TASK ERROR: broken pipe -LOG_HAS_ZSTD_ERROR=false -LOG_HAS_NO_SPACE=false -LOG_HAS_IO_ERROR=false -LOG_HAS_VMA_QUEUE_WRITE=false -LOG_HAS_ABORTING=false -LOG_HAS_GUEST_THAW=true -LOG_HAS_PROGRESS=true -PROGRESS_LINE_COUNT=13 -LAST_PROGRESS_LINE=INFO: 18% (36.9 GiB of 200.0 GiB) in 44s, read: 1.6 GiB/s, write: 56.1 MiB/s -NODE_JOURNAL_RC=0 -NODE_JOURNAL_LINE_COUNT=10002 -NODE_JOURNAL_SHA=b78892f57eea13b97d0d00ae4a3abc977fe08fbd927622c0b4848733dfddfd6e -NODE_JOURNAL_RELEVANT_COUNT=63 -NODE_JOURNAL_WINDOW_COUNT=6 -NODE_JOURNAL_01=Oct 05 19:24:09 pve02 pvesh[4125258]: starting task UPID:pve02:003EF24D:13A18FC0:6AC3CF29:vzdump:160:root@pam: -NODE_JOURNAL_02=Oct 05 19:24:09 pve02 pvesh[4125261]: INFO: starting new backup job: vzdump 160 --notes-template 'WORKERS2 VM160 retirement rollback backup 99615' --mode snapshot --storage local --compress zstd --node pve02 -NODE_JOURNAL_03=Oct 05 19:24:09 pve02 pvesh[4125261]: INFO: Starting Backup of VM 160 (qemu) -NODE_JOURNAL_04=Oct 05 19:24:56 pve02 pvesh[4125261]: ERROR: Backup of VM 160 failed - interrupted by signal -NODE_JOURNAL_05=Oct 05 19:24:56 pve02 pvesh[4125261]: ERROR: Backup job failed - interrupted by signal -NODE_JOURNAL_06=Oct 05 19:24:57 pve02 pvesh[4125258]: end task UPID:pve02:003EF24D:13A18FC0:6AC3CF29:vzdump:160:root@pam: broken pipe -JOURNAL_OOM_SIGNAL=false -JOURNAL_IO_ERROR_SIGNAL=false -JOURNAL_EXT4_ERROR_SIGNAL=false -JOURNAL_ZSTD_SIGNAL=true -JOURNAL_QEMU_SIGNAL=true -LEFTOVER_FIND_RC=0 -LEFTOVER_160_FILE_COUNT=0 -PKG_PVE_MANAGER_VER_SHA=4bc0532243c6a7480dabf2cb6704e8645fef799b9561a78bc98dae955bec9d19 -PKG_QEMU_SERVER_VER_SHA=86b252f483e9d9c1ecf7a79de6136570ddae05a37b103287f5e5d290b6792269 -PKG_PVE_QEMU_KVM_VER_SHA=6f9cdce1fb8c1849f0ed78ae68aadf9744230ece98a33a5bcabcfd4e914dbb38 -PKG_ZSTD_VER_SHA=42f9875a37197a0a9eaa49c6fc729b93cf63e2d3c2dc11e7d28dd8798ca7e5da -HOST_MEMINFO_SHA=890013975f41093d8759e94305a292094c46b0061a63aff051fd908f14499b6e -HOST_MEMTOTAL_KB=32596696 -HOST_MEMAVAILABLE_KB=8851192 -HOST_SWAPTOTAL_KB=10485756 -HOST_SWAPFREE_KB=7745016 -PIPE_ROOT_CLASS=PIPE_CLOSED_WITHOUT_EXPLICIT_CONSUMER_ERROR -RCA_PROVEN=true -DECISION=PASS_VM160_VZDUMP_PIPE_PRODUCER_RCA -NEXT_GATE=VM160_BACKUP_PIPE_SAFE_FIX +NO_BEGET_MUTATION=true +NO_DB_CONTENT_MUTATION=true +PUBLIC_PRE_ALL_SIX_OK=true +CODEVIPE_RU_PUBLIC_PRE_OK=true +CODEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true +HKMODS_RU_PUBLIC_PRE_OK=true +HKMODS_RU_PUBLIC_PRE_BEGET_HINT=true +DSMODS_RU_PUBLIC_PRE_OK=true +DSMODS_RU_PUBLIC_PRE_BEGET_HINT=true +GAMEVIPE_RU_PUBLIC_PRE_OK=true +GAMEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true +KINGOFWOLK_RU_PUBLIC_PRE_OK=true +KINGOFWOLK_RU_PUBLIC_PRE_BEGET_HINT=true +ZAKRUTIM_RU_PUBLIC_PRE_OK=true +ZAKRUTIM_RU_PUBLIC_PRE_BEGET_HINT=true +VM160_PRE_FOUND=true +VM160_PRE_NODE=pve02 +VM160_PRE_STATE=running +PRE_GUEST_QGA_RC=0 +PRE_GUEST_EXITCODE=0 +CRON_ACTIVE_PRESENT=true +CRON_99621_DISABLED_PRESENT=false +STATE_99621_DIR_PRESENT=false +RUNNER_PROCESS_COUNT=0 +MATCH_TIMER_COUNT=1 +MATCH_TIMER_ACTIVE=1 +MATCH_TIMER_ENABLED=1 +DB_ROOT_FILE_COUNT=6 +VM160_PRECHANGE_GUEST_OK=true +ALL_SIX_SOURCE_BOARDS_INACTIVE=true +QUIESCE_UNTOUCHED_PRE=true +BACKUP_STORAGE_CANDIDATE_COUNT=1 +BACKUP_STORAGE_NAME=local +BACKUP_STORAGE_AVAILABLE_BYTES=87478521856 +BACKUP_STORAGE_TOTAL_BYTES=105089261568 +BACKUP_REQUIRED_MIN_FREE_BYTES=34359738368 +BACKUP_SPACE_PREFLIGHT_OK=true +VM160_BACKUP_TASK_ALREADY_RUNNING=false +VM160_BACKUP_COUNT_PRE=0 +TRANSIENT_UNIT_PREEXISTS=false +BACKUP_START_EPOCH=1791222811 +DETACHED_LAUNCH_RC=0 +DETACHED_LAUNCH_OUTPUT_SHA=dff3498d460ccce6681bd424902dec8994b870f02d9ffb229900bb3c122499c3 +BACKUP_TASK_DISCOVERED=false +BACKUP_TASK_UPID_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +DECISION=HOLD_BACKUP_TASK_NOT_DISCOVERED +NEXT_GATE=VM160_BACKUP_TASK_RECONCILE COMMANDS_REMAINING_AFTER_SUCCESS=2 -TASK_RESULT=VM160_VZDUMP_PIPE_PRODUCER_RCA -CHANGES_MADE=false +TASK_RESULT=VM160_BACKUP_QUIESCE_WATCHDOG_FIX +CHANGES_MADE=true ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=NO_MUTATION +MUTATION_OUTCOME=DETACHED_LAUNCH_SUBMITTED_TASK_NOT_DISCOVERED +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END