From 3ad8789af44c1edf5aed05e9b46c83f489383ebe Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Mon, 5 Oct 2026 18:06:16 +0000 Subject: [PATCH] runtime: publish SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1 --- ...160-DETACHED-BACKUP-RECONCILE-99622R1.json | 38 ++++ ...M160-DETACHED-BACKUP-RECONCILE-99622R1.txt | 133 ++++++++++++++ runtime/latest.json | 32 ++-- runtime/latest.txt | 167 +++++++++++------- 4 files changed, 289 insertions(+), 81 deletions(-) create mode 100644 runtime/history/SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1.json create mode 100644 runtime/history/SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1.txt diff --git a/runtime/history/SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1.json b/runtime/history/SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1.json new file mode 100644 index 00000000..b7751de0 --- /dev/null +++ b/runtime/history/SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "vm160-detached-backup-reconcile", + "started_at_utc": "2026-10-05T18:06:04Z", + "finished_at_utc": "2026-10-05T18:06:15Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "e9e452bc37b34b5debb06af949b6021c45323639da2ecfe78074a74c28e1c5d0", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "fbd4bd5f8bc59a62f82272571e97763ab766f44dd0ea83ad003ad4ff9529d457", + "sanitized_output_sha256": "fbd4bd5f8bc59a62f82272571e97763ab766f44dd0ea83ad003ad4ff9529d457", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "VM160_DETACHED_BACKUP_RECONCILE=V1\nMODE=read-only\nNO_VM_MUTATION=true\nNO_BACKUP_CREATE=true\nNO_BACKUP_DELETE=true\nNO_CONFIG_WRITE=true\nNO_SERVICE_RESTART=true\nNO_GUEST_MUTATION=true\nNO_DNS_MUTATION=true\nNO_PROXY_MUTATION=true\nVM160_FOUND=true\nVM160_NODE=pve02\nVM160_STATE=running\nTRANSIENT_SYSTEMCTL_SHOW_RC=0\nTRANSIENT_LOADSTATE=MISSING\nTRANSIENT_ACTIVESTATE=MISSING\nTRANSIENT_SUBSTATE=MISSING\nTRANSIENT_RESULT=MISSING\nTRANSIENT_EXECMAINCODE=MISSING\nTRANSIENT_EXECMAINSTATUS=MISSING\nTRANSIENT_EXECMAINSTARTTIMESTAMP=MISSING\nTRANSIENT_EXECMAINEXITTIMESTAMP=MISSING\nTRANSIENT_JOURNAL_RC=0\nTRANSIENT_JOURNAL_LINE_COUNT=4\nTRANSIENT_JOURNAL_SHA=2d5bd1f2dedb7adcfca35ec5ced2669d94b9fb4e9b05de61993fadb0dd8d7402\nTRANSIENT_JOURNAL_01=2026-10-05T20:53:31+03:00 pve01 systemd[1]: Started workers2-vm160-vzdump-99621.service - [systemd-run] /usr/bin/pvesh create /nodes/pve02/vzdump --vmid 160 --storage local --mode snapshot --compress zstd --notes-template \"WORKERS2 VM160 retirement rollback backup 99621\" --output-format json.\nTRANSIENT_JOURNAL_02=2026-10-05T20:56:31+03:00 pve01 pvesh[742777]: \"INFO: starting new backup job: vzdump 160 --mode snapshot --node pve02 --compress zstd --storage local --notes-template 'WORKERS2 VM160 retirement rollback backup 99621'INFO: Starting Backup of VM 160 (qemu)INFO: Backup started at 2026-10-05 20:53:33INFO: status = runningINFO: VM Name: forum-prodINFO: include disk 'scsi0' 'local-lvm:vm-160-disk-0' 200GINFO: backup mode: snapshotINFO: ionice priority: 7INFO: creating vzdump archive '/var/lib/vz/dump/vzdump-qemu-160-2026_10_05-20_53_32.vma.zst'INFO: issuing guest-agent 'fs-freeze' commandINFO: starting backup via QMP commandINFO: issuing guest-agent 'fs-thaw' commandINFO: started backup task '1d65b799-ca0b-4367-9e10-ac7daf5c0fd4'INFO: resuming VM againINFO: 0% (1.2 GiB of 200.0 GiB) in 3s, read: 419.0 MiB/s, write: 174.3 MiB/sINFO: 1% (3.4 GiB of 200.0 GiB) in 6s, read: 746.7 MiB/s, write\nTRANSIENT_JOURNAL_03=2026-10-05T20:56:31+03:00 pve01 systemd[1]: workers2-vm160-vzdump-99621.service: Deactivated successfully.\nTRANSIENT_JOURNAL_04=2026-10-05T20:56:31+03:00 pve01 systemd[1]: workers2-vm160-vzdump-99621.service: Consumed 623ms CPU time, 143M memory peak.\nPVE01_TASK_API_RC=0\nPVE02_TASK_API_RC=0\nCLUSTER_TASK_API_RC=255\nRECONCILE_VZDUMP_TASK_COUNT=1\nTASK_1_SOURCE=PVE02\nTASK_1_UPID_SHA=3664f926f82f37a33bdf206b1c143a9be85506d90db2fbd929988f6c9fcb7b46\nTASK_1_START=1791222812\nTASK_1_END=1791222991\nTASK_1_STATUS_LIST=OK\nTASK_1_EXITSTATUS_LIST=\nTASK_1_STATUS_API_RC=0\nTASK_1_STATUS_API_STATUS=stopped\nTASK_1_STATUS_API_EXITSTATUS=OK\nTASK_1_LOG_RC=0\nTASK_1_LOG_SHA=7fd6e449501c5c1ab989335618e616330941bf468c7ae409ce9e617f89cd14ab\nTASK_1_LOG_01=INFO: 16% (32.2 GiB of 200.0 GiB) in 42s, read: 1.3 GiB/s, write: 123.6 MiB/s\nTASK_1_LOG_02=INFO: 17% (35.9 GiB of 200.0 GiB) in 45s, read: 1.2 GiB/s, write: 43.2 MiB/s\nTASK_1_LOG_03=INFO: 18% (38.0 GiB of 200.0 GiB) in 48s, read: 699.0 MiB/s, write: 27.3 MiB/s\nTASK_1_LOG_04=INFO: 20% (40.9 GiB of 200.0 GiB) in 51s, read: 991.3 MiB/s, write: 19.5 MiB/s\nTASK_1_LOG_05=INFO: 21% (43.6 GiB of 200.0 GiB) in 54s, read: 914.0 MiB/s, write: 75.2 MiB/s\nTASK_1_LOG_06=INFO: 24% (49.5 GiB of 200.0 GiB) in 57s, read: 2.0 GiB/s, write: 43.2 MiB/s\nTASK_1_LOG_07=INFO: 25% (51.3 GiB of 200.0 GiB) in 1m, read: 613.6 MiB/s, write: 42.5 MiB/s\nTASK_1_LOG_08=INFO: 26% (52.3 GiB of 200.0 GiB) in 1m 7s, read: 141.1 MiB/s, write: 88.0 MiB/s\nTASK_1_LOG_09=INFO: 27% (54.1 GiB of 200.0 GiB) in 1m 13s, read: 318.3 MiB/s, write: 18.5 MiB/s\nTASK_1_LOG_10=INFO: 29% (59.8 GiB of 200.0 GiB) in 1m 16s, read: 1.9 GiB/s, write: 18.4 MiB/s\nTASK_1_LOG_11=INFO: 31% (63.2 GiB of 200.0 GiB) in 1m 19s, read: 1.1 GiB/s, write: 27.9 MiB/s\nTASK_1_LOG_12=INFO: 34% (69.3 GiB of 200.0 GiB) in 1m 22s, read: 2.0 GiB/s, write: 9.0 MiB/s\nTASK_1_LOG_13=INFO: 35% (70.3 GiB of 200.0 GiB) in 1m 25s, read: 351.3 MiB/s, write: 44.6 MiB/s\nTASK_1_LOG_14=INFO: 38% (76.1 GiB of 200.0 GiB) in 1m 29s, read: 1.5 GiB/s, write: 15.9 MiB/s\nTASK_1_LOG_15=INFO: 39% (78.3 GiB of 200.0 GiB) in 1m 32s, read: 748.9 MiB/s, write: 11.9 MiB/s\nTASK_1_LOG_16=INFO: 40% (80.1 GiB of 200.0 GiB) in 1m 38s, read: 312.2 MiB/s, write: 21.2 MiB/s\nTASK_1_LOG_17=INFO: 41% (82.1 GiB of 200.0 GiB) in 1m 42s, read: 514.0 MiB/s, write: 17.5 MiB/s\nTASK_1_LOG_18=INFO: 42% (84.1 GiB of 200.0 GiB) in 1m 51s, read: 227.7 MiB/s, write: 19.6 MiB/s\nTASK_1_LOG_19=INFO: 43% (87.3 GiB of 200.0 GiB) in 1m 57s, read: 548.2 MiB/s, write: 15.7 MiB/s\nTASK_1_LOG_20=INFO: 46% (93.4 GiB of 200.0 GiB) in 2m, read: 2.0 GiB/s, write: 27.9 MiB/s\nTASK_1_LOG_21=INFO: 50% (100.7 GiB of 200.0 GiB) in 2m 4s, read: 1.8 GiB/s, write: [PRIVATE_IP] MiB/s\nTASK_1_LOG_22=INFO: 53% (107.1 GiB of 200.0 GiB) in 2m 7s, read: 2.2 GiB/s, write: 14.1 MiB/s\nTASK_1_LOG_23=INFO: 56% (113.5 GiB of 200.0 GiB) in 2m 10s, read: 2.1 GiB/s, write: 5.3 KiB/s\nTASK_1_LOG_24=INFO: 59% (118.7 GiB of 200.0 GiB) in 2m 13s, read: 1.7 GiB/s, write: 5.9 MiB/s\nTASK_1_LOG_25=INFO: 63% (126.1 GiB of 200.0 GiB) in 2m 16s, read: 2.5 GiB/s, write: 4.5 MiB/s\nANY_TASK_OK=true\nANY_TASK_RUNNING=false\nANY_TASK_FAILED=false\nVM160_BACKUP_COUNT_NOW=1\nVM160_BACKUP_FRESH_MATCH_COUNT=1\nFRESH_BACKUP_CTIME=1791222812\nFRESH_BACKUP_SIZE=3905927732\nFRESH_BACKUP_FORMAT=vma.zst\nFRESH_BACKUP_VOLID_SHA=82623957583673e0406fb0db7ad1cbb7b07c3189cd136dc397362d9fc23d0fdb\nFRESH_BACKUP_VERIFIED=true\nDUMP_FILE_SCAN_RC=0\nDUMP_160_FILE_COUNT=0\nVM160_QGA_RC=0\nVM160_GUEST_EXITCODE=0\nCRON_ACTIVE_PRESENT=true\nCRON_99621_DISABLED_PRESENT=false\nSTATE_99621_DIR_PRESENT=false\nRUNNER_PROCESS_COUNT=0\nMATCH_TIMER_COUNT=1\nMATCH_TIMER_ACTIVE=1\nMATCH_TIMER_ENABLED=1\nVM160_GUEST_RECONCILE_OK=true\nQUIESCE_UNTOUCHED=true\nQUIESCE_DONE=false\nDECISION=PASS_99621_BACKUP_COMPLETE_QUIESCE_PENDING\nNEXT_GATE=VM160_QUIESCE_ONLY_AFTER_VERIFIED_BACKUP\nCOMMANDS_REMAINING_AFTER_SUCCESS=2\nTASK_RESULT=VM160_DETACHED_BACKUP_RECONCILE\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\n" +} diff --git a/runtime/history/SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1.txt b/runtime/history/SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1.txt new file mode 100644 index 00000000..a5d6986f --- /dev/null +++ b/runtime/history/SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1.txt @@ -0,0 +1,133 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=vm160-detached-backup-reconcile +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=e9e452bc37b34b5debb06af949b6021c45323639da2ecfe78074a74c28e1c5d0 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=fbd4bd5f8bc59a62f82272571e97763ab766f44dd0ea83ad003ad4ff9529d457 +SANITIZED_OUTPUT_SHA256=fbd4bd5f8bc59a62f82272571e97763ab766f44dd0ea83ad003ad4ff9529d457 +OUTPUT_BEGIN +VM160_DETACHED_BACKUP_RECONCILE=V1 +MODE=read-only +NO_VM_MUTATION=true +NO_BACKUP_CREATE=true +NO_BACKUP_DELETE=true +NO_CONFIG_WRITE=true +NO_SERVICE_RESTART=true +NO_GUEST_MUTATION=true +NO_DNS_MUTATION=true +NO_PROXY_MUTATION=true +VM160_FOUND=true +VM160_NODE=pve02 +VM160_STATE=running +TRANSIENT_SYSTEMCTL_SHOW_RC=0 +TRANSIENT_LOADSTATE=MISSING +TRANSIENT_ACTIVESTATE=MISSING +TRANSIENT_SUBSTATE=MISSING +TRANSIENT_RESULT=MISSING +TRANSIENT_EXECMAINCODE=MISSING +TRANSIENT_EXECMAINSTATUS=MISSING +TRANSIENT_EXECMAINSTARTTIMESTAMP=MISSING +TRANSIENT_EXECMAINEXITTIMESTAMP=MISSING +TRANSIENT_JOURNAL_RC=0 +TRANSIENT_JOURNAL_LINE_COUNT=4 +TRANSIENT_JOURNAL_SHA=2d5bd1f2dedb7adcfca35ec5ced2669d94b9fb4e9b05de61993fadb0dd8d7402 +TRANSIENT_JOURNAL_01=2026-10-05T20:53:31+03:00 pve01 systemd[1]: Started workers2-vm160-vzdump-99621.service - [systemd-run] /usr/bin/pvesh create /nodes/pve02/vzdump --vmid 160 --storage local --mode snapshot --compress zstd --notes-template "WORKERS2 VM160 retirement rollback backup 99621" --output-format json. +TRANSIENT_JOURNAL_02=2026-10-05T20:56:31+03:00 pve01 pvesh[742777]: "INFO: starting new backup job: vzdump 160 --mode snapshot --node pve02 --compress zstd --storage local --notes-template 'WORKERS2 VM160 retirement rollback backup 99621'INFO: Starting Backup of VM 160 (qemu)INFO: Backup started at 2026-10-05 20:53:33INFO: status = runningINFO: VM Name: forum-prodINFO: include disk 'scsi0' 'local-lvm:vm-160-disk-0' 200GINFO: backup mode: snapshotINFO: ionice priority: 7INFO: creating vzdump archive '/var/lib/vz/dump/vzdump-qemu-160-2026_10_05-20_53_32.vma.zst'INFO: issuing guest-agent 'fs-freeze' commandINFO: starting backup via QMP commandINFO: issuing guest-agent 'fs-thaw' commandINFO: started backup task '1d65b799-ca0b-4367-9e10-ac7daf5c0fd4'INFO: resuming VM againINFO: 0% (1.2 GiB of 200.0 GiB) in 3s, read: 419.0 MiB/s, write: 174.3 MiB/sINFO: 1% (3.4 GiB of 200.0 GiB) in 6s, read: 746.7 MiB/s, write +TRANSIENT_JOURNAL_03=2026-10-05T20:56:31+03:00 pve01 systemd[1]: workers2-vm160-vzdump-99621.service: Deactivated successfully. +TRANSIENT_JOURNAL_04=2026-10-05T20:56:31+03:00 pve01 systemd[1]: workers2-vm160-vzdump-99621.service: Consumed 623ms CPU time, 143M memory peak. +PVE01_TASK_API_RC=0 +PVE02_TASK_API_RC=0 +CLUSTER_TASK_API_RC=255 +RECONCILE_VZDUMP_TASK_COUNT=1 +TASK_1_SOURCE=PVE02 +TASK_1_UPID_SHA=3664f926f82f37a33bdf206b1c143a9be85506d90db2fbd929988f6c9fcb7b46 +TASK_1_START=1791222812 +TASK_1_END=1791222991 +TASK_1_STATUS_LIST=OK +TASK_1_EXITSTATUS_LIST= +TASK_1_STATUS_API_RC=0 +TASK_1_STATUS_API_STATUS=stopped +TASK_1_STATUS_API_EXITSTATUS=OK +TASK_1_LOG_RC=0 +TASK_1_LOG_SHA=7fd6e449501c5c1ab989335618e616330941bf468c7ae409ce9e617f89cd14ab +TASK_1_LOG_01=INFO: 16% (32.2 GiB of 200.0 GiB) in 42s, read: 1.3 GiB/s, write: 123.6 MiB/s +TASK_1_LOG_02=INFO: 17% (35.9 GiB of 200.0 GiB) in 45s, read: 1.2 GiB/s, write: 43.2 MiB/s +TASK_1_LOG_03=INFO: 18% (38.0 GiB of 200.0 GiB) in 48s, read: 699.0 MiB/s, write: 27.3 MiB/s +TASK_1_LOG_04=INFO: 20% (40.9 GiB of 200.0 GiB) in 51s, read: 991.3 MiB/s, write: 19.5 MiB/s +TASK_1_LOG_05=INFO: 21% (43.6 GiB of 200.0 GiB) in 54s, read: 914.0 MiB/s, write: 75.2 MiB/s +TASK_1_LOG_06=INFO: 24% (49.5 GiB of 200.0 GiB) in 57s, read: 2.0 GiB/s, write: 43.2 MiB/s +TASK_1_LOG_07=INFO: 25% (51.3 GiB of 200.0 GiB) in 1m, read: 613.6 MiB/s, write: 42.5 MiB/s +TASK_1_LOG_08=INFO: 26% (52.3 GiB of 200.0 GiB) in 1m 7s, read: 141.1 MiB/s, write: 88.0 MiB/s +TASK_1_LOG_09=INFO: 27% (54.1 GiB of 200.0 GiB) in 1m 13s, read: 318.3 MiB/s, write: 18.5 MiB/s +TASK_1_LOG_10=INFO: 29% (59.8 GiB of 200.0 GiB) in 1m 16s, read: 1.9 GiB/s, write: 18.4 MiB/s +TASK_1_LOG_11=INFO: 31% (63.2 GiB of 200.0 GiB) in 1m 19s, read: 1.1 GiB/s, write: 27.9 MiB/s +TASK_1_LOG_12=INFO: 34% (69.3 GiB of 200.0 GiB) in 1m 22s, read: 2.0 GiB/s, write: 9.0 MiB/s +TASK_1_LOG_13=INFO: 35% (70.3 GiB of 200.0 GiB) in 1m 25s, read: 351.3 MiB/s, write: 44.6 MiB/s +TASK_1_LOG_14=INFO: 38% (76.1 GiB of 200.0 GiB) in 1m 29s, read: 1.5 GiB/s, write: 15.9 MiB/s +TASK_1_LOG_15=INFO: 39% (78.3 GiB of 200.0 GiB) in 1m 32s, read: 748.9 MiB/s, write: 11.9 MiB/s +TASK_1_LOG_16=INFO: 40% (80.1 GiB of 200.0 GiB) in 1m 38s, read: 312.2 MiB/s, write: 21.2 MiB/s +TASK_1_LOG_17=INFO: 41% (82.1 GiB of 200.0 GiB) in 1m 42s, read: 514.0 MiB/s, write: 17.5 MiB/s +TASK_1_LOG_18=INFO: 42% (84.1 GiB of 200.0 GiB) in 1m 51s, read: 227.7 MiB/s, write: 19.6 MiB/s +TASK_1_LOG_19=INFO: 43% (87.3 GiB of 200.0 GiB) in 1m 57s, read: 548.2 MiB/s, write: 15.7 MiB/s +TASK_1_LOG_20=INFO: 46% (93.4 GiB of 200.0 GiB) in 2m, read: 2.0 GiB/s, write: 27.9 MiB/s +TASK_1_LOG_21=INFO: 50% (100.7 GiB of 200.0 GiB) in 2m 4s, read: 1.8 GiB/s, write: [PRIVATE_IP] MiB/s +TASK_1_LOG_22=INFO: 53% (107.1 GiB of 200.0 GiB) in 2m 7s, read: 2.2 GiB/s, write: 14.1 MiB/s +TASK_1_LOG_23=INFO: 56% (113.5 GiB of 200.0 GiB) in 2m 10s, read: 2.1 GiB/s, write: 5.3 KiB/s +TASK_1_LOG_24=INFO: 59% (118.7 GiB of 200.0 GiB) in 2m 13s, read: 1.7 GiB/s, write: 5.9 MiB/s +TASK_1_LOG_25=INFO: 63% (126.1 GiB of 200.0 GiB) in 2m 16s, read: 2.5 GiB/s, write: 4.5 MiB/s +ANY_TASK_OK=true +ANY_TASK_RUNNING=false +ANY_TASK_FAILED=false +VM160_BACKUP_COUNT_NOW=1 +VM160_BACKUP_FRESH_MATCH_COUNT=1 +FRESH_BACKUP_CTIME=1791222812 +FRESH_BACKUP_SIZE=3905927732 +FRESH_BACKUP_FORMAT=vma.zst +FRESH_BACKUP_VOLID_SHA=82623957583673e0406fb0db7ad1cbb7b07c3189cd136dc397362d9fc23d0fdb +FRESH_BACKUP_VERIFIED=true +DUMP_FILE_SCAN_RC=0 +DUMP_160_FILE_COUNT=0 +VM160_QGA_RC=0 +VM160_GUEST_EXITCODE=0 +CRON_ACTIVE_PRESENT=true +CRON_99621_DISABLED_PRESENT=false +STATE_99621_DIR_PRESENT=false +RUNNER_PROCESS_COUNT=0 +MATCH_TIMER_COUNT=1 +MATCH_TIMER_ACTIVE=1 +MATCH_TIMER_ENABLED=1 +VM160_GUEST_RECONCILE_OK=true +QUIESCE_UNTOUCHED=true +QUIESCE_DONE=false +DECISION=PASS_99621_BACKUP_COMPLETE_QUIESCE_PENDING +NEXT_GATE=VM160_QUIESCE_ONLY_AFTER_VERIFIED_BACKUP +COMMANDS_REMAINING_AFTER_SUCCESS=2 +TASK_RESULT=VM160_DETACHED_BACKUP_RECONCILE +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 904f1efd..b7751de0 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1", - "status": "FAIL", - "rc": 68, + "command_id": "SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "vm160-backup-quiesce-watchdog-fix", - "started_at_utc": "2026-10-05T17:53:19Z", - "finished_at_utc": "2026-10-05T17:55:03Z", + "mode": "read-only", + "component": "vm160-detached-backup-reconcile", + "started_at_utc": "2026-10-05T18:06:04Z", + "finished_at_utc": "2026-10-05T18:06:15Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "d8edc0bcd4d08329eae8a011660c73b1a355cfb2a456146ef6590ed72bc36b2d", + "command_sha256": "e9e452bc37b34b5debb06af949b6021c45323639da2ecfe78074a74c28e1c5d0", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, + "change_declared": false, "result_contract_valid": true, - "result_contract_status": "FAIL", + "result_contract_status": null, "result_contract_error": null, - "command_rc": 68, - "changes_made": true, + "command_rc": 0, + "changes_made": false, "rollback_started": false, "rollback_restored": null, - "mutation_outcome": "MUTATED", + "mutation_outcome": "NO_MUTATION", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d", - "sanitized_output_sha256": "3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d", + "raw_evidence_sha256": "fbd4bd5f8bc59a62f82272571e97763ab766f44dd0ea83ad003ad4ff9529d457", + "sanitized_output_sha256": "fbd4bd5f8bc59a62f82272571e97763ab766f44dd0ea83ad003ad4ff9529d457", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "VM160_BACKUP_QUIESCE_WATCHDOG_FIX=V1\nMODE=change\nTARGET_VM=160\nTARGET_NODE=pve02\nWATCHDOG_RCA_PROVEN=true\nWATCHDOG_RCA=CONTROLLER_PVESH_TIMEOUT_45S_INTERRUPTED_BACKUP\nBACKUP_LAUNCH_MODEL=DETACHED_SYSTEMD_TRANSIENT_PVESH\nNO_VM_SHUTDOWN=true\nNO_DNS_MUTATION=true\nNO_PROXY_MUTATION=true\nNO_BEGET_MUTATION=true\nNO_DB_CONTENT_MUTATION=true\nPUBLIC_PRE_ALL_SIX_OK=true\nCODEVIPE_RU_PUBLIC_PRE_OK=true\nCODEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true\nHKMODS_RU_PUBLIC_PRE_OK=true\nHKMODS_RU_PUBLIC_PRE_BEGET_HINT=true\nDSMODS_RU_PUBLIC_PRE_OK=true\nDSMODS_RU_PUBLIC_PRE_BEGET_HINT=true\nGAMEVIPE_RU_PUBLIC_PRE_OK=true\nGAMEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true\nKINGOFWOLK_RU_PUBLIC_PRE_OK=true\nKINGOFWOLK_RU_PUBLIC_PRE_BEGET_HINT=true\nZAKRUTIM_RU_PUBLIC_PRE_OK=true\nZAKRUTIM_RU_PUBLIC_PRE_BEGET_HINT=true\nVM160_PRE_FOUND=true\nVM160_PRE_NODE=pve02\nVM160_PRE_STATE=running\nPRE_GUEST_QGA_RC=0\nPRE_GUEST_EXITCODE=0\nCRON_ACTIVE_PRESENT=true\nCRON_99621_DISABLED_PRESENT=false\nSTATE_99621_DIR_PRESENT=false\nRUNNER_PROCESS_COUNT=0\nMATCH_TIMER_COUNT=1\nMATCH_TIMER_ACTIVE=1\nMATCH_TIMER_ENABLED=1\nDB_ROOT_FILE_COUNT=6\nVM160_PRECHANGE_GUEST_OK=true\nALL_SIX_SOURCE_BOARDS_INACTIVE=true\nQUIESCE_UNTOUCHED_PRE=true\nBACKUP_STORAGE_CANDIDATE_COUNT=1\nBACKUP_STORAGE_NAME=local\nBACKUP_STORAGE_AVAILABLE_BYTES=87478521856\nBACKUP_STORAGE_TOTAL_BYTES=105089261568\nBACKUP_REQUIRED_MIN_FREE_BYTES=34359738368\nBACKUP_SPACE_PREFLIGHT_OK=true\nVM160_BACKUP_TASK_ALREADY_RUNNING=false\nVM160_BACKUP_COUNT_PRE=0\nTRANSIENT_UNIT_PREEXISTS=false\nBACKUP_START_EPOCH=1791222811\nDETACHED_LAUNCH_RC=0\nDETACHED_LAUNCH_OUTPUT_SHA=dff3498d460ccce6681bd424902dec8994b870f02d9ffb229900bb3c122499c3\nBACKUP_TASK_DISCOVERED=false\nBACKUP_TASK_UPID_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nDECISION=HOLD_BACKUP_TASK_NOT_DISCOVERED\nNEXT_GATE=VM160_BACKUP_TASK_RECONCILE\nCOMMANDS_REMAINING_AFTER_SUCCESS=2\nTASK_RESULT=VM160_BACKUP_QUIESCE_WATCHDOG_FIX\nCHANGES_MADE=true\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=DETACHED_LAUNCH_SUBMITTED_TASK_NOT_DISCOVERED\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\n" + "output": "VM160_DETACHED_BACKUP_RECONCILE=V1\nMODE=read-only\nNO_VM_MUTATION=true\nNO_BACKUP_CREATE=true\nNO_BACKUP_DELETE=true\nNO_CONFIG_WRITE=true\nNO_SERVICE_RESTART=true\nNO_GUEST_MUTATION=true\nNO_DNS_MUTATION=true\nNO_PROXY_MUTATION=true\nVM160_FOUND=true\nVM160_NODE=pve02\nVM160_STATE=running\nTRANSIENT_SYSTEMCTL_SHOW_RC=0\nTRANSIENT_LOADSTATE=MISSING\nTRANSIENT_ACTIVESTATE=MISSING\nTRANSIENT_SUBSTATE=MISSING\nTRANSIENT_RESULT=MISSING\nTRANSIENT_EXECMAINCODE=MISSING\nTRANSIENT_EXECMAINSTATUS=MISSING\nTRANSIENT_EXECMAINSTARTTIMESTAMP=MISSING\nTRANSIENT_EXECMAINEXITTIMESTAMP=MISSING\nTRANSIENT_JOURNAL_RC=0\nTRANSIENT_JOURNAL_LINE_COUNT=4\nTRANSIENT_JOURNAL_SHA=2d5bd1f2dedb7adcfca35ec5ced2669d94b9fb4e9b05de61993fadb0dd8d7402\nTRANSIENT_JOURNAL_01=2026-10-05T20:53:31+03:00 pve01 systemd[1]: Started workers2-vm160-vzdump-99621.service - [systemd-run] /usr/bin/pvesh create /nodes/pve02/vzdump --vmid 160 --storage local --mode snapshot --compress zstd --notes-template \"WORKERS2 VM160 retirement rollback backup 99621\" --output-format json.\nTRANSIENT_JOURNAL_02=2026-10-05T20:56:31+03:00 pve01 pvesh[742777]: \"INFO: starting new backup job: vzdump 160 --mode snapshot --node pve02 --compress zstd --storage local --notes-template 'WORKERS2 VM160 retirement rollback backup 99621'INFO: Starting Backup of VM 160 (qemu)INFO: Backup started at 2026-10-05 20:53:33INFO: status = runningINFO: VM Name: forum-prodINFO: include disk 'scsi0' 'local-lvm:vm-160-disk-0' 200GINFO: backup mode: snapshotINFO: ionice priority: 7INFO: creating vzdump archive '/var/lib/vz/dump/vzdump-qemu-160-2026_10_05-20_53_32.vma.zst'INFO: issuing guest-agent 'fs-freeze' commandINFO: starting backup via QMP commandINFO: issuing guest-agent 'fs-thaw' commandINFO: started backup task '1d65b799-ca0b-4367-9e10-ac7daf5c0fd4'INFO: resuming VM againINFO: 0% (1.2 GiB of 200.0 GiB) in 3s, read: 419.0 MiB/s, write: 174.3 MiB/sINFO: 1% (3.4 GiB of 200.0 GiB) in 6s, read: 746.7 MiB/s, write\nTRANSIENT_JOURNAL_03=2026-10-05T20:56:31+03:00 pve01 systemd[1]: workers2-vm160-vzdump-99621.service: Deactivated successfully.\nTRANSIENT_JOURNAL_04=2026-10-05T20:56:31+03:00 pve01 systemd[1]: workers2-vm160-vzdump-99621.service: Consumed 623ms CPU time, 143M memory peak.\nPVE01_TASK_API_RC=0\nPVE02_TASK_API_RC=0\nCLUSTER_TASK_API_RC=255\nRECONCILE_VZDUMP_TASK_COUNT=1\nTASK_1_SOURCE=PVE02\nTASK_1_UPID_SHA=3664f926f82f37a33bdf206b1c143a9be85506d90db2fbd929988f6c9fcb7b46\nTASK_1_START=1791222812\nTASK_1_END=1791222991\nTASK_1_STATUS_LIST=OK\nTASK_1_EXITSTATUS_LIST=\nTASK_1_STATUS_API_RC=0\nTASK_1_STATUS_API_STATUS=stopped\nTASK_1_STATUS_API_EXITSTATUS=OK\nTASK_1_LOG_RC=0\nTASK_1_LOG_SHA=7fd6e449501c5c1ab989335618e616330941bf468c7ae409ce9e617f89cd14ab\nTASK_1_LOG_01=INFO: 16% (32.2 GiB of 200.0 GiB) in 42s, read: 1.3 GiB/s, write: 123.6 MiB/s\nTASK_1_LOG_02=INFO: 17% (35.9 GiB of 200.0 GiB) in 45s, read: 1.2 GiB/s, write: 43.2 MiB/s\nTASK_1_LOG_03=INFO: 18% (38.0 GiB of 200.0 GiB) in 48s, read: 699.0 MiB/s, write: 27.3 MiB/s\nTASK_1_LOG_04=INFO: 20% (40.9 GiB of 200.0 GiB) in 51s, read: 991.3 MiB/s, write: 19.5 MiB/s\nTASK_1_LOG_05=INFO: 21% (43.6 GiB of 200.0 GiB) in 54s, read: 914.0 MiB/s, write: 75.2 MiB/s\nTASK_1_LOG_06=INFO: 24% (49.5 GiB of 200.0 GiB) in 57s, read: 2.0 GiB/s, write: 43.2 MiB/s\nTASK_1_LOG_07=INFO: 25% (51.3 GiB of 200.0 GiB) in 1m, read: 613.6 MiB/s, write: 42.5 MiB/s\nTASK_1_LOG_08=INFO: 26% (52.3 GiB of 200.0 GiB) in 1m 7s, read: 141.1 MiB/s, write: 88.0 MiB/s\nTASK_1_LOG_09=INFO: 27% (54.1 GiB of 200.0 GiB) in 1m 13s, read: 318.3 MiB/s, write: 18.5 MiB/s\nTASK_1_LOG_10=INFO: 29% (59.8 GiB of 200.0 GiB) in 1m 16s, read: 1.9 GiB/s, write: 18.4 MiB/s\nTASK_1_LOG_11=INFO: 31% (63.2 GiB of 200.0 GiB) in 1m 19s, read: 1.1 GiB/s, write: 27.9 MiB/s\nTASK_1_LOG_12=INFO: 34% (69.3 GiB of 200.0 GiB) in 1m 22s, read: 2.0 GiB/s, write: 9.0 MiB/s\nTASK_1_LOG_13=INFO: 35% (70.3 GiB of 200.0 GiB) in 1m 25s, read: 351.3 MiB/s, write: 44.6 MiB/s\nTASK_1_LOG_14=INFO: 38% (76.1 GiB of 200.0 GiB) in 1m 29s, read: 1.5 GiB/s, write: 15.9 MiB/s\nTASK_1_LOG_15=INFO: 39% (78.3 GiB of 200.0 GiB) in 1m 32s, read: 748.9 MiB/s, write: 11.9 MiB/s\nTASK_1_LOG_16=INFO: 40% (80.1 GiB of 200.0 GiB) in 1m 38s, read: 312.2 MiB/s, write: 21.2 MiB/s\nTASK_1_LOG_17=INFO: 41% (82.1 GiB of 200.0 GiB) in 1m 42s, read: 514.0 MiB/s, write: 17.5 MiB/s\nTASK_1_LOG_18=INFO: 42% (84.1 GiB of 200.0 GiB) in 1m 51s, read: 227.7 MiB/s, write: 19.6 MiB/s\nTASK_1_LOG_19=INFO: 43% (87.3 GiB of 200.0 GiB) in 1m 57s, read: 548.2 MiB/s, write: 15.7 MiB/s\nTASK_1_LOG_20=INFO: 46% (93.4 GiB of 200.0 GiB) in 2m, read: 2.0 GiB/s, write: 27.9 MiB/s\nTASK_1_LOG_21=INFO: 50% (100.7 GiB of 200.0 GiB) in 2m 4s, read: 1.8 GiB/s, write: [PRIVATE_IP] MiB/s\nTASK_1_LOG_22=INFO: 53% (107.1 GiB of 200.0 GiB) in 2m 7s, read: 2.2 GiB/s, write: 14.1 MiB/s\nTASK_1_LOG_23=INFO: 56% (113.5 GiB of 200.0 GiB) in 2m 10s, read: 2.1 GiB/s, write: 5.3 KiB/s\nTASK_1_LOG_24=INFO: 59% (118.7 GiB of 200.0 GiB) in 2m 13s, read: 1.7 GiB/s, write: 5.9 MiB/s\nTASK_1_LOG_25=INFO: 63% (126.1 GiB of 200.0 GiB) in 2m 16s, read: 2.5 GiB/s, write: 4.5 MiB/s\nANY_TASK_OK=true\nANY_TASK_RUNNING=false\nANY_TASK_FAILED=false\nVM160_BACKUP_COUNT_NOW=1\nVM160_BACKUP_FRESH_MATCH_COUNT=1\nFRESH_BACKUP_CTIME=1791222812\nFRESH_BACKUP_SIZE=3905927732\nFRESH_BACKUP_FORMAT=vma.zst\nFRESH_BACKUP_VOLID_SHA=82623957583673e0406fb0db7ad1cbb7b07c3189cd136dc397362d9fc23d0fdb\nFRESH_BACKUP_VERIFIED=true\nDUMP_FILE_SCAN_RC=0\nDUMP_160_FILE_COUNT=0\nVM160_QGA_RC=0\nVM160_GUEST_EXITCODE=0\nCRON_ACTIVE_PRESENT=true\nCRON_99621_DISABLED_PRESENT=false\nSTATE_99621_DIR_PRESENT=false\nRUNNER_PROCESS_COUNT=0\nMATCH_TIMER_COUNT=1\nMATCH_TIMER_ACTIVE=1\nMATCH_TIMER_ENABLED=1\nVM160_GUEST_RECONCILE_OK=true\nQUIESCE_UNTOUCHED=true\nQUIESCE_DONE=false\nDECISION=PASS_99621_BACKUP_COMPLETE_QUIESCE_PENDING\nNEXT_GATE=VM160_QUIESCE_ONLY_AFTER_VERIFIED_BACKUP\nCOMMANDS_REMAINING_AFTER_SUCCESS=2\nTASK_RESULT=VM160_DETACHED_BACKUP_RECONCILE\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nMUTATION_OUTCOME=NO_MUTATION\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index bdac9087..a5d6986f 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,62 +1,115 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1 -STATUS=FAIL -RC=68 +COMMAND_ID=SUPPORT-261005-VM160-DETACHED-BACKUP-RECONCILE-99622R1 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=vm160-backup-quiesce-watchdog-fix +MODE=read-only +COMPONENT=vm160-detached-backup-reconcile REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=d8edc0bcd4d08329eae8a011660c73b1a355cfb2a456146ef6590ed72bc36b2d +COMMAND_SHA256=e9e452bc37b34b5debb06af949b6021c45323639da2ecfe78074a74c28e1c5d0 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true +CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=68 -CHANGES_MADE=true +COMMAND_RC=0 +CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=MUTATED +MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d -SANITIZED_OUTPUT_SHA256=3afa104abddc92122f0870efb89bae6a39d11851ed62e98ae42d859fc782a43d +RAW_EVIDENCE_SHA256=fbd4bd5f8bc59a62f82272571e97763ab766f44dd0ea83ad003ad4ff9529d457 +SANITIZED_OUTPUT_SHA256=fbd4bd5f8bc59a62f82272571e97763ab766f44dd0ea83ad003ad4ff9529d457 OUTPUT_BEGIN -VM160_BACKUP_QUIESCE_WATCHDOG_FIX=V1 -MODE=change -TARGET_VM=160 -TARGET_NODE=pve02 -WATCHDOG_RCA_PROVEN=true -WATCHDOG_RCA=CONTROLLER_PVESH_TIMEOUT_45S_INTERRUPTED_BACKUP -BACKUP_LAUNCH_MODEL=DETACHED_SYSTEMD_TRANSIENT_PVESH -NO_VM_SHUTDOWN=true +VM160_DETACHED_BACKUP_RECONCILE=V1 +MODE=read-only +NO_VM_MUTATION=true +NO_BACKUP_CREATE=true +NO_BACKUP_DELETE=true +NO_CONFIG_WRITE=true +NO_SERVICE_RESTART=true +NO_GUEST_MUTATION=true NO_DNS_MUTATION=true NO_PROXY_MUTATION=true -NO_BEGET_MUTATION=true -NO_DB_CONTENT_MUTATION=true -PUBLIC_PRE_ALL_SIX_OK=true -CODEVIPE_RU_PUBLIC_PRE_OK=true -CODEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true -HKMODS_RU_PUBLIC_PRE_OK=true -HKMODS_RU_PUBLIC_PRE_BEGET_HINT=true -DSMODS_RU_PUBLIC_PRE_OK=true -DSMODS_RU_PUBLIC_PRE_BEGET_HINT=true -GAMEVIPE_RU_PUBLIC_PRE_OK=true -GAMEVIPE_RU_PUBLIC_PRE_BEGET_HINT=true -KINGOFWOLK_RU_PUBLIC_PRE_OK=true -KINGOFWOLK_RU_PUBLIC_PRE_BEGET_HINT=true -ZAKRUTIM_RU_PUBLIC_PRE_OK=true -ZAKRUTIM_RU_PUBLIC_PRE_BEGET_HINT=true -VM160_PRE_FOUND=true -VM160_PRE_NODE=pve02 -VM160_PRE_STATE=running -PRE_GUEST_QGA_RC=0 -PRE_GUEST_EXITCODE=0 +VM160_FOUND=true +VM160_NODE=pve02 +VM160_STATE=running +TRANSIENT_SYSTEMCTL_SHOW_RC=0 +TRANSIENT_LOADSTATE=MISSING +TRANSIENT_ACTIVESTATE=MISSING +TRANSIENT_SUBSTATE=MISSING +TRANSIENT_RESULT=MISSING +TRANSIENT_EXECMAINCODE=MISSING +TRANSIENT_EXECMAINSTATUS=MISSING +TRANSIENT_EXECMAINSTARTTIMESTAMP=MISSING +TRANSIENT_EXECMAINEXITTIMESTAMP=MISSING +TRANSIENT_JOURNAL_RC=0 +TRANSIENT_JOURNAL_LINE_COUNT=4 +TRANSIENT_JOURNAL_SHA=2d5bd1f2dedb7adcfca35ec5ced2669d94b9fb4e9b05de61993fadb0dd8d7402 +TRANSIENT_JOURNAL_01=2026-10-05T20:53:31+03:00 pve01 systemd[1]: Started workers2-vm160-vzdump-99621.service - [systemd-run] /usr/bin/pvesh create /nodes/pve02/vzdump --vmid 160 --storage local --mode snapshot --compress zstd --notes-template "WORKERS2 VM160 retirement rollback backup 99621" --output-format json. +TRANSIENT_JOURNAL_02=2026-10-05T20:56:31+03:00 pve01 pvesh[742777]: "INFO: starting new backup job: vzdump 160 --mode snapshot --node pve02 --compress zstd --storage local --notes-template 'WORKERS2 VM160 retirement rollback backup 99621'INFO: Starting Backup of VM 160 (qemu)INFO: Backup started at 2026-10-05 20:53:33INFO: status = runningINFO: VM Name: forum-prodINFO: include disk 'scsi0' 'local-lvm:vm-160-disk-0' 200GINFO: backup mode: snapshotINFO: ionice priority: 7INFO: creating vzdump archive '/var/lib/vz/dump/vzdump-qemu-160-2026_10_05-20_53_32.vma.zst'INFO: issuing guest-agent 'fs-freeze' commandINFO: starting backup via QMP commandINFO: issuing guest-agent 'fs-thaw' commandINFO: started backup task '1d65b799-ca0b-4367-9e10-ac7daf5c0fd4'INFO: resuming VM againINFO: 0% (1.2 GiB of 200.0 GiB) in 3s, read: 419.0 MiB/s, write: 174.3 MiB/sINFO: 1% (3.4 GiB of 200.0 GiB) in 6s, read: 746.7 MiB/s, write +TRANSIENT_JOURNAL_03=2026-10-05T20:56:31+03:00 pve01 systemd[1]: workers2-vm160-vzdump-99621.service: Deactivated successfully. +TRANSIENT_JOURNAL_04=2026-10-05T20:56:31+03:00 pve01 systemd[1]: workers2-vm160-vzdump-99621.service: Consumed 623ms CPU time, 143M memory peak. +PVE01_TASK_API_RC=0 +PVE02_TASK_API_RC=0 +CLUSTER_TASK_API_RC=255 +RECONCILE_VZDUMP_TASK_COUNT=1 +TASK_1_SOURCE=PVE02 +TASK_1_UPID_SHA=3664f926f82f37a33bdf206b1c143a9be85506d90db2fbd929988f6c9fcb7b46 +TASK_1_START=1791222812 +TASK_1_END=1791222991 +TASK_1_STATUS_LIST=OK +TASK_1_EXITSTATUS_LIST= +TASK_1_STATUS_API_RC=0 +TASK_1_STATUS_API_STATUS=stopped +TASK_1_STATUS_API_EXITSTATUS=OK +TASK_1_LOG_RC=0 +TASK_1_LOG_SHA=7fd6e449501c5c1ab989335618e616330941bf468c7ae409ce9e617f89cd14ab +TASK_1_LOG_01=INFO: 16% (32.2 GiB of 200.0 GiB) in 42s, read: 1.3 GiB/s, write: 123.6 MiB/s +TASK_1_LOG_02=INFO: 17% (35.9 GiB of 200.0 GiB) in 45s, read: 1.2 GiB/s, write: 43.2 MiB/s +TASK_1_LOG_03=INFO: 18% (38.0 GiB of 200.0 GiB) in 48s, read: 699.0 MiB/s, write: 27.3 MiB/s +TASK_1_LOG_04=INFO: 20% (40.9 GiB of 200.0 GiB) in 51s, read: 991.3 MiB/s, write: 19.5 MiB/s +TASK_1_LOG_05=INFO: 21% (43.6 GiB of 200.0 GiB) in 54s, read: 914.0 MiB/s, write: 75.2 MiB/s +TASK_1_LOG_06=INFO: 24% (49.5 GiB of 200.0 GiB) in 57s, read: 2.0 GiB/s, write: 43.2 MiB/s +TASK_1_LOG_07=INFO: 25% (51.3 GiB of 200.0 GiB) in 1m, read: 613.6 MiB/s, write: 42.5 MiB/s +TASK_1_LOG_08=INFO: 26% (52.3 GiB of 200.0 GiB) in 1m 7s, read: 141.1 MiB/s, write: 88.0 MiB/s +TASK_1_LOG_09=INFO: 27% (54.1 GiB of 200.0 GiB) in 1m 13s, read: 318.3 MiB/s, write: 18.5 MiB/s +TASK_1_LOG_10=INFO: 29% (59.8 GiB of 200.0 GiB) in 1m 16s, read: 1.9 GiB/s, write: 18.4 MiB/s +TASK_1_LOG_11=INFO: 31% (63.2 GiB of 200.0 GiB) in 1m 19s, read: 1.1 GiB/s, write: 27.9 MiB/s +TASK_1_LOG_12=INFO: 34% (69.3 GiB of 200.0 GiB) in 1m 22s, read: 2.0 GiB/s, write: 9.0 MiB/s +TASK_1_LOG_13=INFO: 35% (70.3 GiB of 200.0 GiB) in 1m 25s, read: 351.3 MiB/s, write: 44.6 MiB/s +TASK_1_LOG_14=INFO: 38% (76.1 GiB of 200.0 GiB) in 1m 29s, read: 1.5 GiB/s, write: 15.9 MiB/s +TASK_1_LOG_15=INFO: 39% (78.3 GiB of 200.0 GiB) in 1m 32s, read: 748.9 MiB/s, write: 11.9 MiB/s +TASK_1_LOG_16=INFO: 40% (80.1 GiB of 200.0 GiB) in 1m 38s, read: 312.2 MiB/s, write: 21.2 MiB/s +TASK_1_LOG_17=INFO: 41% (82.1 GiB of 200.0 GiB) in 1m 42s, read: 514.0 MiB/s, write: 17.5 MiB/s +TASK_1_LOG_18=INFO: 42% (84.1 GiB of 200.0 GiB) in 1m 51s, read: 227.7 MiB/s, write: 19.6 MiB/s +TASK_1_LOG_19=INFO: 43% (87.3 GiB of 200.0 GiB) in 1m 57s, read: 548.2 MiB/s, write: 15.7 MiB/s +TASK_1_LOG_20=INFO: 46% (93.4 GiB of 200.0 GiB) in 2m, read: 2.0 GiB/s, write: 27.9 MiB/s +TASK_1_LOG_21=INFO: 50% (100.7 GiB of 200.0 GiB) in 2m 4s, read: 1.8 GiB/s, write: [PRIVATE_IP] MiB/s +TASK_1_LOG_22=INFO: 53% (107.1 GiB of 200.0 GiB) in 2m 7s, read: 2.2 GiB/s, write: 14.1 MiB/s +TASK_1_LOG_23=INFO: 56% (113.5 GiB of 200.0 GiB) in 2m 10s, read: 2.1 GiB/s, write: 5.3 KiB/s +TASK_1_LOG_24=INFO: 59% (118.7 GiB of 200.0 GiB) in 2m 13s, read: 1.7 GiB/s, write: 5.9 MiB/s +TASK_1_LOG_25=INFO: 63% (126.1 GiB of 200.0 GiB) in 2m 16s, read: 2.5 GiB/s, write: 4.5 MiB/s +ANY_TASK_OK=true +ANY_TASK_RUNNING=false +ANY_TASK_FAILED=false +VM160_BACKUP_COUNT_NOW=1 +VM160_BACKUP_FRESH_MATCH_COUNT=1 +FRESH_BACKUP_CTIME=1791222812 +FRESH_BACKUP_SIZE=3905927732 +FRESH_BACKUP_FORMAT=vma.zst +FRESH_BACKUP_VOLID_SHA=82623957583673e0406fb0db7ad1cbb7b07c3189cd136dc397362d9fc23d0fdb +FRESH_BACKUP_VERIFIED=true +DUMP_FILE_SCAN_RC=0 +DUMP_160_FILE_COUNT=0 +VM160_QGA_RC=0 +VM160_GUEST_EXITCODE=0 CRON_ACTIVE_PRESENT=true CRON_99621_DISABLED_PRESENT=false STATE_99621_DIR_PRESENT=false @@ -64,33 +117,17 @@ RUNNER_PROCESS_COUNT=0 MATCH_TIMER_COUNT=1 MATCH_TIMER_ACTIVE=1 MATCH_TIMER_ENABLED=1 -DB_ROOT_FILE_COUNT=6 -VM160_PRECHANGE_GUEST_OK=true -ALL_SIX_SOURCE_BOARDS_INACTIVE=true -QUIESCE_UNTOUCHED_PRE=true -BACKUP_STORAGE_CANDIDATE_COUNT=1 -BACKUP_STORAGE_NAME=local -BACKUP_STORAGE_AVAILABLE_BYTES=87478521856 -BACKUP_STORAGE_TOTAL_BYTES=105089261568 -BACKUP_REQUIRED_MIN_FREE_BYTES=34359738368 -BACKUP_SPACE_PREFLIGHT_OK=true -VM160_BACKUP_TASK_ALREADY_RUNNING=false -VM160_BACKUP_COUNT_PRE=0 -TRANSIENT_UNIT_PREEXISTS=false -BACKUP_START_EPOCH=1791222811 -DETACHED_LAUNCH_RC=0 -DETACHED_LAUNCH_OUTPUT_SHA=dff3498d460ccce6681bd424902dec8994b870f02d9ffb229900bb3c122499c3 -BACKUP_TASK_DISCOVERED=false -BACKUP_TASK_UPID_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -DECISION=HOLD_BACKUP_TASK_NOT_DISCOVERED -NEXT_GATE=VM160_BACKUP_TASK_RECONCILE +VM160_GUEST_RECONCILE_OK=true +QUIESCE_UNTOUCHED=true +QUIESCE_DONE=false +DECISION=PASS_99621_BACKUP_COMPLETE_QUIESCE_PENDING +NEXT_GATE=VM160_QUIESCE_ONLY_AFTER_VERIFIED_BACKUP COMMANDS_REMAINING_AFTER_SUCCESS=2 -TASK_RESULT=VM160_BACKUP_QUIESCE_WATCHDOG_FIX -CHANGES_MADE=true +TASK_RESULT=VM160_DETACHED_BACKUP_RECONCILE +CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=DETACHED_LAUNCH_SUBMITTED_TASK_NOT_DISCOVERED -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261005-VM160-BACKUP-QUIESCE-WATCHDOG-FIX-99621R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null} +MUTATION_OUTCOME=NO_MUTATION OUTPUT_END CHAT_OUTPUT_END