homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-09-13 08:15:55 +00:00
parent 6967f7b832
commit bd16ac2bfe
4 changed files with 420 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1.json
+38
View File
File diff suppressed because one or more lines are too long
runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1.txt
+204
View File
@@ -0,0 +1,204 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1
STATUS=FAIL
RC=2
HOST=pve01
MODE=change
COMPONENT=infra-dr-backupv2-pve02-sequential-backup-nodebundle-taskwait
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=5c9a4ef394ef9771f83f28c484bf58e1477fba3a658452a05c0c7322a9ba82f2
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=true
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=FAIL
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=2
CHANGES_MADE=true
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=MUTATED
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d
SANITIZED_OUTPUT_SHA256=9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d
OUTPUT_BEGIN
PVE303_BEGIN=
COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1
ANCHOR302_COMMIT=6967f7b832bef65ee5877aeaf9ab7f17d0bdd2ca
RECONCILE298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d
MODE=CHANGE_PVE02_MIXED_GUEST_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET_TASKWAIT
SELFTESTS=22
AUTHORITY_FRESH_READ=OK
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GLOBAL_PRESTATE","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
PVE02_STRICT_SSH_STREAM_PREFLIGHT=PASS
PVE02_INVENTORY=[{"config_sha256":"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b","guard":{"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true},"maxdisk_bytes":8350298112,"name":"dns2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":111},{"config_sha256":"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1","guard":{"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true},"maxdisk_bytes":8350298112,"name":"unbound2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":113},{"config_sha256":"a3668bdf650c6e90f0dd9b684c272c16dc8243dfe4bbf90de0ae90cbe064d5a8","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0},"maxdisk_bytes":214748364800,"name":"forum-prod","node":"pve02","status":"running","template":false,"type":"qemu","vmid":160},{"config_sha256":"d59cd51387dce644c10925d11d3bb49bfea57eb9bc4bee9ebb1d69faae768295","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0},"maxdisk_bytes":103079215104,"name":"edge-cold-standby","node":"pve02","status":"stopped","template":false,"type":"qemu","vmid":9130}]
PVE02_GUEST_COUNT=4
STOPPED_GUEST_POLICY={"policy":"INCLUDE_BACKUP_PRESERVE_STOPPED_STATE","source_status":"stopped","vmid":9130}
CRYPT_COUNT=13
PHYSICAL_COUNT=13
BACKUP_ORDER=[111,113,9130,160]
GUEST_BACKUP_BEGIN=111
GUEST_SOURCE={"maxdisk_bytes":8350298112,"name":"dns2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":111}
LXC_MOUNT_GUARD={"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true}
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST111_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":8350298112,"staging_available_bytes":296977117184,"staging_required_bytes":26067038208,"streaming_single_copy_fits":true,"vmid":111}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871526,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871500,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":8891363328}
CLOUD_PRECAPACITY_ADMISSION=YES
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST111_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GENERATION=G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3
LOGICAL_ID=guest-pve02-lxc-111
WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3
PAYLOAD_NAME=vzdump-lxc-111-2026_09_13-08_08_39.tar.zst
PROXMOX_ARCHIVE_VERIFY=PASS
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871526,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871500,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":929354534}
REPLICA_1_CRYPT=pve03-mail-01-crypt
REPLICA_1_PHYSICAL=pve03-mail-01
REPLICA_1_FREE_BYTES_PREUPLOAD=1085606871526
REPLICA_1_UPLOAD=PASS
REPLICA_2_CRYPT=pve03-mail-02-crypt
REPLICA_2_PHYSICAL=pve03-mail-02
REPLICA_2_FREE_BYTES_PREUPLOAD=1085606871500
REPLICA_2_UPLOAD=PASS
REPLICA_1_READBACK=PASS
REPLICA_2_READBACK=PASS
REPLICA_1_SEALED_READBACK=PASS
REPLICA_2_SEALED_READBACK=PASS
SEALED_QUORUM_VERIFIED=YES
LOCAL_PAYLOAD_FILES_REMAINING=0
SOURCE_CONFIG_UNCHANGED=YES
GUEST_BACKUP_COMPLETE=111
GUEST_BACKUP_BEGIN=113
GUEST_SOURCE={"maxdisk_bytes":8350298112,"name":"unbound2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":113}
LXC_MOUNT_GUARD={"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true}
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST113_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":8350298112,"staging_available_bytes":296977117184,"staging_required_bytes":26067038208,"streaming_single_copy_fits":true,"vmid":113}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485999,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485973,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":8891363328}
CLOUD_PRECAPACITY_ADMISSION=YES
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST113_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GENERATION=G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad
LOGICAL_ID=guest-pve02-lxc-113
WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad
PAYLOAD_NAME=vzdump-lxc-113-2026_09_13-08_10_14.tar.zst
PROXMOX_ARCHIVE_VERIFY=PASS
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485999,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485973,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":762734583}
REPLICA_1_CRYPT=pve03-mail-01-crypt
REPLICA_1_PHYSICAL=pve03-mail-01
REPLICA_1_FREE_BYTES_PREUPLOAD=1085218485999
REPLICA_1_UPLOAD=PASS
REPLICA_2_CRYPT=pve03-mail-02-crypt
REPLICA_2_PHYSICAL=pve03-mail-02
REPLICA_2_FREE_BYTES_PREUPLOAD=1085218485973
REPLICA_2_UPLOAD=PASS
REPLICA_1_READBACK=PASS
REPLICA_2_READBACK=PASS
REPLICA_1_SEALED_READBACK=PASS
REPLICA_2_SEALED_READBACK=PASS
SEALED_QUORUM_VERIFIED=YES
LOCAL_PAYLOAD_FILES_REMAINING=0
SOURCE_CONFIG_UNCHANGED=YES
GUEST_BACKUP_COMPLETE=113
GUEST_BACKUP_BEGIN=9130
GUEST_SOURCE={"maxdisk_bytes":103079215104,"name":"edge-cold-standby","node":"pve02","status":"stopped","template":false,"type":"qemu","vmid":9130}
QEMU_DISK_GUARD={"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0}
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST9130_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":103079215104,"staging_available_bytes":296977117184,"staging_required_bytes":120795955200,"streaming_single_copy_fits":true,"vmid":9130}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1084996761091,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1084996761065,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":103620280320}
CLOUD_PRECAPACITY_ADMISSION=YES
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST9130_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GENERATION=G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84
LOGICAL_ID=guest-pve02-qemu-9130
WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84
PAYLOAD_NAME=vzdump-qemu-9130-2026_09_13-08_11_26.vma.zst
STATUS=FAIL
PHASE=GUEST9130_STREAM_VZDUMP
REASON=SOURCE_GUEST_STATUS
COMPLETED_BEFORE_FAILURE=[{"generation":"G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3","logical_id":"guest-pve02-lxc-111","manifest_sha256":"362b9abbe0f3d1105ee353768fb2e60fbc5a82222228d9b30b720a282315c37b","name":"dns2","payload_bytes":388289318,"payload_sha256":"649785b391b35f0ba7c17577ca51be1541dbf3e772480fac3b821f8a63680a4b","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"],"source_config_sha256":"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b","source_status":"running","type":"lxc","vmid":111},{"generation":"G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad","logical_id":"guest-pve02-lxc-113","manifest_sha256":"21e60227e12baf7663384f4ba14ab980948ced2ded47a2a57386b20a54ed8233","name":"unbound2","payload_bytes":221669367,"payload_sha256":"c6205058bf56ef2ac320cf5774d9d29b3c3ec98b71087203c8df01c94ea659ff","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"],"source_config_sha256":"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1","source_status":"running","type":"lxc","vmid":113}]
CURRENT_SOURCE_CONFIG_UNCHANGED=NO
CURRENT_LOCAL_PAYLOAD_FILES_REMAINING=21
FAILED_WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84
FAILURE_DIAGNOSTIC_SUMMARY={"cloud_write_attempts":8,"completed_vmids":[111,113],"current_type":"qemu","current_vmid":9130,"failed_rc":"NONE","failed_tool":"NONE","phase":"GUEST9130_STREAM_VZDUMP","reason":"SOURCE_GUEST_STATUS"}
GENERATION_DISCOVERY_REJECT_SUMMARY={"not_applicable":"backup_command","phase":"GUEST9130_STREAM_VZDUMP"}
FAILED_TOOL=NONE
FAILED_RC=NONE
FAILED_STDERR_BYTES=0
FAILED_STDERR_SHA256=NONE
CLOUD_CALLS=112
CLOUD_WRITE_ATTEMPTS=8
CLOUD_DELETIONS=0
BACKUP_EXECUTIONS=3
NODE_BUNDLE_EXECUTIONS=0
RESTORE_EXECUTIONS=0
SOURCE_GUEST_MUTATIONS=0
SYSTEMD_MUTATIONS=0
LIVE_RCLONE_CONFIG_READS=26
LIVE_RCLONE_CONFIG_WRITES=0
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null}
PVE303_END=
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+12 -12
View File
File diff suppressed because one or more lines are too long
runtime/latest.txt
+166 -101
View File
@@ -1,22 +1,22 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-RESTORE-PROOF-302R1
STATUS=OK
RC=0
COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1
STATUS=FAIL
RC=2
HOST=pve01
MODE=change
COMPONENT=infra-dr-backupv2-pve03-sequential-restore-proof
COMPONENT=infra-dr-backupv2-pve02-sequential-backup-nodebundle-taskwait
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=d8885fb6a1bf2da2494b56b2e193edc054925b16877dffea2ebf90aadce179b5
COMMAND_SHA256=5c9a4ef394ef9771f83f28c484bf58e1477fba3a658452a05c0c7322a9ba82f2
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=true
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=OK
RESULT_CONTRACT_STATUS=FAIL
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
COMMAND_RC=2
CHANGES_MADE=true
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
@@ -24,116 +24,181 @@ MUTATION_OUTCOME=MUTATED
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=9571cf9070a804ea41f7ca7f900b60c52026dc5f44e8b0cc9e7e64bf703c1efa
SANITIZED_OUTPUT_SHA256=9571cf9070a804ea41f7ca7f900b60c52026dc5f44e8b0cc9e7e64bf703c1efa
RAW_EVIDENCE_SHA256=9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d
SANITIZED_OUTPUT_SHA256=9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d
OUTPUT_BEGIN
PVE302_BEGIN=
COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-RESTORE-PROOF-302R1
ANCHOR301_COMMIT=44008e8e152c13bc078564a46e23d92eb037bb62
MODE=CHANGE_PVE03_CLOUD_ONLY_SEQUENTIAL_RESTORE_BOOT_NODEBUNDLE_NODESET_VERIFY
SELFTESTS=17
PVE303_BEGIN=
COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1
ANCHOR302_COMMIT=6967f7b832bef65ee5877aeaf9ab7f17d0bdd2ca
RECONCILE298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d
MODE=CHANGE_PVE02_MIXED_GUEST_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET_TASKWAIT
SELFTESTS=22
AUTHORITY_FRESH_READ=OK
SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":211}
SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":190}
SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":130}
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GLOBAL_PRESTATE","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
PVE02_STRICT_SSH_STREAM_PREFLIGHT=PASS
PVE02_INVENTORY=[{"config_sha256":"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b","guard":{"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true},"maxdisk_bytes":8350298112,"name":"dns2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":111},{"config_sha256":"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1","guard":{"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true},"maxdisk_bytes":8350298112,"name":"unbound2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":113},{"config_sha256":"a3668bdf650c6e90f0dd9b684c272c16dc8243dfe4bbf90de0ae90cbe064d5a8","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0},"maxdisk_bytes":214748364800,"name":"forum-prod","node":"pve02","status":"running","template":false,"type":"qemu","vmid":160},{"config_sha256":"d59cd51387dce644c10925d11d3bb49bfea57eb9bc4bee9ebb1d69faae768295","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0},"maxdisk_bytes":103079215104,"name":"edge-cold-standby","node":"pve02","status":"stopped","template":false,"type":"qemu","vmid":9130}]
PVE02_GUEST_COUNT=4
STOPPED_GUEST_POLICY={"policy":"INCLUDE_BACKUP_PRESERVE_STOPPED_STATE","source_status":"stopped","vmid":9130}
CRYPT_COUNT=13
PHYSICAL_COUNT=13
NODE_BUNDLE_RECONSTRUCTION_VERIFIED=YES
NODE_BUNDLE_PVE_REFERENCE_VERIFIED=YES
NODE_BUNDLE_MEMBER_COUNT=226
VM_RESTORE_BEGIN=211
TEST_VMID=900311
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM211_SOURCE_PRESTATE","max_conflicts":0,"samples":3,"status":"PASS"}
SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":211}
GENERATION_DISCOVERY_SUMMARY={"generation":"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd","listed_replicas":2,"reject_counts":{"REMOTE_GENERATION_ABSENT":11},"valid_hits":2,"vmid":211}
GENERATION_DISCOVERED={"generation":"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd","manifest_sha256":"76dd08d6e858042eabd48b8dc6578d8ec6a335827a2d4c19e69031d22beda490","replica_count":2,"vmid":211}
RESTORE_ADMISSION={"local_lvm_available":751570296557,"maxdisk_bytes":53687091200,"payload_bytes":618301056,"staging_available":296977117184,"test_node":"pve01","vmid":211}
LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM211_LOCAL_PREP","max_conflicts":0,"samples":3,"status":"PASS"}
RESTORE_ARCHIVE_VERIFIED={"bytes":618301056,"generation":"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd","sha256":"e76393b03366751b461b130361923cf1f42f416886045eb29fd4212a737eac99","vmid":211}
LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM211_LOCAL_PRE_QMRESTORE","max_conflicts":0,"samples":3,"status":"PASS"}
TEST_VM_ISOLATION={"network_devices_remaining":0,"protection":1,"removed_keys":["net0","serial0"],"test_vmid":900311}
QEMU_GUEST_AGENT_PING=PASS
GUEST_OSINFO_QUERY=PASS
GUEST_HOSTNAME_QUERY=PASS
GUEST_RESTORE_BOOT_VERIFIED=YES
TEST_VM_AND_VOLUMES_REMOVED=YES
RESTORE_PAYLOAD_REMOVED=YES
BACKUP_ORDER=[111,113,9130,160]
GUEST_BACKUP_BEGIN=111
GUEST_SOURCE={"maxdisk_bytes":8350298112,"name":"dns2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":111}
LXC_MOUNT_GUARD={"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true}
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST111_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":8350298112,"staging_available_bytes":296977117184,"staging_required_bytes":26067038208,"streaming_single_copy_fits":true,"vmid":111}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871526,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871500,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":8891363328}
CLOUD_PRECAPACITY_ADMISSION=YES
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST111_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GENERATION=G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3
LOGICAL_ID=guest-pve02-lxc-111
WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3
PAYLOAD_NAME=vzdump-lxc-111-2026_09_13-08_08_39.tar.zst
PROXMOX_ARCHIVE_VERIFY=PASS
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871526,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871500,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":929354534}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":929354534}
REPLICA_1_CRYPT=pve03-mail-01-crypt
REPLICA_1_PHYSICAL=pve03-mail-01
REPLICA_1_FREE_BYTES_PREUPLOAD=1085606871526
REPLICA_1_UPLOAD=PASS
REPLICA_2_CRYPT=pve03-mail-02-crypt
REPLICA_2_PHYSICAL=pve03-mail-02
REPLICA_2_FREE_BYTES_PREUPLOAD=1085606871500
REPLICA_2_UPLOAD=PASS
REPLICA_1_READBACK=PASS
REPLICA_2_READBACK=PASS
REPLICA_1_SEALED_READBACK=PASS
REPLICA_2_SEALED_READBACK=PASS
SEALED_QUORUM_VERIFIED=YES
LOCAL_PAYLOAD_FILES_REMAINING=0
SOURCE_CONFIG_UNCHANGED=YES
VM_RESTORE_PROOF_COMPLETE=211
VM_RESTORE_BEGIN=190
TEST_VMID=900312
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM190_SOURCE_PRESTATE","max_conflicts":0,"samples":3,"status":"PASS"}
SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":190}
GENERATION_DISCOVERY_SUMMARY={"generation":"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360","listed_replicas":2,"reject_counts":{"REMOTE_GENERATION_ABSENT":11},"valid_hits":2,"vmid":190}
GENERATION_DISCOVERED={"generation":"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360","manifest_sha256":"6a954f3beb94e35b289c10717f5ac4387e8461df1a5f5e03db4329cade8b1fcc","replica_count":2,"vmid":190}
RESTORE_ADMISSION={"local_lvm_available":751048795253,"maxdisk_bytes":85899345920,"payload_bytes":1550687863,"staging_available":296977117184,"test_node":"pve01","vmid":190}
LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM190_LOCAL_PREP","max_conflicts":0,"samples":3,"status":"PASS"}
RESTORE_ARCHIVE_VERIFIED={"bytes":1550687863,"generation":"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360","sha256":"a3669dcc4de755ad650f363a4aa5578b00162bdcfe77e8bd37d60ffaa52ddb0c","vmid":190}
LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM190_LOCAL_PRE_QMRESTORE","max_conflicts":0,"samples":3,"status":"PASS"}
TEST_VM_ISOLATION={"network_devices_remaining":0,"protection":1,"removed_keys":["net0","serial0"],"test_vmid":900312}
QEMU_GUEST_AGENT_PING=PASS
GUEST_OSINFO_QUERY=PASS
GUEST_HOSTNAME_QUERY=PASS
GUEST_RESTORE_BOOT_VERIFIED=YES
TEST_VM_AND_VOLUMES_REMOVED=YES
RESTORE_PAYLOAD_REMOVED=YES
GUEST_BACKUP_COMPLETE=111
GUEST_BACKUP_BEGIN=113
GUEST_SOURCE={"maxdisk_bytes":8350298112,"name":"unbound2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":113}
LXC_MOUNT_GUARD={"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true}
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST113_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":8350298112,"staging_available_bytes":296977117184,"staging_required_bytes":26067038208,"streaming_single_copy_fits":true,"vmid":113}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485999,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485973,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":8891363328}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":8891363328}
CLOUD_PRECAPACITY_ADMISSION=YES
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST113_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GENERATION=G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad
LOGICAL_ID=guest-pve02-lxc-113
WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad
PAYLOAD_NAME=vzdump-lxc-113-2026_09_13-08_10_14.tar.zst
PROXMOX_ARCHIVE_VERIFY=PASS
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485999,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485973,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":762734583}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":762734583}
REPLICA_1_CRYPT=pve03-mail-01-crypt
REPLICA_1_PHYSICAL=pve03-mail-01
REPLICA_1_FREE_BYTES_PREUPLOAD=1085218485999
REPLICA_1_UPLOAD=PASS
REPLICA_2_CRYPT=pve03-mail-02-crypt
REPLICA_2_PHYSICAL=pve03-mail-02
REPLICA_2_FREE_BYTES_PREUPLOAD=1085218485973
REPLICA_2_UPLOAD=PASS
REPLICA_1_READBACK=PASS
REPLICA_2_READBACK=PASS
REPLICA_1_SEALED_READBACK=PASS
REPLICA_2_SEALED_READBACK=PASS
SEALED_QUORUM_VERIFIED=YES
LOCAL_PAYLOAD_FILES_REMAINING=0
SOURCE_CONFIG_UNCHANGED=YES
VM_RESTORE_PROOF_COMPLETE=190
VM_RESTORE_BEGIN=130
TEST_VMID=900313
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM130_SOURCE_PRESTATE","max_conflicts":0,"samples":3,"status":"PASS"}
SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":130}
GENERATION_DISCOVERY_SUMMARY={"generation":"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f","listed_replicas":2,"reject_counts":{"REMOTE_GENERATION_ABSENT":11},"valid_hits":2,"vmid":130}
GENERATION_DISCOVERED={"generation":"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f","manifest_sha256":"5f61523d3563096c5e78429c8a63bb347d559ec76de152d896bd2c263146c0ab","replica_count":2,"vmid":130}
RESTORE_ADMISSION={"local_lvm_available":751048795253,"maxdisk_bytes":103079215104,"payload_bytes":10849353784,"staging_available":296977117184,"test_node":"pve01","vmid":130}
LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM130_LOCAL_PREP","max_conflicts":0,"samples":3,"status":"PASS"}
RESTORE_ARCHIVE_VERIFIED={"bytes":10849353784,"generation":"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f","sha256":"4567b3c0322b530079bccec6789524ac0291024dbf50942bf3e550d67c70c1f9","vmid":130}
LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM130_LOCAL_PRE_QMRESTORE","max_conflicts":0,"samples":3,"status":"PASS"}
TEST_VM_ISOLATION={"network_devices_remaining":0,"protection":1,"removed_keys":["net0","serial0","startup"],"test_vmid":900313}
QEMU_GUEST_AGENT_PING=PASS
GUEST_OSINFO_QUERY=PASS
GUEST_HOSTNAME_QUERY=PASS
GUEST_RESTORE_BOOT_VERIFIED=YES
TEST_VM_AND_VOLUMES_REMOVED=YES
RESTORE_PAYLOAD_REMOVED=YES
SOURCE_CONFIG_UNCHANGED=YES
VM_RESTORE_PROOF_COMPLETE=130
PVE03_NODESET_MANIFEST_VERIFIED=YES
PVE03_NODESET_RESTORE_COVERAGE_COMPLETE=YES
PVE03_SEQUENTIAL_RESTORE_PROOFS_COMPLETE=[211,190,130]
PVE03_NODE_BUNDLE_RESTORE_VALIDATED=YES
PVE03_NODESET_RESTORE_COVERAGE_COMPLETE=YES
COMPLETED_RESTORE_PROOFS=[{"generation":"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd","manifest_sha256":"76dd08d6e858042eabd48b8dc6578d8ec6a335827a2d4c19e69031d22beda490","payload_sha256":"e76393b03366751b461b130361923cf1f42f416886045eb29fd4212a737eac99","test_vmid":900311,"vmid":211},{"generation":"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360","manifest_sha256":"6a954f3beb94e35b289c10717f5ac4387e8461df1a5f5e03db4329cade8b1fcc","payload_sha256":"a3669dcc4de755ad650f363a4aa5578b00162bdcfe77e8bd37d60ffaa52ddb0c","test_vmid":900312,"vmid":190},{"generation":"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f","manifest_sha256":"5f61523d3563096c5e78429c8a63bb347d559ec76de152d896bd2c263146c0ab","payload_sha256":"4567b3c0322b530079bccec6789524ac0291024dbf50942bf3e550d67c70c1f9","test_vmid":900313,"vmid":130}]
STATUS=OK
PHASE=COMPLETE
NEXT_GATE=PVE02_SEQUENTIAL_BACKUP_WITH_NODE_BUNDLE
CURRENT_SOURCE_VMID=NONE
CURRENT_TEST_VMID=NONE
CURRENT_WORKDIR=NONE
CURRENT_LOCAL_PAYLOAD_FILES_REMAINING=0
LAST_GENERATION_DISCOVERY_SUMMARY={"generation":"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f","listed_replicas":2,"reject_counts":{"REMOTE_GENERATION_ABSENT":11},"valid_hits":2,"vmid":130}
LAST_SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM130_SOURCE_PRESTATE","max_conflicts":0,"samples":3,"status":"PASS"}
LAST_LOCAL_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM130_LOCAL_PRE_QMRESTORE","max_conflicts":0,"samples":3,"status":"PASS"}
NODE_BUNDLE_VERIFIED=YES
NODESET_VERIFIED=YES
GUEST_BACKUP_COMPLETE=113
GUEST_BACKUP_BEGIN=9130
GUEST_SOURCE={"maxdisk_bytes":103079215104,"name":"edge-cold-standby","node":"pve02","status":"stopped","template":false,"type":"qemu","vmid":9130}
QEMU_DISK_GUARD={"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0}
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST9130_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":103079215104,"staging_available_bytes":296977117184,"staging_required_bytes":120795955200,"streaming_single_copy_fits":true,"vmid":9130}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1084996761091,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1084996761065,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":103620280320}
ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":103620280320}
CLOUD_PRECAPACITY_ADMISSION=YES
SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST9130_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"}
GENERATION=G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84
LOGICAL_ID=guest-pve02-qemu-9130
WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84
PAYLOAD_NAME=vzdump-qemu-9130-2026_09_13-08_11_26.vma.zst
STATUS=FAIL
PHASE=GUEST9130_STREAM_VZDUMP
REASON=SOURCE_GUEST_STATUS
COMPLETED_BEFORE_FAILURE=[{"generation":"G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3","logical_id":"guest-pve02-lxc-111","manifest_sha256":"362b9abbe0f3d1105ee353768fb2e60fbc5a82222228d9b30b720a282315c37b","name":"dns2","payload_bytes":388289318,"payload_sha256":"649785b391b35f0ba7c17577ca51be1541dbf3e772480fac3b821f8a63680a4b","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"],"source_config_sha256":"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b","source_status":"running","type":"lxc","vmid":111},{"generation":"G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad","logical_id":"guest-pve02-lxc-113","manifest_sha256":"21e60227e12baf7663384f4ba14ab980948ced2ded47a2a57386b20a54ed8233","name":"unbound2","payload_bytes":221669367,"payload_sha256":"c6205058bf56ef2ac320cf5774d9d29b3c3ec98b71087203c8df01c94ea659ff","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"],"source_config_sha256":"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1","source_status":"running","type":"lxc","vmid":113}]
CURRENT_SOURCE_CONFIG_UNCHANGED=NO
CURRENT_LOCAL_PAYLOAD_FILES_REMAINING=21
FAILED_WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84
FAILURE_DIAGNOSTIC_SUMMARY={"cloud_write_attempts":8,"completed_vmids":[111,113],"current_type":"qemu","current_vmid":9130,"failed_rc":"NONE","failed_tool":"NONE","phase":"GUEST9130_STREAM_VZDUMP","reason":"SOURCE_GUEST_STATUS"}
GENERATION_DISCOVERY_REJECT_SUMMARY={"not_applicable":"backup_command","phase":"GUEST9130_STREAM_VZDUMP"}
FAILED_TOOL=NONE
FAILED_RC=NONE
FAILED_STDERR_BYTES=0
FAILED_STDERR_SHA256=NONE
CLOUD_CALLS=146
CLOUD_WRITE_ATTEMPTS=0
CLOUD_CALLS=112
CLOUD_WRITE_ATTEMPTS=8
CLOUD_DELETIONS=0
BACKUP_EXECUTIONS=0
BACKUP_EXECUTIONS=3
NODE_BUNDLE_EXECUTIONS=0
RESTORE_EXECUTIONS=3
RESTORE_EXECUTIONS=0
SOURCE_GUEST_MUTATIONS=0
SYSTEMD_MUTATIONS=0
LIVE_RCLONE_CONFIG_READS=26
LIVE_RCLONE_CONFIG_WRITES=0
ROLLBACK_STARTED=NO
ROLLBACK_RESTORED=NOT_APPLICABLE
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-RESTORE-PROOF-302R1","status":"OK","changes_made":true,"rollback_started":false,"rollback_restored":null}
PVE302_END=
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null}
PVE303_END=
OUTPUT_END
CHAT_OUTPUT_END