diff --git a/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1.json b/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1.json new file mode 100644 index 00000000..c2de91bc --- /dev/null +++ b/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "change", + "component": "infra-dr-backupv2-pve03-sequential-backup-nodebundle-taskwait", + "started_at_utc": "2026-09-13T06:44:51Z", + "finished_at_utc": "2026-09-13T07:04:51Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "abf9dd74b89dddd6887c5c1f334050a1c0efecf269d917fb975e7c28034e3aae", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": true, + "result_contract_valid": true, + "result_contract_status": "OK", + "result_contract_error": null, + "command_rc": 0, + "changes_made": true, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "MUTATED", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "5650d358cc698ddf57465e379374c891002d100b6b7ccd071043c4ba0860fad1", + "sanitized_output_sha256": "5650d358cc698ddf57465e379374c891002d100b6b7ccd071043c4ba0860fad1", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "PVE301_BEGIN=\nCOMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1\nANCHOR298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d\nFAILED300_COMMIT=114f1d4732136a3b57717a4354fed893a1403ccb\nMODE=CHANGE_PVE03_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET_TASKWAIT\nSELFTESTS=18\nAUTHORITY_FRESH_READ=OK\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GLOBAL_PRESTATE\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nPVE03_STRICT_SSH_STREAM_PREFLIGHT=PASS\nPVE03_INVENTORY=[{\"config_sha256\":\"814d7661a1fc727491a9d31637fd3604b29ee3b7156cace4aa9f7f9714a4aa1f\",\"guard\":{\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\",\"scsi1\"],\"unused_disk_count\":0},\"maxdisk_bytes\":103079215104,\"name\":\"edge-vm\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":130},{\"config_sha256\":\"0abbf7381ccedc3b65fd52840baaa9a1d0790268cb90d3e10cdc9829b429b447\",\"guard\":{\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\"],\"unused_disk_count\":0},\"maxdisk_bytes\":85899345920,\"name\":\"homelab-ops-worker\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":190},{\"config_sha256\":\"22de890f3ac01bf08f592f77d01d2dd2e1d29d9a901a2ca40314d0a565766338\",\"guard\":{\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\"],\"unused_disk_count\":0},\"maxdisk_bytes\":53687091200,\"name\":\"newfi-staging\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":211}]\nPVE03_GUEST_COUNT=3\nCRYPT_COUNT=13\nPHYSICAL_COUNT=13\nBACKUP_ORDER=[211,190,130]\nGUEST_BACKUP_BEGIN=211\nGUEST_SOURCE={\"maxdisk_bytes\":53687091200,\"name\":\"newfi-staging\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":211}\nQEMU_DISK_GUARD={\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\"],\"unused_disk_count\":0}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST211_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":53687091200,\"staging_available_bytes\":296977125376,\"staging_required_bytes\":71403831296,\"streaming_single_copy_fits\":true,\"vmid\":211}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354014148,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353950289,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087776449247,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087776449221,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":54228156416}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST211_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd\nLOGICAL_ID=guest-pve03-qemu-211\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd\nPAYLOAD_NAME=vzdump-qemu-211-2026_09_13-06_45_40.vma.zst\nPROXMOX_ARCHIVE_VERIFY=PASS\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354014148,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353950289,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087776449247,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087776449221,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":1159366272}\nREPLICA_1_CRYPT=pve03-mail-01-crypt\nREPLICA_1_PHYSICAL=pve03-mail-01\nREPLICA_1_FREE_BYTES_PREUPLOAD=1087776449247\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-02-crypt\nREPLICA_2_PHYSICAL=pve03-mail-02\nREPLICA_2_FREE_BYTES_PREUPLOAD=1087776449221\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nSOURCE_CONFIG_UNCHANGED=YES\nGUEST_BACKUP_COMPLETE=211\nGUEST_BACKUP_BEGIN=190\nGUEST_SOURCE={\"maxdisk_bytes\":85899345920,\"name\":\"homelab-ops-worker\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":190}\nQEMU_DISK_GUARD={\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\"],\"unused_disk_count\":0}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST190_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":85899345920,\"staging_available_bytes\":296977121280,\"staging_required_bytes\":103616086016,\"streaming_single_copy_fits\":true,\"vmid\":190}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354014148,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353950289,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087157995658,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087157995632,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":86440411136}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST190_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360\nLOGICAL_ID=guest-pve03-qemu-190\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360\nPAYLOAD_NAME=vzdump-qemu-190-2026_09_13-06_47_40.vma.zst\nPROXMOX_ARCHIVE_VERIFY=PASS\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087157995658,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087157995632,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":2091753079}\nREPLICA_1_CRYPT=pve03-mail-01-crypt\nREPLICA_1_PHYSICAL=pve03-mail-01\nREPLICA_1_FREE_BYTES_PREUPLOAD=1087157995658\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-02-crypt\nREPLICA_2_PHYSICAL=pve03-mail-02\nREPLICA_2_FREE_BYTES_PREUPLOAD=1087157995632\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nSOURCE_CONFIG_UNCHANGED=YES\nGUEST_BACKUP_COMPLETE=190\nGUEST_BACKUP_BEGIN=130\nGUEST_SOURCE={\"maxdisk_bytes\":103079215104,\"name\":\"edge-vm\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":130}\nQEMU_DISK_GUARD={\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\",\"scsi1\"],\"unused_disk_count\":0}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST130_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":103079215104,\"staging_available_bytes\":296977121280,\"staging_required_bytes\":120795955200,\"streaming_single_copy_fits\":true,\"vmid\":130}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927471,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927445,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":103620280320}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST130_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f\nLOGICAL_ID=guest-pve03-qemu-130\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f\nPAYLOAD_NAME=vzdump-qemu-130-2026_09_13-06_50_51.vma.zst\nPROXMOX_ARCHIVE_VERIFY=PASS\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927471,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927445,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":11390419000}\nREPLICA_1_CRYPT=pve01-mail-05-crypt\nREPLICA_1_PHYSICAL=pve01-mail-05\nREPLICA_1_FREE_BYTES_PREUPLOAD=1085882288719\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-04-crypt\nREPLICA_2_PHYSICAL=pve03-mail-04\nREPLICA_2_FREE_BYTES_PREUPLOAD=1085882288719\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nSOURCE_CONFIG_UNCHANGED=YES\nGUEST_BACKUP_COMPLETE=130\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927471,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927445,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":4294967296}\nNODE_BUNDLE_CLOUD_PRECAPACITY=YES\nNODE_BUNDLE_MEMBER_COUNT=226\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927471,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927445,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":541115660}\nREPLICA_1_CRYPT=pve03-mail-01-crypt\nREPLICA_1_PHYSICAL=pve03-mail-01\nREPLICA_1_FREE_BYTES_PREUPLOAD=1085606927471\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-02-crypt\nREPLICA_2_PHYSICAL=pve03-mail-02\nREPLICA_2_FREE_BYTES_PREUPLOAD=1085606927445\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nNODE_BUNDLE_COMPLETE=YES\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606875329,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606875303,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":536870912}\nNODESET_GUESTS_COMPLETE=[130,190,211]\nPVE03_NODESET_SEALED=YES\nPVE03_GUEST_BACKUPS_COMPLETE=[211,190,130]\nNODE_BUNDLE_RECORD={\"generation\":\"N301-20260913T070421Z-2ace9051dc62ad81accd6052aa3692fd\",\"logical_id\":\"node-pve03\",\"manifest_sha256\":\"e9a028f53069e088a5ca69f404468ddc50ce0d8caedaea2c529a41aabfc5c192\",\"node\":\"pve03\",\"payload_bytes\":50444,\"payload_sha256\":\"c99bccb00d24af53fa1e264c7744398c4be906454d414e7bf057a8c46c5dddd2\",\"recovery_policy\":\"CLEAN_INSTALL_JOIN_RECONCILE_NOT_BLIND_PMXCFS_OVERWRITE\",\"replicas\":[\"pve03-mail-01-crypt\",\"pve03-mail-02-crypt\"]}\nNODE_SET_RECORD={\"generation\":\"S301-20260913T070439Z-f040b9c10175c6f22b9c79b1a2a2eca2\",\"logical_id\":\"nodeset-pve03\",\"manifest_sha256\":\"c12abc4b45597b73608e74a443e471b29bde9cb00c7c733f9d34ebb8386f238d\",\"replicas\":[\"pve03-mail-01-crypt\",\"pve03-mail-02-crypt\"]}\nSTATUS=OK\nPHASE=COMPLETE\nNEXT_GATE=PVE03_SEQUENTIAL_RESTORE_PROOF\nCURRENT_SOURCE_CONFIG_UNCHANGED=YES\nCURRENT_LOCAL_PAYLOAD_FILES_REMAINING=0\nFAILED_WORK_DIRECTORY=NONE\nFAILURE_DIAGNOSTIC_SUMMARY={\"cloud_write_attempts\":20,\"completed_vmids\":[211,190,130],\"current_type\":null,\"current_vmid\":null,\"failed_rc\":\"NONE\",\"failed_tool\":\"NONE\",\"phase\":\"FINAL_GUARD\",\"reason\":\"NONE\"}\nGENERATION_DISCOVERY_REJECT_SUMMARY={\"not_applicable\":\"backup_command\",\"phase\":\"FINAL_GUARD\"}\nFAILED_TOOL=NONE\nFAILED_RC=NONE\nFAILED_STDERR_BYTES=0\nFAILED_STDERR_SHA256=NONE\nCLOUD_CALLS=238\nCLOUD_WRITE_ATTEMPTS=20\nCLOUD_DELETIONS=0\nBACKUP_EXECUTIONS=3\nNODE_BUNDLE_EXECUTIONS=1\nRESTORE_EXECUTIONS=0\nSOURCE_GUEST_MUTATIONS=0\nSYSTEMD_MUTATIONS=0\nLIVE_RCLONE_CONFIG_READS=26\nLIVE_RCLONE_CONFIG_WRITES=0\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1\",\"status\":\"OK\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\nPVE301_END=\n" +} diff --git a/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1.txt b/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1.txt new file mode 100644 index 00000000..38c97e02 --- /dev/null +++ b/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1.txt @@ -0,0 +1,293 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=change +COMPONENT=infra-dr-backupv2-pve03-sequential-backup-nodebundle-taskwait +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=abf9dd74b89dddd6887c5c1f334050a1c0efecf269d917fb975e7c28034e3aae +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=true +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=OK +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=true +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=MUTATED +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=5650d358cc698ddf57465e379374c891002d100b6b7ccd071043c4ba0860fad1 +SANITIZED_OUTPUT_SHA256=5650d358cc698ddf57465e379374c891002d100b6b7ccd071043c4ba0860fad1 +OUTPUT_BEGIN +PVE301_BEGIN= +COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1 +ANCHOR298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d +FAILED300_COMMIT=114f1d4732136a3b57717a4354fed893a1403ccb +MODE=CHANGE_PVE03_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET_TASKWAIT +SELFTESTS=18 +AUTHORITY_FRESH_READ=OK +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GLOBAL_PRESTATE","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +PVE03_STRICT_SSH_STREAM_PREFLIGHT=PASS +PVE03_INVENTORY=[{"config_sha256":"814d7661a1fc727491a9d31637fd3604b29ee3b7156cace4aa9f7f9714a4aa1f","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0},"maxdisk_bytes":103079215104,"name":"edge-vm","node":"pve03","status":"running","template":false,"type":"qemu","vmid":130},{"config_sha256":"0abbf7381ccedc3b65fd52840baaa9a1d0790268cb90d3e10cdc9829b429b447","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0},"maxdisk_bytes":85899345920,"name":"homelab-ops-worker","node":"pve03","status":"running","template":false,"type":"qemu","vmid":190},{"config_sha256":"22de890f3ac01bf08f592f77d01d2dd2e1d29d9a901a2ca40314d0a565766338","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0},"maxdisk_bytes":53687091200,"name":"newfi-staging","node":"pve03","status":"running","template":false,"type":"qemu","vmid":211}] +PVE03_GUEST_COUNT=3 +CRYPT_COUNT=13 +PHYSICAL_COUNT=13 +BACKUP_ORDER=[211,190,130] +GUEST_BACKUP_BEGIN=211 +GUEST_SOURCE={"maxdisk_bytes":53687091200,"name":"newfi-staging","node":"pve03","status":"running","template":false,"type":"qemu","vmid":211} +QEMU_DISK_GUARD={"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST211_PRECAPACITY","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":53687091200,"staging_available_bytes":296977125376,"staging_required_bytes":71403831296,"streaming_single_copy_fits":true,"vmid":211} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354014148,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353950289,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087776449247,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087776449221,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":54228156416} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST211_PRESTREAM","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GENERATION=G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd +LOGICAL_ID=guest-pve03-qemu-211 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd +PAYLOAD_NAME=vzdump-qemu-211-2026_09_13-06_45_40.vma.zst +PROXMOX_ARCHIVE_VERIFY=PASS +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354014148,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353950289,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087776449247,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087776449221,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":1159366272} +REPLICA_1_CRYPT=pve03-mail-01-crypt +REPLICA_1_PHYSICAL=pve03-mail-01 +REPLICA_1_FREE_BYTES_PREUPLOAD=1087776449247 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-02-crypt +REPLICA_2_PHYSICAL=pve03-mail-02 +REPLICA_2_FREE_BYTES_PREUPLOAD=1087776449221 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 +SOURCE_CONFIG_UNCHANGED=YES +GUEST_BACKUP_COMPLETE=211 +GUEST_BACKUP_BEGIN=190 +GUEST_SOURCE={"maxdisk_bytes":85899345920,"name":"homelab-ops-worker","node":"pve03","status":"running","template":false,"type":"qemu","vmid":190} +QEMU_DISK_GUARD={"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST190_PRECAPACITY","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":85899345920,"staging_available_bytes":296977121280,"staging_required_bytes":103616086016,"streaming_single_copy_fits":true,"vmid":190} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354014148,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353950289,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087157995658,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087157995632,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":86440411136} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST190_PRESTREAM","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GENERATION=G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360 +LOGICAL_ID=guest-pve03-qemu-190 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360 +PAYLOAD_NAME=vzdump-qemu-190-2026_09_13-06_47_40.vma.zst +PROXMOX_ARCHIVE_VERIFY=PASS +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087157995658,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087157995632,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":2091753079} +REPLICA_1_CRYPT=pve03-mail-01-crypt +REPLICA_1_PHYSICAL=pve03-mail-01 +REPLICA_1_FREE_BYTES_PREUPLOAD=1087157995658 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-02-crypt +REPLICA_2_PHYSICAL=pve03-mail-02 +REPLICA_2_FREE_BYTES_PREUPLOAD=1087157995632 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 +SOURCE_CONFIG_UNCHANGED=YES +GUEST_BACKUP_COMPLETE=190 +GUEST_BACKUP_BEGIN=130 +GUEST_SOURCE={"maxdisk_bytes":103079215104,"name":"edge-vm","node":"pve03","status":"running","template":false,"type":"qemu","vmid":130} +QEMU_DISK_GUARD={"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST130_PRECAPACITY","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":103079215104,"staging_available_bytes":296977121280,"staging_required_bytes":120795955200,"streaming_single_copy_fits":true,"vmid":130} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927471,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927445,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":103620280320} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST130_PRESTREAM","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GENERATION=G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f +LOGICAL_ID=guest-pve03-qemu-130 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f +PAYLOAD_NAME=vzdump-qemu-130-2026_09_13-06_50_51.vma.zst +PROXMOX_ARCHIVE_VERIFY=PASS +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927471,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927445,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":11390419000} +REPLICA_1_CRYPT=pve01-mail-05-crypt +REPLICA_1_PHYSICAL=pve01-mail-05 +REPLICA_1_FREE_BYTES_PREUPLOAD=1085882288719 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-04-crypt +REPLICA_2_PHYSICAL=pve03-mail-04 +REPLICA_2_FREE_BYTES_PREUPLOAD=1085882288719 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 +SOURCE_CONFIG_UNCHANGED=YES +GUEST_BACKUP_COMPLETE=130 +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927471,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927445,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":4294967296} +NODE_BUNDLE_CLOUD_PRECAPACITY=YES +NODE_BUNDLE_MEMBER_COUNT=226 +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927471,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927445,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":541115660} +REPLICA_1_CRYPT=pve03-mail-01-crypt +REPLICA_1_PHYSICAL=pve03-mail-01 +REPLICA_1_FREE_BYTES_PREUPLOAD=1085606927471 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-02-crypt +REPLICA_2_PHYSICAL=pve03-mail-02 +REPLICA_2_FREE_BYTES_PREUPLOAD=1085606927445 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 +NODE_BUNDLE_COMPLETE=YES +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606875329,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606875303,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":536870912} +NODESET_GUESTS_COMPLETE=[130,190,211] +PVE03_NODESET_SEALED=YES +PVE03_GUEST_BACKUPS_COMPLETE=[211,190,130] +NODE_BUNDLE_RECORD={"generation":"N301-20260913T070421Z-2ace9051dc62ad81accd6052aa3692fd","logical_id":"node-pve03","manifest_sha256":"e9a028f53069e088a5ca69f404468ddc50ce0d8caedaea2c529a41aabfc5c192","node":"pve03","payload_bytes":50444,"payload_sha256":"c99bccb00d24af53fa1e264c7744398c4be906454d414e7bf057a8c46c5dddd2","recovery_policy":"CLEAN_INSTALL_JOIN_RECONCILE_NOT_BLIND_PMXCFS_OVERWRITE","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"]} +NODE_SET_RECORD={"generation":"S301-20260913T070439Z-f040b9c10175c6f22b9c79b1a2a2eca2","logical_id":"nodeset-pve03","manifest_sha256":"c12abc4b45597b73608e74a443e471b29bde9cb00c7c733f9d34ebb8386f238d","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"]} +STATUS=OK +PHASE=COMPLETE +NEXT_GATE=PVE03_SEQUENTIAL_RESTORE_PROOF +CURRENT_SOURCE_CONFIG_UNCHANGED=YES +CURRENT_LOCAL_PAYLOAD_FILES_REMAINING=0 +FAILED_WORK_DIRECTORY=NONE +FAILURE_DIAGNOSTIC_SUMMARY={"cloud_write_attempts":20,"completed_vmids":[211,190,130],"current_type":null,"current_vmid":null,"failed_rc":"NONE","failed_tool":"NONE","phase":"FINAL_GUARD","reason":"NONE"} +GENERATION_DISCOVERY_REJECT_SUMMARY={"not_applicable":"backup_command","phase":"FINAL_GUARD"} +FAILED_TOOL=NONE +FAILED_RC=NONE +FAILED_STDERR_BYTES=0 +FAILED_STDERR_SHA256=NONE +CLOUD_CALLS=238 +CLOUD_WRITE_ATTEMPTS=20 +CLOUD_DELETIONS=0 +BACKUP_EXECUTIONS=3 +NODE_BUNDLE_EXECUTIONS=1 +RESTORE_EXECUTIONS=0 +SOURCE_GUEST_MUTATIONS=0 +SYSTEMD_MUTATIONS=0 +LIVE_RCLONE_CONFIG_READS=26 +LIVE_RCLONE_CONFIG_WRITES=0 +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1","status":"OK","changes_made":true,"rollback_started":false,"rollback_restored":null} +PVE301_END= + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 398295cf..c2de91bc 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-QUOTEFIX-300R1", - "status": "FAIL", - "rc": 2, + "command_id": "SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1", + "status": "OK", + "rc": 0, "host": "pve01", "mode": "change", - "component": "infra-dr-backupv2-pve03-sequential-backup-nodebundle-quotefix", - "started_at_utc": "2026-09-13T06:30:23Z", - "finished_at_utc": "2026-09-13T06:30:24Z", + "component": "infra-dr-backupv2-pve03-sequential-backup-nodebundle-taskwait", + "started_at_utc": "2026-09-13T06:44:51Z", + "finished_at_utc": "2026-09-13T07:04:51Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "a7543da5b7da8d9882a85192442cae722fbb9be6d8246fc8fe996769cfb78c2f", + "command_sha256": "abf9dd74b89dddd6887c5c1f334050a1c0efecf269d917fb975e7c28034e3aae", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, "change_declared": true, "result_contract_valid": true, - "result_contract_status": "FAIL", + "result_contract_status": "OK", "result_contract_error": null, - "command_rc": 2, - "changes_made": false, + "command_rc": 0, + "changes_made": true, "rollback_started": false, "rollback_restored": null, - "mutation_outcome": "NO_MUTATION", + "mutation_outcome": "MUTATED", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "a1ed1c45166700de6da5d29a2691cf91643b5512bba8dcfd580d08761692ca71", - "sanitized_output_sha256": "a1ed1c45166700de6da5d29a2691cf91643b5512bba8dcfd580d08761692ca71", + "raw_evidence_sha256": "5650d358cc698ddf57465e379374c891002d100b6b7ccd071043c4ba0860fad1", + "sanitized_output_sha256": "5650d358cc698ddf57465e379374c891002d100b6b7ccd071043c4ba0860fad1", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "PVE300_BEGIN=\nCOMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-QUOTEFIX-300R1\nANCHOR298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d\nMODE=CHANGE_PVE03_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET\nSELFTESTS=14\nAUTHORITY_FRESH_READ=OK\nSTATUS=FAIL\nPHASE=GLOBAL_PRESTATE\nREASON=COMPETING_PVE_TASK\nCOMPLETED_BEFORE_FAILURE=[]\nCURRENT_SOURCE_CONFIG_UNCHANGED=YES\nCURRENT_LOCAL_PAYLOAD_FILES_REMAINING=0\nFAILED_WORK_DIRECTORY=NONE\nFAILURE_DIAGNOSTIC_SUMMARY={\"cloud_write_attempts\":0,\"completed_vmids\":[],\"current_type\":null,\"current_vmid\":null,\"failed_rc\":\"NONE\",\"failed_tool\":\"NONE\",\"phase\":\"GLOBAL_PRESTATE\",\"reason\":\"COMPETING_PVE_TASK\"}\nGENERATION_DISCOVERY_REJECT_SUMMARY={\"not_applicable\":\"backup_command\",\"phase\":\"GLOBAL_PRESTATE\"}\nFAILED_TOOL=NONE\nFAILED_RC=NONE\nFAILED_STDERR_BYTES=0\nFAILED_STDERR_SHA256=NONE\nCLOUD_CALLS=0\nCLOUD_WRITE_ATTEMPTS=0\nCLOUD_DELETIONS=0\nBACKUP_EXECUTIONS=0\nNODE_BUNDLE_EXECUTIONS=0\nRESTORE_EXECUTIONS=0\nSOURCE_GUEST_MUTATIONS=0\nSYSTEMD_MUTATIONS=0\nLIVE_RCLONE_CONFIG_READS=0\nLIVE_RCLONE_CONFIG_WRITES=0\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-QUOTEFIX-300R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nPVE300_END=\n" + "output": "PVE301_BEGIN=\nCOMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1\nANCHOR298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d\nFAILED300_COMMIT=114f1d4732136a3b57717a4354fed893a1403ccb\nMODE=CHANGE_PVE03_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET_TASKWAIT\nSELFTESTS=18\nAUTHORITY_FRESH_READ=OK\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GLOBAL_PRESTATE\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nPVE03_STRICT_SSH_STREAM_PREFLIGHT=PASS\nPVE03_INVENTORY=[{\"config_sha256\":\"814d7661a1fc727491a9d31637fd3604b29ee3b7156cace4aa9f7f9714a4aa1f\",\"guard\":{\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\",\"scsi1\"],\"unused_disk_count\":0},\"maxdisk_bytes\":103079215104,\"name\":\"edge-vm\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":130},{\"config_sha256\":\"0abbf7381ccedc3b65fd52840baaa9a1d0790268cb90d3e10cdc9829b429b447\",\"guard\":{\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\"],\"unused_disk_count\":0},\"maxdisk_bytes\":85899345920,\"name\":\"homelab-ops-worker\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":190},{\"config_sha256\":\"22de890f3ac01bf08f592f77d01d2dd2e1d29d9a901a2ca40314d0a565766338\",\"guard\":{\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\"],\"unused_disk_count\":0},\"maxdisk_bytes\":53687091200,\"name\":\"newfi-staging\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":211}]\nPVE03_GUEST_COUNT=3\nCRYPT_COUNT=13\nPHYSICAL_COUNT=13\nBACKUP_ORDER=[211,190,130]\nGUEST_BACKUP_BEGIN=211\nGUEST_SOURCE={\"maxdisk_bytes\":53687091200,\"name\":\"newfi-staging\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":211}\nQEMU_DISK_GUARD={\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\"],\"unused_disk_count\":0}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST211_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":53687091200,\"staging_available_bytes\":296977125376,\"staging_required_bytes\":71403831296,\"streaming_single_copy_fits\":true,\"vmid\":211}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354014148,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353950289,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087776449247,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087776449221,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":54228156416}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":54228156416}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST211_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd\nLOGICAL_ID=guest-pve03-qemu-211\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd\nPAYLOAD_NAME=vzdump-qemu-211-2026_09_13-06_45_40.vma.zst\nPROXMOX_ARCHIVE_VERIFY=PASS\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354014148,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353950289,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087776449247,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087776449221,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":1159366272}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":1159366272}\nREPLICA_1_CRYPT=pve03-mail-01-crypt\nREPLICA_1_PHYSICAL=pve03-mail-01\nREPLICA_1_FREE_BYTES_PREUPLOAD=1087776449247\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-02-crypt\nREPLICA_2_PHYSICAL=pve03-mail-02\nREPLICA_2_FREE_BYTES_PREUPLOAD=1087776449221\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nSOURCE_CONFIG_UNCHANGED=YES\nGUEST_BACKUP_COMPLETE=211\nGUEST_BACKUP_BEGIN=190\nGUEST_SOURCE={\"maxdisk_bytes\":85899345920,\"name\":\"homelab-ops-worker\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":190}\nQEMU_DISK_GUARD={\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\"],\"unused_disk_count\":0}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST190_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":85899345920,\"staging_available_bytes\":296977121280,\"staging_required_bytes\":103616086016,\"streaming_single_copy_fits\":true,\"vmid\":190}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354014148,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353950289,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087157995658,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087157995632,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":86440411136}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":86440411136}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST190_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360\nLOGICAL_ID=guest-pve03-qemu-190\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360\nPAYLOAD_NAME=vzdump-qemu-190-2026_09_13-06_47_40.vma.zst\nPROXMOX_ARCHIVE_VERIFY=PASS\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087157995658,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1087157995632,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":2091753079}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":2091753079}\nREPLICA_1_CRYPT=pve03-mail-01-crypt\nREPLICA_1_PHYSICAL=pve03-mail-01\nREPLICA_1_FREE_BYTES_PREUPLOAD=1087157995658\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-02-crypt\nREPLICA_2_PHYSICAL=pve03-mail-02\nREPLICA_2_FREE_BYTES_PREUPLOAD=1087157995632\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nSOURCE_CONFIG_UNCHANGED=YES\nGUEST_BACKUP_COMPLETE=190\nGUEST_BACKUP_BEGIN=130\nGUEST_SOURCE={\"maxdisk_bytes\":103079215104,\"name\":\"edge-vm\",\"node\":\"pve03\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":130}\nQEMU_DISK_GUARD={\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\",\"scsi1\"],\"unused_disk_count\":0}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST130_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":103079215104,\"staging_available_bytes\":296977121280,\"staging_required_bytes\":120795955200,\"streaming_single_copy_fits\":true,\"vmid\":130}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927471,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927445,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":103620280320}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST130_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve03\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f\nLOGICAL_ID=guest-pve03-qemu-130\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f\nPAYLOAD_NAME=vzdump-qemu-130-2026_09_13-06_50_51.vma.zst\nPROXMOX_ARCHIVE_VERIFY=PASS\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927471,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927445,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":11390419000}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085882288719,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":11390419000}\nREPLICA_1_CRYPT=pve01-mail-05-crypt\nREPLICA_1_PHYSICAL=pve01-mail-05\nREPLICA_1_FREE_BYTES_PREUPLOAD=1085882288719\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-04-crypt\nREPLICA_2_PHYSICAL=pve03-mail-04\nREPLICA_2_FREE_BYTES_PREUPLOAD=1085882288719\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nSOURCE_CONFIG_UNCHANGED=YES\nGUEST_BACKUP_COMPLETE=130\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927471,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927445,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":4294967296}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":4294967296}\nNODE_BUNDLE_CLOUD_PRECAPACITY=YES\nNODE_BUNDLE_MEMBER_COUNT=226\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927471,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606927445,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":541115660}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":541115660}\nREPLICA_1_CRYPT=pve03-mail-01-crypt\nREPLICA_1_PHYSICAL=pve03-mail-01\nREPLICA_1_FREE_BYTES_PREUPLOAD=1085606927471\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-02-crypt\nREPLICA_2_PHYSICAL=pve03-mail-02\nREPLICA_2_FREE_BYTES_PREUPLOAD=1085606927445\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nNODE_BUNDLE_COMPLETE=YES\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606875329,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606875303,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":536870912}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":536870912}\nNODESET_GUESTS_COMPLETE=[130,190,211]\nPVE03_NODESET_SEALED=YES\nPVE03_GUEST_BACKUPS_COMPLETE=[211,190,130]\nNODE_BUNDLE_RECORD={\"generation\":\"N301-20260913T070421Z-2ace9051dc62ad81accd6052aa3692fd\",\"logical_id\":\"node-pve03\",\"manifest_sha256\":\"e9a028f53069e088a5ca69f404468ddc50ce0d8caedaea2c529a41aabfc5c192\",\"node\":\"pve03\",\"payload_bytes\":50444,\"payload_sha256\":\"c99bccb00d24af53fa1e264c7744398c4be906454d414e7bf057a8c46c5dddd2\",\"recovery_policy\":\"CLEAN_INSTALL_JOIN_RECONCILE_NOT_BLIND_PMXCFS_OVERWRITE\",\"replicas\":[\"pve03-mail-01-crypt\",\"pve03-mail-02-crypt\"]}\nNODE_SET_RECORD={\"generation\":\"S301-20260913T070439Z-f040b9c10175c6f22b9c79b1a2a2eca2\",\"logical_id\":\"nodeset-pve03\",\"manifest_sha256\":\"c12abc4b45597b73608e74a443e471b29bde9cb00c7c733f9d34ebb8386f238d\",\"replicas\":[\"pve03-mail-01-crypt\",\"pve03-mail-02-crypt\"]}\nSTATUS=OK\nPHASE=COMPLETE\nNEXT_GATE=PVE03_SEQUENTIAL_RESTORE_PROOF\nCURRENT_SOURCE_CONFIG_UNCHANGED=YES\nCURRENT_LOCAL_PAYLOAD_FILES_REMAINING=0\nFAILED_WORK_DIRECTORY=NONE\nFAILURE_DIAGNOSTIC_SUMMARY={\"cloud_write_attempts\":20,\"completed_vmids\":[211,190,130],\"current_type\":null,\"current_vmid\":null,\"failed_rc\":\"NONE\",\"failed_tool\":\"NONE\",\"phase\":\"FINAL_GUARD\",\"reason\":\"NONE\"}\nGENERATION_DISCOVERY_REJECT_SUMMARY={\"not_applicable\":\"backup_command\",\"phase\":\"FINAL_GUARD\"}\nFAILED_TOOL=NONE\nFAILED_RC=NONE\nFAILED_STDERR_BYTES=0\nFAILED_STDERR_SHA256=NONE\nCLOUD_CALLS=238\nCLOUD_WRITE_ATTEMPTS=20\nCLOUD_DELETIONS=0\nBACKUP_EXECUTIONS=3\nNODE_BUNDLE_EXECUTIONS=1\nRESTORE_EXECUTIONS=0\nSOURCE_GUEST_MUTATIONS=0\nSYSTEMD_MUTATIONS=0\nLIVE_RCLONE_CONFIG_READS=26\nLIVE_RCLONE_CONFIG_WRITES=0\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1\",\"status\":\"OK\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\nPVE301_END=\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 0de74350..38c97e02 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,63 +1,293 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-QUOTEFIX-300R1 -STATUS=FAIL -RC=2 +COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1 +STATUS=OK +RC=0 HOST=pve01 MODE=change -COMPONENT=infra-dr-backupv2-pve03-sequential-backup-nodebundle-quotefix +COMPONENT=infra-dr-backupv2-pve03-sequential-backup-nodebundle-taskwait REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=a7543da5b7da8d9882a85192442cae722fbb9be6d8246fc8fe996769cfb78c2f +COMMAND_SHA256=abf9dd74b89dddd6887c5c1f334050a1c0efecf269d917fb975e7c28034e3aae DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=true RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_STATUS=OK RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=2 -CHANGES_MADE=false +COMMAND_RC=0 +CHANGES_MADE=true ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=NO_MUTATION +MUTATION_OUTCOME=MUTATED SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=a1ed1c45166700de6da5d29a2691cf91643b5512bba8dcfd580d08761692ca71 -SANITIZED_OUTPUT_SHA256=a1ed1c45166700de6da5d29a2691cf91643b5512bba8dcfd580d08761692ca71 +RAW_EVIDENCE_SHA256=5650d358cc698ddf57465e379374c891002d100b6b7ccd071043c4ba0860fad1 +SANITIZED_OUTPUT_SHA256=5650d358cc698ddf57465e379374c891002d100b6b7ccd071043c4ba0860fad1 OUTPUT_BEGIN -PVE300_BEGIN= -COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-QUOTEFIX-300R1 +PVE301_BEGIN= +COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1 ANCHOR298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d -MODE=CHANGE_PVE03_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET -SELFTESTS=14 +FAILED300_COMMIT=114f1d4732136a3b57717a4354fed893a1403ccb +MODE=CHANGE_PVE03_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET_TASKWAIT +SELFTESTS=18 AUTHORITY_FRESH_READ=OK -STATUS=FAIL -PHASE=GLOBAL_PRESTATE -REASON=COMPETING_PVE_TASK -COMPLETED_BEFORE_FAILURE=[] +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GLOBAL_PRESTATE","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +PVE03_STRICT_SSH_STREAM_PREFLIGHT=PASS +PVE03_INVENTORY=[{"config_sha256":"814d7661a1fc727491a9d31637fd3604b29ee3b7156cace4aa9f7f9714a4aa1f","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0},"maxdisk_bytes":103079215104,"name":"edge-vm","node":"pve03","status":"running","template":false,"type":"qemu","vmid":130},{"config_sha256":"0abbf7381ccedc3b65fd52840baaa9a1d0790268cb90d3e10cdc9829b429b447","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0},"maxdisk_bytes":85899345920,"name":"homelab-ops-worker","node":"pve03","status":"running","template":false,"type":"qemu","vmid":190},{"config_sha256":"22de890f3ac01bf08f592f77d01d2dd2e1d29d9a901a2ca40314d0a565766338","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0},"maxdisk_bytes":53687091200,"name":"newfi-staging","node":"pve03","status":"running","template":false,"type":"qemu","vmid":211}] +PVE03_GUEST_COUNT=3 +CRYPT_COUNT=13 +PHYSICAL_COUNT=13 +BACKUP_ORDER=[211,190,130] +GUEST_BACKUP_BEGIN=211 +GUEST_SOURCE={"maxdisk_bytes":53687091200,"name":"newfi-staging","node":"pve03","status":"running","template":false,"type":"qemu","vmid":211} +QEMU_DISK_GUARD={"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST211_PRECAPACITY","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":53687091200,"staging_available_bytes":296977125376,"staging_required_bytes":71403831296,"streaming_single_copy_fits":true,"vmid":211} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354014148,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353950289,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087776449247,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087776449221,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":54228156416} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":54228156416} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST211_PRESTREAM","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GENERATION=G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd +LOGICAL_ID=guest-pve03-qemu-211 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd +PAYLOAD_NAME=vzdump-qemu-211-2026_09_13-06_45_40.vma.zst +PROXMOX_ARCHIVE_VERIFY=PASS +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354014148,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353950289,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087776449247,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087776449221,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":1159366272} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":1159366272} +REPLICA_1_CRYPT=pve03-mail-01-crypt +REPLICA_1_PHYSICAL=pve03-mail-01 +REPLICA_1_FREE_BYTES_PREUPLOAD=1087776449247 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-02-crypt +REPLICA_2_PHYSICAL=pve03-mail-02 +REPLICA_2_FREE_BYTES_PREUPLOAD=1087776449221 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 +SOURCE_CONFIG_UNCHANGED=YES +GUEST_BACKUP_COMPLETE=211 +GUEST_BACKUP_BEGIN=190 +GUEST_SOURCE={"maxdisk_bytes":85899345920,"name":"homelab-ops-worker","node":"pve03","status":"running","template":false,"type":"qemu","vmid":190} +QEMU_DISK_GUARD={"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST190_PRECAPACITY","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":85899345920,"staging_available_bytes":296977121280,"staging_required_bytes":103616086016,"streaming_single_copy_fits":true,"vmid":190} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354014148,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353950289,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087157995658,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087157995632,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":86440411136} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":86440411136} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST190_PRESTREAM","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GENERATION=G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360 +LOGICAL_ID=guest-pve03-qemu-190 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360 +PAYLOAD_NAME=vzdump-qemu-190-2026_09_13-06_47_40.vma.zst +PROXMOX_ARCHIVE_VERIFY=PASS +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087157995658,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1087157995632,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":2091753079} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":2091753079} +REPLICA_1_CRYPT=pve03-mail-01-crypt +REPLICA_1_PHYSICAL=pve03-mail-01 +REPLICA_1_FREE_BYTES_PREUPLOAD=1087157995658 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-02-crypt +REPLICA_2_PHYSICAL=pve03-mail-02 +REPLICA_2_FREE_BYTES_PREUPLOAD=1087157995632 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 +SOURCE_CONFIG_UNCHANGED=YES +GUEST_BACKUP_COMPLETE=190 +GUEST_BACKUP_BEGIN=130 +GUEST_SOURCE={"maxdisk_bytes":103079215104,"name":"edge-vm","node":"pve03","status":"running","template":false,"type":"qemu","vmid":130} +QEMU_DISK_GUARD={"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST130_PRECAPACITY","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":103079215104,"staging_available_bytes":296977121280,"staging_required_bytes":120795955200,"streaming_single_copy_fits":true,"vmid":130} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927471,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927445,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":103620280320} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST130_PRESTREAM","max_conflicts":0,"node":"pve03","samples":3,"status":"PASS"} +GENERATION=G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f +LOGICAL_ID=guest-pve03-qemu-130 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f +PAYLOAD_NAME=vzdump-qemu-130-2026_09_13-06_50_51.vma.zst +PROXMOX_ARCHIVE_VERIFY=PASS +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927471,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927445,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":11390419000} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085882288719,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":11390419000} +REPLICA_1_CRYPT=pve01-mail-05-crypt +REPLICA_1_PHYSICAL=pve01-mail-05 +REPLICA_1_FREE_BYTES_PREUPLOAD=1085882288719 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-04-crypt +REPLICA_2_PHYSICAL=pve03-mail-04 +REPLICA_2_FREE_BYTES_PREUPLOAD=1085882288719 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 +SOURCE_CONFIG_UNCHANGED=YES +GUEST_BACKUP_COMPLETE=130 +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927471,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927445,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":4294967296} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":4294967296} +NODE_BUNDLE_CLOUD_PRECAPACITY=YES +NODE_BUNDLE_MEMBER_COUNT=226 +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927471,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606927445,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":541115660} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":541115660} +REPLICA_1_CRYPT=pve03-mail-01-crypt +REPLICA_1_PHYSICAL=pve03-mail-01 +REPLICA_1_FREE_BYTES_PREUPLOAD=1085606927471 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-02-crypt +REPLICA_2_PHYSICAL=pve03-mail-02 +REPLICA_2_FREE_BYTES_PREUPLOAD=1085606927445 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 +NODE_BUNDLE_COMPLETE=YES +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606875329,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606875303,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":536870912} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":536870912} +NODESET_GUESTS_COMPLETE=[130,190,211] +PVE03_NODESET_SEALED=YES +PVE03_GUEST_BACKUPS_COMPLETE=[211,190,130] +NODE_BUNDLE_RECORD={"generation":"N301-20260913T070421Z-2ace9051dc62ad81accd6052aa3692fd","logical_id":"node-pve03","manifest_sha256":"e9a028f53069e088a5ca69f404468ddc50ce0d8caedaea2c529a41aabfc5c192","node":"pve03","payload_bytes":50444,"payload_sha256":"c99bccb00d24af53fa1e264c7744398c4be906454d414e7bf057a8c46c5dddd2","recovery_policy":"CLEAN_INSTALL_JOIN_RECONCILE_NOT_BLIND_PMXCFS_OVERWRITE","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"]} +NODE_SET_RECORD={"generation":"S301-20260913T070439Z-f040b9c10175c6f22b9c79b1a2a2eca2","logical_id":"nodeset-pve03","manifest_sha256":"c12abc4b45597b73608e74a443e471b29bde9cb00c7c733f9d34ebb8386f238d","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"]} +STATUS=OK +PHASE=COMPLETE +NEXT_GATE=PVE03_SEQUENTIAL_RESTORE_PROOF CURRENT_SOURCE_CONFIG_UNCHANGED=YES CURRENT_LOCAL_PAYLOAD_FILES_REMAINING=0 FAILED_WORK_DIRECTORY=NONE -FAILURE_DIAGNOSTIC_SUMMARY={"cloud_write_attempts":0,"completed_vmids":[],"current_type":null,"current_vmid":null,"failed_rc":"NONE","failed_tool":"NONE","phase":"GLOBAL_PRESTATE","reason":"COMPETING_PVE_TASK"} -GENERATION_DISCOVERY_REJECT_SUMMARY={"not_applicable":"backup_command","phase":"GLOBAL_PRESTATE"} +FAILURE_DIAGNOSTIC_SUMMARY={"cloud_write_attempts":20,"completed_vmids":[211,190,130],"current_type":null,"current_vmid":null,"failed_rc":"NONE","failed_tool":"NONE","phase":"FINAL_GUARD","reason":"NONE"} +GENERATION_DISCOVERY_REJECT_SUMMARY={"not_applicable":"backup_command","phase":"FINAL_GUARD"} FAILED_TOOL=NONE FAILED_RC=NONE FAILED_STDERR_BYTES=0 FAILED_STDERR_SHA256=NONE -CLOUD_CALLS=0 -CLOUD_WRITE_ATTEMPTS=0 +CLOUD_CALLS=238 +CLOUD_WRITE_ATTEMPTS=20 CLOUD_DELETIONS=0 -BACKUP_EXECUTIONS=0 -NODE_BUNDLE_EXECUTIONS=0 +BACKUP_EXECUTIONS=3 +NODE_BUNDLE_EXECUTIONS=1 RESTORE_EXECUTIONS=0 SOURCE_GUEST_MUTATIONS=0 SYSTEMD_MUTATIONS=0 -LIVE_RCLONE_CONFIG_READS=0 +LIVE_RCLONE_CONFIG_READS=26 LIVE_RCLONE_CONFIG_WRITES=0 -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-QUOTEFIX-300R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} -PVE300_END= +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-301R1","status":"OK","changes_made":true,"rollback_started":false,"rollback_restored":null} +PVE301_END= OUTPUT_END CHAT_OUTPUT_END