diff --git a/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1.json b/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1.json new file mode 100644 index 00000000..a70c7385 --- /dev/null +++ b/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1", + "status": "FAIL", + "rc": 2, + "host": "pve01", + "mode": "change", + "component": "infra-dr-backupv2-pve02-sequential-backup-nodebundle-taskwait", + "started_at_utc": "2026-09-13T08:07:49Z", + "finished_at_utc": "2026-09-13T08:15:54Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "5c9a4ef394ef9771f83f28c484bf58e1477fba3a658452a05c0c7322a9ba82f2", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": true, + "result_contract_valid": true, + "result_contract_status": "FAIL", + "result_contract_error": null, + "command_rc": 2, + "changes_made": true, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "MUTATED", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d", + "sanitized_output_sha256": "9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "PVE303_BEGIN=\nCOMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1\nANCHOR302_COMMIT=6967f7b832bef65ee5877aeaf9ab7f17d0bdd2ca\nRECONCILE298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d\nMODE=CHANGE_PVE02_MIXED_GUEST_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET_TASKWAIT\nSELFTESTS=22\nAUTHORITY_FRESH_READ=OK\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GLOBAL_PRESTATE\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nPVE02_STRICT_SSH_STREAM_PREFLIGHT=PASS\nPVE02_INVENTORY=[{\"config_sha256\":\"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b\",\"guard\":{\"additional_mount_not_explicit_backup_one_count\":0,\"mountpoint_keys\":[],\"rootfs_present\":true},\"maxdisk_bytes\":8350298112,\"name\":\"dns2\",\"node\":\"pve02\",\"status\":\"running\",\"template\":false,\"type\":\"lxc\",\"vmid\":111},{\"config_sha256\":\"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1\",\"guard\":{\"additional_mount_not_explicit_backup_one_count\":0,\"mountpoint_keys\":[],\"rootfs_present\":true},\"maxdisk_bytes\":8350298112,\"name\":\"unbound2\",\"node\":\"pve02\",\"status\":\"running\",\"template\":false,\"type\":\"lxc\",\"vmid\":113},{\"config_sha256\":\"a3668bdf650c6e90f0dd9b684c272c16dc8243dfe4bbf90de0ae90cbe064d5a8\",\"guard\":{\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\"],\"unused_disk_count\":0},\"maxdisk_bytes\":214748364800,\"name\":\"forum-prod\",\"node\":\"pve02\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":160},{\"config_sha256\":\"d59cd51387dce644c10925d11d3bb49bfea57eb9bc4bee9ebb1d69faae768295\",\"guard\":{\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\",\"scsi1\"],\"unused_disk_count\":0},\"maxdisk_bytes\":103079215104,\"name\":\"edge-cold-standby\",\"node\":\"pve02\",\"status\":\"stopped\",\"template\":false,\"type\":\"qemu\",\"vmid\":9130}]\nPVE02_GUEST_COUNT=4\nSTOPPED_GUEST_POLICY={\"policy\":\"INCLUDE_BACKUP_PRESERVE_STOPPED_STATE\",\"source_status\":\"stopped\",\"vmid\":9130}\nCRYPT_COUNT=13\nPHYSICAL_COUNT=13\nBACKUP_ORDER=[111,113,9130,160]\nGUEST_BACKUP_BEGIN=111\nGUEST_SOURCE={\"maxdisk_bytes\":8350298112,\"name\":\"dns2\",\"node\":\"pve02\",\"status\":\"running\",\"template\":false,\"type\":\"lxc\",\"vmid\":111}\nLXC_MOUNT_GUARD={\"additional_mount_not_explicit_backup_one_count\":0,\"mountpoint_keys\":[],\"rootfs_present\":true}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST111_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":8350298112,\"staging_available_bytes\":296977117184,\"staging_required_bytes\":26067038208,\"streaming_single_copy_fits\":true,\"vmid\":111}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606871526,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606871500,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":8891363328}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST111_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3\nLOGICAL_ID=guest-pve02-lxc-111\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3\nPAYLOAD_NAME=vzdump-lxc-111-2026_09_13-08_08_39.tar.zst\nPROXMOX_ARCHIVE_VERIFY=PASS\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606871526,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606871500,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":929354534}\nREPLICA_1_CRYPT=pve03-mail-01-crypt\nREPLICA_1_PHYSICAL=pve03-mail-01\nREPLICA_1_FREE_BYTES_PREUPLOAD=1085606871526\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-02-crypt\nREPLICA_2_PHYSICAL=pve03-mail-02\nREPLICA_2_FREE_BYTES_PREUPLOAD=1085606871500\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nSOURCE_CONFIG_UNCHANGED=YES\nGUEST_BACKUP_COMPLETE=111\nGUEST_BACKUP_BEGIN=113\nGUEST_SOURCE={\"maxdisk_bytes\":8350298112,\"name\":\"unbound2\",\"node\":\"pve02\",\"status\":\"running\",\"template\":false,\"type\":\"lxc\",\"vmid\":113}\nLXC_MOUNT_GUARD={\"additional_mount_not_explicit_backup_one_count\":0,\"mountpoint_keys\":[],\"rootfs_present\":true}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST113_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":8350298112,\"staging_available_bytes\":296977117184,\"staging_required_bytes\":26067038208,\"streaming_single_copy_fits\":true,\"vmid\":113}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085218485999,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085218485973,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":8891363328}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST113_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad\nLOGICAL_ID=guest-pve02-lxc-113\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad\nPAYLOAD_NAME=vzdump-lxc-113-2026_09_13-08_10_14.tar.zst\nPROXMOX_ARCHIVE_VERIFY=PASS\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085218485999,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085218485973,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":762734583}\nREPLICA_1_CRYPT=pve03-mail-01-crypt\nREPLICA_1_PHYSICAL=pve03-mail-01\nREPLICA_1_FREE_BYTES_PREUPLOAD=1085218485999\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-02-crypt\nREPLICA_2_PHYSICAL=pve03-mail-02\nREPLICA_2_FREE_BYTES_PREUPLOAD=1085218485973\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nSOURCE_CONFIG_UNCHANGED=YES\nGUEST_BACKUP_COMPLETE=113\nGUEST_BACKUP_BEGIN=9130\nGUEST_SOURCE={\"maxdisk_bytes\":103079215104,\"name\":\"edge-cold-standby\",\"node\":\"pve02\",\"status\":\"stopped\",\"template\":false,\"type\":\"qemu\",\"vmid\":9130}\nQEMU_DISK_GUARD={\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\",\"scsi1\"],\"unused_disk_count\":0}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST9130_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":103079215104,\"staging_available_bytes\":296977117184,\"staging_required_bytes\":120795955200,\"streaming_single_copy_fits\":true,\"vmid\":9130}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1084996761091,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1084996761065,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":103620280320}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST9130_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84\nLOGICAL_ID=guest-pve02-qemu-9130\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84\nPAYLOAD_NAME=vzdump-qemu-9130-2026_09_13-08_11_26.vma.zst\nSTATUS=FAIL\nPHASE=GUEST9130_STREAM_VZDUMP\nREASON=SOURCE_GUEST_STATUS\nCOMPLETED_BEFORE_FAILURE=[{\"generation\":\"G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3\",\"logical_id\":\"guest-pve02-lxc-111\",\"manifest_sha256\":\"362b9abbe0f3d1105ee353768fb2e60fbc5a82222228d9b30b720a282315c37b\",\"name\":\"dns2\",\"payload_bytes\":388289318,\"payload_sha256\":\"649785b391b35f0ba7c17577ca51be1541dbf3e772480fac3b821f8a63680a4b\",\"replicas\":[\"pve03-mail-01-crypt\",\"pve03-mail-02-crypt\"],\"source_config_sha256\":\"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b\",\"source_status\":\"running\",\"type\":\"lxc\",\"vmid\":111},{\"generation\":\"G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad\",\"logical_id\":\"guest-pve02-lxc-113\",\"manifest_sha256\":\"21e60227e12baf7663384f4ba14ab980948ced2ded47a2a57386b20a54ed8233\",\"name\":\"unbound2\",\"payload_bytes\":221669367,\"payload_sha256\":\"c6205058bf56ef2ac320cf5774d9d29b3c3ec98b71087203c8df01c94ea659ff\",\"replicas\":[\"pve03-mail-01-crypt\",\"pve03-mail-02-crypt\"],\"source_config_sha256\":\"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1\",\"source_status\":\"running\",\"type\":\"lxc\",\"vmid\":113}]\nCURRENT_SOURCE_CONFIG_UNCHANGED=NO\nCURRENT_LOCAL_PAYLOAD_FILES_REMAINING=21\nFAILED_WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84\nFAILURE_DIAGNOSTIC_SUMMARY={\"cloud_write_attempts\":8,\"completed_vmids\":[111,113],\"current_type\":\"qemu\",\"current_vmid\":9130,\"failed_rc\":\"NONE\",\"failed_tool\":\"NONE\",\"phase\":\"GUEST9130_STREAM_VZDUMP\",\"reason\":\"SOURCE_GUEST_STATUS\"}\nGENERATION_DISCOVERY_REJECT_SUMMARY={\"not_applicable\":\"backup_command\",\"phase\":\"GUEST9130_STREAM_VZDUMP\"}\nFAILED_TOOL=NONE\nFAILED_RC=NONE\nFAILED_STDERR_BYTES=0\nFAILED_STDERR_SHA256=NONE\nCLOUD_CALLS=112\nCLOUD_WRITE_ATTEMPTS=8\nCLOUD_DELETIONS=0\nBACKUP_EXECUTIONS=3\nNODE_BUNDLE_EXECUTIONS=0\nRESTORE_EXECUTIONS=0\nSOURCE_GUEST_MUTATIONS=0\nSYSTEMD_MUTATIONS=0\nLIVE_RCLONE_CONFIG_READS=26\nLIVE_RCLONE_CONFIG_WRITES=0\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\nPVE303_END=\n" +} diff --git a/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1.txt b/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1.txt new file mode 100644 index 00000000..38c18f4d --- /dev/null +++ b/runtime/history/SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1.txt @@ -0,0 +1,204 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1 +STATUS=FAIL +RC=2 +HOST=pve01 +MODE=change +COMPONENT=infra-dr-backupv2-pve02-sequential-backup-nodebundle-taskwait +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=5c9a4ef394ef9771f83f28c484bf58e1477fba3a658452a05c0c7322a9ba82f2 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=true +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=2 +CHANGES_MADE=true +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=MUTATED +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d +SANITIZED_OUTPUT_SHA256=9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d +OUTPUT_BEGIN +PVE303_BEGIN= +COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1 +ANCHOR302_COMMIT=6967f7b832bef65ee5877aeaf9ab7f17d0bdd2ca +RECONCILE298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d +MODE=CHANGE_PVE02_MIXED_GUEST_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET_TASKWAIT +SELFTESTS=22 +AUTHORITY_FRESH_READ=OK +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GLOBAL_PRESTATE","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +PVE02_STRICT_SSH_STREAM_PREFLIGHT=PASS +PVE02_INVENTORY=[{"config_sha256":"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b","guard":{"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true},"maxdisk_bytes":8350298112,"name":"dns2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":111},{"config_sha256":"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1","guard":{"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true},"maxdisk_bytes":8350298112,"name":"unbound2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":113},{"config_sha256":"a3668bdf650c6e90f0dd9b684c272c16dc8243dfe4bbf90de0ae90cbe064d5a8","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0},"maxdisk_bytes":214748364800,"name":"forum-prod","node":"pve02","status":"running","template":false,"type":"qemu","vmid":160},{"config_sha256":"d59cd51387dce644c10925d11d3bb49bfea57eb9bc4bee9ebb1d69faae768295","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0},"maxdisk_bytes":103079215104,"name":"edge-cold-standby","node":"pve02","status":"stopped","template":false,"type":"qemu","vmid":9130}] +PVE02_GUEST_COUNT=4 +STOPPED_GUEST_POLICY={"policy":"INCLUDE_BACKUP_PRESERVE_STOPPED_STATE","source_status":"stopped","vmid":9130} +CRYPT_COUNT=13 +PHYSICAL_COUNT=13 +BACKUP_ORDER=[111,113,9130,160] +GUEST_BACKUP_BEGIN=111 +GUEST_SOURCE={"maxdisk_bytes":8350298112,"name":"dns2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":111} +LXC_MOUNT_GUARD={"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST111_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":8350298112,"staging_available_bytes":296977117184,"staging_required_bytes":26067038208,"streaming_single_copy_fits":true,"vmid":111} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871526,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871500,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":8891363328} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST111_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GENERATION=G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3 +LOGICAL_ID=guest-pve02-lxc-111 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3 +PAYLOAD_NAME=vzdump-lxc-111-2026_09_13-08_08_39.tar.zst +PROXMOX_ARCHIVE_VERIFY=PASS +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871526,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871500,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":929354534} +REPLICA_1_CRYPT=pve03-mail-01-crypt +REPLICA_1_PHYSICAL=pve03-mail-01 +REPLICA_1_FREE_BYTES_PREUPLOAD=1085606871526 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-02-crypt +REPLICA_2_PHYSICAL=pve03-mail-02 +REPLICA_2_FREE_BYTES_PREUPLOAD=1085606871500 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 +SOURCE_CONFIG_UNCHANGED=YES +GUEST_BACKUP_COMPLETE=111 +GUEST_BACKUP_BEGIN=113 +GUEST_SOURCE={"maxdisk_bytes":8350298112,"name":"unbound2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":113} +LXC_MOUNT_GUARD={"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST113_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":8350298112,"staging_available_bytes":296977117184,"staging_required_bytes":26067038208,"streaming_single_copy_fits":true,"vmid":113} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485999,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485973,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":8891363328} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST113_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GENERATION=G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad +LOGICAL_ID=guest-pve02-lxc-113 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad +PAYLOAD_NAME=vzdump-lxc-113-2026_09_13-08_10_14.tar.zst +PROXMOX_ARCHIVE_VERIFY=PASS +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485999,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485973,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":762734583} +REPLICA_1_CRYPT=pve03-mail-01-crypt +REPLICA_1_PHYSICAL=pve03-mail-01 +REPLICA_1_FREE_BYTES_PREUPLOAD=1085218485999 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-02-crypt +REPLICA_2_PHYSICAL=pve03-mail-02 +REPLICA_2_FREE_BYTES_PREUPLOAD=1085218485973 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 +SOURCE_CONFIG_UNCHANGED=YES +GUEST_BACKUP_COMPLETE=113 +GUEST_BACKUP_BEGIN=9130 +GUEST_SOURCE={"maxdisk_bytes":103079215104,"name":"edge-cold-standby","node":"pve02","status":"stopped","template":false,"type":"qemu","vmid":9130} +QEMU_DISK_GUARD={"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST9130_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":103079215104,"staging_available_bytes":296977117184,"staging_required_bytes":120795955200,"streaming_single_copy_fits":true,"vmid":9130} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1084996761091,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1084996761065,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":103620280320} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST9130_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GENERATION=G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84 +LOGICAL_ID=guest-pve02-qemu-9130 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84 +PAYLOAD_NAME=vzdump-qemu-9130-2026_09_13-08_11_26.vma.zst +STATUS=FAIL +PHASE=GUEST9130_STREAM_VZDUMP +REASON=SOURCE_GUEST_STATUS +COMPLETED_BEFORE_FAILURE=[{"generation":"G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3","logical_id":"guest-pve02-lxc-111","manifest_sha256":"362b9abbe0f3d1105ee353768fb2e60fbc5a82222228d9b30b720a282315c37b","name":"dns2","payload_bytes":388289318,"payload_sha256":"649785b391b35f0ba7c17577ca51be1541dbf3e772480fac3b821f8a63680a4b","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"],"source_config_sha256":"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b","source_status":"running","type":"lxc","vmid":111},{"generation":"G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad","logical_id":"guest-pve02-lxc-113","manifest_sha256":"21e60227e12baf7663384f4ba14ab980948ced2ded47a2a57386b20a54ed8233","name":"unbound2","payload_bytes":221669367,"payload_sha256":"c6205058bf56ef2ac320cf5774d9d29b3c3ec98b71087203c8df01c94ea659ff","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"],"source_config_sha256":"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1","source_status":"running","type":"lxc","vmid":113}] +CURRENT_SOURCE_CONFIG_UNCHANGED=NO +CURRENT_LOCAL_PAYLOAD_FILES_REMAINING=21 +FAILED_WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84 +FAILURE_DIAGNOSTIC_SUMMARY={"cloud_write_attempts":8,"completed_vmids":[111,113],"current_type":"qemu","current_vmid":9130,"failed_rc":"NONE","failed_tool":"NONE","phase":"GUEST9130_STREAM_VZDUMP","reason":"SOURCE_GUEST_STATUS"} +GENERATION_DISCOVERY_REJECT_SUMMARY={"not_applicable":"backup_command","phase":"GUEST9130_STREAM_VZDUMP"} +FAILED_TOOL=NONE +FAILED_RC=NONE +FAILED_STDERR_BYTES=0 +FAILED_STDERR_SHA256=NONE +CLOUD_CALLS=112 +CLOUD_WRITE_ATTEMPTS=8 +CLOUD_DELETIONS=0 +BACKUP_EXECUTIONS=3 +NODE_BUNDLE_EXECUTIONS=0 +RESTORE_EXECUTIONS=0 +SOURCE_GUEST_MUTATIONS=0 +SYSTEMD_MUTATIONS=0 +LIVE_RCLONE_CONFIG_READS=26 +LIVE_RCLONE_CONFIG_WRITES=0 +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null} +PVE303_END= + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index ce38cbcd..a70c7385 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,27 +1,27 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-RESTORE-PROOF-302R1", - "status": "OK", - "rc": 0, + "command_id": "SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1", + "status": "FAIL", + "rc": 2, "host": "pve01", "mode": "change", - "component": "infra-dr-backupv2-pve03-sequential-restore-proof", - "started_at_utc": "2026-09-13T07:19:34Z", - "finished_at_utc": "2026-09-13T07:38:13Z", + "component": "infra-dr-backupv2-pve02-sequential-backup-nodebundle-taskwait", + "started_at_utc": "2026-09-13T08:07:49Z", + "finished_at_utc": "2026-09-13T08:15:54Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "d8885fb6a1bf2da2494b56b2e193edc054925b16877dffea2ebf90aadce179b5", + "command_sha256": "5c9a4ef394ef9771f83f28c484bf58e1477fba3a658452a05c0c7322a9ba82f2", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, "change_declared": true, "result_contract_valid": true, - "result_contract_status": "OK", + "result_contract_status": "FAIL", "result_contract_error": null, - "command_rc": 0, + "command_rc": 2, "changes_made": true, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "9571cf9070a804ea41f7ca7f900b60c52026dc5f44e8b0cc9e7e64bf703c1efa", - "sanitized_output_sha256": "9571cf9070a804ea41f7ca7f900b60c52026dc5f44e8b0cc9e7e64bf703c1efa", + "raw_evidence_sha256": "9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d", + "sanitized_output_sha256": "9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "PVE302_BEGIN=\nCOMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-RESTORE-PROOF-302R1\nANCHOR301_COMMIT=44008e8e152c13bc078564a46e23d92eb037bb62\nMODE=CHANGE_PVE03_CLOUD_ONLY_SEQUENTIAL_RESTORE_BOOT_NODEBUNDLE_NODESET_VERIFY\nSELFTESTS=17\nAUTHORITY_FRESH_READ=OK\nSOURCE_QGA_PREFLIGHT={\"node\":\"pve03\",\"osinfo\":\"PASS\",\"ping\":\"PASS\",\"vmid\":211}\nSOURCE_QGA_PREFLIGHT={\"node\":\"pve03\",\"osinfo\":\"PASS\",\"ping\":\"PASS\",\"vmid\":190}\nSOURCE_QGA_PREFLIGHT={\"node\":\"pve03\",\"osinfo\":\"PASS\",\"ping\":\"PASS\",\"vmid\":130}\nCRYPT_COUNT=13\nPHYSICAL_COUNT=13\nNODE_BUNDLE_RECONSTRUCTION_VERIFIED=YES\nNODE_BUNDLE_PVE_REFERENCE_VERIFIED=YES\nNODE_BUNDLE_MEMBER_COUNT=226\nVM_RESTORE_BEGIN=211\nTEST_VMID=900311\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM211_SOURCE_PRESTATE\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nSOURCE_QGA_PREFLIGHT={\"node\":\"pve03\",\"osinfo\":\"PASS\",\"ping\":\"PASS\",\"vmid\":211}\nGENERATION_DISCOVERY_SUMMARY={\"generation\":\"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd\",\"listed_replicas\":2,\"reject_counts\":{\"REMOTE_GENERATION_ABSENT\":11},\"valid_hits\":2,\"vmid\":211}\nGENERATION_DISCOVERED={\"generation\":\"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd\",\"manifest_sha256\":\"76dd08d6e858042eabd48b8dc6578d8ec6a335827a2d4c19e69031d22beda490\",\"replica_count\":2,\"vmid\":211}\nRESTORE_ADMISSION={\"local_lvm_available\":751570296557,\"maxdisk_bytes\":53687091200,\"payload_bytes\":618301056,\"staging_available\":296977117184,\"test_node\":\"pve01\",\"vmid\":211}\nLOCAL_RESTORE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM211_LOCAL_PREP\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nRESTORE_ARCHIVE_VERIFIED={\"bytes\":618301056,\"generation\":\"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd\",\"sha256\":\"e76393b03366751b461b130361923cf1f42f416886045eb29fd4212a737eac99\",\"vmid\":211}\nLOCAL_RESTORE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM211_LOCAL_PRE_QMRESTORE\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nTEST_VM_ISOLATION={\"network_devices_remaining\":0,\"protection\":1,\"removed_keys\":[\"net0\",\"serial0\"],\"test_vmid\":900311}\nQEMU_GUEST_AGENT_PING=PASS\nGUEST_OSINFO_QUERY=PASS\nGUEST_HOSTNAME_QUERY=PASS\nGUEST_RESTORE_BOOT_VERIFIED=YES\nTEST_VM_AND_VOLUMES_REMOVED=YES\nRESTORE_PAYLOAD_REMOVED=YES\nSOURCE_CONFIG_UNCHANGED=YES\nVM_RESTORE_PROOF_COMPLETE=211\nVM_RESTORE_BEGIN=190\nTEST_VMID=900312\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM190_SOURCE_PRESTATE\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nSOURCE_QGA_PREFLIGHT={\"node\":\"pve03\",\"osinfo\":\"PASS\",\"ping\":\"PASS\",\"vmid\":190}\nGENERATION_DISCOVERY_SUMMARY={\"generation\":\"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360\",\"listed_replicas\":2,\"reject_counts\":{\"REMOTE_GENERATION_ABSENT\":11},\"valid_hits\":2,\"vmid\":190}\nGENERATION_DISCOVERED={\"generation\":\"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360\",\"manifest_sha256\":\"6a954f3beb94e35b289c10717f5ac4387e8461df1a5f5e03db4329cade8b1fcc\",\"replica_count\":2,\"vmid\":190}\nRESTORE_ADMISSION={\"local_lvm_available\":751048795253,\"maxdisk_bytes\":85899345920,\"payload_bytes\":1550687863,\"staging_available\":296977117184,\"test_node\":\"pve01\",\"vmid\":190}\nLOCAL_RESTORE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM190_LOCAL_PREP\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nRESTORE_ARCHIVE_VERIFIED={\"bytes\":1550687863,\"generation\":\"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360\",\"sha256\":\"a3669dcc4de755ad650f363a4aa5578b00162bdcfe77e8bd37d60ffaa52ddb0c\",\"vmid\":190}\nLOCAL_RESTORE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM190_LOCAL_PRE_QMRESTORE\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nTEST_VM_ISOLATION={\"network_devices_remaining\":0,\"protection\":1,\"removed_keys\":[\"net0\",\"serial0\"],\"test_vmid\":900312}\nQEMU_GUEST_AGENT_PING=PASS\nGUEST_OSINFO_QUERY=PASS\nGUEST_HOSTNAME_QUERY=PASS\nGUEST_RESTORE_BOOT_VERIFIED=YES\nTEST_VM_AND_VOLUMES_REMOVED=YES\nRESTORE_PAYLOAD_REMOVED=YES\nSOURCE_CONFIG_UNCHANGED=YES\nVM_RESTORE_PROOF_COMPLETE=190\nVM_RESTORE_BEGIN=130\nTEST_VMID=900313\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM130_SOURCE_PRESTATE\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nSOURCE_QGA_PREFLIGHT={\"node\":\"pve03\",\"osinfo\":\"PASS\",\"ping\":\"PASS\",\"vmid\":130}\nGENERATION_DISCOVERY_SUMMARY={\"generation\":\"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f\",\"listed_replicas\":2,\"reject_counts\":{\"REMOTE_GENERATION_ABSENT\":11},\"valid_hits\":2,\"vmid\":130}\nGENERATION_DISCOVERED={\"generation\":\"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f\",\"manifest_sha256\":\"5f61523d3563096c5e78429c8a63bb347d559ec76de152d896bd2c263146c0ab\",\"replica_count\":2,\"vmid\":130}\nRESTORE_ADMISSION={\"local_lvm_available\":751048795253,\"maxdisk_bytes\":103079215104,\"payload_bytes\":10849353784,\"staging_available\":296977117184,\"test_node\":\"pve01\",\"vmid\":130}\nLOCAL_RESTORE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM130_LOCAL_PREP\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nRESTORE_ARCHIVE_VERIFIED={\"bytes\":10849353784,\"generation\":\"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f\",\"sha256\":\"4567b3c0322b530079bccec6789524ac0291024dbf50942bf3e550d67c70c1f9\",\"vmid\":130}\nLOCAL_RESTORE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM130_LOCAL_PRE_QMRESTORE\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nTEST_VM_ISOLATION={\"network_devices_remaining\":0,\"protection\":1,\"removed_keys\":[\"net0\",\"serial0\",\"startup\"],\"test_vmid\":900313}\nQEMU_GUEST_AGENT_PING=PASS\nGUEST_OSINFO_QUERY=PASS\nGUEST_HOSTNAME_QUERY=PASS\nGUEST_RESTORE_BOOT_VERIFIED=YES\nTEST_VM_AND_VOLUMES_REMOVED=YES\nRESTORE_PAYLOAD_REMOVED=YES\nSOURCE_CONFIG_UNCHANGED=YES\nVM_RESTORE_PROOF_COMPLETE=130\nPVE03_NODESET_MANIFEST_VERIFIED=YES\nPVE03_NODESET_RESTORE_COVERAGE_COMPLETE=YES\nPVE03_SEQUENTIAL_RESTORE_PROOFS_COMPLETE=[211,190,130]\nPVE03_NODE_BUNDLE_RESTORE_VALIDATED=YES\nPVE03_NODESET_RESTORE_COVERAGE_COMPLETE=YES\nCOMPLETED_RESTORE_PROOFS=[{\"generation\":\"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd\",\"manifest_sha256\":\"76dd08d6e858042eabd48b8dc6578d8ec6a335827a2d4c19e69031d22beda490\",\"payload_sha256\":\"e76393b03366751b461b130361923cf1f42f416886045eb29fd4212a737eac99\",\"test_vmid\":900311,\"vmid\":211},{\"generation\":\"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360\",\"manifest_sha256\":\"6a954f3beb94e35b289c10717f5ac4387e8461df1a5f5e03db4329cade8b1fcc\",\"payload_sha256\":\"a3669dcc4de755ad650f363a4aa5578b00162bdcfe77e8bd37d60ffaa52ddb0c\",\"test_vmid\":900312,\"vmid\":190},{\"generation\":\"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f\",\"manifest_sha256\":\"5f61523d3563096c5e78429c8a63bb347d559ec76de152d896bd2c263146c0ab\",\"payload_sha256\":\"4567b3c0322b530079bccec6789524ac0291024dbf50942bf3e550d67c70c1f9\",\"test_vmid\":900313,\"vmid\":130}]\nSTATUS=OK\nPHASE=COMPLETE\nNEXT_GATE=PVE02_SEQUENTIAL_BACKUP_WITH_NODE_BUNDLE\nCURRENT_SOURCE_VMID=NONE\nCURRENT_TEST_VMID=NONE\nCURRENT_WORKDIR=NONE\nCURRENT_LOCAL_PAYLOAD_FILES_REMAINING=0\nLAST_GENERATION_DISCOVERY_SUMMARY={\"generation\":\"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f\",\"listed_replicas\":2,\"reject_counts\":{\"REMOTE_GENERATION_ABSENT\":11},\"valid_hits\":2,\"vmid\":130}\nLAST_SOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM130_SOURCE_PRESTATE\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nLAST_LOCAL_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"VM130_LOCAL_PRE_QMRESTORE\",\"max_conflicts\":0,\"samples\":3,\"status\":\"PASS\"}\nNODE_BUNDLE_VERIFIED=YES\nNODESET_VERIFIED=YES\nFAILED_TOOL=NONE\nFAILED_RC=NONE\nFAILED_STDERR_BYTES=0\nFAILED_STDERR_SHA256=NONE\nCLOUD_CALLS=146\nCLOUD_WRITE_ATTEMPTS=0\nCLOUD_DELETIONS=0\nBACKUP_EXECUTIONS=0\nNODE_BUNDLE_EXECUTIONS=0\nRESTORE_EXECUTIONS=3\nSOURCE_GUEST_MUTATIONS=0\nSYSTEMD_MUTATIONS=0\nLIVE_RCLONE_CONFIG_READS=26\nLIVE_RCLONE_CONFIG_WRITES=0\nROLLBACK_STARTED=NO\nROLLBACK_RESTORED=NOT_APPLICABLE\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-RESTORE-PROOF-302R1\",\"status\":\"OK\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\nPVE302_END=\n" + "output": "PVE303_BEGIN=\nCOMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1\nANCHOR302_COMMIT=6967f7b832bef65ee5877aeaf9ab7f17d0bdd2ca\nRECONCILE298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d\nMODE=CHANGE_PVE02_MIXED_GUEST_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET_TASKWAIT\nSELFTESTS=22\nAUTHORITY_FRESH_READ=OK\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GLOBAL_PRESTATE\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nPVE02_STRICT_SSH_STREAM_PREFLIGHT=PASS\nPVE02_INVENTORY=[{\"config_sha256\":\"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b\",\"guard\":{\"additional_mount_not_explicit_backup_one_count\":0,\"mountpoint_keys\":[],\"rootfs_present\":true},\"maxdisk_bytes\":8350298112,\"name\":\"dns2\",\"node\":\"pve02\",\"status\":\"running\",\"template\":false,\"type\":\"lxc\",\"vmid\":111},{\"config_sha256\":\"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1\",\"guard\":{\"additional_mount_not_explicit_backup_one_count\":0,\"mountpoint_keys\":[],\"rootfs_present\":true},\"maxdisk_bytes\":8350298112,\"name\":\"unbound2\",\"node\":\"pve02\",\"status\":\"running\",\"template\":false,\"type\":\"lxc\",\"vmid\":113},{\"config_sha256\":\"a3668bdf650c6e90f0dd9b684c272c16dc8243dfe4bbf90de0ae90cbe064d5a8\",\"guard\":{\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\"],\"unused_disk_count\":0},\"maxdisk_bytes\":214748364800,\"name\":\"forum-prod\",\"node\":\"pve02\",\"status\":\"running\",\"template\":false,\"type\":\"qemu\",\"vmid\":160},{\"config_sha256\":\"d59cd51387dce644c10925d11d3bb49bfea57eb9bc4bee9ebb1d69faae768295\",\"guard\":{\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\",\"scsi1\"],\"unused_disk_count\":0},\"maxdisk_bytes\":103079215104,\"name\":\"edge-cold-standby\",\"node\":\"pve02\",\"status\":\"stopped\",\"template\":false,\"type\":\"qemu\",\"vmid\":9130}]\nPVE02_GUEST_COUNT=4\nSTOPPED_GUEST_POLICY={\"policy\":\"INCLUDE_BACKUP_PRESERVE_STOPPED_STATE\",\"source_status\":\"stopped\",\"vmid\":9130}\nCRYPT_COUNT=13\nPHYSICAL_COUNT=13\nBACKUP_ORDER=[111,113,9130,160]\nGUEST_BACKUP_BEGIN=111\nGUEST_SOURCE={\"maxdisk_bytes\":8350298112,\"name\":\"dns2\",\"node\":\"pve02\",\"status\":\"running\",\"template\":false,\"type\":\"lxc\",\"vmid\":111}\nLXC_MOUNT_GUARD={\"additional_mount_not_explicit_backup_one_count\":0,\"mountpoint_keys\":[],\"rootfs_present\":true}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST111_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":8350298112,\"staging_available_bytes\":296977117184,\"staging_required_bytes\":26067038208,\"streaming_single_copy_fits\":true,\"vmid\":111}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606871526,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606871500,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":8891363328}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST111_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3\nLOGICAL_ID=guest-pve02-lxc-111\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3\nPAYLOAD_NAME=vzdump-lxc-111-2026_09_13-08_08_39.tar.zst\nPROXMOX_ARCHIVE_VERIFY=PASS\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606871526,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085606871500,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":929354534}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":929354534}\nREPLICA_1_CRYPT=pve03-mail-01-crypt\nREPLICA_1_PHYSICAL=pve03-mail-01\nREPLICA_1_FREE_BYTES_PREUPLOAD=1085606871526\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-02-crypt\nREPLICA_2_PHYSICAL=pve03-mail-02\nREPLICA_2_FREE_BYTES_PREUPLOAD=1085606871500\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nSOURCE_CONFIG_UNCHANGED=YES\nGUEST_BACKUP_COMPLETE=111\nGUEST_BACKUP_BEGIN=113\nGUEST_SOURCE={\"maxdisk_bytes\":8350298112,\"name\":\"unbound2\",\"node\":\"pve02\",\"status\":\"running\",\"template\":false,\"type\":\"lxc\",\"vmid\":113}\nLXC_MOUNT_GUARD={\"additional_mount_not_explicit_backup_one_count\":0,\"mountpoint_keys\":[],\"rootfs_present\":true}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST113_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":8350298112,\"staging_available_bytes\":296977117184,\"staging_required_bytes\":26067038208,\"streaming_single_copy_fits\":true,\"vmid\":113}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085218485999,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085218485973,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":8891363328}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":8891363328}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST113_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad\nLOGICAL_ID=guest-pve02-lxc-113\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad\nPAYLOAD_NAME=vzdump-lxc-113-2026_09_13-08_10_14.tar.zst\nPROXMOX_ARCHIVE_VERIFY=PASS\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085218485999,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1085218485973,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":762734583}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":762734583}\nREPLICA_1_CRYPT=pve03-mail-01-crypt\nREPLICA_1_PHYSICAL=pve03-mail-01\nREPLICA_1_FREE_BYTES_PREUPLOAD=1085218485999\nREPLICA_1_UPLOAD=PASS\nREPLICA_2_CRYPT=pve03-mail-02-crypt\nREPLICA_2_PHYSICAL=pve03-mail-02\nREPLICA_2_FREE_BYTES_PREUPLOAD=1085218485973\nREPLICA_2_UPLOAD=PASS\nREPLICA_1_READBACK=PASS\nREPLICA_2_READBACK=PASS\nREPLICA_1_SEALED_READBACK=PASS\nREPLICA_2_SEALED_READBACK=PASS\nSEALED_QUORUM_VERIFIED=YES\nLOCAL_PAYLOAD_FILES_REMAINING=0\nSOURCE_CONFIG_UNCHANGED=YES\nGUEST_BACKUP_COMPLETE=113\nGUEST_BACKUP_BEGIN=9130\nGUEST_SOURCE={\"maxdisk_bytes\":103079215104,\"name\":\"edge-cold-standby\",\"node\":\"pve02\",\"status\":\"stopped\",\"template\":false,\"type\":\"qemu\",\"vmid\":9130}\nQEMU_DISK_GUARD={\"backup_zero_count\":0,\"included_disk_keys\":[\"scsi0\",\"scsi1\"],\"unused_disk_count\":0}\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST9130_PRECAPACITY\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGUEST_ADMISSION={\"compression_credit_used\":false,\"maxdisk_bytes\":103079215104,\"staging_available_bytes\":296977117184,\"staging_required_bytes\":120795955200,\"streaming_single_copy_fits\":true,\"vmid\":9130}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433354013023,\"physical\":\"pve01-mail-01\",\"remote\":\"pve01-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":433353949164,\"physical\":\"pve01-mail-02\",\"remote\":\"pve01-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341735,\"physical\":\"pve01-mail-03\",\"remote\":\"pve01-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1014495341705,\"physical\":\"pve01-mail-04\",\"remote\":\"pve01-mail-04-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve01-mail-05\",\"remote\":\"pve01-mail-05-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413245,\"physical\":\"pve02-mail-01\",\"remote\":\"pve02-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":880030413267,\"physical\":\"pve02-mail-02\",\"remote\":\"pve02-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805279,\"physical\":\"pve02-mail-03\",\"remote\":\"pve02-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":892314805249,\"physical\":\"pve02-mail-04\",\"remote\":\"pve02-mail-04-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1084996761091,\"physical\":\"pve03-mail-01\",\"remote\":\"pve03-mail-01-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1084996761065,\"physical\":\"pve03-mail-02\",\"remote\":\"pve03-mail-02-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1008191778231,\"physical\":\"pve03-mail-03\",\"remote\":\"pve03-mail-03-crypt\",\"required_bytes\":103620280320}\nALLOCATOR_CANDIDATE={\"eligible\":true,\"free_bytes\":1075030281693,\"physical\":\"pve03-mail-04\",\"remote\":\"pve03-mail-04-crypt\",\"required_bytes\":103620280320}\nCLOUD_PRECAPACITY_ADMISSION=YES\nSOURCE_TASK_GUARD={\"clean_samples\":3,\"conflict_samples\":0,\"label\":\"GUEST9130_PRESTREAM\",\"max_conflicts\":0,\"node\":\"pve02\",\"samples\":3,\"status\":\"PASS\"}\nGENERATION=G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84\nLOGICAL_ID=guest-pve02-qemu-9130\nWORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84\nPAYLOAD_NAME=vzdump-qemu-9130-2026_09_13-08_11_26.vma.zst\nSTATUS=FAIL\nPHASE=GUEST9130_STREAM_VZDUMP\nREASON=SOURCE_GUEST_STATUS\nCOMPLETED_BEFORE_FAILURE=[{\"generation\":\"G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3\",\"logical_id\":\"guest-pve02-lxc-111\",\"manifest_sha256\":\"362b9abbe0f3d1105ee353768fb2e60fbc5a82222228d9b30b720a282315c37b\",\"name\":\"dns2\",\"payload_bytes\":388289318,\"payload_sha256\":\"649785b391b35f0ba7c17577ca51be1541dbf3e772480fac3b821f8a63680a4b\",\"replicas\":[\"pve03-mail-01-crypt\",\"pve03-mail-02-crypt\"],\"source_config_sha256\":\"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b\",\"source_status\":\"running\",\"type\":\"lxc\",\"vmid\":111},{\"generation\":\"G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad\",\"logical_id\":\"guest-pve02-lxc-113\",\"manifest_sha256\":\"21e60227e12baf7663384f4ba14ab980948ced2ded47a2a57386b20a54ed8233\",\"name\":\"unbound2\",\"payload_bytes\":221669367,\"payload_sha256\":\"c6205058bf56ef2ac320cf5774d9d29b3c3ec98b71087203c8df01c94ea659ff\",\"replicas\":[\"pve03-mail-01-crypt\",\"pve03-mail-02-crypt\"],\"source_config_sha256\":\"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1\",\"source_status\":\"running\",\"type\":\"lxc\",\"vmid\":113}]\nCURRENT_SOURCE_CONFIG_UNCHANGED=NO\nCURRENT_LOCAL_PAYLOAD_FILES_REMAINING=21\nFAILED_WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84\nFAILURE_DIAGNOSTIC_SUMMARY={\"cloud_write_attempts\":8,\"completed_vmids\":[111,113],\"current_type\":\"qemu\",\"current_vmid\":9130,\"failed_rc\":\"NONE\",\"failed_tool\":\"NONE\",\"phase\":\"GUEST9130_STREAM_VZDUMP\",\"reason\":\"SOURCE_GUEST_STATUS\"}\nGENERATION_DISCOVERY_REJECT_SUMMARY={\"not_applicable\":\"backup_command\",\"phase\":\"GUEST9130_STREAM_VZDUMP\"}\nFAILED_TOOL=NONE\nFAILED_RC=NONE\nFAILED_STDERR_BYTES=0\nFAILED_STDERR_SHA256=NONE\nCLOUD_CALLS=112\nCLOUD_WRITE_ATTEMPTS=8\nCLOUD_DELETIONS=0\nBACKUP_EXECUTIONS=3\nNODE_BUNDLE_EXECUTIONS=0\nRESTORE_EXECUTIONS=0\nSOURCE_GUEST_MUTATIONS=0\nSYSTEMD_MUTATIONS=0\nLIVE_RCLONE_CONFIG_READS=26\nLIVE_RCLONE_CONFIG_WRITES=0\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\nPVE303_END=\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index c147ee56..38c18f4d 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-RESTORE-PROOF-302R1 -STATUS=OK -RC=0 +COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1 +STATUS=FAIL +RC=2 HOST=pve01 MODE=change -COMPONENT=infra-dr-backupv2-pve03-sequential-restore-proof +COMPONENT=infra-dr-backupv2-pve02-sequential-backup-nodebundle-taskwait REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=d8885fb6a1bf2da2494b56b2e193edc054925b16877dffea2ebf90aadce179b5 +COMMAND_SHA256=5c9a4ef394ef9771f83f28c484bf58e1477fba3a658452a05c0c7322a9ba82f2 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=true RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=OK +RESULT_CONTRACT_STATUS=FAIL RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=0 +COMMAND_RC=2 CHANGES_MADE=true ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,116 +24,181 @@ MUTATION_OUTCOME=MUTATED SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=9571cf9070a804ea41f7ca7f900b60c52026dc5f44e8b0cc9e7e64bf703c1efa -SANITIZED_OUTPUT_SHA256=9571cf9070a804ea41f7ca7f900b60c52026dc5f44e8b0cc9e7e64bf703c1efa +RAW_EVIDENCE_SHA256=9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d +SANITIZED_OUTPUT_SHA256=9cc856bee6dd23fcbbfc48bf9b3acd4bd5eb94a184634ba3edbebf37af581c4d OUTPUT_BEGIN -PVE302_BEGIN= -COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-RESTORE-PROOF-302R1 -ANCHOR301_COMMIT=44008e8e152c13bc078564a46e23d92eb037bb62 -MODE=CHANGE_PVE03_CLOUD_ONLY_SEQUENTIAL_RESTORE_BOOT_NODEBUNDLE_NODESET_VERIFY -SELFTESTS=17 +PVE303_BEGIN= +COMMAND_ID=SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1 +ANCHOR302_COMMIT=6967f7b832bef65ee5877aeaf9ab7f17d0bdd2ca +RECONCILE298_COMMIT=63188e62e4c8b6ea2706f432b7f6d5b722529f5d +MODE=CHANGE_PVE02_MIXED_GUEST_SEQUENTIAL_STREAMING_BACKUP_NODEBUNDLE_NODESET_TASKWAIT +SELFTESTS=22 AUTHORITY_FRESH_READ=OK -SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":211} -SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":190} -SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":130} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GLOBAL_PRESTATE","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +PVE02_STRICT_SSH_STREAM_PREFLIGHT=PASS +PVE02_INVENTORY=[{"config_sha256":"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b","guard":{"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true},"maxdisk_bytes":8350298112,"name":"dns2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":111},{"config_sha256":"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1","guard":{"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true},"maxdisk_bytes":8350298112,"name":"unbound2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":113},{"config_sha256":"a3668bdf650c6e90f0dd9b684c272c16dc8243dfe4bbf90de0ae90cbe064d5a8","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0"],"unused_disk_count":0},"maxdisk_bytes":214748364800,"name":"forum-prod","node":"pve02","status":"running","template":false,"type":"qemu","vmid":160},{"config_sha256":"d59cd51387dce644c10925d11d3bb49bfea57eb9bc4bee9ebb1d69faae768295","guard":{"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0},"maxdisk_bytes":103079215104,"name":"edge-cold-standby","node":"pve02","status":"stopped","template":false,"type":"qemu","vmid":9130}] +PVE02_GUEST_COUNT=4 +STOPPED_GUEST_POLICY={"policy":"INCLUDE_BACKUP_PRESERVE_STOPPED_STATE","source_status":"stopped","vmid":9130} CRYPT_COUNT=13 PHYSICAL_COUNT=13 -NODE_BUNDLE_RECONSTRUCTION_VERIFIED=YES -NODE_BUNDLE_PVE_REFERENCE_VERIFIED=YES -NODE_BUNDLE_MEMBER_COUNT=226 -VM_RESTORE_BEGIN=211 -TEST_VMID=900311 -SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM211_SOURCE_PRESTATE","max_conflicts":0,"samples":3,"status":"PASS"} -SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":211} -GENERATION_DISCOVERY_SUMMARY={"generation":"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd","listed_replicas":2,"reject_counts":{"REMOTE_GENERATION_ABSENT":11},"valid_hits":2,"vmid":211} -GENERATION_DISCOVERED={"generation":"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd","manifest_sha256":"76dd08d6e858042eabd48b8dc6578d8ec6a335827a2d4c19e69031d22beda490","replica_count":2,"vmid":211} -RESTORE_ADMISSION={"local_lvm_available":751570296557,"maxdisk_bytes":53687091200,"payload_bytes":618301056,"staging_available":296977117184,"test_node":"pve01","vmid":211} -LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM211_LOCAL_PREP","max_conflicts":0,"samples":3,"status":"PASS"} -RESTORE_ARCHIVE_VERIFIED={"bytes":618301056,"generation":"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd","sha256":"e76393b03366751b461b130361923cf1f42f416886045eb29fd4212a737eac99","vmid":211} -LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM211_LOCAL_PRE_QMRESTORE","max_conflicts":0,"samples":3,"status":"PASS"} -TEST_VM_ISOLATION={"network_devices_remaining":0,"protection":1,"removed_keys":["net0","serial0"],"test_vmid":900311} -QEMU_GUEST_AGENT_PING=PASS -GUEST_OSINFO_QUERY=PASS -GUEST_HOSTNAME_QUERY=PASS -GUEST_RESTORE_BOOT_VERIFIED=YES -TEST_VM_AND_VOLUMES_REMOVED=YES -RESTORE_PAYLOAD_REMOVED=YES +BACKUP_ORDER=[111,113,9130,160] +GUEST_BACKUP_BEGIN=111 +GUEST_SOURCE={"maxdisk_bytes":8350298112,"name":"dns2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":111} +LXC_MOUNT_GUARD={"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST111_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":8350298112,"staging_available_bytes":296977117184,"staging_required_bytes":26067038208,"streaming_single_copy_fits":true,"vmid":111} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871526,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871500,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":8891363328} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST111_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GENERATION=G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3 +LOGICAL_ID=guest-pve02-lxc-111 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3 +PAYLOAD_NAME=vzdump-lxc-111-2026_09_13-08_08_39.tar.zst +PROXMOX_ARCHIVE_VERIFY=PASS +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871526,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085606871500,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":929354534} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":929354534} +REPLICA_1_CRYPT=pve03-mail-01-crypt +REPLICA_1_PHYSICAL=pve03-mail-01 +REPLICA_1_FREE_BYTES_PREUPLOAD=1085606871526 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-02-crypt +REPLICA_2_PHYSICAL=pve03-mail-02 +REPLICA_2_FREE_BYTES_PREUPLOAD=1085606871500 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 SOURCE_CONFIG_UNCHANGED=YES -VM_RESTORE_PROOF_COMPLETE=211 -VM_RESTORE_BEGIN=190 -TEST_VMID=900312 -SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM190_SOURCE_PRESTATE","max_conflicts":0,"samples":3,"status":"PASS"} -SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":190} -GENERATION_DISCOVERY_SUMMARY={"generation":"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360","listed_replicas":2,"reject_counts":{"REMOTE_GENERATION_ABSENT":11},"valid_hits":2,"vmid":190} -GENERATION_DISCOVERED={"generation":"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360","manifest_sha256":"6a954f3beb94e35b289c10717f5ac4387e8461df1a5f5e03db4329cade8b1fcc","replica_count":2,"vmid":190} -RESTORE_ADMISSION={"local_lvm_available":751048795253,"maxdisk_bytes":85899345920,"payload_bytes":1550687863,"staging_available":296977117184,"test_node":"pve01","vmid":190} -LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM190_LOCAL_PREP","max_conflicts":0,"samples":3,"status":"PASS"} -RESTORE_ARCHIVE_VERIFIED={"bytes":1550687863,"generation":"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360","sha256":"a3669dcc4de755ad650f363a4aa5578b00162bdcfe77e8bd37d60ffaa52ddb0c","vmid":190} -LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM190_LOCAL_PRE_QMRESTORE","max_conflicts":0,"samples":3,"status":"PASS"} -TEST_VM_ISOLATION={"network_devices_remaining":0,"protection":1,"removed_keys":["net0","serial0"],"test_vmid":900312} -QEMU_GUEST_AGENT_PING=PASS -GUEST_OSINFO_QUERY=PASS -GUEST_HOSTNAME_QUERY=PASS -GUEST_RESTORE_BOOT_VERIFIED=YES -TEST_VM_AND_VOLUMES_REMOVED=YES -RESTORE_PAYLOAD_REMOVED=YES +GUEST_BACKUP_COMPLETE=111 +GUEST_BACKUP_BEGIN=113 +GUEST_SOURCE={"maxdisk_bytes":8350298112,"name":"unbound2","node":"pve02","status":"running","template":false,"type":"lxc","vmid":113} +LXC_MOUNT_GUARD={"additional_mount_not_explicit_backup_one_count":0,"mountpoint_keys":[],"rootfs_present":true} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST113_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":8350298112,"staging_available_bytes":296977117184,"staging_required_bytes":26067038208,"streaming_single_copy_fits":true,"vmid":113} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485999,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485973,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":8891363328} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":8891363328} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST113_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GENERATION=G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad +LOGICAL_ID=guest-pve02-lxc-113 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad +PAYLOAD_NAME=vzdump-lxc-113-2026_09_13-08_10_14.tar.zst +PROXMOX_ARCHIVE_VERIFY=PASS +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485999,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1085218485973,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":762734583} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":762734583} +REPLICA_1_CRYPT=pve03-mail-01-crypt +REPLICA_1_PHYSICAL=pve03-mail-01 +REPLICA_1_FREE_BYTES_PREUPLOAD=1085218485999 +REPLICA_1_UPLOAD=PASS +REPLICA_2_CRYPT=pve03-mail-02-crypt +REPLICA_2_PHYSICAL=pve03-mail-02 +REPLICA_2_FREE_BYTES_PREUPLOAD=1085218485973 +REPLICA_2_UPLOAD=PASS +REPLICA_1_READBACK=PASS +REPLICA_2_READBACK=PASS +REPLICA_1_SEALED_READBACK=PASS +REPLICA_2_SEALED_READBACK=PASS +SEALED_QUORUM_VERIFIED=YES +LOCAL_PAYLOAD_FILES_REMAINING=0 SOURCE_CONFIG_UNCHANGED=YES -VM_RESTORE_PROOF_COMPLETE=190 -VM_RESTORE_BEGIN=130 -TEST_VMID=900313 -SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM130_SOURCE_PRESTATE","max_conflicts":0,"samples":3,"status":"PASS"} -SOURCE_QGA_PREFLIGHT={"node":"pve03","osinfo":"PASS","ping":"PASS","vmid":130} -GENERATION_DISCOVERY_SUMMARY={"generation":"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f","listed_replicas":2,"reject_counts":{"REMOTE_GENERATION_ABSENT":11},"valid_hits":2,"vmid":130} -GENERATION_DISCOVERED={"generation":"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f","manifest_sha256":"5f61523d3563096c5e78429c8a63bb347d559ec76de152d896bd2c263146c0ab","replica_count":2,"vmid":130} -RESTORE_ADMISSION={"local_lvm_available":751048795253,"maxdisk_bytes":103079215104,"payload_bytes":10849353784,"staging_available":296977117184,"test_node":"pve01","vmid":130} -LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM130_LOCAL_PREP","max_conflicts":0,"samples":3,"status":"PASS"} -RESTORE_ARCHIVE_VERIFIED={"bytes":10849353784,"generation":"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f","sha256":"4567b3c0322b530079bccec6789524ac0291024dbf50942bf3e550d67c70c1f9","vmid":130} -LOCAL_RESTORE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM130_LOCAL_PRE_QMRESTORE","max_conflicts":0,"samples":3,"status":"PASS"} -TEST_VM_ISOLATION={"network_devices_remaining":0,"protection":1,"removed_keys":["net0","serial0","startup"],"test_vmid":900313} -QEMU_GUEST_AGENT_PING=PASS -GUEST_OSINFO_QUERY=PASS -GUEST_HOSTNAME_QUERY=PASS -GUEST_RESTORE_BOOT_VERIFIED=YES -TEST_VM_AND_VOLUMES_REMOVED=YES -RESTORE_PAYLOAD_REMOVED=YES -SOURCE_CONFIG_UNCHANGED=YES -VM_RESTORE_PROOF_COMPLETE=130 -PVE03_NODESET_MANIFEST_VERIFIED=YES -PVE03_NODESET_RESTORE_COVERAGE_COMPLETE=YES -PVE03_SEQUENTIAL_RESTORE_PROOFS_COMPLETE=[211,190,130] -PVE03_NODE_BUNDLE_RESTORE_VALIDATED=YES -PVE03_NODESET_RESTORE_COVERAGE_COMPLETE=YES -COMPLETED_RESTORE_PROOFS=[{"generation":"G301-20260913T064540Z-2e6fd0fc524e4e4c39c7a29cd25577cd","manifest_sha256":"76dd08d6e858042eabd48b8dc6578d8ec6a335827a2d4c19e69031d22beda490","payload_sha256":"e76393b03366751b461b130361923cf1f42f416886045eb29fd4212a737eac99","test_vmid":900311,"vmid":211},{"generation":"G301-20260913T064740Z-ddfb0edd3f15d173167e680c54d32360","manifest_sha256":"6a954f3beb94e35b289c10717f5ac4387e8461df1a5f5e03db4329cade8b1fcc","payload_sha256":"a3669dcc4de755ad650f363a4aa5578b00162bdcfe77e8bd37d60ffaa52ddb0c","test_vmid":900312,"vmid":190},{"generation":"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f","manifest_sha256":"5f61523d3563096c5e78429c8a63bb347d559ec76de152d896bd2c263146c0ab","payload_sha256":"4567b3c0322b530079bccec6789524ac0291024dbf50942bf3e550d67c70c1f9","test_vmid":900313,"vmid":130}] -STATUS=OK -PHASE=COMPLETE -NEXT_GATE=PVE02_SEQUENTIAL_BACKUP_WITH_NODE_BUNDLE -CURRENT_SOURCE_VMID=NONE -CURRENT_TEST_VMID=NONE -CURRENT_WORKDIR=NONE -CURRENT_LOCAL_PAYLOAD_FILES_REMAINING=0 -LAST_GENERATION_DISCOVERY_SUMMARY={"generation":"G301-20260913T065051Z-eca2524f5cd97abfe630c87db08d733f","listed_replicas":2,"reject_counts":{"REMOTE_GENERATION_ABSENT":11},"valid_hits":2,"vmid":130} -LAST_SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM130_SOURCE_PRESTATE","max_conflicts":0,"samples":3,"status":"PASS"} -LAST_LOCAL_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"VM130_LOCAL_PRE_QMRESTORE","max_conflicts":0,"samples":3,"status":"PASS"} -NODE_BUNDLE_VERIFIED=YES -NODESET_VERIFIED=YES +GUEST_BACKUP_COMPLETE=113 +GUEST_BACKUP_BEGIN=9130 +GUEST_SOURCE={"maxdisk_bytes":103079215104,"name":"edge-cold-standby","node":"pve02","status":"stopped","template":false,"type":"qemu","vmid":9130} +QEMU_DISK_GUARD={"backup_zero_count":0,"included_disk_keys":["scsi0","scsi1"],"unused_disk_count":0} +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST9130_PRECAPACITY","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GUEST_ADMISSION={"compression_credit_used":false,"maxdisk_bytes":103079215104,"staging_available_bytes":296977117184,"staging_required_bytes":120795955200,"streaming_single_copy_fits":true,"vmid":9130} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433354013023,"physical":"pve01-mail-01","remote":"pve01-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":433353949164,"physical":"pve01-mail-02","remote":"pve01-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341735,"physical":"pve01-mail-03","remote":"pve01-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1014495341705,"physical":"pve01-mail-04","remote":"pve01-mail-04-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve01-mail-05","remote":"pve01-mail-05-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413245,"physical":"pve02-mail-01","remote":"pve02-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":880030413267,"physical":"pve02-mail-02","remote":"pve02-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805279,"physical":"pve02-mail-03","remote":"pve02-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":892314805249,"physical":"pve02-mail-04","remote":"pve02-mail-04-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1084996761091,"physical":"pve03-mail-01","remote":"pve03-mail-01-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1084996761065,"physical":"pve03-mail-02","remote":"pve03-mail-02-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1008191778231,"physical":"pve03-mail-03","remote":"pve03-mail-03-crypt","required_bytes":103620280320} +ALLOCATOR_CANDIDATE={"eligible":true,"free_bytes":1075030281693,"physical":"pve03-mail-04","remote":"pve03-mail-04-crypt","required_bytes":103620280320} +CLOUD_PRECAPACITY_ADMISSION=YES +SOURCE_TASK_GUARD={"clean_samples":3,"conflict_samples":0,"label":"GUEST9130_PRESTREAM","max_conflicts":0,"node":"pve02","samples":3,"status":"PASS"} +GENERATION=G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84 +LOGICAL_ID=guest-pve02-qemu-9130 +WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84 +PAYLOAD_NAME=vzdump-qemu-9130-2026_09_13-08_11_26.vma.zst +STATUS=FAIL +PHASE=GUEST9130_STREAM_VZDUMP +REASON=SOURCE_GUEST_STATUS +COMPLETED_BEFORE_FAILURE=[{"generation":"G303-20260913T080839Z-e51fce4368b87d4cb30c93b4c24b41b3","logical_id":"guest-pve02-lxc-111","manifest_sha256":"362b9abbe0f3d1105ee353768fb2e60fbc5a82222228d9b30b720a282315c37b","name":"dns2","payload_bytes":388289318,"payload_sha256":"649785b391b35f0ba7c17577ca51be1541dbf3e772480fac3b821f8a63680a4b","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"],"source_config_sha256":"c1a4698faa4f61fec8080e4203d129c09a819cb6e19cafae2d4d345bfbb7b65b","source_status":"running","type":"lxc","vmid":111},{"generation":"G303-20260913T081014Z-c2e53d284eb505aa3a70c00279e600ad","logical_id":"guest-pve02-lxc-113","manifest_sha256":"21e60227e12baf7663384f4ba14ab980948ced2ded47a2a57386b20a54ed8233","name":"unbound2","payload_bytes":221669367,"payload_sha256":"c6205058bf56ef2ac320cf5774d9d29b3c3ec98b71087203c8df01c94ea659ff","replicas":["pve03-mail-01-crypt","pve03-mail-02-crypt"],"source_config_sha256":"7a654250b5d359cc50462105688b4e6dd49ae8cd0f4161ab2b62c6740a620ac1","source_status":"running","type":"lxc","vmid":113}] +CURRENT_SOURCE_CONFIG_UNCHANGED=NO +CURRENT_LOCAL_PAYLOAD_FILES_REMAINING=21 +FAILED_WORK_DIRECTORY=/mnt/staging/backup-v2-cross-node/G303-20260913T081126Z-b4b4a6a370763710dd904447b3629e84 +FAILURE_DIAGNOSTIC_SUMMARY={"cloud_write_attempts":8,"completed_vmids":[111,113],"current_type":"qemu","current_vmid":9130,"failed_rc":"NONE","failed_tool":"NONE","phase":"GUEST9130_STREAM_VZDUMP","reason":"SOURCE_GUEST_STATUS"} +GENERATION_DISCOVERY_REJECT_SUMMARY={"not_applicable":"backup_command","phase":"GUEST9130_STREAM_VZDUMP"} FAILED_TOOL=NONE FAILED_RC=NONE FAILED_STDERR_BYTES=0 FAILED_STDERR_SHA256=NONE -CLOUD_CALLS=146 -CLOUD_WRITE_ATTEMPTS=0 +CLOUD_CALLS=112 +CLOUD_WRITE_ATTEMPTS=8 CLOUD_DELETIONS=0 -BACKUP_EXECUTIONS=0 +BACKUP_EXECUTIONS=3 NODE_BUNDLE_EXECUTIONS=0 -RESTORE_EXECUTIONS=3 +RESTORE_EXECUTIONS=0 SOURCE_GUEST_MUTATIONS=0 SYSTEMD_MUTATIONS=0 LIVE_RCLONE_CONFIG_READS=26 LIVE_RCLONE_CONFIG_WRITES=0 -ROLLBACK_STARTED=NO -ROLLBACK_RESTORED=NOT_APPLICABLE -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE03-SEQUENTIAL-RESTORE-PROOF-302R1","status":"OK","changes_made":true,"rollback_started":false,"rollback_restored":null} -PVE302_END= +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260913-INFRA-DR-BACKUPV2-PVE02-SEQUENTIAL-BACKUP-NODEBUNDLE-TASKWAIT-303R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null} +PVE303_END= OUTPUT_END CHAT_OUTPUT_END