Stage HIP device weight uploads through a bounded host buffer

include/engine/framework/core/backend.h

@@ -56,6 +56,8 @@ std::vector<ggml_backend_buffer_type_t> cpu_extra_buffer_types(ggml_backend_t ba
 bool buffer_type_supports_matmul_weight(
     ggml_backend_t backend, ggml_backend_buffer_type_t buffer_type, ggml_type type, int64_t ne0, int64_t ne1);
 BackendType backend_type(ggml_backend_t backend);
+// Whether `buffer` lives on a device owned by the HIP (ROCm) registry.
+bool is_hip_buffer(ggml_backend_buffer_t buffer);
 bool is_host_backend(ggml_backend_t backend);
 bool uses_host_graph_plan(BackendType type);
 bool uses_host_graph_plan(ggml_backend_t backend);

src/framework/assets/tensor_source.cpp

@@ -1,6 +1,7 @@
 #include "engine/framework/assets/tensor_source.h"
 #include "engine/framework/assets/weight_adapter.h"
 #include "engine/framework/assets/torch_bin.h"
+#include "engine/framework/core/backend.h"
 
 #include "engine/framework/io/binary.h"
 #include "engine/framework/io/filesystem.h"
@@ -275,7 +276,23 @@ void set_tensor_bytes(ggml_tensor * tensor, const void * data, size_t bytes, std
     if (bytes != ggml_nbytes(tensor)) {
         throw std::runtime_error("backend tensor byte size mismatch for " + std::string(name));
     }
-    ggml_backend_tensor_set(tensor, data, 0, bytes);
+    ggml_backend_buffer_t buffer = tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer;
+    if (buffer == nullptr || ggml_backend_buffer_is_host(buffer) || !core::is_hip_buffer(buffer)) {
+        ggml_backend_tensor_set(tensor, data, 0, bytes);
+        return;
+    }
+    // HIP device buffers are filled through a bounded host staging buffer instead of straight from
+    // the memory-mapped model file. On ROCm iGPUs without XNACK (Strix Halo, gfx1151) large direct
+    // copies leave the pageable source pages registered with the GPU, the file pages are never
+    // released, and the load crawls once they approach the memory limit.
+    constexpr size_t kUploadChunk = size_t{64} << 20;
+    std::vector<std::byte> staging(std::min(kUploadChunk, bytes));
+    const auto * source = static_cast<const std::byte *>(data);
+    for (size_t offset = 0; offset < bytes; offset += staging.size()) {
+        const size_t count = std::min(staging.size(), bytes - offset);
+        std::memcpy(staging.data(), source + offset, count);
+        ggml_backend_tensor_set(tensor, staging.data(), offset, count);
+    }
 }
 
 std::vector<float> decode_tensor_data_f32(std::string_view name, const TensorData & tensor) {

src/framework/core/backend.cpp

@@ -317,6 +317,15 @@ bool buffer_type_supports_matmul_weight(
     return supported;
 }
 
+bool is_hip_buffer(ggml_backend_buffer_t buffer) {
+    if (buffer == nullptr) return false;
+    ggml_backend_buffer_type_t buffer_type = ggml_backend_buffer_get_type(buffer);
+    ggml_backend_dev_t device = buffer_type != nullptr ? ggml_backend_buft_get_device(buffer_type) : nullptr;
+    if (device == nullptr) return false;
+    ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(device);
+    return reg != nullptr && reg_name_matches(BackendType::Hip, ggml_backend_reg_name(reg));
+}
+
 BackendType backend_type(ggml_backend_t backend) {
     if (is_host_backend(backend)) {
         return BackendType::Cpu;