diff --git a/.gitignore b/.gitignore index 9a8cb0a..beb6022 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,6 @@ libtool # Debug files *.dSYM/ *.su + +# Editor files +*.swp diff --git a/Makefile.am b/Makefile.am index a66dfda..cfe6099 100644 --- a/Makefile.am +++ b/Makefile.am @@ -24,7 +24,7 @@ libgdsyncinclude_HEADERS = include/gdsync/core.h include/gdsync/device.cuh incl src_libgdsync_la_CFLAGS = $(AM_CFLAGS) src_libgdsync_la_SOURCES = src/gdsync.cpp src/memmgr.cpp src/mem.cpp src/objs.cpp src/apis.cpp src/mlx5.cpp include/gdsync.h -src_libgdsync_la_LDFLAGS = -version-info @VERSION_INFO@ +src_libgdsync_la_LDFLAGS = -version-info @VERSION_INFO@ -lmlx5 noinst_HEADERS = src/mem.hpp src/memmgr.hpp src/objs.hpp src/rangeset.hpp src/utils.hpp src/archutils.h src/mlnxutils.h diff --git a/configure.ac b/configure.ac index 0f1c4fc..37f5e25 100644 --- a/configure.ac +++ b/configure.ac @@ -160,11 +160,6 @@ dnl Checks for Verbs support AC_CHECK_LIB(ibverbs, ibv_get_device_list, [], AC_MSG_ERROR([ibv_get_device_list() not found. libgdsync requires libibverbs.])) -AC_CHECK_LIB(ibverbs, ibv_exp_create_qp, - AC_MSG_ERROR([ibv_exp_create_qp not found. libgdsync requires verbs extension support.])) - -AC_CHECK_HEADER(infiniband/peer_ops.h, [], - AC_MSG_ERROR([ not found. libgdsync requires verbs peer-direct support.])) AC_HEADER_STDC dnl Checks for typedefs, structures, and compiler characteristics. diff --git a/include/gdsync.h b/include/gdsync.h index 7d6a45b..f2ed858 100644 --- a/include/gdsync.h +++ b/include/gdsync.h @@ -33,8 +33,6 @@ */ #include -#include -#include #include #include diff --git a/include/gdsync/core.h b/include/gdsync/core.h index 7ff0cbb..3e7681e 100644 --- a/include/gdsync/core.h +++ b/include/gdsync/core.h @@ -32,43 +32,64 @@ #error "don't include directly this header, use gdsync.h always" #endif -#define GDS_API_MAJOR_VERSION 2 -#define GDS_API_MINOR_VERSION 2 +#define GDS_API_MAJOR_VERSION 3 +#define GDS_API_MINOR_VERSION 0 #define GDS_API_VERSION (((unsigned)GDS_API_MAJOR_VERSION << 16) | (unsigned)GDS_API_MINOR_VERSION) #define GDS_API_VERSION_COMPATIBLE(v) \ ( ((((v) & 0xffff0000U) >> 16) == (unsigned)GDS_API_MAJOR_VERSION) && \ ((((v) & 0x0000ffffU) >> 0 ) >= (unsigned)GDS_API_MINOR_VERSION) ) +#include + typedef enum gds_param { - GDS_PARAM_VERSION, - GDS_NUM_PARAMS + GDS_PARAM_VERSION, + GDS_NUM_PARAMS } gds_param_t; int gds_query_param(gds_param_t param, int *value); enum gds_create_qp_flags { - GDS_CREATE_QP_DEFAULT = 0, - GDS_CREATE_QP_WQ_ON_GPU = 1<<0, - GDS_CREATE_QP_TX_CQ_ON_GPU = 1<<1, - GDS_CREATE_QP_RX_CQ_ON_GPU = 1<<2, - GDS_CREATE_QP_WQ_DBREC_ON_GPU = 1<<5, + GDS_CREATE_QP_DEFAULT = 0, + GDS_CREATE_QP_WQ_ON_GPU = 1<<0, + GDS_CREATE_QP_TX_CQ_ON_GPU = 1<<1, + GDS_CREATE_QP_RX_CQ_ON_GPU = 1<<2, + GDS_CREATE_QP_WQ_DBREC_ON_GPU = 1<<5, }; -typedef struct ibv_exp_qp_init_attr gds_qp_init_attr_t; -typedef struct ibv_exp_send_wr gds_send_wr; - -struct gds_cq { - struct ibv_cq *cq; - uint32_t curr_offset; -}; +typedef struct ibv_qp_init_attr gds_qp_init_attr_t; +typedef struct ibv_send_wr gds_send_wr; + +typedef enum gds_cq_type { + GDS_CQ_TYPE_SQ, + GDS_CQ_TYPE_RQ +} gds_cq_type_t; + +typedef enum gds_driver_type { + GDS_DRIVER_TYPE_UNKNOW = 0, + GDS_DRIVER_TYPE_MLX5 +} gds_driver_type_t; + +typedef struct gds_cq { + struct ibv_cq *ibcq; + uint32_t curr_offset; + gds_cq_type_t ctype; + gds_driver_type_t dtype; +} gds_cq_t; + +typedef struct gds_qp { + struct ibv_qp *ibqp; + gds_cq_t *send_cq; + gds_cq_t *recv_cq; + gds_driver_type_t dtype; +} gds_qp_t; + +/* \brief: Poll a peer-enabled CQ. + * + * It works similarly to ibv_poll_cq while blocking CPU access to the CQ + * entries until the GPU has a chance to observe them. + */ -struct gds_qp { - struct ibv_qp *qp; - struct gds_cq send_cq; - struct gds_cq recv_cq; - struct ibv_exp_res_domain * res_domain; - struct ibv_context *dev_context; -}; +int gds_poll_cq(gds_cq_t *cq, int ne, struct ibv_wc *wc); /* \brief: Create a peer-enabled QP attached to the specified GPU id. * @@ -80,11 +101,16 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, gds_qp_init_attr_t *qp_init_attr, int gpu_id, int flags); +/* \brief: Destroy a peer-enabled CQ + * + */ +void gds_destroy_cq(gds_cq_t *cq); + /* \brief: Destroy a peer-enabled QP * * The associated CQs are destroyed as well. */ -int gds_destroy_qp(struct gds_qp *qp); +void gds_destroy_qp(gds_qp_t *qp); /* \brief: CPU-synchronous post send for peer QPs * @@ -114,10 +140,10 @@ int gds_stream_queue_send(CUstream stream, struct gds_qp *qp, gds_send_wr *p_ewr // batched submission APIs typedef enum gds_memory_type { - GDS_MEMORY_GPU = 1, /*< use this flag for both cudaMalloc/cuMemAlloc and cudaMallocHost/cuMemHostAlloc */ + GDS_MEMORY_GPU = 1, /*< use this flag for both cudaMalloc/cuMemAlloc and cudaMallocHost/cuMemHostAlloc */ GDS_MEMORY_HOST = 2, GDS_MEMORY_IO = 4, - GDS_MEMORY_MASK = 0x7 + GDS_MEMORY_MASK = 0x7 } gds_memory_type_t; // Note: those flags below must not overlap with gds_memory_type_t @@ -148,27 +174,27 @@ enum { GDS_WAIT_INFO_MAX_OPS = 32 }; + /** * Represents a posted send operation on a particular QP */ typedef struct gds_send_request { - struct ibv_exp_peer_commit commit; - struct peer_op_wr wr[GDS_SEND_INFO_MAX_OPS]; + uint8_t reserve0[32]; + uint8_t reserve1[56 * GDS_SEND_INFO_MAX_OPS]; } gds_send_request_t; int gds_prepare_send(struct gds_qp *qp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr, gds_send_request_t *request); int gds_stream_post_send(CUstream stream, gds_send_request_t *request); int gds_stream_post_send_all(CUstream stream, int count, gds_send_request_t *request); - /** * Represents a wait operation on a particular CQ */ typedef struct gds_wait_request { - struct ibv_exp_peer_peek peek; - struct peer_op_wr wr[GDS_WAIT_INFO_MAX_OPS]; + uint8_t reserve0[40]; + uint8_t reserve1[56 * GDS_WAIT_INFO_MAX_OPS]; } gds_wait_request_t; /** diff --git a/include/gdsync/device.cuh b/include/gdsync/device.cuh index 25f5dfd..afbf062 100644 --- a/include/gdsync/device.cuh +++ b/include/gdsync/device.cuh @@ -33,129 +33,127 @@ namespace gdsync { - static const clock_t large_timeout = 1ULL<<32; - enum { - ERROR_TIMEOUT = 11, // same as EAGAIN - ERROR_INVALID = 22, // EINVAL - }; - - //typedef enum wait_cond { WAIT_GEQ, WAIT_EQ, WAIT_AND, WAIT_NOR } wait_cond_t; - typedef gds_wait_cond_flag_t wait_cond_t; - - struct sem32 { - typedef uint32_t T; - T sem; - T value; - - __host__ __device__ inline volatile T *access_once() { - return (volatile T *)&sem; - } - }; - typedef struct sem32 sem32_t; - - // indirect 32-bit semaphore - struct isem32 { - typedef uint32_t T; - typedef int32_t Tsigned; - T *ptr; - T value; - - __host__ __device__ inline volatile T *access_once() { - return (volatile T *)ptr; - } - __host__ __device__ isem32() : ptr(NULL), value(0) {} - }; - typedef struct isem32 isem32_t; - - struct isem64 { - typedef uint64_t T; - T *ptr; - T value; - - __host__ __device__ inline volatile T *access_once() { - return (volatile T *)ptr; - } - __host__ __device__ isem64() : ptr(NULL), value(0) {} - }; - typedef struct isem64 isem64_t; + static const clock_t large_timeout = 1ULL<<32; + enum { + ERROR_TIMEOUT = 11, // same as EAGAIN + ERROR_INVALID = 22, // EINVAL + }; + + typedef gds_wait_cond_flag_t wait_cond_t; + + struct sem32 { + typedef uint32_t T; + T sem; + T value; + + __host__ __device__ inline volatile T *access_once() { + return (volatile T *)&sem; + } + }; + typedef struct sem32 sem32_t; + + // indirect 32-bit semaphore + struct isem32 { + typedef uint32_t T; + typedef int32_t Tsigned; + T *ptr; + T value; + + __host__ __device__ inline volatile T *access_once() { + return (volatile T *)ptr; + } + __host__ __device__ isem32() : ptr(NULL), value(0) {} + }; + typedef struct isem32 isem32_t; + + struct isem64 { + typedef uint64_t T; + T *ptr; + T value; + + __host__ __device__ inline volatile T *access_once() { + return (volatile T *)ptr; + } + __host__ __device__ isem64() : ptr(NULL), value(0) {} + }; + typedef struct isem64 isem64_t; #if defined(__CUDACC__) - namespace device { - - // NOTE: fences must be added by caller - template __device__ inline void release(S &sem) { - //printf("[%d:%d] release %p=%08x\n", blockIdx.x, threadIdx.x, sem.access_once(), sem.value); - assert(0 != sem.access_once()); - *sem.access_once() = sem.value; - } - - template __device__ inline int wait(S &sem, wait_cond_t cond) { - int ret = 0; - switch(cond) { - case GDS_WAIT_COND_EQ: ret = wait_eq(sem); break; - case GDS_WAIT_COND_GEQ: ret = wait_geq(sem); break; - case GDS_WAIT_COND_AND: ret = wait_and(sem); break; - case GDS_WAIT_COND_NOR: ret = wait_nor(sem); break; - default: ret = ERROR_INVALID; break; - } - return ret; - } - - template __device__ inline int wait_eq(S &sem) { - int ret = ERROR_TIMEOUT; - volatile clock_t tmout = clock() + large_timeout; - do { - if (*sem.access_once() == sem.value) { - ret = 0; - break; + namespace device { + + // NOTE: fences must be added by caller + template __device__ inline void release(S &sem) { + assert(0 != sem.access_once()); + *sem.access_once() = sem.value; } - __threadfence_block(); - } while(clock() < tmout); - return ret; - } - - template __device__ inline int wait_geq(S &sem) { - int ret = ERROR_TIMEOUT; - volatile clock_t tmout = clock() + large_timeout; - do { - //printf("ptr=%p\n", sem.access_once()); - typedef typename S::Tsigned Ts; - if ((Ts)*sem.access_once() - (Ts)sem.value >= 0) { - ret = 0; - break; + + template __device__ inline int wait_eq(S &sem) { + int ret = ERROR_TIMEOUT; + volatile clock_t tmout = clock() + large_timeout; + do { + if (*sem.access_once() == sem.value) { + ret = 0; + break; + } + __threadfence_block(); + } while(clock() < tmout); + return ret; + } + + template __device__ inline int wait_geq(S &sem) { + int ret = ERROR_TIMEOUT; + volatile clock_t tmout = clock() + large_timeout; + do { + typedef typename S::Tsigned Ts; + if ((Ts)*sem.access_once() - (Ts)sem.value >= 0) { + ret = 0; + break; + } + __threadfence_block(); + } while(clock() < tmout); + return ret; + } + + template __device__ static inline int wait_and(S &sem) { + int ret = ERROR_TIMEOUT; + volatile clock_t tmout = clock() + large_timeout; + do { + if (*sem.access_once() & sem.value) { + ret = 0; + break; + } + __threadfence_block(); + } while(clock() < tmout); + return ret; } - __threadfence_block(); - } while(clock() < tmout); - return ret; - } - - template __device__ static inline int wait_and(S &sem) { - int ret = ERROR_TIMEOUT; - volatile clock_t tmout = clock() + large_timeout; - do { - if (*sem.access_once() & sem.value) { - ret = 0; - break; + + template __device__ static inline int wait_nor(S &sem) { + int ret = ERROR_TIMEOUT; + volatile clock_t tmout = clock() + large_timeout; + do { + if (0 != ~(*sem.access_once() | sem.value)) { + ret = 0; + break; + } + __threadfence_block(); + } while(clock() < tmout); + return ret; } - __threadfence_block(); - } while(clock() < tmout); - return ret; - } - - template __device__ static inline int wait_nor(S &sem) { - int ret = ERROR_TIMEOUT; - volatile clock_t tmout = clock() + large_timeout; - do { - if (0 != ~(*sem.access_once() | sem.value)) { - ret = 0; - break; + + template __device__ inline int wait(S &sem, wait_cond_t cond) { + int ret = 0; + switch(cond) { + case GDS_WAIT_COND_EQ: ret = wait_eq(sem); break; + case GDS_WAIT_COND_GEQ: ret = wait_geq(sem); break; + case GDS_WAIT_COND_AND: ret = wait_and(sem); break; + case GDS_WAIT_COND_NOR: ret = wait_nor(sem); break; + default: ret = ERROR_INVALID; break; + } + return ret; } - __threadfence_block(); - } while(clock() < tmout); - return ret; - } - } // namespace device + + } // namespace device #endif } // namespace gdsync diff --git a/include/gdsync/mlx5.h b/include/gdsync/mlx5.h index ccd4249..df5a6b1 100644 --- a/include/gdsync/mlx5.h +++ b/include/gdsync/mlx5.h @@ -52,7 +52,7 @@ typedef struct gds_mlx5_send_info { int gds_mlx5_get_send_info(int count, const gds_send_request_t *requests, gds_mlx5_send_info_t *mlx5_infos); typedef struct gds_mlx5_wait_info { - gds_wait_cond_flag_t cond; + gds_wait_cond_flag_t cond; uint32_t *cqe_ptr; uint32_t cqe_value; uint32_t *flag_ptr; diff --git a/include/gdsync/tools.h b/include/gdsync/tools.h index bc3b13d..871d202 100644 --- a/include/gdsync/tools.h +++ b/include/gdsync/tools.h @@ -36,12 +36,13 @@ GDS_BEGIN_DECLS typedef struct gds_mem_desc { - CUdeviceptr d_ptr; - void *h_ptr; - void *bar_ptr; - int flags; - size_t alloc_size; - gdr_mh_t mh; + CUdeviceptr original_d_ptr; + CUdeviceptr d_ptr; + void *h_ptr; + void *bar_ptr; + int flags; + size_t alloc_size; + gdr_mh_t mh; } gds_mem_desc_t; int gds_alloc_mapped_memory(gds_mem_desc_t *desc, size_t size, int flags); int gds_free_mapped_memory(gds_mem_desc_t *desc); diff --git a/src/apis.cpp b/src/apis.cpp index cd532d7..743a7c4 100644 --- a/src/apis.cpp +++ b/src/apis.cpp @@ -33,21 +33,13 @@ #include #include #include - -//#include -//#include -//#include -//using namespace std; - -//#include -//#include -//#include +#include #include "gdsync.h" #include "gdsync/tools.h" -#include "objs.hpp" -#include "utils.hpp" #include "memmgr.hpp" +#include "mlx5.hpp" +#include "objs.hpp" #include "utils.hpp" #include "archutils.h" #include "mlnxutils.h" @@ -55,7 +47,7 @@ //----------------------------------------------------------------------------- -static void gds_init_ops(struct peer_op_wr *op, int count) +static void gds_init_ops(struct gds_mlx5_peer_op_wr *op, int count) { int i = count; while (--i) @@ -65,46 +57,23 @@ static void gds_init_ops(struct peer_op_wr *op, int count) //----------------------------------------------------------------------------- -static void gds_init_send_info(gds_send_request_t *info) +static int gds_rollback_qp(struct gds_qp *gqp, gds_send_request_t *p_sreq, enum gds_mlx5_rollback_flags flag) { - gds_dbg("send_request=%p\n", info); - memset(info, 0, sizeof(*info)); - - info->commit.storage = info->wr; - info->commit.entries = sizeof(info->wr)/sizeof(info->wr[0]); - gds_init_ops(info->commit.storage, info->commit.entries); -} - -//----------------------------------------------------------------------------- - -static void gds_init_wait_request(gds_wait_request_t *request, uint32_t offset) -{ - gds_dbg("wait_request=%p offset=%08x\n", request, offset); - memset(request, 0, sizeof(*request)); - request->peek.storage = request->wr; - request->peek.entries = sizeof(request->wr)/sizeof(request->wr[0]); - request->peek.whence = IBV_EXP_PEER_PEEK_ABSOLUTE; - request->peek.offset = offset; - gds_init_ops(request->peek.storage, request->peek.entries); -} + struct gds_mlx5_rollback_ctx rollback; + gds_mlx5_send_request_t *send_info; + int ret = 0; -//----------------------------------------------------------------------------- + assert(gqp); + assert(gqp->ibqp); + assert(p_sreq); -static int gds_rollback_qp(struct gds_qp *qp, gds_send_request_t * send_info, enum ibv_exp_rollback_flags flag) -{ - struct ibv_exp_rollback_ctx rollback; - int ret=0; - - assert(qp); - assert(qp->qp); - assert(send_info); - if( - flag != IBV_EXP_ROLLBACK_ABORT_UNCOMMITED && - flag != IBV_EXP_ROLLBACK_ABORT_LATE - ) - { - gds_err("erroneous ibv_exp_rollback_flags flag input value\n"); - ret=EINVAL; + send_info = to_gds_msreq(p_sreq); + if ( + flag != GDS_MLX5_ROLLBACK_ABORT_UNCOMMITED && + flag != GDS_MLX5_ROLLBACK_ABORT_LATE + ) { + gds_err("erroneous gds_mlx5_rollback_flags flag input value\n"); + ret = EINVAL; goto out; } @@ -115,7 +84,7 @@ static int gds_rollback_qp(struct gds_qp *qp, gds_send_request_t * send_info, en /* Reserved for future expensions, must be 0 */ rollback.comp_mask = 0; gds_warn("Need to rollback WQE %lx\n", rollback.rollback_id); - ret = ibv_exp_rollback_qp(qp->qp, &rollback); + ret = gds_mlx5_rollback_send(to_gds_mqp(gqp), &rollback); if(ret) gds_err("error %d in ibv_exp_rollback_qp\n", ret); @@ -125,11 +94,36 @@ static int gds_rollback_qp(struct gds_qp *qp, gds_send_request_t * send_info, en //----------------------------------------------------------------------------- -int gds_post_send(struct gds_qp *qp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr) +static void gds_init_send_info(gds_mlx5_send_request_t *info) +{ + gds_dbg("send_request=%p\n", info); + memset(info, 0, sizeof(*info)); + + info->commit.storage = info->wr; + info->commit.entries = sizeof(info->wr)/sizeof(info->wr[0]); + gds_init_ops(info->commit.storage, info->commit.entries); +} + +//----------------------------------------------------------------------------- + +static void gds_init_wait_request(gds_mlx5_wait_request_t *request, uint32_t offset) +{ + gds_dbg("wait_request=%p offset=%08x\n", request, offset); + memset(request, 0, sizeof(*request)); + request->peek.storage = request->wr; + request->peek.entries = sizeof(request->wr)/sizeof(request->wr[0]); + request->peek.whence = GDS_MLX5_PEER_PEEK_ABSOLUTE; + request->peek.offset = offset; + gds_init_ops(request->peek.storage, request->peek.entries); +} + +//----------------------------------------------------------------------------- + +int gds_post_send(struct gds_qp *gqp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr) { int ret = 0, ret_roll=0; gds_send_request_t send_info; - ret = gds_prepare_send(qp, p_ewr, bad_ewr, &send_info); + ret = gds_prepare_send(gqp, p_ewr, bad_ewr, &send_info); if (ret) { gds_err("error %d in gds_prepare_send\n", ret); goto out; @@ -138,11 +132,9 @@ int gds_post_send(struct gds_qp *qp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr) ret = gds_post_pokes_on_cpu(1, &send_info, NULL, 0); if (ret) { gds_err("error %d in gds_post_pokes_on_cpu\n", ret); - ret_roll = gds_rollback_qp(qp, &send_info, IBV_EXP_ROLLBACK_ABORT_LATE); - if (ret_roll) { + ret_roll = gds_rollback_qp(gqp, &send_info, GDS_MLX5_ROLLBACK_ABORT_LATE); + if (ret_roll) gds_err("error %d in gds_rollback_qp\n", ret_roll); - } - goto out; } @@ -152,14 +144,14 @@ int gds_post_send(struct gds_qp *qp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr) //----------------------------------------------------------------------------- -int gds_post_recv(struct gds_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr) +int gds_post_recv(struct gds_qp *gqp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr) { int ret = 0; - gds_dbg("qp=%p wr=%p\n", qp, wr); - assert(qp); - assert(qp->qp); - ret = ibv_post_recv(qp->qp, wr, bad_wr); + gds_dbg("gqp=%p wr=%p\n", gqp, wr); + assert(gqp); + assert(gqp->ibqp); + ret = ibv_post_recv(gqp->ibqp, wr, bad_wr); if (ret) { gds_err("error %d in ibv_post_recv\n", ret); goto out; @@ -171,48 +163,38 @@ int gds_post_recv(struct gds_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr //----------------------------------------------------------------------------- -int gds_prepare_send(struct gds_qp *qp, gds_send_wr *p_ewr, - gds_send_wr **bad_ewr, - gds_send_request_t *request) +int gds_prepare_send(struct gds_qp *gqp, gds_send_wr *p_ewr, + gds_send_wr **bad_ewr, + gds_send_request_t *p_sreq) { + gds_mlx5_send_request_t *request; int ret = 0; + + assert(p_sreq); + request = to_gds_msreq(p_sreq); + gds_init_send_info(request); - assert(qp); - assert(qp->qp); - ret = ibv_exp_post_send(qp->qp, p_ewr, bad_ewr); - if (ret) { + assert(gqp); + assert(gqp->ibqp); + assert(request->commit.entries >= 3); + + ret = gds_mlx5_post_send(to_gds_mqp(gqp), p_ewr, bad_ewr, &request->commit); - if (ret == ENOMEM) { - // out of space error can happen too often to report - gds_dbg("ENOMEM error %d in ibv_exp_post_send\n", ret); - } else { - gds_err("error %d in ibv_exp_post_send\n", ret); - } - goto out; - } - - ret = ibv_exp_peer_commit_qp(qp->qp, &request->commit); - if (ret) { - gds_err("error %d in ibv_exp_peer_commit_qp\n", ret); - //gds_wait_kernel(); - goto out; - } -out: return ret; } //----------------------------------------------------------------------------- -int gds_stream_queue_send(CUstream stream, struct gds_qp *qp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr) +int gds_stream_queue_send(CUstream stream, struct gds_qp *gqp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr) { int ret = 0, ret_roll = 0; gds_send_request_t send_info; gds_descriptor_t descs[1]; - assert(qp); + assert(gqp); assert(p_ewr); - ret = gds_prepare_send(qp, p_ewr, bad_ewr, &send_info); + ret = gds_prepare_send(gqp, p_ewr, bad_ewr, &send_info); if (ret) { gds_err("error %d in gds_prepare_send\n", ret); goto out; @@ -221,13 +203,13 @@ int gds_stream_queue_send(CUstream stream, struct gds_qp *qp, gds_send_wr *p_ewr descs[0].tag = GDS_TAG_SEND; descs[0].send = &send_info; - ret=gds_stream_post_descriptors(stream, 1, descs, 0); + ret = gds_stream_post_descriptors(stream, 1, descs, 0); if (ret) { gds_err("error %d in gds_stream_post_descriptors\n", ret); goto out; } - out: +out: return ret; } @@ -254,7 +236,7 @@ int gds_stream_post_send_all(CUstream stream, int count, gds_send_request_t *req assert(count); descs = (gds_descriptor_t *) calloc(count, sizeof(gds_descriptor_t)); - if(!descs) + if (!descs) { gds_err("Calloc for %d elements\n", count); ret=ENOMEM; @@ -272,67 +254,32 @@ int gds_stream_post_send_all(CUstream stream, int count, gds_send_request_t *req goto out; } - out: - if(descs) free(descs); - return ret; +out: + if(descs) free(descs); + return ret; } //----------------------------------------------------------------------------- -int gds_prepare_wait_cq(struct gds_cq *cq, gds_wait_request_t *request, int flags) +int gds_prepare_wait_cq(struct gds_cq *cq, gds_wait_request_t *p_wreq, int flags) { - int retcode = 0; - if (flags != 0) { - gds_err("invalid flags != 0\n"); - return EINVAL; - } + int ret = 0; + gds_mlx5_wait_request_t *request; - gds_init_wait_request(request, cq->curr_offset++); + assert(p_wreq); + request = to_gds_mwreq(p_wreq); - retcode = ibv_exp_peer_peek_cq(cq->cq, &request->peek); - if (retcode == -ENOSPC) { - // TODO: handle too few entries - gds_err("not enough ops in peer_peek_cq\n"); - goto out; - } else if (retcode) { - gds_err("error %d in peer_peek_cq\n", retcode); + if (flags != 0) { + gds_err("invalid flags != 0\n"); + ret = EINVAL; goto out; } - //gds_dump_wait_request(request, 1); - out: - return retcode; -} - -//----------------------------------------------------------------------------- -int gds_append_wait_cq(gds_wait_request_t *request, uint32_t *dw, uint32_t val) -{ - int ret = 0; - unsigned MAX_NUM_ENTRIES = sizeof(request->wr)/sizeof(request->wr[0]); - unsigned n = request->peek.entries; - struct peer_op_wr *wr = request->peek.storage; - - if (n + 1 > MAX_NUM_ENTRIES) { - gds_err("no space left to stuff a poke\n"); - ret = ENOMEM; - goto out; - } - - // at least 1 op - assert(n); - assert(wr); - - for (; n; --n) wr = wr->next; - assert(wr); - - wr->type = IBV_EXP_PEER_OP_STORE_DWORD; - wr->wr.dword_va.data = val; - wr->wr.dword_va.target_id = 0; // direct mapping, offset IS the address - wr->wr.dword_va.offset = (ptrdiff_t)(dw-(uint32_t*)0); + gds_init_wait_request(request, cq->curr_offset++); - ++request->peek.entries; + gds_mlx5_peer_peek_cq(to_gds_mcq(cq), &request->peek); - out: +out: return ret; } @@ -340,26 +287,24 @@ int gds_append_wait_cq(gds_wait_request_t *request, uint32_t *dw, uint32_t val) int gds_stream_post_wait_cq(CUstream stream, gds_wait_request_t *request) { - return gds_stream_post_wait_cq_multi(stream, 1, request, NULL, 0); + return gds_stream_post_wait_cq_multi(stream, 1, request, NULL, 0); } //----------------------------------------------------------------------------- int gds_stream_post_wait_cq_all(CUstream stream, int count, gds_wait_request_t *requests) { - return gds_stream_post_wait_cq_multi(stream, count, requests, NULL, 0); + return gds_stream_post_wait_cq_multi(stream, count, requests, NULL, 0); } //----------------------------------------------------------------------------- -static int gds_abort_wait_cq(struct gds_cq *cq, gds_wait_request_t *request) +static int gds_abort_wait_cq(struct gds_cq *cq, gds_mlx5_wait_request_t *request) { assert(cq); assert(request); - struct ibv_exp_peer_abort_peek abort_ctx; - abort_ctx.peek_id = request->peek.peek_id; - abort_ctx.comp_mask = 0; - return ibv_exp_peer_abort_peek_cq(cq->cq, &abort_ctx); + ((struct gds_mlx5_peek_entry *)request->peek.peek_id)->busy = 0; + return 0; } //----------------------------------------------------------------------------- @@ -387,23 +332,26 @@ int gds_stream_wait_cq(CUstream stream, struct gds_cq *cq, int flags) ret = gds_stream_post_wait_cq(stream, &request); if (ret) { gds_err("error %d in gds_stream_post_wait_cq_ex\n", ret); - int retcode2 = gds_abort_wait_cq(cq, &request); - if (retcode2) { + int retcode2 = gds_abort_wait_cq(cq, to_gds_mwreq(&request)); + if (retcode2) gds_err("nested error %d while aborting request\n", retcode2); - } retcode = ret; goto out; } out: - return retcode; + return retcode; } //----------------------------------------------------------------------------- -int gds_post_wait_cq(struct gds_cq *cq, gds_wait_request_t *request, int flags) +int gds_post_wait_cq(struct gds_cq *cq, gds_wait_request_t *p_wreq, int flags) { int retcode = 0; + gds_mlx5_wait_request_t *request; + + assert(p_wreq); + request = to_gds_mwreq(p_wreq); if (flags) { retcode = EINVAL; @@ -423,7 +371,7 @@ int gds_prepare_wait_value32(gds_wait_value32_t *desc, uint32_t *ptr, uint32_t v assert(desc); gds_dbg("desc=%p ptr=%p value=0x%08x cond_flags=0x%x flags=0x%x\n", - desc, ptr, value, cond_flags, flags); + desc, ptr, value, cond_flags, flags); if (flags & ~(GDS_WAIT_POST_FLUSH_REMOTE|GDS_MEMORY_MASK)) { gds_err("invalid flags\n"); @@ -504,17 +452,17 @@ static bool no_network_descs_after_entry(size_t n_descs, gds_descriptor_t *descs for(i = idx+1; i < n_descs; ++i) { gds_descriptor_t *desc = descs + i; switch(desc->tag) { - case GDS_TAG_SEND: - case GDS_TAG_WAIT: - ret = false; - goto out; - case GDS_TAG_WAIT_VALUE32: - case GDS_TAG_WRITE_VALUE32: - break; - default: - gds_err("invalid tag\n"); - ret = EINVAL; - goto out; + case GDS_TAG_SEND: + case GDS_TAG_WAIT: + ret = false; + goto out; + case GDS_TAG_WAIT_VALUE32: + case GDS_TAG_WRITE_VALUE32: + break; + default: + gds_err("invalid tag\n"); + ret = EINVAL; + goto out; } } out: @@ -528,18 +476,18 @@ static int get_wait_info(size_t n_descs, gds_descriptor_t *descs, size_t &n_wait for(i = 0; i < n_descs; ++i) { gds_descriptor_t *desc = descs + i; switch(desc->tag) { - case GDS_TAG_WAIT: - ++n_waits; - last_wait = i; - break; - case GDS_TAG_SEND: - case GDS_TAG_WAIT_VALUE32: - case GDS_TAG_WRITE_VALUE32: - case GDS_TAG_WRITE_MEMORY: - break; - default: - gds_err("invalid tag\n"); - ret = EINVAL; + case GDS_TAG_WAIT: + ++n_waits; + last_wait = i; + break; + case GDS_TAG_SEND: + case GDS_TAG_WAIT_VALUE32: + case GDS_TAG_WRITE_VALUE32: + case GDS_TAG_WRITE_MEMORY: + break; + default: + gds_err("invalid tag\n"); + ret = EINVAL; } } return ret; @@ -553,20 +501,20 @@ static int calc_n_mem_ops(size_t n_descs, gds_descriptor_t *descs, size_t &n_mem for(i = 0; i < n_descs; ++i) { gds_descriptor_t *desc = descs + i; switch(desc->tag) { - case GDS_TAG_SEND: - n_mem_ops += desc->send->commit.entries + 2; // extra space, ugly - break; - case GDS_TAG_WAIT: - n_mem_ops += desc->wait->peek.entries + 2; // ditto - break; - case GDS_TAG_WAIT_VALUE32: - case GDS_TAG_WRITE_VALUE32: - case GDS_TAG_WRITE_MEMORY: - n_mem_ops += 2; // ditto - break; - default: - gds_err("invalid tag\n"); - ret = EINVAL; + case GDS_TAG_SEND: + n_mem_ops += to_gds_msreq(desc->send)->commit.entries + 2; // extra space, ugly + break; + case GDS_TAG_WAIT: + n_mem_ops += to_gds_mwreq(desc->wait)->peek.entries + 2; // ditto + break; + case GDS_TAG_WAIT_VALUE32: + case GDS_TAG_WRITE_VALUE32: + case GDS_TAG_WRITE_MEMORY: + n_mem_ops += 2; // ditto + break; + default: + gds_err("invalid tag\n"); + ret = EINVAL; } } return ret; @@ -615,60 +563,60 @@ int gds_stream_post_descriptors(CUstream stream, size_t n_descs, gds_descriptor_ for(i = 0; i < n_descs; ++i) { gds_descriptor_t *desc = descs + i; switch(desc->tag) { - case GDS_TAG_SEND: { - gds_send_request_t *sreq = desc->send; - retcode = gds_post_ops(peer, sreq->commit.entries, sreq->commit.storage, params); - if (retcode) { - gds_err("error %d in gds_post_ops\n", retcode); - ret = retcode; - goto out; - } - break; - } - case GDS_TAG_WAIT: { - gds_wait_request_t *wreq = desc->wait; - int flags = 0; - if (move_flush && i != last_wait) { - gds_dbg("discarding FLUSH!\n"); - flags = GDS_POST_OPS_DISCARD_WAIT_FLUSH; - } - retcode = gds_post_ops(peer, wreq->peek.entries, wreq->peek.storage, params, flags); - if (retcode) { - gds_err("error %d in gds_post_ops\n", retcode); - ret = retcode; - goto out; - } - break; - } - case GDS_TAG_WAIT_VALUE32: - retcode = gds_fill_poll(peer, params, desc->wait32.ptr, desc->wait32.value, desc->wait32.cond_flags, desc->wait32.flags); - if (retcode) { - gds_err("error %d in gds_fill_poll\n", retcode); - ret = retcode; - goto out; + case GDS_TAG_SEND: { + gds_mlx5_send_request_t *sreq = to_gds_msreq(desc->send); + retcode = gds_mlx5_post_ops(peer, sreq->commit.entries, sreq->commit.storage, params); + if (retcode) { + gds_err("error %d in gds_mlx5_post_ops\n", retcode); + ret = retcode; + goto out; + } + break; } - break; - case GDS_TAG_WRITE_VALUE32: - retcode = gds_fill_poke(peer, params, desc->write32.ptr, desc->write32.value, desc->write32.flags); - if (retcode) { - gds_err("error %d in gds_fill_poke\n", retcode); - ret = retcode; - goto out; + case GDS_TAG_WAIT: { + gds_mlx5_wait_request_t *wreq = to_gds_mwreq(desc->wait); + int flags = 0; + if (move_flush && i != last_wait) { + gds_dbg("discarding FLUSH!\n"); + flags = GDS_POST_OPS_DISCARD_WAIT_FLUSH; + } + retcode = gds_mlx5_post_ops(peer, wreq->peek.entries, wreq->peek.storage, params, flags); + if (retcode) { + gds_err("error %d in gds_mlx5_post_ops\n", retcode); + ret = retcode; + goto out; + } + break; } - break; - case GDS_TAG_WRITE_MEMORY: - retcode = gds_fill_inlcpy(peer, params, desc->writemem.dest, desc->writemem.src, desc->writemem.count, desc->writemem.flags); - if (retcode) { - gds_err("error %d in gds_fill_inlcpy\n", retcode); - ret = retcode; + case GDS_TAG_WAIT_VALUE32: + retcode = gds_fill_poll(peer, params, desc->wait32.ptr, desc->wait32.value, desc->wait32.cond_flags, desc->wait32.flags); + if (retcode) { + gds_err("error %d in gds_fill_poll\n", retcode); + ret = retcode; + goto out; + } + break; + case GDS_TAG_WRITE_VALUE32: + retcode = gds_fill_poke(peer, params, desc->write32.ptr, desc->write32.value, desc->write32.flags); + if (retcode) { + gds_err("error %d in gds_fill_poke\n", retcode); + ret = retcode; + goto out; + } + break; + case GDS_TAG_WRITE_MEMORY: + retcode = gds_fill_inlcpy(peer, params, desc->writemem.dest, desc->writemem.src, desc->writemem.count, desc->writemem.flags); + if (retcode) { + gds_err("error %d in gds_fill_inlcpy\n", retcode); + ret = retcode; + goto out; + } + break; + default: + gds_err("invalid tag for %zu entry\n", i); + ret = EINVAL; goto out; - } - break; - default: - gds_err("invalid tag for %zu entry\n", i); - ret = EINVAL; - goto out; - break; + break; } } retcode = gds_stream_batch_ops(peer, stream, params, 0); @@ -692,131 +640,131 @@ int gds_post_descriptors(size_t n_descs, gds_descriptor_t *descs, int flags) for(i = 0; i < n_descs; ++i) { gds_descriptor_t *desc = descs + i; switch(desc->tag) { - case GDS_TAG_SEND: { - gds_dbg("desc[%zu] SEND\n", i); - gds_send_request_t *sreq = desc->send; - retcode = gds_post_ops_on_cpu(sreq->commit.entries, sreq->commit.storage, flags); - if (retcode) { - gds_err("error %d in gds_post_ops_on_cpu\n", retcode); - ret = retcode; - goto out; - } - break; - } - case GDS_TAG_WAIT: { - gds_dbg("desc[%zu] WAIT\n", i); - gds_wait_request_t *wreq = desc->wait; - retcode = gds_post_ops_on_cpu(wreq->peek.entries, wreq->peek.storage, flags); - if (retcode) { - gds_err("error %d in gds_post_ops_on_cpu\n", retcode); - ret = retcode; - goto out; - } - break; - } - case GDS_TAG_WAIT_VALUE32: { - gds_dbg("desc[%zu] WAIT_VALUE32\n", i); - uint32_t *ptr = desc->wait32.ptr; - uint32_t value = desc->wait32.value; - bool flush = false; - if (desc->wait32.flags & GDS_WAIT_POST_FLUSH_REMOTE) { - gds_err("GDS_WAIT_POST_FLUSH_REMOTE flag is not supported yet\n"); - flush = true; - } - gds_memory_type_t mem_type = (gds_memory_type_t)(desc->wait32.flags & GDS_MEMORY_MASK); - switch(mem_type) { - case GDS_MEMORY_GPU: - // dereferencing ptr may fail if ptr points to CUDA device memory - case GDS_MEMORY_HOST: - case GDS_MEMORY_IO: - break; - default: - gds_err("invalid memory type 0x%02x in WAIT_VALUE32\n", mem_type); - ret = EINVAL; - goto out; + case GDS_TAG_SEND: { + gds_dbg("desc[%zu] SEND\n", i); + gds_mlx5_send_request_t *sreq = to_gds_msreq(desc->send); + retcode = gds_mlx5_post_ops_on_cpu(sreq->commit.entries, sreq->commit.storage, flags); + if (retcode) { + gds_err("error %d in gds_mlx5_post_ops_on_cpu\n", retcode); + ret = retcode; + goto out; + } break; } - bool done = false; - do { - uint32_t data = gds_atomic_get(ptr); - switch(desc->wait32.cond_flags) { - case GDS_WAIT_COND_GEQ: - done = ((int32_t)data - (int32_t)value >= 0); - break; - case GDS_WAIT_COND_EQ: - done = (data == value); - break; - case GDS_WAIT_COND_AND: - done = (data & value); - break; - case GDS_WAIT_COND_NOR: - done = ~(data | value); - break; - default: - gds_err("invalid condition flags 0x%02x in WAIT_VALUE32\n", desc->wait32.cond_flags); + case GDS_TAG_WAIT: { + gds_dbg("desc[%zu] WAIT\n", i); + gds_mlx5_wait_request_t *wreq = to_gds_mwreq(desc->wait); + retcode = gds_mlx5_post_ops_on_cpu(wreq->peek.entries, wreq->peek.storage, flags); + if (retcode) { + gds_err("error %d in gds_mlx5_post_ops_on_cpu\n", retcode); + ret = retcode; goto out; - break; } - if (done) - break; - // TODO: more aggressive CPU relaxing needed here to avoid starving I/O fabric - arch_cpu_relax(); - } while(true); - break; - } - case GDS_TAG_WRITE_VALUE32: { - gds_dbg("desc[%zu] WRITE_VALUE32\n", i); - uint32_t *ptr = desc->write32.ptr; - uint32_t value = desc->write32.value; - gds_memory_type_t mem_type = (gds_memory_type_t)(desc->write32.flags & GDS_MEMORY_MASK); - switch(mem_type) { - case GDS_MEMORY_GPU: - // dereferencing ptr may fail if ptr points to CUDA device memory - case GDS_MEMORY_HOST: - case GDS_MEMORY_IO: break; - default: - gds_err("invalid memory type 0x%02x in WRITE_VALUE32\n", mem_type); - ret = EINVAL; - goto out; + } + case GDS_TAG_WAIT_VALUE32: { + gds_dbg("desc[%zu] WAIT_VALUE32\n", i); + uint32_t *ptr = desc->wait32.ptr; + uint32_t value = desc->wait32.value; + bool flush = false; + if (desc->wait32.flags & GDS_WAIT_POST_FLUSH_REMOTE) { + gds_err("GDS_WAIT_POST_FLUSH_REMOTE flag is not supported yet\n"); + flush = true; + } + gds_memory_type_t mem_type = (gds_memory_type_t)(desc->wait32.flags & GDS_MEMORY_MASK); + switch(mem_type) { + case GDS_MEMORY_GPU: + // dereferencing ptr may fail if ptr points to CUDA device memory + case GDS_MEMORY_HOST: + case GDS_MEMORY_IO: + break; + default: + gds_err("invalid memory type 0x%02x in WAIT_VALUE32\n", mem_type); + ret = EINVAL; + goto out; + break; + } + bool done = false; + do { + uint32_t data = gds_atomic_get(ptr); + switch(desc->wait32.cond_flags) { + case GDS_WAIT_COND_GEQ: + done = ((int32_t)data - (int32_t)value >= 0); + break; + case GDS_WAIT_COND_EQ: + done = (data == value); + break; + case GDS_WAIT_COND_AND: + done = (data & value); + break; + case GDS_WAIT_COND_NOR: + done = ~(data | value); + break; + default: + gds_err("invalid condition flags 0x%02x in WAIT_VALUE32\n", desc->wait32.cond_flags); + goto out; + break; + } + if (done) + break; + // TODO: more aggressive CPU relaxing needed here to avoid starving I/O fabric + arch_cpu_relax(); + } while(true); break; } - bool barrier = (desc->write32.flags & GDS_WRITE_PRE_BARRIER_SYS); - if (barrier) - wmb(); - gds_atomic_set(ptr, value); - break; - } - case GDS_TAG_WRITE_MEMORY: { - void *dest = desc->writemem.dest; - const void *src = desc->writemem.src; - size_t nbytes = desc->writemem.count; - bool barrier = (desc->writemem.flags & GDS_WRITE_MEMORY_POST_BARRIER_SYS); - gds_memory_type_t mem_type = memtype_from_flags(desc->writemem.flags); - gds_dbg("desc[%zu] WRITE_MEMORY dest=%p src=%p len=%zu memtype=%02x\n", i, dest, src, nbytes, mem_type); - switch(mem_type) { - case GDS_MEMORY_GPU: - case GDS_MEMORY_HOST: - memcpy(dest, src, nbytes); + case GDS_TAG_WRITE_VALUE32: { + gds_dbg("desc[%zu] WRITE_VALUE32\n", i); + uint32_t *ptr = desc->write32.ptr; + uint32_t value = desc->write32.value; + gds_memory_type_t mem_type = (gds_memory_type_t)(desc->write32.flags & GDS_MEMORY_MASK); + switch(mem_type) { + case GDS_MEMORY_GPU: + // dereferencing ptr may fail if ptr points to CUDA device memory + case GDS_MEMORY_HOST: + case GDS_MEMORY_IO: + break; + default: + gds_err("invalid memory type 0x%02x in WRITE_VALUE32\n", mem_type); + ret = EINVAL; + goto out; + break; + } + bool barrier = (desc->write32.flags & GDS_WRITE_PRE_BARRIER_SYS); + if (barrier) + wmb(); + gds_atomic_set(ptr, value); break; - case GDS_MEMORY_IO: - assert(nbytes % sizeof(uint64_t)); - assert(((unsigned long)dest & 0x7) == 0); - gds_bf_copy((uint64_t*)dest, (uint64_t*)src, nbytes); + } + case GDS_TAG_WRITE_MEMORY: { + void *dest = desc->writemem.dest; + const void *src = desc->writemem.src; + size_t nbytes = desc->writemem.count; + bool barrier = (desc->writemem.flags & GDS_WRITE_MEMORY_POST_BARRIER_SYS); + gds_memory_type_t mem_type = memtype_from_flags(desc->writemem.flags); + gds_dbg("desc[%zu] WRITE_MEMORY dest=%p src=%p len=%zu memtype=%02x\n", i, dest, src, nbytes, mem_type); + switch(mem_type) { + case GDS_MEMORY_GPU: + case GDS_MEMORY_HOST: + memcpy(dest, src, nbytes); + break; + case GDS_MEMORY_IO: + assert(nbytes % sizeof(uint64_t)); + assert(((unsigned long)dest & 0x7) == 0); + gds_bf_copy((uint64_t*)dest, (uint64_t*)src, nbytes); + break; + default: + assert(!"invalid mem type"); + break; + } + if (barrier) + wmb(); break; + } default: - assert(!"invalid mem type"); + gds_err("invalid tag for %zu entry\n", i); + ret = EINVAL; + goto out; break; - } - if (barrier) - wmb(); - break; - } - default: - gds_err("invalid tag for %zu entry\n", i); - ret = EINVAL; - goto out; - break; } } out: diff --git a/src/archutils.h b/src/archutils.h index 40058a1..2f58695 100644 --- a/src/archutils.h +++ b/src/archutils.h @@ -1,4 +1,29 @@ -// TODO: ADD LICENCE HERE +/* Copyright (c) 2016 - 2020, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ #pragma once @@ -25,12 +50,12 @@ static void arch_cpu_relax(void) static void wmb(void) __attribute__((unused)) ; static void wmb(void) { - asm volatile("sync") ; + asm volatile("sync") ; } static void rmb(void) __attribute__((unused)) ; static void rmb(void) { - asm volatile("sync") ; + asm volatile("sync") ; } #else diff --git a/src/gdsync.cpp b/src/gdsync.cpp index 90d5508..3d7ba0b 100644 --- a/src/gdsync.cpp +++ b/src/gdsync.cpp @@ -25,7 +25,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#if HAVE_CONFIG_H +#ifdef HAVE_CONFIG_H # include #endif /* HAVE_CONFIG_H */ @@ -36,9 +36,12 @@ #include #include +#include + #include "utils.hpp" #include "memmgr.hpp" #include "mem.hpp" +#include "mlx5.hpp" #include "objs.hpp" #include "archutils.h" #include "mlnxutils.h" @@ -60,31 +63,12 @@ int gds_dbg_enabled() if (env) { int en = atoi(env); gds_dbg_is_enabled = !!en; - //printf("GDS_ENABLE_DEBUG=%s\n", env); } else gds_dbg_is_enabled = 0; } return gds_dbg_is_enabled; } -#if 0 -int gds_flusher_enabled() -{ - static int gds_flusher_is_enabled = -1; - if (-1 == gds_flusher_is_enabled) { - const char *env = getenv("GDS_ENABLE_FLUSHER"); - if (env) { - int en = atoi(env); - gds_flusher_is_enabled = !!en; - } else - gds_flusher_is_enabled = 0; - - gds_warn("GDS_ENABLE_FLUSHER=%d\n", gds_flusher_is_enabled); - } - return gds_flusher_is_enabled; -} -#endif - //----------------------------------------------------------------------------- // detect Async APIs @@ -92,16 +76,12 @@ int gds_flusher_enabled() #define CU_STREAM_BATCH_MEM_OP_RELAXED_ORDERING 0x1 #endif -// TODO: use correct value -// TODO: make it dependent upon the particular GPU -const size_t GDS_GPU_MAX_INLINE_SIZE = 256; - //----------------------------------------------------------------------------- // Note: these are default overrides, i.e. allow to disable/enable the features // in case the GPU supports them -static bool gds_enable_write64() +bool gds_enable_write64() { static int gds_disable_write64 = -1; if (-1 == gds_disable_write64) { @@ -115,7 +95,7 @@ static bool gds_enable_write64() return !gds_disable_write64; } -static bool gds_enable_wait_nor() +bool gds_enable_wait_nor() { static int gds_disable_wait_nor = -1; if (-1 == gds_disable_wait_nor) { @@ -129,7 +109,7 @@ static bool gds_enable_wait_nor() return !gds_disable_wait_nor; } -static bool gds_enable_remote_flush() +bool gds_enable_remote_flush() { static int gds_disable_remote_flush = -1; if (-1 == gds_disable_remote_flush) { @@ -143,7 +123,7 @@ static bool gds_enable_remote_flush() return !gds_disable_remote_flush; } -static bool gds_enable_wait_checker() +bool gds_enable_wait_checker() { static int gds_enable_wait_checker = -1; if (-1 == gds_enable_wait_checker) { @@ -157,7 +137,7 @@ static bool gds_enable_wait_checker() return gds_enable_wait_checker; } -static bool gds_enable_inlcpy() +bool gds_enable_inlcpy() { static int gds_disable_inlcpy = -1; if (-1 == gds_disable_inlcpy) { @@ -172,7 +152,7 @@ static bool gds_enable_inlcpy() } // simulate 64-bits writes with inlcpy -static bool gds_simulate_write64() +bool gds_simulate_write64() { static int gds_simulate_write64 = -1; if (-1 == gds_simulate_write64) { @@ -192,7 +172,7 @@ static bool gds_simulate_write64() return gds_simulate_write64; } -static bool gds_enable_membar() +bool gds_enable_membar() { static int gds_disable_membar = -1; if (-1 == gds_disable_membar) { @@ -206,7 +186,7 @@ static bool gds_enable_membar() return !gds_disable_membar; } -static bool gds_enable_weak_consistency() +bool gds_enable_weak_consistency() { static int gds_disable_weak_consistency = -1; if (-1 == gds_disable_weak_consistency) { @@ -218,22 +198,20 @@ static bool gds_enable_weak_consistency() gds_dbg("GDS_DISABLE_WEAK_CONSISTENCY=%d\n", gds_disable_weak_consistency); } gds_dbg("gds_disable_weak_consistency=%d\n", - gds_disable_weak_consistency); + gds_disable_weak_consistency); return !gds_disable_weak_consistency; } -//----------------------------------------------------------------------------- - -static bool gds_enable_dump_memops() +bool gds_enable_dump_memops() { static int gds_enable_dump_memops = -1; if (-1 == gds_enable_dump_memops) { - const char *env = getenv("GDS_ENABLE_DUMP_MEMOPS"); - if (env) - gds_enable_dump_memops = !!atoi(env); - else - gds_enable_dump_memops = 0; // disabled by default - gds_dbg("GDS_ENABLE_DUMP_MEMOPS=%d\n", gds_enable_dump_memops); + const char *env = getenv("GDS_ENABLE_DUMP_MEMOPS"); + if (env) + gds_enable_dump_memops = !!atoi(env); + else + gds_enable_dump_memops = 0; // disabled by default + gds_dbg("GDS_ENABLE_DUMP_MEMOPS=%d\n", gds_enable_dump_memops); } return gds_enable_dump_memops; } @@ -243,48 +221,48 @@ static bool gds_enable_dump_memops() void gds_dump_param(CUstreamBatchMemOpParams *param) { switch(param->operation) { - case CU_STREAM_MEM_OP_WAIT_VALUE_32: - gds_info("WAIT32 addr:%p alias:%p value:%08x flags:%08x\n", - (void*)param->waitValue.address, - (void*)param->writeValue.alias, - param->waitValue.value, - param->waitValue.flags); - break; - - case CU_STREAM_MEM_OP_WRITE_VALUE_32: - gds_info("WRITE32 addr:%p alias:%p value:%08x flags:%08x\n", - (void*)param->writeValue.address, - (void*)param->writeValue.alias, - param->writeValue.value, - param->writeValue.flags); - break; - - case CU_STREAM_MEM_OP_FLUSH_REMOTE_WRITES: - gds_dbg("FLUSH\n"); - break; + case CU_STREAM_MEM_OP_WAIT_VALUE_32: + gds_info("WAIT32 addr:%p alias:%p value:%08x flags:%08x\n", + (void*)param->waitValue.address, + (void*)param->writeValue.alias, + param->waitValue.value, + param->waitValue.flags); + break; + + case CU_STREAM_MEM_OP_WRITE_VALUE_32: + gds_info("WRITE32 addr:%p alias:%p value:%08x flags:%08x\n", + (void*)param->writeValue.address, + (void*)param->writeValue.alias, + param->writeValue.value, + param->writeValue.flags); + break; + + case CU_STREAM_MEM_OP_FLUSH_REMOTE_WRITES: + gds_dbg("FLUSH\n"); + break; #if HAVE_DECL_CU_STREAM_MEM_OP_WRITE_MEMORY - case CU_STREAM_MEM_OP_WRITE_MEMORY: - gds_info("INLINECOPY addr:%p alias:%p src:%p len=%zu flags:%08x\n", - (void*)param->writeMemory.address, - (void*)param->writeMemory.alias, - (void*)param->writeMemory.src, - param->writeMemory.byteCount, - param->writeMemory.flags); - break; + case CU_STREAM_MEM_OP_WRITE_MEMORY: + gds_info("INLINECOPY addr:%p alias:%p src:%p len=%zu flags:%08x\n", + (void*)param->writeMemory.address, + (void*)param->writeMemory.alias, + (void*)param->writeMemory.src, + param->writeMemory.byteCount, + param->writeMemory.flags); + break; #endif #if HAVE_DECL_CU_STREAM_MEM_OP_MEMORY_BARRIER - case CU_STREAM_MEM_OP_MEMORY_BARRIER: - gds_info("MEMORY_BARRIER scope:%02x set_before=%02x set_after=%02x\n", - param->memoryBarrier.scope, - param->memoryBarrier.set_before, - param->memoryBarrier.set_after); - break; + case CU_STREAM_MEM_OP_MEMORY_BARRIER: + gds_info("MEMORY_BARRIER scope:%02x set_before=%02x set_after=%02x\n", + param->memoryBarrier.scope, + param->memoryBarrier.set_before, + param->memoryBarrier.set_after); + break; #endif - default: - gds_err("unsupported operation=%d\n", param->operation); - break; + default: + gds_err("unsupported operation=%d\n", param->operation); + break; } } @@ -311,8 +289,8 @@ int gds_fill_membar(gds_peer *peer, gds_op_list_t &ops, int flags) param.operation = CU_STREAM_MEM_OP_FLUSH_REMOTE_WRITES; param.flushRemoteWrites.flags = 0; gds_dbg("op=%d flush_remote flags=%08x\n", - param.operation, - param.flushRemoteWrites.flags); + param.operation, + param.flushRemoteWrites.flags); } else { param.operation = CU_STREAM_MEM_OP_MEMORY_BARRIER; if (flags & GDS_MEMBAR_MLX5) { @@ -331,10 +309,10 @@ int gds_fill_membar(gds_peer *peer, gds_op_list_t &ops, int flags) goto out; } gds_dbg("op=%d membar scope:%02x set_before=%02x set_after=%02x\n", - param.operation, - param.memoryBarrier.scope, - param.memoryBarrier.set_before, - param.memoryBarrier.set_after); + param.operation, + param.memoryBarrier.scope, + param.memoryBarrier.set_before, + param.memoryBarrier.set_after); } ops.push_back(param); @@ -348,7 +326,7 @@ int gds_fill_membar(gds_peer *peer, gds_op_list_t &ops, int flags) //----------------------------------------------------------------------------- -static int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, const void *data, size_t n_bytes, int flags) +int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, const void *data, size_t n_bytes, int flags) { int retcode = 0; #if HAVE_DECL_CU_STREAM_MEM_OP_WRITE_MEMORY @@ -379,11 +357,11 @@ static int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, else param.writeMemory.flags = CU_STREAM_WRITE_MEMORY_NO_MEMORY_BARRIER; gds_dbg("op=%d addr=%p src=%p size=%zd flags=%08x\n", - param.operation, - (void*)param.writeMemory.address, - param.writeMemory.src, - param.writeMemory.byteCount, - param.writeMemory.flags); + param.operation, + (void*)param.writeMemory.address, + param.writeMemory.src, + param.writeMemory.byteCount, + param.writeMemory.flags); ops.push_back(param); #else gds_err("CU_STREAM_MEM_OP_WRITE_MEMORY not supported nor enabled on this GPU\n"); @@ -409,7 +387,7 @@ int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t &ops, void *ptr, const void *d //----------------------------------------------------------------------------- -static void gds_enable_barrier_for_inlcpy(CUstreamBatchMemOpParams *param) +void gds_enable_barrier_for_inlcpy(CUstreamBatchMemOpParams *param) { #if HAVE_DECL_CU_STREAM_MEM_OP_WRITE_MEMORY assert(param->operation == CU_STREAM_MEM_OP_WRITE_MEMORY); @@ -419,7 +397,7 @@ static void gds_enable_barrier_for_inlcpy(CUstreamBatchMemOpParams *param) //----------------------------------------------------------------------------- -static int gds_fill_poke(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint32_t value, int flags) +int gds_fill_poke(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint32_t value, int flags) { int retcode = 0; CUdeviceptr dev_ptr = addr; @@ -438,10 +416,10 @@ static int gds_fill_poke(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, u if (need_barrier) param.writeValue.flags = 0; gds_dbg("op=%d addr=%p value=%08x flags=%08x\n", - param.operation, - (void*)param.writeValue.address, - param.writeValue.value, - param.writeValue.flags); + param.operation, + (void*)param.writeValue.address, + param.writeValue.value, + param.writeValue.flags); ops.push_back(param); return retcode; } @@ -466,7 +444,7 @@ int gds_fill_poke(gds_peer *peer, gds_op_list_t &ops, uint32_t *ptr, uint32_t va //----------------------------------------------------------------------------- -static int gds_fill_poke64(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint64_t value, int flags) +int gds_fill_poke64(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint64_t value, int flags) { int retcode = 0; #if HAVE_DECL_CU_STREAM_MEM_OP_WRITE_VALUE_64 @@ -487,10 +465,10 @@ static int gds_fill_poke64(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, if (need_barrier) param.writeValue.flags = 0; gds_dbg("op=%d addr=%p value=%08x flags=%08x\n", - param.operation, - (void*)param.writeValue.address, - param.writeValue.value, - param.writeValue.flags); + param.operation, + (void*)param.writeValue.address, + param.writeValue.value, + param.writeValue.flags); ops.push_back(param); #else gds_err("CU_STREAM_WRITE_VALUE_NO_MEMORY_BARRIER not supported nor enabled on this GPU\n"); @@ -550,6 +528,7 @@ struct poll_checker { gds_dbg("%d injecting pre poke\n", m_idx); gds_fill_poke(peer, ops, &m_buf->state, 1, GDS_MEMORY_HOST); } + void post(gds_peer *peer, gds_op_list_t &ops) { gds_dbg("%d injecting post poke\n", m_idx); gds_fill_poke(peer, ops, &m_buf->state, 2, GDS_MEMORY_HOST); @@ -580,7 +559,7 @@ unsigned poll_checker::m_global_index = 0; //----------------------------------------------------------------------------- -static int gds_fill_poll(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr ptr, uint32_t magic, int cond_flag, int flags) +int gds_fill_poll(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr ptr, uint32_t magic, int cond_flag, int flags) { int retcode = 0; const char *cond_str = NULL; @@ -602,50 +581,50 @@ static int gds_fill_poll(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr ptr, ui param.waitValue.address = dev_ptr; param.waitValue.value = magic; switch(cond_flag) { - case GDS_WAIT_COND_GEQ: - param.waitValue.flags = CU_STREAM_WAIT_VALUE_GEQ; - cond_str = "CU_STREAM_WAIT_VALUE_GEQ"; - break; - case GDS_WAIT_COND_EQ: - param.waitValue.flags = CU_STREAM_WAIT_VALUE_EQ; - cond_str = "CU_STREAM_WAIT_VALUE_EQ"; - break; - case GDS_WAIT_COND_AND: - param.waitValue.flags = CU_STREAM_WAIT_VALUE_AND; - cond_str = "CU_STREAM_WAIT_VALUE_AND"; - break; - - case GDS_WAIT_COND_NOR: + case GDS_WAIT_COND_GEQ: + param.waitValue.flags = CU_STREAM_WAIT_VALUE_GEQ; + cond_str = "CU_STREAM_WAIT_VALUE_GEQ"; + break; + case GDS_WAIT_COND_EQ: + param.waitValue.flags = CU_STREAM_WAIT_VALUE_EQ; + cond_str = "CU_STREAM_WAIT_VALUE_EQ"; + break; + case GDS_WAIT_COND_AND: + param.waitValue.flags = CU_STREAM_WAIT_VALUE_AND; + cond_str = "CU_STREAM_WAIT_VALUE_AND"; + break; + + case GDS_WAIT_COND_NOR: #if HAVE_DECL_CU_STREAM_WAIT_VALUE_NOR - if (!peer->has_wait_nor) { - gds_err("GDS_WAIT_COND_NOR is not supported nor enabled on this GPU\n"); - retcode = EINVAL; - goto out; - } - param.waitValue.flags = CU_STREAM_WAIT_VALUE_NOR; - if (gds_enable_wait_checker()) - ck = new poll_checker(); + if (!peer->has_wait_nor) { + gds_err("GDS_WAIT_COND_NOR is not supported nor enabled on this GPU\n"); + retcode = EINVAL; + goto out; + } + param.waitValue.flags = CU_STREAM_WAIT_VALUE_NOR; + if (gds_enable_wait_checker()) + ck = new poll_checker(); #else - gds_err("GDS_WAIT_COND_NOR requires CUDA 9.0 at least\n"); - retcode = EINVAL; + gds_err("GDS_WAIT_COND_NOR requires CUDA 9.0 at least\n"); + retcode = EINVAL; #endif - cond_str = "CU_STREAM_WAIT_VALUE_NOR"; - break; - default: - gds_err("invalid wait condition flag\n"); - retcode = EINVAL; - goto out; + cond_str = "CU_STREAM_WAIT_VALUE_NOR"; + break; + default: + gds_err("invalid wait condition flag\n"); + retcode = EINVAL; + goto out; } if (need_flush) param.waitValue.flags |= CU_STREAM_WAIT_VALUE_FLUSH; gds_dbg("op=%d addr=%p value=%08x cond=%s flags=%08x\n", - param.operation, - (void*)param.waitValue.address, - param.waitValue.value, - cond_str, - param.waitValue.flags); + param.operation, + (void*)param.waitValue.address, + param.waitValue.value, + cond_str, + param.waitValue.flags); if (ck) ck->pre(peer, ops, ptr, magic, cond_flag); @@ -670,7 +649,7 @@ int gds_fill_poll(gds_peer *peer, gds_op_list_t &ops, uint32_t *ptr, uint32_t ma gds_err("could not lookup %p\n", ptr); goto out; } - + retcode = gds_fill_poll(peer, ops, dev_ptr, magic, cond_flag, flags); out: return retcode; @@ -692,7 +671,6 @@ int gds_stream_batch_ops(gds_peer *peer, CUstream stream, gds_op_list_t &ops, in if (nops > peer->max_batch_size) { gds_warn("batch size might be too big, stream=%p nops=%zu flags=%08x\n", stream, nops, flags); - //return EINVAL; } result = cuStreamBatchMemOp(stream, nops, &ops[0], cuflags); @@ -717,441 +695,19 @@ int gds_stream_batch_ops(gds_peer *peer, CUstream stream, gds_op_list_t &ops, in //----------------------------------------------------------------------------- -/* - A) plain+membar: - WR32 - MEMBAR - WR32 - WR32 - - B) plain: - WR32 - WR32+PREBARRIER - WR32 - - C) sim64+membar: - WR32 - MEMBAR - INLCPY 8B - - D) sim64: - INLCPY 4B + POSTBARRIER - INLCPY 8B - - E) inlcpy+membar: - WR32 - MEMBAR - INLCPY XB - - F) inlcpy: - INLCPY 4B + POSTBARRIER - INLCPY 128B -*/ - -int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_list_t &ops, int post_flags) -{ - int retcode = 0; - size_t n = 0; - bool prev_was_fence = false; - bool use_inlcpy_for_dword = false; - //size_t n_ops = ops.size(); - CUstreamBatchMemOpParams param; - - gds_dbg("n_ops=%zu\n", n_ops); - - if (!peer->has_memops) { - gds_err("CUDA MemOps are required\n"); - return EINVAL; - } - - // divert the request to the same engine handling 64bits - // to avoid out-of-order execution - // caveat: can't use membar if inlcpy is used for 4B writes (to simulate 8B writes) - if (peer->has_inlcpy) { - if (!peer->has_membar) - use_inlcpy_for_dword = true; // F - } - if (gds_simulate_write64()) { - if (!peer->has_membar) { - gds_warn_once("enabling use_inlcpy_for_dword\n"); - use_inlcpy_for_dword = true; // D - } - } - - for (; op && n < n_ops; op = op->next, ++n) { - //int flags = 0; - gds_dbg("op[%zu] type:%08x\n", n, op->type); - switch(op->type) { - case IBV_EXP_PEER_OP_FENCE: { - gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); - - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { - gds_dbg("nothing to do for read fences\n"); - //retcode = EINVAL; - break; - } - else { - if (!peer->has_membar) { - if (use_inlcpy_for_dword) { - assert(ops.size() > 0); - gds_dbg("patching previous param\n"); - gds_enable_barrier_for_inlcpy(&ops.back()); - } - else { - gds_dbg("recording fence event\n"); - prev_was_fence = true; - } - //retcode = 0; - } - else { - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { - gds_err("unexpected from fence\n"); - retcode = EINVAL; - break; - } - int flags = 0; - if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { - gds_dbg("using light membar\n"); - flags = GDS_MEMBAR_DEFAULT | GDS_MEMBAR_MLX5; - } - else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { - gds_dbg("using heavy membar\n"); - flags = GDS_MEMBAR_SYS | GDS_MEMBAR_MLX5; - } - else { - gds_err("unsupported fence combination\n"); - retcode = EINVAL; - break; - } - retcode = gds_fill_membar(peer, ops, flags); - } - } - break; - } - case IBV_EXP_PEER_OP_STORE_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - int flags = 0; - gds_dbg("OP_STORE_DWORD dev_ptr=%llx data=%" PRIx32 "\n", dev_ptr, data); - if (use_inlcpy_for_dword) { // F || D - // membar may be out of order WRT inlcpy - if (peer->has_membar) { - gds_err("invalid feature combination, inlcpy + membar\n"); - retcode = EINVAL; - break; - } - // tail flush is set when following fence is met - // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; - retcode = gds_fill_inlcpy(peer, ops, dev_ptr, &data, sizeof(data), flags); - } - else { // A || B || C || E - // can't guarantee ordering of write32+inlcpy unless - // a membar is there - // TODO: fix driver when !weak - if (peer->has_inlcpy && !peer->has_membar) { - gds_err("invalid feature combination, inlcpy needs membar\n"); - retcode = EINVAL; - break; - } - if (prev_was_fence) { - gds_dbg("using PRE_BARRIER as fence\n"); - flags |= GDS_WRITE_PRE_BARRIER; - prev_was_fence = false; - } - retcode = gds_fill_poke(peer, ops, dev_ptr, data, flags); - } - break; - } - case IBV_EXP_PEER_OP_STORE_QWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + - op->wr.qword_va.offset; - uint64_t data = op->wr.qword_va.data; - int flags = 0; - gds_dbg("OP_STORE_QWORD dev_ptr=%llx data=%" PRIx64 "\n", dev_ptr, data); - // C || D - if (gds_simulate_write64()) { - // simulate 64-bit poke by inline copy - if (!peer->has_membar) { - gds_err("invalid feature combination, inlcpy needs membar\n"); - retcode = EINVAL; - break; - } - - // tail flush is never useful here - //flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; - retcode = gds_fill_inlcpy(peer, ops, dev_ptr, &data, sizeof(data), flags); - } - else if (peer->has_write64) { - retcode = gds_fill_poke64(peer, ops, dev_ptr, data, flags); - } - else { - uint32_t datalo = gds_qword_lo(op->wr.qword_va.data); - uint32_t datahi = gds_qword_hi(op->wr.qword_va.data); - - if (prev_was_fence) { - gds_dbg("enabling PRE_BARRIER\n"); - flags |= GDS_WRITE_PRE_BARRIER; - prev_was_fence = false; - } - retcode = gds_fill_poke(peer, ops, dev_ptr, datalo, flags); - - // get rid of the barrier, if there - flags &= ~GDS_WRITE_PRE_BARRIER; - - // advance to next DWORD - dev_ptr += sizeof(uint32_t); - retcode = gds_fill_poke(peer, ops, dev_ptr, datahi, flags); - } - - break; - } - case IBV_EXP_PEER_OP_COPY_BLOCK: { - CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + - op->wr.copy_op.offset; - size_t len = op->wr.copy_op.len; - void *src = op->wr.copy_op.src; - int flags = 0; - gds_dbg("OP_COPY_BLOCK dev_ptr=%llx src=%p len=%zu\n", dev_ptr, src, len); - // catching any other size here - if (!peer->has_inlcpy) { - gds_err("inline copy is not supported\n"); - retcode = EINVAL; - break; - } - // IB Verbs bug - assert(len <= GDS_GPU_MAX_INLINE_SIZE); - //if (desc->need_flush) { - // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; - //} - retcode = gds_fill_inlcpy(peer, ops, dev_ptr, src, len, flags); - break; - } - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { - int poll_cond; - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - // TODO: properly handle a following fence instead of blidly flushing - int flags = 0; - if (!(post_flags & GDS_POST_OPS_DISCARD_WAIT_FLUSH)) - flags |= GDS_WAIT_POST_FLUSH_REMOTE; - - gds_dbg("OP_WAIT_DWORD dev_ptr=%llx data=%" PRIx32 " type=%" PRIx32 "\n", dev_ptr, data, (uint32_t)op->type); - - switch(op->type) { - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: - poll_cond = GDS_WAIT_COND_NOR; - break; - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - poll_cond = GDS_WAIT_COND_GEQ; - break; - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - poll_cond = GDS_WAIT_COND_AND; - break; - default: - assert(!"cannot happen"); - retcode = EINVAL; - goto out; - } - retcode = gds_fill_poll(peer, ops, dev_ptr, data, poll_cond, flags); - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - retcode = EINVAL; - break; - } - if (retcode) { - gds_err("error in fill func at entry n=%zu\n", n); - goto out; - } - } - - assert(n_ops == n); - -out: - return retcode; -} - -//----------------------------------------------------------------------------- - -int gds_post_pokes(CUstream stream, int count, gds_send_request_t *info, uint32_t *dw, uint32_t val) -{ - int retcode = 0; - //CUstreamBatchMemOpParams params[poke_count+1]; - gds_op_list_t ops; - - assert(info); - assert(dw); - - gds_peer *peer = peer_from_stream(stream); - if (!peer) { - return EINVAL; - } - - for (int j=0; jnext, ++n) { - //int flags = 0; - gds_dbg("op[%zu]=%p\n", n, op); - //gds_dbg("op[%zu]=%p type:%08x\n", n, op, op->type); - switch(op->type) { - case IBV_EXP_PEER_OP_FENCE: { - gds_dbg("FENCE flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); - - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { - gds_warnc(1, "nothing to do for read fences\n"); - //retcode = EINVAL; - break; - } - else { - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { - gds_err("unexpected from %08x fence, expected FROM_HCA\n", fence_from); - retcode = EINVAL; - break; - } - if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { - gds_dbg("using light membar\n"); - wmb(); - } - else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { - gds_dbg("using heavy membar\n"); - wmb(); - } - else { - gds_err("unsupported fence combination\n"); - retcode = EINVAL; - break; - } - } - break; - } - case IBV_EXP_PEER_OP_STORE_DWORD: { - uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); - uint32_t data = op->wr.dword_va.data; - // A || B || C || E - gds_dbg("STORE_DWORD ptr=%p data=%08" PRIx32 "\n", ptr, data); - gds_atomic_set(ptr, data); - break; - } - case IBV_EXP_PEER_OP_STORE_QWORD: { - uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.qword_va.target_id)->va + op->wr.qword_va.offset); - uint64_t data = op->wr.qword_va.data; - gds_dbg("STORE_QWORD ptr=%p data=%016" PRIx64 "\n", ptr, data); - gds_atomic_set(ptr, data); - break; - } - case IBV_EXP_PEER_OP_COPY_BLOCK: { - uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.copy_op.target_id)->va + op->wr.copy_op.offset); - uint64_t *src = (uint64_t*)op->wr.copy_op.src; - size_t n_bytes = op->wr.copy_op.len; - gds_dbg("COPY_BLOCK ptr=%p src=%p len=%zu\n", ptr, src, n_bytes); - gds_bf_copy(ptr, src, n_bytes); - break; - } - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { - int poll_cond; - uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); - uint32_t value = op->wr.dword_va.data; - bool flush = true; - if (post_flags & GDS_POST_OPS_DISCARD_WAIT_FLUSH) - flush = false; - gds_dbg("WAIT_32 dev_ptr=%p data=%" PRIx32 " type=%" PRIx32 "\n", ptr, value, (uint32_t)op->type); - bool done = false; - do { - uint32_t data = gds_atomic_get(ptr); - switch(op->type) { - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: - done = (0 != ~(data | value)); - break; - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - done = ((int32_t)data - (int32_t)value >= 0); - break; - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - done = (0 != (data & value)); - break; - default: - gds_err("invalid op type %02x\n", op->type); - retcode = EINVAL; - goto out; - } - if (done) - break; - // TODO: more aggressive CPU relaxing needed here to avoid starving I/O fabric - arch_cpu_relax(); - } while(true); - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - retcode = EINVAL; - break; - } - if (retcode) { - gds_err("error %d at entry n=%zu\n", retcode, n); - goto out; - } - } - -out: - return retcode; -} - -//----------------------------------------------------------------------------- - -int gds_post_pokes_on_cpu(int count, gds_send_request_t *info, uint32_t *dw, uint32_t val) +int gds_post_pokes_on_cpu(int count, gds_send_request_t *p_info, uint32_t *dw, uint32_t val) { int retcode = 0; int idx = 0; + gds_mlx5_send_request_t *info; + + assert(p_info); - assert(info); + info = to_gds_msreq(p_info); for (int j=0; jnext, ++n) { - gds_dbg("op[%zu] type:%d\n", n, op->type); - switch(op->type) { - case IBV_EXP_PEER_OP_FENCE: { - gds_dbg("FENCE flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - break; - } - case IBV_EXP_PEER_OP_STORE_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - gds_dbg("STORE_QWORD data:%x target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", - op->wr.dword_va.data, op->wr.dword_va.target_id, - op->wr.dword_va.offset, dev_ptr); - break; - } - case IBV_EXP_PEER_OP_STORE_QWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + - op->wr.qword_va.offset; - gds_dbg("STORE_QWORD data:%" PRIx64 " target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", - op->wr.qword_va.data, op->wr.qword_va.target_id, - op->wr.qword_va.offset, dev_ptr); - break; - } - case IBV_EXP_PEER_OP_COPY_BLOCK: { - CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + - op->wr.copy_op.offset; - gds_dbg("COPY_BLOCK src:%p len:%zu target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", - op->wr.copy_op.src, op->wr.copy_op.len, - op->wr.copy_op.target_id, op->wr.copy_op.offset, - dev_ptr); - break; - } - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - gds_dbg("%s data:%08x target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", - (op->type==IBV_EXP_PEER_OP_POLL_AND_DWORD) ? "POLL_AND_DW" : "POLL_NOR_SDW", - op->wr.dword_va.data, - op->wr.dword_va.target_id, - op->wr.dword_va.offset, - dev_ptr); - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - break; - } - } - - assert(count == n); -} + gds_mlx5_wait_request_t *request; -//----------------------------------------------------------------------------- + assert(p_wreq); -void gds_dump_wait_request(gds_wait_request_t *request, size_t count) -{ - for (size_t j=0; jentries, peek->whence, peek->offset, - peek->peek_id, peek->comp_mask); - gds_dump_ops(peek->storage, peek->entries); + j, peek->entries, peek->whence, peek->offset, + peek->peek_id, peek->comp_mask); + gds_mlx5_dump_ops(peek->storage, peek->entries); } } @@ -1263,15 +766,15 @@ int gds_stream_post_wait_cq_multi(CUstream stream, int count, gds_wait_request_t descs[k].wait = &request[k]; } - retcode=gds_stream_post_descriptors(stream, count, descs, 0); + retcode = gds_stream_post_descriptors(stream, count, descs, 0); if (retcode) { gds_err("error %d in gds_stream_post_descriptors\n", retcode); goto out; } - out: - if(descs) free(descs); - return retcode; +out: + if(descs) free(descs); + return retcode; } //----------------------------------------------------------------------------- @@ -1279,40 +782,40 @@ int gds_stream_post_wait_cq_multi(CUstream stream, int count, gds_wait_request_t // If NULL returned then buffer will be allocated in system memory // by ibverbs driver. -static struct ibv_exp_peer_buf *gds_buf_alloc(ibv_exp_peer_buf_alloc_attr *attr) +static struct gds_buf *gds_buf_alloc(gds_buf_alloc_attr *attr) { assert(attr); gds_peer *peer = peer_from_id(attr->peer_id); assert(peer); gds_dbg("alloc mem peer:{type=%d gpu_id=%d} attr{len=%zu dir=%d alignment=%d peer_id=%" PRIx64 "}\n", - peer->alloc_type, peer->gpu_id, attr->length, attr->dir, attr->alignment, attr->peer_id); + peer->alloc_type, peer->gpu_id, attr->length, attr->dir, attr->alignment, attr->peer_id); return peer->buf_alloc(peer->alloc_type, attr->length, attr->dir, attr->alignment, peer->alloc_flags); } -static int gds_buf_release(struct ibv_exp_peer_buf *pb) +static int gds_buf_release(struct gds_buf *pb) { gds_dbg("freeing pb=%p\n", pb); - gds_buf *buf = static_cast(pb); + struct gds_buf *buf = pb; gds_peer *peer = buf->peer; peer->free(buf); return 0; } -static uint64_t gds_register_va(void *start, size_t length, uint64_t peer_id, struct ibv_exp_peer_buf *pb) +static uint64_t gds_register_va(void *start, size_t length, uint64_t peer_id, struct gds_buf *pb) { gds_peer *peer = peer_from_id(peer_id); gds_range *range = NULL; gds_dbg("start=%p length=%zu peer_id=%" PRIx64 " peer_buf=%p\n", start, length, peer_id, pb); - if (IBV_EXP_PEER_IOMEMORY == pb) { + if (GDS_PEER_IOMEMORY == pb) { // register as IOMEM range = peer->register_range(start, length, GDS_MEMORY_IO); } else if (pb) { - gds_buf *buf = static_cast(pb); + struct gds_buf *buf = pb; // should have been allocated via gds_buf_alloc // assume GDR mapping already created // associate range to peer_buf @@ -1435,7 +938,7 @@ static bool support_weak_consistency(CUdevice dev) do { gds_dbg("testing hidden weak flag\n"); - + CUstreamBatchMemOpParams params[2]; CUresult res; res = cuStreamBatchMemOp(0, 0, params, 0); @@ -1509,28 +1012,27 @@ static void gds_init_peer(gds_peer *peer, CUdevice dev, int gpu_id) peer->attr.register_va = gds_register_va; peer->attr.unregister_va = gds_unregister_va; - peer->attr.caps = ( IBV_EXP_PEER_OP_STORE_DWORD_CAP | - IBV_EXP_PEER_OP_STORE_QWORD_CAP | - IBV_EXP_PEER_OP_FENCE_CAP | - IBV_EXP_PEER_OP_POLL_AND_DWORD_CAP ); + peer->attr.caps = ( GDS_PEER_OP_STORE_DWORD_CAP | + GDS_PEER_OP_STORE_QWORD_CAP | + GDS_PEER_OP_FENCE_CAP | + GDS_PEER_OP_POLL_AND_DWORD_CAP ); if (peer->has_wait_nor) { gds_dbg("enabling NOR feature\n"); - peer->attr.caps |= IBV_EXP_PEER_OP_POLL_NOR_DWORD_CAP; + peer->attr.caps |= GDS_PEER_OP_POLL_NOR_DWORD_CAP; } else - peer->attr.caps |= IBV_EXP_PEER_OP_POLL_GEQ_DWORD_CAP; + peer->attr.caps |= GDS_PEER_OP_POLL_GEQ_DWORD_CAP; if (peer->has_inlcpy) { gds_dbg("enabling COPY BLOCK feature\n"); - peer->attr.caps |= IBV_EXP_PEER_OP_COPY_BLOCK_CAP; + peer->attr.caps |= GDS_PEER_OP_COPY_BLOCK_CAP; } else if (peer->has_write64 || gds_simulate_write64()) { gds_dbg("enabling STORE QWORD feature\n"); - peer->attr.caps |= IBV_EXP_PEER_OP_STORE_QWORD_CAP; + peer->attr.caps |= GDS_PEER_OP_STORE_QWORD_CAP; } gds_dbg("caps=%016lx\n", peer->attr.caps); peer->attr.peer_dma_op_map_len = GDS_GPU_MAX_INLINE_SIZE; - peer->attr.comp_mask = IBV_EXP_PEER_DIRECT_VERSION; peer->attr.version = 1; peer->tq = new task_queue; @@ -1547,7 +1049,7 @@ static int gds_register_peer(CUdevice dev, unsigned gpu_id, gds_peer **p_peer, g int ret = 0; gds_dbg("GPU%u: registering peer\n", gpu_id); - + if (gpu_id >= max_gpus) { gds_err("invalid gpu_id %d\n", gpu_id); return EINVAL; @@ -1678,59 +1180,56 @@ gds_peer *peer_from_stream(CUstream stream) //----------------------------------------------------------------------------- -static ibv_exp_res_domain *gds_create_res_domain(struct ibv_context *context) +/* \brief: Get the underlying driver associated with the ibdev. + * + */ +static inline gds_driver_type gds_get_driver_type(struct ibv_device *ibdev) { - if (!context) { - gds_err("invalid context"); - return NULL; - } - - ibv_exp_res_domain_init_attr res_domain_attr; - memset(&res_domain_attr, 0, sizeof(res_domain_attr)); + const char *dev_name = ibv_get_device_name(ibdev); - res_domain_attr.comp_mask |= IBV_EXP_RES_DOMAIN_THREAD_MODEL; - res_domain_attr.thread_model = IBV_EXP_THREAD_SINGLE; - - ibv_exp_res_domain *res_domain = ibv_exp_create_res_domain(context, &res_domain_attr); - if (!res_domain) { - gds_warn("Can't create resource domain\n"); - } - - return res_domain; + // Heuristically guess the driver by the device name. + // Until we find a better way to do so... + if (strstr(dev_name, "mlx5") != NULL) + return GDS_DRIVER_TYPE_MLX5; + return GDS_DRIVER_TYPE_UNKNOW; } //----------------------------------------------------------------------------- -static struct gds_cq * -gds_create_cq_internal(struct ibv_context *context, int cqe, - void *cq_context, struct ibv_comp_channel *channel, - int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags, - struct ibv_exp_res_domain * res_domain) +gds_cq_t *gds_create_cq(struct ibv_context *context, int cqe, + void *cq_context, struct ibv_comp_channel *channel, + int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags) { - struct gds_cq *gcq = NULL; - ibv_exp_cq_init_attr attr; + gds_mlx5_cq_t *mcq = NULL; + struct ibv_cq *ibcq = NULL; gds_peer *peer = NULL; gds_peer_attr *peer_attr = NULL; - int ret=0; + gds_driver_type dtype; + int ret = 0; - if(!context) - { - gds_dbg("Invalid input context\n"); - return NULL; + gds_dbg("cqe=%d gpu_id=%d cq_flags=%08x\n", cqe, gpu_id, flags); + + if (!context) { + gds_dbg("Invalid input context\n"); + goto err; } - gcq = (struct gds_cq*)calloc(1, sizeof(struct gds_cq)); - if (!gcq) { - gds_err("cannot allocate memory\n"); - return NULL; + if (flags & GDS_ALLOC_CQ_ON_GPU) { + gds_err("Allocating CQ on GPU is currently not supported\n"); + goto err; } - //Here we need to recover peer and peer_attr pointers to set alloc_type and alloc_flags - //before ibv_exp_create_cq + dtype = gds_get_driver_type(context->device); + if (dtype != GDS_DRIVER_TYPE_MLX5) { + gds_err("Not supported IB device. Currently only mlx5 devices are supported.\n"); + return NULL; + } + + // Here we need to recover peer and peer_attr pointers to set alloc_type and alloc_flags. ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); if (ret) { - gds_err("error %d while registering GPU peer\n", ret); - return NULL; + gds_err("error %d while registering GPU peer\n", ret); + goto err; } assert(peer); assert(peer_attr); @@ -1738,83 +1237,65 @@ gds_create_cq_internal(struct ibv_context *context, int cqe, peer->alloc_type = gds_peer::CQ; peer->alloc_flags = flags; - attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_PEER_DIRECT; - attr.flags = 0; // see ibv_exp_cq_create_flags - attr.peer_direct_attrs = peer_attr; - if (res_domain) { - gds_dbg("using peer->res_domain %p for CQ\n", res_domain); - attr.res_domain = res_domain; - attr.comp_mask |= IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN; + ibcq = ibv_create_cq(context, cqe, cq_context, channel, comp_vector); + if (!ibcq) { + gds_err("error %d in ibv_create_cq\n", errno); + goto err; } - - int old_errno = errno; - gcq->cq = ibv_exp_create_cq(context, cqe, cq_context, channel, comp_vector, &attr); - if (!gcq->cq) { - gds_err("error %d in ibv_exp_create_cq, old errno %d\n", errno, old_errno); - return NULL; + + ret = gds_mlx5_create_cq(ibcq, peer_attr, &mcq); + if (ret) { + gds_err("error %d in gds_mlx5_create_cq\n", ret); + goto err; } - return gcq; -} + return &mcq->gcq; -//Note: general create cq function, not really used for now! -struct gds_cq * -gds_create_cq(struct ibv_context *context, int cqe, - void *cq_context, struct ibv_comp_channel *channel, - int comp_vector, int gpu_id, gds_alloc_cq_flags_t flags) -{ - int ret = 0; - struct gds_cq *gcq = NULL; - //TODO: leak of res_domain - struct ibv_exp_res_domain * res_domain; - gds_dbg("cqe=%d gpu_id=%d cq_flags=%08x\n", cqe, gpu_id, flags); +err: + if (ibcq) + ibv_destroy_cq(ibcq); - gds_peer *peer = NULL; - gds_peer_attr *peer_attr = NULL; - ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); - if (ret) { - gds_err("error %d while registering GPU peer\n", ret); - return NULL; - } - assert(peer); - assert(peer_attr); + return NULL; +} - peer->alloc_type = gds_peer::CQ; - peer->alloc_flags = flags; +//----------------------------------------------------------------------------- - res_domain = gds_create_res_domain(context); - if (res_domain) - gds_dbg("using res_domain %p\n", res_domain); - else - gds_warn("NOT using res_domain\n"); +int gds_poll_cq(struct gds_cq *gcq, int ne, struct ibv_wc *wc) +{ + assert(gcq->dtype == GDS_DRIVER_TYPE_MLX5); - - gcq = gds_create_cq_internal(context, cqe, cq_context, channel, comp_vector, gpu_id, flags, res_domain); + gds_mlx5_cq_t *mcq = to_gds_mcq(gcq); - if (!gcq) { - gds_err("error in gds_create_cq_internal\n"); - return NULL; + switch (gcq->ctype) { + case GDS_CQ_TYPE_SQ: + return gds_mlx5_poll_cq(mcq, ne, wc); + case GDS_CQ_TYPE_RQ: + return ibv_poll_cq(gcq->ibcq, ne, wc); } - return gcq; + return -EINVAL; } //----------------------------------------------------------------------------- -struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, - gds_qp_init_attr_t *qp_attr, int gpu_id, int flags) +gds_qp_t *gds_create_qp(struct ibv_pd *p_pd, struct ibv_context *context, + gds_qp_init_attr_t *qp_attr, int gpu_id, int flags) { int ret = 0; - struct gds_qp *gqp = NULL; - struct ibv_qp *qp = NULL; - struct gds_cq *rx_gcq = NULL, *tx_gcq = NULL; + struct ibv_pd *pd = NULL; + gds_mlx5_qp_t *mqp = NULL; + gds_qp_t *gqp = NULL; + struct ibv_qp *ibqp = NULL; + gds_cq_t *rx_gcq = NULL, *tx_gcq = NULL; gds_peer *peer = NULL; gds_peer_attr *peer_attr = NULL; - int old_errno = errno; + gds_mlx5_qp_peer_t *qp_peer = NULL; + gds_driver_type dtype; - gds_dbg("pd=%p context=%p gpu_id=%d flags=%08x current errno=%d\n", pd, context, gpu_id, flags, errno); - assert(pd); + gds_dbg("pd=%p context=%p gpu_id=%d flags=%08x current errno=%d\n", p_pd, context, gpu_id, flags, errno); + assert(p_pd); assert(context); + assert(context->device); assert(qp_attr); if (flags & ~(GDS_CREATE_QP_WQ_ON_GPU|GDS_CREATE_QP_TX_CQ_ON_GPU|GDS_CREATE_QP_RX_CQ_ON_GPU|GDS_CREATE_QP_WQ_DBREC_ON_GPU)) { @@ -1822,40 +1303,47 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, return NULL; } - gqp = (struct gds_qp*)calloc(1, sizeof(struct gds_qp)); - if (!gqp) { - gds_err("cannot allocate memory\n"); - return NULL; + dtype = gds_get_driver_type(context->device); + if (dtype != GDS_DRIVER_TYPE_MLX5) { + gds_err("Not supported IB device. Currently only mlx5 devices are supported.\n"); + return NULL; } - gqp->dev_context=context; - // peer registration - gds_dbg("before gds_register_peer_ex\n"); + gds_dbg("before gds_register_peer_by_ordinal\n"); ret = gds_register_peer_by_ordinal(gpu_id, &peer, &peer_attr); if (ret) { - gds_err("error %d in gds_register_peer_ex\n", ret); - goto err; + gds_err("error %d in gds_register_peer_by_ordinal\n", ret); + goto err; } - gqp->res_domain = gds_create_res_domain(context); - if (gqp->res_domain) - gds_dbg("using gqp->res_domain %p\n", gqp->res_domain); - else - gds_warn("NOT using gqp->res_domain\n"); + peer->alloc_type = gds_peer::WQ; + peer->alloc_flags = GDS_ALLOC_WQ_DEFAULT | GDS_ALLOC_DBREC_DEFAULT; + if (flags & GDS_CREATE_QP_WQ_ON_GPU) { + gds_err("error, QP WQ on GPU is not supported yet\n"); + goto err; + } + if (flags & GDS_CREATE_QP_WQ_DBREC_ON_GPU) { + gds_warn("QP WQ DBREC on GPU\n"); + peer->alloc_flags |= GDS_ALLOC_DBREC_ON_GPU; + } - tx_gcq = gds_create_cq_internal(context, qp_attr->cap.max_send_wr, NULL, NULL, 0, gpu_id, - (flags & GDS_CREATE_QP_TX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT, - gqp->res_domain); + ret = gds_mlx5_alloc_parent_domain(p_pd, context, peer_attr, &pd, &qp_peer); + if (ret) { + gds_err("error %d in gds_mlx5_alloc_parent_domain\n", ret); + goto err; + } + + tx_gcq = gds_create_cq(context, qp_attr->cap.max_send_wr, NULL, NULL, 0, gpu_id, + (flags & GDS_CREATE_QP_TX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT); if (!tx_gcq) { ret = errno; - gds_err("error %d while creating TX CQ, old_errno=%d\n", ret, old_errno); + gds_err("error %d while creating TX CQ\n", ret); goto err; } - rx_gcq = gds_create_cq_internal(context, qp_attr->cap.max_recv_wr, NULL, NULL, 0, gpu_id, - (flags & GDS_CREATE_QP_RX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT, - gqp->res_domain); + rx_gcq = gds_create_cq(context, qp_attr->cap.max_recv_wr, NULL, NULL, 0, gpu_id, + (flags & GDS_CREATE_QP_RX_CQ_ON_GPU) ? GDS_ALLOC_CQ_ON_GPU : GDS_ALLOC_CQ_DEFAULT); if (!rx_gcq) { ret = errno; gds_err("error %d while creating RX CQ\n", ret); @@ -1863,97 +1351,68 @@ struct gds_qp *gds_create_qp(struct ibv_pd *pd, struct ibv_context *context, } // peer registration - qp_attr->send_cq = tx_gcq->cq; - qp_attr->recv_cq = rx_gcq->cq; - qp_attr->pd = pd; - qp_attr->comp_mask |= IBV_EXP_QP_INIT_ATTR_PD; + qp_attr->send_cq = tx_gcq->ibcq; + qp_attr->recv_cq = rx_gcq->ibcq; - peer->alloc_type = gds_peer::WQ; - peer->alloc_flags = GDS_ALLOC_WQ_DEFAULT | GDS_ALLOC_DBREC_DEFAULT; - if (flags & GDS_CREATE_QP_WQ_ON_GPU) { - gds_err("error, QP WQ on GPU is not supported yet\n"); + ibqp = ibv_create_qp(pd, qp_attr); + if (!ibqp) { + ret = EINVAL; + gds_err("error in ibv_create_qp\n"); goto err; } - if (flags & GDS_CREATE_QP_WQ_DBREC_ON_GPU) { - gds_warn("QP WQ DBREC on GPU\n"); - peer->alloc_flags |= GDS_ALLOC_DBREC_ON_GPU; - } - qp_attr->comp_mask |= IBV_EXP_QP_INIT_ATTR_PEER_DIRECT; - qp_attr->peer_direct_attrs = peer_attr; - qp = ibv_exp_create_qp(context, qp_attr); - if (!qp) { - ret = EINVAL; - gds_err("error in ibv_exp_create_qp\n"); + ret = gds_mlx5_create_qp(ibqp, qp_attr, to_gds_mcq(tx_gcq), to_gds_mcq(rx_gcq), qp_peer, &mqp); + if (ret) { + gds_err("error in gds_mlx5_create_qp\n"); goto err; } - gqp->qp = qp; - gqp->send_cq.cq = qp->send_cq; - gqp->send_cq.curr_offset = 0; - gqp->recv_cq.cq = qp->recv_cq; - gqp->recv_cq.curr_offset = 0; + gqp = &mqp->gqp; gds_dbg("created gds_qp=%p\n", gqp); return gqp; err: - gds_dbg("destroying QP\n"); - gds_destroy_qp(gqp); + if (qp_peer) + free(qp_peer); + + if (ibqp) + ibv_destroy_qp(ibqp); + + if (rx_gcq) + gds_destroy_cq(rx_gcq); + + if (tx_gcq) + gds_destroy_cq(tx_gcq); return NULL; } //----------------------------------------------------------------------------- -int gds_destroy_qp(struct gds_qp *gqp) +void gds_destroy_cq(gds_cq_t *gcq) { - int retcode = 0; - int ret; - - if(!gqp) return retcode; + gds_mlx5_cq_t *mcq; - if(gqp->qp) - { - ret = ibv_destroy_qp(gqp->qp); - if (ret) { - gds_err("error %d in destroy_qp\n", ret); - retcode = ret; - } - } + if (!gcq) + return; - if(gqp->send_cq.cq) - { - ret = ibv_destroy_cq(gqp->send_cq.cq); - if (ret) { - gds_err("error %d in destroy_cq send_cq\n", ret); - retcode = ret; - } - } + assert(gcq->dtype == GDS_DRIVER_TYPE_MLX5); + + gds_mlx5_destroy_cq(to_gds_mcq(gcq)); +} - if(gqp->recv_cq.cq) - { - ret = ibv_destroy_cq(gqp->recv_cq.cq); - if (ret) { - gds_err("error %d in destroy_cq recv_cq\n", ret); - retcode = ret; - } - } +//----------------------------------------------------------------------------- - if(gqp->res_domain) { - struct ibv_exp_destroy_res_domain_attr attr; //IBV_EXP_DESTROY_RES_DOMAIN_RESERVED - attr.comp_mask=0; - ret = ibv_exp_destroy_res_domain(gqp->dev_context, gqp->res_domain, &attr); - if (ret) { - gds_err("ibv_exp_destroy_res_domain error %d: %s\n", ret, strerror(ret)); - retcode = ret; - } - } +void gds_destroy_qp(gds_qp_t *gqp) +{ + if (!gqp) + return; - free(gqp); + assert(gqp->dtype == GDS_DRIVER_TYPE_MLX5); - return retcode; + gds_mlx5_destroy_qp(to_gds_mqp(gqp)); } //----------------------------------------------------------------------------- @@ -1965,12 +1424,12 @@ int gds_query_param(gds_param_t param, int *value) return EINVAL; switch (param) { - case GDS_PARAM_VERSION: - *value = (GDS_API_MAJOR_VERSION << 16)|GDS_API_MINOR_VERSION; - break; - default: - ret = EINVAL; - break; + case GDS_PARAM_VERSION: + *value = (GDS_API_MAJOR_VERSION << 16)|GDS_API_MINOR_VERSION; + break; + default: + ret = EINVAL; + break; }; return ret; } diff --git a/src/gdsync_debug_hostregister_bug.cpp b/src/gdsync_debug_hostregister_bug.cpp index 1e36d08..1d44c2b 100644 --- a/src/gdsync_debug_hostregister_bug.cpp +++ b/src/gdsync_debug_hostregister_bug.cpp @@ -262,36 +262,36 @@ void gds_dump_params(unsigned int nops, CUstreamBatchMemOpParams *params) static int gds_fill_membar(CUstreamBatchMemOpParams *param, int flags) { - int retcode = 0; + int retcode = 0; #if GDS_HAS_MEMBAR - if (flags & GDS_MEMBAR_FLUSH_REMOTE) { - param->operation = CU_STREAM_MEM_OP_FLUSH_REMOTE_WRITES; - param->flushRemoteWrites.flags = 0; - gds_dbg("op=%d flush_remote flags=%08x\n", - param->operation, - param->flushRemoteWrites.flags); + if (flags & GDS_MEMBAR_FLUSH_REMOTE) { + param->operation = CU_STREAM_MEM_OP_FLUSH_REMOTE_WRITES; + param->flushRemoteWrites.flags = 0; + gds_dbg("op=%d flush_remote flags=%08x\n", + param->operation, + param->flushRemoteWrites.flags); + } else { + if (flags & GDS_MEMBAR_DEFAULT) { + param->operation = CU_STREAM_MEM_OP_MEMORY_BARRIER; + param->memoryBarrier.flags = CU_STREAM_MEMORY_BARRIER_DEFAULT; + } else if (flags & GDS_MEMBAR_SYS) { + param->operation = CU_STREAM_MEM_OP_MEMORY_BARRIER; + param->memoryBarrier.flags = CU_STREAM_MEMORY_BARRIER_SYS; } else { - if (flags & GDS_MEMBAR_DEFAULT) { - param->operation = CU_STREAM_MEM_OP_MEMORY_BARRIER; - param->memoryBarrier.flags = CU_STREAM_MEMORY_BARRIER_DEFAULT; - } else if (flags & GDS_MEMBAR_SYS) { - param->operation = CU_STREAM_MEM_OP_MEMORY_BARRIER; - param->memoryBarrier.flags = CU_STREAM_MEMORY_BARRIER_SYS; - } else { - gds_err("error, unsupported membar\n"); - retcode = EINVAL; - goto out; - } - gds_dbg("op=%d membar flags=%08x\n", - param->operation, - param->memoryBarrier.flags); + gds_err("error, unsupported membar\n"); + retcode = EINVAL; + goto out; } + gds_dbg("op=%d membar flags=%08x\n", + param->operation, + param->memoryBarrier.flags); + } out: #else - gds_err("error, inline copy is unsupported\n"); - retcode = EINVAL; + gds_err("error, inline copy is unsupported\n"); + retcode = EINVAL; #endif - return retcode; + return retcode; } //----------------------------------------------------------------------------- @@ -677,242 +677,242 @@ enum { static int gds_post_ops(size_t n_ops, struct peer_op_wr *op, CUstreamBatchMemOpParams *params, int &idx, int post_flags = 0) { - int retcode = 0; - size_t n = 0; - bool prev_was_fence = false; - bool use_inlcpy_for_dword = false; - - gds_dbg("n_ops=%zu idx=%d\n", n_ops, idx); - - // divert the request to the same engine handling 64bits - // to avoid out-of-order execution - // caveat: can't use membar if inlcpy is used for 4B writes (to simulate 8B writes) - if (gds_enable_inlcpy()) { - if (!gds_enable_membar()) - use_inlcpy_for_dword = true; // F - } - if (gds_simulate_write64()) { - if (!gds_enable_membar()) { - gds_warn_once("enabling use_inlcpy_for_dword\n"); - use_inlcpy_for_dword = true; // D - } - } - - for (; op && n < n_ops; op = op->next, ++n) { - //int flags = 0; - gds_dbg("op[%zu] type:%08x\n", n, op->type); - switch(op->type) { - case IBV_PEER_OP_FENCE: { - gds_dbg("OP_FENCE: fence_flags=%"PRIu64"\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); - - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { - gds_dbg("nothing to do for read fences\n"); - //retcode = EINVAL; - break; - } - else { - if (!gds_enable_membar()) { - if (use_inlcpy_for_dword) { - assert(idx-1 >= 0); - gds_dbg("patching previous param\n"); - gds_enable_barrier_for_inlcpy(params+idx-1); - } - else { - gds_dbg("recording fence event\n"); - prev_was_fence = true; - } - //retcode = 0; - } - else { - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { - gds_err("unexpected from fence\n"); - retcode = EINVAL; - break; - } - int flags = 0; - if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { - gds_dbg("using light membar\n"); - flags = GDS_MEMBAR_DEFAULT; - } - else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { - gds_dbg("using heavy membar\n"); - flags = GDS_MEMBAR_SYS; - } - else { - gds_err("unsupported fence combination\n"); - retcode = EINVAL; - break; - } - retcode = gds_fill_membar(params+idx, flags); - ++idx; - } - } + int retcode = 0; + size_t n = 0; + bool prev_was_fence = false; + bool use_inlcpy_for_dword = false; + + gds_dbg("n_ops=%zu idx=%d\n", n_ops, idx); + + // divert the request to the same engine handling 64bits + // to avoid out-of-order execution + // caveat: can't use membar if inlcpy is used for 4B writes (to simulate 8B writes) + if (gds_enable_inlcpy()) { + if (!gds_enable_membar()) + use_inlcpy_for_dword = true; // F + } + if (gds_simulate_write64()) { + if (!gds_enable_membar()) { + gds_warn_once("enabling use_inlcpy_for_dword\n"); + use_inlcpy_for_dword = true; // D + } + } + + for (; op && n < n_ops; op = op->next, ++n) { + //int flags = 0; + gds_dbg("op[%zu] type:%08x\n", n, op->type); + switch(op->type) { + case IBV_PEER_OP_FENCE: { + gds_dbg("OP_FENCE: fence_flags=%"PRIu64"\n", op->wr.fence.fence_flags); + uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); + + if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { + gds_dbg("nothing to do for read fences\n"); + //retcode = EINVAL; break; - } - case IBV_PEER_OP_STORE_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - int flags = 0; - gds_dbg("OP_STORE_DWORD dev_ptr=%llx data=%"PRIx32"\n", dev_ptr, data); - if (use_inlcpy_for_dword) { // F || D - // membar may be out of order WRT inlcpy - if (gds_enable_membar()) { - gds_err("invalid feature combination, inlcpy + membar\n"); - retcode = EINVAL; - break; - } - // tail flush is set when following fence is met - // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; - retcode = gds_fill_inlcpy(params+idx, dev_ptr, &data, sizeof(data), flags); - ++idx; - } - else { // A || B || C || E - // can't guarantee ordering of write32+inlcpy unless - // a membar is there - // TODO: fix driver when !weak - if (gds_enable_inlcpy() && !gds_enable_membar()) { - gds_err("invalid feature combination, inlcpy needs membar\n"); - retcode = EINVAL; - break; - } - if (prev_was_fence) { - gds_dbg("using PRE_BARRIER as fence\n"); - flags |= GDS_POKE_POST_PRE_BARRIER; - prev_was_fence = false; - } - retcode = gds_fill_poke(params+idx, dev_ptr, data, flags); - ++idx; - } - break; - } - case IBV_PEER_OP_STORE_QWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + - op->wr.qword_va.offset; - uint64_t data = op->wr.qword_va.data; - int flags = 0; - gds_dbg("OP_STORE_QWORD dev_ptr=%llx data=%"PRIx64"\n", dev_ptr, data); - // C || D - if (gds_enable_write64()) { - gds_err("write64 is not supported\n"); - retcode = EINVAL; - break; - } - - // simulate 64-bit poke by inline copy - - if (gds_simulate_write64()){ - if (!gds_enable_membar()) { - gds_err("invalid feature combination, inlcpy needs membar\n"); - retcode = EINVAL; - break; - } - - // tail flush is never useful here - //flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; - retcode = gds_fill_inlcpy(params+idx, dev_ptr, &data, sizeof(data), flags); - ++idx; + } + else { + if (!gds_enable_membar()) { + if (use_inlcpy_for_dword) { + assert(idx-1 >= 0); + gds_dbg("patching previous param\n"); + gds_enable_barrier_for_inlcpy(params+idx-1); + } + else { + gds_dbg("recording fence event\n"); + prev_was_fence = true; + } + //retcode = 0; } else { - uint32_t datalo = gds_qword_lo(op->wr.qword_va.data); - uint32_t datahi = gds_qword_hi(op->wr.qword_va.data); - - if (prev_was_fence) { - gds_dbg("enabling PRE_BARRIER\n"); - flags |= GDS_POKE_POST_PRE_BARRIER; - prev_was_fence = false; - } - retcode = gds_fill_poke(params+idx, dev_ptr, datalo, flags); - ++idx; - - // get rid of the barrier, if there - flags &= ~GDS_POKE_POST_PRE_BARRIER; - - // advance to next DWORD - dev_ptr += sizeof(uint32_t); - retcode = gds_fill_poke(params+idx, dev_ptr, datahi, flags); - ++idx; - } - - break; - } - case IBV_PEER_OP_COPY_BLOCK: { - CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + - op->wr.copy_op.offset; - size_t len = op->wr.copy_op.len; - void *src = op->wr.copy_op.src; - int flags = 0; - gds_dbg("OP_COPY_BLOCK dev_ptr=%llx src=%p len=%zu\n", dev_ptr, src, len); - // catching any other size here - if (!gds_enable_inlcpy()) { - gds_err("inline copy is not supported\n"); + if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { + gds_err("unexpected from fence\n"); retcode = EINVAL; break; - } - // IB Verbs bug - assert(len <= GDS_GPU_MAX_INLINE_SIZE); - //if (desc->need_flush) { - // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; - //} - retcode = gds_fill_inlcpy(params+idx, dev_ptr, src, len, flags); - ++idx; - break; - } - case IBV_PEER_OP_POLL_AND_DWORD: - case IBV_PEER_OP_POLL_GEQ_DWORD: - case IBV_PEER_OP_POLL_NOR_DWORD: { - int poll_cond; - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - // TODO: properly handle a following fence instead of blidly flushing - int flags = 0; - if (!(post_flags & GDS_POST_OPS_DISCARD_WAIT_FLUSH)) - flags |= GDS_POLL_POST_FLUSH; - - gds_dbg("OP_POLL_DWORD dev_ptr=%llx data=%"PRIx32"\n", dev_ptr, data); - - switch(op->type) { - case IBV_PEER_OP_POLL_NOR_DWORD: - //poll_cond = GDS_POLL_COND_NOR; - // TODO: lookup and pass peer down - assert(gpu_does_support_nor(NULL)); - retcode = -EINVAL; - goto out; - break; - case IBV_PEER_OP_POLL_GEQ_DWORD: - poll_cond = GDS_POLL_COND_GEQ; - break; - case IBV_PEER_OP_POLL_AND_DWORD: - poll_cond = GDS_POLL_COND_AND; - break; - default: - assert(!"cannot happen"); + } + int flags = 0; + if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { + gds_dbg("using light membar\n"); + flags = GDS_MEMBAR_DEFAULT; + } + else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { + gds_dbg("using heavy membar\n"); + flags = GDS_MEMBAR_SYS; + } + else { + gds_err("unsupported fence combination\n"); retcode = EINVAL; - goto out; + break; + } + retcode = gds_fill_membar(params+idx, flags); + ++idx; } - retcode = gds_fill_poll(params+idx, dev_ptr, data, poll_cond, flags); - ++idx; - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - retcode = EINVAL; - break; - } - if (retcode) { - gds_err("error in fill func at entry n=%zu (idx=%d)\n", n, idx); - goto out; + } + break; } + case IBV_PEER_OP_STORE_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + int flags = 0; + gds_dbg("OP_STORE_DWORD dev_ptr=%llx data=%"PRIx32"\n", dev_ptr, data); + if (use_inlcpy_for_dword) { // F || D + // membar may be out of order WRT inlcpy + if (gds_enable_membar()) { + gds_err("invalid feature combination, inlcpy + membar\n"); + retcode = EINVAL; + break; + } + // tail flush is set when following fence is met + // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; + retcode = gds_fill_inlcpy(params+idx, dev_ptr, &data, sizeof(data), flags); + ++idx; + } + else { // A || B || C || E + // can't guarantee ordering of write32+inlcpy unless + // a membar is there + // TODO: fix driver when !weak + if (gds_enable_inlcpy() && !gds_enable_membar()) { + gds_err("invalid feature combination, inlcpy needs membar\n"); + retcode = EINVAL; + break; + } + if (prev_was_fence) { + gds_dbg("using PRE_BARRIER as fence\n"); + flags |= GDS_POKE_POST_PRE_BARRIER; + prev_was_fence = false; + } + retcode = gds_fill_poke(params+idx, dev_ptr, data, flags); + ++idx; + } + break; + } + case IBV_PEER_OP_STORE_QWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + + op->wr.qword_va.offset; + uint64_t data = op->wr.qword_va.data; + int flags = 0; + gds_dbg("OP_STORE_QWORD dev_ptr=%llx data=%"PRIx64"\n", dev_ptr, data); + // C || D + if (gds_enable_write64()) { + gds_err("write64 is not supported\n"); + retcode = EINVAL; + break; + } + + // simulate 64-bit poke by inline copy + + if (gds_simulate_write64()){ + if (!gds_enable_membar()) { + gds_err("invalid feature combination, inlcpy needs membar\n"); + retcode = EINVAL; + break; + } + + // tail flush is never useful here + //flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; + retcode = gds_fill_inlcpy(params+idx, dev_ptr, &data, sizeof(data), flags); + ++idx; + } + else { + uint32_t datalo = gds_qword_lo(op->wr.qword_va.data); + uint32_t datahi = gds_qword_hi(op->wr.qword_va.data); + + if (prev_was_fence) { + gds_dbg("enabling PRE_BARRIER\n"); + flags |= GDS_POKE_POST_PRE_BARRIER; + prev_was_fence = false; + } + retcode = gds_fill_poke(params+idx, dev_ptr, datalo, flags); + ++idx; + + // get rid of the barrier, if there + flags &= ~GDS_POKE_POST_PRE_BARRIER; + + // advance to next DWORD + dev_ptr += sizeof(uint32_t); + retcode = gds_fill_poke(params+idx, dev_ptr, datahi, flags); + ++idx; + } + + break; + } + case IBV_PEER_OP_COPY_BLOCK: { + CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + + op->wr.copy_op.offset; + size_t len = op->wr.copy_op.len; + void *src = op->wr.copy_op.src; + int flags = 0; + gds_dbg("OP_COPY_BLOCK dev_ptr=%llx src=%p len=%zu\n", dev_ptr, src, len); + // catching any other size here + if (!gds_enable_inlcpy()) { + gds_err("inline copy is not supported\n"); + retcode = EINVAL; + break; + } + // IB Verbs bug + assert(len <= GDS_GPU_MAX_INLINE_SIZE); + //if (desc->need_flush) { + // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; + //} + retcode = gds_fill_inlcpy(params+idx, dev_ptr, src, len, flags); + ++idx; + break; + } + case IBV_PEER_OP_POLL_AND_DWORD: + case IBV_PEER_OP_POLL_GEQ_DWORD: + case IBV_PEER_OP_POLL_NOR_DWORD: { + int poll_cond; + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + // TODO: properly handle a following fence instead of blidly flushing + int flags = 0; + if (!(post_flags & GDS_POST_OPS_DISCARD_WAIT_FLUSH)) + flags |= GDS_POLL_POST_FLUSH; + + gds_dbg("OP_POLL_DWORD dev_ptr=%llx data=%"PRIx32"\n", dev_ptr, data); + + switch(op->type) { + case IBV_PEER_OP_POLL_NOR_DWORD: + //poll_cond = GDS_POLL_COND_NOR; + // TODO: lookup and pass peer down + assert(gpu_does_support_nor(NULL)); + retcode = -EINVAL; + goto out; + break; + case IBV_PEER_OP_POLL_GEQ_DWORD: + poll_cond = GDS_POLL_COND_GEQ; + break; + case IBV_PEER_OP_POLL_AND_DWORD: + poll_cond = GDS_POLL_COND_AND; + break; + default: + assert(!"cannot happen"); + retcode = EINVAL; + goto out; + } + retcode = gds_fill_poll(params+idx, dev_ptr, data, poll_cond, flags); + ++idx; + break; + } + default: + gds_err("undefined peer op type %d\n", op->type); + retcode = EINVAL; + break; } + if (retcode) { + gds_err("error in fill func at entry n=%zu (idx=%d)\n", n, idx); + goto out; + } + } - assert(n_ops == n); + assert(n_ops == n); out: - return retcode; + return retcode; } //----------------------------------------------------------------------------- @@ -964,91 +964,91 @@ int gds_post_pokes(CUstream stream, int count, gds_send_request_t *info, uint32_ static int gds_post_ops_on_cpu(size_t n_descs, struct peer_op_wr *op) { - int retcode = 0; - size_t n = 0; - - for (; op && n < n_descs; op = op->next, ++n) { - //int flags = 0; - gds_dbg("op[%zu] type:%08x\n", n, op->type); - switch(op->type) { - case IBV_PEER_OP_FENCE: { - gds_dbg("fence_flags=%"PRIu64"\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); - - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { - gds_warnc(1, "nothing to do for read fences\n"); - //retcode = EINVAL; - break; - } - else { - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { - gds_err("unexpected from %08x fence, expected FROM_HCA\n", fence_from); - retcode = EINVAL; - break; - } - if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { - gds_dbg("using light membar\n"); - wmb(); - } - else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { - gds_dbg("using heavy membar\n"); - wmb(); - } - else { - gds_err("unsupported fence combination\n"); - retcode = EINVAL; - break; - } - } - break; - } - case IBV_PEER_OP_STORE_DWORD: { - uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); - uint32_t data = op->wr.dword_va.data; - // A || B || C || E - ACCESS_ONCE(*ptr) = data; - gds_dbg("%p <- %08x\n", ptr, data); - break; + int retcode = 0; + size_t n = 0; + + for (; op && n < n_descs; op = op->next, ++n) { + //int flags = 0; + gds_dbg("op[%zu] type:%08x\n", n, op->type); + switch(op->type) { + case IBV_PEER_OP_FENCE: { + gds_dbg("fence_flags=%"PRIu64"\n", op->wr.fence.fence_flags); + uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); + + if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { + gds_warnc(1, "nothing to do for read fences\n"); + //retcode = EINVAL; + break; } - case IBV_PEER_OP_STORE_QWORD: { - uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.qword_va.target_id)->va + op->wr.qword_va.offset); - uint64_t data = op->wr.qword_va.data; - ACCESS_ONCE(*ptr) = data; - gds_dbg("%p <- %016"PRIx64"\n", ptr, data); - break; - } - case IBV_PEER_OP_COPY_BLOCK: { - uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.copy_op.target_id)->va + op->wr.copy_op.offset); - uint64_t *src = (uint64_t*)op->wr.copy_op.src; - size_t n_bytes = op->wr.copy_op.len; - gds_bf_copy(ptr, src, n_bytes); - gds_dbg("%p <- %p len=%zu\n", ptr, src, n_bytes); - break; - } - case IBV_PEER_OP_POLL_AND_DWORD: - case IBV_PEER_OP_POLL_GEQ_DWORD: - case IBV_PEER_OP_POLL_NOR_DWORD: { - gds_err("polling is not supported\n"); + else { + if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { + gds_err("unexpected from %08x fence, expected FROM_HCA\n", fence_from); retcode = EINVAL; break; - } - default: - gds_err("undefined peer op type %d\n", op->type); + } + if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { + gds_dbg("using light membar\n"); + wmb(); + } + else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { + gds_dbg("using heavy membar\n"); + wmb(); + } + else { + gds_err("unsupported fence combination\n"); retcode = EINVAL; break; + } } - if (retcode) { - gds_err("error in fill func at entry n=%zu\n", n); - goto out; - } + break; + } + case IBV_PEER_OP_STORE_DWORD: { + uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); + uint32_t data = op->wr.dword_va.data; + // A || B || C || E + ACCESS_ONCE(*ptr) = data; + gds_dbg("%p <- %08x\n", ptr, data); + break; + } + case IBV_PEER_OP_STORE_QWORD: { + uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.qword_va.target_id)->va + op->wr.qword_va.offset); + uint64_t data = op->wr.qword_va.data; + ACCESS_ONCE(*ptr) = data; + gds_dbg("%p <- %016"PRIx64"\n", ptr, data); + break; + } + case IBV_PEER_OP_COPY_BLOCK: { + uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.copy_op.target_id)->va + op->wr.copy_op.offset); + uint64_t *src = (uint64_t*)op->wr.copy_op.src; + size_t n_bytes = op->wr.copy_op.len; + gds_bf_copy(ptr, src, n_bytes); + gds_dbg("%p <- %p len=%zu\n", ptr, src, n_bytes); + break; + } + case IBV_PEER_OP_POLL_AND_DWORD: + case IBV_PEER_OP_POLL_GEQ_DWORD: + case IBV_PEER_OP_POLL_NOR_DWORD: { + gds_err("polling is not supported\n"); + retcode = EINVAL; + break; + } + default: + gds_err("undefined peer op type %d\n", op->type); + retcode = EINVAL; + break; + } + if (retcode) { + gds_err("error in fill func at entry n=%zu\n", n); + goto out; } + } - assert(n_descs == n); + assert(n_descs == n); out: - return retcode; + return retcode; } //----------------------------------------------------------------------------- diff --git a/src/mem.cpp b/src/mem.cpp index 7b74652..53b98a1 100644 --- a/src/mem.cpp +++ b/src/mem.cpp @@ -40,7 +40,6 @@ using namespace std; #include -#include #include #include "gdsync.h" @@ -64,6 +63,36 @@ static gdr_t gdr = 0; //----------------------------------------------------------------------------- +static inline int is_gdr_mh_valid(gdr_mh_t mh) +{ + #if defined(GDR_API_MAJOR_VERSION) && (GDR_API_MAJOR_VERSION > 1) + return !!(mh.h); + #else + return !!(mh); + #endif +} + +//----------------------------------------------------------------------------- + +/** + * Check that the version of libgdrapi.so (linked at runtime) and that of + * gdrapi.h (used during compilation) are compatible. + * @return 0 if they are compatible; -1 otherwise + */ +static int check_gdr_compatibility() +{ + #if defined(GDR_API_MAJOR_VERSION) && (GDR_API_MAJOR_VERSION > 1) + int major, minor; + gdr_runtime_get_version(&major, &minor); + return major == GDR_API_MAJOR_VERSION ? 0 : -1; + #else + // gdrapi v1 does not provide a way to check the runtime version. + return 0; + #endif +} + +//----------------------------------------------------------------------------- + static int gds_map_gdr_memory(gds_mem_desc_t *desc, CUdeviceptr d_buf, size_t size, int flags) { gdr_mh_t mh; @@ -80,6 +109,10 @@ static int gds_map_gdr_memory(gds_mem_desc_t *desc, CUdeviceptr d_buf, size_t si assert(size); if (!gdr) { + if (check_gdr_compatibility()) { + gds_err("incompatible gdrapi versions used during compile time and runtime.\n"); + exit(EXIT_FAILURE); + } gdr = gdr_open(); if (!gdr) { gds_err("can't initialize GDRCopy library\n"); @@ -125,10 +158,10 @@ static int gds_map_gdr_memory(gds_mem_desc_t *desc, CUdeviceptr d_buf, size_t si desc->alloc_size = buf_size; desc->mh = mh; gds_dbg("d_ptr=%lx h_ptr=%p bar_ptr=%p flags=0x%08x alloc_size=%zd mh=%x\n", - (unsigned long)desc->d_ptr, desc->h_ptr, desc->bar_ptr, desc->flags, desc->alloc_size, desc->mh); + (unsigned long)desc->d_ptr, desc->h_ptr, desc->bar_ptr, desc->flags, desc->alloc_size, desc->mh); out: if (ret) { - if (mh) { + if (is_gdr_mh_valid(mh)) { if (bar_ptr) gdr_unmap(gdr, mh, bar_ptr, buf_size); gdr_unpin_buffer(gdr, mh); @@ -147,12 +180,12 @@ static int gds_unmap_gdr_memory(gds_mem_desc_t *desc) gds_err("GDRCopy library is not initialized\n"); exit(EXIT_FAILURE); } - if (!desc->d_ptr || !desc->h_ptr || !desc->alloc_size || !desc->mh || !desc->bar_ptr) { + if (!desc->d_ptr || !desc->h_ptr || !desc->alloc_size || !is_gdr_mh_valid(desc->mh) || !desc->bar_ptr) { gds_err("invalid desc\n"); return EINVAL; } gds_dbg("d_ptr=%lx h_ptr=%p alloc_size=%zd mh=%x\n", - (unsigned long)desc->d_ptr, desc->h_ptr, desc->alloc_size, desc->mh); + (unsigned long)desc->d_ptr, desc->h_ptr, desc->alloc_size, desc->mh); gdr_unmap(gdr, desc->mh, desc->bar_ptr, desc->alloc_size); gdr_unpin_buffer(gdr, desc->mh); return ret; @@ -163,20 +196,26 @@ static int gds_unmap_gdr_memory(gds_mem_desc_t *desc) static int gds_alloc_gdr_memory(gds_mem_desc_t *desc, size_t size, int flags) { CUdeviceptr d_buf = 0; - size_t buf_size = size; + CUdeviceptr d_buf_aligned = 0; + size_t buf_size = size + GDS_GPU_PAGE_SIZE - 1; int ret = 0; assert(desc); CUCHECK(cuMemAlloc(&d_buf, buf_size)); - gds_dbg("allocated GPU polling buffer d_buf=%p\n", (void*)d_buf); - //CUCHECK(cuMemsetD8(d_buf, 0, buf_size)); - ret = gds_map_gdr_memory(desc, d_buf, buf_size, flags); + d_buf_aligned = (d_buf + GDS_GPU_PAGE_SIZE - 1) & GDS_GPU_PAGE_MASK; + + gds_dbg("allocated GPU polling buffer d_buf=0x%llx req_size=%zu d_buf_aligned=0x%llx buf_size=%zu\n", d_buf, size, d_buf_aligned, buf_size); + + ret = gds_map_gdr_memory(desc, d_buf_aligned, size, flags); if (ret) { gds_err("error %d while mapping gdr memory\n", ret); CUCHECK(cuMemFree(d_buf)); + return ret; } + + desc->original_d_ptr = d_buf; return ret; } @@ -186,19 +225,19 @@ static int gds_free_gdr_memory(gds_mem_desc_t *desc) { int ret = 0; assert(desc); - if (!desc->d_ptr || !desc->h_ptr || !desc->alloc_size || !desc->mh || !desc->bar_ptr) { + if (!desc->d_ptr || !desc->h_ptr || !desc->alloc_size || !is_gdr_mh_valid(desc->mh) || !desc->bar_ptr) { gds_err("invalid desc\n"); return EINVAL; } gds_dbg("d_ptr=%lx h_ptr=%p alloc_size=%zd mh=%x\n", - (unsigned long)desc->d_ptr, desc->h_ptr, desc->alloc_size, desc->mh); + (unsigned long)desc->d_ptr, desc->h_ptr, desc->alloc_size, desc->mh); ret = gds_unmap_gdr_memory(desc); if (ret) { gds_err("error %d while unmapping gdr, going on anyway\n", ret); } - CUCHECK(cuMemFree(desc->d_ptr)); + CUCHECK(cuMemFree(desc->original_d_ptr)); return ret; } @@ -243,9 +282,9 @@ static int gds_alloc_pinned_memory(gds_mem_desc_t *desc, size_t size, int flags) desc->bar_ptr = NULL; desc->flags = flags; desc->alloc_size = size; - desc->mh = 0; + memset(&desc->mh, 0, sizeof(gdr_mh_t)); gds_dbg("d_ptr=%lx h_ptr=%p flags=0x%08x alloc_size=%zd\n", - (unsigned long)desc->d_ptr, desc->h_ptr, desc->flags, desc->alloc_size); + (unsigned long)desc->d_ptr, desc->h_ptr, desc->flags, desc->alloc_size); out: if (ret) { if (desc->h_ptr) { @@ -272,7 +311,7 @@ static int gds_free_pinned_memory(gds_mem_desc_t *desc) // BUG: TBD #else gds_dbg("d_ptr=%lx h_ptr=%p flags=0x%08x alloc_size=%zd\n", - (unsigned long)desc->d_ptr, desc->h_ptr, desc->flags, desc->alloc_size); + (unsigned long)desc->d_ptr, desc->h_ptr, desc->flags, desc->alloc_size); ret = gds_unregister_mem(desc->h_ptr, desc->alloc_size); free(desc->h_ptr); desc->h_ptr = NULL; @@ -296,16 +335,16 @@ int gds_alloc_mapped_memory(gds_mem_desc_t *desc, size_t size, int flags) return EINVAL; } switch(flags & GDS_MEMORY_MASK) { - case GDS_MEMORY_GPU: - ret = gds_alloc_gdr_memory(desc, size, flags); - break; - case GDS_MEMORY_HOST: - ret = gds_alloc_pinned_memory(desc, size, flags); - break; - default: - gds_err("invalid flags\n"); - ret = EINVAL; - break; + case GDS_MEMORY_GPU: + ret = gds_alloc_gdr_memory(desc, size, flags); + break; + case GDS_MEMORY_HOST: + ret = gds_alloc_pinned_memory(desc, size, flags); + break; + default: + gds_err("invalid flags\n"); + ret = EINVAL; + break; } return ret; } @@ -320,15 +359,15 @@ int gds_free_mapped_memory(gds_mem_desc_t *desc) return EINVAL; } switch(desc->flags & GDS_MEMORY_MASK) { - case GDS_MEMORY_GPU: - ret = gds_free_gdr_memory(desc); - break; - case GDS_MEMORY_HOST: - ret = gds_free_pinned_memory(desc); - break; - default: - ret = EINVAL; - break; + case GDS_MEMORY_GPU: + ret = gds_free_gdr_memory(desc); + break; + case GDS_MEMORY_HOST: + ret = gds_free_pinned_memory(desc); + break; + default: + ret = EINVAL; + break; } return ret; } @@ -336,10 +375,9 @@ int gds_free_mapped_memory(gds_mem_desc_t *desc) //----------------------------------------------------------------------------- #define ROUND_TO(V,PS) ((((V) + (PS) - 1)/(PS)) * (PS)) -//#define ROUND_TO_GDR_GPU_PAGE(V) ROUND_TO(V, GDR_GPU_PAGE_SIZE) // allocate GPU memory with a GDR mapping (CPU can dereference it) -int gds_peer_malloc_ex(int peer_id, uint64_t peer_data, void **host_addr, CUdeviceptr *peer_addr, size_t req_size, void **phandle, bool has_cpu_mapping) +int gds_peer_malloc_ex(int peer_id, uint64_t peer_data, void **host_addr, CUdeviceptr *peer_addr, size_t req_size, void **phandle, gds_memory_type_t mem_type, bool has_cpu_mapping) { int ret = 0; // assume GPUs are the only peers!!! @@ -392,7 +430,7 @@ int gds_peer_malloc_ex(int peer_id, uint64_t peer_data, void **host_addr, CUdevi goto out; } - ret = gds_alloc_mapped_memory(desc, size, GDS_MEMORY_GPU); + ret = gds_alloc_mapped_memory(desc, size, mem_type); if (ret) { gds_err("error %d while allocating mapped GPU buffers\n", ret); goto out; @@ -414,9 +452,9 @@ int gds_peer_malloc_ex(int peer_id, uint64_t peer_data, void **host_addr, CUdevi //----------------------------------------------------------------------------- -int gds_peer_malloc(int peer_id, uint64_t peer_data, void **host_addr, CUdeviceptr *peer_addr, size_t req_size, void **phandle) +int gds_peer_malloc(int peer_id, uint64_t peer_data, void **host_addr, CUdeviceptr *peer_addr, size_t req_size, void **phandle, gds_memory_type_t mem_type) { - return gds_peer_malloc_ex(peer_id, peer_data, host_addr, peer_addr, req_size, phandle, true); + return gds_peer_malloc_ex(peer_id, peer_data, host_addr, peer_addr, req_size, phandle, mem_type, true); } //----------------------------------------------------------------------------- diff --git a/src/mem.hpp b/src/mem.hpp index 639f9a1..6b1ba4c 100644 --- a/src/mem.hpp +++ b/src/mem.hpp @@ -1,8 +1,8 @@ #pragma once int gds_peer_mfree(int peer_id, uint64_t peer_data, void *host_addr, void *handle); -int gds_peer_malloc(int peer_id, uint64_t peer_data, void **host_addr, CUdeviceptr *peer_addr, size_t req_size, void **phandle); -int gds_peer_malloc_ex(int peer_id, uint64_t peer_data, void **host_addr, CUdeviceptr *peer_addr, size_t req_size, void **phandle, bool has_cpu_mapping); +int gds_peer_malloc(int peer_id, uint64_t peer_data, void **host_addr, CUdeviceptr *peer_addr, size_t req_size, void **phandle, gds_memory_type_t mem_type); +int gds_peer_malloc_ex(int peer_id, uint64_t peer_data, void **host_addr, CUdeviceptr *peer_addr, size_t req_size, void **phandle, gds_memory_type_t mem_type, bool has_cpu_mapping); diff --git a/src/memmgr.cpp b/src/memmgr.cpp index ab3e490..7a4b2bc 100644 --- a/src/memmgr.cpp +++ b/src/memmgr.cpp @@ -40,7 +40,6 @@ using namespace std; #include -#include #include #include "gdsync.h" @@ -95,31 +94,31 @@ int gds_map_mem(void *ptr, size_t size, gds_memory_type_t mem_type, CUdeviceptr range_set::find_result res = rset.find(r); switch(res.second) { - case range_set::not_found: - return gds_register_mem_internal(ptr, size, mem_type, dev_ptr); - break; - case range_set::partial_overlap: - gds_err("partial overlap, buffer already registered?\n"); - return EINVAL; - case range_set::fully_contained: { - range r = *res.first; - if (dev_ptr) { - pindown_cache_t::iterator found = pinned_ranges.find(r.first); - if (found != pinned_ranges.end()) { - CUdeviceptr page_dev_ptr = (*found).second; - ptrdiff_t off = (ptrdiff_t)ptr - (ptrdiff_t)r.first; - *dev_ptr = page_dev_ptr + off; - } - else { - gds_err("can't find dev_ptr for page_addr=%lx\n", r.first); - return EINVAL; + case range_set::not_found: + return gds_register_mem_internal(ptr, size, mem_type, dev_ptr); + break; + case range_set::partial_overlap: + gds_err("partial overlap, buffer already registered?\n"); + return EINVAL; + case range_set::fully_contained: { + range r = *res.first; + if (dev_ptr) { + pindown_cache_t::iterator found = pinned_ranges.find(r.first); + if (found != pinned_ranges.end()) { + CUdeviceptr page_dev_ptr = (*found).second; + ptrdiff_t off = (ptrdiff_t)ptr - (ptrdiff_t)r.first; + *dev_ptr = page_dev_ptr + off; + } + else { + gds_err("can't find dev_ptr for page_addr=%lx\n", r.first); + return EINVAL; + } } + break; } - break; - } - default: - gds_err("unexpected result"); - return EINVAL; + default: + gds_err("unexpected result"); + return EINVAL; } return 0; @@ -149,24 +148,24 @@ int gds_register_mem_internal(void *ptr, size_t size, gds_memory_type_t type, CU unsigned long target_page_size = 0; switch (type) { - case GDS_MEMORY_GPU: - gds_dbg("this is GPU memory, no CUDA registration required\n"); - need_cuda_registration = false; - target_page_mask = GDS_GPU_PAGE_MASK; - target_page_off = GDS_GPU_PAGE_OFF; - target_page_size = GDS_GPU_PAGE_SIZE; - break; - case GDS_MEMORY_IO: - flags |= CU_MEMHOSTREGISTER_IOMEMORY; - // fall through - case GDS_MEMORY_HOST: - target_page_mask = GDS_HOST_PAGE_MASK; - target_page_off = GDS_HOST_PAGE_OFF; - target_page_size = GDS_HOST_PAGE_SIZE; - break; - default: - gds_err("invalid mem type %d\n", type); - return EINVAL; + case GDS_MEMORY_GPU: + gds_dbg("this is GPU memory, no CUDA registration required\n"); + need_cuda_registration = false; + target_page_mask = GDS_GPU_PAGE_MASK; + target_page_off = GDS_GPU_PAGE_OFF; + target_page_size = GDS_GPU_PAGE_SIZE; + break; + case GDS_MEMORY_IO: + flags |= CU_MEMHOSTREGISTER_IOMEMORY; + // fall through + case GDS_MEMORY_HOST: + target_page_mask = GDS_HOST_PAGE_MASK; + target_page_off = GDS_HOST_PAGE_OFF; + target_page_size = GDS_HOST_PAGE_SIZE; + break; + default: + gds_err("invalid mem type %d\n", type); + return EINVAL; } unsigned long page_addr = addr & target_page_mask; @@ -180,7 +179,7 @@ int gds_register_mem_internal(void *ptr, size_t size, gds_memory_type_t type, CU // we are good here } else if ((res == CUDA_ERROR_HOST_MEMORY_ALREADY_REGISTERED) || - (res == CUDA_ERROR_ALREADY_MAPPED)) { + (res == CUDA_ERROR_ALREADY_MAPPED)) { const char *err_str = NULL; cuGetErrorString(res, &err_str); // older CUDA driver versions seem to return CUDA_ERROR_ALREADY_MAPPED @@ -192,11 +191,10 @@ int gds_register_mem_internal(void *ptr, size_t size, gds_memory_type_t type, CU return EAGAIN; } else { - //CUCHECK(res); const char *err_str = NULL; cuGetErrorString(res, &err_str); gds_err("Error %d (%s) while register address=%p size=%zu (original size %zu) flags=%08x\n", - res, err_str, (void*)page_addr, len, size, flags); + res, err_str, (void*)page_addr, len, size, flags); // TODO: handle ENOPERM return EINVAL; } @@ -246,127 +244,6 @@ int gds_unregister_mem(void *ptr, size_t size) return 0; } -//----------------------------------------------------------------------------- - -#if 0 -int gds_mem_devptr(void *va, size_t n_bytes, CUdeviceptr *pdev_ptr) -{ - gds_err("not supported anymore\n"); - return EINVAL; -} -#endif - -//----------------------------------------------------------------------------- -#if 0 - -#if 0 - char *ptr = (char*)_ptr; - char *p = ptr; - bool new_reg = false; - bool first_page = true; - - if (!size) { - gds_err("invalid 0 size buffer\n"); - return EINVAL; - } - - while (size) { - unsigned long addr = (unsigned long)p; - unsigned long page_addr = addr & GDS_HOST_PAGE_MASK; - unsigned long off = addr & GDS_HOST_PAGE_OFF; - unsigned long len = min((GDS_HOST_PAGE_SIZE - off), (unsigned long long)size); - CUdeviceptr page_dev_ptr = 0; - //gds_dbg("page_addr=%lx len=%lu\n", page_addr, len); - if (last_pinned.page_addr == page_addr) { - gds_dbg("hit last_pinned cache\n"); - page_dev_ptr = last_pinned.dev_addr; - } else { - //gds_dbg("traversing map\n"); - pindown_cache_t::iterator found = pinned_pages.find(page_addr); - if (found != pinned_pages.end()) { - page_dev_ptr = found->second; - } else { - unsigned int flags = CU_MEMHOSTREGISTER_DEVICEMAP | CU_MEMHOSTREGISTER_PORTABLE; - if (is_iomem) - flags |= CU_MEMHOSTREGISTER_IOMEMORY; - gds_dbg("registering page_addr=%lx iomem=%d\n", page_addr, is_iomem); - CUresult res = cuMemHostRegister((void*)page_addr, GDS_HOST_PAGE_SIZE, flags); - if (res == CUDA_SUCCESS) { - } else if ((res == CUDA_ERROR_HOST_MEMORY_ALREADY_REGISTERED) || - (res == CUDA_ERROR_ALREADY_MAPPED)) { - gds_warn("page=%p size=%llu is already registered with CUDA\n", (void*)page_addr, GDS_HOST_PAGE_SIZE); - } else { - //CUCHECK(res); - const char *err_str = NULL; - cuGetErrorString(res, &err_str); - gds_err("Error '%s' while register address=%p size=%llu flags=%08x\n", - err_str, (void*)page_addr, GDS_HOST_PAGE_SIZE, flags); - // TODO: handle ENOPERM - return EINVAL; - } - CUCHECK(cuMemHostGetDevicePointer(&page_dev_ptr, (void *)page_addr, 0)); - gds_dbg("page_ptr=%lx page_dev_ptr=%lx\n", page_addr, (unsigned long)page_dev_ptr); - pinned_pages[page_addr] = page_dev_ptr; - new_reg = true; - } - last_pinned.page_addr = page_addr; - last_pinned.dev_addr = page_dev_ptr; - } - if (first_page) { - first_page = 0; - *dev_ptr = (CUdeviceptr) (page_dev_ptr + off); - } - size -= min(len, size); - p += len; - } -#if 0 - // consistency check - { - CUdeviceptr my_dev_ptr; - CUCHECK(cuMemHostGetDevicePointer(&my_dev_ptr, _ptr, 0)); - assert(my_dev_ptr == *dev_ptr); - } -#endif - return 0; -#endif - -int gds_lookup_devptr(void *va, CUdeviceptr *dev_ptr) -{ - int retcode = EINVAL; - assert(dev_ptr); - - unsigned long addr = (unsigned long)va; - unsigned long page_addr = addr & GDS_HOST_PAGE_MASK; - unsigned long off = addr & GDS_HOST_PAGE_OFF; - CUdeviceptr page_dev_ptr = 0; - - //gds_dbg("page_addr=%lx len=%lu\n", page_addr, len); - - if (last_pinned.page_addr == page_addr) { - gds_dbg("hit last_pinned cache\n"); - page_dev_ptr = last_pinned.dev_addr; - } else { - //gds_dbg("traversing map\n"); - pindown_cache_t::iterator found = pinned_pages.find(page_addr); - if (found != pinned_pages.end()) { - page_dev_ptr = found->second; - } - } - if (page_dev_ptr) { - *dev_ptr = (CUdeviceptr) (page_dev_ptr + off); - retcode = 0; - } - - return retcode; - -} - -int gds_unmap_mem(void *_ptr, size_t size) -{ - gds_err("not supported anymore\n"); - return EINVAL; -} -#endif //----------------------------------------------------------------------------- /* * Local variables: diff --git a/src/memmgr.hpp b/src/memmgr.hpp index 007598e..6520182 100644 --- a/src/memmgr.hpp +++ b/src/memmgr.hpp @@ -2,13 +2,9 @@ // lookup dev ptr, potentially registering it first with gds_register_mem int gds_map_mem(void *_ptr, size_t size, gds_memory_type_t mem_type, CUdeviceptr *dev_ptr); -// lookup dev ptr, without registering it -//int gds_mem_devptr(void *va, size_t n_bytes, CUdeviceptr *pdev_ptr); // 1st time registration of memory, for HOST and IO int gds_register_mem(void *_ptr, size_t size, gds_memory_type_t type, CUdeviceptr *dev_ptr); int gds_unregister_mem(void *_ptr, size_t size); -//int gds_lookup_devptr(void *va, CUdeviceptr *dev_ptr); -//int gds_unmap_mem(void *_ptr, size_t size); /* * Local variables: diff --git a/src/mlnxutils.h b/src/mlnxutils.h index ffb6f83..964365b 100644 --- a/src/mlnxutils.h +++ b/src/mlnxutils.h @@ -41,28 +41,28 @@ */ #if defined(__x86_64__) #define COPY_64B_NT(dst, src) \ - __asm__ __volatile__ ( \ - " movdqa (%1),%%xmm0\n" \ - " movdqa 16(%1),%%xmm1\n" \ - " movdqa 32(%1),%%xmm2\n" \ - " movdqa 48(%1),%%xmm3\n" \ - " movntdq %%xmm0, (%0)\n" \ - " movntdq %%xmm1, 16(%0)\n" \ - " movntdq %%xmm2, 32(%0)\n" \ - " movntdq %%xmm3, 48(%0)\n" \ - : : "r" (dst), "r" (src) : "memory"); \ - dst += 8; \ - src += 8 + __asm__ __volatile__ ( \ + " movdqa (%1),%%xmm0\n" \ + " movdqa 16(%1),%%xmm1\n" \ + " movdqa 32(%1),%%xmm2\n" \ + " movdqa 48(%1),%%xmm3\n" \ + " movntdq %%xmm0, (%0)\n" \ + " movntdq %%xmm1, 16(%0)\n" \ + " movntdq %%xmm2, 32(%0)\n" \ + " movntdq %%xmm3, 48(%0)\n" \ + : : "r" (dst), "r" (src) : "memory"); \ + dst += 8; \ + src += 8 #else #define COPY_64B_NT(dst, src) \ - *dst++ = *src++; \ - *dst++ = *src++; \ - *dst++ = *src++; \ - *dst++ = *src++; \ - *dst++ = *src++; \ - *dst++ = *src++; \ - *dst++ = *src++; \ - *dst++ = *src++ + *dst++ = *src++; \ + *dst++ = *src++; \ + *dst++ = *src++; \ + *dst++ = *src++; \ + *dst++ = *src++; \ + *dst++ = *src++; \ + *dst++ = *src++; \ + *dst++ = *src++ #endif // no WQ wrap-around check!!! @@ -70,10 +70,10 @@ static inline void gds_bf_copy(uint64_t *dest, uint64_t *src, size_t n_bytes) { assert(n_bytes % sizeof(uint64_t) == 0); assert(n_bytes < 128); - while (n_bytes > 0) { - COPY_64B_NT(dest, src); - n_bytes -= 8 * sizeof(*dest); - } + while (n_bytes > 0) { + COPY_64B_NT(dest, src); + n_bytes -= 8 * sizeof(*dest); + } } diff --git a/src/mlx5.cpp b/src/mlx5.cpp index a2c7b39..7fca5db 100644 --- a/src/mlx5.cpp +++ b/src/mlx5.cpp @@ -32,119 +32,167 @@ #include #include #include +#include +#include +#include #include "gdsync.h" #include "gdsync/mlx5.h" -#include "utils.hpp" + +#include "archutils.h" #include "memmgr.hpp" -//#include "mem.hpp" +#include "mlx5.hpp" +#include "mlnxutils.h" #include "objs.hpp" #include "utils.hpp" -#if 0 - union { uint64_t qw; uint32_t dw[2]; } db_val; - db_val.qw = 0; - db_val.dw[0] = desc->data32; - db_val.dw[1] = desc->data32; - mlx5_i->db_value = db_val.qw; -#endif +//----------------------------------------------------------------------------- + +#define MLX5_ATOMIC_SIZE 8 + +static const uint32_t mlx5_ib_opcode[] = { + [IBV_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE, + [IBV_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM, + [IBV_WR_SEND] = MLX5_OPCODE_SEND, + [IBV_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM, + [IBV_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ, + [IBV_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS, + [IBV_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA, + [IBV_WR_LOCAL_INV] = MLX5_OPCODE_UMR, + [IBV_WR_BIND_MW] = MLX5_OPCODE_UMR, + [IBV_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL, + [IBV_WR_TSO] = MLX5_OPCODE_TSO, + [IBV_WR_DRIVER1] = MLX5_OPCODE_UMR, +}; + +struct mlx5_wqe_xrc_seg { + __be32 xrc_srqn; + uint8_t rsvd[12]; +}; + +struct mlx5_sg_copy_ptr { + int index; + int offset; +}; + +struct mlx5_wqe_eth_pad { + uint8_t rsvd0[16]; +}; + +struct mlx5_wqe_umr_data_seg { + union { + struct mlx5_wqe_umr_klm_seg klm; + uint8_t reserved[64]; + }; +}; + +struct mlx5_wqe_inline_seg { + __be32 byte_count; +}; + +enum { + MLX5_IPOIB_INLINE_MIN_HEADER_SIZE = 4, + MLX5_SOURCE_QPN_INLINE_MAX_HEADER_SIZE = 18, + MLX5_ETH_L2_INLINE_HEADER_SIZE = 18, + MLX5_ETH_L2_MIN_HEADER_SIZE = 14, +}; //----------------------------------------------------------------------------- -int gds_mlx5_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_send_request_t *request) +int gds_mlx5_get_send_descs(gds_mlx5_send_info_t *mlx5_i, const gds_mlx5_send_request_t *request) { int retcode = 0; size_t n_ops = request->commit.entries; - peer_op_wr *op = request->commit.storage; + gds_mlx5_peer_op_wr *op = request->commit.storage; size_t n = 0; memset(mlx5_i, 0, sizeof(*mlx5_i)); for (; op && n < n_ops; op = op->next, ++n) { switch(op->type) { - case IBV_EXP_PEER_OP_FENCE: { - gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { - gds_dbg("nothing to do for read fences\n"); + case GDS_MLX5_PEER_OP_FENCE: { + gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); + if (fence_op == GDS_PEER_FENCE_OP_READ) { + gds_dbg("nothing to do for read fences\n"); + break; + } + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { + gds_err("unexpected from fence\n"); + retcode = EINVAL; + break; + } + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { + gds_dbg("using light membar\n"); + mlx5_i->membar = 1; + } + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { + gds_dbg("using heavy membar\n"); + mlx5_i->membar_full = 1; + } + else { + gds_err("unsupported fence combination\n"); + retcode = EINVAL; + break; + } break; } - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { - gds_err("unexpected from fence\n"); - retcode = EINVAL; + case GDS_MLX5_PEER_OP_STORE_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + gds_dbg("OP_STORE_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); + if (n != 0) { + gds_err("store DWORD is not 1st op\n"); + retcode = EINVAL; + break; + } + mlx5_i->dbrec_ptr = (uint32_t*)dev_ptr; + mlx5_i->dbrec_value = data; break; } - if (fence_mem == IBV_EXP_PEER_FENCE_MEM_PEER) { - gds_dbg("using light membar\n"); - mlx5_i->membar = 1; - } - else if (fence_mem == IBV_EXP_PEER_FENCE_MEM_SYS) { - gds_dbg("using heavy membar\n"); - mlx5_i->membar_full = 1; - } - else { - gds_err("unsupported fence combination\n"); - retcode = EINVAL; + case GDS_MLX5_PEER_OP_STORE_QWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + + op->wr.qword_va.offset; + uint64_t data = op->wr.qword_va.data; + gds_dbg("OP_STORE_QWORD dev_ptr=%" PRIx64 " data=%" PRIx64 "\n", (uint64_t)dev_ptr, (uint64_t)data); + if (n != 2) { + gds_err("store QWORD is not 3rd op\n"); + retcode = EINVAL; + break; + } + mlx5_i->db_ptr = (uint64_t*)dev_ptr; + mlx5_i->db_value = data; break; } - break; - } - case IBV_EXP_PEER_OP_STORE_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - gds_dbg("OP_STORE_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); - if (n != 0) { - gds_err("store DWORD is not 1st op\n"); - retcode = EINVAL; + case GDS_MLX5_PEER_OP_COPY_BLOCK: { + CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + + op->wr.copy_op.offset; + size_t len = op->wr.copy_op.len; + void *src = op->wr.copy_op.src; + gds_dbg("send inline detected\n"); + if (len < 8 || len > 64) { + gds_err("unexpected len %zu\n", len); + retcode = EINVAL; + break; + } + mlx5_i->db_ptr = (uint64_t*)dev_ptr; + mlx5_i->db_value = *(uint64_t*)src; break; } - mlx5_i->dbrec_ptr = (uint32_t*)dev_ptr; - mlx5_i->dbrec_value = data; - break; - } - case IBV_EXP_PEER_OP_STORE_QWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + - op->wr.qword_va.offset; - uint64_t data = op->wr.qword_va.data; - gds_dbg("OP_STORE_QWORD dev_ptr=%" PRIx64 " data=%" PRIx64 "\n", (uint64_t)dev_ptr, (uint64_t)data); - if (n != 2) { - gds_err("store QWORD is not 3rd op\n"); + case GDS_MLX5_PEER_OP_POLL_AND_DWORD: + case GDS_MLX5_PEER_OP_POLL_GEQ_DWORD: + case GDS_MLX5_PEER_OP_POLL_NOR_DWORD: { + gds_err("unexpected polling op in send request\n"); retcode = EINVAL; break; } - mlx5_i->db_ptr = (uint64_t*)dev_ptr; - mlx5_i->db_value = data; - break; - } - case IBV_EXP_PEER_OP_COPY_BLOCK: { - CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + - op->wr.copy_op.offset; - size_t len = op->wr.copy_op.len; - void *src = op->wr.copy_op.src; - gds_dbg("send inline detected\n"); - if (len < 8 || len > 64) { - gds_err("unexpected len %zu\n", len); + default: + gds_err("undefined peer op type %d\n", op->type); retcode = EINVAL; break; - } - mlx5_i->db_ptr = (uint64_t*)dev_ptr; - mlx5_i->db_value = *(uint64_t*)src; - break; - } - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { - gds_err("unexpected polling op in send request\n"); - retcode = EINVAL; - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - retcode = EINVAL; - break; } if (retcode) { @@ -161,120 +209,120 @@ int gds_mlx5_get_send_info(int count, const gds_send_request_t *requests, gds_ml { int retcode = 0; - for (int j=0; jdbrec_ptr, mlx5_i->dbrec_value, mlx5_i->db_ptr, mlx5_i->db_value); - } + mlx5_i->dbrec_ptr, mlx5_i->dbrec_value, mlx5_i->db_ptr, mlx5_i->db_value); + } - return retcode; + return retcode; } //----------------------------------------------------------------------------- -int gds_mlx5_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_wait_request_t *request) +int gds_mlx5_get_wait_descs(gds_mlx5_wait_info_t *mlx5_i, const gds_mlx5_wait_request_t *request) { int retcode = 0; size_t n_ops = request->peek.entries; - peer_op_wr *op = request->peek.storage; + gds_mlx5_peer_op_wr *op = request->peek.storage; size_t n = 0; memset(mlx5_i, 0, sizeof(*mlx5_i)); for (; op && n < n_ops; op = op->next, ++n) { switch(op->type) { - case IBV_EXP_PEER_OP_FENCE: { - gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); - uint32_t fence_op = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_OP_READ|IBV_EXP_PEER_FENCE_OP_WRITE)); - uint32_t fence_from = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_FROM_CPU|IBV_EXP_PEER_FENCE_FROM_HCA)); - uint32_t fence_mem = (op->wr.fence.fence_flags & (IBV_EXP_PEER_FENCE_MEM_SYS|IBV_EXP_PEER_FENCE_MEM_PEER)); - if (fence_op == IBV_EXP_PEER_FENCE_OP_READ) { - gds_dbg("nothing to do for read fences\n"); + case GDS_MLX5_PEER_OP_FENCE: { + gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); + if (fence_op == GDS_PEER_FENCE_OP_READ) { + gds_dbg("nothing to do for read fences\n"); + break; + } + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { + gds_err("unexpected from fence\n"); + retcode = EINVAL; + break; + } + gds_err("unsupported fence combination\n"); + retcode = EINVAL; break; } - if (fence_from != IBV_EXP_PEER_FENCE_FROM_HCA) { - gds_err("unexpected from fence\n"); + case GDS_MLX5_PEER_OP_STORE_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + gds_dbg("OP_STORE_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); + if (n != 1) { + gds_err("store DWORD is not 2nd op\n"); + retcode = EINVAL; + break; + } + mlx5_i->flag_ptr = (uint32_t*)dev_ptr; + mlx5_i->flag_value = data; + break; + } + case GDS_MLX5_PEER_OP_STORE_QWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + + op->wr.qword_va.offset; + uint64_t data = op->wr.qword_va.data; + gds_dbg("OP_STORE_QWORD dev_ptr=%" PRIx64 " data=%" PRIx64 "\n", (uint64_t)dev_ptr, (uint64_t)data); + gds_err("unsupported QWORD op\n"); retcode = EINVAL; break; } - gds_err("unsupported fence combination\n"); - retcode = EINVAL; - break; - } - case IBV_EXP_PEER_OP_STORE_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; - gds_dbg("OP_STORE_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); - if (n != 1) { - gds_err("store DWORD is not 2nd op\n"); + case GDS_MLX5_PEER_OP_COPY_BLOCK: { + CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + + op->wr.copy_op.offset; + size_t len = op->wr.copy_op.len; + void *src = op->wr.copy_op.src; + gds_err("unsupported COPY_BLOCK\n"); retcode = EINVAL; break; } - mlx5_i->flag_ptr = (uint32_t*)dev_ptr; - mlx5_i->flag_value = data; - break; - } - case IBV_EXP_PEER_OP_STORE_QWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + - op->wr.qword_va.offset; - uint64_t data = op->wr.qword_va.data; - gds_dbg("OP_STORE_QWORD dev_ptr=%" PRIx64 " data=%" PRIx64 "\n", (uint64_t)dev_ptr, (uint64_t)data); - gds_err("unsupported QWORD op\n"); - retcode = EINVAL; - break; - } - case IBV_EXP_PEER_OP_COPY_BLOCK: { - CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + - op->wr.copy_op.offset; - size_t len = op->wr.copy_op.len; - void *src = op->wr.copy_op.src; - gds_err("unsupported COPY_BLOCK\n"); - retcode = EINVAL; - break; - } - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: { - CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + - op->wr.dword_va.offset; - uint32_t data = op->wr.dword_va.data; + case GDS_MLX5_PEER_OP_POLL_AND_DWORD: + case GDS_MLX5_PEER_OP_POLL_GEQ_DWORD: + case GDS_MLX5_PEER_OP_POLL_NOR_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; - gds_dbg("OP_POLL_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); + gds_dbg("OP_POLL_DWORD dev_ptr=%" PRIx64 " data=%08x\n", (uint64_t)dev_ptr, data); - mlx5_i->cqe_ptr = (uint32_t *)dev_ptr; - mlx5_i->cqe_value = data; + mlx5_i->cqe_ptr = (uint32_t *)dev_ptr; + mlx5_i->cqe_value = data; - switch(op->type) { - case IBV_EXP_PEER_OP_POLL_NOR_DWORD: - // GPU SMs can always do NOR - mlx5_i->cond = GDS_WAIT_COND_NOR; - break; - case IBV_EXP_PEER_OP_POLL_GEQ_DWORD: - mlx5_i->cond = GDS_WAIT_COND_GEQ; - break; - case IBV_EXP_PEER_OP_POLL_AND_DWORD: - mlx5_i->cond = GDS_WAIT_COND_AND; + switch(op->type) { + case GDS_MLX5_PEER_OP_POLL_NOR_DWORD: + // GPU SMs can always do NOR + mlx5_i->cond = GDS_WAIT_COND_NOR; + break; + case GDS_MLX5_PEER_OP_POLL_GEQ_DWORD: + mlx5_i->cond = GDS_WAIT_COND_GEQ; + break; + case GDS_MLX5_PEER_OP_POLL_AND_DWORD: + mlx5_i->cond = GDS_WAIT_COND_AND; + break; + default: + gds_err("unexpected op type\n"); + retcode = EINVAL; + goto err; + } break; + } default: - gds_err("unexpected op type\n"); + gds_err("undefined peer op type %d\n", op->type); retcode = EINVAL; - goto err; - } - break; - } - default: - gds_err("undefined peer op type %d\n", op->type); - retcode = EINVAL; - break; + break; } - err: +err: if (retcode) { gds_err("error in fill func at entry n=%zu\n", n); break; @@ -289,16 +337,11 @@ int gds_mlx5_get_wait_info(int count, const gds_wait_request_t *requests, gds_ml { int retcode = 0; - for (int j=0; jcqe_ptr, mlx5_i->cqe_value, mlx5_i->flag_ptr, mlx5_i->flag_value); + j, mlx5_i->cqe_ptr, mlx5_i->cqe_value, mlx5_i->flag_ptr, mlx5_i->flag_value); } return retcode; @@ -312,7 +355,6 @@ int gds_mlx5_get_dword_wait_info(uint32_t *ptr, uint32_t value, int flags, gds_m CUdeviceptr dev_ptr = 0; assert(NULL != ptr); - //assert((((unsigned long)ptr) & 0x3) == 0); memset(mlx5_info, 0, sizeof(&mlx5_info)); retcode = gds_map_mem(ptr, sizeof(*ptr), memtype_from_flags(flags), &dev_ptr); @@ -330,6 +372,1842 @@ int gds_mlx5_get_dword_wait_info(uint32_t *ptr, uint32_t value, int flags, gds_m //----------------------------------------------------------------------------- +int gds_mlx5_rollback_send(gds_mlx5_qp_t *mqp, + struct gds_mlx5_rollback_ctx *rollback) +{ + int diff; + + mqp->bf_offset = (rollback->rollback_id & GDS_MLX5_ROLLBACK_ID_PARITY_MASK) ? + mqp->dvqp.bf.size : 0; + rollback->rollback_id &= GDS_MLX5_ROLLBACK_ID_PARITY_MASK - 1; + + if (rollback->flags & GDS_MLX5_ROLLBACK_ABORT_UNCOMMITED) { + diff = (mqp->sq_cur_post & 0xffff) + - ntohl(mqp->dvqp.dbrec[MLX5_SND_DBR]); + if (diff < 0) + diff += 0x10000; + mqp->sq_cur_post -= diff; + } else { + if (!(rollback->flags & GDS_MLX5_ROLLBACK_ABORT_LATE)) { + if (mqp->sq_cur_post != + (rollback->rollback_id >> 32)) + return -ERANGE; + } + mqp->sq_cur_post = rollback->rollback_id & 0xffffffff; + } + return 0; +} + +//----------------------------------------------------------------------------- + +static inline int set_datagram_seg(struct mlx5_wqe_datagram_seg *seg, gds_send_wr *wr) +{ + int ret = 0; + mlx5dv_obj dv_obj; + mlx5dv_ah dv_ah; + + memset(&dv_ah, 0, sizeof(mlx5dv_ah)); + dv_obj.ah.in = wr->wr.ud.ah; + dv_obj.ah.out = &dv_ah; + + ret = mlx5dv_init_obj(&dv_obj, MLX5DV_OBJ_AH); + if (ret || dv_ah.av == NULL) { + gds_err("Error %d in mlx5dv_init_obj(..., MLX5DV_OBJ_AH)\n", ret); + return ret; + } + + memcpy(&seg->av, dv_ah.av, sizeof(struct mlx5_wqe_av)); + seg->av.dqp_dct = htobe32(wr->wr.ud.remote_qpn | MLX5_EXTENDED_UD_AV); + seg->av.key.qkey.qkey = htobe32(wr->wr.ud.remote_qkey); + return ret; +} + +//----------------------------------------------------------------------------- + +static inline int mlx5_wq_overflow(gds_mlx5_qp_t *mqp, int nreq) +{ + unsigned cur; + + cur = mqp->wq->head - mqp->wq->tail; + + return cur + nreq >= mqp->dvqp.sq.wqe_cnt; +} + +//----------------------------------------------------------------------------- + +static inline void *mlx5_get_send_wqe(gds_mlx5_qp_t *mqp, int n) +{ + return (void *)((uintptr_t)mqp->dvqp.sq.buf + (n * mqp->dvqp.sq.stride)); +} + +//----------------------------------------------------------------------------- + +static inline __be32 send_ieth(gds_send_wr *wr) +{ + switch (wr->opcode) { + case IBV_WR_SEND_WITH_IMM: + case IBV_WR_RDMA_WRITE_WITH_IMM: + return wr->imm_data; + case IBV_WR_SEND_WITH_INV: + return htobe32(wr->invalidate_rkey); + default: + return 0; + } +} + +//----------------------------------------------------------------------------- + +static inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg, + uint64_t remote_addr, uint32_t rkey) +{ + rseg->raddr = htobe64(remote_addr); + rseg->rkey = htobe32(rkey); + rseg->reserved = 0; +} + +//----------------------------------------------------------------------------- + +static inline void set_atomic_seg(struct mlx5_wqe_atomic_seg *aseg, + enum ibv_wr_opcode opcode, + uint64_t swap, + uint64_t compare_add) +{ + if (opcode == IBV_WR_ATOMIC_CMP_AND_SWP) { + aseg->swap_add = htobe64(swap); + aseg->compare = htobe64(compare_add); + } else { + aseg->swap_add = htobe64(compare_add); + } +} + +//----------------------------------------------------------------------------- + +#define ALIGN(x, log_a) ((((x) + (1 << (log_a)) - 1)) & ~((1 << (log_a)) - 1)) + +static inline __be16 get_klm_octo(int nentries) +{ + return htobe16(ALIGN(nentries, 3) / 2); +} + +static void set_umr_data_seg(gds_mlx5_qp_t *mqp, enum ibv_mw_type type, + int32_t rkey, + const struct ibv_mw_bind_info *bind_info, + uint32_t qpn, void **seg, int *size) +{ + struct mlx5_wqe_umr_data_seg *data = (struct mlx5_wqe_umr_data_seg *)*seg; + + data->klm.byte_count = htobe32(bind_info->length); + data->klm.mkey = htobe32(bind_info->mr->lkey); + data->klm.address = htobe64(bind_info->addr); + + memset(&data->klm + 1, 0, sizeof(data->reserved) - + sizeof(data->klm)); + + *seg = (void *)((uintptr_t)*seg + sizeof(*data)); + *size += (sizeof(*data) / 16); +} + +static void set_umr_mkey_seg(gds_mlx5_qp_t *mqp, enum ibv_mw_type type, + int32_t rkey, + const struct ibv_mw_bind_info *bind_info, + uint32_t qpn, void **seg, int *size) +{ + struct mlx5_wqe_mkey_context_seg *mkey = (struct mlx5_wqe_mkey_context_seg *)*seg; + + mkey->qpn_mkey = htobe32((rkey & 0xFF) | + ((type == IBV_MW_TYPE_1 || !bind_info->length) ? + 0xFFFFFF00 : qpn << 8)); + if (bind_info->length) { + /* Local read is set in kernel */ + mkey->access_flags = 0; + mkey->free = 0; + if (bind_info->mw_access_flags & IBV_ACCESS_LOCAL_WRITE) + mkey->access_flags |= + MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_LOCAL_WRITE; + if (bind_info->mw_access_flags & IBV_ACCESS_REMOTE_WRITE) + mkey->access_flags |= + MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_REMOTE_WRITE; + if (bind_info->mw_access_flags & IBV_ACCESS_REMOTE_READ) + mkey->access_flags |= + MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_REMOTE_READ; + if (bind_info->mw_access_flags & IBV_ACCESS_REMOTE_ATOMIC) + mkey->access_flags |= + MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_ATOMIC; + if (bind_info->mw_access_flags & IBV_ACCESS_ZERO_BASED) + mkey->start_addr = 0; + else + mkey->start_addr = htobe64(bind_info->addr); + mkey->len = htobe64(bind_info->length); + } else { + mkey->free = MLX5_WQE_MKEY_CONTEXT_FREE; + } + + *seg = (void *)((uintptr_t)*seg + sizeof(struct mlx5_wqe_mkey_context_seg)); + *size += (sizeof(struct mlx5_wqe_mkey_context_seg) / 16); +} + +static inline void set_umr_control_seg(gds_mlx5_qp_t *mqp, enum ibv_mw_type type, + int32_t rkey, + const struct ibv_mw_bind_info *bind_info, + uint32_t qpn, void **seg, int *size) +{ + struct mlx5_wqe_umr_ctrl_seg *ctrl = (struct mlx5_wqe_umr_ctrl_seg *)*seg; + + ctrl->flags = MLX5_WQE_UMR_CTRL_FLAG_TRNSLATION_OFFSET | + MLX5_WQE_UMR_CTRL_FLAG_INLINE; + ctrl->mkey_mask = htobe64(MLX5_WQE_UMR_CTRL_MKEY_MASK_FREE | + MLX5_WQE_UMR_CTRL_MKEY_MASK_MKEY); + ctrl->translation_offset = 0; + memset(ctrl->rsvd0, 0, sizeof(ctrl->rsvd0)); + memset(ctrl->rsvd1, 0, sizeof(ctrl->rsvd1)); + + if (type == IBV_MW_TYPE_2) + ctrl->mkey_mask |= htobe64(MLX5_WQE_UMR_CTRL_MKEY_MASK_QPN); + + if (bind_info->length) { + ctrl->klm_octowords = get_klm_octo(1); + if (type == IBV_MW_TYPE_2) + ctrl->flags |= MLX5_WQE_UMR_CTRL_FLAG_CHECK_FREE; + ctrl->mkey_mask |= htobe64(MLX5_WQE_UMR_CTRL_MKEY_MASK_LEN | + MLX5_WQE_UMR_CTRL_MKEY_MASK_START_ADDR | + MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_LOCAL_WRITE | + MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_REMOTE_READ | + MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_REMOTE_WRITE | + MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_ATOMIC); + } else { + ctrl->klm_octowords = get_klm_octo(0); + if (type == IBV_MW_TYPE_2) + ctrl->flags |= MLX5_WQE_UMR_CTRL_FLAG_CHECK_QPN; + } + + *seg = (void *)((uintptr_t)*seg + sizeof(struct mlx5_wqe_umr_ctrl_seg)); + *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; +} + +static inline int set_bind_wr(gds_mlx5_qp_t *mqp, enum ibv_mw_type type, + int32_t rkey, + const struct ibv_mw_bind_info *bind_info, + uint32_t qpn, void **seg, int *size) +{ + void *qend = mlx5_get_send_wqe(mqp, mqp->dvqp.sq.wqe_cnt); + + /* check that len > 2GB because KLM support only 2GB */ + if (bind_info->length > 1UL << 31) + return EOPNOTSUPP; + + set_umr_control_seg(mqp, type, rkey, bind_info, qpn, seg, size); + if (unlikely((*seg == qend))) + *seg = mlx5_get_send_wqe(mqp, 0); + + set_umr_mkey_seg(mqp, type, rkey, bind_info, qpn, seg, size); + if (!bind_info->length) + return 0; + + if (unlikely((seg == qend))) + *seg = mlx5_get_send_wqe(mqp, 0); + + set_umr_data_seg(mqp, type, rkey, bind_info, qpn, seg, size); + return 0; +} + +//----------------------------------------------------------------------------- + +static inline int mlx5_post_send_underlay(gds_mlx5_qp_t *mqp, gds_send_wr *wr, + void **pseg, int *total_size, + struct mlx5_sg_copy_ptr *sg_copy_ptr) +{ + struct mlx5_wqe_eth_seg *eseg; + int inl_hdr_copy_size; + void *seg = *pseg; + int size = 0; + + if (unlikely(wr->opcode == IBV_WR_SEND_WITH_IMM)) + return EINVAL; + + memset(seg, 0, sizeof(struct mlx5_wqe_eth_pad)); + size += sizeof(struct mlx5_wqe_eth_pad); + seg = (void *)((uintptr_t)seg + sizeof(struct mlx5_wqe_eth_pad)); + eseg = (struct mlx5_wqe_eth_seg *)seg; + *((uint64_t *)eseg) = 0; + eseg->rsvd2 = 0; + + if (wr->send_flags & IBV_SEND_IP_CSUM) { + eseg->cs_flags |= MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; + } + + if (likely(wr->sg_list[0].length >= MLX5_SOURCE_QPN_INLINE_MAX_HEADER_SIZE)) + /* Copying the minimum required data unless inline mode is set */ + inl_hdr_copy_size = (wr->send_flags & IBV_SEND_INLINE) ? + MLX5_SOURCE_QPN_INLINE_MAX_HEADER_SIZE : + MLX5_IPOIB_INLINE_MIN_HEADER_SIZE; + else { + inl_hdr_copy_size = MLX5_IPOIB_INLINE_MIN_HEADER_SIZE; + /* We expect at least 4 bytes as part of first entry to hold the IPoIB header */ + if (unlikely(wr->sg_list[0].length < inl_hdr_copy_size)) + return EINVAL; + } + + memcpy(eseg->inline_hdr_start, (void *)(uintptr_t)wr->sg_list[0].addr, + inl_hdr_copy_size); + eseg->inline_hdr_sz = htobe16(inl_hdr_copy_size); + size += sizeof(struct mlx5_wqe_eth_seg); + seg = (void *)((uintptr_t)seg + sizeof(struct mlx5_wqe_eth_seg)); + + /* If we copied all the sge into the inline-headers, then we need to + * start copying from the next sge into the data-segment. + */ + if (unlikely(wr->sg_list[0].length == inl_hdr_copy_size)) + sg_copy_ptr->index++; + else + sg_copy_ptr->offset = inl_hdr_copy_size; + + *pseg = seg; + *total_size += (size / 16); + return 0; +} + +//----------------------------------------------------------------------------- + +/* Copy tso header to eth segment with considering padding and WQE + * wrap around in WQ buffer. + */ +static inline int set_tso_eth_seg(void **seg, void *hdr, uint16_t hdr_sz, + uint16_t mss, + gds_mlx5_qp_t *mqp, int *size) +{ + struct mlx5_wqe_eth_seg *eseg = (struct mlx5_wqe_eth_seg *)*seg; + int size_of_inl_hdr_start = sizeof(eseg->inline_hdr_start); + uint64_t left, left_len, copy_sz; + + void *qend = mlx5_get_send_wqe(mqp, mqp->dvqp.sq.wqe_cnt); + + if (unlikely(hdr_sz < MLX5_ETH_L2_MIN_HEADER_SIZE)) { + gds_dbg("TSO header size should be at least %d\n", + MLX5_ETH_L2_MIN_HEADER_SIZE); + return EINVAL; + } + + left = hdr_sz; + eseg->mss = htobe16(mss); + eseg->inline_hdr_sz = htobe16(hdr_sz); + + /* Check if there is space till the end of queue, if yes, + * copy all in one shot, otherwise copy till the end of queue, + * rollback and then copy the left + */ + left_len = (uintptr_t)qend - (uintptr_t)eseg->inline_hdr_start; + copy_sz = MIN(left_len, left); + + memcpy(eseg->inline_hdr_start, hdr, copy_sz); + + /* The -1 is because there are already 16 bytes included in + * eseg->inline_hdr[16] + */ + *seg = (void *)((uintptr_t)seg + align(copy_sz - size_of_inl_hdr_start, 16) - 16); + *size += align(copy_sz - size_of_inl_hdr_start, 16) / 16 - 1; + + /* The last wqe in the queue */ + if (unlikely(copy_sz < left)) { + *seg = mlx5_get_send_wqe(mqp, 0); + left -= copy_sz; + hdr = (void *)((uintptr_t)hdr + copy_sz); + memcpy(*seg, hdr, left); + *seg = (void *)((uintptr_t)*seg + align(left, 16)); + *size += align(left, 16) / 16; + } + + return 0; +} + +//----------------------------------------------------------------------------- + +static inline int copy_eth_inline_headers(struct ibv_qp *ibqp, + const void *list, + size_t nelem, + struct mlx5_wqe_eth_seg *eseg, + struct mlx5_sg_copy_ptr *sg_copy_ptr, + bool is_sge) + __attribute__((always_inline)); +static inline int copy_eth_inline_headers(struct ibv_qp *ibqp, + const void *list, + size_t nelem, + struct mlx5_wqe_eth_seg *eseg, + struct mlx5_sg_copy_ptr *sg_copy_ptr, + bool is_sge) +{ + uint32_t inl_hdr_size = MLX5_ETH_L2_INLINE_HEADER_SIZE; + size_t inl_hdr_copy_size = 0; + int j = 0; + size_t length; + void *addr; + + if (unlikely(nelem < 1)) { + gds_dbg("illegal num_sge: %zu, minimum is 1\n", nelem); + return EINVAL; + } + + if (is_sge) { + addr = (void *)(uintptr_t)((struct ibv_sge *)list)[0].addr; + length = (size_t)((struct ibv_sge *)list)[0].length; + } else { + addr = ((struct ibv_data_buf *)list)[0].addr; + length = ((struct ibv_data_buf *)list)[0].length; + } + + if (likely(length >= MLX5_ETH_L2_INLINE_HEADER_SIZE)) { + inl_hdr_copy_size = inl_hdr_size; + memcpy(eseg->inline_hdr_start, addr, inl_hdr_copy_size); + } else { + uint32_t inl_hdr_size_left = inl_hdr_size; + + for (j = 0; j < nelem && inl_hdr_size_left > 0; ++j) { + if (is_sge) { + addr = (void *)(uintptr_t)((struct ibv_sge *)list)[j].addr; + length = (size_t)((struct ibv_sge *)list)[j].length; + } else { + addr = ((struct ibv_data_buf *)list)[j].addr; + length = ((struct ibv_data_buf *)list)[j].length; + } + + inl_hdr_copy_size = MIN(length, inl_hdr_size_left); + memcpy(eseg->inline_hdr_start + + (MLX5_ETH_L2_INLINE_HEADER_SIZE - inl_hdr_size_left), + addr, inl_hdr_copy_size); + inl_hdr_size_left -= inl_hdr_copy_size; + } + if (unlikely(inl_hdr_size_left)) { + gds_dbg("Ethernet headers < 16 bytes\n"); + return EINVAL; + } + if (j) + --j; + } + + eseg->inline_hdr_sz = htobe16(inl_hdr_size); + + /* If we copied all the sge into the inline-headers, then we need to + * start copying from the next sge into the data-segment. + */ + if (unlikely(length == inl_hdr_copy_size)) { + ++j; + inl_hdr_copy_size = 0; + } + + sg_copy_ptr->index = j; + sg_copy_ptr->offset = inl_hdr_copy_size; + + return 0; +} + +//----------------------------------------------------------------------------- + +static inline int set_data_inl_seg(gds_mlx5_qp_t *mqp, gds_send_wr *wr, + void *wqe, int *sz, + struct mlx5_sg_copy_ptr *sg_copy_ptr) +{ + struct mlx5_wqe_inline_seg *seg; + void *addr; + int len; + int i; + int inl = 0; + int copy; + int offset = sg_copy_ptr->offset; + void *qend = mlx5_get_send_wqe(mqp, mqp->dvqp.sq.wqe_cnt); + + seg = (struct mlx5_wqe_inline_seg *)wqe; + wqe = (void *)((uintptr_t)wqe + sizeof *seg); + for (i = sg_copy_ptr->index; i < wr->num_sge; ++i) { + addr = (void *) (unsigned long)(wr->sg_list[i].addr + offset); + len = wr->sg_list[i].length - offset; + inl += len; + offset = 0; + + if (unlikely((void *)((uintptr_t)wqe + len) > qend)) { + copy = (uintptr_t)qend - (uintptr_t)wqe; + memcpy(wqe, addr, copy); + addr = (void *)((uintptr_t)addr + copy); + len -= copy; + wqe = mlx5_get_send_wqe(mqp, 0); + } + memcpy(wqe, addr, len); + wqe = (void *)((uintptr_t)wqe + len); + } + + if (likely(inl)) { + seg->byte_count = htobe32(inl | MLX5_INLINE_SEG); + *sz = align(inl + sizeof seg->byte_count, 16) / 16; + } else + *sz = 0; + + return 0; +} + +//----------------------------------------------------------------------------- + +static inline void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ibv_sge *sg, + int offset) +{ + dseg->byte_count = htobe32(sg->length - offset); + dseg->lkey = htobe32(sg->lkey); + dseg->addr = htobe64(sg->addr + offset); +} + +static inline void set_data_ptr_seg_atomic(struct mlx5_wqe_data_seg *dseg, + struct ibv_sge *sg) +{ + dseg->byte_count = htobe32(MLX5_ATOMIC_SIZE); + dseg->lkey = htobe32(sg->lkey); + dseg->addr = htobe64(sg->addr); +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_post_send(gds_mlx5_qp_t *mqp, gds_send_wr *p_ewr, gds_send_wr **bad_wr, gds_mlx5_peer_commit *commit) +{ + int ret = 0; + unsigned int idx; + int size; + void *seg; + void *qend = mlx5_get_send_wqe(mqp, mqp->dvqp.sq.wqe_cnt); + int i; + int nreq; + int inl = 0; + uint8_t opmod = 0; + uint32_t mlx5_opcode; + + struct mlx5_wqe_ctrl_seg *ctrl; + struct mlx5_wqe_xrc_seg *xrc; + struct mlx5_wqe_eth_seg *eseg; + struct mlx5_wqe_data_seg *dpseg; + struct mlx5_sg_copy_ptr sg_copy_ptr = {.index = 0, .offset = 0}; + + uint8_t fence; + uint8_t next_fence; + + struct gds_mlx5_peer_op_wr *wr; + + if (commit->entries < 3) { + gds_err("not enough entries in gds_mlx5_peer_commit.\n"); + ret = EINVAL; + goto out; + } + + next_fence = mqp->fm_cache; + + for (nreq = 0; p_ewr; ++nreq, p_ewr = p_ewr->next) { + if (unlikely(p_ewr->opcode < 0 || + p_ewr->opcode >= sizeof mlx5_ib_opcode / sizeof mlx5_ib_opcode[0])) { + gds_dbg("bad opcode %d\n", p_ewr->opcode); + ret = EINVAL; + *bad_wr = p_ewr; + goto out; + } + + if (unlikely(mlx5_wq_overflow(mqp, nreq))) { + gds_dbg("work queue overflow\n"); + ret = ENOMEM; + *bad_wr = p_ewr; + goto out; + } + + if (p_ewr->send_flags & IBV_SEND_FENCE) + fence = MLX5_WQE_CTRL_FENCE; + else + fence = next_fence; + next_fence = 0; + idx = mqp->sq_cur_post & (mqp->dvqp.sq.wqe_cnt - 1); + seg = mlx5_get_send_wqe(mqp, idx); + ctrl = (struct mlx5_wqe_ctrl_seg *)seg; + *(uint32_t *)((uintptr_t)seg + 8) = 0; + ctrl->imm = send_ieth(p_ewr); + ctrl->fm_ce_se = mqp->sq_signal_bits | fence | + (p_ewr->send_flags & IBV_SEND_SIGNALED ? + MLX5_WQE_CTRL_CQ_UPDATE : 0) | + (p_ewr->send_flags & IBV_SEND_SOLICITED ? + MLX5_WQE_CTRL_SOLICITED : 0); + + seg = (void *)((uintptr_t)seg + sizeof *ctrl); + size = sizeof *ctrl / 16; + + switch (mqp->gqp.ibqp->qp_type) { + case IBV_QPT_XRC_SEND: + if (unlikely(p_ewr->opcode != IBV_WR_BIND_MW && + p_ewr->opcode != IBV_WR_LOCAL_INV)) { + xrc = (struct mlx5_wqe_xrc_seg *)seg; + xrc->xrc_srqn = htobe32(p_ewr->qp_type.xrc.remote_srqn); + seg = (void *)((uintptr_t)seg + sizeof(*xrc)); + size += sizeof(*xrc) / 16; + } + /* fall through */ + case IBV_QPT_RC: + switch (p_ewr->opcode) { + case IBV_WR_RDMA_READ: + case IBV_WR_RDMA_WRITE: + case IBV_WR_RDMA_WRITE_WITH_IMM: + set_raddr_seg((struct mlx5_wqe_raddr_seg *)seg, + p_ewr->wr.rdma.remote_addr, + p_ewr->wr.rdma.rkey); + seg = (void *)((uintptr_t)seg + sizeof(struct mlx5_wqe_raddr_seg)); + size += sizeof(struct mlx5_wqe_raddr_seg) / 16; + break; + + case IBV_WR_ATOMIC_CMP_AND_SWP: + case IBV_WR_ATOMIC_FETCH_AND_ADD: + set_raddr_seg((struct mlx5_wqe_raddr_seg *)seg, + p_ewr->wr.atomic.remote_addr, + p_ewr->wr.atomic.rkey); + seg = (void *)((uintptr_t)seg + sizeof(struct mlx5_wqe_raddr_seg)); + + set_atomic_seg((struct mlx5_wqe_atomic_seg *)seg, + p_ewr->opcode, + p_ewr->wr.atomic.swap, + p_ewr->wr.atomic.compare_add); + seg = (void *)((uintptr_t)seg + sizeof(struct mlx5_wqe_atomic_seg)); + + size += (sizeof(struct mlx5_wqe_raddr_seg) + + sizeof(struct mlx5_wqe_atomic_seg)) / 16; + break; + + case IBV_WR_BIND_MW: + next_fence = MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE; + ctrl->imm = htobe32(p_ewr->bind_mw.mw->rkey); + ret = set_bind_wr(mqp, p_ewr->bind_mw.mw->type, + p_ewr->bind_mw.rkey, + &p_ewr->bind_mw.bind_info, + mqp->gqp.ibqp->qp_num, &seg, &size); + if (ret) { + *bad_wr = p_ewr; + goto out; + } + break; + case IBV_WR_LOCAL_INV: { + struct ibv_mw_bind_info bind_info = {}; + + next_fence = MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE; + ctrl->imm = htobe32(p_ewr->invalidate_rkey); + ret = set_bind_wr(mqp, IBV_MW_TYPE_2, 0, + &bind_info, mqp->gqp.ibqp->qp_num, + &seg, &size); + if (ret) { + *bad_wr = p_ewr; + goto out; + } + break; + } + + default: + break; + } + break; + + case IBV_QPT_UC: + switch (p_ewr->opcode) { + case IBV_WR_RDMA_WRITE: + case IBV_WR_RDMA_WRITE_WITH_IMM: + set_raddr_seg((struct mlx5_wqe_raddr_seg *)seg, + p_ewr->wr.rdma.remote_addr, + p_ewr->wr.rdma.rkey); + seg = (void *)((uintptr_t)seg + sizeof(struct mlx5_wqe_raddr_seg)); + size += sizeof(struct mlx5_wqe_raddr_seg) / 16; + break; + case IBV_WR_BIND_MW: + next_fence = MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE; + ctrl->imm = htobe32(p_ewr->bind_mw.mw->rkey); + ret = set_bind_wr(mqp, p_ewr->bind_mw.mw->type, + p_ewr->bind_mw.rkey, + &p_ewr->bind_mw.bind_info, + mqp->gqp.ibqp->qp_num, &seg, &size); + if (ret) { + *bad_wr = p_ewr; + goto out; + } + break; + case IBV_WR_LOCAL_INV: { + struct ibv_mw_bind_info bind_info = {}; + + next_fence = MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE; + ctrl->imm = htobe32(p_ewr->invalidate_rkey); + ret = set_bind_wr(mqp, IBV_MW_TYPE_2, 0, + &bind_info, mqp->gqp.ibqp->qp_num, + &seg, &size); + if (ret) { + *bad_wr = p_ewr; + goto out; + } + break; + } + + default: + break; + } + break; + + case IBV_QPT_UD: + set_datagram_seg((struct mlx5_wqe_datagram_seg *)seg, p_ewr); + seg = (void *)((uintptr_t)seg + sizeof(struct mlx5_wqe_datagram_seg)); + size += sizeof(struct mlx5_wqe_datagram_seg) / 16; + if (unlikely((seg == qend))) + seg = mlx5_get_send_wqe(mqp, 0); + break; + + case IBV_QPT_RAW_PACKET: + memset(seg, 0, sizeof(struct mlx5_wqe_eth_seg)); + eseg = (struct mlx5_wqe_eth_seg *)seg; + + if (p_ewr->send_flags & IBV_SEND_IP_CSUM) { + eseg->cs_flags |= MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; + } + + if (p_ewr->opcode == IBV_WR_TSO) { + ret = set_tso_eth_seg(&seg, p_ewr->tso.hdr, + p_ewr->tso.hdr_sz, + p_ewr->tso.mss, mqp, &size); + if (unlikely(ret)) { + *bad_wr = p_ewr; + goto out; + } + + /* For TSO WR we always copy at least MLX5_ETH_L2_MIN_HEADER_SIZE + * bytes of inline header which is included in struct mlx5_wqe_eth_seg. + * If additional bytes are copied, 'seg' and 'size' are adjusted + * inside set_tso_eth_seg(). + */ + + seg = (void *)((uintptr_t)seg + sizeof(struct mlx5_wqe_eth_seg)); + size += sizeof(struct mlx5_wqe_eth_seg) / 16; + } else { + uint32_t inl_hdr_size = MLX5_ETH_L2_INLINE_HEADER_SIZE; + + ret = copy_eth_inline_headers(mqp->gqp.ibqp, p_ewr->sg_list, + p_ewr->num_sge, (struct mlx5_wqe_eth_seg *)seg, + &sg_copy_ptr, 1); + if (unlikely(ret)) { + *bad_wr = p_ewr; + gds_dbg("copy_eth_inline_headers failed, err: %d\n", ret); + goto out; + } + + /* The eth segment size depends on the device's min inline + * header requirement which can be 0 or 18. The basic eth segment + * always includes room for first 2 inline header bytes (even if + * copy size is 0) so the additional seg size is adjusted accordingly. + */ + + seg = (void *)((uintptr_t)seg + (offsetof(struct mlx5_wqe_eth_seg, inline_hdr) + + inl_hdr_size) & ~0xf); + size += (offsetof(struct mlx5_wqe_eth_seg, inline_hdr) + + inl_hdr_size) >> 4; + } + break; + + default: + break; + } + + if (p_ewr->send_flags & IBV_SEND_INLINE && p_ewr->num_sge) { + int uninitialized_var(sz); + + ret = set_data_inl_seg(mqp, p_ewr, seg, &sz, &sg_copy_ptr); + if (unlikely(ret)) { + *bad_wr = p_ewr; + gds_dbg("inline layout failed, err %d\n", ret); + goto out; + } + inl = 1; + size += sz; + } else { + dpseg = (struct mlx5_wqe_data_seg *)seg; + for (i = sg_copy_ptr.index; i < p_ewr->num_sge; ++i) { + if (unlikely(dpseg == qend)) { + seg = mlx5_get_send_wqe(mqp, 0); + dpseg = (struct mlx5_wqe_data_seg *)seg; + } + if (likely(p_ewr->sg_list[i].length)) { + if (unlikely(p_ewr->opcode == + IBV_WR_ATOMIC_CMP_AND_SWP || + p_ewr->opcode == + IBV_WR_ATOMIC_FETCH_AND_ADD)) + set_data_ptr_seg_atomic(dpseg, p_ewr->sg_list + i); + else { + set_data_ptr_seg(dpseg, p_ewr->sg_list + i, + sg_copy_ptr.offset); + } + sg_copy_ptr.offset = 0; + ++dpseg; + size += sizeof(struct mlx5_wqe_data_seg) / 16; + } + } + } + + mlx5_opcode = mlx5_ib_opcode[p_ewr->opcode]; + ctrl->opmod_idx_opcode = htobe32(((mqp->sq_cur_post & 0xffff) << 8) | + mlx5_opcode | + (opmod << 24)); + ctrl->qpn_ds = htobe32(size | (mqp->gqp.ibqp->qp_num << 8)); + + mqp->wq->wrid[idx] = p_ewr->wr_id; + mqp->wq->wqe_head[idx] = mqp->wq->head + nreq; + mqp->sq_cur_post += (size * 16 + mqp->dvqp.sq.stride - 1) / mqp->dvqp.sq.stride; + } + +out: + mqp->fm_cache = next_fence; + + if (likely(nreq > 0)) { + mqp->wq->head += nreq; + + commit->rollback_id = mqp->qp_peer->scur_post | ((uint64_t)mqp->sq_cur_post << 32); + mqp->qp_peer->scur_post = mqp->sq_cur_post; + + wr = commit->storage; + + wr->type = GDS_MLX5_PEER_OP_STORE_DWORD; + wr->wr.dword_va.data = htonl(mqp->sq_cur_post & 0xffff); + wr->wr.dword_va.target_id = mqp->qp_peer->dbr.va_id; + wr->wr.dword_va.offset = sizeof(uint32_t) * MLX5_SND_DBR; + wr = wr->next; + + wr->type = GDS_MLX5_PEER_OP_FENCE; + wr->wr.fence.fence_flags = GDS_PEER_FENCE_OP_WRITE | GDS_PEER_FENCE_FROM_HCA | GDS_PEER_FENCE_MEM_SYS; + wr = wr->next; + + wr->type = GDS_MLX5_PEER_OP_STORE_QWORD; + wr->wr.qword_va.data = *(__be64 *)ctrl; + wr->wr.qword_va.target_id = mqp->qp_peer->bf.va_id; + wr->wr.qword_va.offset = mqp->bf_offset; + + mqp->bf_offset ^= mqp->dvqp.bf.size; + commit->entries = 3; + } + + return ret; +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_peer_peek_cq(gds_mlx5_cq_t *mcq, struct gds_mlx5_peer_peek *peek) +{ + int ret = 0; + + gds_peer_attr *peer_attr = (gds_peer_attr *)mcq->peer_attr; + struct gds_mlx5_peer_op_wr *wr; + int n, cur_own; + void *cqe; + struct mlx5_cqe64 *cqe64; + struct gds_mlx5_peek_entry *tmp; + + if (peek->entries < 2) { + gds_err("not enough entries in gds_mlx5_peek_entry.\n"); + ret = EINVAL; + goto out; + } + + wr = peek->storage; + n = peek->offset; + + cqe = (char *)mcq->dvcq.buf + (n & (mcq->dvcq.cqe_cnt - 1)) * mcq->dvcq.cqe_size; + cur_own = n & mcq->dvcq.cqe_cnt; + cqe64 = (struct mlx5_cqe64 *)((mcq->dvcq.cqe_size == 64) ? cqe : (char *)cqe + 64); + + if (cur_own) { + wr->type = GDS_MLX5_PEER_OP_POLL_AND_DWORD; + wr->wr.dword_va.data = htonl(MLX5_CQE_OWNER_MASK); + } + else if (peer_attr->caps & GDS_PEER_OP_POLL_NOR_DWORD_CAP) { + wr->type = GDS_MLX5_PEER_OP_POLL_NOR_DWORD; + wr->wr.dword_va.data = ~htonl(MLX5_CQE_OWNER_MASK); + } + else if (peer_attr->caps & GDS_PEER_OP_POLL_GEQ_DWORD_CAP) { + wr->type = GDS_MLX5_PEER_OP_POLL_GEQ_DWORD; + wr->wr.dword_va.data = 0; + } + wr->wr.dword_va.target_id = mcq->active_buf_va_id; + wr->wr.dword_va.offset = (uintptr_t)&cqe64->wqe_counter - (uintptr_t)mcq->dvcq.buf; + wr = wr->next; + + tmp = mcq->peer_peek_free; + if (!tmp) { + ret = ENOMEM; + goto out; + } + mcq->peer_peek_free = GDS_MLX5_PEEK_ENTRY(mcq, tmp->next); + tmp->busy = 1; + wmb(); + tmp->next = GDS_MLX5_PEEK_ENTRY_N(mcq, mcq->peer_peek_table[n & (mcq->dvcq.cqe_cnt - 1)]); + mcq->peer_peek_table[n & (mcq->dvcq.cqe_cnt - 1)] = tmp; + + wr->type = GDS_MLX5_PEER_OP_STORE_DWORD; + wr->wr.dword_va.data = 0; + wr->wr.dword_va.target_id = mcq->peer_va_id; + wr->wr.dword_va.offset = (uintptr_t)&tmp->busy - (uintptr_t)mcq->peer_buf->addr; + + peek->entries = 2; + peek->peek_id = (uintptr_t)tmp; + +out: + return ret; +} +//----------------------------------------------------------------------------- + +int gds_mlx5_create_cq(struct ibv_cq *ibcq, gds_peer_attr *peer_attr, gds_mlx5_cq_t **out_mcq) +{ + int ret = 0; + + gds_mlx5_cq_t *mcq = NULL; + gds_cq_t *gcq; + mlx5dv_obj dv_obj; + + struct gds_buf_alloc_attr ba_attr; + + mcq = (gds_mlx5_cq_t *)calloc(1, sizeof(gds_mlx5_cq_t)); + if (!mcq) { + gds_err("cannot allocate memory\n"); + ret = ENOMEM; + goto err; + } + + dv_obj.cq.in = ibcq; + dv_obj.cq.out = &mcq->dvcq; + ret = mlx5dv_init_obj(&dv_obj, MLX5DV_OBJ_CQ); + if (ret) { + gds_err("error %d in mlx5dv_init_obj MLX5DV_OBJ_CQ\n", ret); + goto err; + } + + mcq->peer_attr = peer_attr; + + mcq->active_buf_va_id = peer_attr->register_va( + mcq->dvcq.buf, + (uint64_t)mcq->dvcq.cqe_cnt * (uint64_t)mcq->dvcq.cqe_size, + peer_attr->peer_id, + NULL + ); + if (!mcq->active_buf_va_id) { + gds_err("error in peer_attr->register_va\n"); + ret = EINVAL; + goto err; + } + + mcq->peer_peek_table = (struct gds_mlx5_peek_entry **)malloc(sizeof(struct gds_mlx5_peek_entry *) * mcq->dvcq.cqe_cnt); + if (!mcq->peer_peek_table) { + gds_err("error %d in malloc peer_peek_table\n", errno); + ret = ENOMEM; + goto err; + } + memset(mcq->peer_peek_table, 0, sizeof(struct gds_peek_entry *) * mcq->dvcq.cqe_cnt); + mcq->peer_dir = GDS_PEER_DIRECTION_FROM_PEER | GDS_PEER_DIRECTION_TO_CPU; + + ba_attr = { + .length = sizeof(struct gds_mlx5_peek_entry) * mcq->dvcq.cqe_cnt, + .dir = mcq->peer_dir, + .peer_id = peer_attr->peer_id, + .alignment = (uint32_t)sysconf(_SC_PAGESIZE), + .comp_mask = 0 + }; + mcq->peer_buf = peer_attr->buf_alloc(&ba_attr); + if (!mcq->peer_buf) { + gds_err("error %d in buf_alloc\n", errno); + ret = ENOMEM; + goto err; + } + + mcq->peer_va_id = peer_attr->register_va(mcq->peer_buf->addr, mcq->peer_buf->length, peer_attr->peer_id, mcq->peer_buf); + if (!mcq->peer_va_id) { + gds_err("error %d in register_va\n", errno); + ret = EINVAL; + goto err; + } + + memset(mcq->peer_buf->addr, 0, mcq->peer_buf->length); + + mcq->peer_peek_free = (struct gds_mlx5_peek_entry *)mcq->peer_buf->addr; + for (int i = 0; i < mcq->dvcq.cqe_cnt - 1; ++i) + mcq->peer_peek_free[i].next = i + 1; + mcq->peer_peek_free[mcq->dvcq.cqe_size - 1].next = GDS_MLX5_LAST_PEEK_ENTRY; + + mcq->gcq.ibcq = ibcq; + mcq->gcq.dtype = GDS_DRIVER_TYPE_MLX5; + *out_mcq = mcq; + + return 0; + +err: + if (mcq) { + if (mcq->peer_va_id) + peer_attr->unregister_va(mcq->peer_va_id, peer_attr->peer_id); + + if (mcq->peer_buf) + peer_attr->buf_release(mcq->peer_buf); + + if (mcq->peer_peek_table) + free(mcq->peer_peek_table); + + if (mcq->active_buf_va_id) + peer_attr->unregister_va(mcq->active_buf_va_id, peer_attr->peer_id); + + free(mcq); + } + + return ret; +} + +//----------------------------------------------------------------------------- + +void gds_mlx5_destroy_cq(gds_mlx5_cq_t *mcq) +{ + int status = 0; + if (mcq->peer_peek_table) { + free(mcq->peer_peek_table); + mcq->peer_peek_table = NULL; + } + + if (mcq->wq) + mcq->wq = NULL; + + if (mcq->peer_attr) { + gds_peer_attr *peer_attr = mcq->peer_attr; + if (mcq->active_buf_va_id) { + peer_attr->unregister_va(mcq->active_buf_va_id, peer_attr->peer_id); + mcq->active_buf_va_id = 0; + } + if (mcq->peer_va_id) { + peer_attr->unregister_va(mcq->peer_va_id, peer_attr->peer_id); + mcq->peer_va_id = 0; + } + if (mcq->peer_buf) { + mcq->peer_attr->buf_release(mcq->peer_buf); + mcq->peer_buf = NULL; + } + } + + if (mcq->gcq.ibcq) { + status = ibv_destroy_cq(mcq->gcq.ibcq); + if (status) { + gds_err("error %d in ibv_destroy\n", status); + return; + } + mcq->gcq.ibcq = NULL; + } + + free(mcq); +} + +//----------------------------------------------------------------------------- + +static gds_mlx5_wq_t *mlx5_create_wq(uint32_t wqe_cnt) +{ + gds_mlx5_wq_t *wq = (gds_mlx5_wq_t *)calloc(1, sizeof(gds_mlx5_wq_t)); + if (!wq) { + gds_err("error in calloc wq\n"); + goto err; + } + + wq->wrid = (uint64_t *)malloc(wqe_cnt * sizeof(uint64_t)); + if (!wq->wrid) { + gds_err("error in calloc wq->wrid\n"); + goto err; + } + + wq->wqe_head = (uint64_t *)malloc(wqe_cnt * sizeof(uint64_t)); + if (!wq->wqe_head) { + gds_err("error in calloc wq->wqe_head\n"); + goto err; + } + + wq->wqe_cnt = wqe_cnt; + + return wq; +err: + if (wq) { + if (wq->wrid) + free(wq->wrid); + if (wq->wqe_head) + free(wq->wqe_head); + free(wq); + } + return NULL; +} + +static void mlx5_destroy_wq(gds_mlx5_wq_t *wq) +{ + if (wq) { + if (wq->wrid) { + free(wq->wrid); + wq->wrid = NULL; + } + if (wq->wqe_head) { + free(wq->wqe_head); + wq->wqe_head = NULL; + } + free(wq); + } +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_create_qp(struct ibv_qp *ibqp, gds_qp_init_attr_t *qp_attr, gds_mlx5_cq_t *tx_mcq, gds_mlx5_cq_t *rx_mcq, gds_mlx5_qp_peer_t *qp_peer, gds_mlx5_qp_t **out_mqp) +{ + int ret = 0; + + gds_mlx5_qp_t *mqp = NULL; + gds_qp_t *gqp; + + gds_mlx5_wq_t *wq = NULL; + + gds_peer_attr *peer_attr = qp_peer->peer_attr; + + bool register_peer_dbr = false; + + mlx5dv_obj dv_obj; + + mqp = (gds_mlx5_qp_t *)calloc(1, sizeof(gds_mlx5_qp_t)); + if (!mqp) { + gds_err("cannot allocate memory\n"); + ret = ENOMEM; + goto err; + } + + gqp = &mqp->gqp; + gqp->dtype = GDS_DRIVER_TYPE_MLX5; + gqp->ibqp = ibqp; + + tx_mcq->gcq.ibcq = ibqp->send_cq; + tx_mcq->gcq.curr_offset = 0; + tx_mcq->gcq.ctype = GDS_CQ_TYPE_SQ; + gqp->send_cq = &tx_mcq->gcq; + + rx_mcq->gcq.ibcq = ibqp->recv_cq; + rx_mcq->gcq.curr_offset = 0; + rx_mcq->gcq.ctype = GDS_CQ_TYPE_RQ; + gqp->recv_cq = &rx_mcq->gcq; + + if (qp_attr->sq_sig_all) + mqp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE; + else + mqp->sq_signal_bits = 0; + + dv_obj = { + .qp = { + .in = gqp->ibqp, + .out = &mqp->dvqp + } + }; + ret = mlx5dv_init_obj(&dv_obj, MLX5DV_OBJ_QP); + if (ret != 0) { + gds_err("error in mlx5dv_init_obj MLX5DV_OBJ_QP\n"); + goto err; + } + + if (!qp_peer->dbr.va_id) { + qp_peer->dbr.va_id = peer_attr->register_va( + mqp->dvqp.dbrec, + qp_peer->dbr.size, + peer_attr->peer_id, + NULL + ); + if (!qp_peer->dbr.va_id) { + gds_err("error in register_va\n"); + goto err; + } + register_peer_dbr = true; + } + + qp_peer->bf.va_id = peer_attr->register_va( + (uint32_t *)mqp->dvqp.bf.reg, + mqp->dvqp.bf.size, + peer_attr->peer_id, + GDS_PEER_IOMEMORY + ); + if (!qp_peer->bf.va_id) { + gds_err("error in register_va\n"); + goto err; + } + + wq = mlx5_create_wq(mqp->dvqp.sq.wqe_cnt); + if (!wq) { + gds_err("error in mlx5_create_wq\n"); + ret = ENOMEM; + goto err; + } + + mqp->qp_peer = qp_peer; + mqp->wq = wq; + tx_mcq->wq = wq; + *out_mqp = mqp; + return 0; + +err: + if (register_peer_dbr) + peer_attr->unregister_va(qp_peer->dbr.va_id, peer_attr->peer_id); + + if (qp_peer->bf.va_id) + peer_attr->unregister_va(qp_peer->bf.va_id, peer_attr->peer_id); + + if (wq) + mlx5_destroy_wq(wq); + + if (mqp) + free(mqp); + + return ret; +} + +//----------------------------------------------------------------------------- + +void gds_mlx5_destroy_qp(gds_mlx5_qp_t *mqp) +{ + int status; + gds_mlx5_qp_peer_t *qp_peer = mqp->qp_peer; + + if (qp_peer) { + gds_peer_attr *peer_attr = qp_peer->peer_attr; + if (qp_peer->dbr.va_id) { + peer_attr->unregister_va(qp_peer->dbr.va_id, peer_attr->peer_id); + qp_peer->dbr.va_id = 0; + } + + if (qp_peer->bf.va_id) { + peer_attr->unregister_va(qp_peer->bf.va_id, peer_attr->peer_id); + qp_peer->bf.va_id = 0; + } + } + + if (mqp->gqp.ibqp) { + status = ibv_destroy_qp(mqp->gqp.ibqp); + if (status) + gds_err("error %d in ibv_destroy_qp\n", status); + } + + if (mqp->gqp.send_cq) { + gds_destroy_cq(mqp->gqp.send_cq); + mqp->gqp.send_cq = NULL; + } + + if (mqp->gqp.recv_cq) { + gds_destroy_cq(mqp->gqp.recv_cq); + mqp->gqp.recv_cq = NULL; + } + + if (mqp->wq) { + mlx5_destroy_wq(mqp->wq); + mqp->wq = NULL; + } + + free(mqp); +} + +//----------------------------------------------------------------------------- + +static void *pd_mem_alloc(struct ibv_pd *pd, void *pd_context, size_t size, + size_t alignment, uint64_t resource_type) +{ + assert(pd_context); + + gds_peer_attr *peer_attr = (gds_peer_attr *)pd_context; + gds_buf_alloc_attr buf_attr = { + .length = size, + .dir = GDS_PEER_DIRECTION_FROM_PEER | GDS_PEER_DIRECTION_TO_HCA, + .peer_id = peer_attr->peer_id, + .alignment = (uint32_t)alignment, + .comp_mask = peer_attr->comp_mask + }; + gds_peer *peer = peer_from_id(peer_attr->peer_id); + gds_mlx5_qp_peer_t *qp_peer; + uint64_t range_id; + gds_buf *buf = NULL; + void *ptr = NULL; + + gds_dbg("pd_mem_alloc: pd=%p, pd_context=%p, size=%zu, alignment=%zu, resource_type=0x%lx\n", + pd, pd_context, size, alignment, resource_type); + + assert(peer->obj); + qp_peer = (gds_mlx5_qp_peer_t *)peer->obj; + + switch (resource_type) { + case MLX5DV_RES_TYPE_QP: + break; + case MLX5DV_RES_TYPE_DBR: + buf = peer_attr->buf_alloc(&buf_attr); + qp_peer->dbr.size = size; + break; + default: + gds_err("request allocation with unsupported resource_type\n"); + break; + } + + if (!buf) { + int err; + gds_dbg("alloc on host\n"); + return IBV_ALLOCATOR_USE_DEFAULT; + } + else { + gds_dbg("alloc on GPU\n"); + ptr = buf->addr; + } + + if ((range_id = peer_attr->register_va(ptr, size, peer_attr->peer_id, buf)) == 0) { + gds_err("error in register_va\n"); + peer_attr->buf_release(buf); + return IBV_ALLOCATOR_USE_DEFAULT; + } + + if (resource_type == MLX5DV_RES_TYPE_DBR) { + qp_peer->dbr.va_id = range_id; + qp_peer->dbr.gbuf = buf; + } + + return ptr; +} + +static void pd_mem_free(struct ibv_pd *pd, void *pd_context, void *ptr, + uint64_t resource_type) +{ + gds_dbg("pd_mem_free: pd=%p, pd_context=%p, ptr=%p, resource_type=0x%lx\n", + pd, pd_context, ptr, resource_type); + + assert(pd_context); + + gds_peer_attr *peer_attr = (gds_peer_attr *)pd_context; + gds_peer *peer = peer_from_id(peer_attr->peer_id); + + assert(peer->obj); + gds_mlx5_qp_peer_t *qp_peer = (gds_mlx5_qp_peer_t *)peer->obj; + + if (qp_peer->dbr.gbuf) { + if (qp_peer->dbr.va_id) { + peer_attr->unregister_va(qp_peer->dbr.va_id, peer_attr->peer_id); + qp_peer->dbr.va_id = 0; + } + peer_attr->buf_release(qp_peer->dbr.gbuf); + qp_peer->dbr.gbuf = NULL; + } +} + +int gds_mlx5_alloc_parent_domain(struct ibv_pd *p_pd, struct ibv_context *ibctx, gds_peer_attr *peer_attr, struct ibv_pd **out_pd, gds_mlx5_qp_peer_t **out_qp_peer) +{ + int ret = 0; + + struct ibv_parent_domain_init_attr pd_init_attr; + struct ibv_pd *pd = NULL; + gds_peer *peer = peer_from_id(peer_attr->peer_id); + + gds_mlx5_qp_peer_t *qp_peer = (gds_mlx5_qp_peer_t *)calloc(1, sizeof(gds_mlx5_qp_peer_t)); + if (!qp_peer) { + gds_err("cannot allocate memory\n"); + ret = ENOMEM; + goto err; + } + + qp_peer->peer_attr = peer_attr; + peer->obj = qp_peer; + + memset(&pd_init_attr, 0, sizeof(ibv_parent_domain_init_attr)); + pd_init_attr.pd = p_pd; + pd_init_attr.comp_mask = IBV_PARENT_DOMAIN_INIT_ATTR_ALLOCATORS | IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT; + pd_init_attr.alloc = pd_mem_alloc; + pd_init_attr.free = pd_mem_free; + pd_init_attr.pd_context = peer_attr; + + pd = ibv_alloc_parent_domain(ibctx, &pd_init_attr); + if (!pd) { + gds_err("error in ibv_alloc_parent_domain\n"); + ret = EINVAL; + goto err; + } + + *out_pd = pd; + *out_qp_peer = qp_peer; + return 0; + +err: + if (qp_peer) + free(qp_peer); + return ret; +} + +//----------------------------------------------------------------------------- + +/* + A) plain+membar: + WR32 + MEMBAR + WR32 + WR32 + + B) plain: + WR32 + WR32+PREBARRIER + WR32 + + C) sim64+membar: + WR32 + MEMBAR + INLCPY 8B + + D) sim64: + INLCPY 4B + POSTBARRIER + INLCPY 8B + + E) inlcpy+membar: + WR32 + MEMBAR + INLCPY XB + + F) inlcpy: + INLCPY 4B + POSTBARRIER + INLCPY 128B + */ + +int gds_mlx5_post_ops(gds_peer *peer, size_t n_ops, struct gds_mlx5_peer_op_wr *op, gds_op_list_t &ops, int post_flags) +{ + int retcode = 0; + size_t n = 0; + bool prev_was_fence = false; + bool use_inlcpy_for_dword = false; + CUstreamBatchMemOpParams param; + + gds_dbg("n_ops=%zu\n", n_ops); + + if (!peer->has_memops) { + gds_err("CUDA MemOps are required\n"); + return EINVAL; + } + + // divert the request to the same engine handling 64bits + // to avoid out-of-order execution + // caveat: can't use membar if inlcpy is used for 4B writes (to simulate 8B writes) + if (peer->has_inlcpy) { + if (!peer->has_membar) + use_inlcpy_for_dword = true; // F + } + if (gds_simulate_write64()) { + if (!peer->has_membar) { + gds_warn_once("enabling use_inlcpy_for_dword\n"); + use_inlcpy_for_dword = true; // D + } + } + + for (; op && n < n_ops; op = op->next, ++n) { + gds_dbg("op[%zu] type:%08x\n", n, op->type); + switch(op->type) { + case GDS_MLX5_PEER_OP_FENCE: { + gds_dbg("OP_FENCE: fence_flags=%" PRIu64 "\n", op->wr.fence.fence_flags); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); + + if (fence_op == GDS_PEER_FENCE_OP_READ) { + gds_dbg("nothing to do for read fences\n"); + break; + } + else { + if (!peer->has_membar) { + if (use_inlcpy_for_dword) { + assert(ops.size() > 0); + gds_dbg("patching previous param\n"); + gds_enable_barrier_for_inlcpy(&ops.back()); + } + else { + gds_dbg("recording fence event\n"); + prev_was_fence = true; + } + } + else { + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { + gds_err("unexpected from fence\n"); + retcode = EINVAL; + break; + } + int flags = 0; + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { + gds_dbg("using light membar\n"); + flags = GDS_MEMBAR_DEFAULT | GDS_MEMBAR_MLX5; + } + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { + gds_dbg("using heavy membar\n"); + flags = GDS_MEMBAR_SYS | GDS_MEMBAR_MLX5; + } + else { + gds_err("unsupported fence combination\n"); + retcode = EINVAL; + break; + } + retcode = gds_fill_membar(peer, ops, flags); + } + } + break; + } + case GDS_MLX5_PEER_OP_STORE_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + int flags = 0; + gds_dbg("OP_STORE_DWORD dev_ptr=%llx data=%" PRIx32 "\n", dev_ptr, data); + if (use_inlcpy_for_dword) { // F || D + // membar may be out of order WRT inlcpy + if (peer->has_membar) { + gds_err("invalid feature combination, inlcpy + membar\n"); + retcode = EINVAL; + break; + } + // tail flush is set when following fence is met + // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; + retcode = gds_fill_inlcpy(peer, ops, dev_ptr, &data, sizeof(data), flags); + } + else { // A || B || C || E + // can't guarantee ordering of write32+inlcpy unless + // a membar is there + // TODO: fix driver when !weak + if (peer->has_inlcpy && !peer->has_membar) { + gds_err("invalid feature combination, inlcpy needs membar\n"); + retcode = EINVAL; + break; + } + if (prev_was_fence) { + gds_dbg("using PRE_BARRIER as fence\n"); + flags |= GDS_WRITE_PRE_BARRIER; + prev_was_fence = false; + } + retcode = gds_fill_poke(peer, ops, dev_ptr, data, flags); + } + break; + } + case GDS_MLX5_PEER_OP_STORE_QWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + + op->wr.qword_va.offset; + uint64_t data = op->wr.qword_va.data; + int flags = 0; + gds_dbg("OP_STORE_QWORD dev_ptr=%llx data=%" PRIx64 "\n", dev_ptr, data); + // C || D + if (gds_simulate_write64()) { + // simulate 64-bit poke by inline copy + if (!peer->has_membar) { + gds_err("invalid feature combination, inlcpy needs membar\n"); + retcode = EINVAL; + break; + } + + // tail flush is never useful here + //flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; + retcode = gds_fill_inlcpy(peer, ops, dev_ptr, &data, sizeof(data), flags); + } + else if (peer->has_write64) { + retcode = gds_fill_poke64(peer, ops, dev_ptr, data, flags); + } + else { + uint32_t datalo = gds_qword_lo(op->wr.qword_va.data); + uint32_t datahi = gds_qword_hi(op->wr.qword_va.data); + + if (prev_was_fence) { + gds_dbg("enabling PRE_BARRIER\n"); + flags |= GDS_WRITE_PRE_BARRIER; + prev_was_fence = false; + } + retcode = gds_fill_poke(peer, ops, dev_ptr, datalo, flags); + + // get rid of the barrier, if there + flags &= ~GDS_WRITE_PRE_BARRIER; + + // advance to next DWORD + dev_ptr += sizeof(uint32_t); + retcode = gds_fill_poke(peer, ops, dev_ptr, datahi, flags); + } + + break; + } + case GDS_MLX5_PEER_OP_COPY_BLOCK: { + CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + + op->wr.copy_op.offset; + size_t len = op->wr.copy_op.len; + void *src = op->wr.copy_op.src; + int flags = 0; + gds_dbg("OP_COPY_BLOCK dev_ptr=%llx src=%p len=%zu\n", dev_ptr, src, len); + // catching any other size here + if (!peer->has_inlcpy) { + gds_err("inline copy is not supported\n"); + retcode = EINVAL; + break; + } + // IB Verbs bug + assert(len <= GDS_GPU_MAX_INLINE_SIZE); + //if (desc->need_flush) { + // flags |= GDS_IMMCOPY_POST_TAIL_FLUSH; + //} + retcode = gds_fill_inlcpy(peer, ops, dev_ptr, src, len, flags); + break; + } + case GDS_MLX5_PEER_OP_POLL_AND_DWORD: + case GDS_MLX5_PEER_OP_POLL_GEQ_DWORD: + case GDS_MLX5_PEER_OP_POLL_NOR_DWORD: { + int poll_cond; + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + uint32_t data = op->wr.dword_va.data; + // TODO: properly handle a following fence instead of blidly flushing + int flags = 0; + if (!(post_flags & GDS_POST_OPS_DISCARD_WAIT_FLUSH)) + flags |= GDS_WAIT_POST_FLUSH_REMOTE; + + gds_dbg("OP_WAIT_DWORD dev_ptr=%llx data=%" PRIx32 " type=%" PRIx32 "\n", dev_ptr, data, (uint32_t)op->type); + + switch(op->type) { + case GDS_MLX5_PEER_OP_POLL_NOR_DWORD: + poll_cond = GDS_WAIT_COND_NOR; + break; + case GDS_MLX5_PEER_OP_POLL_GEQ_DWORD: + poll_cond = GDS_WAIT_COND_GEQ; + break; + case GDS_MLX5_PEER_OP_POLL_AND_DWORD: + poll_cond = GDS_WAIT_COND_AND; + break; + default: + assert(!"cannot happen"); + retcode = EINVAL; + goto out; + } + retcode = gds_fill_poll(peer, ops, dev_ptr, data, poll_cond, flags); + break; + } + default: + gds_err("undefined peer op type %d\n", op->type); + retcode = EINVAL; + break; + } + if (retcode) { + gds_err("error in fill func at entry n=%zu\n", n); + goto out; + } + } + + assert(n_ops == n); + +out: + return retcode; +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_post_ops_on_cpu(size_t n_ops, struct gds_mlx5_peer_op_wr *op, int post_flags) +{ + int retcode = 0; + size_t n = 0; + gds_dbg("n_ops=%zu op=%p post_flags=0x%x\n", n_ops, op, post_flags); + for (; op && n < n_ops; op = op->next, ++n) { + gds_dbg("op[%zu]=%p\n", n, op); + switch(op->type) { + case GDS_MLX5_PEER_OP_FENCE: { + gds_dbg("FENCE flags=%" PRIu64 "\n", op->wr.fence.fence_flags); + uint32_t fence_op = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_OP_READ|GDS_PEER_FENCE_OP_WRITE)); + uint32_t fence_from = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_FROM_CPU|GDS_PEER_FENCE_FROM_HCA)); + uint32_t fence_mem = (op->wr.fence.fence_flags & (GDS_PEER_FENCE_MEM_SYS|GDS_PEER_FENCE_MEM_PEER)); + + if (fence_op == GDS_PEER_FENCE_OP_READ) { + gds_warnc(1, "nothing to do for read fences\n"); + break; + } + else { + if (fence_from != GDS_PEER_FENCE_FROM_HCA) { + gds_err("unexpected from %08x fence, expected FROM_HCA\n", fence_from); + retcode = EINVAL; + break; + } + if (fence_mem == GDS_PEER_FENCE_MEM_PEER) { + gds_dbg("using light membar\n"); + wmb(); + } + else if (fence_mem == GDS_PEER_FENCE_MEM_SYS) { + gds_dbg("using heavy membar\n"); + wmb(); + } + else { + gds_err("unsupported fence combination\n"); + retcode = EINVAL; + break; + } + } + break; + } + case GDS_MLX5_PEER_OP_STORE_DWORD: { + uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); + uint32_t data = op->wr.dword_va.data; + // A || B || C || E + gds_dbg("STORE_DWORD ptr=%p data=%08" PRIx32 "\n", ptr, data); + gds_atomic_set(ptr, data); + break; + } + case GDS_MLX5_PEER_OP_STORE_QWORD: { + uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.qword_va.target_id)->va + op->wr.qword_va.offset); + uint64_t data = op->wr.qword_va.data; + gds_dbg("STORE_QWORD ptr=%p data=%016" PRIx64 "\n", ptr, data); + gds_atomic_set(ptr, data); + break; + } + case GDS_MLX5_PEER_OP_COPY_BLOCK: { + uint64_t *ptr = (uint64_t*)((ptrdiff_t)range_from_id(op->wr.copy_op.target_id)->va + op->wr.copy_op.offset); + uint64_t *src = (uint64_t*)op->wr.copy_op.src; + size_t n_bytes = op->wr.copy_op.len; + gds_dbg("COPY_BLOCK ptr=%p src=%p len=%zu\n", ptr, src, n_bytes); + gds_bf_copy(ptr, src, n_bytes); + break; + } + case GDS_MLX5_PEER_OP_POLL_AND_DWORD: + case GDS_MLX5_PEER_OP_POLL_GEQ_DWORD: + case GDS_MLX5_PEER_OP_POLL_NOR_DWORD: { + int poll_cond; + uint32_t *ptr = (uint32_t*)((ptrdiff_t)range_from_id(op->wr.dword_va.target_id)->va + op->wr.dword_va.offset); + uint32_t value = op->wr.dword_va.data; + bool flush = true; + if (post_flags & GDS_POST_OPS_DISCARD_WAIT_FLUSH) + flush = false; + gds_dbg("WAIT_32 dev_ptr=%p data=%" PRIx32 " type=%" PRIx32 "\n", ptr, value, (uint32_t)op->type); + bool done = false; + do { + uint32_t data = gds_atomic_get(ptr); + switch(op->type) { + case GDS_MLX5_PEER_OP_POLL_NOR_DWORD: + done = (0 != ~(data | value)); + break; + case GDS_MLX5_PEER_OP_POLL_GEQ_DWORD: + done = ((int32_t)data - (int32_t)value >= 0); + break; + case GDS_MLX5_PEER_OP_POLL_AND_DWORD: + done = (0 != (data & value)); + break; + default: + gds_err("invalid op type %02x\n", op->type); + retcode = EINVAL; + goto out; + } + if (done) + break; + // TODO: more aggressive CPU relaxing needed here to avoid starving I/O fabric + arch_cpu_relax(); + } while(true); + break; + } + default: + gds_err("undefined peer op type %d\n", op->type); + retcode = EINVAL; + break; + } + if (retcode) { + gds_err("error %d at entry n=%zu\n", retcode, n); + goto out; + } + } + +out: + return retcode; +} + +//----------------------------------------------------------------------------- + +void gds_mlx5_dump_ops(struct gds_mlx5_peer_op_wr *op, size_t count) +{ + size_t n = 0; + for (; op; op = op->next, ++n) { + gds_dbg("op[%zu] type:%d\n", n, op->type); + switch(op->type) { + case GDS_MLX5_PEER_OP_FENCE: { + gds_dbg("FENCE flags=%" PRIu64 "\n", op->wr.fence.fence_flags); + break; + } + case GDS_MLX5_PEER_OP_STORE_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + gds_dbg("STORE_QWORD data:%x target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", + op->wr.dword_va.data, op->wr.dword_va.target_id, + op->wr.dword_va.offset, dev_ptr); + break; + } + case GDS_MLX5_PEER_OP_STORE_QWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.qword_va.target_id)->dptr + + op->wr.qword_va.offset; + gds_dbg("STORE_QWORD data:%" PRIx64 " target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", + op->wr.qword_va.data, op->wr.qword_va.target_id, + op->wr.qword_va.offset, dev_ptr); + break; + } + case GDS_MLX5_PEER_OP_COPY_BLOCK: { + CUdeviceptr dev_ptr = range_from_id(op->wr.copy_op.target_id)->dptr + + op->wr.copy_op.offset; + gds_dbg("COPY_BLOCK src:%p len:%zu target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", + op->wr.copy_op.src, op->wr.copy_op.len, + op->wr.copy_op.target_id, op->wr.copy_op.offset, + dev_ptr); + break; + } + case GDS_MLX5_PEER_OP_POLL_AND_DWORD: + case GDS_MLX5_PEER_OP_POLL_NOR_DWORD: { + CUdeviceptr dev_ptr = range_from_id(op->wr.dword_va.target_id)->dptr + + op->wr.dword_va.offset; + gds_dbg("%s data:%08x target_id:%" PRIx64 " offset:%zu dev_ptr=%llx\n", + (op->type==GDS_MLX5_PEER_OP_POLL_AND_DWORD) ? "POLL_AND_DW" : "POLL_NOR_SDW", + op->wr.dword_va.data, + op->wr.dword_va.target_id, + op->wr.dword_va.offset, + dev_ptr); + break; + } + default: + gds_err("undefined peer op type %d\n", op->type); + break; + } + } + + assert(count == n); +} + +//----------------------------------------------------------------------------- + +int gds_mlx5_poll_cq(gds_mlx5_cq_t *mcq, int ne, struct ibv_wc *wc) +{ + unsigned int idx; + int cnt; + int p_ne; + + void *cqe; + struct mlx5_cqe64 *cqe64; + + uint16_t wqe_ctr; + int wqe_ctr_idx; + + assert(mcq->gcq.ctype == GDS_CQ_TYPE_SQ); + + for (cnt = 0; cnt < ne; ++cnt) { + idx = mcq->cons_index & (mcq->dvcq.cqe_cnt - 1); + while (mcq->peer_peek_table[idx]) { + struct gds_mlx5_peek_entry *tmp; + if (*(volatile uint32_t *)&mcq->peer_peek_table[idx]->busy) { + return cnt; + } + tmp = mcq->peer_peek_table[idx]; + mcq->peer_peek_table[idx] = GDS_MLX5_PEEK_ENTRY(mcq, tmp->next); + tmp->next = GDS_MLX5_PEEK_ENTRY_N(mcq, mcq->peer_peek_free); + mcq->peer_peek_free = tmp; + } + cqe = (void *)((uintptr_t)mcq->dvcq.buf + mcq->cons_index * mcq->dvcq.cqe_size); + cqe64 = (mcq->dvcq.cqe_size == 64) ? (struct mlx5_cqe64 *)cqe : (struct mlx5_cqe64 *)((uintptr_t)cqe + 64); + + wqe_ctr = be16toh(cqe64->wqe_counter); + wqe_ctr_idx = wqe_ctr & (mcq->wq->wqe_cnt - 1); + + p_ne = ibv_poll_cq(mcq->gcq.ibcq, 1, wc + cnt); + if (p_ne <= 0) + return p_ne; + + wc[cnt].wr_id = mcq->wq->wrid[wqe_ctr_idx]; + + mcq->wq->tail = mcq->wq->wqe_head[wqe_ctr_idx] + 1; + + ++mcq->cons_index; + } + return cnt; +} + +//----------------------------------------------------------------------------- + + /* * Local variables: * c-indent-level: 8 diff --git a/src/mlx5.hpp b/src/mlx5.hpp new file mode 100644 index 0000000..5b1afcc --- /dev/null +++ b/src/mlx5.hpp @@ -0,0 +1,264 @@ +/* Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "objs.hpp" +#include "utils.hpp" + +//----------------------------------------------------------------------------- + +#define GDS_MLX5_ROLLBACK_ID_PARITY_MASK (1ULL << 63) +#define GDS_MLX5_LAST_PEEK_ENTRY (-1U) +#define GDS_MLX5_PEEK_ENTRY(cq, n) \ + (n == GDS_MLX5_LAST_PEEK_ENTRY ? NULL : \ + ((struct gds_mlx5_peek_entry *)cq->peer_buf->addr) + n) +#define GDS_MLX5_PEEK_ENTRY_N(cq, pe) \ + (pe == NULL ? GDS_MLX5_LAST_PEEK_ENTRY : \ + ((pe - (struct gds_mlx5_peek_entry *)cq->peer_buf->addr))) + +struct gds_mlx5_peek_entry { + uint32_t busy; + uint32_t next; +}; + +enum { + GDS_MLX5_PEER_PEEK_ABSOLUTE, + GDS_MLX5_PEER_PEEK_RELATIVE +}; + +struct gds_mlx5_peer_peek { + /* IN/OUT - linked list of empty/filled descriptors */ + struct gds_mlx5_peer_op_wr *storage; + /* IN/OUT - number of allocated/filled descriptors */ + uint32_t entries; + /* IN - Which CQ entry does the peer want to peek for + * completion. According to "whence" directive entry + * chosen as follows: + * IBV_EXP_PEER_PEEK_ABSOLUTE - + * "offset" is absolute index of entry wrapped to 32-bit + * IBV_EXP_PEER_PEEK_RELATIVE - + * "offset" is relative to current poll_cq location. + */ + uint32_t whence; + uint32_t offset; + /* OUT - identifier used in ibv_exp_peer_ack_peek_cq to advance CQ */ + uint64_t peek_id; + uint32_t comp_mask; /* Reserved for future expensions, must be 0 */ +}; + +enum gds_mlx5_peer_op { + GDS_MLX5_PEER_OP_RESERVED1 = 1, + + GDS_MLX5_PEER_OP_FENCE = 0, + + GDS_MLX5_PEER_OP_STORE_DWORD = 4, + GDS_MLX5_PEER_OP_STORE_QWORD = 2, + GDS_MLX5_PEER_OP_COPY_BLOCK = 3, + + GDS_MLX5_PEER_OP_POLL_AND_DWORD = 12, + GDS_MLX5_PEER_OP_POLL_NOR_DWORD = 13, + GDS_MLX5_PEER_OP_POLL_GEQ_DWORD = 14, +}; + +enum gds_peer_op_caps { + GDS_PEER_OP_FENCE_CAP = (1 << GDS_MLX5_PEER_OP_FENCE), + GDS_PEER_OP_STORE_DWORD_CAP = (1 << GDS_MLX5_PEER_OP_STORE_DWORD), + GDS_PEER_OP_STORE_QWORD_CAP = (1 << GDS_MLX5_PEER_OP_STORE_QWORD), + GDS_PEER_OP_COPY_BLOCK_CAP = (1 << GDS_MLX5_PEER_OP_COPY_BLOCK), + GDS_PEER_OP_POLL_AND_DWORD_CAP + = (1 << GDS_MLX5_PEER_OP_POLL_AND_DWORD), + GDS_PEER_OP_POLL_NOR_DWORD_CAP + = (1 << GDS_MLX5_PEER_OP_POLL_NOR_DWORD), + GDS_PEER_OP_POLL_GEQ_DWORD_CAP + = (1 << GDS_MLX5_PEER_OP_POLL_GEQ_DWORD), +}; + +struct gds_mlx5_peer_op_wr { + struct gds_mlx5_peer_op_wr *next; + enum gds_mlx5_peer_op type; + union { + struct { + uint64_t fence_flags; /* from gds_peer_fence */ + } fence; + + struct { + uint32_t data; + uint64_t target_id; + size_t offset; + } dword_va; /* Use for all operations targeting dword */ + + struct { + uint64_t data; + uint64_t target_id; + size_t offset; + } qword_va; /* Use for all operations targeting qword */ + + struct { + void *src; + uint64_t target_id; + size_t offset; + size_t len; + } copy_op; + } wr; + uint32_t comp_mask; /* Reserved for future expensions, must be 0 */ +}; + +struct gds_mlx5_peer_commit { + /* IN/OUT - linked list of empty/filled descriptors */ + struct gds_mlx5_peer_op_wr *storage; + /* IN/OUT - number of allocated/filled descriptors */ + uint32_t entries; + /* OUT - identifier used in gds_rollback_qp to rollback WQEs set */ + uint64_t rollback_id; + uint32_t comp_mask; /* Reserved for future expensions, must be 0 */ +}; + +enum gds_mlx5_rollback_flags { + /* Abort all WQEs which were not committed to HW yet. + * rollback_id is ignored. **/ + GDS_MLX5_ROLLBACK_ABORT_UNCOMMITED = (1 << 0), + /* Abort the request even if there are following requests + * being aborted as well. **/ + GDS_MLX5_ROLLBACK_ABORT_LATE = (1 << 1), +}; + +struct gds_mlx5_rollback_ctx { + uint64_t rollback_id; /* from ibv_exp_peer_commit call */ + uint32_t flags; /* from ibv_exp_rollback_flags */ + uint32_t comp_mask; /* Reserved for future expensions, must be 0 */ +}; + +typedef struct gds_mlx5_send_request { + struct gds_mlx5_peer_commit commit; + struct gds_mlx5_peer_op_wr wr[GDS_SEND_INFO_MAX_OPS]; +} gds_mlx5_send_request_t; + +static inline gds_mlx5_send_request_t *to_gds_msreq(gds_send_request_t *req) { + return (gds_mlx5_send_request_t *)req; +} + +typedef struct gds_mlx5_wait_request { + struct gds_mlx5_peer_peek peek; + struct gds_mlx5_peer_op_wr wr[GDS_WAIT_INFO_MAX_OPS]; +} gds_mlx5_wait_request_t; + +static inline gds_mlx5_wait_request_t *to_gds_mwreq(gds_wait_request_t *req) { + return (gds_mlx5_wait_request_t *)req; +} + +typedef struct gds_mlx5_wq { + uint64_t *wrid; + uint64_t *wqe_head; + unsigned int wqe_cnt; + uint64_t head; + uint64_t tail; +} gds_mlx5_wq_t; + +typedef struct gds_mlx5_cq { + gds_cq_t gcq; + uint32_t cons_index; + struct mlx5dv_cq dvcq; + gds_mlx5_wq_t *wq; + uint64_t active_buf_va_id; + gds_peer_attr *peer_attr; + uint64_t peer_va_id; + uint32_t peer_dir; + struct gds_buf *peer_buf; + struct gds_mlx5_peek_entry **peer_peek_table; + struct gds_mlx5_peek_entry *peer_peek_free; +} gds_mlx5_cq_t; + +typedef struct gds_mlx5_qp_peer { + gds_peer_attr *peer_attr; + uint32_t scur_post; + + struct { + uint64_t va_id; + size_t size; + gds_buf *gbuf; + } dbr; + + struct { + uint64_t va_id; + } bf; +} gds_mlx5_qp_peer_t; + +typedef struct gds_mlx5_qp { + gds_qp_t gqp; + + struct mlx5dv_qp dvqp; + + unsigned int sq_cur_post; + uint8_t sq_signal_bits; + + unsigned int bf_offset; + + uint8_t fm_cache; + + gds_mlx5_qp_peer_t *qp_peer; + + gds_mlx5_wq_t *wq; +} gds_mlx5_qp_t; + +static inline gds_mlx5_cq_t *to_gds_mcq(struct gds_cq *gcq) { + return container_of(gcq, gds_mlx5_cq_t, gcq); +} + +static inline gds_mlx5_qp_t *to_gds_mqp(struct gds_qp *gqp) { + return container_of(gqp, gds_mlx5_qp_t, gqp); +} + +int gds_mlx5_rollback_send(gds_mlx5_qp_t *mqp, struct gds_mlx5_rollback_ctx *rollback); +int gds_mlx5_post_send(gds_mlx5_qp_t *mqp, gds_send_wr *p_ewr, gds_send_wr **bad_ewr, gds_mlx5_peer_commit *commit); +int gds_mlx5_peer_peek_cq(gds_mlx5_cq_t *mcq, struct gds_mlx5_peer_peek *peek); + +int gds_mlx5_create_cq(struct ibv_cq *ibcq, gds_peer_attr *peer_attr, gds_mlx5_cq_t **out_mcq); +void gds_mlx5_destroy_cq(gds_mlx5_cq_t *mcq); + +int gds_mlx5_alloc_parent_domain(struct ibv_pd *p_pd, struct ibv_context *ibctx, gds_peer_attr *peer_attr, struct ibv_pd **out_pd, gds_mlx5_qp_peer_t **out_qp_peer); + +int gds_mlx5_create_qp(struct ibv_qp *ibqp, gds_qp_init_attr_t *qp_attr, gds_mlx5_cq_t *tx_mcq, gds_mlx5_cq_t *rx_mcq, gds_mlx5_qp_peer_t *qp_peer, gds_mlx5_qp_t **out_mqp); +void gds_mlx5_destroy_qp(gds_mlx5_qp_t *mqp); + +int gds_mlx5_post_ops_on_cpu(size_t n_ops, struct gds_mlx5_peer_op_wr *op, int post_flags = 0); +int gds_mlx5_post_ops(gds_peer *peer, size_t n_ops, struct gds_mlx5_peer_op_wr *op, gds_op_list_t &ops, int post_flags = 0); + +void gds_mlx5_dump_ops(struct gds_mlx5_peer_op_wr *op, size_t count); + +int gds_mlx5_poll_cq(gds_mlx5_cq_t *mcq, int ne, struct ibv_wc *wc); + +//----------------------------------------------------------------------------- + +/* + * Local variables: + * c-indent-level: 8 + * c-basic-offset: 8 + * tab-width: 8 + * indent-tabs-mode: nil + * End: + */ diff --git a/src/objs.cpp b/src/objs.cpp index 1dac250..b744c53 100644 --- a/src/objs.cpp +++ b/src/objs.cpp @@ -33,13 +33,11 @@ #include #include -//#include #include #include using namespace std; #include -#include #include #include "gdsync.h" @@ -51,14 +49,14 @@ using namespace std; //----------------------------------------------------------------------------- -gds_buf *gds_peer::alloc(size_t sz, uint32_t alignment) +gds_buf *gds_peer::alloc(size_t sz, uint32_t alignment, gds_memory_type_t mem_type) { // TODO: support alignment // TODO: handle exception here gds_buf *buf = new gds_buf(this, sz); if (!buf) return buf; - int ret = gds_peer_malloc(gpu_id, 0, &buf->addr, &buf->peer_addr, buf->length, &buf->handle); + int ret = gds_peer_malloc(gpu_id, 0, &buf->addr, &buf->peer_addr, buf->length, &buf->handle, mem_type); if (ret) { delete buf; buf = NULL; @@ -71,28 +69,30 @@ gds_buf *gds_peer::buf_alloc_cq(size_t length, uint32_t dir, uint32_t alignment, { gds_buf *buf = NULL; switch (dir) { - case (IBV_EXP_PEER_DIRECTION_FROM_HCA|IBV_EXP_PEER_DIRECTION_TO_PEER|IBV_EXP_PEER_DIRECTION_TO_CPU): - // CQ buf - if (GDS_ALLOC_CQ_ON_GPU == (flags & GDS_ALLOC_CQ_MASK)) { - gds_dbg("allocating CQ on GPU mem\n"); - buf = alloc(length, alignment); - } else { - gds_dbg("allocating CQ on Host mem\n"); - } - break; - case (IBV_EXP_PEER_DIRECTION_FROM_PEER|IBV_EXP_PEER_DIRECTION_TO_CPU): - // CQ peer buf, helper buffer - // on SYSMEM for the near future - // GPU does a store to the 'busy' field as part of the peek_cq task - // CPU polls on that field - gds_dbg("allocating CQ peer buf on Host mem\n"); - break; - case (IBV_EXP_PEER_DIRECTION_FROM_PEER|IBV_EXP_PEER_DIRECTION_TO_HCA): - gds_dbg("allocating CQ dbrec on Host mem\n"); - break; - default: - gds_err("unexpected dir 0x%x\n", dir); - break; + case (GDS_PEER_DIRECTION_FROM_HCA|GDS_PEER_DIRECTION_TO_PEER|GDS_PEER_DIRECTION_TO_CPU): + // CQ buf + if (GDS_ALLOC_CQ_ON_GPU == (flags & GDS_ALLOC_CQ_MASK)) { + gds_dbg("allocating CQ on GPU mem\n"); + buf = alloc(length, alignment, GDS_MEMORY_GPU); + } else { + gds_dbg("allocating CQ on Host mem\n"); + } + break; + case (GDS_PEER_DIRECTION_FROM_PEER|GDS_PEER_DIRECTION_TO_CPU): + // CQ peer buf, helper buffer + // on SYSMEM for the near future + // GPU does a store to the 'busy' field as part of the peek_cq task + // CPU polls on that field + gds_dbg("allocating CQ peer buf on Host mem\n"); + buf = alloc(length, alignment, GDS_MEMORY_HOST); + break; + case (GDS_PEER_DIRECTION_FROM_PEER|GDS_PEER_DIRECTION_TO_HCA): + gds_dbg("allocating CQ dbrec on Host mem\n"); + buf = alloc(length, alignment, GDS_MEMORY_HOST); + break; + default: + gds_err("unexpected dir 0x%x\n", dir); + break; } return buf; } @@ -101,18 +101,18 @@ gds_buf *gds_peer::buf_alloc_wq(size_t length, uint32_t dir, uint32_t alignment, { gds_buf *buf = NULL; switch (dir) { - case IBV_EXP_PEER_DIRECTION_FROM_PEER|IBV_EXP_PEER_DIRECTION_TO_HCA: - // dbrec - if (GDS_ALLOC_DBREC_ON_GPU == (flags & GDS_ALLOC_DBREC_MASK)) { - gds_dbg("allocating DBREC on GPU mem\n"); - buf = alloc(length, alignment); - } else { - gds_dbg("allocating DBREC on Host mem\n"); - } - break; - default: - gds_err("unexpected dir=%08x\n", dir); - break; + case GDS_PEER_DIRECTION_FROM_PEER|GDS_PEER_DIRECTION_TO_HCA: + // dbrec + if (GDS_ALLOC_DBREC_ON_GPU == (flags & GDS_ALLOC_DBREC_MASK)) { + gds_dbg("allocating DBREC on GPU mem\n"); + buf = alloc(length, alignment, GDS_MEMORY_GPU); + } else { + gds_dbg("allocating DBREC on Host mem\n"); + } + break; + default: + gds_err("unexpected dir=%08x\n", dir); + break; } return buf; } @@ -122,15 +122,15 @@ gds_buf *gds_peer::buf_alloc(obj_type type, size_t length, uint32_t dir, uint32_ gds_buf *buf = NULL; gds_dbg("type=%d dir=%08x flags=%08x\n", type, dir, flags); switch (type) { - case CQ: - buf = buf_alloc_cq(length, dir, alignment, flags); - break; - case WQ: - buf = buf_alloc_wq(length, dir, alignment, flags); - break; - default: - gds_err("unexpected obj type=%d\n", type); - break; + case CQ: + buf = buf_alloc_cq(length, dir, alignment, flags); + break; + case WQ: + buf = buf_alloc_wq(length, dir, alignment, flags); + break; + default: + gds_err("unexpected obj type=%d\n", type); + break; } return buf; @@ -151,7 +151,7 @@ gds_range *gds_peer::range_from_buf(gds_buf *buf, void *start, size_t length) gds_range *range = new gds_range; gds_dbg("buf=%p\n", buf); assert((ptrdiff_t)start >= (ptrdiff_t)buf->addr && - (ptrdiff_t)start + length <= (ptrdiff_t)buf->addr + buf->length); + (ptrdiff_t)start + length <= (ptrdiff_t)buf->addr + buf->length); range->va = start; // CPU mapping range->dptr = buf->peer_addr + ((ptrdiff_t)start - (ptrdiff_t)buf->addr); range->size = length; diff --git a/src/objs.hpp b/src/objs.hpp index 796b6bd..a43540b 100644 --- a/src/objs.hpp +++ b/src/objs.hpp @@ -27,13 +27,44 @@ #pragma once +#include "task_queue.hpp" + static const size_t max_gpus = 16; -typedef struct ibv_exp_peer_direct_attr gds_peer_attr; +enum gds_peer_fence { + GDS_PEER_FENCE_OP_READ = (1 << 0), + GDS_PEER_FENCE_OP_WRITE = (1 << 1), + GDS_PEER_FENCE_FROM_CPU = (1 << 2), + GDS_PEER_FENCE_FROM_HCA = (1 << 3), + GDS_PEER_FENCE_MEM_SYS = (1 << 4), + GDS_PEER_FENCE_MEM_PEER = (1 << 5), +}; + +enum gds_peer_direction { + GDS_PEER_DIRECTION_FROM_CPU = (1 << 0), + GDS_PEER_DIRECTION_FROM_HCA = (1 << 1), + GDS_PEER_DIRECTION_FROM_PEER = (1 << 2), + GDS_PEER_DIRECTION_TO_CPU = (1 << 3), + GDS_PEER_DIRECTION_TO_HCA = (1 << 4), + GDS_PEER_DIRECTION_TO_PEER = (1 << 5), +}; + +#define GDS_PEER_IOMEMORY ((struct gds_buf *)-1UL) struct gds_peer; -struct gds_buf: ibv_exp_peer_buf { +struct gds_buf_alloc_attr { + size_t length; + uint32_t dir; + uint64_t peer_id; + uint32_t alignment; + uint32_t comp_mask; +}; + +struct gds_buf { + void *addr; + size_t length; + uint32_t comp_mask; gds_peer *peer; CUdeviceptr peer_addr; void *handle; @@ -45,11 +76,24 @@ struct gds_buf: ibv_exp_peer_buf { } }; +typedef struct { + uint64_t peer_id; + struct gds_buf *(*buf_alloc)(struct gds_buf_alloc_attr *attr); + int (*buf_release)(struct gds_buf *pb); + uint64_t (*register_va)(void *start, size_t length, uint64_t peer_id, struct gds_buf *pb); + int (*unregister_va)(uint64_t target_id, uint64_t peer_id); + uint64_t caps; + size_t peer_dma_op_map_len; + uint32_t comp_mask; + uint32_t version; +} gds_peer_attr; + + struct gds_range { void *va; CUdeviceptr dptr; size_t size; - gds_buf *buf; + struct gds_buf *buf; gds_memory_type_t type; }; @@ -82,6 +126,7 @@ struct gds_peer { gds_peer_attr attr; task_queue *tq; + void *obj; enum obj_type { NONE, CQ, WQ, N_IBV_OBJS } alloc_type; // This field works as a ugly run-time parameters passing // mechanism, as it carries tracking info during the QP creation @@ -98,7 +143,7 @@ struct gds_peer { // unregister all kinds of memory void unregister(gds_range *range); - gds_buf *alloc(size_t length, uint32_t alignment); + gds_buf *alloc(size_t length, uint32_t alignment, gds_memory_type_t mem_type); gds_buf *buf_alloc_cq(size_t length, uint32_t dir, uint32_t alignment, int flags); gds_buf *buf_alloc_wq(size_t length, uint32_t dir, uint32_t alignment, int flags); gds_buf *buf_alloc(obj_type type, size_t length, uint32_t dir, uint32_t alignment, int flags); diff --git a/src/task_queue.hpp b/src/task_queue.hpp index 8c57757..4387d03 100644 --- a/src/task_queue.hpp +++ b/src/task_queue.hpp @@ -41,94 +41,94 @@ // single threaded task queue struct task_queue { - task_queue() { - TQDBG("CTOR"); - m_state = idle; - } + task_queue() { + TQDBG("CTOR"); + m_state = idle; + } - ~task_queue() { - TQDBG("DTOR"); - kill_worker(); - } + ~task_queue() { + TQDBG("DTOR"); + kill_worker(); + } - typedef std::function task_t; - typedef std::list task_list_t; - typedef task_list_t::iterator task_handle_t; + typedef std::function task_t; + typedef std::list task_list_t; + typedef task_list_t::iterator task_handle_t; - template - task_handle_t queue(Task t) { - lock_t lock(m_mtx); - task_handle_t handle = m_tasks.insert(m_tasks.end(), t); - if (m_state == idle) { - m_worker = std::thread(std::bind(&task_queue::worker_thread, this)); - while (m_state == idle) { - m_cond.wait(lock); - } - } - m_cond.notify_one(); - return handle; - } + template + task_handle_t queue(Task t) { + lock_t lock(m_mtx); + task_handle_t handle = m_tasks.insert(m_tasks.end(), t); + if (m_state == idle) { + m_worker = std::thread(std::bind(&task_queue::worker_thread, this)); + while (m_state == idle) { + m_cond.wait(lock); + } + } + m_cond.notify_one(); + return handle; + } - // WARNING: dequeuing can race with self-dequeing task - void dequeue(task_handle_t handle) { - TQDBG("dequing task"); - std::lock_guard g(m_mtx); - m_tasks.erase(handle); - } + // WARNING: dequeuing can race with self-dequeing task + void dequeue(task_handle_t handle) { + TQDBG("dequing task"); + std::lock_guard g(m_mtx); + m_tasks.erase(handle); + } -private: - void kill_worker() { - TQDBG("killing worker thread"); - { - lock_t g(m_mtx); - if (m_state == running) { - TQDBG("setting queue state to draining"); - m_state = draining; - m_cond.notify_one(); - } + private: + void kill_worker() { + TQDBG("killing worker thread"); + { + lock_t g(m_mtx); + if (m_state == running) { + TQDBG("setting queue state to draining"); + m_state = draining; + m_cond.notify_one(); + } + } + TQDBG("joining worker thread"); + m_worker.join(); } - TQDBG("joining worker thread"); - m_worker.join(); - } - void worker_thread() { - lock_t lock(m_mtx); - TQDBG("state<-running"); - m_state = running; - m_cond.notify_all(); - while(true) { - if (m_state == draining) { - TQDBG("state<-done"); - m_state = done; - break; - } - if (m_tasks.empty()) - m_cond.wait(lock); - auto f = m_tasks.begin(); - while (f != m_tasks.end()) { - auto fnext = std::next(f); - task_t t = *f; - lock.unlock(); - bool keep = t(); - lock.lock(); - if (!keep) { - TQDBG("removing task"); - m_tasks.erase(f); - TQDBG("done"); + void worker_thread() { + lock_t lock(m_mtx); + TQDBG("state<-running"); + m_state = running; + m_cond.notify_all(); + while(true) { + if (m_state == draining) { + TQDBG("state<-done"); + m_state = done; + break; + } + if (m_tasks.empty()) + m_cond.wait(lock); + auto f = m_tasks.begin(); + while (f != m_tasks.end()) { + auto fnext = std::next(f); + task_t t = *f; + lock.unlock(); + bool keep = t(); + lock.lock(); + if (!keep) { + TQDBG("removing task"); + m_tasks.erase(f); + TQDBG("done"); + } + f = fnext; + } } - f = fnext; - } } - } - enum { idle, running, draining, done } m_state; + enum { idle, running, draining, done } m_state; - // single threaded queue - std::thread m_worker; - std::mutex m_mtx; - typedef std::unique_lock lock_t; - task_list_t m_tasks; - std::condition_variable m_cond; + // single threaded queue + std::thread m_worker; + std::mutex m_mtx; + typedef std::unique_lock lock_t; + task_list_t m_tasks; + std::condition_variable m_cond; }; #undef TQDBG diff --git a/src/utils.hpp b/src/utils.hpp index b501bda..11b10dd 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -31,6 +31,44 @@ #warning "__STDC_FORMAT_MACROS should be defined to pull definition of PRIx64, etc" #endif #include // to pull PRIx64 +#include +#include + +#if (__GNUC__ >= 6 && !defined(__powerpc__)) || defined(__clang__) +#define uninitialized_var(x) x +#else +#define uninitialized_var(x) x = x +#endif + +#ifndef likely +#ifdef __GNUC__ +#define likely(x) __builtin_expect(!!(x), 1) +#else +#define likely(x) (x) +#endif +#endif + +#ifndef unlikely +#ifdef __GNUC__ +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define unlikely(x) (x) +#endif +#endif + +#ifndef container_of +#define container_of(ptr, type, member) ({ \ + void *__mptr = (void *)(ptr); \ + ((type *)((uintptr_t)__mptr - offsetof(type, member))); }) + +#endif + +#define MIN(x, y) ((x) < (y) ? (x) : (y)) + +static inline unsigned long align(unsigned long val, unsigned long align) +{ + return (val + align - 1) & ~(align - 1); +} // internal assert function @@ -39,9 +77,9 @@ void gds_assert(const char *cond, const char *file, unsigned line, const char *f #define GDS_ASSERT2(COND) \ do { \ if (!(COND)) \ - gds_assert(#COND, __FILE__, __LINE__, __FUNCTION__); \ + gds_assert(#COND, __FILE__, __LINE__, __FUNCTION__); \ } \ - while(0) +while(0) #define GDS_ASSERT(COND) GDS_ASSERT2(COND) @@ -49,26 +87,26 @@ void gds_assert(const char *cond, const char *file, unsigned line, const char *f // CUDA error checking #define __CUCHECK(stmt, cond_str) \ - do { \ - CUresult result = (stmt); \ - if (CUDA_SUCCESS != result) { \ - const char *err_str = NULL; \ - cuGetErrorString(result, &err_str); \ - fprintf(stderr, "Assertion \"%s != cudaSuccess\" failed at %s:%d error=%d(%s)\n", \ - cond_str, __FILE__, __LINE__, result, err_str); \ - exit(EXIT_FAILURE); \ - } \ + do { \ + CUresult result = (stmt); \ + if (CUDA_SUCCESS != result) { \ + const char *err_str = NULL; \ + cuGetErrorString(result, &err_str); \ + fprintf(stderr, "Assertion \"%s != cudaSuccess\" failed at %s:%d error=%d(%s)\n", \ + cond_str, __FILE__, __LINE__, result, err_str); \ + exit(EXIT_FAILURE); \ + } \ } while (0) #define CUCHECK(stmt) __CUCHECK(stmt, #stmt) -template + template static inline void gds_atomic_set(T *ptr, T value) { *(volatile T*)ptr = value; } -template + template static inline T gds_atomic_get(T *ptr) { return *(volatile T*)ptr; @@ -94,10 +132,10 @@ static inline T gds_atomic_get(T *ptr) // tracing support enum gds_msg_level { - GDS_MSG_DEBUG = 1, - GDS_MSG_INFO, - GDS_MSG_WARN, - GDS_MSG_ERROR + GDS_MSG_DEBUG = 1, + GDS_MSG_INFO, + GDS_MSG_WARN, + GDS_MSG_ERROR }; #define gds_stream stderr @@ -105,9 +143,9 @@ enum gds_msg_level { int gds_dbg_enabled(); #define gds_msg(LVL, LVLSTR, FMT, ARGS...) do { \ - fprintf(gds_stream, "[%d] GDS " LVLSTR " %s() " FMT, getpid(), __FUNCTION__ ,##ARGS); \ - fflush(gds_stream); \ - } while(0) + fprintf(gds_stream, "[%d] GDS " LVLSTR " %s() " FMT, getpid(), __FUNCTION__ ,##ARGS); \ + fflush(gds_stream); \ +} while(0) #define gds_dbg(FMT, ARGS...) do { if (gds_dbg_enabled()) gds_msg(GDS_MSG_DEBUG, "DBG ", FMT, ## ARGS); } while(0) #define gds_dbgc(CNT, FMT, ARGS...) do { static int __cnt = 0; if (__cnt++ < CNT) gds_dbg(FMT, ## ARGS); } while(0) @@ -121,19 +159,18 @@ int gds_dbg_enabled(); #define gds_err(FMT, ARGS...) gds_msg(GDS_MSG_ERROR, "ERR ", FMT, ##ARGS) - //----------------------------------------------------------------------------- static inline int gds_curesult_to_errno(CUresult result) { int retcode = 0; switch (result) { - case CUDA_SUCCESS: retcode = 0; break; - case CUDA_ERROR_NOT_SUPPORTED: retcode = EPERM; break; - case CUDA_ERROR_INVALID_VALUE: retcode = EINVAL; break; - case CUDA_ERROR_OUT_OF_MEMORY: retcode = ENOMEM; break; - // TODO: add missing cases - default: retcode = EIO; break; + case CUDA_SUCCESS: retcode = 0; break; + case CUDA_ERROR_NOT_SUPPORTED: retcode = EPERM; break; + case CUDA_ERROR_INVALID_VALUE: retcode = EINVAL; break; + case CUDA_ERROR_OUT_OF_MEMORY: retcode = ENOMEM; break; + // TODO: add missing cases + default: retcode = EIO; break; } return retcode; } @@ -189,6 +226,10 @@ typedef enum gds_alloc_qp_flags { GDS_ALLOC_DBREC_MASK = 1<<4 } gds_alloc_qp_flags_t; +// TODO: use correct value +// TODO: make it dependent upon the particular GPU +const size_t GDS_GPU_MAX_INLINE_SIZE = 256; + #include typedef std::vector gds_op_list_t; @@ -204,9 +245,13 @@ void gds_dump_params(gds_op_list_t ¶ms); struct gds_peer; int gds_fill_membar(gds_peer *peer, gds_op_list_t ¶m, int flags); +int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, const void *data, size_t n_bytes, int flags); int gds_fill_inlcpy(gds_peer *peer, gds_op_list_t ¶m, void *ptr, const void *data, size_t n_bytes, int flags); +int gds_fill_poke(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint32_t value, int flags); int gds_fill_poke(gds_peer *peer, gds_op_list_t ¶m, uint32_t *ptr, uint32_t value, int flags); +int gds_fill_poke64(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr addr, uint64_t value, int flags); int gds_fill_poke64(gds_peer *peer, gds_op_list_t ¶m, uint64_t *ptr, uint64_t value, int flags); +int gds_fill_poll(gds_peer *peer, gds_op_list_t &ops, CUdeviceptr ptr, uint32_t magic, int cond_flag, int flags); int gds_fill_poll(gds_peer *peer, gds_op_list_t ¶m, uint32_t *ptr, uint32_t magic, int cond_flag, int flags); int gds_stream_batch_ops(gds_peer *peer, CUstream stream, gds_op_list_t ¶ms, int flags); @@ -216,10 +261,19 @@ enum gds_post_ops_flags { }; struct gds_peer; -int gds_post_ops(gds_peer *peer, size_t n_ops, struct peer_op_wr *op, gds_op_list_t ¶ms, int post_flags = 0); -int gds_post_ops_on_cpu(size_t n_descs, struct peer_op_wr *op, int post_flags = 0); gds_peer *peer_from_stream(CUstream stream); +bool gds_enable_write64(); +bool gds_enable_wait_nor(); +bool gds_enable_remote_flush(); +bool gds_enable_wait_checker(); +bool gds_enable_inlcpy(); +bool gds_simulate_write64(); +bool gds_enable_membar(); +bool gds_enable_weak_consistency(); +bool gds_enable_dump_memops(); +void gds_enable_barrier_for_inlcpy(CUstreamBatchMemOpParams *param); + //----------------------------------------------------------------------------- /* diff --git a/tests/gds_kernel_latency.c b/tests/gds_kernel_latency.c index 63875bf..d98246e 100644 --- a/tests/gds_kernel_latency.c +++ b/tests/gds_kernel_latency.c @@ -67,18 +67,18 @@ #include -#define MPI_CHECK(stmt) \ -do { \ - int result = (stmt); \ - if (MPI_SUCCESS != result) { \ - char string[MPI_MAX_ERROR_STRING]; \ - int resultlen = 0; \ - MPI_Error_string(result, string, &resultlen); \ - fprintf(stderr, " (%s:%d) MPI check failed with %d (%*s)\n", \ - __FILE__, __LINE__, result, resultlen, string); \ - exit(EXIT_FAILURE); \ - } \ -} while(0) +#define MPI_CHECK(stmt) \ + do { \ + int result = (stmt); \ + if (MPI_SUCCESS != result) { \ + char string[MPI_MAX_ERROR_STRING]; \ + int resultlen = 0; \ + MPI_Error_string(result, string, &resultlen); \ + fprintf(stderr, " (%s:%d) MPI check failed with %d (%*s)\n", \ + __FILE__, __LINE__, result, resultlen, string); \ + exit(EXIT_FAILURE); \ + } \ + } while(0) //----------------------------------------------------------------------------- @@ -100,8 +100,8 @@ int prof_idx = 0; #define USE_CUDA_PROFILER 1 enum { - PINGPONG_RECV_WRID = 1, - PINGPONG_SEND_WRID = 2, + PINGPONG_RECV_WRID = 1, + PINGPONG_SEND_WRID = 2, }; static int page_size; @@ -116,28 +116,28 @@ int max_batch_len = 20; int stream_cb_error = 0; struct pingpong_context { - struct ibv_context *context; - struct ibv_comp_channel *channel; - struct ibv_pd *pd; - struct ibv_mr *mr; - struct ibv_cq *tx_cq; - struct ibv_cq *rx_cq; - struct ibv_qp *qp; - struct gds_qp *gds_qp; - struct ibv_ah *ah; - void *buf; - char *txbuf; + struct ibv_context *context; + struct ibv_comp_channel *channel; + struct ibv_pd *pd; + struct ibv_mr *mr; + struct ibv_cq *tx_cq; + struct ibv_cq *rx_cq; + struct ibv_qp *qp; + struct gds_qp *gds_qp; + struct ibv_ah *ah; + void *buf; + char *txbuf; char *rxbuf; char *rx_flag; - int size; + int size; int calc_size; - int rx_depth; - int pending; - struct ibv_port_attr portinfo; - int gpu_id; - int kernel_duration; - int peersync; - int peersync_gpu_cq; + int rx_depth; + int pending; + struct ibv_port_attr portinfo; + int gpu_id; + int kernel_duration; + int peersync; + int peersync_gpu_cq; int consume_rx_cqe; int gpumem; int use_desc_apis; @@ -151,67 +151,67 @@ struct pingpong_context { static int my_rank, comm_size; struct pingpong_dest { - int lid; - int qpn; - int psn; - union ibv_gid gid; + int lid; + int qpn; + int psn; + union ibv_gid gid; }; static inline unsigned long align_to(unsigned long val, unsigned long pow2) { - return (val + pow2 - 1) & ~(pow2 - 1); + return (val + pow2 - 1) & ~(pow2 - 1); } static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, int calc_size, - int rx_depth, int port, - int use_event, - int gpu_id, - int peersync, - int peersync_gpu_cq, - int peersync_gpu_dbrec, - int consume_rx_cqe, - int sched_mode, - int use_gpumem, - int use_desc_apis, - int skip_kernel_launch) + int rx_depth, int port, + int use_event, + int gpu_id, + int peersync, + int peersync_gpu_cq, + int peersync_gpu_dbrec, + int consume_rx_cqe, + int sched_mode, + int use_gpumem, + int use_desc_apis, + int skip_kernel_launch) { - struct pingpong_context *ctx; - - if (gpu_id >=0 && gpu_init(gpu_id, sched_mode)) { - fprintf(stderr, "error in GPU init.\n"); - return NULL; - } - - ctx = malloc(sizeof *ctx); - if (!ctx) - return NULL; - - ctx->size = size; - ctx->calc_size = calc_size; - ctx->rx_depth = rx_depth; - ctx->gpu_id = gpu_id; - ctx->gpumem = use_gpumem; - ctx->use_desc_apis = use_desc_apis; + struct pingpong_context *ctx; + + if (gpu_id >= 0 && gpu_init(gpu_id, sched_mode)) { + fprintf(stderr, "error in GPU init.\n"); + return NULL; + } + + ctx = malloc(sizeof *ctx); + if (!ctx) + return NULL; + + ctx->size = size; + ctx->calc_size = calc_size; + ctx->rx_depth = rx_depth; + ctx->gpu_id = gpu_id; + ctx->gpumem = use_gpumem; + ctx->use_desc_apis = use_desc_apis; ctx->skip_kernel_launch = skip_kernel_launch; size_t alloc_size = 3 * align_to(size + 40, page_size); - if (ctx->gpumem) { - ctx->buf = gpu_malloc(page_size, alloc_size); + + if (ctx->gpumem) { + ctx->buf = gpu_malloc(page_size, alloc_size); printf("allocated GPU memory at %p\n", ctx->buf); - } else { - ctx->buf = memalign(page_size, alloc_size); + } else { + ctx->buf = memalign(page_size, alloc_size); printf("allocated CPU memory at %p\n", ctx->buf); } - if (!ctx->buf) { - fprintf(stderr, "Couldn't allocate work buf.\n"); - goto clean_ctx; - } + if (!ctx->buf) { + fprintf(stderr, "Couldn't allocate work buf.\n"); + goto clean_ctx; + } gpu_info("allocated ctx buffer %p\n", ctx->buf); ctx->rxbuf = (char*)ctx->buf; ctx->txbuf = (char*)ctx->buf + align_to(size + 40, page_size); - //ctx->rx_flag = (char*)ctx->buf + 2 * align_to(size + 40, page_size); ctx->rx_flag = memalign(page_size, alloc_size); if (!ctx->rx_flag) { @@ -219,16 +219,16 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, goto clean_ctx; } - ctx->kernel_duration = 0; - ctx->peersync = peersync; + ctx->kernel_duration = 0; + ctx->peersync = peersync; ctx->peersync_gpu_cq = peersync_gpu_cq; ctx->consume_rx_cqe = consume_rx_cqe; // must be ZERO!!! for rx_flag to work... - if (ctx->gpumem) - gpu_memset(ctx->buf, 0, alloc_size); - else - memset(ctx->buf, 0, alloc_size); + if (ctx->gpumem) + gpu_memset(ctx->buf, 0, alloc_size); + else + memset(ctx->buf, 0, alloc_size); memset(ctx->rx_flag, 0, alloc_size); gpu_register_host_mem(ctx->rx_flag, alloc_size); @@ -239,33 +239,33 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, gpu_launch_kernel(ctx->calc_size, ctx->peersync); CUCHECK(cuCtxSynchronize()); - ctx->context = ibv_open_device(ib_dev); - if (!ctx->context) { - gpu_err("Couldn't get context for %s\n", - ibv_get_device_name(ib_dev)); - goto clean_buffer; - } - - if (use_event) { - ctx->channel = ibv_create_comp_channel(ctx->context); - if (!ctx->channel) { - gpu_err("Couldn't create completion channel\n"); - goto clean_device; - } - } else - ctx->channel = NULL; - - ctx->pd = ibv_alloc_pd(ctx->context); - if (!ctx->pd) { - gpu_err("Couldn't allocate PD\n"); - goto clean_comp_channel; - } - - ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, alloc_size, IBV_ACCESS_LOCAL_WRITE); - if (!ctx->mr) { - gpu_err("Couldn't register MR\n"); - goto clean_pd; - } + ctx->context = ibv_open_device(ib_dev); + if (!ctx->context) { + gpu_err("Couldn't get context for %s\n", + ibv_get_device_name(ib_dev)); + goto clean_buffer; + } + + if (use_event) { + ctx->channel = ibv_create_comp_channel(ctx->context); + if (!ctx->channel) { + gpu_err("Couldn't create completion channel\n"); + goto clean_device; + } + } else + ctx->channel = NULL; + + ctx->pd = ibv_alloc_pd(ctx->context); + if (!ctx->pd) { + gpu_err("Couldn't allocate PD\n"); + goto clean_comp_channel; + } + + ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, alloc_size, IBV_ACCESS_LOCAL_WRITE); + if (!ctx->mr) { + gpu_err("Couldn't register MR\n"); + goto clean_pd; + } int gds_flags = 0; if (peersync_gpu_cq) @@ -290,103 +290,103 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx->gds_qp) { gpu_err("Couldn't create QP\n"); goto clean_mr; - } - ctx->qp = ctx->gds_qp->qp; - ctx->tx_cq = ctx->gds_qp->qp->send_cq; - ctx->rx_cq = ctx->gds_qp->qp->recv_cq; - - { - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_INIT, - .pkey_index = 0, - .port_num = port, - .qkey = 0x11111111, - .qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE - }; - - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_PKEY_INDEX | - IBV_QP_PORT | - ((IBV_QPT_UD == gds_qpt) ? IBV_QP_QKEY : IBV_QP_ACCESS_FLAGS))) { - gpu_err("Failed to modify QP to INIT\n"); - goto clean_qp; - } - } - - return ctx; + } + + ctx->qp = ctx->gds_qp->ibqp; + ctx->tx_cq = ctx->gds_qp->ibqp->send_cq; + ctx->rx_cq = ctx->gds_qp->ibqp->recv_cq; + ctx->pd = ctx->qp->pd; + + { + struct ibv_qp_attr attr = { + .qp_state = IBV_QPS_INIT, + .pkey_index = 0, + .port_num = port, + .qkey = 0x11111111, + .qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE + }; + + if (ibv_modify_qp(ctx->qp, &attr, + IBV_QP_STATE | + IBV_QP_PKEY_INDEX | + IBV_QP_PORT | + ((IBV_QPT_UD == gds_qpt) ? IBV_QP_QKEY : IBV_QP_ACCESS_FLAGS))) { + gpu_err("Failed to modify QP to INIT\n"); + goto clean_qp; + } + } + + return ctx; clean_qp: - gds_destroy_qp(ctx->gds_qp); + gds_destroy_qp(ctx->gds_qp); clean_mr: - ibv_dereg_mr(ctx->mr); + ibv_dereg_mr(ctx->mr); clean_pd: - ibv_dealloc_pd(ctx->pd); + ibv_dealloc_pd(ctx->pd); clean_comp_channel: - if (ctx->channel) - ibv_destroy_comp_channel(ctx->channel); + if (ctx->channel) + ibv_destroy_comp_channel(ctx->channel); clean_device: - ibv_close_device(ctx->context); + ibv_close_device(ctx->context); clean_buffer: - if (ctx->gpumem) - gpu_free(ctx->buf); - else - free(ctx->buf); + if (ctx->gpumem) + gpu_free(ctx->buf); + else + free(ctx->buf); clean_ctx: - if (ctx->gpu_id >= 0) - gpu_finalize(); - free(ctx); + if (ctx->gpu_id >= 0) + gpu_finalize(); + free(ctx); - return NULL; + return NULL; } int pp_close_ctx(struct pingpong_context *ctx) { - if (gds_destroy_qp(ctx->gds_qp)) { - gpu_err("Couldn't destroy QP\n"); - } + gds_destroy_qp(ctx->gds_qp); - if (ibv_dereg_mr(ctx->mr)) { - gpu_err("Couldn't deregister MR\n"); - } + if (ibv_dereg_mr(ctx->mr)) { + gpu_err("Couldn't deregister MR\n"); + } - if (IBV_QPT_UD == gds_qpt) { - if (ibv_destroy_ah(ctx->ah)) { - gpu_err("Couldn't destroy AH\n"); - } - } + if (IBV_QPT_UD == gds_qpt) { + if (ibv_destroy_ah(ctx->ah)) { + gpu_err("Couldn't destroy AH\n"); + } + } - if (ibv_dealloc_pd(ctx->pd)) { - gpu_err("Couldn't deallocate PD\n"); - } + if (ibv_dealloc_pd(ctx->pd)) { + gpu_err("Couldn't deallocate PD\n"); + } - if (ctx->channel) { - if (ibv_destroy_comp_channel(ctx->channel)) { - gpu_err("Couldn't destroy completion channel\n"); - } - } + if (ctx->channel) { + if (ibv_destroy_comp_channel(ctx->channel)) { + gpu_err("Couldn't destroy completion channel\n"); + } + } - if (ibv_close_device(ctx->context)) { - gpu_err("Couldn't release context\n"); - } + if (ibv_close_device(ctx->context)) { + gpu_err("Couldn't release context\n"); + } - if (ctx->gpumem) - gpu_free(ctx->buf); - else - free(ctx->buf); + if (ctx->gpumem) + gpu_free(ctx->buf); + else + free(ctx->buf); - if (ctx->gpu_id >= 0) - gpu_finalize(); + if (ctx->gpu_id >= 0) + gpu_finalize(); - free(ctx); + free(ctx); - return 0; + return 0; } static int poll_send_cq(struct pingpong_context *ctx) @@ -396,7 +396,7 @@ static int poll_send_cq(struct pingpong_context *ctx) struct ibv_wc wc[max_batch_len]; int ne, i; - ne = ibv_poll_cq(ctx->tx_cq, max_batch_len, wc); + ne = gds_poll_cq(ctx->gds_qp->send_cq, max_batch_len, wc); if (ne < 0) { gpu_err("poll TX CQ failed %d\n", ne); return 1; @@ -407,20 +407,21 @@ static int poll_send_cq(struct pingpong_context *ctx) for (i = 0; i < ne; ++i) { if (wc[i].status != IBV_WC_SUCCESS) { gpu_err("Failed status %s (%d) for wr_id %d\n", - ibv_wc_status_str(wc[i].status), - wc[i].status, (int) wc[i].wr_id); + ibv_wc_status_str(wc[i].status), + wc[i].status, (int) wc[i].wr_id); return 1; } switch ((int) wc[i].wr_id) { - case PINGPONG_SEND_WRID: - ++ctx->scnt; - gpu_dbg("got send event scnt=%d\n", ctx->scnt); - break; - default: - gpu_err("Completion for unknown wr_id %d\n", - (int) wc[i].wr_id); - return 1; + case PINGPONG_SEND_WRID: + ++ctx->scnt; + gpu_dbg("got send event scnt=%d\n", ctx->scnt); + break; + default: + gpu_err("Completion for unknown wr_id %d\n", + (int) wc[i].wr_id); + ++ctx->scnt; + return 1; } } @@ -436,7 +437,7 @@ static int poll_recv_cq(struct pingpong_context *ctx) int ne = 0; int i; - ne = ibv_poll_cq(ctx->rx_cq, max_batch_len, wc); + ne = gds_poll_cq(ctx->gds_qp->recv_cq, max_batch_len, wc); if (ne < 0) { gpu_err("poll RX CQ failed %d\n", ne); return 1; @@ -447,55 +448,56 @@ static int poll_recv_cq(struct pingpong_context *ctx) for (i = 0; i < ne; ++i) { if (wc[i].status != IBV_WC_SUCCESS) { gpu_err("[%d] Failed status %s (%d) for wr_id %d\n", - my_rank, ibv_wc_status_str(wc[i].status), - wc[i].status, (int) wc[i].wr_id); + my_rank, ibv_wc_status_str(wc[i].status), + wc[i].status, (int) wc[i].wr_id); return 1; } switch ((int) wc[i].wr_id) { - case PINGPONG_RECV_WRID: - ++ctx->rcnt; - gpu_dbg("[%d] got recv event rcnt=%d\n", my_rank, ctx->rcnt); - break; - default: - gpu_err("[%d] Completion for unknown wr_id %d\n", - my_rank, (int) wc[i].wr_id); - return 1; + case PINGPONG_RECV_WRID: + ++ctx->rcnt; + gpu_dbg("[%d] got recv event rcnt=%d\n", my_rank, ctx->rcnt); + break; + default: + gpu_err("[%d] Completion for unknown wr_id %d\n", + my_rank, (int) wc[i].wr_id); + return 1; } } + return 0; } static int pp_post_recv(struct pingpong_context *ctx, int n) { - struct ibv_sge list = { - .addr = (uintptr_t) ctx->rxbuf, - .length = ctx->size + 40, // good for IBV_QPT_UD - .lkey = ctx->mr->lkey - }; - - if (IBV_QPT_UD != gds_qpt) list.length = ctx->size; - - struct ibv_recv_wr wr = { - .wr_id = PINGPONG_RECV_WRID, - .sg_list = &list, - .num_sge = 1, - }; - struct ibv_recv_wr *bad_wr; - int i; - - for (i = 0; i < n; ++i) - if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) - break; - - return i; + struct ibv_sge list = { + .addr = (uintptr_t) ctx->rxbuf, + .length = ctx->size + 40, // good for IBV_QPT_UD + .lkey = ctx->mr->lkey + }; + + if (IBV_QPT_UD != gds_qpt) list.length = ctx->size; + + struct ibv_recv_wr wr = { + .wr_id = PINGPONG_RECV_WRID, + .sg_list = &list, + .num_sge = 1, + }; + struct ibv_recv_wr *bad_wr; + int i; + + for (i = 0; i < n; ++i) + if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) + break; + + return i; } static int pp_wait_cq(struct pingpong_context *ctx, int is_client) { int ret; if (ctx->peersync) { - ret = gds_stream_wait_cq(gpu_stream, &ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); + ret = gds_stream_wait_cq(gpu_stream, ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); } else { if (is_client) { do { @@ -504,7 +506,7 @@ static int pp_wait_cq(struct pingpong_context *ctx, int is_client) return ret; } } while(ctx->n_tx_ev <= 0); - + do { ret = poll_recv_cq(ctx); if (ret) { @@ -532,76 +534,73 @@ static int pp_wait_cq(struct pingpong_context *ctx, int is_client) static int pp_post_gpu_send(struct pingpong_context *ctx, uint32_t qpn, CUstream *p_gpu_stream) { - int ret = 0; - struct ibv_sge list = { - .addr = (uintptr_t) ctx->txbuf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - gds_send_wr ewr = { - .wr_id = PINGPONG_SEND_WRID, - .sg_list = &list, - .num_sge = 1, - .exp_opcode = IBV_EXP_WR_SEND, - .exp_send_flags = IBV_EXP_SEND_SIGNALED, - .wr = { - .ud = { - .ah = ctx->ah, - .remote_qpn = qpn, - .remote_qkey = 0x11111111 - } - }, - .comp_mask = 0 - }; -#if 0 - if (IBV_QPT_UD != gds_qpt) { - memset(&ewr, 0, sizeof(ewr)); - ewr.num_sge = 1; - ewr.exp_send_flags = IBV_EXP_SEND_SIGNALED; - ewr.exp_opcode = IBV_EXP_WR_SEND; - ewr.wr_id = PINGPONG_SEND_WRID; - ewr.sg_list = &list; - ewr.next = NULL; - } -#endif - gds_send_wr *bad_ewr; + struct ibv_sge list = { + .addr = (uintptr_t) ctx->txbuf, + .length = ctx->size, + .lkey = ctx->mr->lkey + }; + gds_send_wr ewr = { + .wr_id = PINGPONG_SEND_WRID, + .sg_list = &list, + .num_sge = 1, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, + .wr = { + .ud = { + .ah = ctx->ah, + .remote_qpn = qpn, + .remote_qkey = 0x11111111 + } + } + }; + + if (IBV_QPT_UD != gds_qpt) { + memset(&ewr, 0, sizeof(ewr)); + ewr.num_sge = 1; + ewr.opcode = IBV_WR_SEND; + ewr.send_flags = IBV_SEND_SIGNALED, + ewr.wr_id = PINGPONG_SEND_WRID; + ewr.sg_list = &list; + ewr.next = NULL; + } + + gds_send_wr *bad_ewr; return gds_stream_queue_send(*p_gpu_stream, ctx->gds_qp, &ewr, &bad_ewr); } static int pp_prepare_gpu_send(struct pingpong_context *ctx, uint32_t qpn, gds_send_request_t *req) { - int ret = 0; - struct ibv_sge list = { - .addr = (uintptr_t) ctx->txbuf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - gds_send_wr ewr = { - .wr_id = PINGPONG_SEND_WRID, - .sg_list = &list, - .num_sge = 1, - .exp_opcode = IBV_EXP_WR_SEND, - .exp_send_flags = IBV_EXP_SEND_SIGNALED, - .wr = { - .ud = { - .ah = ctx->ah, - .remote_qpn = qpn, - .remote_qkey = 0x11111111 - } - }, - .comp_mask = 0 - }; - - if (IBV_QPT_UD != gds_qpt) { - memset(&ewr, 0, sizeof(ewr)); - ewr.num_sge = 1; - ewr.exp_send_flags = IBV_EXP_SEND_SIGNALED; - ewr.exp_opcode = IBV_EXP_WR_SEND; - ewr.wr_id = PINGPONG_SEND_WRID; - ewr.sg_list = &list; - ewr.next = NULL; - } - gds_send_wr *bad_ewr; + //int ret = 0; + struct ibv_sge list = { + .addr = (uintptr_t) ctx->txbuf, + .length = ctx->size, + .lkey = ctx->mr->lkey + }; + gds_send_wr ewr = { + .wr_id = PINGPONG_SEND_WRID, + .sg_list = &list, + .num_sge = 1, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, + .wr = { + .ud = { + .ah = ctx->ah, + .remote_qpn = qpn, + .remote_qkey = 0x11111111 + } + } + }; + + if (IBV_QPT_UD != gds_qpt) { + memset(&ewr, 0, sizeof(ewr)); + ewr.num_sge = 1; + ewr.opcode = IBV_WR_SEND; + ewr.send_flags = IBV_SEND_SIGNALED, + ewr.wr_id = PINGPONG_SEND_WRID; + ewr.sg_list = &list; + ewr.next = NULL; + } + gds_send_wr *bad_ewr; return gds_prepare_send(ctx->gds_qp, &ewr, &bad_ewr, req); } @@ -639,18 +638,16 @@ static void post_work_cb(CUstream hStream, CUresult status, void *userData)\ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uint32_t qpn, int is_client) { int retcode = 0; - int i, ret = 0; + int i, ret = 0; int posted_recv = 0; - //printf("post_work posting %d\n", n_posts); - if (n_posts <= 0) return 0; posted_recv = pp_post_recv(ctx, n_posts); if (posted_recv < 0) { gpu_err("can't post recv (%d) n_posts=%d is_client=%d\n", - posted_recv, n_posts, is_client); + posted_recv, n_posts, is_client); exit(EXIT_FAILURE); return 0; } else if (posted_recv != n_posts) { @@ -659,33 +656,38 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin return 0; } PROF(&prof, prof_idx++); - for (i = 0; i < posted_recv; ++i) { + + for (i = 0; i < posted_recv; ++i) { if (is_client) { - if (gds_enable_event_prof && (event_idx < MAX_EVENTS)) { - cudaEventRecord(start_time[event_idx], gpu_stream); - } + if (gds_enable_event_prof && (event_idx < MAX_EVENTS)) { + cudaEventRecord(start_time[event_idx], gpu_stream); + } if (ctx->use_desc_apis) { work_desc_t *wdesc = calloc(1, sizeof(*wdesc)); int k = 0; + ret = pp_prepare_gpu_send(ctx, qpn, &wdesc->send_rq); if (ret) { retcode = -ret; break; } + assert(k < N_WORK_DESCS); wdesc->descs[k].tag = GDS_TAG_SEND; wdesc->descs[k].send = &wdesc->send_rq; ++k; - ret = gds_prepare_wait_cq(&ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); + + ret = gds_prepare_wait_cq(ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); if (ret) { retcode = -ret; break; } + assert(k < N_WORK_DESCS); wdesc->descs[k].tag = GDS_TAG_WAIT; wdesc->descs[k].wait = &wdesc->wait_tx_rq; ++k; - ret = gds_prepare_wait_cq(&ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); if (ret) { retcode = -ret; break; @@ -711,22 +713,21 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin ret = pp_post_gpu_send(ctx, qpn, &gpu_stream); if (ret) { gpu_err("error %d in pp_post_gpu_send, posted_recv=%d posted_so_far=%d is_client=%d \n", - ret, posted_recv, i, is_client); + ret, posted_recv, i, is_client); retcode = -ret; break; } - ret = gds_stream_wait_cq(gpu_stream, &ctx->gds_qp->send_cq, 0); + ret = gds_stream_wait_cq(gpu_stream, ctx->gds_qp->send_cq, 0); if (ret) { // TODO: rollback gpu send gpu_err("error %d in gds_stream_wait_cq\n", ret); retcode = -ret; break; } - ret = gds_stream_wait_cq(gpu_stream, &ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); + ret = gds_stream_wait_cq(gpu_stream, ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); if (ret) { // TODO: rollback gpu send and wait send_cq gpu_err("[%d] error %d in gds_stream_wait_cq\n", my_rank, ret); - //exit(EXIT_FAILURE); retcode = -ret; break; } @@ -736,10 +737,10 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin break; } - if (gds_enable_event_prof && (event_idx < MAX_EVENTS)) { - cudaEventRecord(stop_time[event_idx], gpu_stream); - event_idx++; - } + if (gds_enable_event_prof && (event_idx < MAX_EVENTS)) { + cudaEventRecord(stop_time[event_idx], gpu_stream); + event_idx++; + } if (ctx->skip_kernel_launch) { gpu_warn_once("[%d] NOT LAUNCHING ANY KERNEL AT ALL\n", my_rank); } else { @@ -751,7 +752,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin if (ctx->use_desc_apis) { work_desc_t *wdesc = calloc(1, sizeof(*wdesc)); int k = 0; - ret = gds_prepare_wait_cq(&ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); if (ret) { retcode = -ret; break; @@ -773,11 +774,10 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin CUCHECK(cuStreamAddCallback(gpu_stream, post_work_cb, wdesc, 0)); } } else if (ctx->peersync) { - ret = gds_stream_wait_cq(gpu_stream, &ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); + ret = gds_stream_wait_cq(gpu_stream, ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); if (ret) { // TODO: rollback gpu send and wait send_cq gpu_err("error %d in gds_stream_wait_cq\n", ret); - //exit(EXIT_FAILURE); retcode = -ret; break; } @@ -791,9 +791,9 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin } else { gpu_launch_kernel(ctx->calc_size, ctx->peersync); } - if (gds_enable_event_prof && (event_idx < MAX_EVENTS)) { - cudaEventRecord(start_time[event_idx], gpu_stream); - } + if (gds_enable_event_prof && (event_idx < MAX_EVENTS)) { + cudaEventRecord(start_time[event_idx], gpu_stream); + } if (ctx->use_desc_apis) { work_desc_t *wdesc = calloc(1, sizeof(*wdesc)); int k = 0; @@ -806,7 +806,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin wdesc->descs[k].tag = GDS_TAG_SEND; wdesc->descs[k].send = &wdesc->send_rq; ++k; - ret = gds_prepare_wait_cq(&ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); if (ret) { retcode = -ret; break; @@ -831,11 +831,11 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin ret = pp_post_gpu_send(ctx, qpn, &gpu_stream); if (ret) { gpu_err("error %d in pp_post_gpu_send, posted_recv=%d posted_so_far=%d is_client=%d \n", - ret, posted_recv, i, is_client); + ret, posted_recv, i, is_client); retcode = -ret; break; } - ret = gds_stream_wait_cq(gpu_stream, &ctx->gds_qp->send_cq, 0); + ret = gds_stream_wait_cq(gpu_stream, ctx->gds_qp->send_cq, 0); if (ret) { // TODO: rollback gpu send gpu_err("error %d in gds_stream_wait_cq\n", ret); @@ -848,82 +848,79 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin break; } - if (gds_enable_event_prof && (event_idx < MAX_EVENTS)) { - cudaEventRecord(stop_time[event_idx], gpu_stream); - event_idx++; - } + if (gds_enable_event_prof && (event_idx < MAX_EVENTS)) { + cudaEventRecord(stop_time[event_idx], gpu_stream); + event_idx++; + } } } PROF(&prof, prof_idx++); if (!retcode) { retcode = i; gpu_post_release_tracking_event(&gpu_stream_server); - //sleep(1); } - return retcode; + return retcode; } static void usage(const char *argv0) { - printf("Usage:\n"); - printf(" %s start a server and wait for connection\n", argv0); - printf(" %s connect to server at \n", argv0); - printf("\n"); - printf("Options:\n"); - printf(" -p, --port= listen on/connect to port (default 18515)\n"); - printf(" -d, --ib-dev= use IB device (default first device found)\n"); - printf(" -i, --ib-port= use port of IB device (default 1)\n"); - printf(" -s, --size= size of message to exchange (default 1024)\n"); - printf(" -r, --rx-depth= number of receives to post at a time (default 500)\n"); - printf(" -n, --iters= number of exchanges (default 1000)\n"); - printf(" -e, --events sleep on CQ events (default poll)\n"); - printf(" -g, --gid-idx= local port gid index\n"); - printf(" -S, --gpu-calc-size= size of GPU compute buffer (default 128KB)\n"); - printf(" -G, --gpu-id use specified GPU (default 0)\n"); - printf(" -B, --batch-length= max batch length (default 20)\n"); - printf(" -P, --peersync enable GPUDirect PeerSync support (default enabled)\n"); - printf(" -C, --peersync-gpu-cq enable GPUDirect PeerSync GPU CQ support (default disabled)\n"); - printf(" -D, --peersync-gpu-dbrec enable QP DBREC on GPU memory (default disabled)\n"); - printf(" -U, --peersync-desc-apis use batched descriptor APIs (default disabled)\n"); - printf(" -Q, --consume-rx-cqe enable GPU consumes RX CQE support (default disabled)\n"); - printf(" -T, --time-gds-ops evaluate time needed to execute gds operations using cuda events\n"); - printf(" -k, --qp-kind select IB transport kind used by GDS QPs. (-K 1) for UD, (-K 2) for RC\n"); - printf(" -M, --gpu-sched-mode set CUDA context sched mode, default (A)UTO, (S)PIN, (Y)IELD, (B)LOCKING\n"); - printf(" -E, --gpu-mem allocate GPU intead of CPU memory buffers\n"); - printf(" -K, --skip-kernel-launch no GPU kernel computations, only communications\n"); + printf("Usage:\n"); + printf(" %s start a server and wait for connection\n", argv0); + printf(" %s connect to server at \n", argv0); + printf("\n"); + printf("Options:\n"); + printf(" -p, --port= listen on/connect to port (default 18515)\n"); + printf(" -d, --ib-dev= use IB device (default first device found)\n"); + printf(" -i, --ib-port= use port of IB device (default 1)\n"); + printf(" -s, --size= size of message to exchange (default 1024)\n"); + printf(" -r, --rx-depth= number of receives to post at a time (default 500)\n"); + printf(" -n, --iters= number of exchanges (default 1000)\n"); + printf(" -e, --events sleep on CQ events (default poll)\n"); + printf(" -g, --gid-idx= local port gid index\n"); + printf(" -S, --gpu-calc-size= size of GPU compute buffer (default 128KB)\n"); + printf(" -G, --gpu-id use specified GPU (default 0)\n"); + printf(" -B, --batch-length= max batch length (default 20)\n"); + printf(" -P, --peersync enable GPUDirect PeerSync support (default enabled)\n"); + printf(" -C, --peersync-gpu-cq enable GPUDirect PeerSync GPU CQ support (default disabled)\n"); + printf(" -D, --peersync-gpu-dbrec enable QP DBREC on GPU memory (default disabled)\n"); + printf(" -U, --peersync-desc-apis use batched descriptor APIs (default disabled)\n"); + printf(" -Q, --consume-rx-cqe enable GPU consumes RX CQE support (default disabled)\n"); + printf(" -T, --time-gds-ops evaluate time needed to execute gds operations using cuda events\n"); + printf(" -k, --qp-kind select IB transport kind used by GDS QPs. (-K 1) for UD, (-K 2) for RC\n"); + printf(" -M, --gpu-sched-mode set CUDA context sched mode, default (A)UTO, (S)PIN, (Y)IELD, (B)LOCKING\n"); + printf(" -E, --gpu-mem allocate GPU intead of CPU memory buffers\n"); + printf(" -K, --skip-kernel-launch no GPU kernel computations, only communications\n"); } int main(int argc, char *argv[]) { - struct ibv_device **dev_list; - struct ibv_device *ib_dev; - struct pingpong_context *ctx; - struct pingpong_dest my_dest; - struct pingpong_dest *rem_dest = NULL; - struct timeval rstart, start, end; - const char *ib_devname = NULL; - char *servername = NULL; - int port = 18515; - int ib_port = 1; - int size = 1024; - int calc_size = 128*1024; - int rx_depth = 2*512; - int iters = 1000; - int use_event = 0; - int routs; + struct ibv_device **dev_list; + struct ibv_device *ib_dev; + struct pingpong_context *ctx; + struct pingpong_dest my_dest; + struct pingpong_dest *rem_dest = NULL; + struct timeval start, end; + const char *ib_devname = NULL; + char *servername = NULL; + int port = 18515; + int ib_port = 1; + int size = 1024; + int calc_size = 128*1024; + int rx_depth = 2*512; + int iters = 1000; + int use_event = 0; + int routs; int nposted; - int num_cq_events = 0; - int sl = 0; - int gidx = -1; - char gid[INET6_ADDRSTRLEN]; - int gpu_id = 0; + int sl = 0; + int gidx = -1; + char gid[INET6_ADDRSTRLEN]; + int gpu_id = 0; int peersync = 1; int peersync_gpu_cq = 0; int peersync_gpu_dbrec = 0; - int warmup = 10; int consume_rx_cqe = 0; - int gds_qp_type = 1; + int gds_qp_type = 1; int sched_mode = CU_CTX_SCHED_AUTO; int ret = 0; int use_gpumem = 0; @@ -953,169 +950,169 @@ int main(int argc, char *argv[]) MPI_Abort(MPI_COMM_WORLD, -1); } - srand48(getpid() * time(NULL)); - - while (1) { - int c; - - static struct option long_options[] = { - { .name = "port", .has_arg = 1, .val = 'p' }, - { .name = "ib-dev", .has_arg = 1, .val = 'd' }, - { .name = "ib-port", .has_arg = 1, .val = 'i' }, - { .name = "size", .has_arg = 1, .val = 's' }, - { .name = "rx-depth", .has_arg = 1, .val = 'r' }, - { .name = "iters", .has_arg = 1, .val = 'n' }, - { .name = "sl", .has_arg = 1, .val = 'l' }, - { .name = "events", .has_arg = 0, .val = 'e' }, - { .name = "gid-idx", .has_arg = 1, .val = 'g' }, - { .name = "gpu-id", .has_arg = 1, .val = 'G' }, - { .name = "peersync", .has_arg = 0, .val = 'P' }, - { .name = "peersync-gpu-cq", .has_arg = 0, .val = 'C' }, - { .name = "peersync-gpu-dbrec", .has_arg = 1, .val = 'D' }, + srand48(getpid() * time(NULL)); + + while (1) { + int c; + + static struct option long_options[] = { + { .name = "port", .has_arg = 1, .val = 'p' }, + { .name = "ib-dev", .has_arg = 1, .val = 'd' }, + { .name = "ib-port", .has_arg = 1, .val = 'i' }, + { .name = "size", .has_arg = 1, .val = 's' }, + { .name = "rx-depth", .has_arg = 1, .val = 'r' }, + { .name = "iters", .has_arg = 1, .val = 'n' }, + { .name = "sl", .has_arg = 1, .val = 'l' }, + { .name = "events", .has_arg = 0, .val = 'e' }, + { .name = "gid-idx", .has_arg = 1, .val = 'g' }, + { .name = "gpu-id", .has_arg = 1, .val = 'G' }, + { .name = "peersync", .has_arg = 0, .val = 'P' }, + { .name = "peersync-gpu-cq", .has_arg = 0, .val = 'C' }, + { .name = "peersync-gpu-dbrec", .has_arg = 1, .val = 'D' }, { .name = "peersync-desc-apis", .has_arg = 0, .val = 'U' }, - { .name = "gpu-calc-size", .has_arg = 1, .val = 'S' }, - { .name = "batch-length", .has_arg = 1, .val = 'B' }, - { .name = "consume-rx-cqe", .has_arg = 0, .val = 'Q' }, - { .name = "time-gds-ops", .has_arg = 0, .val = 'T' }, - { .name = "qp-kind", .has_arg = 1, .val = 'k' }, - { .name = "gpu-sched-mode", .has_arg = 1, .val = 'M' }, - { .name = "gpu-mem", .has_arg = 0, .val = 'E' }, - { .name = "skip-kernel-launch", .has_arg = 0, .val = 'K' }, - { 0 } - }; - - c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:G:k:S:B:PCDQTM:EUK", long_options, NULL); - if (c == -1) - break; - - switch (c) { - case 'p': - port = strtol(optarg, NULL, 0); - if (port < 0 || port > 65535) { - usage(argv[0]); - ret = 1; - exit(EXIT_FAILURE); - } - break; - - case 'd': - ib_devname = strdupa(optarg); - break; - - case 'i': - ib_port = strtol(optarg, NULL, 0); - if (ib_port < 0) { - usage(argv[0]); - ret = 1; - exit(EXIT_FAILURE); - } - break; - - case 's': - size = strtol(optarg, NULL, 0); - break; - - case 'S': - calc_size = strtol(optarg, NULL, 0); - break; - - case 'r': - rx_depth = strtol(optarg, NULL, 0); - break; - - case 'n': - iters = strtol(optarg, NULL, 0); - break; - - case 'l': - sl = strtol(optarg, NULL, 0); - break; - - case 'e': - ++use_event; - break; - - case 'g': - gidx = strtol(optarg, NULL, 0); - break; - - case 'G': - gpu_id = strtol(optarg, NULL, 0); - printf("INFO: gpu id=%d\n", gpu_id); - break; - - case 'B': - max_batch_len = strtol(optarg, NULL, 0); - printf("INFO: max_batch_len=%d\n", max_batch_len); - break; - - case 'P': - peersync = !peersync; - printf("INFO: switching PeerSync %s\n", peersync?"ON":"OFF"); - break; - - case 'k': - gds_qp_type = (int) strtol(optarg, NULL, 0); - switch (gds_qp_type) { - case 1: printf("INFO: GDS_QPT %s\n","UD"); gds_qpt = IBV_QPT_UD; break; - case 2: printf("INFO: GDS_QPT %s\n","RC"); gds_qpt = IBV_QPT_RC; break; - default: printf("ERROR: unexpected value 1 for UD or 2 for RC \n"); exit(EXIT_FAILURE); break; - } - break; - case 'Q': - consume_rx_cqe = !consume_rx_cqe; - printf("INFO: switching consume_rx_cqe %s\n", consume_rx_cqe?"ON":"OFF"); - break; - - case 'T': - gds_enable_event_prof = !gds_enable_event_prof; - printf("INFO: gds_enable_event_prof %s\n", gds_enable_event_prof?"ON":"OFF"); - break; - - case 'C': - peersync_gpu_cq = !peersync_gpu_cq; - printf("INFO: switching %s PeerSync GPU CQ\n", peersync_gpu_cq?"ON":"OFF"); - break; - - case 'D': - peersync_gpu_dbrec= !peersync_gpu_dbrec; - printf("INFO: switching %s PeerSync GPU QP DBREC\n", peersync_gpu_dbrec?"ON":"OFF"); - break; - - case 'M': - { - char m = *optarg; - printf("INFO: sched mode '%c'\n", m); - switch (m) { - case 'S': sched_mode = CU_CTX_SCHED_SPIN; break; - case 'Y': sched_mode = CU_CTX_SCHED_YIELD; break; - case 'B': sched_mode = CU_CTX_SCHED_BLOCKING_SYNC; break; - case 'A': sched_mode = CU_CTX_SCHED_AUTO; break; - default: printf("ERROR: unexpected value %c\n", m); exit(EXIT_FAILURE); break; - } - } - break; + { .name = "gpu-calc-size", .has_arg = 1, .val = 'S' }, + { .name = "batch-length", .has_arg = 1, .val = 'B' }, + { .name = "consume-rx-cqe", .has_arg = 0, .val = 'Q' }, + { .name = "time-gds-ops", .has_arg = 0, .val = 'T' }, + { .name = "qp-kind", .has_arg = 1, .val = 'k' }, + { .name = "gpu-sched-mode", .has_arg = 1, .val = 'M' }, + { .name = "gpu-mem", .has_arg = 0, .val = 'E' }, + { .name = "skip-kernel-launch", .has_arg = 0, .val = 'K' }, + { 0 } + }; - case 'E': - use_gpumem = !use_gpumem; - printf("INFO: use_gpumem=%d\n", use_gpumem); + c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:G:k:S:B:PCDQTM:EUK", long_options, NULL); + if (c == -1) break; - case 'U': - use_desc_apis = 1; - printf("INFO: use_desc_apis=%d\n", use_desc_apis); - break; - - case 'K': - skip_kernel_launch = 1; - printf("INFO: skip_kernel_launch=%d\n", skip_kernel_launch); - break; + switch (c) { + case 'p': + port = strtol(optarg, NULL, 0); + if (port < 0 || port > 65535) { + usage(argv[0]); + ret = 1; + exit(EXIT_FAILURE); + } + break; + + case 'd': + ib_devname = strdup(optarg); + break; + + case 'i': + ib_port = strtol(optarg, NULL, 0); + if (ib_port < 0) { + usage(argv[0]); + ret = 1; + exit(EXIT_FAILURE); + } + break; + + case 's': + size = strtol(optarg, NULL, 0); + break; - default: - usage(argv[0]); - return 1; - } - } + case 'S': + calc_size = strtol(optarg, NULL, 0); + break; + + case 'r': + rx_depth = strtol(optarg, NULL, 0); + break; + + case 'n': + iters = strtol(optarg, NULL, 0); + break; + + case 'l': + sl = strtol(optarg, NULL, 0); + break; + + case 'e': + ++use_event; + break; + + case 'g': + gidx = strtol(optarg, NULL, 0); + break; + + case 'G': + gpu_id = strtol(optarg, NULL, 0); + printf("INFO: gpu id=%d\n", gpu_id); + break; + + case 'B': + max_batch_len = strtol(optarg, NULL, 0); + printf("INFO: max_batch_len=%d\n", max_batch_len); + break; + + case 'P': + peersync = !peersync; + printf("INFO: switching PeerSync %s\n", peersync?"ON":"OFF"); + break; + + case 'k': + gds_qp_type = (int) strtol(optarg, NULL, 0); + switch (gds_qp_type) { + case 1: printf("INFO: GDS_QPT %s\n","UD"); gds_qpt = IBV_QPT_UD; break; + case 2: printf("INFO: GDS_QPT %s\n","RC"); gds_qpt = IBV_QPT_RC; break; + default: printf("ERROR: unexpected value 1 for UD or 2 for RC \n"); exit(EXIT_FAILURE); break; + } + break; + case 'Q': + consume_rx_cqe = !consume_rx_cqe; + printf("INFO: switching consume_rx_cqe %s\n", consume_rx_cqe?"ON":"OFF"); + break; + + case 'T': + gds_enable_event_prof = !gds_enable_event_prof; + printf("INFO: gds_enable_event_prof %s\n", gds_enable_event_prof?"ON":"OFF"); + break; + + case 'C': + peersync_gpu_cq = !peersync_gpu_cq; + printf("INFO: switching %s PeerSync GPU CQ\n", peersync_gpu_cq?"ON":"OFF"); + break; + + case 'D': + peersync_gpu_dbrec= !peersync_gpu_dbrec; + printf("INFO: switching %s PeerSync GPU QP DBREC\n", peersync_gpu_dbrec?"ON":"OFF"); + break; + + case 'M': + { + char m = *optarg; + printf("INFO: sched mode '%c'\n", m); + switch (m) { + case 'S': sched_mode = CU_CTX_SCHED_SPIN; break; + case 'Y': sched_mode = CU_CTX_SCHED_YIELD; break; + case 'B': sched_mode = CU_CTX_SCHED_BLOCKING_SYNC; break; + case 'A': sched_mode = CU_CTX_SCHED_AUTO; break; + default: printf("ERROR: unexpected value %c\n", m); exit(EXIT_FAILURE); break; + } + } + break; + + case 'E': + use_gpumem = !use_gpumem; + printf("INFO: use_gpumem=%d\n", use_gpumem); + break; + + case 'U': + use_desc_apis = 1; + printf("INFO: use_desc_apis=%d\n", use_desc_apis); + break; + + case 'K': + skip_kernel_launch = 1; + printf("INFO: skip_kernel_launch=%d\n", skip_kernel_launch); + break; + + default: + usage(argv[0]); + return 1; + } + } if (!peersync && !use_desc_apis) { gpu_err("!peersync case only supported when using descriptor APIs, enabling them\n"); @@ -1133,12 +1130,13 @@ int main(int argc, char *argv[]) hostnames, MPI_MAX_PROCESSOR_NAME, MPI_CHAR, MPI_COMM_WORLD)); if (my_rank == 1) { - servername = hostnames[0]; + servername = hostnames[0]; printf("[%d] pid=%d server:%s\n", my_rank, getpid(), servername); } else { printf("[%d] pid=%d client:%s\n", my_rank, getpid(), hostnames[1]); } + const char *tags = NULL; if (peersync) { tags = "wait trk|pollrxcq|polltxcq|postrecv|postwork| poketrk"; @@ -1146,16 +1144,15 @@ int main(int argc, char *argv[]) tags = "krn laun|krn sync|postsend|<------>|<------>| sent ev"; } prof_init(&prof, 10000, 10000, "10us", 60, 2, tags); - //prof_init(&prof, 100, 100, "100ns", 25*4, 2, tags); prof_disable(&prof); - page_size = sysconf(_SC_PAGESIZE); + page_size = sysconf(_SC_PAGESIZE); - dev_list = ibv_get_device_list(NULL); - if (!dev_list) { - perror("Failed to get IB devices list"); - return 1; - } + dev_list = ibv_get_device_list(NULL); + if (!dev_list) { + perror("Failed to get IB devices list"); + return 1; + } if (!ib_devname) { const char *value = getenv("USE_HCA"); @@ -1167,31 +1164,31 @@ int main(int argc, char *argv[]) printf("[%d] requested IB device: <%s>\n", my_rank, ib_devname); } - { - const char *value = getenv("GDS_ENABLE_EVENT_PROF"); - if (value != NULL) { - gds_enable_event_prof = atoi(value); - } - } + { + const char *value = getenv("GDS_ENABLE_EVENT_PROF"); + if (value != NULL) { + gds_enable_event_prof = atoi(value); + } + } - if (!ib_devname) { + if (!ib_devname) { printf("[%d] picking 1st available device\n", my_rank); - ib_dev = *dev_list; - if (!ib_dev) { - gpu_err("[%d] No IB devices found\n", my_rank); - return 1; - } - } else { - int i; - for (i = 0; dev_list[i]; ++i) - if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) - break; - ib_dev = dev_list[i]; - if (!ib_dev) { - gpu_err("IB device %s not found\n", ib_devname); - return 1; - } - } + ib_dev = *dev_list; + if (!ib_dev) { + gpu_err("[%d] No IB devices found\n", my_rank); + return 1; + } + } else { + int i; + for (i = 0; dev_list[i]; ++i) + if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) + break; + ib_dev = dev_list[i]; + if (!ib_dev) { + gpu_err("IB device %s not found\n", ib_devname); + return 1; + } + } { const char *env = getenv("USE_GPU"); @@ -1201,46 +1198,47 @@ int main(int argc, char *argv[]) } } printf("[%d] use gpumem: %d\n", my_rank, use_gpumem); - ctx = pp_init_ctx(ib_dev, size, calc_size, rx_depth, ib_port, 0, gpu_id, peersync, peersync_gpu_cq, peersync_gpu_dbrec, consume_rx_cqe, sched_mode, use_gpumem, use_desc_apis, skip_kernel_launch); - if (!ctx) - return 1; - - int nrecv = pp_post_recv(ctx, max_batch_len); - if (nrecv < max_batch_len) { - gpu_warn("[%d] Could not post all receive, requested %d, actually posted %d\n", my_rank, max_batch_len, nrecv); - return 1; - } - - if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { - gpu_err("[%d] Couldn't get port info\n", my_rank); - return 1; - } - my_dest.lid = ctx->portinfo.lid; - my_dest.qpn = ctx->qp->qp_num; - my_dest.psn = (IBV_QPT_UD == gds_qpt) ? (lrand48() & 0xffffff) : 0; - - if (gidx >= 0) { - if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { - gpu_err("Could not get local gid for gid index " - "%d\n", gidx); - return 1; - } - } else - memset(&my_dest.gid, 0, sizeof my_dest.gid); - - printf("[%d] local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", - my_rank, my_dest.lid, my_dest.qpn, my_dest.psn, gid); - inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); - - struct pingpong_dest all_dest[4] = {{0,}}; + + ctx = pp_init_ctx(ib_dev, size, calc_size, rx_depth, ib_port, 0, gpu_id, peersync, peersync_gpu_cq, peersync_gpu_dbrec, consume_rx_cqe, sched_mode, use_gpumem, use_desc_apis, skip_kernel_launch); + if (!ctx) + return 1; + + int nrecv = pp_post_recv(ctx, max_batch_len); + if (nrecv < max_batch_len) { + gpu_warn("[%d] Could not post all receive, requested %d, actually posted %d\n", my_rank, max_batch_len, nrecv); + return 1; + } + + if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { + gpu_err("[%d] Couldn't get port info\n", my_rank); + return 1; + } + my_dest.lid = ctx->portinfo.lid; + my_dest.qpn = ctx->qp->qp_num; + my_dest.psn = (IBV_QPT_UD == gds_qpt) ? (lrand48() & 0xffffff) : 0; + + if (gidx >= 0) { + if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { + gpu_err("Could not get local gid for gid index " + "%d\n", gidx); + return 1; + } + } else + memset(&my_dest.gid, 0, sizeof my_dest.gid); + + printf("[%d] local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", + my_rank, my_dest.lid, my_dest.qpn, my_dest.psn, gid); + inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); + + struct pingpong_dest all_dest[4] = {{0,}}; all_dest[my_rank] = my_dest; MPI_CHECK(MPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, all_dest, sizeof(all_dest[0]), MPI_CHAR, MPI_COMM_WORLD)); rem_dest = &all_dest[my_rank?0:1]; - inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); + inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); - printf("[%d] remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", - my_rank, rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); + printf("[%d] remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", + my_rank, rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); if (IBV_QPT_UD == gds_qpt) { struct ibv_qp_attr attr = { @@ -1258,8 +1256,8 @@ int main(int argc, char *argv[]) attr.sq_psn = my_dest.psn; if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_SQ_PSN)) { + IBV_QP_STATE | + IBV_QP_SQ_PSN)) { gpu_err("Failed to modify QP to RTS\n"); return 1; } @@ -1287,67 +1285,66 @@ int main(int argc, char *argv[]) } } - else { + else { struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_RTR, - .path_mtu = ctx->portinfo.active_mtu, - .dest_qp_num = rem_dest->qpn, - .rq_psn = rem_dest->psn, - .ah_attr.dlid = rem_dest->lid, - .max_dest_rd_atomic = 1, - .min_rnr_timer = 12, - .ah_attr.is_global = 0, - .ah_attr.sl = 0, - .ah_attr.src_path_bits = 0, - .ah_attr.port_num = ib_port + .qp_state = IBV_QPS_RTR, + .path_mtu = ctx->portinfo.active_mtu, + .dest_qp_num = rem_dest->qpn, + .rq_psn = rem_dest->psn, + .ah_attr.dlid = rem_dest->lid, + .max_dest_rd_atomic = 1, + .min_rnr_timer = 12, + .ah_attr.is_global = 0, + .ah_attr.sl = 0, + .ah_attr.src_path_bits = 0, + .ah_attr.port_num = ib_port }; if (ibv_modify_qp(ctx->qp, &attr, (IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU - | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN - | IBV_QP_MIN_RNR_TIMER | IBV_QP_MAX_DEST_RD_ATOMIC))) { + | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN + | IBV_QP_MIN_RNR_TIMER | IBV_QP_MAX_DEST_RD_ATOMIC))) { gpu_err("Failed to modify QP to RTR\n"); return 1; } - - memset(&attr, 0, sizeof(struct ibv_qp_attr)); - attr.qp_state = IBV_QPS_RTS; - attr.sq_psn = 0; - attr.timeout = 20; - attr.retry_cnt = 7; - attr.rnr_retry = 7; - attr.max_rd_atomic = 1; - - if (ibv_modify_qp(ctx->qp, &attr, (IBV_QP_STATE | IBV_QP_SQ_PSN | IBV_QP_TIMEOUT - | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY - | IBV_QP_MAX_QP_RD_ATOMIC))) { + + memset(&attr, 0, sizeof(struct ibv_qp_attr)); + attr.qp_state = IBV_QPS_RTS; + attr.sq_psn = 0; + attr.timeout = 20; + attr.retry_cnt = 7; + attr.rnr_retry = 7; + attr.max_rd_atomic = 1; + + if (ibv_modify_qp(ctx->qp, &attr, (IBV_QP_STATE | IBV_QP_SQ_PSN | IBV_QP_TIMEOUT + | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY + | IBV_QP_MAX_QP_RD_ATOMIC))) { gpu_err("Failed to modify QP to RTS\n"); return 1; - } - } + } + } MPI_Barrier(MPI_COMM_WORLD); // for performance reasons, multiple batches back-to-back are posted here - ctx->rcnt = 0; + ctx->rcnt = 0; ctx->scnt = 0; ctx->n_tx_ev = 0; ctx->n_rx_ev = 0; nposted = 0; routs = 0; const int n_batches = 3; - //int prev_batch_len = 0; int last_batch_len = 0; int n_post = 0; int n_posted = 0; int batch; - int ii; + int ii; - if (gds_enable_event_prof) { - for (ii = 0; ii < MAX_EVENTS; ii++) { - cudaEventCreate(&start_time[ii]); - cudaEventCreate(&stop_time[ii]); - } - } + if (gds_enable_event_prof) { + for (ii = 0; ii < MAX_EVENTS; ii++) { + cudaEventCreate(&start_time[ii]); + cudaEventCreate(&stop_time[ii]); + } + } float pre_post_us = 0; @@ -1368,7 +1365,6 @@ int main(int argc, char *argv[]) } routs += n_posted; nposted += n_posted; - //prev_batch_len = last_batch_len; last_batch_len = n_posted; printf("[%d] batch %d: posted %d sequences\n", my_rank, batch, n_posted); } @@ -1377,12 +1373,12 @@ int main(int argc, char *argv[]) ret = 1; goto out; } - float usec = (end.tv_sec - start.tv_sec) * 1000000 + - (end.tv_usec - start.tv_usec); - printf("pre-posting took %.2f usec\n", usec); + float usec = (end.tv_sec - start.tv_sec) * 1000000 + + (end.tv_usec - start.tv_usec); + printf("pre-posting took %.2f usec\n", usec); pre_post_us = usec; } - ctx->pending = PINGPONG_RECV_WRID; + ctx->pending = PINGPONG_RECV_WRID; if (!my_rank) { puts(""); @@ -1395,41 +1391,22 @@ int main(int argc, char *argv[]) fflush(stdout); } - if (gettimeofday(&start, NULL)) { - perror("gettimeofday"); - return 1; - } + if (gettimeofday(&start, NULL)) { + perror("gettimeofday"); + return 1; + } + prof_enable(&prof); prof_idx = 0; int got_error = 0; int iter = 0; - while ((ctx->rcnt < iters || ctx->scnt < iters) && !got_error && !stream_cb_error) { + while ((ctx->rcnt < iters || ctx->scnt < iters) && !got_error && !stream_cb_error) { ++iter; PROF(&prof, prof_idx++); -#if 0 - if (!ctx->peersync) { - n_post = 1; - int n = pp_post_work(ctx, n_post, nposted, rem_dest->qpn, servername?1:0); - if (n != n_post) { - gpu_err("[%d] post_work error (%d) rcnt=%d n_post=%d routs=%d\n", my_rank, n, ctx->rcnt, n_post, routs); - return 1; - } - last_batch_len = n; - routs += n; - nposted += n; - - PROF(&prof, prof_idx++); - prof_update(&prof); - prof_idx = 0; - - continue; - } -#endif - int ret = gpu_wait_tracking_event(1000*1000); if (ret == ENOMEM) { - //gpu_info("[%d] gpu_wait_tracking_event reported nothing to do (%d)\n", my_rank, ret); + gpu_info("[%d] gpu_wait_tracking_event reported nothing to do (%d)\n", my_rank, ret); } else if (ret == EAGAIN) { gpu_info("[%d] gpu_wait_tracking_event reported timout (rc=%d), retrying\n", my_rank, ret); prof_reset(&prof); @@ -1470,13 +1447,11 @@ int main(int argc, char *argv[]) if (ctx->n_tx_ev || ctx->n_rx_ev) { // update counters routs -= last_batch_len; - //prev_batch_len = last_batch_len; if (ctx->n_tx_ev != last_batch_len) gpu_info("[%d] iter:%d unexpected tx ev %d, batch len %d\n", my_rank, iter, ctx->n_tx_ev, last_batch_len); if (ctx->n_rx_ev != last_batch_len) gpu_info("[%d] iter:%d unexpected rx ev %d, batch len %d\n", my_rank, iter, ctx->n_rx_ev, last_batch_len); if (nposted < iters) { - //fprintf(stdout, "rcnt=%d scnt=%d routs=%d nposted=%d\n", rcnt, scnt, routs, nposted); fflush(stdout); // potentially submit new work n_post = min(min(ctx->rx_depth/2, iters-nposted), max_batch_len); int n = pp_post_work(ctx, n_post, nposted, rem_dest->qpn, servername?1:0); @@ -1487,27 +1462,22 @@ int main(int argc, char *argv[]) last_batch_len = n; routs += n; nposted += n; - //fprintf(stdout, "n_post=%d n=%d\n", n_post, n); } } - //usleep(10); PROF(&prof, prof_idx++); - prof_update(&prof); - prof_idx = 0; - - //fprintf(stdout, "%d %d\n", rcnt, scnt); fflush(stdout); - + prof_update(&prof); + prof_idx = 0; if (got_error) { gpu_err("exiting for error\n"); return 1; } - } + } - if (gettimeofday(&end, NULL)) { - perror("gettimeofday"); - ret = 1; - } + if (gettimeofday(&end, NULL)) { + perror("gettimeofday"); + ret = 1; + } int rid; @@ -1519,9 +1489,9 @@ int main(int argc, char *argv[]) long long bytes = (long long) size * iters * 2; printf("[%d] %lld bytes in %.2f seconds = %.2f Mbit/sec\n", - my_rank, bytes, usec / 1000000., bytes * 8. / usec); + my_rank, bytes, usec / 1000000., bytes * 8. / usec); printf("[%d] %d iters in %.2f seconds = %.2f usec/iter\n", - my_rank, iters, usec / 1000000., usec / iters); + my_rank, iters, usec / 1000000., usec / iters); if (prof_enabled(&prof)) { printf("[%d] dumping prof\n", my_rank); @@ -1530,32 +1500,29 @@ int main(int argc, char *argv[]) } } - //ibv_ack_cq_events(ctx->cq, num_cq_events); - - //expect work to be completed by now + //expect work to be completed by now - if (gds_enable_event_prof) { - for (ii = 0; ii < event_idx; ii++) { - cudaEventElapsedTime(&elapsed_time, start_time[ii], stop_time[ii]); - gpu_err("[%d] size = %d, time = %f\n", my_rank, ctx->size, 1000 * elapsed_time); - } - for (ii = 0; ii < MAX_EVENTS; ii++) { - cudaEventDestroy(stop_time[ii]); - cudaEventDestroy(start_time[ii]); - } - } + if (gds_enable_event_prof) { + for (ii = 0; ii < event_idx; ii++) { + cudaEventElapsedTime(&elapsed_time, start_time[ii], stop_time[ii]); + gpu_err("[%d] size = %d, time = %f\n", my_rank, ctx->size, 1000 * elapsed_time); + } + for (ii = 0; ii < MAX_EVENTS; ii++) { + cudaEventDestroy(stop_time[ii]); + cudaEventDestroy(start_time[ii]); + } + } MPI_Barrier(MPI_COMM_WORLD); - if (pp_close_ctx(ctx)) - ret = 1; + if (pp_close_ctx(ctx)) + ret = 1; - ibv_free_device_list(dev_list); - //free(rem_dest); + ibv_free_device_list(dev_list); MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); out: - return ret; + return ret; } /* diff --git a/tests/gds_kernel_loopback_latency.c b/tests/gds_kernel_loopback_latency.c index b2d209c..181e3c8 100644 --- a/tests/gds_kernel_loopback_latency.c +++ b/tests/gds_kernel_loopback_latency.c @@ -79,36 +79,36 @@ int prof_idx = 0; #define USE_CUDA_PROFILER 1 enum { - PINGPONG_RECV_WRID = 1, - PINGPONG_SEND_WRID = 2, + PINGPONG_RECV_WRID = 1, + PINGPONG_SEND_WRID = 2, }; static int page_size; int stream_cb_error = 0; struct pingpong_context { - struct ibv_context *context; - struct ibv_comp_channel *channel; - struct ibv_pd *pd; - struct ibv_mr *mr; - struct ibv_cq *tx_cq; - struct ibv_cq *rx_cq; - struct ibv_qp *qp; - struct gds_qp *gds_qp; - struct ibv_ah *ah; - void *buf; - char *txbuf; + struct ibv_context *context; + struct ibv_comp_channel *channel; + struct ibv_pd *pd; + struct ibv_mr *mr; + struct ibv_cq *tx_cq; + struct ibv_cq *rx_cq; + struct ibv_qp *qp; + struct gds_qp *gds_qp; + struct ibv_ah *ah; + void *buf; + char *txbuf; char *rxbuf; char *rx_flag; - int size; + int size; int calc_size; - int rx_depth; - int pending; - struct ibv_port_attr portinfo; - int gpu_id; - int kernel_duration; - int peersync; - int peersync_gpu_cq; + int rx_depth; + int pending; + struct ibv_port_attr portinfo; + int gpu_id; + int kernel_duration; + int peersync; + int peersync_gpu_cq; int consume_rx_cqe; int gpumem; int use_desc_apis; @@ -118,141 +118,129 @@ struct pingpong_context { static int my_rank = 0, comm_size = 1; struct pingpong_dest { - int lid; - int qpn; - int psn; - union ibv_gid gid; + int lid; + int qpn; + int psn; + union ibv_gid gid; }; static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, - int sl, struct pingpong_dest *dest, int sgid_idx) + int sl, struct pingpong_dest *dest, int sgid_idx) { - struct ibv_ah_attr ah_attr = { - .is_global = 0, - .dlid = dest->lid, - .sl = sl, - .src_path_bits = 0, - .port_num = port - }; - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_RTR - }; - - if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) { - fprintf(stderr, "Failed to modify QP to RTR\n"); - return 1; - } - - attr.qp_state = IBV_QPS_RTS; - attr.sq_psn = my_psn; - - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_SQ_PSN)) { - fprintf(stderr, "Failed to modify QP to RTS\n"); - return 1; - } - - if (dest->gid.global.interface_id) { - ah_attr.is_global = 1; - ah_attr.grh.hop_limit = 1; - ah_attr.grh.dgid = dest->gid; - ah_attr.grh.sgid_index = sgid_idx; - } - - ctx->ah = ibv_create_ah(ctx->pd, &ah_attr); - if (!ctx->ah) { - fprintf(stderr, "Failed to create AH\n"); - return 1; - } - - return 0; + struct ibv_ah_attr ah_attr = { + .is_global = 0, + .dlid = dest->lid, + .sl = sl, + .src_path_bits = 0, + .port_num = port + }; + struct ibv_qp_attr attr = { + .qp_state = IBV_QPS_RTR + }; + + if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) { + fprintf(stderr, "Failed to modify QP to RTR\n"); + return 1; + } + + attr.qp_state = IBV_QPS_RTS; + attr.sq_psn = my_psn; + + if (ibv_modify_qp(ctx->qp, &attr, + IBV_QP_STATE | + IBV_QP_SQ_PSN)) { + fprintf(stderr, "Failed to modify QP to RTS\n"); + return 1; + } + + if (dest->gid.global.interface_id) { + ah_attr.is_global = 1; + ah_attr.grh.hop_limit = 1; + ah_attr.grh.dgid = dest->gid; + ah_attr.grh.sgid_index = sgid_idx; + } + + ctx->ah = ibv_create_ah(ctx->pd, &ah_attr); + if (!ctx->ah) { + fprintf(stderr, "Failed to create AH\n"); + return 1; + } + + return 0; } static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, - const struct pingpong_dest *my_dest) + const struct pingpong_dest *my_dest) { - struct addrinfo *res, *t; - struct addrinfo hints = { - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - char *service; - char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; - int n; - int sockfd = -1; - struct pingpong_dest *rem_dest = NULL; - char gid[33]; - - fprintf(stderr, "%04x:%06x:%06x:%s\n", my_dest->lid, my_dest->qpn, - my_dest->psn, (char *)&my_dest->gid); - rem_dest = malloc(sizeof *rem_dest); - if (!rem_dest) - goto out; - memcpy(rem_dest, my_dest, sizeof(struct pingpong_dest)); - //rem_dest->gid = my_dest->gid; - fprintf(stderr, "%04x:%06x:%06x\n", rem_dest->lid, rem_dest->qpn, - rem_dest->psn); + struct pingpong_dest *rem_dest = NULL; + + fprintf(stderr, "%04x:%06x:%06x:%s\n", my_dest->lid, my_dest->qpn, + my_dest->psn, (char *)&my_dest->gid); + rem_dest = malloc(sizeof *rem_dest); + if (!rem_dest) + goto out; + memcpy(rem_dest, my_dest, sizeof(struct pingpong_dest)); + fprintf(stderr, "%04x:%06x:%06x\n", rem_dest->lid, rem_dest->qpn, + rem_dest->psn); out: - return rem_dest; + return rem_dest; } static inline unsigned long align_to(unsigned long val, unsigned long pow2) { - return (val + pow2 - 1) & ~(pow2 - 1); + return (val + pow2 - 1) & ~(pow2 - 1); } static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, int calc_size, - int rx_depth, int port, - int use_event, - int gpu_id, - int peersync, - int peersync_gpu_cq, - int peersync_gpu_dbrec, - int consume_rx_cqe, - int sched_mode, - int use_gpumem, - int use_desc_apis, - int skip_kernel_launch) + int rx_depth, int port, + int use_event, + int gpu_id, + int peersync, + int peersync_gpu_cq, + int peersync_gpu_dbrec, + int consume_rx_cqe, + int sched_mode, + int use_gpumem, + int use_desc_apis, + int skip_kernel_launch) { - struct pingpong_context *ctx; + struct pingpong_context *ctx; - if (gpu_id >=0 && gpu_init(gpu_id, sched_mode)) { - fprintf(stderr, "error in GPU init.\n"); - return NULL; - } + if (gpu_id >=0 && gpu_init(gpu_id, sched_mode)) { + fprintf(stderr, "error in GPU init.\n"); + return NULL; + } - ctx = malloc(sizeof *ctx); - if (!ctx) - return NULL; + ctx = malloc(sizeof *ctx); + if (!ctx) + return NULL; - ctx->size = size; - ctx->calc_size = calc_size; - ctx->rx_depth = rx_depth; - ctx->gpu_id = gpu_id; + ctx->size = size; + ctx->calc_size = calc_size; + ctx->rx_depth = rx_depth; + ctx->gpu_id = gpu_id; ctx->gpumem = use_gpumem; ctx->use_desc_apis = use_desc_apis; ctx->skip_kernel_launch = skip_kernel_launch; size_t alloc_size = 3 * align_to(size + 40, page_size); - if (ctx->gpumem) { - ctx->buf = gpu_malloc(page_size, alloc_size); + if (ctx->gpumem) { + ctx->buf = gpu_malloc(page_size, alloc_size); printf("allocated GPU buffer address at %p\n", ctx->buf); - } else { + } else { printf("allocating CPU memory buf\n"); - ctx->buf = memalign(page_size, alloc_size); + ctx->buf = memalign(page_size, alloc_size); printf("allocated CPU buffer address at %p\n", ctx->buf); } - if (!ctx->buf) { - fprintf(stderr, "Couldn't allocate work buf.\n"); - goto clean_ctx; - } + if (!ctx->buf) { + fprintf(stderr, "Couldn't allocate work buf.\n"); + goto clean_ctx; + } printf("ctx buf=%p\n", ctx->buf); ctx->rxbuf = (char*)ctx->buf; ctx->txbuf = (char*)ctx->buf + align_to(size + 40, page_size); - //ctx->rx_flag = (char*)ctx->buf + 2 * align_to(size + 40, page_size); ctx->rx_flag = memalign(page_size, alloc_size); if (!ctx->rx_flag) { @@ -260,19 +248,18 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, goto clean_ctx; } - ctx->kernel_duration = 0; - ctx->peersync = peersync; + ctx->kernel_duration = 0; + ctx->peersync = peersync; ctx->peersync_gpu_cq = peersync_gpu_cq; ctx->consume_rx_cqe = consume_rx_cqe; // must be ZERO!!! for rx_flag to work... - if (ctx->gpumem) - gpu_memset(ctx->buf, 0, alloc_size); - else - memset(ctx->buf, 0, alloc_size); + if (ctx->gpumem) + gpu_memset(ctx->buf, 0, alloc_size); + else + memset(ctx->buf, 0, alloc_size); memset(ctx->rx_flag, 0, alloc_size); - //gpu_register_host_mem(ctx->rx_flag, alloc_size); if (!ctx->skip_kernel_launch) { // pipe-cleaner @@ -286,35 +273,33 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, CUCHECK(cuCtxSynchronize()); } - ctx->context = ibv_open_device(ib_dev); - if (!ctx->context) { - fprintf(stderr, "Couldn't get context for %s\n", - ibv_get_device_name(ib_dev)); - goto clean_buffer; - } - - if (use_event) { - ctx->channel = ibv_create_comp_channel(ctx->context); - if (!ctx->channel) { - fprintf(stderr, "Couldn't create completion channel\n"); - goto clean_device; - } - } else - ctx->channel = NULL; - - ctx->pd = ibv_alloc_pd(ctx->context); - if (!ctx->pd) { - fprintf(stderr, "Couldn't allocate PD\n"); - goto clean_comp_channel; - } - - //printf("BEFORE reg_mr(), sleeping 2s\n"); sleep(2); - ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, alloc_size, IBV_ACCESS_LOCAL_WRITE); - if (!ctx->mr) { - fprintf(stderr, "Couldn't register MR\n"); - goto clean_pd; - } - //printf("AFTER reg_mr(), sleeping 2s\n"); sleep(2); + ctx->context = ibv_open_device(ib_dev); + if (!ctx->context) { + fprintf(stderr, "Couldn't get context for %s\n", + ibv_get_device_name(ib_dev)); + goto clean_buffer; + } + + if (use_event) { + ctx->channel = ibv_create_comp_channel(ctx->context); + if (!ctx->channel) { + fprintf(stderr, "Couldn't create completion channel\n"); + goto clean_device; + } + } else + ctx->channel = NULL; + + ctx->pd = ibv_alloc_pd(ctx->context); + if (!ctx->pd) { + fprintf(stderr, "Couldn't allocate PD\n"); + goto clean_comp_channel; + } + + ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, alloc_size, IBV_ACCESS_LOCAL_WRITE); + if (!ctx->mr) { + fprintf(stderr, "Couldn't register MR\n"); + goto clean_pd; + } int gds_flags = 0; if (peersync_gpu_cq) @@ -333,8 +318,8 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, }, .qp_type = IBV_QPT_UD, }; - - //why? + + //why? if (my_rank == 1) { printf("sleeping 2s\n"); sleep(2); @@ -344,100 +329,98 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, if (!ctx->gds_qp) { fprintf(stderr, "Couldn't create QP (%d/%s)\n", errno, strerror(errno)); goto clean_mr; - } - ctx->qp = ctx->gds_qp->qp; - ctx->tx_cq = ctx->gds_qp->qp->send_cq; - ctx->rx_cq = ctx->gds_qp->qp->recv_cq; - - { - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_INIT, - .pkey_index = 0, - .port_num = port, - .qkey = 0x11111111 - }; - - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_PKEY_INDEX | - IBV_QP_PORT | - IBV_QP_QKEY)) { - fprintf(stderr, "Failed to modify QP to INIT\n"); - goto clean_qp; - } - } - - return ctx; + } + ctx->qp = ctx->gds_qp->ibqp; + ctx->tx_cq = ctx->gds_qp->ibqp->send_cq; + ctx->rx_cq = ctx->gds_qp->ibqp->recv_cq; + + { + struct ibv_qp_attr attr = { + .qp_state = IBV_QPS_INIT, + .pkey_index = 0, + .port_num = port, + .qkey = 0x11111111 + }; + + if (ibv_modify_qp(ctx->qp, &attr, + IBV_QP_STATE | + IBV_QP_PKEY_INDEX | + IBV_QP_PORT | + IBV_QP_QKEY)) { + fprintf(stderr, "Failed to modify QP to INIT\n"); + goto clean_qp; + } + } + + return ctx; clean_qp: - gds_destroy_qp(ctx->gds_qp); + gds_destroy_qp(ctx->gds_qp); clean_mr: - ibv_dereg_mr(ctx->mr); + ibv_dereg_mr(ctx->mr); clean_pd: - ibv_dealloc_pd(ctx->pd); + ibv_dealloc_pd(ctx->pd); clean_comp_channel: - if (ctx->channel) - ibv_destroy_comp_channel(ctx->channel); + if (ctx->channel) + ibv_destroy_comp_channel(ctx->channel); clean_device: - ibv_close_device(ctx->context); + ibv_close_device(ctx->context); clean_buffer: - if (ctx->gpumem) - gpu_free(ctx->buf); - else - free(ctx->buf); + if (ctx->gpumem) + gpu_free(ctx->buf); + else + free(ctx->buf); clean_ctx: - if (ctx->gpu_id >= 0) - gpu_finalize(); - free(ctx); + if (ctx->gpu_id >= 0) + gpu_finalize(); + free(ctx); - return NULL; + return NULL; } int pp_close_ctx(struct pingpong_context *ctx) { - if (gds_destroy_qp(ctx->gds_qp)) { - fprintf(stderr, "Couldn't destroy QP\n"); - } + gds_destroy_qp(ctx->gds_qp); - if (ibv_dereg_mr(ctx->mr)) { - fprintf(stderr, "Couldn't deregister MR\n"); - } + if (ibv_dereg_mr(ctx->mr)) { + fprintf(stderr, "Couldn't deregister MR\n"); + } - if (ibv_destroy_ah(ctx->ah)) { - fprintf(stderr, "Couldn't destroy AH\n"); - } + if (ibv_destroy_ah(ctx->ah)) { + fprintf(stderr, "Couldn't destroy AH\n"); + } - if (ibv_dealloc_pd(ctx->pd)) { - fprintf(stderr, "Couldn't deallocate PD\n"); - } + if (ibv_dealloc_pd(ctx->pd)) { + fprintf(stderr, "Couldn't deallocate PD\n"); + } - if (ctx->channel) { - if (ibv_destroy_comp_channel(ctx->channel)) { - fprintf(stderr, "Couldn't destroy completion channel\n"); - } - } + if (ctx->channel) { + if (ibv_destroy_comp_channel(ctx->channel)) { + fprintf(stderr, "Couldn't destroy completion channel\n"); + } + } - if (ibv_close_device(ctx->context)) { - fprintf(stderr, "Couldn't release context\n"); - } + if (ibv_close_device(ctx->context)) { + fprintf(stderr, "Couldn't release context\n"); + } - if (ctx->gpumem) - gpu_free(ctx->buf); - else - free(ctx->buf); + if (ctx->gpumem) + gpu_free(ctx->buf); + else + free(ctx->buf); - if (ctx->gpu_id >= 0) - gpu_finalize(); + if (ctx->gpu_id >= 0) + gpu_finalize(); - free(ctx); + free(ctx); - return 0; + return 0; } static int block_server_stream(struct pingpong_context *ctx) @@ -461,123 +444,116 @@ static int unblock_server_stream(struct pingpong_context *ctx) usleep(100); int ret = cuStreamQuery(gpu_stream_server); switch (ret) { - case CUDA_ERROR_NOT_READY: - break; - case CUDA_SUCCESS: - gpu_err("unexpected idle stream\n"); - retcode = EINVAL; - break; - default: - gpu_err("unexpected error %d in stream query\n", ret); - retcode = EINVAL; - break; + case CUDA_ERROR_NOT_READY: + break; + case CUDA_SUCCESS: + gpu_err("unexpected idle stream\n"); + retcode = EINVAL; + break; + default: + gpu_err("unexpected error %d in stream query\n", ret); + retcode = EINVAL; + break; } gds_atomic_set_dword((uint32_t *)ctx->rx_flag, 1); - return 0; + return retcode; } static int pp_post_recv(struct pingpong_context *ctx, int n) { - struct ibv_sge list = { - .addr = (uintptr_t) ctx->rxbuf, - .length = ctx->size + 40, - .lkey = ctx->mr->lkey - }; - struct ibv_recv_wr wr = { - .wr_id = PINGPONG_RECV_WRID, - .sg_list = &list, - .num_sge = 1, - }; - struct ibv_recv_wr *bad_wr; - int i; + struct ibv_sge list = { + .addr = (uintptr_t) ctx->rxbuf, + .length = ctx->size + 40, + .lkey = ctx->mr->lkey + }; + struct ibv_recv_wr wr = { + .wr_id = PINGPONG_RECV_WRID, + .sg_list = &list, + .num_sge = 1, + }; + struct ibv_recv_wr *bad_wr; + int i; gpu_dbg("posting %d recvs\n", n); - for (i = 0; i < n; ++i) - if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) - break; + for (i = 0; i < n; ++i) + if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) + break; gpu_dbg("posted %d recvs\n", i); - return i; + return i; } // will be needed when implementing the !peersync !use_desc_apis case static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn) { - int ret = 0; - struct ibv_sge list = { - .addr = (uintptr_t) ctx->txbuf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - gds_send_wr ewr = { - .wr_id = PINGPONG_SEND_WRID, - .sg_list = &list, - .num_sge = 1, - .exp_opcode = IBV_EXP_WR_SEND, - .exp_send_flags = IBV_EXP_SEND_SIGNALED, - .wr = { - .ud = { - .ah = ctx->ah, - .remote_qpn = qpn, - .remote_qkey = 0x11111111 - } - }, - .comp_mask = 0 - }; - gds_send_wr *bad_ewr; - return gds_post_send(ctx->gds_qp, &ewr, &bad_ewr); + struct ibv_sge list = { + .addr = (uintptr_t) ctx->txbuf, + .length = ctx->size, + .lkey = ctx->mr->lkey + }; + gds_send_wr ewr = { + .wr_id = PINGPONG_SEND_WRID, + .sg_list = &list, + .num_sge = 1, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, + .wr = { + .ud = { + .ah = ctx->ah, + .remote_qpn = qpn, + .remote_qkey = 0x11111111 + } + } + }; + gds_send_wr *bad_ewr; + return gds_post_send(ctx->gds_qp, &ewr, &bad_ewr); } static int pp_post_gpu_send(struct pingpong_context *ctx, uint32_t qpn, CUstream *p_gpu_stream) { - int ret = 0; - struct ibv_sge list = { - .addr = (uintptr_t) ctx->txbuf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - gds_send_wr ewr = { - .wr_id = PINGPONG_SEND_WRID, - .sg_list = &list, - .num_sge = 1, - .exp_opcode = IBV_EXP_WR_SEND, - .exp_send_flags = IBV_EXP_SEND_SIGNALED, - .wr = { - .ud = { - .ah = ctx->ah, - .remote_qpn = qpn, - .remote_qkey = 0x11111111 - } - }, - .comp_mask = 0 - }; - gds_send_wr *bad_ewr; - return gds_stream_queue_send(*p_gpu_stream, ctx->gds_qp, &ewr, &bad_ewr); + struct ibv_sge list = { + .addr = (uintptr_t) ctx->txbuf, + .length = ctx->size, + .lkey = ctx->mr->lkey + }; + gds_send_wr ewr = { + .wr_id = PINGPONG_SEND_WRID, + .sg_list = &list, + .num_sge = 1, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, + .wr = { + .ud = { + .ah = ctx->ah, + .remote_qpn = qpn, + .remote_qkey = 0x11111111 + } + } + }; + gds_send_wr *bad_ewr; + return gds_stream_queue_send(*p_gpu_stream, ctx->gds_qp, &ewr, &bad_ewr); } static int pp_prepare_gpu_send(struct pingpong_context *ctx, uint32_t qpn, gds_send_request_t *req) { - int ret = 0; - struct ibv_sge list = { - .addr = (uintptr_t) ctx->txbuf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - gds_send_wr ewr = { - .wr_id = PINGPONG_SEND_WRID, - .sg_list = &list, - .num_sge = 1, - .exp_opcode = IBV_EXP_WR_SEND, - .exp_send_flags = IBV_EXP_SEND_SIGNALED, - .wr = { - .ud = { - .ah = ctx->ah, - .remote_qpn = qpn, - .remote_qkey = 0x11111111 - } - }, - .comp_mask = 0 - }; - gds_send_wr *bad_ewr; - //printf("gpu_post_send_on_stream\n"); + struct ibv_sge list = { + .addr = (uintptr_t) ctx->txbuf, + .length = ctx->size, + .lkey = ctx->mr->lkey + }; + gds_send_wr ewr = { + .wr_id = PINGPONG_SEND_WRID, + .sg_list = &list, + .num_sge = 1, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, + .wr = { + .ud = { + .ah = ctx->ah, + .remote_qpn = qpn, + .remote_qkey = 0x11111111 + } + } + }; + gds_send_wr *bad_ewr; return gds_prepare_send(ctx->gds_qp, &ewr, &bad_ewr, req); } @@ -614,7 +590,7 @@ static void post_work_cb(CUstream hStream, CUresult status, void *userData)\ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uint32_t qpn, int is_client) { int retcode = 0; - int i, ret = 0; + int i, ret = 0; int posted_recv = 0; gpu_dbg("n_posts=%d rcnt=%d is_client=%d\n", n_posts, rcnt, is_client); @@ -628,7 +604,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin posted_recv = pp_post_recv(ctx, n_posts); if (posted_recv < 0) { fprintf(stderr,"ERROR: can't post recv (%d) n_posts=%d is_client=%d\n", - posted_recv, n_posts, is_client); + posted_recv, n_posts, is_client); exit(EXIT_FAILURE); return 0; } else if (posted_recv != n_posts) { @@ -637,11 +613,11 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin return 0; } NVTX_POP(); - + PROF(&prof, prof_idx++); NVTX_PUSH("post send+wait", 1); - for (i = 0; i < posted_recv; ++i) { + for (i = 0; i < posted_recv; ++i) { if (ctx->use_desc_apis) { work_desc_t *wdesc = calloc(1, sizeof(*wdesc)); int k = 0; @@ -655,7 +631,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin wdesc->descs[k].send = &wdesc->send_rq; ++k; - ret = gds_prepare_wait_cq(&ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->send_cq, &wdesc->wait_tx_rq, 0); if (ret) { retcode = -ret; break; @@ -665,7 +641,7 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin wdesc->descs[k].wait = &wdesc->wait_tx_rq; ++k; - ret = gds_prepare_wait_cq(&ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); + ret = gds_prepare_wait_cq(ctx->gds_qp->recv_cq, &wdesc->wait_rx_rq, 0); if (ret) { retcode = -ret; break; @@ -692,12 +668,12 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin ret = pp_post_gpu_send(ctx, qpn, &gpu_stream_server); if (ret) { gpu_err("error %d in pp_post_gpu_send, posted_recv=%d posted_so_far=%d is_client=%d \n", - ret, posted_recv, i, is_client); + ret, posted_recv, i, is_client); retcode = -ret; break; } - ret = gds_stream_wait_cq(gpu_stream_server, &ctx->gds_qp->send_cq, 0); + ret = gds_stream_wait_cq(gpu_stream_server, ctx->gds_qp->send_cq, 0); if (ret) { // TODO: rollback gpu send gpu_err("error %d in gds_stream_wait_cq\n", ret); @@ -705,11 +681,10 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin break; } - ret = gds_stream_wait_cq(gpu_stream_server, &ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); + ret = gds_stream_wait_cq(gpu_stream_server, ctx->gds_qp->recv_cq, ctx->consume_rx_cqe); if (ret) { // TODO: rollback gpu send and wait send_cq gpu_err("error %d in gds_stream_wait_cq\n", ret); - //exit(EXIT_FAILURE); retcode = -ret; break; } @@ -718,10 +693,10 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin retcode = -EINVAL; break; } - if (ctx->skip_kernel_launch) { + if (ctx->skip_kernel_launch) { gpu_warn_once("NOT LAUNCHING ANY KERNEL AT ALL\n"); } else { - gpu_launch_kernel_on_stream(ctx->calc_size, ctx->peersync, gpu_stream_server); + gpu_launch_kernel_on_stream(ctx->calc_size, ctx->peersync, gpu_stream_server); } } @@ -729,71 +704,68 @@ static int pp_post_work(struct pingpong_context *ctx, int n_posts, int rcnt, uin if (!retcode) { retcode = i; gpu_post_release_tracking_event(&gpu_stream_server); - //sleep(1); } NVTX_POP(); - return retcode; + return retcode; } static void usage(const char *argv0) { - printf("Usage:\n"); - printf(" %s start a server and wait for connection\n", argv0); - printf(" %s connect to server at \n", argv0); - printf("\n"); - printf("Options:\n"); - printf(" -p, --port= listen on/connect to port (default 18515)\n"); - printf(" -d, --ib-dev= use IB device (default first device found)\n"); - printf(" -i, --ib-port= use port of IB device (default 1)\n"); - printf(" -s, --size= size of message to exchange (default 1024)\n"); - printf(" -r, --rx-depth= number of receives to post at a time (default 500)\n"); - printf(" -n, --iters= number of exchanges (default 1000)\n"); - printf(" -e, --events sleep on CQ events (default poll)\n"); - printf(" -g, --gid-idx= local port gid index\n"); - printf(" -S, --gpu-calc-size= size of GPU compute buffer (default 128KB)\n"); - printf(" -G, --gpu-id use specified GPU (default 0)\n"); - printf(" -B, --batch-length= max batch length (default 20)\n"); - printf(" -P, --peersync disable GPUDirect PeerSync support (default enabled)\n"); - printf(" -C, --peersync-gpu-cq enable GPUDirect PeerSync GPU CQ support (default disabled)\n"); - printf(" -D, --peersync-gpu-dbrec enable QP DBREC on GPU memory (default disabled)\n"); - printf(" -U, --peersync-desc-apis use batched descriptor APIs (default disabled)\n"); - printf(" -Q, --consume-rx-cqe enable GPU consumes RX CQE support (default disabled)\n"); - printf(" -M, --gpu-sched-mode set CUDA context sched mode, default (A)UTO, (S)PIN, (Y)IELD, (B)LOCKING\n"); - printf(" -E, --gpu-mem allocate GPU intead of CPU memory buffers\n"); - printf(" -K, --skip-kernel-launch no GPU kernel computations, only communications\n"); - printf(" -L, --hide-cpu-launch-latency try to prelaunch work on blocked stream then unblock\n"); + printf("Usage:\n"); + printf(" %s start a server and wait for connection\n", argv0); + printf(" %s connect to server at \n", argv0); + printf("\n"); + printf("Options:\n"); + printf(" -p, --port= listen on/connect to port (default 18515)\n"); + printf(" -d, --ib-dev= use IB device (default first device found)\n"); + printf(" -i, --ib-port= use port of IB device (default 1)\n"); + printf(" -s, --size= size of message to exchange (default 1024)\n"); + printf(" -r, --rx-depth= number of receives to post at a time (default 500)\n"); + printf(" -n, --iters= number of exchanges (default 1000)\n"); + printf(" -e, --events sleep on CQ events (default poll)\n"); + printf(" -g, --gid-idx= local port gid index\n"); + printf(" -S, --gpu-calc-size= size of GPU compute buffer (default 128KB)\n"); + printf(" -G, --gpu-id use specified GPU (default 0)\n"); + printf(" -B, --batch-length= max batch length (default 20)\n"); + printf(" -P, --peersync disable GPUDirect PeerSync support (default enabled)\n"); + printf(" -C, --peersync-gpu-cq enable GPUDirect PeerSync GPU CQ support (default disabled)\n"); + printf(" -D, --peersync-gpu-dbrec enable QP DBREC on GPU memory (default disabled)\n"); + printf(" -U, --peersync-desc-apis use batched descriptor APIs (default disabled)\n"); + printf(" -Q, --consume-rx-cqe enable GPU consumes RX CQE support (default disabled)\n"); + printf(" -M, --gpu-sched-mode set CUDA context sched mode, default (A)UTO, (S)PIN, (Y)IELD, (B)LOCKING\n"); + printf(" -E, --gpu-mem allocate GPU instead of CPU memory buffers\n"); + printf(" -K, --skip-kernel-launch no GPU kernel computations, only communications\n"); + printf(" -L, --hide-cpu-launch-latency try to prelaunch work on blocked stream then unblock\n"); } int main(int argc, char *argv[]) { - struct ibv_device **dev_list; - struct ibv_device *ib_dev; - struct pingpong_context *ctx; - struct pingpong_dest my_dest; - struct pingpong_dest *rem_dest; - struct timeval rstart, start, end; - const char *ib_devname = NULL; - char *servername = NULL; - int port = 18515; - int ib_port = 1; - int size = 1024; - int calc_size = 128*1024; - int rx_depth = 2*512; - int iters = 1000; - int use_event = 0; - int routs; + struct ibv_device **dev_list; + struct ibv_device *ib_dev; + struct pingpong_context *ctx; + struct pingpong_dest my_dest; + struct pingpong_dest *rem_dest; + struct timeval start, end; + const char *ib_devname = NULL; + char *servername = NULL; + int port = 18515; + int ib_port = 1; + int size = 1024; + int calc_size = 128*1024; + int rx_depth = 2*512; + int iters = 1000; + int use_event = 0; + int routs; int nposted; - int rcnt, scnt; - int num_cq_events = 0; - int sl = 0; - int gidx = -1; - char gid[INET6_ADDRSTRLEN]; - int gpu_id = 0; + int rcnt, scnt; + int sl = 0; + int gidx = -1; + char gid[INET6_ADDRSTRLEN]; + int gpu_id = 0; int peersync = 1; int peersync_gpu_cq = 0; int peersync_gpu_dbrec = 0; - int warmup = 10; int max_batch_len = 20; int consume_rx_cqe = 0; int sched_mode = CU_CTX_SCHED_AUTO; @@ -804,6 +776,12 @@ int main(int argc, char *argv[]) int skip_kernel_launch = 0; int hide_cpu_launch_latency = 0; + /*printf("sizeof(gds_send_request_t)=%zu\n", sizeof(gds_send_request_t)); + printf("sizeof(gds_mlx5_peer_commit)=%zu\n", sizeof(struct gds_mlx5_peer_commit)); + printf("sizeof(gds_mlx5_peer_op_wr)=%zu\n", sizeof(struct gds_mlx5_peer_op_wr)); + printf("sizeof(gds_wait_request_t)=%zu\n", sizeof(gds_wait_request_t)); + printf("sizeof(gds_mlx5_peer_peek)=%zu\n", sizeof(struct gds_mlx5_peer_peek)); + exit(0);*/ fprintf(stdout, "libgdsync build version 0x%08x, major=%d minor=%d\n", GDS_API_VERSION, GDS_API_MAJOR_VERSION, GDS_API_MINOR_VERSION); int version; @@ -818,171 +796,171 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } - srand48(getpid() * time(NULL)); - - while (1) { - int c; - - static struct option long_options[] = { - { .name = "port", .has_arg = 1, .val = 'p' }, - { .name = "ib-dev", .has_arg = 1, .val = 'd' }, - { .name = "ib-port", .has_arg = 1, .val = 'i' }, - { .name = "size", .has_arg = 1, .val = 's' }, - { .name = "rx-depth", .has_arg = 1, .val = 'r' }, - { .name = "iters", .has_arg = 1, .val = 'n' }, - { .name = "sl", .has_arg = 1, .val = 'l' }, - { .name = "events", .has_arg = 0, .val = 'e' }, - { .name = "gid-idx", .has_arg = 1, .val = 'g' }, - { .name = "gpu-id", .has_arg = 1, .val = 'G' }, - { .name = "peersync", .has_arg = 0, .val = 'P' }, - { .name = "peersync-gpu-cq", .has_arg = 0, .val = 'C' }, - { .name = "peersync-gpu-dbrec", .has_arg = 1, .val = 'D' }, + srand48(getpid() * time(NULL)); + + while (1) { + int c; + + static struct option long_options[] = { + { .name = "port", .has_arg = 1, .val = 'p' }, + { .name = "ib-dev", .has_arg = 1, .val = 'd' }, + { .name = "ib-port", .has_arg = 1, .val = 'i' }, + { .name = "size", .has_arg = 1, .val = 's' }, + { .name = "rx-depth", .has_arg = 1, .val = 'r' }, + { .name = "iters", .has_arg = 1, .val = 'n' }, + { .name = "sl", .has_arg = 1, .val = 'l' }, + { .name = "events", .has_arg = 0, .val = 'e' }, + { .name = "gid-idx", .has_arg = 1, .val = 'g' }, + { .name = "gpu-id", .has_arg = 1, .val = 'G' }, + { .name = "peersync", .has_arg = 0, .val = 'P' }, + { .name = "peersync-gpu-cq", .has_arg = 0, .val = 'C' }, + { .name = "peersync-gpu-dbrec", .has_arg = 1, .val = 'D' }, { .name = "peersync-desc-apis", .has_arg = 0, .val = 'U' }, - { .name = "gpu-calc-size", .has_arg = 1, .val = 'S' }, - { .name = "batch-length", .has_arg = 1, .val = 'B' }, - { .name = "consume-rx-cqe", .has_arg = 0, .val = 'Q' }, - { .name = "gpu-sched-mode", .has_arg = 1, .val = 'M' }, - { .name = "gpu-mem", .has_arg = 0, .val = 'E' }, - { .name = "wait-key", .has_arg = 1, .val = 'W' }, - { .name = "skip-kernel-launch", .has_arg = 0, .val = 'K' }, - { .name = "hide-cpu-launch-latency", .has_arg = 0, .val = 'L' }, - { 0 } - }; - - c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:G:S:B:PCDQM:W:EUKL", long_options, NULL); - if (c == -1) - break; - - switch (c) { - case 'p': - port = strtol(optarg, NULL, 0); - if (port < 0 || port > 65535) { - usage(argv[0]); - ret = 1; - exit(EXIT_FAILURE); - } - break; - - case 'd': - ib_devname = strdupa(optarg); - break; - - case 'i': - ib_port = strtol(optarg, NULL, 0); - if (ib_port < 0) { - usage(argv[0]); - ret = 1; - exit(EXIT_FAILURE); - } - break; - - case 's': - size = strtol(optarg, NULL, 0); - printf("INFO: message size=%d\n", size); - break; - - case 'S': - calc_size = strtol(optarg, NULL, 0); - printf("INFO: kernel calc size=%d\n", calc_size); - break; - - case 'r': - rx_depth = strtol(optarg, NULL, 0); - break; - - case 'n': - iters = strtol(optarg, NULL, 0); - break; - - case 'l': - sl = strtol(optarg, NULL, 0); - break; - - case 'e': - ++use_event; - break; - - case 'g': - gidx = strtol(optarg, NULL, 0); - break; - - case 'G': - gpu_id = strtol(optarg, NULL, 0); - printf("INFO: gpu id=%d\n", gpu_id); - break; - - case 'B': - max_batch_len = strtol(optarg, NULL, 0); - printf("INFO: max_batch_len=%d\n", max_batch_len); - break; - - case 'P': - peersync = !peersync; - printf("INFO: switching PeerSync %s\n", peersync?"ON":"OFF"); - if (!peersync) { - printf("WARNING: PeerSync OFF is approximated using CUDA stream callbacks\n"); - } - break; + { .name = "gpu-calc-size", .has_arg = 1, .val = 'S' }, + { .name = "batch-length", .has_arg = 1, .val = 'B' }, + { .name = "consume-rx-cqe", .has_arg = 0, .val = 'Q' }, + { .name = "gpu-sched-mode", .has_arg = 1, .val = 'M' }, + { .name = "gpu-mem", .has_arg = 0, .val = 'E' }, + { .name = "wait-key", .has_arg = 1, .val = 'W' }, + { .name = "skip-kernel-launch", .has_arg = 0, .val = 'K' }, + { .name = "hide-cpu-launch-latency", .has_arg = 0, .val = 'L' }, + { 0 } + }; + + c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:G:S:B:PCDQM:W:EUKL", long_options, NULL); + if (c == -1) + break; - case 'Q': - consume_rx_cqe = !consume_rx_cqe; - printf("INFO: switching consume_rx_cqe %s\n", consume_rx_cqe?"ON":"OFF"); - break; + switch (c) { + case 'p': + port = strtol(optarg, NULL, 0); + if (port < 0 || port > 65535) { + usage(argv[0]); + ret = 1; + exit(EXIT_FAILURE); + } + break; - case 'C': - peersync_gpu_cq = !peersync_gpu_cq; - printf("INFO: switching %s PeerSync GPU CQ\n", peersync_gpu_cq?"ON":"OFF"); - break; + case 'd': + ib_devname = strdup(optarg); + break; - case 'D': - peersync_gpu_dbrec= !peersync_gpu_dbrec; - printf("INFO: switching %s PeerSync GPU QP DBREC\n", peersync_gpu_dbrec?"ON":"OFF"); - break; + case 'i': + ib_port = strtol(optarg, NULL, 0); + if (ib_port < 0) { + usage(argv[0]); + ret = 1; + exit(EXIT_FAILURE); + } + break; - case 'M': - { - char m = *optarg; - printf("INFO: sched mode '%c'\n", m); - switch (m) { - case 'S': sched_mode = CU_CTX_SCHED_SPIN; break; - case 'Y': sched_mode = CU_CTX_SCHED_YIELD; break; - case 'B': sched_mode = CU_CTX_SCHED_BLOCKING_SYNC; break; - case 'A': sched_mode = CU_CTX_SCHED_AUTO; break; - default: printf("ERROR: unexpected value %c\n", m); exit(EXIT_FAILURE); break; - } - } - break; + case 's': + size = strtol(optarg, NULL, 0); + printf("INFO: message size=%d\n", size); + break; - case 'W': - wait_key = strtol(optarg, NULL, 0); - printf("INFO: wait_key=%d\n", wait_key); - break; + case 'S': + calc_size = strtol(optarg, NULL, 0); + printf("INFO: kernel calc size=%d\n", calc_size); + break; - case 'E': - use_gpumem = !use_gpumem; - printf("INFO: use_gpumem=%d\n", use_gpumem); - break; + case 'r': + rx_depth = strtol(optarg, NULL, 0); + break; - case 'U': - use_desc_apis = 1; - printf("INFO: use_desc_apis=%d\n", use_desc_apis); - break; + case 'n': + iters = strtol(optarg, NULL, 0); + break; - case 'K': - skip_kernel_launch = 1; - printf("INFO: skip_kernel_launch=%d\n", skip_kernel_launch); - break; + case 'l': + sl = strtol(optarg, NULL, 0); + break; - case 'L': - hide_cpu_launch_latency = 1; - printf("INFO: hide_cpu_launch_latency=%d\n", hide_cpu_launch_latency); - break; + case 'e': + ++use_event; + break; + + case 'g': + gidx = strtol(optarg, NULL, 0); + break; + + case 'G': + gpu_id = strtol(optarg, NULL, 0); + printf("INFO: gpu id=%d\n", gpu_id); + break; + + case 'B': + max_batch_len = strtol(optarg, NULL, 0); + printf("INFO: max_batch_len=%d\n", max_batch_len); + break; + + case 'P': + peersync = !peersync; + printf("INFO: switching PeerSync %s\n", peersync?"ON":"OFF"); + if (!peersync) { + printf("WARNING: PeerSync OFF is approximated using CUDA stream callbacks\n"); + } + break; + + case 'Q': + consume_rx_cqe = !consume_rx_cqe; + printf("INFO: switching consume_rx_cqe %s\n", consume_rx_cqe?"ON":"OFF"); + break; + + case 'C': + peersync_gpu_cq = !peersync_gpu_cq; + printf("INFO: switching %s PeerSync GPU CQ\n", peersync_gpu_cq?"ON":"OFF"); + break; + + case 'D': + peersync_gpu_dbrec= !peersync_gpu_dbrec; + printf("INFO: switching %s PeerSync GPU QP DBREC\n", peersync_gpu_dbrec?"ON":"OFF"); + break; + + case 'M': + { + char m = *optarg; + printf("INFO: sched mode '%c'\n", m); + switch (m) { + case 'S': sched_mode = CU_CTX_SCHED_SPIN; break; + case 'Y': sched_mode = CU_CTX_SCHED_YIELD; break; + case 'B': sched_mode = CU_CTX_SCHED_BLOCKING_SYNC; break; + case 'A': sched_mode = CU_CTX_SCHED_AUTO; break; + default: printf("ERROR: unexpected value %c\n", m); exit(EXIT_FAILURE); break; + } + } + break; + + case 'W': + wait_key = strtol(optarg, NULL, 0); + printf("INFO: wait_key=%d\n", wait_key); + break; + + case 'E': + use_gpumem = !use_gpumem; + printf("INFO: use_gpumem=%d\n", use_gpumem); + break; + + case 'U': + use_desc_apis = 1; + printf("INFO: use_desc_apis=%d\n", use_desc_apis); + break; - default: - usage(argv[0]); - return 1; - } - } + case 'K': + skip_kernel_launch = 1; + printf("INFO: skip_kernel_launch=%d\n", skip_kernel_launch); + break; + + case 'L': + hide_cpu_launch_latency = 1; + printf("INFO: hide_cpu_launch_latency=%d\n", hide_cpu_launch_latency); + break; + + default: + usage(argv[0]); + return 1; + } + } if (!peersync && !use_desc_apis) { gpu_err("!peersync case only supported when using descriptor APIs, enabling them\n"); @@ -994,23 +972,22 @@ int main(int argc, char *argv[]) char *hostnames[1] = {"localhost"}; if (my_rank == 0) { - servername = hostnames[0]; + servername = hostnames[0]; printf("[%d] pid=%d server:%s\n", my_rank, getpid(), servername); } const char *tags = NULL; tags = "wait trk|pollrxcq|polltxcq|postrecv|postwork| poketrk"; prof_init(&prof, 100000, 100000, "100us", 60, 2, tags); - //prof_init(&prof, 100, 100, "100ns", 25*4, 2, tags); prof_disable(&prof); - page_size = sysconf(_SC_PAGESIZE); + page_size = sysconf(_SC_PAGESIZE); - dev_list = ibv_get_device_list(NULL); - if (!dev_list) { - perror("Failed to get IB devices list"); - return 1; - } + dev_list = ibv_get_device_list(NULL); + if (!dev_list) { + perror("Failed to get IB devices list"); + return 1; + } if (!ib_devname) { const char *value = getenv("USE_HCA"); @@ -1022,24 +999,24 @@ int main(int argc, char *argv[]) printf("[%d] requested IB device: <%s>\n", my_rank, ib_devname); } - if (!ib_devname) { + if (!ib_devname) { printf("picking 1st available device\n"); - ib_dev = *dev_list; - if (!ib_dev) { - fprintf(stderr, "No IB devices found\n"); - return 1; - } - } else { - int i; - for (i = 0; dev_list[i]; ++i) - if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) - break; - ib_dev = dev_list[i]; - if (!ib_dev) { - fprintf(stderr, "IB device %s not found\n", ib_devname); - return 1; - } - } + ib_dev = *dev_list; + if (!ib_dev) { + fprintf(stderr, "No IB devices found\n"); + return 1; + } + } else { + int i; + for (i = 0; dev_list[i]; ++i) + if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) + break; + ib_dev = dev_list[i]; + if (!ib_dev) { + fprintf(stderr, "IB device %s not found\n", ib_devname); + return 1; + } + } { const char *env = getenv("USE_GPU"); @@ -1049,54 +1026,54 @@ int main(int argc, char *argv[]) } } printf("use gpumem: %d\n", use_gpumem); - ctx = pp_init_ctx(ib_dev, size, calc_size, rx_depth, ib_port, 0, gpu_id, peersync, peersync_gpu_cq, peersync_gpu_dbrec, consume_rx_cqe, sched_mode, use_gpumem, use_desc_apis, skip_kernel_launch); - if (!ctx) - return 1; - - //pre-posting - int nrecv = pp_post_recv(ctx, max_batch_len); - if (nrecv < max_batch_len) { - fprintf(stderr, "Couldn't post receive (%d)\n", nrecv); - return 1; - } - - if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { - fprintf(stderr, "Couldn't get port info\n"); - return 1; - } - my_dest.lid = ctx->portinfo.lid; - - my_dest.qpn = ctx->qp->qp_num; - my_dest.psn = lrand48() & 0xffffff; - - if (gidx >= 0) { - if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { - fprintf(stderr, "Could not get local gid for gid index " - "%d\n", gidx); - return 1; - } - } else - memset(&my_dest.gid, 0, sizeof my_dest.gid); - - inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", - my_dest.lid, my_dest.qpn, my_dest.psn, gid); - + ctx = pp_init_ctx(ib_dev, size, calc_size, rx_depth, ib_port, 0, gpu_id, peersync, peersync_gpu_cq, peersync_gpu_dbrec, consume_rx_cqe, sched_mode, use_gpumem, use_desc_apis, skip_kernel_launch); + if (!ctx) + return 1; + + //pre-posting + int nrecv = pp_post_recv(ctx, max_batch_len); + if (nrecv < max_batch_len) { + fprintf(stderr, "Couldn't post receive (%d)\n", nrecv); + return 1; + } + + if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { + fprintf(stderr, "Couldn't get port info\n"); + return 1; + } + my_dest.lid = ctx->portinfo.lid; + + my_dest.qpn = ctx->qp->qp_num; + my_dest.psn = lrand48() & 0xffffff; + + if (gidx >= 0) { + if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { + fprintf(stderr, "Could not get local gid for gid index " + "%d\n", gidx); + return 1; + } + } else + memset(&my_dest.gid, 0, sizeof my_dest.gid); + + inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); + printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", + my_dest.lid, my_dest.qpn, my_dest.psn, gid); + rem_dest = pp_client_exch_dest(servername, port, &my_dest); - if (!rem_dest) { + if (!rem_dest) { fprintf(stderr, "Could not exchange destination\n"); - ret = 1; + ret = 1; goto out; } - inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", - rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); + inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); + printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", + rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); - if (servername) { - if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx)) - return 1; + if (servername) { + if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx)) + return 1; //sleep(1); } @@ -1105,21 +1082,17 @@ int main(int argc, char *argv[]) block_server_stream(ctx); } - if (gettimeofday(&start, NULL)) { - perror("gettimeofday"); - ret = 1; + if (gettimeofday(&start, NULL)) { + perror("gettimeofday"); + ret = 1; goto out; - } - - //printf("sleeping 10s\n"); - //sleep(10); + } // for performance reasons, multiple batches back-to-back are posted here - rcnt = scnt = 0; + rcnt = scnt = 0; nposted = 0; routs = 0; const int n_batches = 3; - //int prev_batch_len = 0; int last_batch_len = 0; int n_post = 0; int n_posted; @@ -1147,26 +1120,25 @@ int main(int argc, char *argv[]) } routs += n_posted; nposted += n_posted; - //prev_batch_len = last_batch_len; last_batch_len = n_posted; printf("[%d] batch %d: posted %d sequences\n", my_rank, batch, n_posted); } - ctx->pending = PINGPONG_RECV_WRID; + ctx->pending = PINGPONG_RECV_WRID; float pre_post_us = 0; - if (gettimeofday(&end, NULL)) { - perror("gettimeofday"); - ret = 1; + if (gettimeofday(&end, NULL)) { + perror("gettimeofday"); + ret = 1; goto out; - } - { - float usec = (end.tv_sec - start.tv_sec) * 1000000 + - (end.tv_usec - start.tv_usec); - printf("pre-posting took %.2f usec\n", usec); + } + { + float usec = (end.tv_sec - start.tv_sec) * 1000000 + + (end.tv_usec - start.tv_usec); + printf("pre-posting took %.2f usec\n", usec); pre_post_us = usec; - } + } if (hide_cpu_launch_latency) { printf("ignoring pre-posting time and unblocking the stream\n"); @@ -1187,19 +1159,18 @@ int main(int argc, char *argv[]) fflush(stdout); } - if (gettimeofday(&start, NULL)) { - perror("gettimeofday"); - return 1; - } + if (gettimeofday(&start, NULL)) { + perror("gettimeofday"); + return 1; + } prof_enable(&prof); prof_idx = 0; int got_error = 0; int iter = 0; - while ((rcnt < iters && scnt < iters) && !got_error && !stream_cb_error) { + while ((rcnt < iters && scnt < iters) && !got_error && !stream_cb_error) { ++iter; PROF(&prof, prof_idx++); - //printf("before tracking\n"); fflush(stdout); int ret = gpu_wait_tracking_event(1000*1000); if (ret == ENOMEM) { gpu_dbg("gpu_wait_tracking_event nothing to do (%d)\n", ret); @@ -1211,7 +1182,6 @@ int main(int argc, char *argv[]) gpu_err("gpu_wait_tracking_event failed (%d)\n", ret); got_error = ret; } - //gpu_infoc(20, "after tracking\n"); PROF(&prof, prof_idx++); @@ -1221,29 +1191,28 @@ int main(int argc, char *argv[]) struct ibv_wc wc[max_batch_len]; int ne = 0, i; - ne = ibv_poll_cq(ctx->rx_cq, max_batch_len, wc); + ne = gds_poll_cq(ctx->gds_qp->recv_cq, max_batch_len, wc); if (ne < 0) { fprintf(stderr, "poll RX CQ failed %d\n", ne); return 1; } n_rx_ev += ne; - //if (ne) printf("ne=%d\n", ne); for (i = 0; i < ne; ++i) { if (wc[i].status != IBV_WC_SUCCESS) { fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", - ibv_wc_status_str(wc[i].status), - wc[i].status, (int) wc[i].wr_id); + ibv_wc_status_str(wc[i].status), + wc[i].status, (int) wc[i].wr_id); return 1; } switch ((int) wc[i].wr_id) { - case PINGPONG_RECV_WRID: - ++rcnt; - break; - default: - fprintf(stderr, "Completion for unknown wr_id %d\n", - (int) wc[i].wr_id); - return 1; + case PINGPONG_RECV_WRID: + ++rcnt; + break; + default: + fprintf(stderr, "Completion for unknown wr_id %d\n", + (int) wc[i].wr_id); + return 1; } } } else { @@ -1257,7 +1226,7 @@ int main(int argc, char *argv[]) struct ibv_wc wc[max_batch_len]; int ne, i; - ne = ibv_poll_cq(ctx->tx_cq, max_batch_len, wc); + ne = gds_poll_cq(ctx->gds_qp->send_cq, max_batch_len, wc); if (ne < 0) { fprintf(stderr, "poll TX CQ failed %d\n", ne); return 1; @@ -1266,20 +1235,20 @@ int main(int argc, char *argv[]) for (i = 0; i < ne; ++i) { if (wc[i].status != IBV_WC_SUCCESS) { fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", - ibv_wc_status_str(wc[i].status), - wc[i].status, (int) wc[i].wr_id); + ibv_wc_status_str(wc[i].status), + wc[i].status, (int) wc[i].wr_id); return 1; } switch ((int) wc[i].wr_id) { - case PINGPONG_SEND_WRID: - ++scnt; - break; - default: - fprintf(stderr, "Completion for unknown wr_id %d\n", - (int) wc[i].wr_id); - ret = 1; - goto out; + case PINGPONG_SEND_WRID: + ++scnt; + break; + default: + fprintf(stderr, "Completion for unknown wr_id %d\n", + (int) wc[i].wr_id); + ret = 1; + goto out; } } } @@ -1291,13 +1260,11 @@ int main(int argc, char *argv[]) if (n_tx_ev || n_rx_ev) { // update counters routs -= last_batch_len; - //prev_batch_len = last_batch_len; if (n_tx_ev != last_batch_len) gpu_dbg("[%d] partially completed batch, got tx ev %d, batch len %d\n", iter, n_tx_ev, last_batch_len); if (n_rx_ev != last_batch_len) gpu_dbg("[%d] partially completed batch, got rx ev %d, batch len %d\n", iter, n_rx_ev, last_batch_len); if (nposted < iters) { - //fprintf(stdout, "rcnt=%d scnt=%d routs=%d nposted=%d\n", rcnt, scnt, routs, nposted); fflush(stdout); // potentially submit new work n_post = min(min(ctx->rx_depth/2, iters-nposted), max_batch_len); int n = pp_post_work(ctx, n_post, nposted, rem_dest->qpn, servername?1:0); @@ -1308,51 +1275,44 @@ int main(int argc, char *argv[]) last_batch_len = n; routs += n; nposted += n; - //fprintf(stdout, "n_post=%d n=%d\n", n_post, n); } } else { PROF(&prof, prof_idx++); PROF(&prof, prof_idx++); } - //usleep(10); PROF(&prof, prof_idx++); - prof_update(&prof); - prof_idx = 0; - - //fprintf(stdout, "%d %d\n", rcnt, scnt); fflush(stdout); + prof_update(&prof); + prof_idx = 0; if (got_error || stream_cb_error) { - //fprintf(stderr, "sleeping 10s then exiting for error\n"); - //sleep(10); gpu_err("[%d] exiting due to error(s)\n", my_rank); return 1; } if (wait_key >= 0) { - int c; if (iter == wait_key) { puts("press any key"); - c = getchar(); + getchar(); } } - } - - if (gettimeofday(&end, NULL)) { - perror("gettimeofday"); - ret = 1; - } + } - { - float usec = (end.tv_sec - start.tv_sec) * 1000000 + - (end.tv_usec - start.tv_usec) + pre_post_us; - long long bytes = (long long) size * iters * 2; + if (gettimeofday(&end, NULL)) { + perror("gettimeofday"); + ret = 1; + } - printf("[%d] %lld bytes in %.2f seconds = %.2f Mbit/sec\n", - my_rank, bytes, usec / 1000000., bytes * 8. / usec); - printf("[%d] %d iters in %.2f seconds = %.2f usec/iter\n", - my_rank, iters, usec / 1000000., usec / iters); - } + { + float usec = (end.tv_sec - start.tv_sec) * 1000000 + + (end.tv_usec - start.tv_usec) + pre_post_us; + long long bytes = (long long) size * iters * 2; + + printf("[%d] %lld bytes in %.2f seconds = %.2f Mbit/sec\n", + my_rank, bytes, usec / 1000000., bytes * 8. / usec); + printf("[%d] %d iters in %.2f seconds = %.2f usec/iter\n", + my_rank, iters, usec / 1000000., usec / iters); + } if (prof_enabled(&prof)) { printf("dumping prof\n"); @@ -1360,21 +1320,18 @@ int main(int argc, char *argv[]) } prof_destroy(&prof); - //ibv_ack_cq_events(ctx->cq, num_cq_events); - - - return 0; + return 0; out: - if (pp_close_ctx(ctx)) - ret = 1; + if (pp_close_ctx(ctx)) + ret = 1; - ibv_free_device_list(dev_list); - free(rem_dest); + ibv_free_device_list(dev_list); + free(rem_dest); - return ret; + return ret; } /* diff --git a/tests/gds_poll_lat.c b/tests/gds_poll_lat.c index cf2147b..4745d99 100644 --- a/tests/gds_poll_lat.c +++ b/tests/gds_poll_lat.c @@ -11,7 +11,7 @@ #include #include -#include +//#include #include #include #include @@ -25,12 +25,12 @@ int prof_idx = 0; int main(int argc, char *argv[]) { int ret = 0; - int gpu_id = 0; + int gpu_id = 0; int num_iters = 1000; // this seems to minimize polling time int sleep_us = 10; - size_t page_size = sysconf(_SC_PAGESIZE); - size_t size = 1024*64; + size_t page_size = sysconf(_SC_PAGESIZE); + size_t size = 1024*64; int use_gpu_buf = 0; int use_flush = 0; int use_combined = 0; @@ -50,52 +50,52 @@ int main(int argc, char *argv[]) break; switch(c) { - case 'd': - gpu_id = strtol(optarg, NULL, 0); - break; - case 'W': - wait_key = strtol(optarg, NULL, 0); - break; - case 'p': - n_bg_streams = strtol(optarg, NULL, 0); - break; - case 'c': - // merge poll and multiple pokes - use_combined = 1; - break; - case 'P': - // multiple pokes - n_pokes = strtol(optarg, NULL, 0); - break; - case 'm': - use_membar = 1; - break; - case 'n': - num_iters = strtol(optarg, NULL, 0); - break; - case 's': - sleep_us = strtol(optarg, NULL, 0); - break; - case 'f': - use_flush = 1; - gpu_info("enabling flush\n"); - break; - case 'g': - use_gpu_buf = 1; - gpu_info("polling on GPU buffer\n"); - break; - case 'w': - use_wrmem = 1; - gpu_info("enabling use of WRITE_MEMORY\n"); - break; - case '?': - case 'h': - printf(" %s [-n ][-s ][-p # bg streams][-P # pokes][ckhfgomW]\n", argv[0]); - exit(EXIT_SUCCESS); - break; - default: - gpu_err("invalid option '%c'\n", c); - exit(EXIT_FAILURE); + case 'd': + gpu_id = strtol(optarg, NULL, 0); + break; + case 'W': + wait_key = strtol(optarg, NULL, 0); + break; + case 'p': + n_bg_streams = strtol(optarg, NULL, 0); + break; + case 'c': + // merge poll and multiple pokes + use_combined = 1; + break; + case 'P': + // multiple pokes + n_pokes = strtol(optarg, NULL, 0); + break; + case 'm': + use_membar = 1; + break; + case 'n': + num_iters = strtol(optarg, NULL, 0); + break; + case 's': + sleep_us = strtol(optarg, NULL, 0); + break; + case 'f': + use_flush = 1; + gpu_info("enabling flush\n"); + break; + case 'g': + use_gpu_buf = 1; + gpu_info("polling on GPU buffer\n"); + break; + case 'w': + use_wrmem = 1; + gpu_info("enabling use of WRITE_MEMORY\n"); + break; + case '?': + case 'h': + printf(" %s [-n ][-s ][-p # bg streams][-P # pokes][ckhfgomW]\n", argv[0]); + exit(EXIT_SUCCESS); + break; + default: + gpu_err("invalid option '%c'\n", c); + exit(EXIT_FAILURE); } } @@ -103,25 +103,25 @@ int main(int argc, char *argv[]) gpu_err("n_pokes must be 1 at least\n"); exit(EXIT_FAILURE); } - + CUstream bg_streams[n_bg_streams]; memset(bg_streams, 0, sizeof(bg_streams)); //if (use_combined && use_pokes) { // fprintf(stderr, "error, incompatible switches\n"); - // exit(EXIT_FAILURE); + // exit(EXIT_FAILURE); //} const char *tags = "postpoll|que poke| sleep| set dw|pollpoke|str sync"; if ( /*prof_init(&prof, 1000, 1000, "1ms", 50, 1, tags)*/ - prof_init(&prof, 100, 100, "100ns", 25*4*2, 5, tags)) { + prof_init(&prof, 100, 100, "100ns", 25*4*2, 5, tags)) { gpu_err("error in prof_init init.\n"); - exit(EXIT_FAILURE); - } + exit(EXIT_FAILURE); + } - if (gpu_init(gpu_id, CU_CTX_SCHED_AUTO)) { - gpu_err("error in GPU init.\n"); - exit(EXIT_FAILURE); - } + if (gpu_init(gpu_id, CU_CTX_SCHED_AUTO)) { + gpu_err("error in GPU init.\n"); + exit(EXIT_FAILURE); + } //CUCHECK(cuStreamCreate(&gpu_stream, 0)); @@ -185,20 +185,19 @@ int main(int argc, char *argv[]) perf_start(); gds_us_t delta_t = 0; int warmup = 5; - for (i = 0, value = 1; i < num_iters; ++i, ++value) { + for (i = 0, value = 1; i < num_iters; ++i, ++value) { ASSERT(value <= INT_MAX); uint32_t *h_ptr = (uint32_t*)h_buf + (i % (size/sizeof(uint32_t))); uint32_t *d_ptr = (uint32_t*)d_buf + (i % (size/sizeof(uint32_t))); gpu_dbg("GEQ h_ptr=%p d_ptr=%p *h_ptr=%08x i=%d value=%d\n", h_ptr, d_ptr, *h_ptr, i, value); if (!use_gpu_buf) { - int c; if (wait_key>=0 && i==wait_key) { puts("press any key"); - c = getchar(); + getchar(); } } - PROF(&prof, prof_idx++); + PROF(&prof, prof_idx++); assert(n_pokes < size/sizeof(uint32_t)); gds_descriptor_t descs[n_pokes+1]; @@ -220,18 +219,18 @@ int main(int argc, char *argv[]) gpu_dbg("poke %d WRITE_VALUE32\n", k); descs[1+k].tag = GDS_TAG_WRITE_VALUE32; GDSCHECK(gds_prepare_write_value32(&descs[1+k].write32, - ptr, - 0xd4d00000|(j<<8)|k, - dflags)); + ptr, + 0xd4d00000|(j<<8)|k, + dflags)); } else { gpu_dbg("poke %d WRITE_MEMORY\n", k); descs[1+k].tag = GDS_TAG_WRITE_MEMORY; uint32_t word = 0xd4d00000|(j<<8)|k; GDSCHECK(gds_prepare_write_memory(&descs[1+k].writemem, - (uint8_t*)ptr, - (uint8_t*)&word, - sizeof(word), - dflags)); + (uint8_t*)ptr, + (uint8_t*)&word, + sizeof(word), + dflags)); } poke_hptrs[k] = h_data + off; @@ -249,13 +248,12 @@ int main(int argc, char *argv[]) PROF(&prof, prof_idx++); GDSCHECK(gds_stream_post_descriptors(gpu_stream, n_pokes, descs+1, 0)); } - PROF(&prof, prof_idx++); + PROF(&prof, prof_idx++); if (use_gpu_buf) { - int c; if (wait_key>=0 && i==wait_key) { puts("press any key"); - c = getchar(); + getchar(); } } // CPU waits some time here to make sure the previous commands @@ -267,12 +265,12 @@ int main(int argc, char *argv[]) gpu_err("error, stream must NOT be idle at this point, iter:%d\n", i); exit(EXIT_FAILURE); } - PROF(&prof, prof_idx++); + PROF(&prof, prof_idx++); // CPU writes to SYS/VIDMEM, triggering the GPU acquire, // which should trigger execution past the sema acquire gpu_dbg("writing h_ptr=%p value=%08x\n", h_ptr, value); - gds_atomic_set_dword(h_ptr, value); - PROF(&prof, prof_idx++); + gds_atomic_set_dword(h_ptr, value); + PROF(&prof, prof_idx++); // CPU polling on zero-copy SYSMEM //if (use_pokes || use_combined) // ret = gpu_poll_pokes(); @@ -285,18 +283,18 @@ int main(int argc, char *argv[]) gpu_fail("error while polling on %zu poke\n", n_pokes-1); } gds_us_t end = gds_get_time_us(); - PROF(&prof, prof_idx++); + PROF(&prof, prof_idx++); // CUDA synchronize - //gpu_wait_kernel(); + //gpu_wait_kernel(); CUCHECK(cuStreamSynchronize(gpu_stream)); - PROF(&prof, prof_idx++); - prof_update(&prof); - prof_idx = 0; - + PROF(&prof, prof_idx++); + prof_update(&prof); + prof_idx = 0; + if (i > warmup) { delta_t += end - start; } - } + } gpu_info("test finished!\n"); if (num_iters > warmup) { @@ -306,7 +304,8 @@ int main(int argc, char *argv[]) perf_stop(); prof_dump(&prof); -err: + + //err: if (n_bg_streams) { gpu_info("signaling %d background polling stream(s)\n", n_bg_streams); int s; @@ -328,7 +327,7 @@ int main(int argc, char *argv[]) GDSCHECK(ret = gds_free_mapped_memory(&desc_data)); out: - gpu_finalize(); + gpu_finalize(); return ret; } diff --git a/tests/gds_sanity.cpp b/tests/gds_sanity.cpp index 910032b..74bf829 100644 --- a/tests/gds_sanity.cpp +++ b/tests/gds_sanity.cpp @@ -14,7 +14,7 @@ #include #include -#include +//#include #include #include #include @@ -68,50 +68,50 @@ int main(int argc, char *argv[]) break; switch(c) { - case 'd': - gpu_id = strtol(optarg, NULL, 0); - break; - case 'm': - use_membar = !use_membar; - break; - case 'n': - num_iters = strtol(optarg, NULL, 0); - break; - case 'f': - use_flush = 1; - printf("INFO enabling flush\n"); - break; - case 'g': - use_gpu_buf = 1; - printf("INFO polling on GPU buffer\n"); - break; - case 'N': - use_nor = 1; - printf("INFO polling using NOR\n"); - break; - case 'h': - printf("Usage:\n" - " %s [options]\n" - "Options:\n" - " -d id use gpu ordinal id\n" - " -n n iterate n times\n" - " -f issue a GPU RDMA flush following each poll\n" - " -g allocate all memory on GPU\n" - " -m issue memory barrier between signal and data stores\n" - " -N poll memory using NOR condition (requires Volta)\n" - " -h this help\n", argv[0]); - exit(EXIT_SUCCESS); - break; - default: - printf("ERROR: invalid option\n"); - exit(EXIT_FAILURE); + case 'd': + gpu_id = strtol(optarg, NULL, 0); + break; + case 'm': + use_membar = !use_membar; + break; + case 'n': + num_iters = strtol(optarg, NULL, 0); + break; + case 'f': + use_flush = 1; + printf("INFO enabling flush\n"); + break; + case 'g': + use_gpu_buf = 1; + printf("INFO polling on GPU buffer\n"); + break; + case 'N': + use_nor = 1; + printf("INFO polling using NOR\n"); + break; + case 'h': + printf("Usage:\n" + " %s [options]\n" + "Options:\n" + " -d id use gpu ordinal id\n" + " -n n iterate n times\n" + " -f issue a GPU RDMA flush following each poll\n" + " -g allocate all memory on GPU\n" + " -m issue memory barrier between signal and data stores\n" + " -N poll memory using NOR condition (requires Volta)\n" + " -h this help\n", argv[0]); + exit(EXIT_SUCCESS); + break; + default: + printf("ERROR: invalid option\n"); + exit(EXIT_FAILURE); } } - if (gpu_init(gpu_id, CU_CTX_SCHED_AUTO)) { - fprintf(stderr, "error in GPU init.\n"); - exit(EXIT_FAILURE); - } + if (gpu_init(gpu_id, CU_CTX_SCHED_AUTO)) { + fprintf(stderr, "error in GPU init.\n"); + exit(EXIT_FAILURE); + } puts(""); printf("number iterations %d\n", num_iters); @@ -121,7 +121,7 @@ int main(int argc, char *argv[]) printf("poll on %s buffer\n", use_gpu_buf?"GPU":"CPU"); printf("write on %s buffer\n", use_gpu_buf?"GPU":"CPU"); puts(""); - + int mem_type = use_gpu_buf ? GDS_MEMORY_GPU : GDS_MEMORY_HOST; @@ -268,7 +268,7 @@ int main(int argc, char *argv[]) ASSERT(gds_atomic_read_dword(h_signal) == (value-1)); } ASSERT(gds_atomic_read_dword(h_done) == (value-1)); - + gpu_dbg("%d: dbg @%p:%08x\n", i, h_dbg, gds_atomic_read_dword(h_dbg)); gpu_dbg("%d: sig @%p:%08x\n", i, h_signal, gds_atomic_read_dword(h_signal)); gpu_dbg("%d: done @%p:%08x\n", i, h_done, gds_atomic_read_dword(h_done)); @@ -339,7 +339,7 @@ int main(int argc, char *argv[]) gpu_fail("error (%d) while freeing mem\n", ret); } - gpu_finalize(); + gpu_finalize(); printf(">>> SUCCESS\n"); return ret; diff --git a/tests/gpu.cpp b/tests/gpu.cpp index 6d7da67..29b022c 100644 --- a/tests/gpu.cpp +++ b/tests/gpu.cpp @@ -29,7 +29,7 @@ #include #include -#include +//#include #include "gdrapi.h" #include "gdsync.h" @@ -72,52 +72,52 @@ int gpu_launch_void_kernel_on_stream(CUstream s); int gpu_init(int gpu_id, int sched_mode) { - int ret = 0; - - CUCHECK(cuInit(0)); - - int deviceCount = 0; - CUCHECK(cuDeviceGetCount(&deviceCount)); - - // This function call returns 0 if there are no CUDA capable devices. - if (deviceCount == 0) { - gpu_err("There are no available device(s) that support CUDA\n"); - ret = 1; - goto out; - } else { - gpu_dbg("There are %d devices supporting CUDA, picking N.%d\n", deviceCount, gpu_id); + int ret = 0; + + CUCHECK(cuInit(0)); + + int deviceCount = 0; + CUCHECK(cuDeviceGetCount(&deviceCount)); + + // This function call returns 0 if there are no CUDA capable devices. + if (deviceCount == 0) { + gpu_err("There are no available device(s) that support CUDA\n"); + ret = 1; + goto out; + } else { + gpu_dbg("There are %d devices supporting CUDA, picking N.%d\n", deviceCount, gpu_id); } if (getenv("USE_GPU")) { gpu_id = atoi(getenv("USE_GPU")); gpu_info("overriding gpu_id with USE_GPU=%d\n", gpu_id); } - if (gpu_id >= deviceCount) { - gpu_err("ERROR: requested GPU gpu_id beyond available\n"); - ret = 1; - goto out; - } + if (gpu_id >= deviceCount) { + gpu_err("ERROR: requested GPU gpu_id beyond available\n"); + ret = 1; + goto out; + } gpu_blocking_sync_mode = (CU_CTX_SCHED_BLOCKING_SYNC == sched_mode) ? 1 : 0; - int i; - for (i=0; i #include -#include +//#include #ifdef USE_PROFILE @@ -42,19 +43,19 @@ #ifdef USE_NVTX #include "nvToolsExt.h" -#define NVTX_PUSH(name,cid) do { \ - uint32_t colors[] = { 0x0000ff00, 0x000000ff, 0x00ffff00, 0x00ff00ff, 0x0000ffff, 0x00ff0000, 0x00ffffff }; \ - int num_colors = sizeof(colors)/sizeof(uint32_t); \ - int color_id = cid; \ - color_id = color_id%num_colors;\ - nvtxEventAttributes_t eventAttrib = {0}; \ - eventAttrib.version = NVTX_VERSION; \ - eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; \ - eventAttrib.colorType = NVTX_COLOR_ARGB; \ - eventAttrib.color = colors[color_id]; \ - eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; \ - eventAttrib.message.ascii = name; \ - nvtxRangePushEx(&eventAttrib); \ +#define NVTX_PUSH(name,cid) do { \ + uint32_t colors[] = { 0x0000ff00, 0x000000ff, 0x00ffff00, 0x00ff00ff, 0x0000ffff, 0x00ff0000, 0x00ffffff }; \ + int num_colors = sizeof(colors)/sizeof(uint32_t); \ + int color_id = cid; \ + color_id = color_id%num_colors; \ + nvtxEventAttributes_t eventAttrib = {0}; \ + eventAttrib.version = NVTX_VERSION; \ + eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; \ + eventAttrib.colorType = NVTX_COLOR_ARGB; \ + eventAttrib.color = colors[color_id]; \ + eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; \ + eventAttrib.message.ascii = name; \ + nvtxRangePushEx(&eventAttrib); \ } while(0) #define NVTX_POP() do { nvtxRangePop(); } while(0) #else @@ -65,10 +66,10 @@ //---- enum gpu_msg_level { - GPU_MSG_DEBUG = 1, - GPU_MSG_INFO, - GPU_MSG_WARN, - GPU_MSG_ERROR + GPU_MSG_DEBUG = 1, + GPU_MSG_INFO, + GPU_MSG_WARN, + GPU_MSG_ERROR }; @@ -92,9 +93,9 @@ enum gpu_msg_level { #define gpu_err(FMT, ARGS...) gpu_msg(GPU_MSG_ERROR, "ERR: ", FMT, ##ARGS) #define gpu_fail_2(FMT, ARGS... ) do { \ - gpu_err(">>> test FAILED in %s at %s:%d\n" FMT, __FUNCTION__, __FILE__, __LINE__, ## ARGS); \ - exit(EXIT_FAILURE); \ - } while(0) + gpu_err(">>> test FAILED in %s at %s:%d\n" FMT, __FUNCTION__, __FILE__, __LINE__, ## ARGS); \ + exit(EXIT_FAILURE); \ +} while(0) #define gpu_fail(FMT, ARGS...) gpu_fail_2(FMT, ##ARGS) //--- @@ -110,12 +111,12 @@ enum gpu_msg_level { #define __GDSCHECK(stmt, cond_str) \ - do { \ - int result = (stmt); \ - if (0 != result) { \ - const char *err_str = strerror(result); \ - gpu_fail("Assertion \"%s returned %s\" failed\n", cond_str, err_str); \ - } \ + do { \ + int result = (stmt); \ + if (0 != result) { \ + const char *err_str = strerror(result); \ + gpu_fail("Assertion \"%s returned %s\" failed\n", cond_str, err_str); \ + } \ } while (0) #define GDSCHECK(stmt) __GDSCHECK(stmt, #stmt) @@ -123,14 +124,14 @@ enum gpu_msg_level { //---- #define __CUCHECK(stmt, cond_str) \ - do { \ - CUresult result = (stmt); \ - if (CUDA_SUCCESS != result) { \ - const char *err_str = NULL; \ - cuGetErrorString(result, &err_str); \ - fprintf(stderr, "Assertion \"%s != cudaSuccess\" failed at %s:%d error=%d(%s)\n", cond_str, __FILE__, __LINE__, result, err_str); \ - exit(EXIT_FAILURE); \ - } \ + do { \ + CUresult result = (stmt); \ + if (CUDA_SUCCESS != result) { \ + const char *err_str = NULL; \ + cuGetErrorString(result, &err_str); \ + fprintf(stderr, "Assertion \"%s != cudaSuccess\" failed at %s:%d error=%d(%s)\n", cond_str, __FILE__, __LINE__, result, err_str); \ + exit(EXIT_FAILURE); \ + } \ } while (0) #define CUCHECK(stmt) __CUCHECK(stmt, #stmt) @@ -138,12 +139,12 @@ enum gpu_msg_level { //---- #define __CUDACHECK(stmt, cond_str) \ - do { \ - cudaError_t result = (stmt); \ - if (cudaSuccess != result) { \ - fprintf(stderr, "Assertion \"%s != cudaSuccess\" failed at %s:%d error=%d(%s)\n", cond_str, __FILE__, __LINE__, result, cudaGetErrorString(result)); \ - exit(EXIT_FAILURE); \ - } \ + do { \ + cudaError_t result = (stmt); \ + if (cudaSuccess != result) { \ + fprintf(stderr, "Assertion \"%s != cudaSuccess\" failed at %s:%d error=%d(%s)\n", cond_str, __FILE__, __LINE__, result, cudaGetErrorString(result)); \ + exit(EXIT_FAILURE); \ + } \ } while (0) #define CUDACHECK(stmt) __CUDACHECK(stmt, #stmt) @@ -156,27 +157,31 @@ extern CUstream gpu_stream_server; extern CUstream gpu_stream_client; extern int gpu_num_sm; -BEGIN_C_DECLS - -int gpu_dbg_enabled(); - -/* sched_mode=(CU_CTX_SCHED_SPIN,CU_CTX_SCHED_YIELD,CU_CTX_SCHED_BLOCKING_SYNC, CU_CTX_SCHED_AUTO) */ -int gpu_init(int gpu_id, int sched_mode); -int gpu_finalize(); -void *gpu_malloc(size_t page_size, size_t min_size); -int gpu_free(void *ptr); -int gpu_memset(void *ptr, const unsigned char c, size_t size); -int gpu_register_host_mem(void *ptr, size_t size); - -int gpu_launch_kernel(size_t size, int is_peersync); -int gpu_launch_kernel_on_stream(size_t size, int is_peersync, CUstream s); -void gpu_post_release_tracking_event(); -int gpu_wait_tracking_event(int tmout_us); - -int gpu_launch_void_kernel(); -int gpu_launch_dummy_kernel(); - -END_C_DECLS +//BEGIN_C_DECLS +#ifdef __cplusplus +extern "C" { +#endif + int gpu_dbg_enabled(); + + /* sched_mode=(CU_CTX_SCHED_SPIN,CU_CTX_SCHED_YIELD,CU_CTX_SCHED_BLOCKING_SYNC, CU_CTX_SCHED_AUTO) */ + int gpu_init(int gpu_id, int sched_mode); + int gpu_finalize(); + void *gpu_malloc(size_t page_size, size_t min_size); + int gpu_free(void *ptr); + int gpu_memset(void *ptr, const unsigned char c, size_t size); + int gpu_register_host_mem(void *ptr, size_t size); + + int gpu_launch_kernel(size_t size, int is_peersync); + int gpu_launch_kernel_on_stream(size_t size, int is_peersync, CUstream s); + void gpu_post_release_tracking_event(); + int gpu_wait_tracking_event(int tmout_us); + + int gpu_launch_void_kernel(); + int gpu_launch_dummy_kernel(); +#ifdef __cplusplus +} +#endif +//END_C_DECLS /* * Local variables: diff --git a/tests/gpu_kernels.cu b/tests/gpu_kernels.cu index 48aac75..096d1ec 100644 --- a/tests/gpu_kernels.cu +++ b/tests/gpu_kernels.cu @@ -19,7 +19,7 @@ int gpu_launch_void_kernel_on_stream(CUstream s) { const int nblocks = 1; const int nthreads = 1; - void_kernel<<>>(); + void_kernel<<>>(); CUDACHECK(cudaGetLastError()); return 0; } @@ -44,7 +44,7 @@ int gpu_launch_dummy_kernel(void) int p0 = 100; float p1 = 1.1f; float *p2 = NULL; - dummy_kernel<<>>(p0, p1, p2); + dummy_kernel<<>>(p0, p1, p2); CUDACHECK(cudaGetLastError()); return 0; } @@ -78,7 +78,7 @@ int gpu_launch_calc_kernel_on_stream(size_t size, CUstream s) // at least 1 thr block int nb = std::min(((n + nthreads - 1) / nthreads), nblocks); assert(nb >= 1); - calc_kernel<<>>(n, 1.0f, in, out); + calc_kernel<<>>(n, 1.0f, in, out); CUDACHECK(cudaGetLastError()); return 0; } diff --git a/tests/pingpong.c b/tests/pingpong.c index 90732ef..ab9a0af 100644 --- a/tests/pingpong.c +++ b/tests/pingpong.c @@ -38,50 +38,50 @@ enum ibv_mtu pp_mtu_to_enum(int mtu) { - switch (mtu) { - case 256: return IBV_MTU_256; - case 512: return IBV_MTU_512; - case 1024: return IBV_MTU_1024; - case 2048: return IBV_MTU_2048; - case 4096: return IBV_MTU_4096; - default: return -1; - } + switch (mtu) { + case 256: return IBV_MTU_256; + case 512: return IBV_MTU_512; + case 1024: return IBV_MTU_1024; + case 2048: return IBV_MTU_2048; + case 4096: return IBV_MTU_4096; + default: return -1; + } } uint16_t pp_get_local_lid(struct ibv_context *context, int port) { - struct ibv_port_attr attr; + struct ibv_port_attr attr; - if (ibv_query_port(context, port, &attr)) - return 0; + if (ibv_query_port(context, port, &attr)) + return 0; - return attr.lid; + return attr.lid; } int pp_get_port_info(struct ibv_context *context, int port, - struct ibv_port_attr *attr) + struct ibv_port_attr *attr) { - return ibv_query_port(context, port, attr); + return ibv_query_port(context, port, attr); } void wire_gid_to_gid(const char *wgid, union ibv_gid *gid) { - char tmp[9]; - uint32_t v32; - int i; + char tmp[9]; + uint32_t v32; + int i; - for (tmp[8] = 0, i = 0; i < 4; ++i) { - memcpy(tmp, wgid + i * 8, 8); - sscanf(tmp, "%x", &v32); - *(uint32_t *)(&gid->raw[i * 4]) = ntohl(v32); - } + for (tmp[8] = 0, i = 0; i < 4; ++i) { + memcpy(tmp, wgid + i * 8, 8); + sscanf(tmp, "%x", &v32); + *(uint32_t *)(&gid->raw[i * 4]) = ntohl(v32); + } } void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]) { - int i; + int i; - for (i = 0; i < 4; ++i) - sprintf(&wgid[i * 8], "%08x", - htonl(*(uint32_t *)(gid->raw + i * 4))); + for (i = 0; i < 4; ++i) + sprintf(&wgid[i * 8], "%08x", + htonl(*(uint32_t *)(gid->raw + i * 4))); } diff --git a/tests/pingpong.h b/tests/pingpong.h index 32f020b..db4f52f 100644 --- a/tests/pingpong.h +++ b/tests/pingpong.h @@ -34,7 +34,7 @@ #define IBV_PINGPONG_H #include -#include +//#include enum ibv_mtu pp_mtu_to_enum(int mtu); uint16_t pp_get_local_lid(struct ibv_context *context, int port); diff --git a/tests/rstest.cpp b/tests/rstest.cpp index 028cd75..26ee089 100644 --- a/tests/rstest.cpp +++ b/tests/rstest.cpp @@ -31,84 +31,84 @@ using namespace std; std::ostream &operator <<(std::ostream &o, std::pair &r) { - o << "[" << r.first << "," << r.second << "]"; - return o; + o << "[" << r.first << "," << r.second << "]"; + return o; } std::ostream &operator << (std::ostream &o, RangeSet &rs) { #if __cplusplus <= 199711L // not C++-11, hopefully C++-98 - for (RangeSet::iterator _s = rs.begin(); _s != rs.end(); ++_s) { Range s = *_s; + for (RangeSet::iterator _s = rs.begin(); _s != rs.end(); ++_s) { Range s = *_s; #else - for (auto s: rs) { + for (auto s: rs) { #endif - std::cout << s << std::endl; - } - return o; + std::cout << s << std::endl; + } + return o; } int main() { - RangeSet rs; - - Range r1(1,2); - Range r2(4,5); - Range r3(8,22); - - rs.insert(r1); - rs.insert(r2); - rs.insert(r3); - - cout << "rs contains:" << endl; - cout << rs; - cout << endl; - - { RangeSet::find_result res = rs.find(r1); Range r = *res.first; assert(r == r1); assert(res.second == RangeSet::fully_contained); } - - { RangeSet::find_result res = rs.find(r2); Range r = *res.first; assert(r == r2); assert(res.second == RangeSet::fully_contained); } - - { - RangeSet::find_result res = rs.find(r3); - Range r = *res.first; - if (r.second == RangeSet::not_found) { - assert(res.first == rs.end()); - cout << "cannot find range " << r3 << endl; + RangeSet rs; + + Range r1(1,2); + Range r2(4,5); + Range r3(8,22); + + rs.insert(r1); + rs.insert(r2); + rs.insert(r3); + + cout << "rs contains:" << endl; + cout << rs; + cout << endl; + + { RangeSet::find_result res = rs.find(r1); Range r = *res.first; assert(r == r1); assert(res.second == RangeSet::fully_contained); } + + { RangeSet::find_result res = rs.find(r2); Range r = *res.first; assert(r == r2); assert(res.second == RangeSet::fully_contained); } + + { + RangeSet::find_result res = rs.find(r3); + Range r = *res.first; + if (r.second == RangeSet::not_found) { + assert(res.first == rs.end()); + cout << "cannot find range " << r3 << endl; + } + else { + if (r != r3) cout << "ERROR: " << r3 << endl; + if (res.second != RangeSet::fully_contained) cout << "ERROR: result " << res.second << endl; + } } - else { - if (r != r3) cout << "ERROR: " << r3 << endl; - if (res.second != RangeSet::fully_contained) cout << "ERROR: result " << res.second << endl; + + { Range r(8,9); RangeSet::find_result res = rs.find(r); Range ro = *res.first; cout << r << " is contained in " << ro << endl; assert(res.second == RangeSet::fully_contained); } + + { Range r(7,9); RangeSet::find_result res = rs.find(r); Range ro = *res.first; cout << r << " partially overlaps with " << ro << endl; assert(res.second == RangeSet::partial_overlap); } + + { + Range r4(5,6); + cout << "range extension test" << endl; + cout << "adding " << r4 << endl; + RangeSet::insert_result res = rs.insert(r4); + cout << "res.first=" << *res.first << " res.second=" << res.second << endl; + assert(*res.first == Range(4,6)); + assert(res.second); } - } - - { Range r(8,9); RangeSet::find_result res = rs.find(r); Range ro = *res.first; cout << r << " is contained in " << ro << endl; assert(res.second == RangeSet::fully_contained); } - - { Range r(7,9); RangeSet::find_result res = rs.find(r); Range ro = *res.first; cout << r << " partially overlaps with " << ro << endl; assert(res.second == RangeSet::partial_overlap); } - - { - Range r4(5,6); - cout << "range extension test" << endl; - cout << "adding " << r4 << endl; - RangeSet::insert_result res = rs.insert(r4); - cout << "res.first=" << *res.first << " res.second=" << res.second << endl; - assert(*res.first == Range(4,6)); - assert(res.second); - } - - cout << "rs contains:" << endl; - cout << rs; - cout << endl; - - { - Range r5(5,9); - cout << "range extension test" << endl; - cout << "adding " << r5 << endl; - RangeSet::insert_result res = rs.insert(r5); - cout << "res.first=" << *res.first << " res.second=" << res.second << endl; - //assert(*res.first == Range(4,6)); - //assert(res.second); - } - - cout << "rs contains:" << endl; - cout << rs; - cout << endl; + + cout << "rs contains:" << endl; + cout << rs; + cout << endl; + + { + Range r5(5,9); + cout << "range extension test" << endl; + cout << "adding " << r5 << endl; + RangeSet::insert_result res = rs.insert(r5); + cout << "res.first=" << *res.first << " res.second=" << res.second << endl; + //assert(*res.first == Range(4,6)); + //assert(res.second); + } + + cout << "rs contains:" << endl; + cout << rs; + cout << endl; } diff --git a/tests/task_queue_test.cpp b/tests/task_queue_test.cpp index 2d30172..42a465c 100644 --- a/tests/task_queue_test.cpp +++ b/tests/task_queue_test.cpp @@ -81,7 +81,7 @@ struct slow_printer { m_last_time = now; m_store_time = false; } - + if (now > m_last_time + std::chrono::milliseconds(1)) { m_last_time = now; msg("%p slow_printer i=%d\n", this, m_i); @@ -106,7 +106,7 @@ main() { task_queue wq; int i = 0; - + slow_printer s; wq.queue(std::bind(&slow_printer::run, &s)); diff --git a/tests/test_utils.h b/tests/test_utils.h index 52e4e4f..5fdf590 100644 --- a/tests/test_utils.h +++ b/tests/test_utils.h @@ -61,7 +61,7 @@ static void gds_cpu_relax(void) static void gds_wmb(void) __attribute__((unused)) ; static void gds_wmb(void) { - asm volatile("sync") ; + asm volatile("sync") ; } #else #error "platform not supported"