From 71eb90581a338242a26123790e5f24df90327465 Mon Sep 17 00:00:00 2001 From: Luoyouming Date: Thu, 11 Aug 2022 20:50:54 +0800 Subject: [PATCH v4 10/10] libhns: Support cqe inline When rq or srq recv data less than or equal to 32 byte in size, roce driver support get data from cqe. Signed-off-by: Luoyouming Reviewed-by: Yangyang Li --- providers/hns/hns_roce_u.c | 3 ++- providers/hns/hns_roce_u.h | 21 +++++++++-------- providers/hns/hns_roce_u_hw_v2.c | 39 ++++++++++++++++++++++++++++++-- providers/hns/hns_roce_u_hw_v2.h | 4 ++-- providers/hns/hns_roce_u_verbs.c | 25 ++++++++++++++++++-- 5 files changed, 75 insertions(+), 17 deletions(-) diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c index 6c9aefa..266e73e 100644 --- a/providers/hns/hns_roce_u.c +++ b/providers/hns/hns_roce_u.c @@ -113,7 +113,8 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, if (!context) return NULL; - cmd.config |= HNS_ROCE_EXSGE_FLAGS | HNS_ROCE_RQ_INLINE_FLAGS; + cmd.config |= HNS_ROCE_EXSGE_FLAGS | HNS_ROCE_RQ_INLINE_FLAGS | + HNS_ROCE_CQE_INLINE_FLAGS; if (ibv_cmd_get_context(&context->ibv_ctx, &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp))) goto err_free; diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 57ebe55..6b64cd0 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -246,10 +246,21 @@ struct hns_roce_idx_que { unsigned int tail; }; +struct hns_roce_rinl_wqe { + struct ibv_sge *sg_list; + unsigned int sge_cnt; +}; + +struct hns_roce_rinl_buf { + struct hns_roce_rinl_wqe *wqe_list; + unsigned int wqe_cnt; +}; + struct hns_roce_srq { struct verbs_srq verbs_srq; struct hns_roce_idx_que idx_que; struct hns_roce_buf wqe_buf; + struct hns_roce_rinl_buf srq_rinl_buf; pthread_spinlock_t lock; unsigned long *wrid; unsigned int srqn; @@ -290,16 +301,6 @@ struct hns_roce_sge_ex { unsigned int sge_shift; }; -struct hns_roce_rinl_wqe { - struct ibv_sge *sg_list; - unsigned int sge_cnt; -}; - -struct hns_roce_rinl_buf { - struct hns_roce_rinl_wqe *wqe_list; - unsigned int wqe_cnt; -}; - struct hns_roce_qp { struct verbs_qp verbs_qp; struct hns_roce_buf buf; diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index 25d8861..7063b26 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -444,6 +444,28 @@ static void handle_recv_inl_data(struct hns_roce_v2_cqe *cqe, } +static void handle_recv_cqe_inl_from_rq(struct hns_roce_v2_cqe *cqe, + struct hns_roce_qp *cur_qp) +{ + uint32_t wr_num; + + wr_num = hr_reg_read(cqe, CQE_WQE_IDX) & (cur_qp->rq.wqe_cnt - 1); + + handle_recv_inl_data(cqe, &(cur_qp->rq_rinl_buf), wr_num, + (uint8_t *)cqe->payload); +} + +static void handle_recv_cqe_inl_from_srq(struct hns_roce_v2_cqe *cqe, + struct hns_roce_srq *srq) +{ + uint32_t wr_num; + + wr_num = hr_reg_read(cqe, CQE_WQE_IDX) & (srq->wqe_cnt - 1); + + handle_recv_inl_data(cqe, &(srq->srq_rinl_buf), wr_num, + (uint8_t *)cqe->payload); +} + static void handle_recv_rq_inl(struct hns_roce_v2_cqe *cqe, struct hns_roce_qp *cur_qp) { @@ -473,6 +495,9 @@ static void parse_cqe_for_srq(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX); wc->wr_id = srq->wrid[wqe_idx & (srq->wqe_cnt - 1)]; hns_roce_free_srq_wqe(srq, wqe_idx); + + if (hr_reg_read(cqe, CQE_CQE_INLINE)) + handle_recv_cqe_inl_from_srq(cqe, srq); } static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, @@ -487,7 +512,9 @@ static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD) parse_for_ud_qp(cqe, wc); - if (hr_reg_read(cqe, CQE_RQ_INLINE)) + if (hr_reg_read(cqe, CQE_CQE_INLINE)) + handle_recv_cqe_inl_from_rq(cqe, hr_qp); + else if (hr_reg_read(cqe, CQE_RQ_INLINE)) handle_recv_rq_inl(cqe, hr_qp); return 0; @@ -559,6 +586,9 @@ static void cqe_proc_srq(struct hns_roce_srq *srq, uint32_t wqe_idx, { cq->verbs_cq.cq_ex.wr_id = srq->wrid[wqe_idx & (srq->wqe_cnt - 1)]; hns_roce_free_srq_wqe(srq, wqe_idx); + + if (hr_reg_read(cq->cqe, CQE_CQE_INLINE)) + handle_recv_cqe_inl_from_srq(cq->cqe, srq); } static void cqe_proc_rq(struct hns_roce_qp *hr_qp, struct hns_roce_cq *cq) @@ -568,7 +598,9 @@ static void cqe_proc_rq(struct hns_roce_qp *hr_qp, struct hns_roce_cq *cq) cq->verbs_cq.cq_ex.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; ++wq->tail; - if (hr_reg_read(cq->cqe, CQE_RQ_INLINE)) + if (hr_reg_read(cq->cqe, CQE_CQE_INLINE)) + handle_recv_cqe_inl_from_rq(cq->cqe, hr_qp); + else if (hr_reg_read(cq->cqe, CQE_RQ_INLINE)) handle_recv_rq_inl(cq->cqe, hr_qp); } @@ -1725,6 +1757,9 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, wqe = get_srq_wqe(srq, wqe_idx); fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge); + + fill_recv_inl_buf(&srq->srq_rinl_buf, wqe_idx, wr); + fill_wqe_idx(srq, wqe_idx); srq->wrid[wqe_idx] = wr->wr_id; diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h index 098dbdf..d71c695 100644 --- a/providers/hns/hns_roce_u_hw_v2.h +++ b/providers/hns/hns_roce_u_hw_v2.h @@ -157,7 +157,7 @@ struct hns_roce_v2_cqe { __le32 smac; __le32 byte_28; __le32 byte_32; - __le32 rsv[8]; + __le32 payload[8]; }; #define CQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_cqe, h, l) @@ -170,7 +170,7 @@ struct hns_roce_v2_cqe { #define CQE_WQE_IDX CQE_FIELD_LOC(31, 16) #define CQE_RKEY_IMMTDATA CQE_FIELD_LOC(63, 32) #define CQE_XRC_SRQN CQE_FIELD_LOC(87, 64) -#define CQE_RSV0 CQE_FIELD_LOC(95, 88) +#define CQE_CQE_INLINE CQE_FIELD_LOC(89, 88) #define CQE_LCL_QPN CQE_FIELD_LOC(119, 96) #define CQE_SUB_STATUS CQE_FIELD_LOC(127, 120) #define CQE_BYTE_CNT CQE_FIELD_LOC(159, 128) diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 1d661dd..cff9d1d 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -522,6 +522,8 @@ static int verify_srq_create_attr(struct hns_roce_context *context, static void set_srq_param(struct ibv_context *context, struct hns_roce_srq *srq, struct ibv_srq_init_attr_ex *attr) { + struct hns_roce_context *ctx = to_hr_ctx(context); + if (to_hr_dev(context->device)->hw_version == HNS_ROCE_HW_VER2) srq->rsv_sge = 1; @@ -531,6 +533,10 @@ static void set_srq_param(struct ibv_context *context, struct hns_roce_srq *srq, srq->max_gs)); attr->attr.max_sge = srq->max_gs; attr->attr.srq_limit = 0; + + srq->srq_rinl_buf.wqe_cnt = 0; + if (ctx->config & HNS_ROCE_RSP_CQE_INLINE_FLAGS) + srq->srq_rinl_buf.wqe_cnt = srq->wqe_cnt; } static int alloc_srq_idx_que(struct hns_roce_srq *srq) @@ -570,6 +576,11 @@ static int alloc_srq_wqe_buf(struct hns_roce_srq *srq) return hns_roce_alloc_buf(&srq->wqe_buf, buf_size, HNS_HW_PAGE_SIZE); } +static int alloc_recv_rinl_buf(uint32_t max_sge, + struct hns_roce_rinl_buf *rinl_buf); + +static void free_recv_rinl_buf(struct hns_roce_rinl_buf *rinl_buf); + static int alloc_srq_buf(struct hns_roce_srq *srq) { int ret; @@ -582,14 +593,22 @@ static int alloc_srq_buf(struct hns_roce_srq *srq) if (ret) goto err_idx_que; + if (srq->srq_rinl_buf.wqe_cnt) { + ret = alloc_recv_rinl_buf(srq->max_gs, &srq->srq_rinl_buf); + if (ret) + goto err_wqe_buf; + } + srq->wrid = calloc(srq->wqe_cnt, sizeof(*srq->wrid)); if (!srq->wrid) { ret = -ENOMEM; - goto err_wqe_buf; + goto err_inl_buf; } return 0; +err_inl_buf: + free_recv_rinl_buf(&srq->srq_rinl_buf); err_wqe_buf: hns_roce_free_buf(&srq->wqe_buf); err_idx_que: @@ -603,6 +622,7 @@ static void free_srq_buf(struct hns_roce_srq *srq) { free(srq->wrid); hns_roce_free_buf(&srq->wqe_buf); + free_recv_rinl_buf(&srq->srq_rinl_buf); hns_roce_free_buf(&srq->idx_que.buf); free(srq->idx_que.bitmap); } @@ -1082,7 +1102,8 @@ static void hns_roce_set_qp_params(struct ibv_qp_init_attr_ex *attr, qp->rq.wqe_cnt = cnt; qp->rq.shift = hr_ilog32(cnt); qp->rq_rinl_buf.wqe_cnt = 0; - if (ctx->config & HNS_ROCE_RSP_RQ_INLINE_FLAGS) + if (ctx->config & (HNS_ROCE_RSP_RQ_INLINE_FLAGS | + HNS_ROCE_RSP_CQE_INLINE_FLAGS)) qp->rq_rinl_buf.wqe_cnt = cnt; attr->cap.max_recv_wr = qp->rq.wqe_cnt; -- 2.30.0