From 9e5f5d39757a5479a1a4e1170978d2e09acb995b Mon Sep 17 00:00:00 2001 From: Luoyouming Date: Fri, 9 Sep 2022 17:42:38 +0800 Subject: [PATCH v4 07/10] libhns: Refactor rq inline The ibv_sge struct is enough, there is no need to customize the hns_roce_rinl_sge struct. Refactored structures and functions for reuse in cqe inline (rq, srq scenarios). Signed-off-by: Luoyouming --- providers/hns/hns_roce_u.h | 7 +-- providers/hns/hns_roce_u_hw_v2.c | 103 +++++++++++++++---------------- providers/hns/hns_roce_u_verbs.c | 46 +++++++------- 3 files changed, 75 insertions(+), 81 deletions(-) diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 5388f9c..57ebe55 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -290,13 +290,8 @@ struct hns_roce_sge_ex { unsigned int sge_shift; }; -struct hns_roce_rinl_sge { - void *addr; - unsigned int len; -}; - struct hns_roce_rinl_wqe { - struct hns_roce_rinl_sge *sg_list; + struct ibv_sge *sg_list; unsigned int sge_cnt; }; diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index ebe68bc..73acc9e 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "hns_roce_u.h" #include "hns_roce_u_db.h" #include "hns_roce_u_hw_v2.h" @@ -417,46 +418,42 @@ static void get_opcode_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, wc->opcode = wc_rcv_op_map[opcode]; } -static int handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, - struct hns_roce_qp **cur_qp, uint32_t opcode) +static void handle_recv_inl_data(struct hns_roce_v2_cqe *cqe, + struct hns_roce_rinl_buf *rinl_buf, + uint32_t wr_cnt, uint8_t *buf) { - if (((*cur_qp)->verbs_qp.qp.qp_type == IBV_QPT_RC) && - (opcode == HNS_ROCE_RECV_OP_SEND || - opcode == HNS_ROCE_RECV_OP_SEND_WITH_IMM || - opcode == HNS_ROCE_RECV_OP_SEND_WITH_INV) && - hr_reg_read(cqe, CQE_RQ_INLINE)) { - struct hns_roce_rinl_sge *sge_list; - uint32_t wr_num, wr_cnt, sge_num, data_len; - uint8_t *wqe_buf; - uint32_t sge_cnt, size; + struct ibv_sge *sge_list; + uint32_t sge_num, data_len; + uint32_t sge_cnt, size; - wr_num = hr_reg_read(cqe, CQE_WQE_IDX); - wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1); + sge_list = rinl_buf->wqe_list[wr_cnt].sg_list; + sge_num = rinl_buf->wqe_list[wr_cnt].sge_cnt; - sge_list = (*cur_qp)->rq_rinl_buf.wqe_list[wr_cnt].sg_list; - sge_num = (*cur_qp)->rq_rinl_buf.wqe_list[wr_cnt].sge_cnt; - wqe_buf = (uint8_t *)get_recv_wqe_v2(*cur_qp, wr_cnt); + data_len = le32toh(cqe->byte_cnt); - data_len = wc->byte_len; + for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) { + size = min(sge_list[sge_cnt].length, data_len); - for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); - sge_cnt++) { - size = sge_list[sge_cnt].len < data_len ? - sge_list[sge_cnt].len : data_len; + memcpy((void *)(uintptr_t)sge_list[sge_cnt].addr, (void *)buf, size); + data_len -= size; + buf += size; + } - memcpy((void *)sge_list[sge_cnt].addr, - (void *)wqe_buf, size); - data_len -= size; - wqe_buf += size; - } + if (data_len) + hr_reg_write(cqe, CQE_STATUS, HNS_ROCE_V2_CQE_LOCAL_LENGTH_ERR); - if (data_len) { - wc->status = IBV_WC_LOC_LEN_ERR; - return V2_CQ_POLL_ERR; - } - } +} - return V2_CQ_OK; +static void handle_recv_rq_inl(struct hns_roce_v2_cqe *cqe, + struct hns_roce_qp *cur_qp) +{ + uint8_t *wqe_buf; + uint32_t wr_num; + + wr_num = hr_reg_read(cqe, CQE_WQE_IDX) & (cur_qp->rq.wqe_cnt - 1); + + wqe_buf = (uint8_t *)get_recv_wqe_v2(cur_qp, wr_num); + handle_recv_inl_data(cqe, &(cur_qp->rq_rinl_buf), wr_num, wqe_buf); } static void parse_for_ud_qp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc) @@ -479,10 +476,9 @@ static void parse_cqe_for_srq(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, } static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, - struct hns_roce_qp *hr_qp, uint8_t opcode) + struct hns_roce_qp *hr_qp) { struct hns_roce_wq *wq; - int ret; wq = &hr_qp->rq; wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; @@ -491,12 +487,8 @@ static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD) parse_for_ud_qp(cqe, wc); - ret = handle_recv_inl_wqe(cqe, wc, &hr_qp, opcode); - if (ret) { - verbs_err(verbs_get_ctx(hr_qp->verbs_qp.qp.context), - PFX "failed to handle recv inline wqe!\n"); - return ret; - } + if (hr_reg_read(cqe, CQE_RQ_INLINE)) + handle_recv_rq_inl(cqe, hr_qp); return 0; } @@ -626,7 +618,7 @@ static int parse_cqe_for_cq(struct hns_roce_context *ctx, struct hns_roce_cq *cq if (srq) parse_cqe_for_srq(cqe, wc, srq); else - parse_cqe_for_resp(cqe, wc, cur_qp, opcode); + parse_cqe_for_resp(cqe, wc, cur_qp); } return 0; @@ -1355,26 +1347,31 @@ static void fill_recv_sge_to_wqe(struct ibv_recv_wr *wr, void *wqe, } } +static void fill_recv_inl_buf(struct hns_roce_rinl_buf *rinl_buf, + unsigned int wqe_idx, struct ibv_recv_wr *wr) +{ + struct ibv_sge *sge_list; + unsigned int i; + + if (!rinl_buf->wqe_cnt) + return; + + sge_list = rinl_buf->wqe_list[wqe_idx].sg_list; + rinl_buf->wqe_list[wqe_idx].sge_cnt = (unsigned int)wr->num_sge; + for (i = 0; i < wr->num_sge; i++) + memcpy((void *)&sge_list[i], (void *)&wr->sg_list[i], + sizeof(struct ibv_sge)); +} + static void fill_rq_wqe(struct hns_roce_qp *qp, struct ibv_recv_wr *wr, unsigned int wqe_idx, unsigned int max_sge) { - struct hns_roce_rinl_sge *sge_list; - unsigned int i; void *wqe; wqe = get_recv_wqe_v2(qp, wqe_idx); fill_recv_sge_to_wqe(wr, wqe, max_sge, qp->rq.rsv_sge); - if (!qp->rq_rinl_buf.wqe_cnt) - return; - - /* QP support receive inline wqe */ - sge_list = qp->rq_rinl_buf.wqe_list[wqe_idx].sg_list; - qp->rq_rinl_buf.wqe_list[wqe_idx].sge_cnt = (unsigned int)wr->num_sge; - for (i = 0; i < wr->num_sge; i++) { - sge_list[i].addr = (void *)(uintptr_t)wr->sg_list[i].addr; - sge_list[i].len = wr->sg_list[i].length; - } + fill_recv_inl_buf(&qp->rq_rinl_buf, wqe_idx, wr); } static int hns_roce_u_v2_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 3e9a306..1d661dd 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -855,43 +855,45 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx, return verify_qp_create_cap(ctx, attr); } -static int qp_alloc_recv_inl_buf(struct ibv_qp_cap *cap, - struct hns_roce_qp *qp) +static int alloc_recv_rinl_buf(uint32_t max_sge, + struct hns_roce_rinl_buf *rinl_buf) { unsigned int cnt; int i; - cnt = qp->rq_rinl_buf.wqe_cnt; - qp->rq_rinl_buf.wqe_list = calloc(cnt, - sizeof(struct hns_roce_rinl_wqe)); - if (!qp->rq_rinl_buf.wqe_list) + cnt = rinl_buf->wqe_cnt; + rinl_buf->wqe_list = calloc(cnt, + sizeof(struct hns_roce_rinl_wqe)); + if (!rinl_buf->wqe_list) return ENOMEM; - qp->rq_rinl_buf.wqe_list[0].sg_list = calloc(cnt * cap->max_recv_sge, - sizeof(struct hns_roce_rinl_sge)); - if (!qp->rq_rinl_buf.wqe_list[0].sg_list) + rinl_buf->wqe_list[0].sg_list = calloc(cnt * max_sge, + sizeof(struct ibv_sge)); + if (!rinl_buf->wqe_list[0].sg_list) { + free(rinl_buf->wqe_list); return ENOMEM; + } for (i = 0; i < cnt; i++) { - int wqe_size = i * cap->max_recv_sge; + int wqe_size = i * max_sge; - qp->rq_rinl_buf.wqe_list[i].sg_list = - &(qp->rq_rinl_buf.wqe_list[0].sg_list[wqe_size]); + rinl_buf->wqe_list[i].sg_list = + &(rinl_buf->wqe_list[0].sg_list[wqe_size]); } return 0; } -static void qp_free_recv_inl_buf(struct hns_roce_qp *qp) +static void free_recv_rinl_buf(struct hns_roce_rinl_buf *rinl_buf) { - if (qp->rq_rinl_buf.wqe_list) { - if (qp->rq_rinl_buf.wqe_list[0].sg_list) { - free(qp->rq_rinl_buf.wqe_list[0].sg_list); - qp->rq_rinl_buf.wqe_list[0].sg_list = NULL; + if (rinl_buf->wqe_list) { + if (rinl_buf->wqe_list[0].sg_list) { + free(rinl_buf->wqe_list[0].sg_list); + rinl_buf->wqe_list[0].sg_list = NULL; } - free(qp->rq_rinl_buf.wqe_list); - qp->rq_rinl_buf.wqe_list = NULL; + free(rinl_buf->wqe_list); + rinl_buf->wqe_list = NULL; } } @@ -930,7 +932,7 @@ static int calc_qp_buff_size(struct hns_roce_device *hr_dev, static void qp_free_wqe(struct hns_roce_qp *qp) { - qp_free_recv_inl_buf(qp); + free_recv_rinl_buf(&qp->rq_rinl_buf); if (qp->sq.wqe_cnt) free(qp->sq.wrid); @@ -958,7 +960,7 @@ static int qp_alloc_wqe(struct ibv_qp_cap *cap, struct hns_roce_qp *qp, } if (qp->rq_rinl_buf.wqe_cnt) { - if (qp_alloc_recv_inl_buf(cap, qp)) + if (alloc_recv_rinl_buf(cap->max_recv_sge, &qp->rq_rinl_buf)) goto err_alloc; } @@ -968,7 +970,7 @@ static int qp_alloc_wqe(struct ibv_qp_cap *cap, struct hns_roce_qp *qp, return 0; err_alloc: - qp_free_recv_inl_buf(qp); + free_recv_rinl_buf(&qp->rq_rinl_buf); if (qp->rq.wrid) free(qp->rq.wrid); -- 2.30.0