From f5f54bf889825da254b2a5df859da1c471a40314 Mon Sep 17 00:00:00 2001 From: zzry <1245464216@qq.com> Date: Fri, 8 Mar 2024 15:56:09 +0800 Subject: [PATCH 08/10] libhns: Add support for lock-free QP driver inclusion category: feature bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I97WST ------------------------------------------------------------------ Drop QP locks when associated to a PAD holding a TD. Signed-off-by: Yixing Liu Signed-off-by: Junxian Huang --- providers/hns/hns_roce_u.h | 2 +- providers/hns/hns_roce_u_hw_v2.c | 26 ++++++++++------- providers/hns/hns_roce_u_verbs.c | 49 +++++++++++++++++++++++++++++--- 3 files changed, 61 insertions(+), 16 deletions(-) diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 5d3f480..5732e39 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -305,7 +305,7 @@ struct hns_roce_srq { struct hns_roce_wq { unsigned long *wrid; - pthread_spinlock_t lock; + struct hns_roce_spinlock hr_lock; unsigned int wqe_cnt; int max_post; unsigned int head; diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index dd13049..90a76e2 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -1270,7 +1270,7 @@ int hns_roce_u_v2_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, return ret; } - pthread_spin_lock(&qp->sq.lock); + hns_roce_spin_lock(&qp->sq.hr_lock); sge_info.start_idx = qp->next_sge; /* start index of extend sge */ @@ -1331,7 +1331,7 @@ out: *(qp->sdb) = qp->sq.head & 0xffff; } - pthread_spin_unlock(&qp->sq.lock); + hns_roce_spin_unlock(&qp->sq.hr_lock); if (ibvqp->state == IBV_QPS_ERR) { attr.qp_state = IBV_QPS_ERR; @@ -1420,7 +1420,7 @@ static int hns_roce_u_v2_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, return ret; } - pthread_spin_lock(&qp->rq.lock); + hns_roce_spin_lock(&qp->rq.hr_lock); max_sge = qp->rq.max_gs - qp->rq.rsv_sge; for (nreq = 0; wr; ++nreq, wr = wr->next) { @@ -1454,7 +1454,7 @@ out: hns_roce_update_rq_db(ctx, ibvqp->qp_num, qp->rq.head); } - pthread_spin_unlock(&qp->rq.lock); + hns_roce_spin_unlock(&qp->rq.hr_lock); if (ibvqp->state == IBV_QPS_ERR) { attr.qp_state = IBV_QPS_ERR; @@ -1549,8 +1549,8 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int ret; if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_ERR)) { - pthread_spin_lock(&hr_qp->sq.lock); - pthread_spin_lock(&hr_qp->rq.lock); + hns_roce_spin_lock(&hr_qp->sq.hr_lock); + hns_roce_spin_lock(&hr_qp->rq.hr_lock); flag = true; } @@ -1561,8 +1561,8 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, if (flag) { if (!ret) qp->state = IBV_QPS_ERR; - pthread_spin_unlock(&hr_qp->rq.lock); - pthread_spin_unlock(&hr_qp->sq.lock); + hns_roce_spin_unlock(&hr_qp->sq.hr_lock); + hns_roce_spin_unlock(&hr_qp->rq.hr_lock); } if (ret) @@ -1640,6 +1640,7 @@ static void hns_roce_unlock_cqs(struct ibv_qp *qp) static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp) { struct hns_roce_context *ctx = to_hr_ctx(ibqp->context); + struct hns_roce_pad *pad = to_hr_pad(ibqp->pd); struct hns_roce_qp *qp = to_hr_qp(ibqp); int ret; @@ -1666,6 +1667,9 @@ static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp) hns_roce_free_qp_buf(qp, ctx); + if (pad) + atomic_fetch_sub(&pad->pd.refcount, 1); + free(qp); return ret; @@ -2555,7 +2559,7 @@ static void wr_start(struct ibv_qp_ex *ibv_qp) return; } - pthread_spin_lock(&qp->sq.lock); + hns_roce_spin_lock(&qp->sq.hr_lock); qp->sge_info.start_idx = qp->next_sge; qp->rb_sq_head = qp->sq.head; qp->err = 0; @@ -2588,7 +2592,7 @@ static int wr_complete(struct ibv_qp_ex *ibv_qp) } out: - pthread_spin_unlock(&qp->sq.lock); + hns_roce_spin_unlock(&qp->sq.hr_lock); if (ibv_qp->qp_base.state == IBV_QPS_ERR) { attr.qp_state = IBV_QPS_ERR; hns_roce_u_v2_modify_qp(&ibv_qp->qp_base, &attr, IBV_QP_STATE); @@ -2603,7 +2607,7 @@ static void wr_abort(struct ibv_qp_ex *ibv_qp) qp->sq.head = qp->rb_sq_head; - pthread_spin_unlock(&qp->sq.lock); + hns_roce_spin_unlock(&qp->sq.hr_lock); } enum { diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index ecf8666..d503031 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -1033,6 +1033,41 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx, return verify_qp_create_cap(ctx, attr); } +static int hns_roce_qp_spinlock_init(struct hns_roce_context *ctx, + struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp) +{ + bool sq_need_lock; + bool rq_need_lock; + int ret; + + sq_need_lock = hns_roce_whether_need_lock(attr->pd); + if (!sq_need_lock) + verbs_info(&ctx->ibv_ctx, "configure sq as no lock.\n"); + + rq_need_lock = hns_roce_whether_need_lock(attr->pd); + if (!rq_need_lock) + verbs_info(&ctx->ibv_ctx, "configure rq as no lock.\n"); + + ret = hns_roce_spinlock_init(&qp->sq.hr_lock, sq_need_lock); + if (ret) + return ret; + + ret = hns_roce_spinlock_init(&qp->rq.hr_lock, rq_need_lock); + if (ret) { + hns_roce_spinlock_destroy(&qp->sq.hr_lock); + return ret; + } + + return 0; +} + +void hns_roce_qp_spinlock_destroy(struct hns_roce_qp *qp) +{ + hns_roce_spinlock_destroy(&qp->rq.hr_lock); + hns_roce_spinlock_destroy(&qp->sq.hr_lock); +} + static int alloc_recv_rinl_buf(uint32_t max_sge, struct hns_roce_rinl_buf *rinl_buf) { @@ -1435,10 +1470,6 @@ static int hns_roce_alloc_qp_buf(struct ibv_qp_init_attr_ex *attr, { int ret; - if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE) || - pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE)) - return -ENOMEM; - ret = qp_alloc_wqe(&attr->cap, qp, ctx); if (ret) return ret; @@ -1466,6 +1497,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, struct hnsdv_qp_init_attr *hns_attr) { struct hns_roce_context *context = to_hr_ctx(ibv_ctx); + struct hns_roce_pad *pad = to_hr_pad(attr->pd); struct hns_roce_qp *qp; uint64_t dwqe_mmap_key; int ret; @@ -1482,6 +1514,13 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, hns_roce_set_qp_params(attr, qp, context); + if (pad) + atomic_fetch_add(&pad->pd.refcount, 1); + + ret = hns_roce_qp_spinlock_init(context, attr, qp); + if (ret) + goto err_spinlock; + ret = hns_roce_alloc_qp_buf(attr, qp, context); if (ret) goto err_buf; @@ -1515,6 +1554,8 @@ err_ops: err_cmd: hns_roce_free_qp_buf(qp, context); err_buf: + hns_roce_qp_spinlock_destroy(qp); +err_spinlock: free(qp); err: if (ret < 0) -- 2.33.0