From 812372fadc96c6c59d460d84cdc72d60014f220d Mon Sep 17 00:00:00 2001 From: Yixing Liu Date: Mon, 27 Jun 2022 14:52:04 +0800 Subject: [PATCH rdma-core] libhns: Add support for the thread domain and the parent domain Internal locks will be disabled for queues under the thread domain with the parent domain. driver inclusion category: feature bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I63NGA ------------------------------------------------------------------ Signed-off-by: Yixing Liu Reviewed-by: Yangyang Li --- providers/hns/hns_roce_u.c | 5 +- providers/hns/hns_roce_u.h | 75 ++++++++- providers/hns/hns_roce_u_hw_v2.c | 77 ++++----- providers/hns/hns_roce_u_verbs.c | 267 ++++++++++++++++++++++++++++--- 4 files changed, 357 insertions(+), 67 deletions(-) diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c index 266e73e..e3c72bb 100644 --- a/providers/hns/hns_roce_u.c +++ b/providers/hns/hns_roce_u.c @@ -67,7 +67,7 @@ static const struct verbs_context_ops hns_common_ops = { .create_qp = hns_roce_u_create_qp, .create_qp_ex = hns_roce_u_create_qp_ex, .dealloc_mw = hns_roce_u_dealloc_mw, - .dealloc_pd = hns_roce_u_free_pd, + .dealloc_pd = hns_roce_u_dealloc_pd, .dereg_mr = hns_roce_u_dereg_mr, .destroy_cq = hns_roce_u_destroy_cq, .modify_cq = hns_roce_u_modify_cq, @@ -88,6 +88,9 @@ static const struct verbs_context_ops hns_common_ops = { .close_xrcd = hns_roce_u_close_xrcd, .open_qp = hns_roce_u_open_qp, .get_srq_num = hns_roce_u_get_srq_num, + .alloc_td = hns_roce_u_alloc_td, + .dealloc_td = hns_roce_u_dealloc_td, + .alloc_parent_domain = hns_roce_u_alloc_pad, }; static uint32_t calc_table_shift(uint32_t entry_count, uint32_t size_shift) diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 8c1cb1e..8181da7 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -188,6 +188,11 @@ struct hns_roce_db_page { unsigned long *bitmap; }; +struct hns_roce_spinlock { + pthread_spinlock_t lock; + int need_lock; +}; + struct hns_roce_context { struct verbs_context ibv_ctx; void *uar; @@ -222,15 +227,27 @@ struct hns_roce_context { unsigned int max_inline_data; }; +struct hns_roce_td { + struct ibv_td ibv_td; + atomic_int refcount; +}; + struct hns_roce_pd { struct ibv_pd ibv_pd; unsigned int pdn; + atomic_int refcount; + struct hns_roce_pd *protection_domain; +}; + +struct hns_roce_pad { + struct hns_roce_pd pd; + struct hns_roce_td *td; }; struct hns_roce_cq { struct verbs_cq verbs_cq; struct hns_roce_buf buf; - pthread_spinlock_t lock; + struct hns_roce_spinlock hr_lock; unsigned int cqn; unsigned int cq_depth; unsigned int cons_index; @@ -266,7 +283,7 @@ struct hns_roce_srq { struct hns_roce_idx_que idx_que; struct hns_roce_buf wqe_buf; struct hns_roce_rinl_buf srq_rinl_buf; - pthread_spinlock_t lock; + struct hns_roce_spinlock hr_lock; unsigned long *wrid; unsigned int srqn; unsigned int wqe_cnt; @@ -279,7 +296,7 @@ struct hns_roce_srq { struct hns_roce_wq { unsigned long *wrid; - pthread_spinlock_t lock; + struct hns_roce_spinlock hr_lock; unsigned int wqe_cnt; int max_post; unsigned int head; @@ -397,9 +414,35 @@ static inline struct hns_roce_context *to_hr_ctx(struct ibv_context *ibv_ctx) return container_of(ibv_ctx, struct hns_roce_context, ibv_ctx.context); } +static inline struct hns_roce_td *to_hr_td(struct ibv_td *ibv_td) +{ + return container_of(ibv_td, struct hns_roce_td, ibv_td); +} + +/* to_hr_pd always returns the real hns_roce_pd obj. */ static inline struct hns_roce_pd *to_hr_pd(struct ibv_pd *ibv_pd) { - return container_of(ibv_pd, struct hns_roce_pd, ibv_pd); + struct hns_roce_pd *pd = + container_of(ibv_pd, struct hns_roce_pd, ibv_pd); + + if (pd->protection_domain) + return pd->protection_domain; + + return pd; +} + +static inline struct hns_roce_pad *to_hr_pad(struct ibv_pd *ibv_pd) +{ + struct hns_roce_pad *pad = + ibv_pd ? + container_of(ibv_pd, struct hns_roce_pad, pd.ibv_pd) : + NULL; + + if (pad && pad->pd.protection_domain) + return pad; + + /* Otherwise ibv_pd isn't a parent_domain */ + return NULL; } static inline struct hns_roce_cq *to_hr_cq(struct ibv_cq *ibv_cq) @@ -422,14 +465,35 @@ static inline struct hns_roce_ah *to_hr_ah(struct ibv_ah *ibv_ah) return container_of(ibv_ah, struct hns_roce_ah, ibv_ah); } +static inline int hns_roce_spin_lock(struct hns_roce_spinlock *hr_lock) +{ + if (hr_lock->need_lock) + return pthread_spin_lock(&hr_lock->lock); + + return 0; +} + +static inline int hns_roce_spin_unlock(struct hns_roce_spinlock *hr_lock) +{ + if (hr_lock->need_lock) + return pthread_spin_unlock(&hr_lock->lock); + + return 0; +} + int hns_roce_u_query_device(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr, size_t attr_size); int hns_roce_u_query_port(struct ibv_context *context, uint8_t port, struct ibv_port_attr *attr); +struct ibv_td *hns_roce_u_alloc_td(struct ibv_context *context, + struct ibv_td_init_attr *attr); +int hns_roce_u_dealloc_td(struct ibv_td *ibv_td); +struct ibv_pd *hns_roce_u_alloc_pad(struct ibv_context *context, + struct ibv_parent_domain_init_attr *attr); struct ibv_pd *hns_roce_u_alloc_pd(struct ibv_context *context); -int hns_roce_u_free_pd(struct ibv_pd *pd); +int hns_roce_u_dealloc_pd(struct ibv_pd *pd); struct ibv_mr *hns_roce_u_reg_mr(struct ibv_pd *pd, void *addr, size_t length, uint64_t hca_va, int access); @@ -488,6 +552,7 @@ int hns_roce_u_close_xrcd(struct ibv_xrcd *ibv_xrcd); int hns_roce_alloc_buf(struct hns_roce_buf *buf, unsigned int size, int page_size); void hns_roce_free_buf(struct hns_roce_buf *buf); +void hns_roce_qp_spinlock_destroy(struct hns_roce_qp *qp); void hns_roce_free_qp_buf(struct hns_roce_qp *qp, struct hns_roce_context *ctx); diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index c652eea..80e836d 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -227,14 +227,14 @@ static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, uint16_t ind) uint32_t bitmap_num; int bit_num; - pthread_spin_lock(&srq->lock); + hns_roce_spin_lock(&srq->hr_lock); bitmap_num = ind / BIT_CNT_PER_LONG; bit_num = ind % BIT_CNT_PER_LONG; srq->idx_que.bitmap[bitmap_num] |= (1ULL << bit_num); srq->idx_que.tail++; - pthread_spin_unlock(&srq->lock); + hns_roce_spin_unlock(&srq->hr_lock); } static int get_srq_from_cqe(struct hns_roce_v2_cqe *cqe, @@ -266,9 +266,9 @@ static int hns_roce_v2_wq_overflow(struct hns_roce_wq *wq, unsigned int nreq, if (cur + nreq < wq->max_post) return 0; - pthread_spin_lock(&cq->lock); + hns_roce_spin_lock(&cq->hr_lock); cur = wq->head - wq->tail; - pthread_spin_unlock(&cq->lock); + hns_roce_spin_unlock(&cq->hr_lock); return cur + nreq >= wq->max_post; } @@ -721,7 +721,7 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne, int err = V2_CQ_OK; int npolled; - pthread_spin_lock(&cq->lock); + hns_roce_spin_lock(&cq->hr_lock); for (npolled = 0; npolled < ne; ++npolled) { err = hns_roce_poll_one(ctx, &qp, cq, wc + npolled); @@ -736,7 +736,7 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne, update_cq_db(ctx, cq); } - pthread_spin_unlock(&cq->lock); + hns_roce_spin_unlock(&cq->hr_lock); return err == V2_CQ_POLL_ERR ? err : npolled; } @@ -1273,7 +1273,7 @@ int hns_roce_u_v2_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, return ret; } - pthread_spin_lock(&qp->sq.lock); + hns_roce_spin_lock(&qp->sq.hr_lock); sge_info.start_idx = qp->next_sge; /* start index of extend sge */ @@ -1333,7 +1333,7 @@ out: *(qp->sdb) = qp->sq.head & 0xffff; } - pthread_spin_unlock(&qp->sq.lock); + hns_roce_spin_unlock(&qp->sq.hr_lock); if (ibvqp->state == IBV_QPS_ERR) { attr.qp_state = IBV_QPS_ERR; @@ -1426,7 +1426,7 @@ static int hns_roce_u_v2_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, return ret; } - pthread_spin_lock(&qp->rq.lock); + hns_roce_spin_lock(&qp->rq.hr_lock); max_sge = qp->rq.max_gs - qp->rq.rsv_sge; for (nreq = 0; wr; ++nreq, wr = wr->next) { @@ -1460,7 +1460,7 @@ out: hns_roce_update_rq_db(ctx, ibvqp->qp_num, qp->rq.head); } - pthread_spin_unlock(&qp->rq.lock); + hns_roce_spin_unlock(&qp->rq.hr_lock); if (ibvqp->state == IBV_QPS_ERR) { attr.qp_state = IBV_QPS_ERR; @@ -1515,9 +1515,9 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *cq, uint32_t qpn, static void hns_roce_v2_cq_clean(struct hns_roce_cq *cq, unsigned int qpn, struct hns_roce_srq *srq) { - pthread_spin_lock(&cq->lock); + hns_roce_spin_lock(&cq->hr_lock); __hns_roce_v2_cq_clean(cq, qpn, srq); - pthread_spin_unlock(&cq->lock); + hns_roce_spin_unlock(&cq->hr_lock); } static void record_qp_attr(struct ibv_qp *qp, struct ibv_qp_attr *attr, @@ -1550,8 +1550,8 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int ret; if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_ERR)) { - pthread_spin_lock(&hr_qp->sq.lock); - pthread_spin_lock(&hr_qp->rq.lock); + hns_roce_spin_lock(&hr_qp->sq.hr_lock); + hns_roce_spin_lock(&hr_qp->rq.hr_lock); flag = true; } @@ -1560,8 +1560,8 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, sizeof(resp_ex)); if (flag) { - pthread_spin_unlock(&hr_qp->rq.lock); - pthread_spin_unlock(&hr_qp->sq.lock); + hns_roce_spin_unlock(&hr_qp->sq.hr_lock); + hns_roce_spin_unlock(&hr_qp->rq.hr_lock); } if (ret) @@ -1602,18 +1602,18 @@ static void hns_roce_lock_cqs(struct ibv_qp *qp) if (send_cq && recv_cq) { if (send_cq == recv_cq) { - pthread_spin_lock(&send_cq->lock); + hns_roce_spin_lock(&send_cq->hr_lock); } else if (send_cq->cqn < recv_cq->cqn) { - pthread_spin_lock(&send_cq->lock); - pthread_spin_lock(&recv_cq->lock); + hns_roce_spin_lock(&send_cq->hr_lock); + hns_roce_spin_lock(&recv_cq->hr_lock); } else { - pthread_spin_lock(&recv_cq->lock); - pthread_spin_lock(&send_cq->lock); + hns_roce_spin_lock(&recv_cq->hr_lock); + hns_roce_spin_lock(&send_cq->hr_lock); } } else if (send_cq) { - pthread_spin_lock(&send_cq->lock); + hns_roce_spin_lock(&send_cq->hr_lock); } else if (recv_cq) { - pthread_spin_lock(&recv_cq->lock); + hns_roce_spin_lock(&recv_cq->hr_lock); } } @@ -1624,18 +1624,18 @@ static void hns_roce_unlock_cqs(struct ibv_qp *qp) if (send_cq && recv_cq) { if (send_cq == recv_cq) { - pthread_spin_unlock(&send_cq->lock); + hns_roce_spin_unlock(&send_cq->hr_lock); } else if (send_cq->cqn < recv_cq->cqn) { - pthread_spin_unlock(&recv_cq->lock); - pthread_spin_unlock(&send_cq->lock); + hns_roce_spin_unlock(&recv_cq->hr_lock); + hns_roce_spin_unlock(&send_cq->hr_lock); } else { - pthread_spin_unlock(&send_cq->lock); - pthread_spin_unlock(&recv_cq->lock); + hns_roce_spin_unlock(&send_cq->hr_lock); + hns_roce_spin_unlock(&recv_cq->hr_lock); } } else if (send_cq) { - pthread_spin_unlock(&send_cq->lock); + hns_roce_spin_unlock(&send_cq->hr_lock); } else if (recv_cq) { - pthread_spin_unlock(&recv_cq->lock); + hns_roce_spin_unlock(&recv_cq->hr_lock); } } @@ -1750,7 +1750,7 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, int ret = 0; void *wqe; - pthread_spin_lock(&srq->lock); + hns_roce_spin_lock(&srq->hr_lock); max_sge = srq->max_gs - srq->rsv_sge; for (nreq = 0; wr; ++nreq, wr = wr->next) { @@ -1789,7 +1789,7 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, (__le32 *)&srq_db); } - pthread_spin_unlock(&srq->lock); + hns_roce_spin_unlock(&srq->hr_lock); return ret; } @@ -1805,11 +1805,11 @@ static int wc_start_poll_cq(struct ibv_cq_ex *current, if (attr->comp_mask) return EINVAL; - pthread_spin_lock(&cq->lock); + hns_roce_spin_lock(&cq->hr_lock); err = hns_roce_poll_one(ctx, &qp, cq, NULL); if (err != V2_CQ_OK) - pthread_spin_unlock(&cq->lock); + hns_roce_spin_unlock(&cq->hr_lock); return err; } @@ -1843,7 +1843,7 @@ static void wc_end_poll_cq(struct ibv_cq_ex *current) else update_cq_db(ctx, cq); - pthread_spin_unlock(&cq->lock); + hns_roce_spin_unlock(&cq->hr_lock); } static enum ibv_wc_opcode wc_read_opcode(struct ibv_cq_ex *current) @@ -2558,7 +2558,7 @@ static void wr_start(struct ibv_qp_ex *ibv_qp) return; } - pthread_spin_lock(&qp->sq.lock); + hns_roce_spin_lock(&qp->sq.hr_lock); qp->sge_info.start_idx = qp->next_sge; qp->rb_sq_head = qp->sq.head; qp->err = 0; @@ -2591,7 +2591,8 @@ static int wr_complete(struct ibv_qp_ex *ibv_qp) } out: - pthread_spin_unlock(&qp->sq.lock); + hns_roce_spin_unlock(&qp->sq.hr_lock); + if (ibv_qp->qp_base.state == IBV_QPS_ERR) { attr.qp_state = IBV_QPS_ERR; hns_roce_u_v2_modify_qp(&ibv_qp->qp_base, &attr, IBV_QP_STATE); @@ -2606,7 +2607,7 @@ static void wr_abort(struct ibv_qp_ex *ibv_qp) qp->sq.head = qp->rb_sq_head; - pthread_spin_unlock(&qp->sq.lock); + hns_roce_spin_unlock(&qp->sq.hr_lock); } enum { diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 3b7a67d..f6c7423 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,38 @@ #include "hns_roce_u_db.h" #include "hns_roce_u_hw_v2.h" +static int hns_roce_whether_need_lock(struct ibv_pd *pd) +{ + struct hns_roce_pad *pad; + bool need_lock = true; + + pad = to_hr_pad(pd); + if (pad && pad->td) + need_lock = false; + + return need_lock; +} + +static int hns_roce_spinlock_init(struct hns_roce_spinlock *hr_lock, + bool need_lock) +{ + hr_lock->need_lock = need_lock; + + if (need_lock) + return pthread_spin_init(&hr_lock->lock, + PTHREAD_PROCESS_PRIVATE); + + return 0; +} + +static int hns_roce_spinlock_destroy(struct hns_roce_spinlock *hr_lock) +{ + if (hr_lock->need_lock) + return pthread_spin_destroy(&hr_lock->lock); + + return 0; +} + void hns_roce_init_qp_indices(struct hns_roce_qp *qp) { qp->sq.head = 0; @@ -85,40 +118,153 @@ int hns_roce_u_query_port(struct ibv_context *context, uint8_t port, return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd)); } +struct ibv_td *hns_roce_u_alloc_td(struct ibv_context *context, + struct ibv_td_init_attr *attr) +{ + struct hns_roce_td *td; + + if (attr->comp_mask) { + errno = EINVAL; + return NULL; + } + + td = calloc(1, sizeof(*td)); + if (!td) { + errno = ENOMEM; + return NULL; + } + + td->ibv_td.context = context; + atomic_init(&td->refcount, 1); + + return &td->ibv_td; +} + +int hns_roce_u_dealloc_td(struct ibv_td *ibv_td) +{ + struct hns_roce_td *td; + int ret = 0; + + td = to_hr_td(ibv_td); + if (atomic_load(&td->refcount) > 1) { + ret = -EBUSY; + goto err; + } + + free(td); + +err: + errno = abs(ret); + return ret; +} + struct ibv_pd *hns_roce_u_alloc_pd(struct ibv_context *context) { + struct hns_roce_alloc_pd_resp resp = {}; struct ibv_alloc_pd cmd; struct hns_roce_pd *pd; - struct hns_roce_alloc_pd_resp resp = {}; - - pd = malloc(sizeof(*pd)); - if (!pd) - return NULL; + int ret; - if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof(cmd), - &resp.ibv_resp, sizeof(resp))) { - free(pd); + pd = calloc(1, sizeof(*pd)); + if (!pd) { + errno = ENOMEM; return NULL; } + ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof(cmd), + &resp.ibv_resp, sizeof(resp)); + + if (ret) + goto err; + + atomic_init(&pd->refcount, 1); pd->pdn = resp.pdn; return &pd->ibv_pd; + +err: + free(pd); + errno = abs(ret); + return NULL; } -int hns_roce_u_free_pd(struct ibv_pd *pd) +struct ibv_pd *hns_roce_u_alloc_pad(struct ibv_context *context, + struct ibv_parent_domain_init_attr *attr) +{ + struct hns_roce_pad *pad; + + if (ibv_check_alloc_parent_domain(attr)) + return NULL; + + if (attr->comp_mask) { + errno = EINVAL; + return NULL; + } + + pad = calloc(1, sizeof(*pad)); + if (!pad) { + errno = ENOMEM; + return NULL; + } + + if (attr->td) { + pad->td = to_hr_td(attr->td); + atomic_fetch_add(&pad->td->refcount, 1); + } + + pad->pd.protection_domain = to_hr_pd(attr->pd); + atomic_fetch_add(&pad->pd.protection_domain->refcount, 1); + + ibv_initialize_parent_domain(&pad->pd.ibv_pd, + &pad->pd.protection_domain->ibv_pd); + + return &pad->pd.ibv_pd; +} + +static void hns_roce_free_pad(struct hns_roce_pad *pad) +{ + atomic_fetch_sub(&pad->pd.protection_domain->refcount, 1); + + if (pad->td) + atomic_fetch_sub(&pad->td->refcount, 1); + + free(pad); +} + +static int hns_roce_free_pd(struct hns_roce_pd *pd) { int ret; - ret = ibv_cmd_dealloc_pd(pd); + if (atomic_load(&pd->refcount) > 1) { + ret = -EBUSY; + goto err; + } + + ret = ibv_cmd_dealloc_pd(&pd->ibv_pd); if (ret) - return ret; + goto err; - free(to_hr_pd(pd)); + free(pd); + +err: + errno = abs(ret); return ret; } +int hns_roce_u_dealloc_pd(struct ibv_pd *ibv_pd) +{ + struct hns_roce_pad *pad = to_hr_pad(ibv_pd); + struct hns_roce_pd *pd = to_hr_pd(ibv_pd); + + if (pad) { + hns_roce_free_pad(pad); + return 0; + } + + return hns_roce_free_pd(pd); +} + struct ibv_xrcd *hns_roce_u_open_xrcd(struct ibv_context *context, struct ibv_xrcd_init_attr *xrcd_init_attr) { @@ -275,6 +421,11 @@ int hns_roce_u_dealloc_mw(struct ibv_mw *mw) return 0; } +enum { + CREATE_CQ_SUPPORTED_COMP_MASK = IBV_CQ_INIT_ATTR_MASK_FLAGS | + IBV_CQ_INIT_ATTR_MASK_PD, +}; + enum { CREATE_CQ_SUPPORTED_WC_FLAGS = IBV_WC_STANDARD_FLAGS | IBV_WC_EX_WITH_CVLAN, @@ -286,12 +437,22 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, if (!attr->cqe || attr->cqe > context->max_cqe) return -EINVAL; - if (attr->comp_mask) - return -EOPNOTSUPP; + if (!check_comp_mask(attr->comp_mask, CREATE_CQ_SUPPORTED_COMP_MASK)) { + verbs_err(&context->ibv_ctx, "unsupported cq comps 0x%x\n", + attr->comp_mask); + return EOPNOTSUPP; + } if (!check_comp_mask(attr->wc_flags, CREATE_CQ_SUPPORTED_WC_FLAGS)) return -EOPNOTSUPP; + if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) { + if (!to_hr_pad(attr->parent_domain)) { + verbs_err(&context->ibv_ctx, "failed to check the pad of cq.\n"); + return EINVAL; + } + } + attr->cqe = max_t(uint32_t, HNS_ROCE_MIN_CQE_NUM, roundup_pow_of_two(attr->cqe)); @@ -341,7 +502,9 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context, struct ibv_cq_init_attr_ex *attr) { struct hns_roce_context *hr_ctx = to_hr_ctx(context); + struct hns_roce_pad *pad = NULL; struct hns_roce_cq *cq; + int need_lock; int ret; ret = verify_cq_create_attr(attr, hr_ctx); @@ -354,7 +517,14 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context, goto err; } - ret = pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE); + if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) + pad = to_hr_pad(attr->parent_domain); + + need_lock = hns_roce_whether_need_lock(pad ? &pad->pd.ibv_pd : NULL); + if (!need_lock) + verbs_info(verbs_get_ctx(context), "configure cq as no lock.\n"); + + ret = hns_roce_spinlock_init(&cq->hr_lock, need_lock); if (ret) goto err_lock; @@ -385,14 +555,12 @@ err_cmd: hns_roce_free_db(hr_ctx, cq->db, HNS_ROCE_CQ_TYPE_DB); err_db: hns_roce_free_buf(&cq->buf); -err_lock: err_buf: + hns_roce_spinlock_destroy(&cq->hr_lock); +err_lock: free(cq); err: - if (ret < 0) - ret = -ret; - - errno = ret; + errno = abs(ret); return NULL; } @@ -655,6 +823,7 @@ static struct ibv_srq *create_srq(struct ibv_context *context, { struct hns_roce_context *hr_ctx = to_hr_ctx(context); struct hns_roce_srq *srq; + int need_lock; int ret; ret = verify_srq_create_attr(hr_ctx, init_attr); @@ -667,7 +836,11 @@ static struct ibv_srq *create_srq(struct ibv_context *context, goto err; } - if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE)) + need_lock = hns_roce_whether_need_lock(init_attr->pd); + if (!need_lock) + verbs_info(verbs_get_ctx(context), "configure srq as no lock.\n"); + + if (hns_roce_spinlock_init(&srq->hr_lock, need_lock)) goto err_free_srq; set_srq_param(context, srq, init_attr); @@ -875,6 +1048,48 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx, return verify_qp_create_cap(ctx, attr); } +static int hns_roce_qp_spinlock_init(struct hns_roce_context *ctx, + struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp) +{ + int sq_need_lock; + int rq_need_lock; + int ret; + + sq_need_lock = hns_roce_whether_need_lock(attr->pd); + if (!sq_need_lock) + verbs_warn(&ctx->ibv_ctx, "configure sq as no lock.\n"); + + rq_need_lock = hns_roce_whether_need_lock(attr->pd); + if (!rq_need_lock) + verbs_warn(&ctx->ibv_ctx, "configure rq as no lock.\n"); + + ret = hns_roce_spinlock_init(&qp->sq.hr_lock, sq_need_lock); + if (ret) { + verbs_err(&ctx->ibv_ctx, "failed to init sq spinlock.\n"); + return ret; + } + + ret = hns_roce_spinlock_init(&qp->rq.hr_lock, rq_need_lock); + if (ret) { + verbs_err(&ctx->ibv_ctx, "failed to init rq spinlock.\n"); + goto err_rq_lock; + } + + return 0; + +err_rq_lock: + hns_roce_spinlock_destroy(&qp->sq.hr_lock); + + return ret; +} + +void hns_roce_qp_spinlock_destroy(struct hns_roce_qp *qp) +{ + hns_roce_spinlock_destroy(&qp->rq.hr_lock); + hns_roce_spinlock_destroy(&qp->sq.hr_lock); +} + static int alloc_recv_rinl_buf(uint32_t max_sge, struct hns_roce_rinl_buf *rinl_buf) { @@ -1248,8 +1463,8 @@ static int hns_roce_alloc_qp_buf(struct ibv_qp_init_attr_ex *attr, { int ret; - if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE) || - pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE)) + if (pthread_spin_init(&qp->sq.hr_lock.lock, PTHREAD_PROCESS_PRIVATE) || + pthread_spin_init(&qp->rq.hr_lock.lock, PTHREAD_PROCESS_PRIVATE)) return -ENOMEM; ret = qp_alloc_wqe(&attr->cap, qp, ctx); @@ -1294,6 +1509,10 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, hns_roce_set_qp_params(attr, qp, context); + ret = hns_roce_qp_spinlock_init(context, attr, qp); + if (ret) + goto err_spinlock; + ret = hns_roce_alloc_qp_buf(attr, qp, context); if (ret) goto err_buf; @@ -1327,6 +1546,8 @@ err_ops: err_cmd: hns_roce_free_qp_buf(qp, context); err_buf: + hns_roce_qp_spinlock_destroy(qp); +err_spinlock: free(qp); err: if (ret < 0) -- 2.30.0