From b914c76318f5b95e3157c3cbf1ccb49ec6d27635 Mon Sep 17 00:00:00 2001 From: Wenpeng Liang Date: Tue, 11 May 2021 19:06:39 +0800 Subject: libhns: Refactor the process of create_srq Reorganize create_srq() as several sub-functions to make the process clearer. Signed-off-by: Wenpeng Liang Signed-off-by: Weihang Li --- providers/hns/hns_roce_u.h | 7 +- providers/hns/hns_roce_u_hw_v2.c | 2 +- providers/hns/hns_roce_u_verbs.c | 178 ++++++++++++++++++------------- 3 files changed, 105 insertions(+), 82 deletions(-) diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index b3f48113..a437727c 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -211,7 +211,8 @@ struct hns_roce_idx_que { struct hns_roce_srq { struct verbs_srq verbs_srq; - struct hns_roce_buf buf; + struct hns_roce_idx_que idx_que; + struct hns_roce_buf wqe_buf; pthread_spinlock_t lock; unsigned long *wrid; unsigned int srqn; @@ -221,7 +222,6 @@ struct hns_roce_srq { unsigned int wqe_shift; unsigned int *db; unsigned short counter; - struct hns_roce_idx_que idx_que; }; struct hns_roce_wq { @@ -343,8 +343,7 @@ static inline struct hns_roce_cq *to_hr_cq(struct ibv_cq *ibv_cq) static inline struct hns_roce_srq *to_hr_srq(struct ibv_srq *ibv_srq) { - return container_of(container_of(ibv_srq, struct verbs_srq, srq), - struct hns_roce_srq, verbs_srq); + return container_of(ibv_srq, struct hns_roce_srq, verbs_srq.srq); } static inline struct hns_roce_qp *to_hr_qp(struct ibv_qp *ibv_qp) diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index b622eaef..d4e7e4f9 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -244,7 +244,7 @@ static void *get_send_sge_ex(struct hns_roce_qp *qp, unsigned int n) static void *get_srq_wqe(struct hns_roce_srq *srq, unsigned int n) { - return srq->buf.buf + (n << srq->wqe_shift); + return srq->wqe_buf.buf + (n << srq->wqe_shift); } static void *get_idx_buf(struct hns_roce_idx_que *idx_que, int n) diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 107da753..75b9e530 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -432,17 +432,23 @@ static int hns_roce_store_srq(struct hns_roce_context *ctx, uint32_t tind = (srq->srqn & (ctx->num_srqs - 1)) >> ctx->srq_table_shift; + pthread_mutex_lock(&ctx->srq_table_mutex); + if (!ctx->srq_table[tind].refcnt) { ctx->srq_table[tind].table = calloc(ctx->srq_table_mask + 1, sizeof(struct hns_roce_srq *)); - if (!ctx->srq_table[tind].table) + if (!ctx->srq_table[tind].table) { + pthread_mutex_unlock(&ctx->srq_table_mutex); return -ENOMEM; + } } ++ctx->srq_table[tind].refcnt; ctx->srq_table[tind].table[srq->srqn & ctx->srq_table_mask] = srq; + pthread_mutex_unlock(&ctx->srq_table_mutex); + return 0; } @@ -461,13 +467,46 @@ static void hns_roce_clear_srq(struct hns_roce_context *ctx, uint32_t srqn) { uint32_t tind = (srqn & (ctx->num_srqs - 1)) >> ctx->srq_table_shift; + pthread_mutex_lock(&ctx->srq_table_mutex); + if (!--ctx->srq_table[tind].refcnt) free(ctx->srq_table[tind].table); else ctx->srq_table[tind].table[srqn & ctx->srq_table_mask] = NULL; + + pthread_mutex_unlock(&ctx->srq_table_mutex); +} + +static int verify_srq_create_attr(struct hns_roce_context *context, + struct ibv_srq_init_attr_ex *attr) +{ + if (attr->srq_type != IBV_SRQT_BASIC && + attr->srq_type != IBV_SRQT_XRC) + return -EINVAL; + + if (!attr->attr.max_sge || + attr->attr.max_wr > context->max_srq_wr || + attr->attr.max_sge > context->max_srq_sge) + return -EINVAL; + + return 0; +} + +static void set_srq_param(struct ibv_context *context, struct hns_roce_srq *srq, + struct ibv_srq_init_attr_ex *attr) +{ + if (to_hr_dev(context->device)->hw_version == HNS_ROCE_HW_VER2) + srq->rsv_sge = 1; + + srq->wqe_cnt = roundup_pow_of_two(attr->attr.max_wr + 1); + srq->max_gs = roundup_pow_of_two(attr->attr.max_sge + srq->rsv_sge); + srq->wqe_shift = hr_ilog32(roundup_pow_of_two(HNS_ROCE_SGE_SIZE * + srq->max_gs)); + attr->attr.max_sge = srq->max_gs; + attr->attr.srq_limit = 0; } -static int hns_roce_create_idx_que(struct hns_roce_srq *srq) +static int alloc_srq_idx_que(struct hns_roce_srq *srq) { struct hns_roce_idx_que *idx_que = &srq->idx_que; unsigned int buf_size; @@ -478,13 +517,13 @@ static int hns_roce_create_idx_que(struct hns_roce_srq *srq) BIT_CNT_PER_LONG; idx_que->bitmap = calloc(idx_que->bitmap_cnt, sizeof(unsigned long)); if (!idx_que->bitmap) - return ENOMEM; + return -ENOMEM; buf_size = to_hr_hem_entries_size(srq->wqe_cnt, idx_que->entry_shift); if (hns_roce_alloc_buf(&idx_que->buf, buf_size, HNS_HW_PAGE_SIZE)) { free(idx_que->bitmap); idx_que->bitmap = NULL; - return ENOMEM; + return -ENOMEM; } /* init the idx_que bitmap */ @@ -497,40 +536,48 @@ static int hns_roce_create_idx_que(struct hns_roce_srq *srq) return 0; } -static int hns_roce_alloc_srq_buf(struct hns_roce_srq *srq) +static int alloc_srq_wqe_buf(struct hns_roce_srq *srq) { - int srq_buf_size; + int buf_size = to_hr_hem_entries_size(srq->wqe_cnt, srq->wqe_shift); - srq->wrid = calloc(srq->wqe_cnt, sizeof(unsigned long)); - if (!srq->wrid) - return ENOMEM; + return hns_roce_alloc_buf(&srq->wqe_buf, buf_size, HNS_HW_PAGE_SIZE); +} - srq->wqe_shift = hr_ilog32(roundup_pow_of_two(HNS_ROCE_SGE_SIZE * - srq->max_gs)); - srq_buf_size = to_hr_hem_entries_size(srq->wqe_cnt, srq->wqe_shift); +static int alloc_srq_buf(struct hns_roce_srq *srq) +{ + int ret; - /* allocate srq wqe buf */ - if (hns_roce_alloc_buf(&srq->buf, srq_buf_size, HNS_HW_PAGE_SIZE)) { - free(srq->wrid); - return ENOMEM; + ret = alloc_srq_idx_que(srq); + if (ret) + return ret; + + ret = alloc_srq_wqe_buf(srq); + if (ret) + goto err_idx_que; + + srq->wrid = calloc(srq->wqe_cnt, sizeof(*srq->wrid)); + if (!srq->wrid) { + ret = -ENOMEM; + goto err_wqe_buf; } return 0; -} -static int hns_roce_verify_srq(struct hns_roce_context *context, - struct ibv_srq_init_attr_ex *init_attr) -{ - if (init_attr->srq_type != IBV_SRQT_BASIC && - init_attr->srq_type != IBV_SRQT_XRC) - return -EINVAL; +err_wqe_buf: + hns_roce_free_buf(&srq->wqe_buf); +err_idx_que: + hns_roce_free_buf(&srq->idx_que.buf); + free(srq->idx_que.bitmap); - if (!init_attr->attr.max_sge || - init_attr->attr.max_wr > context->max_srq_wr || - init_attr->attr.max_sge > context->max_srq_sge) - return -EINVAL; + return ret; +} - return 0; +static void free_srq_buf(struct hns_roce_srq *srq) +{ + free(srq->wrid); + hns_roce_free_buf(&srq->wqe_buf); + hns_roce_free_buf(&srq->idx_que.buf); + free(srq->idx_que.bitmap); } static int exec_srq_create_cmd(struct ibv_context *context, @@ -541,7 +588,7 @@ static int exec_srq_create_cmd(struct ibv_context *context, struct hns_roce_create_srq_ex cmd_ex = {}; int ret; - cmd_ex.buf_addr = (uintptr_t)srq->buf.buf; + cmd_ex.buf_addr = (uintptr_t)srq->wqe_buf.buf; cmd_ex.que_addr = (uintptr_t)srq->idx_que.buf.buf; cmd_ex.db_addr = (uintptr_t)srq->db; @@ -559,57 +606,44 @@ static int exec_srq_create_cmd(struct ibv_context *context, static struct ibv_srq *create_srq(struct ibv_context *context, struct ibv_srq_init_attr_ex *init_attr) { - struct hns_roce_context *ctx = to_hr_ctx(context); - struct ibv_srq_attr *attr = &init_attr->attr; + struct hns_roce_context *hr_ctx = to_hr_ctx(context); struct hns_roce_srq *srq; int ret; - if (hns_roce_verify_srq(ctx, init_attr)) - return NULL; + ret = verify_srq_create_attr(hr_ctx, init_attr); + if (ret) + goto err; srq = calloc(1, sizeof(*srq)); - if (!srq) - return NULL; + if (!srq) { + ret = -ENOMEM; + goto err; + } if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE)) goto err_free_srq; - if (to_hr_dev(context->device)->hw_version == HNS_ROCE_HW_VER2) - srq->rsv_sge = 1; - - srq->wqe_cnt = roundup_pow_of_two(attr->max_wr + 1); - srq->max_gs = roundup_pow_of_two(attr->max_sge + srq->rsv_sge); - attr->max_sge = srq->max_gs; - attr->srq_limit = 0; - - ret = hns_roce_create_idx_que(srq); - if (ret) + set_srq_param(context, srq, init_attr); + if (alloc_srq_buf(srq)) goto err_free_srq; - ret = hns_roce_alloc_srq_buf(srq); - if (ret) - goto err_idx_que; - - srq->db = hns_roce_alloc_db(ctx, HNS_ROCE_QP_TYPE_DB); + srq->db = hns_roce_alloc_db(hr_ctx, HNS_ROCE_QP_TYPE_DB); if (!srq->db) goto err_srq_buf; - *(srq->db) = 0; - - pthread_mutex_lock(&ctx->srq_table_mutex); + *srq->db = 0; ret = exec_srq_create_cmd(context, srq, init_attr); if (ret) goto err_srq_db; - ret = hns_roce_store_srq(ctx, srq); + ret = hns_roce_store_srq(hr_ctx, srq); if (ret) goto err_destroy_srq; - pthread_mutex_unlock(&ctx->srq_table_mutex); - - srq->max_gs = attr->max_sge; - attr->max_sge = min(attr->max_sge - srq->rsv_sge, ctx->max_srq_sge); + srq->max_gs = init_attr->attr.max_sge; + init_attr->attr.max_sge = + min(init_attr->attr.max_sge - srq->rsv_sge, hr_ctx->max_srq_sge); return &srq->verbs_srq.srq; @@ -617,20 +651,19 @@ err_destroy_srq: ibv_cmd_destroy_srq(&srq->verbs_srq.srq); err_srq_db: - pthread_mutex_unlock(&ctx->srq_table_mutex); - hns_roce_free_db(ctx, srq->db, HNS_ROCE_QP_TYPE_DB); + hns_roce_free_db(hr_ctx, srq->db, HNS_ROCE_QP_TYPE_DB); err_srq_buf: - free(srq->wrid); - hns_roce_free_buf(&srq->buf); - -err_idx_que: - free(srq->idx_que.bitmap); - hns_roce_free_buf(&srq->idx_que.buf); + free_srq_buf(srq); err_free_srq: free(srq); +err: + if (ret < 0) + ret = -ret; + + errno = ret; return NULL; } @@ -690,23 +723,14 @@ int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq) struct hns_roce_srq *srq = to_hr_srq(ibv_srq); int ret; - pthread_mutex_lock(&ctx->srq_table_mutex); - ret = ibv_cmd_destroy_srq(ibv_srq); - if (ret) { - pthread_mutex_unlock(&ctx->srq_table_mutex); + if (ret) return ret; - } hns_roce_clear_srq(ctx, srq->srqn); - pthread_mutex_unlock(&ctx->srq_table_mutex); - hns_roce_free_db(ctx, srq->db, HNS_ROCE_QP_TYPE_DB); - hns_roce_free_buf(&srq->buf); - free(srq->wrid); - hns_roce_free_buf(&srq->idx_que.buf); - free(srq->idx_que.bitmap); + free_srq_buf(srq); free(srq); return 0; -- 2.30.0