From 4cc15f4ef3dadb3219719376822cf427df338f2a Mon Sep 17 00:00:00 2001 From: Junxian Huang Date: Tue, 5 Mar 2024 13:57:24 +0800 Subject: [PATCH 7/7] libhns: Support congestion control algorithm configuration with direct verbs driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I95UWO ------------------------------------------------------------------ Add support for configuration of congestion control algorithms in QP granularity with direct verbs hnsdv_create_qp(). Signed-off-by: Junxian Huang Signed-off-by: Ran Zhou --- providers/hns/hns_roce_u.c | 1 + providers/hns/hns_roce_u.h | 1 + providers/hns/hns_roce_u_verbs.c | 45 ++++++++++++++++++++++++++++---- providers/hns/hnsdv.h | 25 ++++++++++++++++-- 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c index 69f7d3f..90f250e 100644 --- a/providers/hns/hns_roce_u.c +++ b/providers/hns/hns_roce_u.c @@ -135,6 +135,7 @@ static int set_context_attr(struct hns_roce_device *hr_dev, return EIO; hr_dev->hw_version = dev_attrs.hw_ver; + hr_dev->congest_cap = resp->congest_type; context->max_qp_wr = dev_attrs.max_qp_wr; context->max_sge = dev_attrs.max_sge; context->max_cqe = dev_attrs.max_cqe; diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 99fa23f..c73e5c0 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -158,6 +158,7 @@ struct hns_roce_device { int page_size; const struct hns_roce_u_hw *u_hw; int hw_version; + uint8_t congest_cap; }; struct hns_roce_buf { diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 997b7e0..dcdc722 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -786,7 +786,7 @@ int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq) } enum { - HNSDV_QP_SUP_COMP_MASK = 0, + HNSDV_QP_SUP_COMP_MASK = HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE, }; static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, @@ -1209,10 +1209,33 @@ static int hns_roce_store_qp(struct hns_roce_context *ctx, return 0; } +static int to_cmd_cong_type(uint8_t cong_type, __u64 *cmd_cong_type) +{ + switch (cong_type) { + case HNSDV_QP_CREATE_ENABLE_DCQCN: + *cmd_cong_type = HNS_ROCE_CREATE_QP_FLAGS_DCQCN; + break; + case HNSDV_QP_CREATE_ENABLE_LDCP: + *cmd_cong_type = HNS_ROCE_CREATE_QP_FLAGS_LDCP; + break; + case HNSDV_QP_CREATE_ENABLE_HC3: + *cmd_cong_type = HNS_ROCE_CREATE_QP_FLAGS_HC3; + break; + case HNSDV_QP_CREATE_ENABLE_DIP: + *cmd_cong_type = HNS_ROCE_CREATE_QP_FLAGS_DIP; + break; + default: + return EINVAL; + } + + return 0; +} + static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr, struct hns_roce_qp *qp, struct hns_roce_context *ctx, - uint64_t *dwqe_mmap_key) + uint64_t *dwqe_mmap_key, + struct hnsdv_qp_init_attr *hns_attr) { struct hns_roce_create_qp_ex_resp resp_ex = {}; struct hns_roce_create_qp_ex cmd_ex = {}; @@ -1224,6 +1247,15 @@ static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr, cmd_ex.log_sq_stride = qp->sq.wqe_shift; cmd_ex.log_sq_bb_count = hr_ilog32(qp->sq.wqe_cnt); + if (hns_attr && + hns_attr->comp_mask & HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE) { + ret = to_cmd_cong_type(hns_attr->congest_type, + &cmd_ex.cong_type_flags); + if (ret) + return ret; + cmd_ex.comp_mask |= HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE; + } + ret = ibv_cmd_create_qp_ex2(&ctx->ibv_ctx.context, &qp->verbs_qp, attr, &cmd_ex.ibv_cmd, sizeof(cmd_ex), &resp_ex.ibv_resp, sizeof(resp_ex)); @@ -1322,7 +1354,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, if (ret) goto err_buf; - ret = qp_exec_create_cmd(attr, qp, context, &dwqe_mmap_key); + ret = qp_exec_create_cmd(attr, qp, context, &dwqe_mmap_key, hns_attr); if (ret) goto err_cmd; @@ -1403,9 +1435,9 @@ struct ibv_qp *hnsdv_create_qp(struct ibv_context *context, int hnsdv_query_device(struct ibv_context *context, struct hnsdv_context *attrs_out) { - struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL; + struct hns_roce_device *hr_dev = to_hr_dev(context->device); - if (!ctx || !attrs_out) + if (!hr_dev || !attrs_out) return EINVAL; if (!is_hns_dev(context->device)) { @@ -1414,6 +1446,9 @@ int hnsdv_query_device(struct ibv_context *context, } memset(attrs_out, 0, sizeof(*attrs_out)); + attrs_out->comp_mask |= HNSDV_CONTEXT_MASK_CONGEST_TYPE; + attrs_out->congest_type = hr_dev->congest_cap; + return 0; } diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h index 49ba08a..451b26e 100644 --- a/providers/hns/hnsdv.h +++ b/providers/hns/hnsdv.h @@ -15,12 +15,33 @@ extern "C" { #endif +enum hnsdv_qp_congest_ctrl_type { + HNSDV_QP_CREATE_ENABLE_DCQCN = 1 << 0, + HNSDV_QP_CREATE_ENABLE_LDCP = 1 << 1, + HNSDV_QP_CREATE_ENABLE_HC3 = 1 << 2, + HNSDV_QP_CREATE_ENABLE_DIP = 1 << 3, +}; + +enum hnsdv_qp_init_attr_mask { + HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE = 1 << 1, +}; + struct hnsdv_qp_init_attr { - uint64_t comp_mask; + uint64_t comp_mask; /* Use enum hnsdv_qp_init_attr_mask */ + uint32_t create_flags; + uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */ + uint8_t reserved[3]; +}; + +enum hnsdv_query_context_comp_mask { + HNSDV_CONTEXT_MASK_CONGEST_TYPE = 1 << 0, }; struct hnsdv_context { - uint64_t comp_mask; + uint64_t comp_mask; /* Use enum hnsdv_query_context_comp_mask */ + uint64_t flags; + uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */ + uint8_t reserved[7]; }; bool hnsdv_is_supported(struct ibv_device *device); -- 2.33.0