From b88e6ae3e144651092bce923123ca20361cdacab Mon Sep 17 00:00:00 2001 From: Yixing Liu Date: Tue, 27 Sep 2022 19:06:00 +0800 Subject: [PATCH 12/12] libhns: Support DSCP This patch adds user mode DSCP function through the mapping of dscp-tc configured in kernel mode. Signed-off-by: Yixing Liu --- providers/hns/hns_roce_u.h | 7 +++++++ providers/hns/hns_roce_u_abi.h | 6 ++++++ providers/hns/hns_roce_u_hw_v2.c | 19 +++++++++++++++---- providers/hns/hns_roce_u_verbs.c | 7 +++++-- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 6b64cd0..8c1cb1e 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -175,6 +175,11 @@ enum hns_roce_db_type { HNS_ROCE_DB_TYPE_NUM }; +enum hns_roce_tc_map_mode { + HNS_ROCE_TC_MAP_MODE_PRIO, + HNS_ROCE_TC_MAP_MODE_DSCP, +}; + struct hns_roce_db_page { struct hns_roce_db_page *prev, *next; struct hns_roce_buf buf; @@ -315,6 +320,8 @@ struct hns_roce_qp { unsigned int next_sge; int port_num; uint8_t sl; + uint8_t tc_mode; + uint8_t priority; unsigned int qkey; enum ibv_mtu path_mtu; diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h index 2753d30..0519ac7 100644 --- a/providers/hns/hns_roce_u_abi.h +++ b/providers/hns/hns_roce_u_abi.h @@ -49,6 +49,9 @@ DECLARE_DRV_CMD(hns_roce_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ, DECLARE_DRV_CMD(hns_roce_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, hns_roce_ib_alloc_ucontext, hns_roce_ib_alloc_ucontext_resp); +DECLARE_DRV_CMD(hns_roce_create_ah, IB_USER_VERBS_CMD_CREATE_AH, empty, + hns_roce_ib_create_ah_resp); + DECLARE_DRV_CMD(hns_roce_create_qp, IB_USER_VERBS_CMD_CREATE_QP, hns_roce_ib_create_qp, hns_roce_ib_create_qp_resp); @@ -61,4 +64,7 @@ DECLARE_DRV_CMD(hns_roce_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ, DECLARE_DRV_CMD(hns_roce_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ, hns_roce_ib_create_srq, hns_roce_ib_create_srq_resp); +DECLARE_DRV_CMD(hns_roce_modify_qp_ex, IB_USER_VERBS_EX_CMD_MODIFY_QP, + empty, hns_roce_ib_modify_qp_resp); + #endif /* _HNS_ROCE_U_ABI_H */ diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index a30d461..c652eea 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -1543,10 +1543,11 @@ static void record_qp_attr(struct ibv_qp *qp, struct ibv_qp_attr *attr, static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) { - int ret; - struct ibv_modify_qp cmd; + struct hns_roce_modify_qp_ex_resp resp_ex = {}; + struct hns_roce_modify_qp_ex cmd_ex = {}; struct hns_roce_qp *hr_qp = to_hr_qp(qp); bool flag = false; /* modify qp to error */ + int ret; if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_ERR)) { pthread_spin_lock(&hr_qp->sq.lock); @@ -1554,7 +1555,9 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, flag = true; } - ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof(cmd)); + ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask, &cmd_ex.ibv_cmd, + sizeof(cmd_ex), &resp_ex.ibv_resp, + sizeof(resp_ex)); if (flag) { pthread_spin_unlock(&hr_qp->rq.lock); @@ -1564,8 +1567,13 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, if (ret) return ret; - if (attr_mask & IBV_QP_STATE) + if (attr_mask & IBV_QP_STATE) { qp->state = attr->qp_state; + if (attr->qp_state == IBV_QPS_RTR) { + hr_qp->tc_mode = resp_ex.drv_payload.tc_mode; + hr_qp->priority = resp_ex.drv_payload.priority; + } + } if ((attr_mask & IBV_QP_STATE) && attr->qp_state == IBV_QPS_RESET) { if (qp->recv_cq) @@ -1579,6 +1587,9 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, hns_roce_init_qp_indices(to_hr_qp(qp)); } + if (hr_qp->tc_mode == HNS_ROCE_TC_MAP_MODE_DSCP) + hr_qp->sl = hr_qp->priority; + record_qp_attr(qp, attr, attr_mask); return ret; diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index cff9d1d..3b7a67d 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -1449,7 +1449,7 @@ static int get_tclass(struct ibv_context *context, struct ibv_ah_attr *attr, struct ibv_ah *hns_roce_u_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) { struct hns_roce_device *hr_dev = to_hr_dev(pd->context->device); - struct ib_uverbs_create_ah_resp resp = {}; + struct hns_roce_create_ah_resp resp = {}; struct hns_roce_ah *ah; /* HIP08 don't support create ah */ @@ -1477,12 +1477,15 @@ struct ibv_ah *hns_roce_u_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) memcpy(ah->av.dgid, attr->grh.dgid.raw, ARRAY_SIZE(ah->av.dgid)); } - if (ibv_cmd_create_ah(pd, &ah->ibv_ah, attr, &resp, sizeof(resp))) + if (ibv_cmd_create_ah(pd, &ah->ibv_ah, attr, &resp.ibv_resp, sizeof(resp))) goto err; if (ibv_resolve_eth_l2_from_gid(pd->context, attr, ah->av.mac, NULL)) goto err; + if (resp.tc_mode == HNS_ROCE_TC_MAP_MODE_DSCP) + ah->av.sl = resp.priority; + ah->av.udp_sport = get_ah_udp_sport(attr); return &ah->ibv_ah; -- 2.30.0