From 71eb90581a338242a26123790e5f24df90327465 Mon Sep 17 00:00:00 2001
From: Luoyouming <luoyouming@huawei.com>
Date: Thu, 11 Aug 2022 20:50:54 +0800
Subject: [PATCH v4 10/10] libhns: Support cqe inline

When rq or srq recv data less than or equal to 32 byte in size, roce driver
support get data from cqe.

Signed-off-by: Luoyouming <luoyouming@huawei.com>
Reviewed-by: Yangyang Li <liyangyang20@huawei.com>
---
 providers/hns/hns_roce_u.c       |  3 ++-
 providers/hns/hns_roce_u.h       | 21 +++++++++--------
 providers/hns/hns_roce_u_hw_v2.c | 39 ++++++++++++++++++++++++++++++--
 providers/hns/hns_roce_u_hw_v2.h |  4 ++--
 providers/hns/hns_roce_u_verbs.c | 25 ++++++++++++++++++--
 5 files changed, 75 insertions(+), 17 deletions(-)

diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c
index 6c9aefa..266e73e 100644
--- a/providers/hns/hns_roce_u.c
+++ b/providers/hns/hns_roce_u.c
@@ -113,7 +113,8 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
 	if (!context)
 		return NULL;
 
-	cmd.config |= HNS_ROCE_EXSGE_FLAGS | HNS_ROCE_RQ_INLINE_FLAGS;
+	cmd.config |= HNS_ROCE_EXSGE_FLAGS | HNS_ROCE_RQ_INLINE_FLAGS |
+		      HNS_ROCE_CQE_INLINE_FLAGS;
 	if (ibv_cmd_get_context(&context->ibv_ctx, &cmd.ibv_cmd, sizeof(cmd),
 				&resp.ibv_resp, sizeof(resp)))
 		goto err_free;
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index 57ebe55..6b64cd0 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -246,10 +246,21 @@ struct hns_roce_idx_que {
 	unsigned int			tail;
 };
 
+struct hns_roce_rinl_wqe {
+	struct ibv_sge			*sg_list;
+	unsigned int			sge_cnt;
+};
+
+struct hns_roce_rinl_buf {
+	struct hns_roce_rinl_wqe	*wqe_list;
+	unsigned int			wqe_cnt;
+};
+
 struct hns_roce_srq {
 	struct verbs_srq		verbs_srq;
 	struct hns_roce_idx_que		idx_que;
 	struct hns_roce_buf		wqe_buf;
+	struct hns_roce_rinl_buf	srq_rinl_buf;
 	pthread_spinlock_t		lock;
 	unsigned long			*wrid;
 	unsigned int			srqn;
@@ -290,16 +301,6 @@ struct hns_roce_sge_ex {
 	unsigned int			sge_shift;
 };
 
-struct hns_roce_rinl_wqe {
-	struct ibv_sge			*sg_list;
-	unsigned int			sge_cnt;
-};
-
-struct hns_roce_rinl_buf {
-	struct hns_roce_rinl_wqe	*wqe_list;
-	unsigned int			wqe_cnt;
-};
-
 struct hns_roce_qp {
 	struct verbs_qp			verbs_qp;
 	struct hns_roce_buf		buf;
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 25d8861..7063b26 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -444,6 +444,28 @@ static void handle_recv_inl_data(struct hns_roce_v2_cqe *cqe,
 
 }
 
+static void handle_recv_cqe_inl_from_rq(struct hns_roce_v2_cqe *cqe,
+					struct hns_roce_qp *cur_qp)
+{
+	uint32_t wr_num;
+
+	wr_num = hr_reg_read(cqe, CQE_WQE_IDX) & (cur_qp->rq.wqe_cnt - 1);
+
+	handle_recv_inl_data(cqe, &(cur_qp->rq_rinl_buf), wr_num,
+			     (uint8_t *)cqe->payload);
+}
+
+static void handle_recv_cqe_inl_from_srq(struct hns_roce_v2_cqe *cqe,
+					 struct hns_roce_srq *srq)
+{
+	uint32_t wr_num;
+
+	wr_num = hr_reg_read(cqe, CQE_WQE_IDX) & (srq->wqe_cnt - 1);
+
+	handle_recv_inl_data(cqe, &(srq->srq_rinl_buf), wr_num,
+			     (uint8_t *)cqe->payload);
+}
+
 static void handle_recv_rq_inl(struct hns_roce_v2_cqe *cqe,
 			       struct hns_roce_qp *cur_qp)
 {
@@ -473,6 +495,9 @@ static void parse_cqe_for_srq(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
 	wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX);
 	wc->wr_id = srq->wrid[wqe_idx & (srq->wqe_cnt - 1)];
 	hns_roce_free_srq_wqe(srq, wqe_idx);
+
+	if (hr_reg_read(cqe, CQE_CQE_INLINE))
+		handle_recv_cqe_inl_from_srq(cqe, srq);
 }
 
 static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
@@ -487,7 +512,9 @@ static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
 	if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD)
 		parse_for_ud_qp(cqe, wc);
 
-	if (hr_reg_read(cqe, CQE_RQ_INLINE))
+	if (hr_reg_read(cqe, CQE_CQE_INLINE))
+		handle_recv_cqe_inl_from_rq(cqe, hr_qp);
+	else if (hr_reg_read(cqe, CQE_RQ_INLINE))
 		handle_recv_rq_inl(cqe, hr_qp);
 
 	return 0;
@@ -559,6 +586,9 @@ static void cqe_proc_srq(struct hns_roce_srq *srq, uint32_t wqe_idx,
 {
 	cq->verbs_cq.cq_ex.wr_id = srq->wrid[wqe_idx & (srq->wqe_cnt - 1)];
 	hns_roce_free_srq_wqe(srq, wqe_idx);
+
+	if (hr_reg_read(cq->cqe, CQE_CQE_INLINE))
+		handle_recv_cqe_inl_from_srq(cq->cqe, srq);
 }
 
 static void cqe_proc_rq(struct hns_roce_qp *hr_qp, struct hns_roce_cq *cq)
@@ -568,7 +598,9 @@ static void cqe_proc_rq(struct hns_roce_qp *hr_qp, struct hns_roce_cq *cq)
 	cq->verbs_cq.cq_ex.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
 	++wq->tail;
 
-	if (hr_reg_read(cq->cqe, CQE_RQ_INLINE))
+	if (hr_reg_read(cq->cqe, CQE_CQE_INLINE))
+		handle_recv_cqe_inl_from_rq(cq->cqe, hr_qp);
+	else if (hr_reg_read(cq->cqe, CQE_RQ_INLINE))
 		handle_recv_rq_inl(cq->cqe, hr_qp);
 }
 
@@ -1725,6 +1757,9 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq,
 
 		wqe = get_srq_wqe(srq, wqe_idx);
 		fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge);
+
+		fill_recv_inl_buf(&srq->srq_rinl_buf, wqe_idx, wr);
+
 		fill_wqe_idx(srq, wqe_idx);
 
 		srq->wrid[wqe_idx] = wr->wr_id;
diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h
index 098dbdf..d71c695 100644
--- a/providers/hns/hns_roce_u_hw_v2.h
+++ b/providers/hns/hns_roce_u_hw_v2.h
@@ -157,7 +157,7 @@ struct hns_roce_v2_cqe {
 	__le32	smac;
 	__le32	byte_28;
 	__le32	byte_32;
-	__le32	rsv[8];
+	__le32	payload[8];
 };
 
 #define CQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_cqe, h, l)
@@ -170,7 +170,7 @@ struct hns_roce_v2_cqe {
 #define CQE_WQE_IDX CQE_FIELD_LOC(31, 16)
 #define CQE_RKEY_IMMTDATA CQE_FIELD_LOC(63, 32)
 #define CQE_XRC_SRQN CQE_FIELD_LOC(87, 64)
-#define CQE_RSV0 CQE_FIELD_LOC(95, 88)
+#define CQE_CQE_INLINE CQE_FIELD_LOC(89, 88)
 #define CQE_LCL_QPN CQE_FIELD_LOC(119, 96)
 #define CQE_SUB_STATUS CQE_FIELD_LOC(127, 120)
 #define CQE_BYTE_CNT CQE_FIELD_LOC(159, 128)
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index 1d661dd..cff9d1d 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -522,6 +522,8 @@ static int verify_srq_create_attr(struct hns_roce_context *context,
 static void set_srq_param(struct ibv_context *context, struct hns_roce_srq *srq,
 			  struct ibv_srq_init_attr_ex *attr)
 {
+	struct hns_roce_context *ctx = to_hr_ctx(context);
+
 	if (to_hr_dev(context->device)->hw_version == HNS_ROCE_HW_VER2)
 		srq->rsv_sge = 1;
 
@@ -531,6 +533,10 @@ static void set_srq_param(struct ibv_context *context, struct hns_roce_srq *srq,
 						      srq->max_gs));
 	attr->attr.max_sge = srq->max_gs;
 	attr->attr.srq_limit = 0;
+
+	srq->srq_rinl_buf.wqe_cnt = 0;
+	if (ctx->config & HNS_ROCE_RSP_CQE_INLINE_FLAGS)
+		srq->srq_rinl_buf.wqe_cnt = srq->wqe_cnt;
 }
 
 static int alloc_srq_idx_que(struct hns_roce_srq *srq)
@@ -570,6 +576,11 @@ static int alloc_srq_wqe_buf(struct hns_roce_srq *srq)
 	return hns_roce_alloc_buf(&srq->wqe_buf, buf_size, HNS_HW_PAGE_SIZE);
 }
 
+static int alloc_recv_rinl_buf(uint32_t max_sge,
+				 struct hns_roce_rinl_buf *rinl_buf);
+
+static void free_recv_rinl_buf(struct hns_roce_rinl_buf *rinl_buf);
+
 static int alloc_srq_buf(struct hns_roce_srq *srq)
 {
 	int ret;
@@ -582,14 +593,22 @@ static int alloc_srq_buf(struct hns_roce_srq *srq)
 	if (ret)
 		goto err_idx_que;
 
+	if (srq->srq_rinl_buf.wqe_cnt) {
+		ret = alloc_recv_rinl_buf(srq->max_gs, &srq->srq_rinl_buf);
+		if (ret)
+			goto err_wqe_buf;
+	}
+
 	srq->wrid = calloc(srq->wqe_cnt, sizeof(*srq->wrid));
 	if (!srq->wrid) {
 		ret = -ENOMEM;
-		goto err_wqe_buf;
+		goto err_inl_buf;
 	}
 
 	return 0;
 
+err_inl_buf:
+	free_recv_rinl_buf(&srq->srq_rinl_buf);
 err_wqe_buf:
 	hns_roce_free_buf(&srq->wqe_buf);
 err_idx_que:
@@ -603,6 +622,7 @@ static void free_srq_buf(struct hns_roce_srq *srq)
 {
 	free(srq->wrid);
 	hns_roce_free_buf(&srq->wqe_buf);
+	free_recv_rinl_buf(&srq->srq_rinl_buf);
 	hns_roce_free_buf(&srq->idx_que.buf);
 	free(srq->idx_que.bitmap);
 }
@@ -1082,7 +1102,8 @@ static void hns_roce_set_qp_params(struct ibv_qp_init_attr_ex *attr,
 		qp->rq.wqe_cnt = cnt;
 		qp->rq.shift = hr_ilog32(cnt);
 		qp->rq_rinl_buf.wqe_cnt = 0;
-		if (ctx->config & HNS_ROCE_RSP_RQ_INLINE_FLAGS)
+		if (ctx->config & (HNS_ROCE_RSP_RQ_INLINE_FLAGS |
+				   HNS_ROCE_RSP_CQE_INLINE_FLAGS))
 			qp->rq_rinl_buf.wqe_cnt = cnt;
 
 		attr->cap.max_recv_wr = qp->rq.wqe_cnt;
-- 
2.30.0

