diff mbox series

[v3,08/17] rdma_rxe: Added mw object

Message ID 20200820224638.3212-9-rpearson@hpe.com (mailing list archive)
State Changes Requested
Delegated to: Jason Gunthorpe
Headers show
Series Memory window support for rdma_rxe | expand

Commit Message

Bob Pearson Aug. 20, 2020, 10:46 p.m. UTC 
Set max_mw device attribute.
Changed rxe_param.c so we can create mw objects (RXE_MAX_MW != 0).
Changed mw_pool.c to use struct rxe_mw size for the mw pool.
Added new mw struct to rxe_verbs.h. Can share state enum with mr.

Signed-off-by: Bob Pearson <rpearson@hpe.com>
---
 drivers/infiniband/sw/rxe/rxe.c       |  1 +
 drivers/infiniband/sw/rxe/rxe_param.h | 10 ++++++----
 drivers/infiniband/sw/rxe/rxe_pool.c  |  2 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h | 13 +++++++++++++
 4 files changed, 21 insertions(+), 5 deletions(-)

Comments

Zhu Yanjun Aug. 22, 2020, 3:39 a.m. UTC | #1 
On 8/21/2020 6:46 AM, Bob Pearson wrote:
> Set max_mw device attribute.
> Changed rxe_param.c so we can create mw objects (RXE_MAX_MW != 0).
> Changed mw_pool.c to use struct rxe_mw size for the mw pool.
> Added new mw struct to rxe_verbs.h. Can share state enum with mr.
>
> Signed-off-by: Bob Pearson <rpearson@hpe.com>
> ---
>   drivers/infiniband/sw/rxe/rxe.c       |  1 +
>   drivers/infiniband/sw/rxe/rxe_param.h | 10 ++++++----
>   drivers/infiniband/sw/rxe/rxe_pool.c  |  2 +-
>   drivers/infiniband/sw/rxe/rxe_verbs.h | 13 +++++++++++++
>   4 files changed, 21 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
> index 6c2100c71874..10f441ac7203 100644
> --- a/drivers/infiniband/sw/rxe/rxe.c
> +++ b/drivers/infiniband/sw/rxe/rxe.c
> @@ -54,6 +54,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
>   	rxe->attr.max_cq			= RXE_MAX_CQ;
>   	rxe->attr.max_cqe			= (1 << RXE_MAX_LOG_CQE) - 1;
>   	rxe->attr.max_mr			= RXE_MAX_MR;
> +	rxe->attr.max_mw			= RXE_MAX_MW;
>   	rxe->attr.max_pd			= RXE_MAX_PD;
>   	rxe->attr.max_qp_rd_atom		= RXE_MAX_QP_RD_ATOM;
>   	rxe->attr.max_res_rd_atom		= RXE_MAX_RES_RD_ATOM;
> diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
> index 96516e251bc1..e1d485bdd6af 100644
> --- a/drivers/infiniband/sw/rxe/rxe_param.h
> +++ b/drivers/infiniband/sw/rxe/rxe_param.h
> @@ -60,7 +60,8 @@ enum rxe_device_param {
>   	RXE_MAX_SGE_RD			= 32,
>   	RXE_MAX_CQ			= 16384,
>   	RXE_MAX_LOG_CQE			= 15,
> -	RXE_MAX_MR			= 256 * 1024,
> +	RXE_MAX_MR			= 0x40000,
> +	RXE_MAX_MW			= 0x40000,

How to get this 0x40000?

Thanks


>   	RXE_MAX_PD			= 0x7ffc,
>   	RXE_MAX_QP_RD_ATOM		= 128,
>   	RXE_MAX_RES_RD_ATOM		= 0x3f000,
> @@ -89,9 +90,10 @@ enum rxe_device_param {
>   	RXE_MAX_SRQ_INDEX		= 0x00040000,
>   
>   	RXE_MIN_MR_INDEX		= 0x00000001,
> -	RXE_MAX_MR_INDEX		= 0x00040000,
> -	RXE_MIN_MW_INDEX		= 0x00040001,
> -	RXE_MAX_MW_INDEX		= 0x00060000,
> +	RXE_MAX_MR_INDEX		= RXE_MIN_MR_INDEX + RXE_MAX_MR - 1,
> +	RXE_MIN_MW_INDEX		= RXE_MIN_MR_INDEX + RXE_MAX_MR,
> +	RXE_MAX_MW_INDEX		= RXE_MIN_MW_INDEX + RXE_MAX_MW - 1,
> +
>   	RXE_MAX_PKT_PER_ACK		= 64,
>   
>   	RXE_MAX_UNACKED_PSNS		= 128,
> diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
> index fe652ce488f3..ad6f50e8b57a 100644
> --- a/drivers/infiniband/sw/rxe/rxe_pool.c
> +++ b/drivers/infiniband/sw/rxe/rxe_pool.c
> @@ -60,7 +60,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
>   	},
>   	[RXE_TYPE_MW] = {
>   		.name		= "rxe-mw",
> -		.size		= sizeof(struct rxe_mr),
> +		.size		= sizeof(struct rxe_mw),
>   		.flags		= RXE_POOL_INDEX,
>   		.max_index	= RXE_MAX_MW_INDEX,
>   		.min_index	= RXE_MIN_MW_INDEX,
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
> index c866fc5a80d6..44da74a41ccb 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.h
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
> @@ -264,6 +264,7 @@ struct rxe_qp {
>   	struct execute_work	cleanup_work;
>   };
>   
> +/* common state values for mr and mw */
>   enum rxe_mem_state {
>   	RXE_MEM_STATE_ZOMBIE,
>   	RXE_MEM_STATE_INVALID,
> @@ -321,6 +322,18 @@ struct rxe_mr {
>   	struct rxe_map		**map;
>   };
>   
> +struct rxe_mw {
> +	struct rxe_pool_entry	pelem;
> +	struct ib_mw		ibmw;
> +	struct rxe_qp		*qp;	/* type 2B only */
> +	struct rxe_mr		*mr;
> +	spinlock_t		lock;
> +	enum rxe_mem_state	state;
> +	u32			access;
> +	u64			addr;
> +	u64			length;
> +};
> +
>   struct rxe_mc_grp {
>   	struct rxe_pool_entry	pelem;
>   	spinlock_t		mcg_lock; /* guard group */
diff mbox series

Patch

diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 6c2100c71874..10f441ac7203 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -54,6 +54,7 @@  static void rxe_init_device_param(struct rxe_dev *rxe)
 	rxe->attr.max_cq			= RXE_MAX_CQ;
 	rxe->attr.max_cqe			= (1 << RXE_MAX_LOG_CQE) - 1;
 	rxe->attr.max_mr			= RXE_MAX_MR;
+	rxe->attr.max_mw			= RXE_MAX_MW;
 	rxe->attr.max_pd			= RXE_MAX_PD;
 	rxe->attr.max_qp_rd_atom		= RXE_MAX_QP_RD_ATOM;
 	rxe->attr.max_res_rd_atom		= RXE_MAX_RES_RD_ATOM;
diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
index 96516e251bc1..e1d485bdd6af 100644
--- a/drivers/infiniband/sw/rxe/rxe_param.h
+++ b/drivers/infiniband/sw/rxe/rxe_param.h
@@ -60,7 +60,8 @@  enum rxe_device_param {
 	RXE_MAX_SGE_RD			= 32,
 	RXE_MAX_CQ			= 16384,
 	RXE_MAX_LOG_CQE			= 15,
-	RXE_MAX_MR			= 256 * 1024,
+	RXE_MAX_MR			= 0x40000,
+	RXE_MAX_MW			= 0x40000,
 	RXE_MAX_PD			= 0x7ffc,
 	RXE_MAX_QP_RD_ATOM		= 128,
 	RXE_MAX_RES_RD_ATOM		= 0x3f000,
@@ -89,9 +90,10 @@  enum rxe_device_param {
 	RXE_MAX_SRQ_INDEX		= 0x00040000,
 
 	RXE_MIN_MR_INDEX		= 0x00000001,
-	RXE_MAX_MR_INDEX		= 0x00040000,
-	RXE_MIN_MW_INDEX		= 0x00040001,
-	RXE_MAX_MW_INDEX		= 0x00060000,
+	RXE_MAX_MR_INDEX		= RXE_MIN_MR_INDEX + RXE_MAX_MR - 1,
+	RXE_MIN_MW_INDEX		= RXE_MIN_MR_INDEX + RXE_MAX_MR,
+	RXE_MAX_MW_INDEX		= RXE_MIN_MW_INDEX + RXE_MAX_MW - 1,
+
 	RXE_MAX_PKT_PER_ACK		= 64,
 
 	RXE_MAX_UNACKED_PSNS		= 128,
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
index fe652ce488f3..ad6f50e8b57a 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.c
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -60,7 +60,7 @@  struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	},
 	[RXE_TYPE_MW] = {
 		.name		= "rxe-mw",
-		.size		= sizeof(struct rxe_mr),
+		.size		= sizeof(struct rxe_mw),
 		.flags		= RXE_POOL_INDEX,
 		.max_index	= RXE_MAX_MW_INDEX,
 		.min_index	= RXE_MIN_MW_INDEX,
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index c866fc5a80d6..44da74a41ccb 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -264,6 +264,7 @@  struct rxe_qp {
 	struct execute_work	cleanup_work;
 };
 
+/* common state values for mr and mw */
 enum rxe_mem_state {
 	RXE_MEM_STATE_ZOMBIE,
 	RXE_MEM_STATE_INVALID,
@@ -321,6 +322,18 @@  struct rxe_mr {
 	struct rxe_map		**map;
 };
 
+struct rxe_mw {
+	struct rxe_pool_entry	pelem;
+	struct ib_mw		ibmw;
+	struct rxe_qp		*qp;	/* type 2B only */
+	struct rxe_mr		*mr;
+	spinlock_t		lock;
+	enum rxe_mem_state	state;
+	u32			access;
+	u64			addr;
+	u64			length;
+};
+
 struct rxe_mc_grp {
 	struct rxe_pool_entry	pelem;
 	spinlock_t		mcg_lock; /* guard group */