diff mbox series

[v2,for-next,4/4] RDMA/hns: Add SRQ asynchronous event support

Message ID 1542957560-59868-5-git-send-email-oulijun@huawei.com (mailing list archive)
State Superseded
Headers show
Series SRQ support for hip08 | expand

Commit Message

Lijun Ou Nov. 23, 2018, 7:19 a.m. UTC 
This patch implements the process flow of SRQ asynchronous
event.

Signed-off-by: Lijun Ou <oulijun@huawei.com>
---
V1->V2:
1. Use spin lock instead of rcu lock
2. Use atomic_refcount instead of refcount_inc
---
 drivers/infiniband/hw/hns/hns_roce_device.h |  7 +++++++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  |  7 ++++++-
 drivers/infiniband/hw/hns/hns_roce_srq.c    | 23 +++++++++++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)

Comments

Jason Gunthorpe Nov. 23, 2018, 6:49 p.m. UTC | #1 
On Fri, Nov 23, 2018 at 03:19:20PM +0800, Lijun Ou wrote:

> +void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type)
> +{
> +	struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
> +	struct hns_roce_srq *srq;
> +
> +	spin_lock(&srq_table->lock);
> +	srq = radix_tree_lookup(&srq_table->tree,
> +				srqn & (hr_dev->caps.num_srqs - 1));
> +	spin_unlock(&srq_table->lock);
> +	if (srq) {
> +		atomic_inc(&srq->refcount);

This locking arrangment still looks wrong.

What prevents srq from becoming freed before the atomic_inc is run?

If you put the atomic_inc inside the spinlock then this:

+	spin_lock_irq(&srq_table->lock);
+	radix_tree_delete(&srq_table->tree, srq->srqn);
+	spin_unlock_irq(&srq_table->lock);
+
+	if (atomic_dec_and_test(&srq->refcount))

Is serialized and doesn't have a race anymore.

Jason
Lijun Ou Nov. 24, 2018, 8:15 a.m. UTC | #2 
在 2018/11/24 2:49, Jason Gunthorpe 写道:
> On Fri, Nov 23, 2018 at 03:19:20PM +0800, Lijun Ou wrote:
>
>> +void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type)
>> +{
>> +	struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
>> +	struct hns_roce_srq *srq;
>> +
>> +	spin_lock(&srq_table->lock);
>> +	srq = radix_tree_lookup(&srq_table->tree,
>> +				srqn & (hr_dev->caps.num_srqs - 1));
>> +	spin_unlock(&srq_table->lock);
>> +	if (srq) {
>> +		atomic_inc(&srq->refcount);
> This locking arrangment still looks wrong.
>
> What prevents srq from becoming freed before the atomic_inc is run?
>
> If you put the atomic_inc inside the spinlock then this:
>
> +	spin_lock_irq(&srq_table->lock);
> +	radix_tree_delete(&srq_table->tree, srq->srqn);
> +	spin_unlock_irq(&srq_table->lock);
> +
> +	if (atomic_dec_and_test(&srq->refcount))
>
> Is serialized and doesn't have a race anymore.
>
> Jason
Thanks. I will fix it.
>
diff mbox series

Patch

diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index fa72e4d..83e3078 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -648,6 +648,12 @@  struct hns_roce_aeqe {
 		} qp_event;
 
 		struct {
+			__le32 srq;
+			u32 rsv0;
+			u32 rsv1;
+		} srq_event;
+
+		struct {
 			__le32 cq;
 			u32 rsv0;
 			u32 rsv1;
@@ -1136,6 +1142,7 @@  int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db,
 void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn);
 void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);
 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type);
+void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type);
 int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
 int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 6c9baf9..835b783 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -4459,6 +4459,7 @@  static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
 	int aeqe_found = 0;
 	int event_type;
 	int sub_type;
+	u32 srqn;
 	u32 qpn;
 	u32 cqn;
 
@@ -4481,6 +4482,9 @@  static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
 		cqn = roce_get_field(aeqe->event.cq_event.cq,
 				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
 				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
+		srqn = roce_get_field(aeqe->event.srq_event.srq,
+				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
+				     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
 
 		switch (event_type) {
 		case HNS_ROCE_EVENT_TYPE_PATH_MIG:
@@ -4488,13 +4492,14 @@  static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
 		case HNS_ROCE_EVENT_TYPE_COMM_EST:
 		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
 		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
+		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
 		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
 		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
 			hns_roce_qp_event(hr_dev, qpn, event_type);
 			break;
 		case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
-		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
 		case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
+			hns_roce_srq_event(hr_dev, srqn, event_type);
 			break;
 		case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
 		case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c
index 2cd170b..c475701 100644
--- a/drivers/infiniband/hw/hns/hns_roce_srq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_srq.c
@@ -9,6 +9,29 @@ 
 #include "hns_roce_cmd.h"
 #include "hns_roce_hem.h"
 
+void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type)
+{
+	struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
+	struct hns_roce_srq *srq;
+
+	spin_lock(&srq_table->lock);
+	srq = radix_tree_lookup(&srq_table->tree,
+				srqn & (hr_dev->caps.num_srqs - 1));
+	spin_unlock(&srq_table->lock);
+	if (srq) {
+		atomic_inc(&srq->refcount);
+	} else {
+		dev_warn(hr_dev->dev, "Async event for bogus SRQ %08x\n", srqn);
+		return;
+	}
+
+	srq->event(srq, event_type);
+
+	if (atomic_dec_and_test(&srq->refcount))
+		complete(&srq->free);
+}
+EXPORT_SYMBOL_GPL(hns_roce_srq_event);
+
 static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,
 				  enum hns_roce_event event_type)
 {