[v3,25/28] cxl/bus: Populate the target list at decoder create

Commit Message

Dan Williams Aug. 24, 2021, 4:07 p.m. UTC

As found by cxl_test, the implementation populated the target_list for
the single dport exceptional case, it missed populating the target_list
for the typical multi-dport case.

Walk the hosting port's dport list and populate based on the passed in
map.

Move devm_cxl_add_passthrough_decoder() out of line now that it does the
work of generating a target_map.

Before:
$ cat /sys/bus/cxl/devices/root2/decoder*/target_list
0

0


After:
$ cat /sys/bus/cxl/devices/root2/decoder*/target_list
0
0,1,2,3
0
0,1,2,3

Where root2 is a CXL topology root object generated by 'cxl_test'.

Acked-by: Ben Widawsky <[e-mail]>
Signed-off-by: Dan Williams <[e-mail]>
---
 drivers/cxl/acpi.c     |   13 +++++++++-
 drivers/cxl/core/bus.c |   65 +++++++++++++++++++++++++++++++++++++++---------
 drivers/cxl/cxl.h      |   25 +++++++-----------
 3 files changed, 75 insertions(+), 28 deletions(-)

Comments

Jonathan Cameron Sept. 3, 2021, 12:59 p.m. UTC | #1

On Tue, 24 Aug 2021 09:07:39 -0700
Dan Williams <[e-mail]> wrote:

> As found by cxl_test, the implementation populated the target_list for
> the single dport exceptional case, it missed populating the target_list
> for the typical multi-dport case.

Description makes this sound like a fix, rather than what I think it is
which is implementing a new feature...

> 
> Walk the hosting port's dport list and populate based on the passed in
> map.
> 
> Move devm_cxl_add_passthrough_decoder() out of line now that it does the
> work of generating a target_map.
> 
> Before:
> $ cat /sys/bus/cxl/devices/root2/decoder*/target_list
> 0
> 
> 0
> 
> 
> After:
> $ cat /sys/bus/cxl/devices/root2/decoder*/target_list
> 0
> 0,1,2,3
> 0
> 0,1,2,3
> 
> Where root2 is a CXL topology root object generated by 'cxl_test'.
> 
> Acked-by: Ben Widawsky <[e-mail]>
> Signed-off-by: Dan Williams <[e-mail]>

one trivial inline.
Otherwise looks fine to me.
Reviewed-by: Jonathan Cameron <[e-mail]>

J
> ---
>  drivers/cxl/acpi.c     |   13 +++++++++-
>  drivers/cxl/core/bus.c |   65 +++++++++++++++++++++++++++++++++++++++---------
>  drivers/cxl/cxl.h      |   25 +++++++-----------
>  3 files changed, 75 insertions(+), 28 deletions(-)
> 
> diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c
> index 3fcb704c647f..6aea27c8fd4c 100644
> --- a/drivers/cxl/acpi.c
> +++ b/drivers/cxl/acpi.c
> @@ -52,6 +52,12 @@ static int cxl_acpi_cfmws_verify(struct device *dev,
>  		return -EINVAL;
>  	}
>  
> +	if (CFMWS_INTERLEAVE_WAYS(cfmws) > CXL_DECODER_MAX_INTERLEAVE) {
> +		dev_err(dev, "CFMWS Interleave Ways (%d) too large\n",
> +			CFMWS_INTERLEAVE_WAYS(cfmws));
> +		return -EINVAL;
> +	}
> +
>  	expected_len = struct_size((cfmws), interleave_targets,
>  				   CFMWS_INTERLEAVE_WAYS(cfmws));
>  
> @@ -71,6 +77,7 @@ static int cxl_acpi_cfmws_verify(struct device *dev,
>  static void cxl_add_cfmws_decoders(struct device *dev,
>  				   struct cxl_port *root_port)
>  {
> +	int target_map[CXL_DECODER_MAX_INTERLEAVE];
>  	struct acpi_cedt_cfmws *cfmws;
>  	struct cxl_decoder *cxld;
>  	acpi_size len, cur = 0;
> @@ -83,6 +90,7 @@ static void cxl_add_cfmws_decoders(struct device *dev,
>  
>  	while (cur < len) {
>  		struct acpi_cedt_header *c = cedt_subtable + cur;
> +		int i;
>  
>  		if (c->type != ACPI_CEDT_TYPE_CFMWS) {
>  			cur += c->length;
> @@ -108,6 +116,9 @@ static void cxl_add_cfmws_decoders(struct device *dev,
>  			continue;
>  		}
>  
> +		for (i = 0; i < CFMWS_INTERLEAVE_WAYS(cfmws); i++)
> +			target_map[i] = cfmws->interleave_targets[i];
> +
>  		flags = cfmws_to_decoder_flags(cfmws->restrictions);
>  		cxld = devm_cxl_add_decoder(dev, root_port,
>  					    CFMWS_INTERLEAVE_WAYS(cfmws),
> @@ -115,7 +126,7 @@ static void cxl_add_cfmws_decoders(struct device *dev,
>  					    CFMWS_INTERLEAVE_WAYS(cfmws),
>  					    CFMWS_INTERLEAVE_GRANULARITY(cfmws),
>  					    CXL_DECODER_EXPANDER,
> -					    flags);
> +					    flags, target_map);
>  
>  		if (IS_ERR(cxld)) {
>  			dev_err(dev, "Failed to add decoder for %#llx-%#llx\n",
> diff --git a/drivers/cxl/core/bus.c b/drivers/cxl/core/bus.c
> index 8073354ba232..9a755a37eadf 100644
> --- a/drivers/cxl/core/bus.c
> +++ b/drivers/cxl/core/bus.c
> @@ -454,14 +454,15 @@ int cxl_add_dport(struct cxl_port *port, struct device *dport_dev, int port_id,
>  EXPORT_SYMBOL_GPL(cxl_add_dport);
>  
>  static struct cxl_decoder *
> -cxl_decoder_alloc(struct cxl_port *port, int nr_targets, resource_size_t base,
> -		  resource_size_t len, int interleave_ways,
> -		  int interleave_granularity, enum cxl_decoder_type type,
> -		  unsigned long flags)
> +cxl_decoder_alloc(struct device *host, struct cxl_port *port, int nr_targets,
> +		  resource_size_t base, resource_size_t len,
> +		  int interleave_ways, int interleave_granularity,
> +		  enum cxl_decoder_type type, unsigned long flags,
> +		  int *target_map)
>  {
>  	struct cxl_decoder *cxld;
>  	struct device *dev;
> -	int rc = 0;
> +	int rc = 0, i;
>  
>  	if (interleave_ways < 1)
>  		return ERR_PTR(-EINVAL);
> @@ -493,10 +494,19 @@ cxl_decoder_alloc(struct cxl_port *port, int nr_targets, resource_size_t base,
>  		.target_type = type,
>  	};
>  
> -	/* handle implied target_list */
> -	if (interleave_ways == 1)
> -		cxld->target[0] =
> -			list_first_entry(&port->dports, struct cxl_dport, list);
> +	device_lock(&port->dev);
> +	for (i = 0; target_map && i < nr_targets; i++) {

Perhaps move target map check much earlier rather than putting it
int he loop condition?  I don't think the loop is modifying it...

> +		struct cxl_dport *dport = find_dport(port, target_map[i]);
> +
> +		if (!dport) {
> +			rc = -ENXIO;
> +			goto err;
> +		}
> +		dev_dbg(host, "%s: target: %d\n", dev_name(dport->dport), i);
> +		cxld->target[i] = dport;
> +	}
> +	device_unlock(&port->dev);
> +
>  	dev = &cxld->dev;
>  	device_initialize(dev);
>  	device_set_pm_not_required(dev);
> @@ -519,14 +529,19 @@ struct cxl_decoder *
>  devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets,
>  		     resource_size_t base, resource_size_t len,
>  		     int interleave_ways, int interleave_granularity,
> -		     enum cxl_decoder_type type, unsigned long flags)
> +		     enum cxl_decoder_type type, unsigned long flags,
> +		     int *target_map)
>  {
>  	struct cxl_decoder *cxld;
>  	struct device *dev;
>  	int rc;
>  
> -	cxld = cxl_decoder_alloc(port, nr_targets, base, len, interleave_ways,
> -				 interleave_granularity, type, flags);
> +	if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
> +		return ERR_PTR(-EINVAL);
> +
> +	cxld = cxl_decoder_alloc(host, port, nr_targets, base, len,
> +				 interleave_ways, interleave_granularity, type,
> +				 flags, target_map);
>  	if (IS_ERR(cxld))
>  		return cxld;
>  
> @@ -550,6 +565,32 @@ devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets,
>  }
>  EXPORT_SYMBOL_GPL(devm_cxl_add_decoder);
>  
> +/*
> + * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
> + * single ported host-bridges need not publish a decoder capability when a
> + * passthrough decode can be assumed, i.e. all transactions that the uport sees
> + * are claimed and passed to the single dport. Default the range a 0-base
> + * 0-length until the first CXL region is activated.
> + */
> +struct cxl_decoder *devm_cxl_add_passthrough_decoder(struct device *host,
> +						     struct cxl_port *port)
> +{
> +	struct cxl_dport *dport;
> +	int target_map[1];
> +
> +	device_lock(&port->dev);
> +	dport = list_first_entry_or_null(&port->dports, typeof(*dport), list);
> +	device_unlock(&port->dev);
> +
> +	if (!dport)
> +		return ERR_PTR(-ENXIO);
> +
> +	target_map[0] = dport->port_id;
> +	return devm_cxl_add_decoder(host, port, 1, 0, 0, 1, PAGE_SIZE,
> +				    CXL_DECODER_EXPANDER, 0, target_map);
> +}
> +EXPORT_SYMBOL_GPL(devm_cxl_add_passthrough_decoder);
> +
>  /**
>   * __cxl_driver_register - register a driver for the cxl bus
>   * @cxl_drv: cxl driver structure to attach
> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
> index a9d4051722bf..1064427e3eb5 100644
> --- a/drivers/cxl/cxl.h
> +++ b/drivers/cxl/cxl.h
> @@ -180,6 +180,12 @@ enum cxl_decoder_type {
>         CXL_DECODER_EXPANDER = 3,
>  };
>  
> +/*
> + * Current specification goes up to 8, double that seems a reasonable
> + * software max for the foreseeable future
> + */
> +#define CXL_DECODER_MAX_INTERLEAVE 16
> +
>  /**
>   * struct cxl_decoder - CXL address range decode configuration
>   * @dev: this decoder's device
> @@ -284,22 +290,11 @@ struct cxl_decoder *
>  devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets,
>  		     resource_size_t base, resource_size_t len,
>  		     int interleave_ways, int interleave_granularity,
> -		     enum cxl_decoder_type type, unsigned long flags);
> -
> -/*
> - * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
> - * single ported host-bridges need not publish a decoder capability when a
> - * passthrough decode can be assumed, i.e. all transactions that the uport sees
> - * are claimed and passed to the single dport. Default the range a 0-base
> - * 0-length until the first CXL region is activated.
> - */
> -static inline struct cxl_decoder *
> -devm_cxl_add_passthrough_decoder(struct device *host, struct cxl_port *port)
> -{
> -	return devm_cxl_add_decoder(host, port, 1, 0, 0, 1, PAGE_SIZE,
> -				    CXL_DECODER_EXPANDER, 0);
> -}
> +		     enum cxl_decoder_type type, unsigned long flags,
> +		     int *target_map);
>  
> +struct cxl_decoder *devm_cxl_add_passthrough_decoder(struct device *host,
> +						     struct cxl_port *port);
>  extern struct bus_type cxl_bus_type;
>  
>  struct cxl_driver {
>

Dan Williams Sept. 3, 2021, 10:43 p.m. UTC | #2

On Fri, Sep 3, 2021 at 5:59 AM Jonathan Cameron
<[e-mail]> wrote:
>
> On Tue, 24 Aug 2021 09:07:39 -0700
> Dan Williams <[e-mail]> wrote:
>
> > As found by cxl_test, the implementation populated the target_list for
> > the single dport exceptional case, it missed populating the target_list
> > for the typical multi-dport case.
>
> Description makes this sound like a fix, rather than what I think it is
> which is implementing a new feature...

It is finishing a feature where the unfinished state is broken. It
should never be the case that target_list_show() returns nothing.

[..]
> > diff --git a/drivers/cxl/core/bus.c b/drivers/cxl/core/bus.c
> > index 8073354ba232..9a755a37eadf 100644
> > --- a/drivers/cxl/core/bus.c
> > +++ b/drivers/cxl/core/bus.c
[..]
> > @@ -493,10 +494,19 @@ cxl_decoder_alloc(struct cxl_port *port, int nr_targets, resource_size_t base,
> >               .target_type = type,
> >       };
> >
> > -     /* handle implied target_list */
> > -     if (interleave_ways == 1)
> > -             cxld->target[0] =
> > -                     list_first_entry(&port->dports, struct cxl_dport, list);
> > +     device_lock(&port->dev);
> > +     for (i = 0; target_map && i < nr_targets; i++) {
>
> Perhaps move target map check much earlier rather than putting it
> int he loop condition?  I don't think the loop is modifying it...

The loop is not modifying target_map, but target_map is allowed to be
NULL. I was trying to avoid a non-error goto, but a better way to
solve that would be to make the loop a helper function taken under the
lock.

Jonathan Cameron Sept. 6, 2021, 8:52 a.m. UTC | #3

On Fri, 3 Sep 2021 15:43:25 -0700
Dan Williams <[e-mail]> wrote:

> On Fri, Sep 3, 2021 at 5:59 AM Jonathan Cameron
> <[e-mail]> wrote:
> >
> > On Tue, 24 Aug 2021 09:07:39 -0700
> > Dan Williams <[e-mail]> wrote:
> >  
> > > As found by cxl_test, the implementation populated the target_list for
> > > the single dport exceptional case, it missed populating the target_list
> > > for the typical multi-dport case.  
> >
> > Description makes this sound like a fix, rather than what I think it is
> > which is implementing a new feature...  
> 
> It is finishing a feature where the unfinished state is broken. It
> should never be the case that target_list_show() returns nothing.
> 
> [..]
> > > diff --git a/drivers/cxl/core/bus.c b/drivers/cxl/core/bus.c
> > > index 8073354ba232..9a755a37eadf 100644
> > > --- a/drivers/cxl/core/bus.c
> > > +++ b/drivers/cxl/core/bus.c  
> [..]
> > > @@ -493,10 +494,19 @@ cxl_decoder_alloc(struct cxl_port *port, int nr_targets, resource_size_t base,
> > >               .target_type = type,
> > >       };
> > >
> > > -     /* handle implied target_list */
> > > -     if (interleave_ways == 1)
> > > -             cxld->target[0] =
> > > -                     list_first_entry(&port->dports, struct cxl_dport, list);
> > > +     device_lock(&port->dev);
> > > +     for (i = 0; target_map && i < nr_targets; i++) {  
> >
> > Perhaps move target map check much earlier rather than putting it
> > int he loop condition?  I don't think the loop is modifying it...  
> 
> The loop is not modifying target_map, but target_map is allowed to be
> NULL. I was trying to avoid a non-error goto, but a better way to
> solve that would be to make the loop a helper function taken under the
> lock.
Ah. Understood.  I was not appreciating that check need to be under
the device_lock(). Helper function would indeed make this clean - good plan.

Thanks,

Jonathan

Patch

diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c
index 3fcb704c647f..6aea27c8fd4c 100644
--- a/drivers/cxl/acpi.c
+++ b/drivers/cxl/acpi.c
@@ -52,6 +52,12 @@  static int cxl_acpi_cfmws_verify(struct device *dev,
 		return -EINVAL;
 	}
 
+	if (CFMWS_INTERLEAVE_WAYS(cfmws) > CXL_DECODER_MAX_INTERLEAVE) {
+		dev_err(dev, "CFMWS Interleave Ways (%d) too large\n",
+			CFMWS_INTERLEAVE_WAYS(cfmws));
+		return -EINVAL;
+	}
+
 	expected_len = struct_size((cfmws), interleave_targets,
 				   CFMWS_INTERLEAVE_WAYS(cfmws));
 
@@ -71,6 +77,7 @@  static int cxl_acpi_cfmws_verify(struct device *dev,
 static void cxl_add_cfmws_decoders(struct device *dev,
 				   struct cxl_port *root_port)
 {
+	int target_map[CXL_DECODER_MAX_INTERLEAVE];
 	struct acpi_cedt_cfmws *cfmws;
 	struct cxl_decoder *cxld;
 	acpi_size len, cur = 0;
@@ -83,6 +90,7 @@  static void cxl_add_cfmws_decoders(struct device *dev,
 
 	while (cur < len) {
 		struct acpi_cedt_header *c = cedt_subtable + cur;
+		int i;
 
 		if (c->type != ACPI_CEDT_TYPE_CFMWS) {
 			cur += c->length;
@@ -108,6 +116,9 @@  static void cxl_add_cfmws_decoders(struct device *dev,
 			continue;
 		}
 
+		for (i = 0; i < CFMWS_INTERLEAVE_WAYS(cfmws); i++)
+			target_map[i] = cfmws->interleave_targets[i];
+
 		flags = cfmws_to_decoder_flags(cfmws->restrictions);
 		cxld = devm_cxl_add_decoder(dev, root_port,
 					    CFMWS_INTERLEAVE_WAYS(cfmws),
@@ -115,7 +126,7 @@  static void cxl_add_cfmws_decoders(struct device *dev,
 					    CFMWS_INTERLEAVE_WAYS(cfmws),
 					    CFMWS_INTERLEAVE_GRANULARITY(cfmws),
 					    CXL_DECODER_EXPANDER,
-					    flags);
+					    flags, target_map);
 
 		if (IS_ERR(cxld)) {
 			dev_err(dev, "Failed to add decoder for %#llx-%#llx\n",
diff --git a/drivers/cxl/core/bus.c b/drivers/cxl/core/bus.c
index 8073354ba232..9a755a37eadf 100644
--- a/drivers/cxl/core/bus.c
+++ b/drivers/cxl/core/bus.c
@@ -454,14 +454,15 @@  int cxl_add_dport(struct cxl_port *port, struct device *dport_dev, int port_id,
 EXPORT_SYMBOL_GPL(cxl_add_dport);
 
 static struct cxl_decoder *
-cxl_decoder_alloc(struct cxl_port *port, int nr_targets, resource_size_t base,
-		  resource_size_t len, int interleave_ways,
-		  int interleave_granularity, enum cxl_decoder_type type,
-		  unsigned long flags)
+cxl_decoder_alloc(struct device *host, struct cxl_port *port, int nr_targets,
+		  resource_size_t base, resource_size_t len,
+		  int interleave_ways, int interleave_granularity,
+		  enum cxl_decoder_type type, unsigned long flags,
+		  int *target_map)
 {
 	struct cxl_decoder *cxld;
 	struct device *dev;
-	int rc = 0;
+	int rc = 0, i;
 
 	if (interleave_ways < 1)
 		return ERR_PTR(-EINVAL);
@@ -493,10 +494,19 @@  cxl_decoder_alloc(struct cxl_port *port, int nr_targets, resource_size_t base,
 		.target_type = type,
 	};
 
-	/* handle implied target_list */
-	if (interleave_ways == 1)
-		cxld->target[0] =
-			list_first_entry(&port->dports, struct cxl_dport, list);
+	device_lock(&port->dev);
+	for (i = 0; target_map && i < nr_targets; i++) {
+		struct cxl_dport *dport = find_dport(port, target_map[i]);
+
+		if (!dport) {
+			rc = -ENXIO;
+			goto err;
+		}
+		dev_dbg(host, "%s: target: %d\n", dev_name(dport->dport), i);
+		cxld->target[i] = dport;
+	}
+	device_unlock(&port->dev);
+
 	dev = &cxld->dev;
 	device_initialize(dev);
 	device_set_pm_not_required(dev);
@@ -519,14 +529,19 @@  struct cxl_decoder *
 devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets,
 		     resource_size_t base, resource_size_t len,
 		     int interleave_ways, int interleave_granularity,
-		     enum cxl_decoder_type type, unsigned long flags)
+		     enum cxl_decoder_type type, unsigned long flags,
+		     int *target_map)
 {
 	struct cxl_decoder *cxld;
 	struct device *dev;
 	int rc;
 
-	cxld = cxl_decoder_alloc(port, nr_targets, base, len, interleave_ways,
-				 interleave_granularity, type, flags);
+	if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
+		return ERR_PTR(-EINVAL);
+
+	cxld = cxl_decoder_alloc(host, port, nr_targets, base, len,
+				 interleave_ways, interleave_granularity, type,
+				 flags, target_map);
 	if (IS_ERR(cxld))
 		return cxld;
 
@@ -550,6 +565,32 @@  devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets,
 }
 EXPORT_SYMBOL_GPL(devm_cxl_add_decoder);
 
+/*
+ * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
+ * single ported host-bridges need not publish a decoder capability when a
+ * passthrough decode can be assumed, i.e. all transactions that the uport sees
+ * are claimed and passed to the single dport. Default the range a 0-base
+ * 0-length until the first CXL region is activated.
+ */
+struct cxl_decoder *devm_cxl_add_passthrough_decoder(struct device *host,
+						     struct cxl_port *port)
+{
+	struct cxl_dport *dport;
+	int target_map[1];
+
+	device_lock(&port->dev);
+	dport = list_first_entry_or_null(&port->dports, typeof(*dport), list);
+	device_unlock(&port->dev);
+
+	if (!dport)
+		return ERR_PTR(-ENXIO);
+
+	target_map[0] = dport->port_id;
+	return devm_cxl_add_decoder(host, port, 1, 0, 0, 1, PAGE_SIZE,
+				    CXL_DECODER_EXPANDER, 0, target_map);
+}
+EXPORT_SYMBOL_GPL(devm_cxl_add_passthrough_decoder);
+
 /**
  * __cxl_driver_register - register a driver for the cxl bus
  * @cxl_drv: cxl driver structure to attach
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index a9d4051722bf..1064427e3eb5 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -180,6 +180,12 @@  enum cxl_decoder_type {
        CXL_DECODER_EXPANDER = 3,
 };
 
+/*
+ * Current specification goes up to 8, double that seems a reasonable
+ * software max for the foreseeable future
+ */
+#define CXL_DECODER_MAX_INTERLEAVE 16
+
 /**
  * struct cxl_decoder - CXL address range decode configuration
  * @dev: this decoder's device
@@ -284,22 +290,11 @@  struct cxl_decoder *
 devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets,
 		     resource_size_t base, resource_size_t len,
 		     int interleave_ways, int interleave_granularity,
-		     enum cxl_decoder_type type, unsigned long flags);
-
-/*
- * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
- * single ported host-bridges need not publish a decoder capability when a
- * passthrough decode can be assumed, i.e. all transactions that the uport sees
- * are claimed and passed to the single dport. Default the range a 0-base
- * 0-length until the first CXL region is activated.
- */
-static inline struct cxl_decoder *
-devm_cxl_add_passthrough_decoder(struct device *host, struct cxl_port *port)
-{
-	return devm_cxl_add_decoder(host, port, 1, 0, 0, 1, PAGE_SIZE,
-				    CXL_DECODER_EXPANDER, 0);
-}
+		     enum cxl_decoder_type type, unsigned long flags,
+		     int *target_map);
 
+struct cxl_decoder *devm_cxl_add_passthrough_decoder(struct device *host,
+						     struct cxl_port *port);
 extern struct bus_type cxl_bus_type;
 
 struct cxl_driver {